From c677424edba09bd64e86c1aa24c3148abecbd352 Mon Sep 17 00:00:00 2001 From: Colin Johnson Date: Thu, 18 Jun 2026 15:17:46 -0400 Subject: [PATCH] qa-lab: add evidence artifact gallery (#94283) * qa-lab: add evidence artifact gallery * qa-lab: harden evidence gallery artifacts * qa-lab: share evidence gallery view types * qa-lab: disable evidence preview caching * refactor(qa-lab): resolve evidence artifacts once and trim gallery render/test duplication * fix(qa-lab): build evidence model before sending /api/evidence success headers --------- Co-authored-by: Dallin Romney --- .../qa-lab/shared/evidence-gallery-types.ts | 88 +++ .../qa-lab/src/evidence-gallery.test.ts | 521 +++++++++++++ extensions/qa-lab/src/evidence-gallery.ts | 711 ++++++++++++++++++ extensions/qa-lab/src/lab-server.test.ts | 121 ++- extensions/qa-lab/src/lab-server.ts | 94 +++ extensions/qa-lab/web/src/app.ts | 124 ++- extensions/qa-lab/web/src/styles.css | 665 ++++++++++++++++ extensions/qa-lab/web/src/ui-render.test.ts | 258 +++++++ extensions/qa-lab/web/src/ui-render.ts | 503 ++++++++++++- scripts/check-no-raw-channel-fetch.mjs | 6 +- 10 files changed, 3079 insertions(+), 12 deletions(-) create mode 100644 extensions/qa-lab/shared/evidence-gallery-types.ts create mode 100644 extensions/qa-lab/src/evidence-gallery.test.ts create mode 100644 extensions/qa-lab/src/evidence-gallery.ts create mode 100644 extensions/qa-lab/web/src/ui-render.test.ts diff --git a/extensions/qa-lab/shared/evidence-gallery-types.ts b/extensions/qa-lab/shared/evidence-gallery-types.ts new file mode 100644 index 00000000000..5905b3ec452 --- /dev/null +++ b/extensions/qa-lab/shared/evidence-gallery-types.ts @@ -0,0 +1,88 @@ +export type QaEvidenceGalleryStatus = "pass" | "fail" | "blocked" | "skipped"; + +export type QaEvidenceCoverageView = { + id: string; + role: string; +}; + +export type QaEvidenceProducerContextFile = { + href: string; + path: string; + preview: string | null; +}; + +export type QaEvidenceMatrixCellView = { + artifactKinds: string[]; + artifactPaths: string[]; + coverageIds: string[]; + runner: { + availability: string | null; + command: string | null; + lane: string | null; + workflow: string | null; + } | null; + stage: string; + status: string; + surface: string; + testId: string | null; + title: string | null; +}; + +export type QaEvidenceArtifactView = { + exists: boolean; + error: string | null; + href: string | null; + kind: string; + mediaKind: "image" | "video" | "json" | "text" | "file"; + path: string; + preview: string | null; + source: string; +}; + +export type QaEvidenceGalleryEntryView = { + artifacts: QaEvidenceArtifactView[]; + coverage: QaEvidenceCoverageView[]; + failureReason: string | null; + id: string; + kind: string; + sourcePath: string | null; + status: QaEvidenceGalleryStatus; + title: string; +}; + +export type QaEvidenceProducerContext = { + commands: QaEvidenceProducerContextFile | null; + kind: "ux-matrix"; + manifest: + | (QaEvidenceProducerContextFile & { + path: string; + runStatus: string | null; + runId: string | null; + }) + | null; + matrix: { + cells: QaEvidenceMatrixCellView[]; + counts: Record; + path: string; + stages: string[]; + surfaces: string[]; + } | null; + preflight: { + adbDevices: QaEvidenceProducerContextFile | null; + memory: QaEvidenceProducerContextFile | null; + }; + releaseLedger: (QaEvidenceProducerContextFile & { counts: Record }) | null; + rootPath: string; + scorecard: QaEvidenceProducerContextFile | null; +}; + +export type QaEvidenceGalleryModel = { + counts: Record; + entries: QaEvidenceGalleryEntryView[]; + evidenceMode: string; + evidencePath: string; + generatedAt: string; + profile: string | null; + producerContext: QaEvidenceProducerContext | null; + schemaVersion: number; +}; diff --git a/extensions/qa-lab/src/evidence-gallery.test.ts b/extensions/qa-lab/src/evidence-gallery.test.ts new file mode 100644 index 00000000000..e4abdf8d449 --- /dev/null +++ b/extensions/qa-lab/src/evidence-gallery.test.ts @@ -0,0 +1,521 @@ +// Qa Lab tests cover generic QA evidence gallery behavior. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + buildQaEvidenceGalleryModel, + resolveQaEvidenceArtifactFile, + resolveQaEvidenceFile, +} from "./evidence-gallery.js"; +import { + QA_EVIDENCE_FILENAME, + buildVitestEvidenceSummary, + type QaEvidenceSummaryJson, +} from "./evidence-summary.js"; + +async function createTempRepo() { + return fs.mkdtemp(path.join(os.tmpdir(), "qa-evidence-gallery-")); +} + +async function writeJson(filePath: string, value: unknown) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function vitestArtifactEvidence(params: { + id: string; + title: string; + artifact: { kind: string; path: string }; +}) { + return { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-17T12:00:00.000Z", + evidenceMode: "full", + entries: [ + { + test: { kind: "vitest-test", id: params.id, title: params.title }, + coverage: [{ id: "qa.artifact", role: "primary" }], + execution: { + runner: "vitest", + environment: { ref: "gallery-test", os: "darwin", nodeVersion: "v24.0.0" }, + provider: { + id: "mock-openai", + live: false, + model: { name: "mock-openai/gpt-5.5", ref: "mock-openai/gpt-5.5" }, + }, + packageSource: { kind: "source-checkout" }, + artifacts: [{ ...params.artifact, source: "vitest" }], + }, + result: { status: "pass" }, + }, + ], + }; +} + +describe("evidence gallery", () => { + it("builds a generic gallery model for non-UX QA Lab evidence", async () => { + const repoRoot = await createTempRepo(); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "vitest"); + await fs.mkdir(path.join(outputDir, "runner"), { recursive: true }); + await fs.writeFile(path.join(outputDir, "runner", "result.json"), '{"ok":true}\n', "utf8"); + await fs.writeFile(path.join(outputDir, "runner", "output.log"), "vitest pass\n", "utf8"); + + const evidence: QaEvidenceSummaryJson = buildVitestEvidenceSummary({ + artifactPaths: [ + { kind: "runner-result", path: "runner/result.json" }, + { kind: "log", path: "runner/output.log" }, + ], + env: { + OPENCLAW_QA_REF: "gallery-test", + } as NodeJS.ProcessEnv, + generatedAt: "2026-06-17T12:00:00.000Z", + primaryModel: "mock-openai/gpt-5.5", + providerMode: "mock-openai", + targets: [ + { + id: "qa-lab.generic-vitest", + title: "Generic Vitest evidence", + sourcePath: "extensions/qa-lab/src/generic.test.ts", + primaryCoverageIds: ["qa.generic"], + }, + { + id: "qa-lab.no-artifacts", + title: "Generic entry without artifacts", + sourcePath: "extensions/qa-lab/src/no-artifacts.test.ts", + primaryCoverageIds: ["qa.empty"], + }, + ], + results: [ + { + id: "qa-lab.generic-vitest", + status: "pass", + durationMs: 42, + }, + { + id: "qa-lab.no-artifacts", + status: "skipped", + durationMs: 1, + }, + ], + }); + evidence.entries[1] = { + ...evidence.entries[1], + execution: { + ...evidence.entries[1].execution!, + artifacts: [], + }, + }; + const evidencePath = path.join(outputDir, QA_EVIDENCE_FILENAME); + await writeJson(evidencePath, evidence); + + const model = await buildQaEvidenceGalleryModel({ + evidencePath: outputDir, + repoRoot, + }); + + expect(model.counts).toMatchObject({ pass: 1, skipped: 1, fail: 0, blocked: 0 }); + expect(model.evidencePath).toBe(".artifacts/qa-e2e/vitest/qa-evidence.json"); + expect(model.producerContext).toBeNull(); + expect(model.entries).toHaveLength(2); + expect(model.entries[0]).toMatchObject({ + id: "qa-lab.generic-vitest", + kind: "vitest-test", + artifacts: [ + expect.objectContaining({ + exists: true, + kind: "runner-result", + href: "/api/evidence/artifact?evidencePath=.artifacts%2Fqa-e2e%2Fvitest%2Fqa-evidence.json&artifactPath=runner%2Fresult.json", + mediaKind: "json", + preview: '{\n "ok": true\n}', + }), + expect.objectContaining({ + exists: true, + kind: "log", + mediaKind: "text", + preview: "vitest pass\n", + }), + ], + }); + expect(model.entries[1]).toMatchObject({ + id: "qa-lab.no-artifacts", + artifacts: [], + }); + }); + + it("detects UX Matrix producer context from suite-level evidence artifacts", async () => { + const repoRoot = await createTempRepo(); + const suiteDir = path.join(repoRoot, ".artifacts", "qa-e2e", "suite"); + const runDir = path.join(suiteDir, "script", "ux-matrix-evidence-dashboard", "run-1"); + await fs.mkdir(path.join(runDir, "surfaces", "web-ui", "stages", "first-run"), { + recursive: true, + }); + await fs.mkdir(path.join(runDir, "surfaces", "cli", "stages", "error-state"), { + recursive: true, + }); + await fs.writeFile( + path.join(runDir, "surfaces", "web-ui", "stages", "first-run", "screenshot.png"), + "png", + ); + await fs.writeFile( + path.join(runDir, "surfaces", "cli", "stages", "error-state", "logs.txt"), + "cli blocked\n", + "utf8", + ); + await writeJson(path.join(runDir, "manifest.json"), { + run: { + runId: "run-1", + status: "pass", + }, + }); + await writeJson(path.join(runDir, "matrix.json"), { + counts: { + pass: 1, + blocked: 1, + "proof-gap": 1, + }, + stages: [ + { id: "first-run", label: "First run" }, + { id: "error-state", label: "Error state" }, + ], + surfaces: [ + { id: "web-ui", label: "Web UI" }, + { id: "cli", label: "CLI" }, + ], + cells: [ + null, + { + coverageIds: ["ui.control"], + runner: { + availability: "local", + command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + lane: "web-ui-playwright", + workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + }, + stage: "first-run", + status: "pass", + surface: "web-ui", + }, + { + coverageIds: ["cli-entrypoint"], + runner: { + availability: "local", + command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + lane: "cli-status", + workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + }, + stage: "first-run", + status: "proof-gap", + surface: "cli", + }, + { stage: "error-state", status: "blocked", surface: "cli" }, + ], + }); + await writeJson(path.join(runDir, "release-ledger.json"), { + counts: { + pass: 1, + blocked: 1, + "proof-gap": 1, + }, + }); + await fs.writeFile(path.join(runDir, "scorecard.md"), "# UX Matrix\n\n- pass: 1\n", "utf8"); + await fs.writeFile(path.join(runDir, "commands.txt"), "node ux matrix\n", "utf8"); + await fs.mkdir(path.join(runDir, "preflight"), { recursive: true }); + await fs.writeFile(path.join(runDir, "preflight", "memory.txt"), "memory ok\n", "utf8"); + await fs.writeFile( + path.join(runDir, "preflight", "adb-devices.txt"), + "List of devices\n", + "utf8", + ); + + await writeJson(path.join(suiteDir, QA_EVIDENCE_FILENAME), { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-17T12:00:00.000Z", + evidenceMode: "full", + entries: [ + { + test: { + kind: "ux-matrix-cell", + id: "ux-matrix.web-ui.first-run", + title: "UX Matrix: web-ui / first-run", + source: { path: "scripts/ux-matrix/dashboard.ts" }, + }, + coverage: [{ id: "ui.control", role: "primary" }], + execution: { + runner: "ux-matrix-dashboard", + environment: { + ref: "gallery-test", + os: "darwin", + nodeVersion: "v24.0.0", + }, + provider: { + id: "ux-matrix", + live: false, + model: { name: null, ref: null }, + fixture: "mocked-control-ui-and-isolated-cli", + }, + packageSource: { kind: "source-checkout", sha: "abc123" }, + artifacts: [ + { + kind: "screenshot", + path: ".artifacts/qa-e2e/suite/script/ux-matrix-evidence-dashboard/run-1/surfaces/web-ui/stages/first-run/screenshot.png", + source: "ux-matrix:web-ui:first-run", + }, + ], + }, + result: { status: "pass", timing: { wallMs: 1 } }, + }, + { + test: { + kind: "ux-matrix-cell", + id: "qa-lab.wrapper-cli-error", + title: "UX Matrix: cli / error-state", + source: { path: "scripts/ux-matrix/dashboard.ts" }, + }, + coverage: [{ id: "status-snapshots", role: "primary" }], + execution: { + runner: "ux-matrix-dashboard", + environment: { + ref: "gallery-test", + os: "darwin", + nodeVersion: "v24.0.0", + }, + provider: { + id: "ux-matrix", + live: false, + model: { name: null, ref: null }, + fixture: "mocked-control-ui-and-isolated-cli", + }, + packageSource: { kind: "source-checkout", sha: "abc123" }, + artifacts: [ + { + kind: "log", + path: ".artifacts/qa-e2e/suite/script/ux-matrix-evidence-dashboard/run-1/surfaces/cli/stages/error-state/logs.txt", + source: "ux-matrix:cli:error-state", + }, + ], + }, + result: { + status: "blocked", + failure: { + class: "blocked", + reason: "CLI error-state proof captured a blocked result.", + }, + timing: { wallMs: 2 }, + }, + }, + ], + }); + + const model = await buildQaEvidenceGalleryModel({ + evidencePath: suiteDir, + repoRoot, + }); + + expect(model.producerContext).toMatchObject({ + kind: "ux-matrix", + manifest: { + runId: "run-1", + runStatus: "pass", + }, + matrix: { + counts: { + pass: 1, + blocked: 1, + "proof-gap": 1, + }, + stages: ["first-run", "error-state"], + surfaces: ["web-ui", "cli"], + }, + releaseLedger: { + counts: { + pass: 1, + blocked: 1, + "proof-gap": 1, + }, + }, + }); + expect(model.producerContext?.matrix?.cells).toEqual([ + { + artifactKinds: ["screenshot"], + artifactPaths: [ + ".artifacts/qa-e2e/suite/script/ux-matrix-evidence-dashboard/run-1/surfaces/web-ui/stages/first-run/screenshot.png", + ], + coverageIds: ["ui.control"], + runner: { + availability: "local", + command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + lane: "web-ui-playwright", + workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + }, + stage: "first-run", + status: "pass", + surface: "web-ui", + testId: "ux-matrix.web-ui.first-run", + title: "UX Matrix: web-ui / first-run", + }, + { + artifactKinds: [], + artifactPaths: [], + coverageIds: ["cli-entrypoint"], + runner: { + availability: "local", + command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + lane: "cli-status", + workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + }, + stage: "first-run", + status: "proof-gap", + surface: "cli", + testId: null, + title: null, + }, + { + artifactKinds: ["log"], + artifactPaths: [ + ".artifacts/qa-e2e/suite/script/ux-matrix-evidence-dashboard/run-1/surfaces/cli/stages/error-state/logs.txt", + ], + coverageIds: [], + runner: null, + stage: "error-state", + status: "blocked", + surface: "cli", + testId: "qa-lab.wrapper-cli-error", + title: "UX Matrix: cli / error-state", + }, + ]); + expect(model.producerContext?.scorecard?.preview).toContain("# UX Matrix"); + expect(model.producerContext?.scorecard?.href).toContain("/api/evidence/artifact?"); + expect(model.producerContext?.scorecard?.href).not.toContain(repoRoot); + expect(model.producerContext?.commands?.preview).toBe("node ux matrix\n"); + expect(model.producerContext?.commands?.path).toContain("commands.txt"); + expect(model.producerContext?.manifest?.preview).toContain('"runId": "run-1"'); + expect(model.producerContext?.releaseLedger?.preview).toContain('"proof-gap": 1'); + expect(model.producerContext?.preflight.memory?.path).toContain("preflight/memory.txt"); + expect(model.producerContext?.preflight.memory?.preview).toBe("memory ok\n"); + expect(model.producerContext?.preflight.adbDevices?.path).toContain( + "preflight/adb-devices.txt", + ); + expect(model.producerContext?.preflight.adbDevices?.preview).toBe("List of devices\n"); + expect(model.evidencePath).toBe(".artifacts/qa-e2e/suite/qa-evidence.json"); + const outsideDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-evidence-outside-")); + const outsideCommands = path.join(outsideDir, "commands.txt"); + await fs.writeFile(outsideCommands, "outside secret\n", "utf8"); + await fs.unlink(path.join(runDir, "commands.txt")); + await fs.symlink(outsideCommands, path.join(runDir, "commands.txt")); + const symlinkModel = await buildQaEvidenceGalleryModel({ + evidencePath: suiteDir, + repoRoot, + }); + expect(symlinkModel.producerContext?.commands).toBeNull(); + expect(JSON.stringify(symlinkModel)).not.toContain("outside secret"); + await expect( + resolveQaEvidenceArtifactFile({ + artifactPath: + ".artifacts/qa-e2e/suite/script/ux-matrix-evidence-dashboard/run-1/scorecard.md", + evidencePath: suiteDir, + repoRoot, + }), + ).resolves.toBe(await fs.realpath(path.join(runDir, "scorecard.md"))); + }); + + it("resolves evidence and declared artifacts inside the repo root only", async () => { + const repoRoot = await createTempRepo(); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "suite"); + const evidencePath = path.join(outputDir, QA_EVIDENCE_FILENAME); + await fs.writeFile(path.join(repoRoot, "package.json"), '{"private":true}\n', "utf8"); + await fs.mkdir(outputDir, { recursive: true }); + await fs.writeFile(path.join(outputDir, "artifact.log"), "ok\n", "utf8"); + await writeJson( + evidencePath, + vitestArtifactEvidence({ + id: "qa-lab.declared-artifact", + title: "Declared artifact", + artifact: { kind: "log", path: "artifact.log" }, + }), + ); + + await expect(resolveQaEvidenceFile({ inputPath: outputDir, repoRoot })).resolves.toBe( + await fs.realpath(evidencePath), + ); + await expect( + resolveQaEvidenceArtifactFile({ + artifactPath: "artifact.log", + evidencePath, + repoRoot, + }), + ).resolves.toBe(await fs.realpath(path.join(outputDir, "artifact.log"))); + await fs.mkdir(path.join(repoRoot, "runner"), { recursive: true }); + await fs.mkdir(path.join(outputDir, "runner"), { recursive: true }); + await fs.writeFile(path.join(repoRoot, "runner", "result.json"), '{"from":"repo"}\n', "utf8"); + await fs.writeFile( + path.join(outputDir, "runner", "result.json"), + '{"from":"evidence"}\n', + "utf8", + ); + const collisionEvidence = vitestArtifactEvidence({ + id: "qa-lab.colliding-artifact", + title: "Colliding artifact", + artifact: { kind: "runner-result", path: "runner/result.json" }, + }); + await writeJson(evidencePath, collisionEvidence); + await expect( + resolveQaEvidenceArtifactFile({ + artifactPath: "runner/result.json", + evidencePath, + repoRoot, + }), + ).resolves.toBe(await fs.realpath(path.join(outputDir, "runner", "result.json"))); + await fs.rm(path.join(outputDir, "runner", "result.json")); + const missingBundleModel = await buildQaEvidenceGalleryModel({ evidencePath, repoRoot }); + expect(missingBundleModel.entries[0].artifacts[0]).toMatchObject({ + exists: false, + error: "Evidence artifact not found.", + preview: null, + }); + expect(JSON.stringify(missingBundleModel)).not.toContain('"from":"repo"'); + await expect( + resolveQaEvidenceArtifactFile({ + artifactPath: "package.json", + evidencePath, + repoRoot, + }), + ).rejects.toThrow("Evidence artifact not found."); + await fs.writeFile(path.join(outputDir, "undeclared.log"), "undeclared\n", "utf8"); + await expect( + resolveQaEvidenceArtifactFile({ + artifactPath: "undeclared.log", + evidencePath, + repoRoot, + }), + ).rejects.toThrow("Evidence artifact is not declared by this evidence summary."); + const outsideDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-evidence-outside-artifact-")); + const outsideArtifact = path.join(outsideDir, "artifact.log"); + await fs.writeFile(outsideArtifact, "outside secret\n", "utf8"); + await fs.symlink(outsideArtifact, path.join(outputDir, "escape.log")); + await writeJson(evidencePath, { + ...collisionEvidence, + entries: [ + { + ...collisionEvidence.entries[0], + execution: { + ...collisionEvidence.entries[0].execution, + artifacts: [{ kind: "log", path: "escape.log", source: "vitest" }], + }, + }, + ], + }); + await expect( + resolveQaEvidenceArtifactFile({ + artifactPath: "escape.log", + evidencePath, + repoRoot, + }), + ).rejects.toThrow("Evidence artifact not found."); + await expect( + resolveQaEvidenceFile({ inputPath: "/tmp/not-openclaw-evidence.json", repoRoot }), + ).rejects.toThrow("Evidence path not found."); + }); +}); diff --git a/extensions/qa-lab/src/evidence-gallery.ts b/extensions/qa-lab/src/evidence-gallery.ts new file mode 100644 index 00000000000..9be9f501ff4 --- /dev/null +++ b/extensions/qa-lab/src/evidence-gallery.ts @@ -0,0 +1,711 @@ +// Qa Lab plugin module implements generic QA evidence gallery data. +import fs from "node:fs/promises"; +import path from "node:path"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import type { + QaEvidenceArtifactView, + QaEvidenceGalleryEntryView, + QaEvidenceGalleryModel, + QaEvidenceMatrixCellView, + QaEvidenceProducerContext, + QaEvidenceProducerContextFile, +} from "../shared/evidence-gallery-types.js"; +import { toRepoRelativePath } from "./cli-paths.js"; +import { + QA_EVIDENCE_FILENAME, + validateQaEvidenceSummaryJson, + type QaEvidenceStatus, + type QaEvidenceSummaryEntry, +} from "./evidence-summary.js"; + +export type { + QaEvidenceArtifactView, + QaEvidenceGalleryEntryView, + QaEvidenceGalleryModel, + QaEvidenceMatrixCellView, + QaEvidenceProducerContext, + QaEvidenceProducerContextFile, +} from "../shared/evidence-gallery-types.js"; + +const TEXT_PREVIEW_BYTES = 12 * 1024; +const ARTIFACT_VIEW_CONCURRENCY = 8; + +const UX_MATRIX_PRODUCER_FILES = [ + { key: "commands", path: "commands.txt", previewKind: "text" }, + { key: "manifest", path: "manifest.json", previewKind: "json" }, + { key: "matrix", path: "matrix.json", previewKind: "json" }, + { key: "releaseLedger", path: "release-ledger.json", previewKind: "json" }, + { key: "scorecard", path: "scorecard.md", previewKind: "text" }, + { key: "memory", path: path.join("preflight", "memory.txt"), previewKind: "text" }, + { key: "adbDevices", path: path.join("preflight", "adb-devices.txt"), previewKind: "text" }, +] as const; + +type QaEvidenceArtifact = NonNullable["artifacts"][number]; + +export class QaEvidenceGalleryError extends Error { + readonly statusCode: number; + + constructor(message: string, statusCode: number) { + super(message); + this.name = "QaEvidenceGalleryError"; + this.statusCode = statusCode; + } +} + +function evidenceError(message: string, statusCode: number): QaEvidenceGalleryError { + return new QaEvidenceGalleryError(message, statusCode); +} + +function isInside(root: string, candidate: string) { + const relative = path.relative(root, candidate); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +async function realpathIfExists(filePath: string): Promise { + return fs.realpath(filePath).catch(() => null); +} + +async function resolveContainedFileIfExists( + filePath: string, + allowedRoots: readonly string[], +): Promise { + const realFile = await realpathIfExists(filePath); + if (!realFile) { + return null; + } + if (!allowedRoots.some((root) => isInside(root, realFile))) { + return null; + } + const stats = await fs.stat(realFile).catch(() => null); + return stats?.isFile() ? realFile : null; +} + +export async function resolveQaEvidenceFile(params: { + inputPath: string; + repoRoot: string; +}): Promise { + const repoRoot = await fs.realpath(path.resolve(params.repoRoot)); + const raw = params.inputPath.trim(); + if (!raw) { + throw evidenceError("Evidence path is required.", 400); + } + const candidate = path.resolve(repoRoot, raw); + const realCandidate = await realpathIfExists(candidate); + if (!realCandidate) { + throw evidenceError("Evidence path not found.", 404); + } + if (!isInside(repoRoot, realCandidate)) { + throw evidenceError("Evidence path must stay inside the repo root.", 403); + } + const stats = await fs.stat(realCandidate); + const evidencePath = stats.isDirectory() + ? path.join(realCandidate, QA_EVIDENCE_FILENAME) + : realCandidate; + const realEvidencePath = await realpathIfExists(evidencePath); + if (!realEvidencePath) { + throw evidenceError("qa-evidence.json not found.", 404); + } + if (!isInside(repoRoot, realEvidencePath)) { + throw evidenceError("qa-evidence.json must stay inside the repo root.", 403); + } + return realEvidencePath; +} + +export async function resolveQaEvidenceArtifactFile(params: { + artifactPath: string; + evidencePath: string; + repoRoot: string; +}): Promise { + const repoRoot = await fs.realpath(path.resolve(params.repoRoot)); + const evidencePath = await resolveQaEvidenceFile({ inputPath: params.evidencePath, repoRoot }); + if (!params.artifactPath.trim()) { + throw evidenceError("Artifact path is required.", 400); + } + const summary = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(evidencePath, "utf8")) as unknown, + ); + const artifactFile = await resolveArtifactFileWithinRoots({ + artifactPath: params.artifactPath, + evidenceDir: path.dirname(evidencePath), + repoRoot, + }); + if (!artifactFile) { + throw evidenceError("Evidence artifact not found.", 404); + } + const allowedArtifactFiles = await collectDeclaredQaEvidenceArtifactFiles({ + evidencePath, + repoRoot, + summaryEntries: summary.entries, + }); + if (allowedArtifactFiles.has(artifactFile)) { + return artifactFile; + } + throw evidenceError("Evidence artifact is not declared by this evidence summary.", 403); +} + +function isExplicitRepoRootArtifactPath(raw: string): boolean { + const normalized = raw.split(/[\\/]+/u).join("/"); + return normalized.startsWith(".artifacts/"); +} + +// Resolve an artifact path against pre-resolved roots without re-reading the evidence file. +// Returns null when the path is missing or escapes both roots; callers map that to an error. +async function resolveArtifactFileWithinRoots(params: { + artifactPath: string; + evidenceDir: string; + repoRoot: string; +}): Promise { + const raw = params.artifactPath.trim(); + if (!raw) { + return null; + } + const candidates = path.isAbsolute(raw) ? [raw] : [path.resolve(params.evidenceDir, raw)]; + if (!path.isAbsolute(raw) && isExplicitRepoRootArtifactPath(raw)) { + candidates.push(path.resolve(params.repoRoot, raw)); + } + for (const candidate of candidates) { + const realCandidate = await realpathIfExists(candidate); + if (!realCandidate) { + continue; + } + if (!isInside(params.repoRoot, realCandidate) && !isInside(params.evidenceDir, realCandidate)) { + continue; + } + const stats = await fs.stat(realCandidate).catch(() => null); + if (stats?.isFile()) { + return realCandidate; + } + } + return null; +} + +async function collectDeclaredQaEvidenceArtifactFiles(params: { + evidencePath: string; + repoRoot: string; + summaryEntries: readonly QaEvidenceSummaryEntry[]; +}): Promise> { + const repoRoot = await fs.realpath(path.resolve(params.repoRoot)); + const evidenceDir = path.dirname(params.evidencePath); + const allowed = new Set(); + for (const entry of params.summaryEntries) { + for (const artifact of entry.execution?.artifacts ?? []) { + const artifactPath = await resolveArtifactFileWithinRoots({ + artifactPath: artifact.path, + evidenceDir, + repoRoot, + }); + if (artifactPath) { + allowed.add(artifactPath); + } + } + } + const producerRoot = await findUxMatrixProducerRoot({ + evidencePath: params.evidencePath, + repoRoot: params.repoRoot, + summaryEntries: params.summaryEntries, + }); + if (producerRoot) { + const producerFiles = [ + ...UX_MATRIX_PRODUCER_FILES.map((file) => file.path), + QA_EVIDENCE_FILENAME, + ]; + for (const producerFile of producerFiles) { + const realProducerFile = await realpathIfExists(path.join(producerRoot, producerFile)); + if (realProducerFile) { + allowed.add(realProducerFile); + } + } + } + return allowed; +} + +function classifyArtifact(kind: string, filePath: string): QaEvidenceArtifactView["mediaKind"] { + const normalizedKind = kind.toLowerCase(); + const ext = path.extname(filePath).toLowerCase(); + if ( + normalizedKind.includes("screenshot") || + normalizedKind.includes("gif") || + [".png", ".jpg", ".jpeg", ".gif", ".webp"].includes(ext) + ) { + return "image"; + } + if (normalizedKind.includes("video") || [".webm", ".mp4", ".mov"].includes(ext)) { + return "video"; + } + if ( + normalizedKind.includes("validation") || + normalizedKind.includes("json") || + ext === ".json" || + ext === ".jsonl" + ) { + return "json"; + } + if ( + normalizedKind.includes("log") || + normalizedKind.includes("report") || + [".log", ".md", ".txt"].includes(ext) + ) { + return "text"; + } + return "file"; +} + +async function readPreview(filePath: string, mediaKind: QaEvidenceArtifactView["mediaKind"]) { + if (mediaKind !== "json" && mediaKind !== "text") { + return null; + } + const handle = await fs.open(filePath, "r"); + try { + const buffer = Buffer.alloc(TEXT_PREVIEW_BYTES); + const { bytesRead } = await handle.read(buffer, 0, TEXT_PREVIEW_BYTES, 0); + const text = buffer.subarray(0, bytesRead).toString("utf8"); + if (mediaKind !== "json") { + return text; + } + try { + return JSON.stringify(JSON.parse(text), null, 2); + } catch { + return text; + } + } finally { + await handle.close(); + } +} + +async function readJsonIfExists( + filePath: string, + allowedRoots: readonly string[], +): Promise | null> { + const realFile = await resolveContainedFileIfExists(filePath, allowedRoots); + if (!realFile) { + return null; + } + try { + const value = JSON.parse(await fs.readFile(realFile, "utf8")) as unknown; + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; + } catch { + return null; + } +} + +function artifactHref(evidencePath: string, artifactPath: string) { + const params = new URLSearchParams({ + evidencePath, + artifactPath, + }); + return `/api/evidence/artifact?${params.toString()}`; +} + +async function buildProducerContextFile(params: { + allowedRoots: readonly string[]; + artifactPath: string; + filePath: string; + hrefEvidencePath: string; + previewKind: "json" | "text"; + repoRoot: string; +}): Promise { + const realFile = await resolveContainedFileIfExists(params.filePath, params.allowedRoots); + if (!realFile) { + return null; + } + const repoPath = toRepoRelativePath(params.repoRoot, params.filePath); + return { + href: artifactHref(params.hrefEvidencePath, params.artifactPath), + path: repoPath, + preview: await readPreview(realFile, params.previewKind).catch(() => null), + }; +} + +async function buildArtifactView(params: { + allowedArtifactFiles: ReadonlySet; + artifact: QaEvidenceArtifact; + evidenceDir: string; + hrefEvidencePath: string; + repoRoot: string; +}): Promise { + const mediaKind = classifyArtifact(params.artifact.kind, params.artifact.path); + const realFile = await resolveArtifactFileWithinRoots({ + artifactPath: params.artifact.path, + evidenceDir: params.evidenceDir, + repoRoot: params.repoRoot, + }).catch(() => null); + if (!realFile || !params.allowedArtifactFiles.has(realFile)) { + return { + exists: false, + error: realFile + ? "Evidence artifact is not declared by this evidence summary." + : "Evidence artifact not found.", + href: null, + kind: params.artifact.kind, + mediaKind, + path: params.artifact.path, + preview: null, + source: params.artifact.source, + }; + } + return { + exists: true, + error: null, + href: artifactHref(params.hrefEvidencePath, params.artifact.path), + kind: params.artifact.kind, + mediaKind, + path: params.artifact.path, + preview: await readPreview(realFile, mediaKind).catch( + (error: unknown) => `Preview unavailable: ${formatErrorMessage(error)}`, + ), + source: params.artifact.source, + }; +} + +function readString(value: unknown): string | null { + return typeof value === "string" ? value : null; +} + +function readRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function readCountRecord(value: unknown): Record { + const record = readRecord(value); + if (!record) { + return {}; + } + return Object.fromEntries( + Object.entries(record).filter( + (entry): entry is [string, number] => typeof entry[1] === "number", + ), + ); +} + +function readOrderedStringArray(values: Iterable) { + return Array.from( + new Set(Array.from(values).filter((value): value is string => typeof value === "string")), + ); +} + +function readStringArray(values: Iterable) { + return readOrderedStringArray(values).toSorted(); +} + +function readMatrixDimensionIds(value: unknown, fallback: readonly string[]): string[] { + if (!Array.isArray(value)) { + return readOrderedStringArray(fallback); + } + const ids = readOrderedStringArray( + value.map((entry) => { + if (typeof entry === "string") { + return entry; + } + return readString(readRecord(entry)?.id); + }), + ); + for (const fallbackId of fallback) { + if (!ids.includes(fallbackId)) { + ids.push(fallbackId); + } + } + return ids; +} + +function uxMatrixEntryKey( + entry: QaEvidenceSummaryEntry, +): { stage: string; surface: string } | null { + const idMatch = /^ux-matrix\.([a-z0-9-]+)\.([a-z0-9-]+)$/u.exec(entry.test.id); + if (idMatch) { + return { surface: idMatch[1], stage: idMatch[2] }; + } + for (const artifact of entry.execution?.artifacts ?? []) { + const sourceMatch = /^ux-matrix:([a-z0-9-]+):([a-z0-9-]+)$/u.exec(artifact.source); + if (sourceMatch) { + return { surface: sourceMatch[1], stage: sourceMatch[2] }; + } + } + return null; +} + +function buildUxMatrixEvidenceEntryIndex(entries: readonly QaEvidenceSummaryEntry[]) { + const indexed = new Map(); + for (const entry of entries) { + const key = uxMatrixEntryKey(entry); + if (key) { + indexed.set(`${key.surface}:${key.stage}`, entry); + } + } + return indexed; +} + +function readMatrixCells(params: { + matrix: Record | null; + summaryEntries: readonly QaEvidenceSummaryEntry[]; +}): QaEvidenceMatrixCellView[] { + const rawCells = Array.isArray(params.matrix?.cells) + ? params.matrix.cells + .map(readRecord) + .filter((cell): cell is Record => Boolean(cell)) + : []; + const entriesByCell = buildUxMatrixEvidenceEntryIndex(params.summaryEntries); + return rawCells.flatMap((cell): QaEvidenceMatrixCellView[] => { + const surface = readString(cell.surface); + const stage = readString(cell.stage); + const status = readString(cell.status) ?? "proof-gap"; + if (!surface || !stage) { + return []; + } + const entry = + status === "proof-gap" ? null : (entriesByCell.get(`${surface}:${stage}`) ?? null); + const artifacts = entry?.execution?.artifacts ?? []; + const runner = readRecord(cell.runner); + return [ + { + artifactKinds: readStringArray(artifacts.map((artifact) => artifact.kind)), + artifactPaths: artifacts.map((artifact) => artifact.path), + coverageIds: readStringArray(Array.isArray(cell.coverageIds) ? cell.coverageIds : []), + runner: runner + ? { + availability: readString(runner.availability), + command: readString(runner.command), + lane: readString(runner.lane), + workflow: readString(runner.workflow), + } + : null, + stage, + status, + surface, + testId: entry?.test.id ?? null, + title: entry?.test.title ?? null, + }, + ]; + }); +} + +async function candidateProducerRoots(params: { + evidencePath: string; + repoRoot: string; + summaryEntries: readonly QaEvidenceSummaryEntry[]; +}) { + const repoRoot = await fs.realpath(path.resolve(params.repoRoot)); + const evidenceDir = path.dirname(params.evidencePath); + const roots = new Set([evidenceDir]); + for (const entry of params.summaryEntries) { + for (const artifact of entry.execution?.artifacts ?? []) { + const artifactPath = await resolveArtifactFileWithinRoots({ + artifactPath: artifact.path, + evidenceDir, + repoRoot, + }); + if (!artifactPath) { + continue; + } + let current = path.dirname(artifactPath); + while (isInside(repoRoot, current)) { + roots.add(current); + const parent = path.dirname(current); + if (parent === current) { + break; + } + current = parent; + } + } + } + return Array.from(roots); +} + +async function findUxMatrixProducerRoot(params: { + evidencePath: string; + repoRoot: string; + summaryEntries: readonly QaEvidenceSummaryEntry[]; +}) { + for (const candidate of await candidateProducerRoots(params)) { + const [manifest, matrix] = await Promise.all([ + realpathIfExists(path.join(candidate, "manifest.json")), + realpathIfExists(path.join(candidate, "matrix.json")), + ]); + if (manifest && matrix) { + return candidate; + } + } + return null; +} + +async function buildProducerContext(params: { + evidencePath: string; + hrefEvidencePath: string; + repoRoot: string; + summaryEntries: readonly QaEvidenceSummaryEntry[]; +}): Promise { + const rootPath = await findUxMatrixProducerRoot(params); + if (!rootPath) { + return null; + } + const repoRoot = await fs.realpath(path.resolve(params.repoRoot)); + const evidenceDir = path.dirname( + await resolveQaEvidenceFile({ inputPath: params.evidencePath, repoRoot }), + ); + const allowedRoots = [repoRoot, evidenceDir]; + const producerPaths = Object.fromEntries( + UX_MATRIX_PRODUCER_FILES.map((file) => [file.key, path.join(rootPath, file.path)]), + ) as Record<(typeof UX_MATRIX_PRODUCER_FILES)[number]["key"], string>; + const manifestPath = producerPaths.manifest; + const matrixPath = producerPaths.matrix; + const releaseLedgerPath = producerPaths.releaseLedger; + const manifest = await readJsonIfExists(manifestPath, allowedRoots); + const matrix = await readJsonIfExists(matrixPath, allowedRoots); + const releaseLedger = await readJsonIfExists(releaseLedgerPath, allowedRoots); + const producerFiles = Object.fromEntries( + await Promise.all( + UX_MATRIX_PRODUCER_FILES.map(async (file) => [ + file.key, + await buildProducerContextFile({ + allowedRoots, + artifactPath: toRepoRelativePath(repoRoot, producerPaths[file.key]), + filePath: producerPaths[file.key], + hrefEvidencePath: params.hrefEvidencePath, + previewKind: file.previewKind, + repoRoot, + }), + ]), + ), + ) as Record< + (typeof UX_MATRIX_PRODUCER_FILES)[number]["key"], + QaEvidenceProducerContextFile | null + >; + const matrixCells = readMatrixCells({ + matrix, + summaryEntries: params.summaryEntries, + }); + return { + commands: producerFiles.commands, + kind: "ux-matrix", + manifest: + manifest && producerFiles.manifest + ? { + ...producerFiles.manifest, + runId: readString(readRecord(manifest.run)?.runId), + runStatus: readString(readRecord(manifest.run)?.status), + } + : null, + matrix: matrix + ? { + cells: matrixCells, + counts: readCountRecord(matrix.counts), + path: toRepoRelativePath(repoRoot, matrixPath), + stages: readMatrixDimensionIds( + matrix.stages, + matrixCells.map((cell) => cell.stage), + ), + surfaces: readMatrixDimensionIds( + matrix.surfaces, + matrixCells.map((cell) => cell.surface), + ), + } + : null, + preflight: { + adbDevices: producerFiles.adbDevices, + memory: producerFiles.memory, + }, + releaseLedger: + releaseLedger && producerFiles.releaseLedger + ? { + ...producerFiles.releaseLedger, + counts: readCountRecord(releaseLedger.counts), + } + : null, + rootPath: toRepoRelativePath(repoRoot, rootPath), + scorecard: producerFiles.scorecard, + }; +} + +function createConcurrencyLimit(limit: number) { + let active = 0; + const queue: Array<() => void> = []; + return async function runLimited(task: () => Promise): Promise { + if (active >= limit) { + await new Promise((resolve) => { + queue.push(resolve); + }); + } + active += 1; + try { + return await task(); + } finally { + active -= 1; + queue.shift()?.(); + } + }; +} + +export async function buildQaEvidenceGalleryModel(params: { + evidencePath: string; + repoRoot: string; +}): Promise { + const repoRoot = await fs.realpath(path.resolve(params.repoRoot)); + const evidencePath = await resolveQaEvidenceFile({ + inputPath: params.evidencePath, + repoRoot, + }); + const hrefEvidencePath = toRepoRelativePath(repoRoot, evidencePath); + const summary = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(evidencePath, "utf8")) as unknown, + ); + const counts: Record = { + pass: 0, + fail: 0, + blocked: 0, + skipped: 0, + }; + // Resolve the declared-artifact allowlist once; buildArtifactView then only checks membership + // instead of re-reading the evidence file and re-collecting the allowlist per artifact. + const evidenceDir = path.dirname(evidencePath); + const allowedArtifactFiles = await collectDeclaredQaEvidenceArtifactFiles({ + evidencePath, + repoRoot, + summaryEntries: summary.entries, + }); + const limitArtifactView = createConcurrencyLimit(ARTIFACT_VIEW_CONCURRENCY); + const entries = await Promise.all( + summary.entries.map(async (entry): Promise => { + counts[entry.result.status] += 1; + return { + artifacts: await Promise.all( + (entry.execution?.artifacts ?? []).map((artifact) => + limitArtifactView(() => + buildArtifactView({ + allowedArtifactFiles, + artifact, + evidenceDir, + hrefEvidencePath, + repoRoot, + }), + ), + ), + ), + coverage: entry.coverage, + failureReason: entry.result.failure?.reason ?? null, + id: entry.test.id, + kind: entry.test.kind, + sourcePath: entry.test.source?.path ?? null, + status: entry.result.status, + title: entry.test.title, + }; + }), + ); + return { + counts, + entries, + evidenceMode: summary.evidenceMode, + evidencePath: hrefEvidencePath, + generatedAt: summary.generatedAt, + profile: summary.profile ?? null, + producerContext: await buildProducerContext({ + evidencePath, + hrefEvidencePath, + repoRoot, + summaryEntries: summary.entries, + }), + schemaVersion: summary.schemaVersion, + }; +} diff --git a/extensions/qa-lab/src/lab-server.test.ts b/extensions/qa-lab/src/lab-server.test.ts index d48c4239135..3a81f0cb224 100644 --- a/extensions/qa-lab/src/lab-server.test.ts +++ b/extensions/qa-lab/src/lab-server.test.ts @@ -354,11 +354,9 @@ describe("qa-lab server", () => { reject(new Error("gateway stop failed")); return; } - abortSignal.addEventListener( - "abort", - () => reject(new Error("gateway stop failed")), - { once: true }, - ); + abortSignal.addEventListener("abort", () => reject(new Error("gateway stop failed")), { + once: true, + }); }), ); @@ -450,6 +448,119 @@ describe("qa-lab server", () => { await expectFileMissing(outputPath); }); + it("serves evidence artifact HEAD metadata and streams GET bodies", async () => { + const repoRoot = await createQaLabRepoRootFixture(); + const evidenceDir = path.join(repoRoot, ".artifacts", "qa-e2e", "server"); + await mkdir(evidenceDir, { recursive: true }); + await writeFile(path.join(evidenceDir, "artifact.log"), "streamed body\n", "utf8"); + await writeFile( + path.join(evidenceDir, "qa-evidence.json"), + `${JSON.stringify( + { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-17T12:00:00.000Z", + evidenceMode: "full", + entries: [ + { + test: { + kind: "vitest-test", + id: "qa-lab.server-artifact", + title: "Server artifact", + }, + coverage: [{ id: "qa.artifact", role: "primary" }], + execution: { + runner: "vitest", + environment: { + ref: "server-test", + os: "darwin", + nodeVersion: "v24.0.0", + }, + provider: { + id: "mock-openai", + live: false, + model: { name: "mock-openai/gpt-5.5", ref: "mock-openai/gpt-5.5" }, + }, + packageSource: { kind: "source-checkout" }, + artifacts: [{ kind: "log", path: "artifact.log", source: "vitest" }], + }, + result: { status: "pass" }, + }, + ], + }, + null, + 2, + )}\n`, + "utf8", + ); + const lab = await startQaLabServerForTest({ + host: "127.0.0.1", + port: 0, + repoRoot, + }); + cleanups.push(async () => { + await lab.stop(); + }); + const evidenceUrl = new URL("/api/evidence", lab.baseUrl); + evidenceUrl.searchParams.set("path", ".artifacts/qa-e2e/server/qa-evidence.json"); + + const evidenceResponse = await fetchWithRetry(evidenceUrl.toString()); + expect(evidenceResponse.status).toBe(200); + expect(evidenceResponse.headers.get("cache-control")).toBe("no-store"); + expect((await evidenceResponse.json()) as unknown).toMatchObject({ + evidence: { + counts: { + pass: 1, + }, + entries: [{ id: "qa-lab.server-artifact" }], + }, + }); + + // A missing evidence path must return a controlled JSON error, not a reset connection + // (the model must build before any success header is written). + const missingEvidenceUrl = new URL("/api/evidence", lab.baseUrl); + missingEvidenceUrl.searchParams.set("path", ".artifacts/qa-e2e/server/does-not-exist.json"); + const missingEvidenceResponse = await fetchWithRetry(missingEvidenceUrl.toString()); + expect(missingEvidenceResponse.status).toBe(404); + expect(await missingEvidenceResponse.text()).not.toBe(""); + + const artifactUrl = new URL("/api/evidence/artifact", lab.baseUrl); + artifactUrl.searchParams.set("evidencePath", ".artifacts/qa-e2e/server/qa-evidence.json"); + artifactUrl.searchParams.set("artifactPath", "artifact.log"); + + const headResponse = await fetchWithRetry(artifactUrl.toString(), { method: "HEAD" }); + expect(headResponse.status).toBe(200); + expect(headResponse.headers.get("content-type")).toBe("text/plain; charset=utf-8"); + expect(headResponse.headers.get("content-length")).toBe("14"); + expect(headResponse.headers.get("cache-control")).toBe("no-store"); + expect(headResponse.headers.get("x-content-type-options")).toBe("nosniff"); + expect(await headResponse.text()).toBe(""); + + const getResponse = await fetchWithRetry(artifactUrl.toString()); + expect(getResponse.status).toBe(200); + expect(getResponse.headers.get("content-length")).toBe("14"); + expect(getResponse.headers.get("cache-control")).toBe("no-store"); + expect(getResponse.headers.get("x-content-type-options")).toBe("nosniff"); + expect(await getResponse.text()).toBe("streamed body\n"); + + await writeFile(path.join(evidenceDir, "undeclared.log"), "hidden\n", "utf8"); + const undeclaredUrl = new URL(artifactUrl); + undeclaredUrl.searchParams.set("artifactPath", "undeclared.log"); + const undeclaredResponse = await fetchWithRetry(undeclaredUrl.toString()); + expect(undeclaredResponse.status).toBe(403); + + const outsideDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-outside-artifact-")); + cleanups.push(async () => { + await rm(outsideDir, { recursive: true, force: true }); + }); + const outsideArtifact = path.join(outsideDir, "outside.log"); + await writeFile(outsideArtifact, "outside\n", "utf8"); + const outsideUrl = new URL(artifactUrl); + outsideUrl.searchParams.set("artifactPath", outsideArtifact); + const outsideResponse = await fetchWithRetry(outsideUrl.toString()); + expect(outsideResponse.status).toBe(404); + }); + it("returns controlled errors for oversized JSON body reads", async () => { const req = { headers: { "content-length": String(1024 * 1024 + 1) }, diff --git a/extensions/qa-lab/src/lab-server.ts b/extensions/qa-lab/src/lab-server.ts index 918a6840d8d..a9e4979287c 100644 --- a/extensions/qa-lab/src/lab-server.ts +++ b/extensions/qa-lab/src/lab-server.ts @@ -16,6 +16,11 @@ import { writeQaRequestBodyLimitError, } from "./bus-server.js"; import { createQaBusState, type QaBusState } from "./bus-state.js"; +import { + QaEvidenceGalleryError, + buildQaEvidenceGalleryModel, + resolveQaEvidenceArtifactFile, +} from "./evidence-gallery.js"; import { createQaRunnerRuntime } from "./harness-runtime.js"; import { isCaptureQueryPreset, @@ -69,6 +74,10 @@ export function writeQaLabServerError(res: Parameters[0], err if (writeQaRequestBodyLimitError(res, error)) { return; } + if (error instanceof QaEvidenceGalleryError) { + writeError(res, error.statusCode, error.message); + return; + } writeError(res, 500, error); } @@ -172,6 +181,38 @@ function normalizeQaLabCleanupError(error: unknown): Error { return error instanceof Error ? error : new Error(formatErrorMessage(error)); } +function detectQaEvidenceArtifactContentType(filePath: string): string { + const lower = filePath.toLowerCase(); + if (lower.endsWith(".png")) { + return "image/png"; + } + if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) { + return "image/jpeg"; + } + if (lower.endsWith(".gif")) { + return "image/gif"; + } + if (lower.endsWith(".webp")) { + return "image/webp"; + } + if (lower.endsWith(".webm")) { + return "video/webm"; + } + if (lower.endsWith(".mp4")) { + return "video/mp4"; + } + if (lower.endsWith(".mov")) { + return "video/quicktime"; + } + if (lower.endsWith(".json") || lower.endsWith(".jsonl")) { + return "application/json; charset=utf-8"; + } + if (lower.endsWith(".md") || lower.endsWith(".txt") || lower.endsWith(".log")) { + return "text/plain; charset=utf-8"; + } + return "application/octet-stream"; +} + async function startQaGatewayLoop(params: { state: QaBusState; baseUrl: string }) { const runtime = createQaRunnerRuntime(); setQaChannelRuntime(runtime); @@ -384,6 +425,59 @@ export async function startQaLabServer( writeJson(res, 200, { run: latestScenarioRun }); return; } + if (req.method === "GET" && url.pathname === "/api/evidence") { + const evidencePath = + url.searchParams.get("path")?.trim() || runnerSnapshot.artifacts?.evidencePath; + if (!evidencePath) { + res.writeHead(200, { + "content-type": "application/json; charset=utf-8", + "cache-control": "no-store", + }); + res.end(JSON.stringify({ evidence: null })); + return; + } + // Build the model before sending any headers so a thrown QaEvidenceGalleryError + // still routes through writeQaLabServerError (writing headers first would make the + // error response throw ERR_HTTP_HEADERS_SENT and reset the connection). + const evidence = await buildQaEvidenceGalleryModel({ evidencePath, repoRoot }); + res.writeHead(200, { + "content-type": "application/json; charset=utf-8", + "cache-control": "no-store", + }); + res.end(JSON.stringify({ evidence })); + return; + } + if ( + (req.method === "GET" || req.method === "HEAD") && + url.pathname === "/api/evidence/artifact" + ) { + const evidencePath = url.searchParams.get("evidencePath")?.trim(); + const artifactPath = url.searchParams.get("artifactPath")?.trim(); + if (!evidencePath || !artifactPath) { + writeError(res, 400, "Missing evidencePath or artifactPath"); + return; + } + const artifactFile = await resolveQaEvidenceArtifactFile({ + artifactPath, + evidencePath, + repoRoot, + }); + const artifactStats = await fs.promises.stat(artifactFile); + res.writeHead(200, { + "content-type": detectQaEvidenceArtifactContentType(artifactFile), + "content-length": artifactStats.size, + "cache-control": "no-store", + "x-content-type-options": "nosniff", + }); + if (req.method === "HEAD") { + res.end(); + return; + } + fs.createReadStream(artifactFile) + .on("error", (error) => res.destroy(normalizeQaLabCleanupError(error))) + .pipe(res); + return; + } if (req.method === "GET" && url.pathname === "/api/capture/sessions") { writeJson(res, 200, { sessions: captureStore.listSessions(50), diff --git a/extensions/qa-lab/web/src/app.ts b/extensions/qa-lab/web/src/app.ts index 613f37a0378..0af085cde78 100644 --- a/extensions/qa-lab/web/src/app.ts +++ b/extensions/qa-lab/web/src/app.ts @@ -4,6 +4,7 @@ import { normalizeCaptureSavedView, normalizeCaptureSavedViews } from "./capture import { formatErrorMessage } from "./errors.js"; import { type Bootstrap, + type EvidenceEnvelope, type OutcomesEnvelope, type ReportEnvelope, type RunnerSelection, @@ -160,6 +161,11 @@ function isEditableElement(target: EventTarget | null): boolean { } export async function createQaLabApp(root: HTMLDivElement) { + const initialUrl = new URL(window.location.href); + const initialEvidencePath = + initialUrl.searchParams.get("evidencePath")?.trim() || + initialUrl.searchParams.get("path")?.trim() || + ""; const state: UiState = { theme: detectTheme(), bootstrap: null, @@ -203,6 +209,13 @@ export async function createQaLabApp(root: HTMLDivElement) { captureErrorsOnly: false, captureCoverage: null, captureStartupStatus: null, + evidence: null, + evidenceArtifactFilter: "all", + evidenceError: null, + evidenceLoading: false, + evidencePathDraft: initialEvidencePath, + evidenceSearchText: "", + evidenceStatusFilter: "all", captureControlsExpanded: false, captureSummaryExpanded: false, captureSavedViews: loadCaptureSavedViews(), @@ -213,10 +226,11 @@ export async function createQaLabApp(root: HTMLDivElement) { capturePinnedLaneIds: [], selectedCaptureSessionIds: [], selectedCaptureEventKey: null, + selectedEvidenceEntryId: null, selectedConversationId: null, selectedThreadId: null, selectedScenarioId: null, - activeTab: "chat", + activeTab: initialUrl.pathname === "/evidence" || initialEvidencePath ? "evidence" : "chat", runnerDraft: null, runnerDraftDirty: false, composer: { @@ -316,6 +330,17 @@ export async function createQaLabApp(root: HTMLDivElement) { ccli: state.captureCollapsedLaneIds.join(","), ccpi: state.capturePinnedLaneIds.join(","), er: state.error, + el: state.evidenceLoading, + ee: state.evidenceError, + ep: state.evidence?.evidencePath ?? null, + eg: state.evidence?.generatedAt ?? null, + ecnt: state.evidence?.entries.length ?? 0, + eac: state.evidence?.entries.reduce((sum, entry) => sum + entry.artifacts.length, 0) ?? 0, + epr: state.evidence?.producerContext?.rootPath ?? null, + esf: state.evidenceStatusFilter, + eaf: state.evidenceArtifactFilter, + esq: state.evidenceSearchText, + ese: state.selectedEvidenceEntryId, }); } @@ -338,6 +363,9 @@ export async function createQaLabApp(root: HTMLDivElement) { state.snapshot = snapshot; state.latestReport = report.report ?? bootstrap.latestReport; state.scenarioRun = outcomes.run; + if (!state.evidencePathDraft.trim() && bootstrap.runner.artifacts?.evidencePath) { + state.evidencePathDraft = bootstrap.runner.artifacts.evidencePath; + } if (!state.runnerDraft || !state.runnerDraftDirty) { state.runnerDraft = { ...bootstrap.runner.selection, @@ -617,6 +645,37 @@ export async function createQaLabApp(root: HTMLDivElement) { } } + async function loadEvidence(pathOverride?: string) { + const evidencePath = (pathOverride ?? state.evidencePathDraft).trim(); + if (!evidencePath) { + state.evidenceError = "Evidence path is required."; + render(); + return; + } + state.evidenceLoading = true; + state.evidenceError = null; + render(); + try { + const payload = await getJson( + `/api/evidence?path=${encodeURIComponent(evidencePath)}`, + ); + state.evidence = payload.evidence; + state.evidencePathDraft = payload.evidence?.evidencePath ?? evidencePath; + state.selectedEvidenceEntryId = payload.evidence?.entries[0]?.id ?? null; + const url = new URL(window.location.href); + url.pathname = "/evidence"; + url.searchParams.set("path", state.evidencePathDraft); + window.history.replaceState(null, "", `${url.pathname}${url.search}`); + } catch (error) { + state.evidence = null; + state.selectedEvidenceEntryId = null; + state.evidenceError = formatErrorMessage(error); + } finally { + state.evidenceLoading = false; + render(); + } + } + async function sendKickoff() { state.busy = true; state.error = null; @@ -832,6 +891,19 @@ export async function createQaLabApp(root: HTMLDivElement) { root .querySelector("[data-action='download-report']") ?.addEventListener("click", downloadReport); + root + .querySelector("[data-action='load-evidence']") + ?.addEventListener("click", () => void loadEvidence()); + root + .querySelector("[data-action='open-run-evidence']") + ?.addEventListener("click", () => { + const evidencePath = state.bootstrap?.runner.artifacts?.evidencePath; + if (!evidencePath) { + return; + } + state.activeTab = "evidence"; + void loadEvidence(evidencePath); + }); /* Scenario All/None */ root @@ -899,6 +971,53 @@ export async function createQaLabApp(root: HTMLDivElement) { })); }); + root.querySelector("#evidence-path")?.addEventListener("input", (e) => { + state.evidencePathDraft = (e.currentTarget as HTMLInputElement).value; + }); + root.querySelector("#evidence-path")?.addEventListener("keydown", (e) => { + if (e.key === "Enter") { + e.preventDefault(); + void loadEvidence(); + } + }); + root + .querySelector("#evidence-status-filter") + ?.addEventListener("change", (e) => { + const value = (e.currentTarget as HTMLSelectElement).value; + state.evidenceStatusFilter = + value === "pass" || value === "fail" || value === "blocked" || value === "skipped" + ? value + : "all"; + state.selectedEvidenceEntryId = null; + render(); + }); + root + .querySelector("#evidence-artifact-filter") + ?.addEventListener("change", (e) => { + const value = (e.currentTarget as HTMLSelectElement).value; + state.evidenceArtifactFilter = + value === "image" || + value === "video" || + value === "json" || + value === "text" || + value === "file" + ? value + : "all"; + state.selectedEvidenceEntryId = null; + render(); + }); + root.querySelector("#evidence-search")?.addEventListener("input", (e) => { + state.evidenceSearchText = (e.currentTarget as HTMLInputElement).value; + state.selectedEvidenceEntryId = null; + render(); + }); + root.querySelectorAll("[data-evidence-entry-id]").forEach((node) => { + node.addEventListener("click", () => { + state.selectedEvidenceEntryId = node.dataset.evidenceEntryId ?? null; + render(); + }); + }); + root.querySelector("#capture-session")?.addEventListener("change", (e) => { state.selectedCaptureSessionIds = readMultiSelect(e.currentTarget as HTMLSelectElement); state.selectedCaptureEventKey = null; @@ -1643,6 +1762,9 @@ export async function createQaLabApp(root: HTMLDivElement) { render(); await refresh(); + if (initialEvidencePath) { + await loadEvidence(initialEvidencePath); + } void pollUiVersion(); setInterval(() => void refresh(), 1_000); setInterval(() => void pollUiVersion(), 1_000); diff --git a/extensions/qa-lab/web/src/styles.css b/extensions/qa-lab/web/src/styles.css index 21db950ebe3..bdd532aba00 100644 --- a/extensions/qa-lab/web/src/styles.css +++ b/extensions/qa-lab/web/src/styles.css @@ -371,6 +371,27 @@ select { border-right-color: transparent; } +.app-shell--evidence-focus .sidebar { + width: 0; + min-width: 0; + border-right-color: transparent; +} + +.app-shell--evidence-focus .main-content { + background: + linear-gradient( + 180deg, + color-mix(in srgb, var(--bg-app) 88%, var(--bg-surface)) 0, + var(--bg-app) 240px + ), + var(--bg-app); +} + +.app-shell--evidence-focus .tab-content { + overflow-x: hidden; + overflow-y: auto; +} + .sidebar-panel-tabs { display: flex; gap: 6px; @@ -1522,6 +1543,650 @@ select { border: 1px solid var(--border); } +/* --- Evidence view --- */ +.evidence-view { + display: flex; + flex-direction: column; + height: 100%; + overflow-x: hidden; + overflow-y: auto; + padding: 12px 16px 16px; + gap: 10px; + max-width: 1680px; + width: 100%; + margin: 0 auto; +} + +.app-shell--evidence-focus .evidence-view { + height: auto; + min-height: 100%; + overflow: visible; +} + +.evidence-toolbar { + flex-shrink: 0; + display: grid; + grid-template-columns: minmax(220px, 0.65fr) minmax(0, 1.35fr) minmax(380px, 0.9fr); + gap: 12px; + align-items: end; + padding: 10px 12px; + border: 1px solid var(--border); + border-radius: 10px; + background: color-mix(in srgb, var(--bg-surface) 94%, transparent); + box-shadow: var(--shadow); +} + +.evidence-toolbar-intro { + min-width: 0; +} + +.evidence-toolbar-intro p { + margin: 0; + color: var(--text-secondary); + font-size: 13px; + line-height: 1.4; +} + +.evidence-toolbar-main, +.evidence-filters { + display: grid; + gap: 10px; + align-items: end; +} + +.evidence-toolbar-main { + grid-template-columns: minmax(0, 1fr) auto; +} + +.evidence-filters { + grid-template-columns: minmax(112px, 0.8fr) minmax(112px, 0.8fr) minmax(0, 1.4fr); +} + +.evidence-filters label { + display: grid; + gap: 4px; + color: var(--text-tertiary); + font-size: 11px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.evidence-summary-row { + flex-shrink: 0; + display: grid; + grid-template-columns: repeat(6, minmax(0, 1fr)); + gap: 6px; +} + +.evidence-metric { + padding: 8px 10px; + border: 1px solid var(--border); + border-radius: 7px; + background: color-mix(in srgb, var(--bg-surface) 92%, transparent); +} + +.evidence-metric span { + display: block; + color: var(--text-tertiary); + font-size: 10px; + font-weight: 800; + text-transform: uppercase; + letter-spacing: 0.06em; +} + +.evidence-metric strong { + display: block; + margin-top: 1px; + font-size: 18px; + line-height: 1.1; +} + +.evidence-metric-pass strong { + color: var(--success); +} + +.evidence-metric-fail strong { + color: var(--danger); +} + +.evidence-metric-blocked strong { + color: var(--warning); +} + +.evidence-metric-skipped strong { + color: var(--text-secondary); +} + +.evidence-producer-panel { + flex-shrink: 0; + display: grid; + gap: 10px; + padding: 12px; + border: 1px solid var(--border); + border-radius: 10px; + background: color-mix(in srgb, var(--bg-surface) 94%, transparent); + box-shadow: var(--shadow); +} + +.evidence-producer-head { + display: flex; + justify-content: space-between; + gap: 18px; + align-items: flex-start; +} + +.evidence-producer-head h2 { + margin-top: 2px; + color: var(--text); + font-size: 17px; + line-height: 1.2; +} + +.evidence-producer-head p { + margin-top: 3px; + max-width: 720px; + color: var(--text-secondary); + font-size: 12px; +} + +.evidence-producer-run { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 8px; + align-items: center; + color: var(--text-tertiary); + font-size: 12px; +} + +.evidence-producer-grid { + display: grid; + grid-template-columns: repeat(6, minmax(0, 1fr)); + gap: 6px; +} + +.evidence-producer-metric { + min-width: 0; + padding: 8px 10px; + border: 1px solid var(--border); + border-radius: 7px; + background: color-mix(in srgb, var(--bg-surface) 72%, var(--bg-inset)); +} + +.evidence-producer-metric span { + display: block; + overflow: hidden; + color: var(--text-tertiary); + font-size: 10px; + font-weight: 800; + text-transform: uppercase; + text-overflow: ellipsis; + white-space: nowrap; + letter-spacing: 0.06em; +} + +.evidence-producer-metric strong { + display: block; + margin-top: 1px; + color: var(--text); + font-size: 17px; + line-height: 1.1; +} + +.evidence-producer-status-row { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 10px; +} + +.evidence-producer-status-row > div { + min-width: 0; + display: grid; + gap: 6px; +} + +.evidence-producer-drilldowns { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 8px; + align-items: start; +} + +.evidence-producer-drilldown { + min-width: 0; + overflow: hidden; + border: 1px solid var(--border); + border-radius: 8px; + background: color-mix(in srgb, var(--bg-surface) 78%, var(--bg-inset)); +} + +.evidence-producer-drilldown summary { + display: grid; + grid-template-columns: minmax(100px, auto) minmax(0, 1fr); + gap: 8px; + align-items: center; + min-height: 34px; + padding: 8px 10px; + cursor: pointer; + color: var(--text); + font-size: 12px; + font-weight: 800; +} + +.evidence-producer-drilldown summary::marker { + color: var(--text-tertiary); +} + +.evidence-producer-drilldown summary .capture-mono { + overflow: hidden; + color: var(--text-tertiary); + font-size: 11px; + font-weight: 500; + text-overflow: ellipsis; + white-space: nowrap; +} + +.evidence-producer-drilldown-body { + display: grid; + gap: 8px; + padding: 0 10px 10px; +} + +.evidence-producer-preview { + max-height: 220px; + overflow: auto; + margin: 0; + border-radius: 6px; + font-size: 11px; +} + +.evidence-producer-links { + display: flex; + flex-wrap: wrap; + gap: 6px; + min-width: 0; +} + +.evidence-matrix-mini { + overflow: visible; + border: 1px solid var(--border); + border-radius: 8px; + background: color-mix(in srgb, var(--bg-surface) 68%, var(--bg-inset)); +} + +.evidence-matrix-mini table { + width: 100%; + min-width: 980px; + border-collapse: collapse; + table-layout: fixed; +} + +.evidence-matrix-mini th, +.evidence-matrix-mini td { + padding: 4px; + border-bottom: 1px solid var(--border); + border-left: 1px solid var(--border); + text-align: center; + vertical-align: middle; +} + +.evidence-matrix-mini th:first-child, +.evidence-matrix-mini td:first-child { + border-left: 0; +} + +.evidence-matrix-mini thead th, +.evidence-matrix-mini tbody th { + color: var(--text-secondary); + background: color-mix(in srgb, var(--bg-surface) 84%, var(--bg-inset)); + font-size: 10px; + font-weight: 800; + line-height: 1.15; + text-transform: uppercase; +} + +.evidence-matrix-mini thead th { + position: sticky; + top: 0; + z-index: 1; +} + +.evidence-matrix-mini tbody th { + position: sticky; + left: 0; + z-index: 1; + width: 116px; + text-align: left; +} + +.evidence-matrix-cell { + display: inline-flex; + align-items: center; + justify-content: center; + width: 100%; + min-height: 22px; + padding: 2px 4px; + border: 1px solid var(--border); + border-radius: 5px; + color: var(--text-secondary); + background: var(--bg-surface); + font-size: 10px; + font-weight: 800; + line-height: 1.1; + text-transform: uppercase; +} + +button.evidence-matrix-cell { + white-space: normal; +} + +.evidence-matrix-cell-action:hover { + border-color: var(--accent); + background: var(--accent-soft); +} + +.evidence-matrix-cell-pass, +.evidence-matrix-cell-fixed-in-pr { + color: var(--success-text); + border-color: color-mix(in srgb, var(--success) 42%, var(--border)); + background: var(--success-soft); +} + +.evidence-matrix-cell-fail, +.evidence-matrix-cell-automation-issue { + color: var(--danger-text); + border-color: color-mix(in srgb, var(--danger) 44%, var(--border)); + background: var(--danger-soft); +} + +.evidence-matrix-cell-blocked, +.evidence-matrix-cell-environment-issue { + color: var(--warning-text); + border-color: color-mix(in srgb, var(--warning) 44%, var(--border)); + background: var(--warning-soft); +} + +.evidence-matrix-cell-proof-gap { + color: var(--text-tertiary); + border-style: dashed; + background: transparent; +} + +.evidence-matrix-cell-not-applicable, +.evidence-matrix-cell-missing { + color: var(--text-tertiary); + background: color-mix(in srgb, var(--bg-surface) 58%, var(--bg-inset)); +} + +.evidence-meta-line { + flex-shrink: 0; + display: flex; + gap: 10px; + align-items: center; + min-width: 0; + color: var(--text-tertiary); + font-size: 12px; +} + +.evidence-meta-line span { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.evidence-workspace { + flex: 0 0 min(640px, calc(100vh - 220px)); + min-height: 420px; + display: grid; + grid-template-columns: minmax(320px, 0.34fr) minmax(0, 1fr); + gap: 10px; + overflow: hidden; +} + +.evidence-list, +.evidence-inspector { + min-height: 0; + border: 1px solid var(--border); + border-radius: 10px; + background: var(--bg-surface); + overflow: hidden; + box-shadow: var(--shadow); +} + +.evidence-list { + display: flex; + flex-direction: column; +} + +.evidence-list-header { + flex-shrink: 0; + display: flex; + justify-content: space-between; + gap: 10px; + padding: 10px 12px; + border-bottom: 1px solid var(--border); + color: var(--text-tertiary); + font-size: 12px; + font-weight: 600; +} + +.evidence-list-scroll, +.evidence-inspector { + overflow: auto; +} + +.evidence-list-scroll { + display: grid; + gap: 6px; + padding: 8px; +} + +.evidence-entry-card { + display: grid; + gap: 8px; + width: 100%; + padding: 10px; + text-align: left; + background: color-mix(in srgb, var(--bg-surface) 70%, var(--bg-inset)); + border-color: var(--border); + border-radius: 8px; +} + +.evidence-entry-card:hover, +.evidence-entry-card.selected { + border-color: var(--accent); + background: color-mix(in srgb, var(--accent-soft) 72%, var(--bg-surface)); +} + +.evidence-entry-card-top { + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + gap: 9px; + align-items: start; +} + +.evidence-entry-title { + color: var(--text); + font-size: 13px; + font-weight: 700; + line-height: 1.3; + white-space: normal; +} + +.evidence-entry-meta, +.evidence-artifact-source, +.evidence-source-path { + color: var(--text-tertiary); + font-size: 12px; + overflow-wrap: anywhere; +} + +.evidence-entry-artifacts { + display: flex; + flex-wrap: wrap; + gap: 5px; +} + +.evidence-artifact-badge { + max-width: 150px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + padding: 2px 6px; + border-radius: 999px; + border: 1px solid var(--border); + color: var(--text-secondary); + background: var(--bg-surface); + font-size: 10px; + font-weight: 700; +} + +.evidence-artifact-badge-missing { + color: var(--danger); + border-color: var(--danger); +} + +.evidence-detail { + min-height: 100%; + display: grid; + align-content: start; + gap: 12px; + padding: 16px; +} + +.evidence-detail-header { + display: flex; + justify-content: space-between; + gap: 16px; + align-items: start; +} + +.evidence-detail-header h2 { + margin-top: 4px; + font-size: 22px; + line-height: 1.18; + letter-spacing: -0.02em; +} + +.evidence-detail-section { + display: grid; + gap: 8px; +} + +.evidence-artifact-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); + gap: 10px; +} + +.evidence-artifact-card { + min-width: 0; + overflow: hidden; + border: 1px solid var(--border); + border-radius: 10px; + background: color-mix(in srgb, var(--bg-surface) 72%, var(--bg-inset)); +} + +.evidence-artifact-card header { + display: flex; + justify-content: space-between; + gap: 10px; + padding: 10px; + border-bottom: 1px solid var(--border); +} + +.evidence-artifact-title { + font-size: 13px; + font-weight: 800; +} + +.evidence-artifact-card img, +.evidence-artifact-card video { + display: block; + width: 100%; + max-height: 430px; + aspect-ratio: 16 / 10; + object-fit: contain; + background: var(--bg-app); +} + +.evidence-artifact-deferred { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + padding: 12px; + color: var(--text-secondary); + font-size: 12px; +} + +.evidence-artifact-preview-shell { + border-top: 1px solid var(--border); +} + +.evidence-artifact-preview-shell summary { + cursor: pointer; + padding: 10px 12px; + color: var(--text-secondary); + font-size: 12px; + font-weight: 700; +} + +.evidence-artifact-preview { + max-height: 340px; + overflow: auto; + border-radius: 0; + border: 0; +} + +.evidence-artifact-card footer { + padding: 8px 10px; + border-top: 1px solid var(--border); + color: var(--text-tertiary); + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 11px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.evidence-empty { + flex: 1; + display: grid; + place-content: center; + justify-items: center; + gap: 8px; + color: var(--text-secondary); + text-align: center; +} + +.evidence-empty h2 { + color: var(--text); + font-size: 20px; +} + +.evidence-empty p { + max-width: 620px; +} + +@media (max-width: 980px) { + .evidence-summary-row, + .evidence-filters, + .evidence-toolbar, + .evidence-producer-grid, + .evidence-producer-status-row, + .evidence-producer-drilldowns, + .evidence-workspace { + grid-template-columns: 1fr; + } + + .evidence-workspace { + overflow: auto; + } +} + /* --- Events view --- */ .events-view { display: flex; diff --git a/extensions/qa-lab/web/src/ui-render.test.ts b/extensions/qa-lab/web/src/ui-render.test.ts new file mode 100644 index 00000000000..de2669b17b7 --- /dev/null +++ b/extensions/qa-lab/web/src/ui-render.test.ts @@ -0,0 +1,258 @@ +// Qa Lab UI render tests cover evidence gallery affordances. +import { describe, expect, it } from "vitest"; +import { renderQaLabUi, type UiState } from "./ui-render.js"; + +function evidenceState(overrides: Partial = {}): UiState { + return { + activeTab: "evidence", + bootstrap: null, + busy: false, + captureCollapsedLaneIds: [], + captureControlsExpanded: true, + captureCoverage: null, + captureDetailPlacement: "right", + captureDetailSplitDragging: false, + captureDetailSplitPct: 35, + captureDetailView: "overview", + captureErrorsOnly: false, + captureEvents: [], + captureFlowDetailLayout: null, + captureGroupMode: "none", + captureHeaderMode: "key", + captureHostFilter: [], + captureKindFilter: [], + capturePayloadDetailLayout: null, + capturePayloadEventFilter: "", + capturePayloadEventSort: "stream", + capturePayloadExtent: "preview", + capturePinnedLaneIds: [], + capturePreferredDetailView: null, + captureProviderFilter: [], + captureQueryPreset: "none", + captureQueryRows: [], + captureSavedViews: [], + captureSearchText: "", + captureSelectedSessionsExpanded: true, + captureSessions: [], + captureStartupStatus: null, + captureSummaryExpanded: true, + captureTimelineBrushAnchorPct: null, + captureTimelineBrushCurrentPct: null, + captureTimelineFocusSelectedFlow: false, + captureTimelineFocusedLaneMode: "all", + captureTimelineFocusedLaneThreshold: "any", + captureTimelineLaneMode: "domain", + captureTimelineLaneSearch: "", + captureTimelineLaneSort: "most-events", + captureTimelinePreviousLaneSort: null, + captureTimelineSparklineMode: "session-relative", + captureTimelineWindowEndPct: null, + captureTimelineWindowStartPct: null, + captureTimelineZoom: 100, + captureViewMode: "list", + composer: { + conversationId: "", + conversationKind: "direct", + senderId: "", + senderName: "", + text: "", + }, + error: null, + evidence: null, + evidenceArtifactFilter: "all", + evidenceError: null, + evidenceLoading: false, + evidencePathDraft: "", + evidenceSearchText: "", + evidenceStatusFilter: "all", + latestReport: null, + runnerDraft: null, + runnerDraftDirty: false, + scenarioRun: null, + selectedCaptureEventKey: null, + selectedCaptureSessionIds: [], + selectedConversationId: null, + selectedEvidenceEntryId: null, + selectedScenarioId: null, + selectedThreadId: null, + sidebarCollapsed: false, + sidebarPanel: "scenarios", + snapshot: null, + theme: "light", + ...overrides, + }; +} + +describe("QA Lab UI evidence render", () => { + it("maps blocked and skipped evidence statuses to styled tones", () => { + const html = renderQaLabUi( + evidenceState({ + evidence: { + counts: { blocked: 1, fail: 0, pass: 0, skipped: 1 }, + entries: [ + { + artifacts: [], + coverage: [{ id: "qa.blocked", role: "primary" }], + failureReason: "Environment unavailable", + id: "qa-lab.blocked", + kind: "script-test", + sourcePath: "scripts/blocked.ts", + status: "blocked", + title: "Blocked evidence", + }, + { + artifacts: [], + coverage: [{ id: "qa.skipped", role: "primary" }], + failureReason: null, + id: "qa-lab.skipped", + kind: "vitest-test", + sourcePath: "extensions/qa-lab/src/skipped.test.ts", + status: "skipped", + title: "Skipped evidence", + }, + ], + evidenceMode: "full", + evidencePath: ".artifacts/qa-e2e/suite/qa-evidence.json", + generatedAt: "2026-06-17T12:00:00.000Z", + producerContext: null, + profile: null, + schemaVersion: 2, + }, + selectedEvidenceEntryId: "qa-lab.blocked", + }), + ); + + expect(html).toContain("badge-pending"); + expect(html).toContain("badge-skip"); + expect(html).toContain("scenario-item-dot-pending"); + expect(html).toContain("scenario-item-dot-skip"); + expect(html).not.toContain("badge-blocked"); + expect(html).not.toContain("badge-skipped"); + expect(html).not.toContain("scenario-item-dot-blocked"); + }); + + it("links executed UX Matrix cells to evidence entries and leaves proof gaps unlinked", () => { + const html = renderQaLabUi( + evidenceState({ + evidence: { + counts: { blocked: 0, fail: 0, pass: 1, skipped: 0 }, + entries: [ + { + artifacts: [ + { + error: null, + exists: true, + href: "/api/evidence/artifact?artifactPath=screenshot.png", + kind: "screenshot", + mediaKind: "image", + path: "screenshot.png", + preview: null, + source: "ux-matrix:web-ui:first-run", + }, + { + error: null, + exists: true, + href: "/api/evidence/artifact?artifactPath=recording.gif", + kind: "motion-preview-gif", + mediaKind: "image", + path: "recording.gif", + preview: null, + source: "ux-matrix:web-ui:first-run", + }, + { + error: null, + exists: true, + href: "/api/evidence/artifact?artifactPath=recording.webm", + kind: "video", + mediaKind: "video", + path: "recording.webm", + preview: null, + source: "ux-matrix:web-ui:first-run", + }, + ], + coverage: [{ id: "ui.control", role: "primary" }], + failureReason: null, + id: "ux-matrix.web-ui.first-run", + kind: "ux-matrix-cell", + sourcePath: "scripts/ux-matrix/dashboard.ts", + status: "pass", + title: "UX Matrix: web-ui / first-run", + }, + ], + evidenceMode: "full", + evidencePath: ".artifacts/qa-e2e/suite/qa-evidence.json", + generatedAt: "2026-06-17T12:00:00.000Z", + producerContext: { + commands: null, + kind: "ux-matrix", + manifest: { + href: "/api/evidence/artifact?artifactPath=manifest.json", + path: "manifest.json", + preview: null, + runId: "run-1", + runStatus: "pass", + }, + matrix: { + cells: [ + { + artifactKinds: ["screenshot"], + artifactPaths: ["screenshot.png"], + coverageIds: ["ui.control"], + runner: { + availability: "local", + command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + lane: "web-ui-playwright", + workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + }, + stage: "first-run", + status: "pass", + surface: "web-ui", + testId: "ux-matrix.web-ui.first-run", + title: "UX Matrix: web-ui / first-run", + }, + { + artifactKinds: [], + artifactPaths: [], + coverageIds: ["cli-entrypoint"], + runner: { + availability: "local", + command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + lane: "cli-status", + workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + }, + stage: "first-run", + status: "proof-gap", + surface: "cli", + testId: null, + title: null, + }, + ], + counts: { pass: 1, "proof-gap": 1 }, + path: "matrix.json", + stages: ["first-run"], + surfaces: ["cli", "web-ui"], + }, + preflight: { adbDevices: null, memory: null }, + releaseLedger: null, + rootPath: ".artifacts/qa-e2e/suite/script/ux-matrix-evidence-dashboard/run-1", + scorecard: null, + }, + profile: null, + schemaVersion: 2, + }, + selectedEvidenceEntryId: "ux-matrix.web-ui.first-run", + }), + ); + + expect(html).toContain('data-evidence-entry-id="ux-matrix.web-ui.first-run"'); + expect(html).toContain("evidence-matrix-cell-proof-gap"); + expect(html).toContain("not executed in this run"); + expect(html).toContain("Coverage: cli-entrypoint"); + expect(html).toContain("Runner: cli-status"); + expect(html).toContain("Open media artifact"); + expect(html).toContain("Open video artifact"); + expect(html).not.toContain('src="/api/evidence/artifact?artifactPath=recording.gif"'); + expect(html).not.toContain("