From 61d4ff782eeaab0b451c96ebac84f5f900f28bb5 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 24 Jun 2026 17:32:08 -0700 Subject: [PATCH] docs: clarify maturity scorecard scoring (#96594) * docs: clarify maturity scorecard scoring * chore: split qa profile workflow change * docs: keep maturity coverage values stable * test: keep maturity renderer fixture in core boundary --- docs/maturity/scorecard.md | 35 ++--- docs/style.css | 2 +- scripts/qa/render-maturity-docs.ts | 20 +-- test/scripts/render-maturity-docs.test.ts | 167 +++++++++++++++++----- 4 files changed, 154 insertions(+), 70 deletions(-) diff --git a/docs/maturity/scorecard.md b/docs/maturity/scorecard.md index 654110b286e..677a996fdd3 100644 --- a/docs/maturity/scorecard.md +++ b/docs/maturity/scorecard.md @@ -19,42 +19,23 @@ Use this page to answer one question: which OpenClaw surfaces are credible choic ## At a glance
-
-
- 4% - Coverage -
-
-
- Experimental - QA profile evidence -
-
- 63% - Quality + 67% + Maturity score
-
+
Alpha - Reliability and operator confidence -
-
-
-
- 70% - Completeness -
-
-
- Beta - Expected workflow coverage + Quality + completeness + Coverage Experimental - 4% + Quality Alpha - 63% + Completeness Beta - 70%
-Coverage is deliberately evidence-led: an area does not become "ready" just because the implementation exists. +Coverage is deliberately evidence-led: an area does not become "ready" just because the implementation exists. It is not an input to the maturity score, but OpenClaw aims to keep end-to-end coverage above 90% for mature Stable-or-better features over time. ## Score bands diff --git a/docs/style.css b/docs/style.css index a042ffad2f9..a0500623807 100644 --- a/docs/style.css +++ b/docs/style.css @@ -269,7 +269,7 @@ html.dark .nav-tabs-underline { .maturity-summary-grid { display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); + grid-template-columns: repeat(auto-fit, minmax(min(220px, 100%), 1fr)); margin: 14px 0 20px; border-top: 1px solid color-mix(in oklab, rgb(var(--primary)) 18%, transparent); border-bottom: 1px solid color-mix(in oklab, rgb(var(--primary)) 18%, transparent); diff --git a/scripts/qa/render-maturity-docs.ts b/scripts/qa/render-maturity-docs.ts index c54d85b9b77..b4f7eca7a50 100644 --- a/scripts/qa/render-maturity-docs.ts +++ b/scripts/qa/render-maturity-docs.ts @@ -299,6 +299,7 @@ function scoreSummary( title: string, value: QaMaturityScoreObject | undefined, description: string, + details: readonly string[] = [], ): string[] { const score = scorePercent(value); const displayScore = score === undefined ? "-" : `${score}%`; @@ -313,6 +314,7 @@ function scoreSummary( '
', ` ${maturityLabelPill(value?.label ?? "Unscored")}`, ` ${markdownEscape(description)}`, + ...details.map((detail) => ` ${markdownEscape(detail)}`), "
", "", ]; @@ -964,6 +966,7 @@ function renderMaturityScorecard({ const surfaceAverage = coverage.rollups.surface_average; const qualityAverage = scores.rollups.surface_average.quality; const completenessAverage = scores.rollups.surface_average.completeness; + const maturityAverage = averageScores([qualityAverage, completenessAverage]); const lines = [ ...frontmatter( "Maturity scorecard", @@ -985,18 +988,17 @@ function renderMaturityScorecard({ "## At a glance", "", '
', - ...indentMarkdown(scoreSummary("Coverage", surfaceAverage, "QA profile evidence"), 2), ...indentMarkdown( - scoreSummary("Quality", qualityAverage, "Reliability and operator confidence"), - 2, - ), - ...indentMarkdown( - scoreSummary("Completeness", completenessAverage, "Expected workflow coverage"), + scoreSummary("Maturity score", maturityAverage, "Quality + completeness", [ + `Coverage ${scoreLabel(surfaceAverage)}`, + `Quality ${scoreLabel(qualityAverage)}`, + `Completeness ${scoreLabel(completenessAverage)}`, + ]), 2, ), "
", "", - 'Coverage is deliberately evidence-led: an area does not become "ready" just because the implementation exists.', + 'Coverage is deliberately evidence-led: an area does not become "ready" just because the implementation exists. It is not an input to the maturity score, but OpenClaw aims to keep end-to-end coverage above 90% for mature Stable-or-better features over time.', "", ...renderScoreBands(), ]; @@ -1212,9 +1214,7 @@ function main(): void { } const evidenceSummaries = readEvidenceSummaries(args.evidenceDir); - if (args.strictInputs) { - rejectBlockingEvidence(evidenceSummaries); - } + rejectBlockingEvidence(evidenceSummaries); const coverage = deriveCoverageScores(taxonomy, evidenceSummaries); const { scores, warnings: scoreWarnings } = readValidatedQaMaturityScoreSources({ coverageScores: coverage, diff --git a/test/scripts/render-maturity-docs.test.ts b/test/scripts/render-maturity-docs.test.ts index ac32f1547e6..2d949f0c218 100644 --- a/test/scripts/render-maturity-docs.test.ts +++ b/test/scripts/render-maturity-docs.test.ts @@ -3,11 +3,32 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { parse as parseYaml } from "yaml"; import { createTempDirTracker } from "../helpers/temp-dir.js"; const repoRoot = path.resolve(__dirname, "../.."); const tempDirs = createTempDirTracker(); +type TaxonomyFixture = { + surfaces?: TaxonomySurfaceFixture[]; +}; + +type TaxonomySurfaceFixture = { + id?: string; + status?: string; + categories?: TaxonomyCategoryFixture[]; +}; + +type TaxonomyCategoryFixture = { + id?: string; + name?: string; + features?: TaxonomyFeatureFixture[]; +}; + +type TaxonomyFeatureFixture = { + coverageIds?: string[]; +}; + afterEach(() => { tempDirs.cleanup(); }); @@ -26,7 +47,33 @@ function runCli(...args: string[]) { function writeQaEvidence(params: { dir: string; entries: Array<{ id: string; status: "pass" | "fail" | "blocked" | "skipped" }>; + scorecard?: unknown; }) { + const scorecard = params.scorecard ?? { + filters: { surface: null, category: null }, + run: { evidenceEntryCount: params.entries.length }, + categories: { + total: 0, + fulfilled: 0, + partial: 0, + missing: 0, + fulfillmentPercent: 0, + }, + features: { + total: 0, + fulfilled: 0, + partial: 0, + missing: 0, + fulfillmentPercent: 0, + }, + coverageIds: { + total: 0, + fulfilled: 0, + missing: 0, + fulfillmentPercent: 0, + }, + categoryReports: [], + }; fs.mkdirSync(params.dir, { recursive: true }); fs.writeFileSync( path.join(params.dir, "qa-evidence.json"), @@ -47,31 +94,7 @@ function writeQaEvidence(params: { coverage: [{ id: "tools.evidence", role: "primary" }], result: { status: entry.status }, })), - scorecard: { - filters: { surface: null, category: null }, - run: { evidenceEntryCount: params.entries.length }, - categories: { - total: 0, - fulfilled: 0, - partial: 0, - missing: 0, - fulfillmentPercent: 0, - }, - features: { - total: 0, - fulfilled: 0, - partial: 0, - missing: 0, - fulfillmentPercent: 0, - }, - coverageIds: { - total: 0, - fulfilled: 0, - missing: 0, - fulfillmentPercent: 0, - }, - categoryReports: [], - }, + scorecard, }, null, 2, @@ -80,6 +103,73 @@ function writeQaEvidence(params: { ); } +function allProfileScorecardFixture() { + const taxonomy = parseYaml( + fs.readFileSync(path.join(repoRoot, "taxonomy.yaml"), "utf8"), + ) as TaxonomyFixture; + const activeSurfaces = (taxonomy.surfaces ?? []).filter( + (surface) => surface.status !== "retired", + ); + const categoryReports = activeSurfaces.flatMap((surface) => + (surface.categories ?? []).map((category) => { + const coverageIds = [ + ...new Set((category.features ?? []).flatMap((feature) => feature.coverageIds ?? [])), + ].sort(); + return { + id: `${surface.id}.${category.id}`, + surfaceId: surface.id, + name: category.name, + status: "missing", + features: { + total: category.features.length, + fulfilled: 0, + partial: 0, + missing: category.features.length, + fulfillmentPercent: 0, + }, + coverageIds: { + total: coverageIds.length, + fulfilled: 0, + missing: coverageIds.length, + fulfillmentPercent: 0, + secondaryOnly: 0, + }, + missingCoverageIds: coverageIds, + }; + }), + ); + const featureCount = categoryReports.reduce((count, report) => count + report.features.total, 0); + const coverageIdCount = categoryReports.reduce( + (count, report) => count + report.coverageIds.total, + 0, + ); + return { + filters: { surface: null, category: null }, + run: { evidenceEntryCount: 1 }, + categories: { + total: categoryReports.length, + fulfilled: 0, + partial: 0, + missing: categoryReports.length, + fulfillmentPercent: 0, + }, + features: { + total: featureCount, + fulfilled: 0, + partial: 0, + missing: featureCount, + fulfillmentPercent: 0, + }, + coverageIds: { + total: coverageIdCount, + fulfilled: 0, + missing: coverageIdCount, + fulfillmentPercent: 0, + }, + categoryReports, + }; +} + describe("maturity docs renderer CLI", () => { it("checks maturity inputs without requiring QA evidence artifacts", () => { const result = runCli("--check"); @@ -113,13 +203,7 @@ describe("maturity docs renderer CLI", () => { ], }); - const result = runCli( - "--output-dir", - outputDir, - "--evidence-dir", - evidenceDir, - "--strict-inputs", - ); + const result = runCli("--output-dir", outputDir, "--evidence-dir", evidenceDir); expect(result.status).toBe(1); expect(result.stdout).toBe(""); @@ -147,4 +231,23 @@ describe("maturity docs renderer CLI", () => { expect(scorecard).not.toContain("0 failed"); expect(scorecard).not.toContain("0 blocked"); }); + + it("renders the maturity score from quality and completeness without coverage", () => { + const outputDir = tempDirs.make("openclaw-maturity-docs-output-"); + const evidenceDir = tempDirs.make("openclaw-maturity-docs-evidence-"); + writeQaEvidence({ + dir: evidenceDir, + entries: [{ id: "passing-scenario", status: "pass" }], + scorecard: allProfileScorecardFixture(), + }); + + const result = runCli("--output-dir", outputDir, "--evidence-dir", evidenceDir); + + expect(result.status).toBe(0); + const scorecard = fs.readFileSync(path.join(outputDir, "maturity", "scorecard.md"), "utf8"); + expect(scorecard).toContain("Maturity score"); + expect(scorecard).toContain('67%'); + expect(scorecard).toContain("Coverage Experimental - 0%"); + expect(scorecard).toContain("end-to-end coverage above 90%"); + }); });