mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
docs: clarify maturity scorecard scoring (#96594)
* docs: clarify maturity scorecard scoring * chore: split qa profile workflow change * docs: keep maturity coverage values stable * test: keep maturity renderer fixture in core boundary
This commit is contained in:
Dallin Romney
GitHub
parent
3ab7a72764
commit
61d4ff782e
4 changed files with 154 additions and 70 deletions
|
|
@ -19,42 +19,23 @@ Use this page to answer one question: which OpenClaw surfaces are credible choic
|
|||
## At a glance
|
||||
|
||||
<div className="maturity-summary-grid">
|
||||
<div className="maturity-summary-item maturity-score-experimental">
|
||||
<div className="maturity-summary-heading">
|
||||
<span className="maturity-summary-value">4%</span>
|
||||
<span>Coverage</span>
|
||||
</div>
|
||||
<div className="maturity-summary-bar" style={{ "--score": "4" }}><span /></div>
|
||||
<div className="maturity-summary-meta">
|
||||
<span className="maturity-level-pill maturity-level-experimental">Experimental</span>
|
||||
<span>QA profile evidence</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="maturity-summary-item maturity-score-alpha">
|
||||
<div className="maturity-summary-heading">
|
||||
<span className="maturity-summary-value">63%</span>
|
||||
<span>Quality</span>
|
||||
<span className="maturity-summary-value">67%</span>
|
||||
<span>Maturity score</span>
|
||||
</div>
|
||||
<div className="maturity-summary-bar" style={{ "--score": "63" }}><span /></div>
|
||||
<div className="maturity-summary-bar" style={{ "--score": "67" }}><span /></div>
|
||||
<div className="maturity-summary-meta">
|
||||
<span className="maturity-level-pill maturity-level-alpha">Alpha</span>
|
||||
<span>Reliability and operator confidence</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="maturity-summary-item maturity-score-beta">
|
||||
<div className="maturity-summary-heading">
|
||||
<span className="maturity-summary-value">70%</span>
|
||||
<span>Completeness</span>
|
||||
</div>
|
||||
<div className="maturity-summary-bar" style={{ "--score": "70" }}><span /></div>
|
||||
<div className="maturity-summary-meta">
|
||||
<span className="maturity-level-pill maturity-level-beta">Beta</span>
|
||||
<span>Expected workflow coverage</span>
|
||||
<span>Quality + completeness</span>
|
||||
<span>Coverage Experimental - 4%</span>
|
||||
<span>Quality Alpha - 63%</span>
|
||||
<span>Completeness Beta - 70%</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Coverage is deliberately evidence-led: an area does not become "ready" just because the implementation exists.
|
||||
Coverage is deliberately evidence-led: an area does not become "ready" just because the implementation exists. It is not an input to the maturity score, but OpenClaw aims to keep end-to-end coverage above 90% for mature Stable-or-better features over time.
|
||||
|
||||
## Score bands
|
||||
|
||||
|
|
|
|||
|
|
@ -269,7 +269,7 @@ html.dark .nav-tabs-underline {
|
|||
|
||||
.maturity-summary-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
grid-template-columns: repeat(auto-fit, minmax(min(220px, 100%), 1fr));
|
||||
margin: 14px 0 20px;
|
||||
border-top: 1px solid color-mix(in oklab, rgb(var(--primary)) 18%, transparent);
|
||||
border-bottom: 1px solid color-mix(in oklab, rgb(var(--primary)) 18%, transparent);
|
||||
|
|
|
|||
|
|
@ -299,6 +299,7 @@ function scoreSummary(
|
|||
title: string,
|
||||
value: QaMaturityScoreObject | undefined,
|
||||
description: string,
|
||||
details: readonly string[] = [],
|
||||
): string[] {
|
||||
const score = scorePercent(value);
|
||||
const displayScore = score === undefined ? "-" : `${score}%`;
|
||||
|
|
@ -313,6 +314,7 @@ function scoreSummary(
|
|||
' <div className="maturity-summary-meta">',
|
||||
` ${maturityLabelPill(value?.label ?? "Unscored")}`,
|
||||
` <span>${markdownEscape(description)}</span>`,
|
||||
...details.map((detail) => ` <span>${markdownEscape(detail)}</span>`),
|
||||
" </div>",
|
||||
"</div>",
|
||||
];
|
||||
|
|
@ -964,6 +966,7 @@ function renderMaturityScorecard({
|
|||
const surfaceAverage = coverage.rollups.surface_average;
|
||||
const qualityAverage = scores.rollups.surface_average.quality;
|
||||
const completenessAverage = scores.rollups.surface_average.completeness;
|
||||
const maturityAverage = averageScores([qualityAverage, completenessAverage]);
|
||||
const lines = [
|
||||
...frontmatter(
|
||||
"Maturity scorecard",
|
||||
|
|
@ -985,18 +988,17 @@ function renderMaturityScorecard({
|
|||
"## At a glance",
|
||||
"",
|
||||
'<div className="maturity-summary-grid">',
|
||||
...indentMarkdown(scoreSummary("Coverage", surfaceAverage, "QA profile evidence"), 2),
|
||||
...indentMarkdown(
|
||||
scoreSummary("Quality", qualityAverage, "Reliability and operator confidence"),
|
||||
2,
|
||||
),
|
||||
...indentMarkdown(
|
||||
scoreSummary("Completeness", completenessAverage, "Expected workflow coverage"),
|
||||
scoreSummary("Maturity score", maturityAverage, "Quality + completeness", [
|
||||
`Coverage ${scoreLabel(surfaceAverage)}`,
|
||||
`Quality ${scoreLabel(qualityAverage)}`,
|
||||
`Completeness ${scoreLabel(completenessAverage)}`,
|
||||
]),
|
||||
2,
|
||||
),
|
||||
"</div>",
|
||||
"",
|
||||
'Coverage is deliberately evidence-led: an area does not become "ready" just because the implementation exists.',
|
||||
'Coverage is deliberately evidence-led: an area does not become "ready" just because the implementation exists. It is not an input to the maturity score, but OpenClaw aims to keep end-to-end coverage above 90% for mature Stable-or-better features over time.',
|
||||
"",
|
||||
...renderScoreBands(),
|
||||
];
|
||||
|
|
@ -1212,9 +1214,7 @@ function main(): void {
|
|||
}
|
||||
|
||||
const evidenceSummaries = readEvidenceSummaries(args.evidenceDir);
|
||||
if (args.strictInputs) {
|
||||
rejectBlockingEvidence(evidenceSummaries);
|
||||
}
|
||||
rejectBlockingEvidence(evidenceSummaries);
|
||||
const coverage = deriveCoverageScores(taxonomy, evidenceSummaries);
|
||||
const { scores, warnings: scoreWarnings } = readValidatedQaMaturityScoreSources({
|
||||
coverageScores: coverage,
|
||||
|
|
|
|||
|
|
@ -3,11 +3,32 @@ import { spawnSync } from "node:child_process";
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { parse as parseYaml } from "yaml";
|
||||
import { createTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const repoRoot = path.resolve(__dirname, "../..");
|
||||
const tempDirs = createTempDirTracker();
|
||||
|
||||
type TaxonomyFixture = {
|
||||
surfaces?: TaxonomySurfaceFixture[];
|
||||
};
|
||||
|
||||
type TaxonomySurfaceFixture = {
|
||||
id?: string;
|
||||
status?: string;
|
||||
categories?: TaxonomyCategoryFixture[];
|
||||
};
|
||||
|
||||
type TaxonomyCategoryFixture = {
|
||||
id?: string;
|
||||
name?: string;
|
||||
features?: TaxonomyFeatureFixture[];
|
||||
};
|
||||
|
||||
type TaxonomyFeatureFixture = {
|
||||
coverageIds?: string[];
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
tempDirs.cleanup();
|
||||
});
|
||||
|
|
@ -26,7 +47,33 @@ function runCli(...args: string[]) {
|
|||
function writeQaEvidence(params: {
|
||||
dir: string;
|
||||
entries: Array<{ id: string; status: "pass" | "fail" | "blocked" | "skipped" }>;
|
||||
scorecard?: unknown;
|
||||
}) {
|
||||
const scorecard = params.scorecard ?? {
|
||||
filters: { surface: null, category: null },
|
||||
run: { evidenceEntryCount: params.entries.length },
|
||||
categories: {
|
||||
total: 0,
|
||||
fulfilled: 0,
|
||||
partial: 0,
|
||||
missing: 0,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
features: {
|
||||
total: 0,
|
||||
fulfilled: 0,
|
||||
partial: 0,
|
||||
missing: 0,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
coverageIds: {
|
||||
total: 0,
|
||||
fulfilled: 0,
|
||||
missing: 0,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
categoryReports: [],
|
||||
};
|
||||
fs.mkdirSync(params.dir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(params.dir, "qa-evidence.json"),
|
||||
|
|
@ -47,31 +94,7 @@ function writeQaEvidence(params: {
|
|||
coverage: [{ id: "tools.evidence", role: "primary" }],
|
||||
result: { status: entry.status },
|
||||
})),
|
||||
scorecard: {
|
||||
filters: { surface: null, category: null },
|
||||
run: { evidenceEntryCount: params.entries.length },
|
||||
categories: {
|
||||
total: 0,
|
||||
fulfilled: 0,
|
||||
partial: 0,
|
||||
missing: 0,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
features: {
|
||||
total: 0,
|
||||
fulfilled: 0,
|
||||
partial: 0,
|
||||
missing: 0,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
coverageIds: {
|
||||
total: 0,
|
||||
fulfilled: 0,
|
||||
missing: 0,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
categoryReports: [],
|
||||
},
|
||||
scorecard,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
|
|
@ -80,6 +103,73 @@ function writeQaEvidence(params: {
|
|||
);
|
||||
}
|
||||
|
||||
function allProfileScorecardFixture() {
|
||||
const taxonomy = parseYaml(
|
||||
fs.readFileSync(path.join(repoRoot, "taxonomy.yaml"), "utf8"),
|
||||
) as TaxonomyFixture;
|
||||
const activeSurfaces = (taxonomy.surfaces ?? []).filter(
|
||||
(surface) => surface.status !== "retired",
|
||||
);
|
||||
const categoryReports = activeSurfaces.flatMap((surface) =>
|
||||
(surface.categories ?? []).map((category) => {
|
||||
const coverageIds = [
|
||||
...new Set((category.features ?? []).flatMap((feature) => feature.coverageIds ?? [])),
|
||||
].sort();
|
||||
return {
|
||||
id: `${surface.id}.${category.id}`,
|
||||
surfaceId: surface.id,
|
||||
name: category.name,
|
||||
status: "missing",
|
||||
features: {
|
||||
total: category.features.length,
|
||||
fulfilled: 0,
|
||||
partial: 0,
|
||||
missing: category.features.length,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
coverageIds: {
|
||||
total: coverageIds.length,
|
||||
fulfilled: 0,
|
||||
missing: coverageIds.length,
|
||||
fulfillmentPercent: 0,
|
||||
secondaryOnly: 0,
|
||||
},
|
||||
missingCoverageIds: coverageIds,
|
||||
};
|
||||
}),
|
||||
);
|
||||
const featureCount = categoryReports.reduce((count, report) => count + report.features.total, 0);
|
||||
const coverageIdCount = categoryReports.reduce(
|
||||
(count, report) => count + report.coverageIds.total,
|
||||
0,
|
||||
);
|
||||
return {
|
||||
filters: { surface: null, category: null },
|
||||
run: { evidenceEntryCount: 1 },
|
||||
categories: {
|
||||
total: categoryReports.length,
|
||||
fulfilled: 0,
|
||||
partial: 0,
|
||||
missing: categoryReports.length,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
features: {
|
||||
total: featureCount,
|
||||
fulfilled: 0,
|
||||
partial: 0,
|
||||
missing: featureCount,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
coverageIds: {
|
||||
total: coverageIdCount,
|
||||
fulfilled: 0,
|
||||
missing: coverageIdCount,
|
||||
fulfillmentPercent: 0,
|
||||
},
|
||||
categoryReports,
|
||||
};
|
||||
}
|
||||
|
||||
describe("maturity docs renderer CLI", () => {
|
||||
it("checks maturity inputs without requiring QA evidence artifacts", () => {
|
||||
const result = runCli("--check");
|
||||
|
|
@ -113,13 +203,7 @@ describe("maturity docs renderer CLI", () => {
|
|||
],
|
||||
});
|
||||
|
||||
const result = runCli(
|
||||
"--output-dir",
|
||||
outputDir,
|
||||
"--evidence-dir",
|
||||
evidenceDir,
|
||||
"--strict-inputs",
|
||||
);
|
||||
const result = runCli("--output-dir", outputDir, "--evidence-dir", evidenceDir);
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
|
|
@ -147,4 +231,23 @@ describe("maturity docs renderer CLI", () => {
|
|||
expect(scorecard).not.toContain("0 failed");
|
||||
expect(scorecard).not.toContain("0 blocked");
|
||||
});
|
||||
|
||||
it("renders the maturity score from quality and completeness without coverage", () => {
|
||||
const outputDir = tempDirs.make("openclaw-maturity-docs-output-");
|
||||
const evidenceDir = tempDirs.make("openclaw-maturity-docs-evidence-");
|
||||
writeQaEvidence({
|
||||
dir: evidenceDir,
|
||||
entries: [{ id: "passing-scenario", status: "pass" }],
|
||||
scorecard: allProfileScorecardFixture(),
|
||||
});
|
||||
|
||||
const result = runCli("--output-dir", outputDir, "--evidence-dir", evidenceDir);
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
const scorecard = fs.readFileSync(path.join(outputDir, "maturity", "scorecard.md"), "utf8");
|
||||
expect(scorecard).toContain("<span>Maturity score</span>");
|
||||
expect(scorecard).toContain('<span className="maturity-summary-value">67%</span>');
|
||||
expect(scorecard).toContain("Coverage Experimental - 0%");
|
||||
expect(scorecard).toContain("end-to-end coverage above 90%");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue