openclaw/scripts/full-release-validation-state.mjs
Peter Steinberger f34c2a21db
feat(release): let a release lead record an exact-job flake instead of blocking publication (#161515)
* feat(release): accept exact-job recorded flakes in release validation

A release lead can classify one failed Normal CI job of a Full Release
Validation run as a flake through the trusted classification workflow. The
receipt binds the exact job id and attempt, CI child run, FRV parent run and
attempt, and Release SHA, and carries a tracking issue or PR plus a reason.
Release Decision, the manifest, the publisher's live re-derivation, the step
summary, and the GitHub release notes tail treat it as a visible advisory.
Required classes and every other child stay blocking.

* docs(release): fold recorded flakes into the shared release boundaries

* fix(release): scope flake receipt discovery to the CI child run

* fix(release): require main lineage for flake receipt producers

* test(release): copy the flake classification module into tooling fixtures

* fix(release): keep advisory-only release notes verifiable
2026-09-30 04:31:07 +00:00

1647 lines
59 KiB
JavaScript

#!/usr/bin/env node
import { execFile } from "node:child_process";
import {
appendFileSync,
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { dirname, join } from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import {
validateFullReleaseCandidateRequest,
validateRecordedFullReleaseCandidateRequest,
} from "./full-release-candidate-contract.mjs";
import { loadFlakeClassifications } from "./full-release-flake-classification.mjs";
import {
createPublicationAdmission,
publicationObservationJson,
publicationSourceReuseIdentity,
validatePublicationAdmissionBinding,
validatePublicationSourceBinding,
} from "./full-release-publication-contract.mjs";
import {
assertReleasePublicationKnownBudget,
affectedActiveRunIds,
buildReleaseExecutionPlan,
buildReleaseExecutionPlanArtifact,
buildReleaseStateArtifact,
buildReleaseValidationManifest,
classifyReleaseGhTransportError,
classifyReleaseSnapshot,
composeReleaseChildAttemptEvidence,
formatReleaseStateOutcome,
releasePlanGateFailures,
releaseChildClassificationEvidence,
MAX_RELEASE_ARTIFACT_BYTES,
serializeReleaseArtifact,
selectReleaseStateArtifacts,
validateReleaseChildRunProvenance,
validateReleaseCoveragePolicyBinding,
validateReleaseExecutionPlanArtifact,
validateReleaseTelegramWaiverBinding,
verifyReleaseStateArtifacts,
} from "./full-release-validation-policy.mjs";
import { sortJsonValueKeys } from "./lib/canonical-json.mjs";
import { validateReusableReleaseChild } from "./lib/full-release-child-reuse.mjs";
import { createReleasePublishInputs } from "./lib/release-publish-inputs.mjs";
import { downloadFullReleaseNpmPreflight } from "./npm-preflight-tooling-identity.mjs";
export * from "./full-release-validation-policy.mjs";
const execFileAsync = promisify(execFile);
const RELEASE_SUMMARY_PATH =
process.env.OPENCLAW_RELEASE_CI_SUMMARY_VALIDATOR ??
fileURLToPath(new URL("./release-ci-summary.mjs", import.meta.url));
const API_ERROR_PATTERN =
/HTTP [45][0-9][0-9]|API|Bad credentials|rate limit|network|connection|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN/u;
const DEFAULT_POLL_INTERVAL_MS = 60_000;
const DEFAULT_HEARTBEAT_INTERVAL_MS = 5 * 60_000;
const GH_TIMEOUT_MS = 60_000;
const TRANSPORT_UNCERTAINTY_MS = 15 * 60_000;
let ghRetryDeadline;
function stringValue(value, fallback = "") {
return typeof value === "string" ? value : fallback;
}
function requiredString(value, label) {
const normalized = stringValue(value).trim();
if (!normalized) {
throw new Error(`${label} is required`);
}
return normalized;
}
function positiveInteger(value, label) {
const normalized = Number(value);
if (!Number.isSafeInteger(normalized) || normalized < 1) {
throw new Error(`${label} must be a positive integer`);
}
return normalized;
}
async function abortableSleep(milliseconds, signal) {
let abortError;
await new Promise((resolve) => {
const timer = setTimeout(() => {
signal?.removeEventListener("abort", abort);
resolve();
}, milliseconds);
const abort = () => {
clearTimeout(timer);
abortError = signal?.reason instanceof Error ? signal.reason : new Error("operation aborted");
resolve();
};
signal?.addEventListener("abort", abort, { once: true });
});
if (abortError instanceof Error) {
throw new Error(abortError.message, { cause: abortError });
}
}
const deadlineDelayMs = (delay, deadline, now) =>
Number.isFinite(deadline) ? Math.max(0, Math.min(delay, deadline - now)) : delay;
export function releaseGhRetryDelayMs(attempt, deadlineMonotonicMs, nowMonotonicMs) {
return deadlineDelayMs(Math.min(attempt * 10_000, 60_000), deadlineMonotonicMs, nowMonotonicMs);
}
async function runGh(args, options = {}) {
const attempts = options.attempts ?? 6;
let lastError;
for (let attempt = 1; attempt <= attempts; attempt += 1) {
try {
const result = await execFileAsync("gh", args, {
encoding: "utf8",
env: process.env,
killSignal: "SIGKILL",
maxBuffer: 64 * 1024 * 1024,
signal: options.signal,
timeout: GH_TIMEOUT_MS,
});
return result.stdout;
} catch (error) {
lastError = error;
if (options.signal?.aborted) {
throw options.signal.reason ?? error;
}
if (attempt === attempts || classifyReleaseGhTransportError(error) !== "transient") {
throw error;
}
const delay = releaseGhRetryDelayMs(attempt, ghRetryDeadline, performance.now());
if (delay === 0) {
throw error;
}
await abortableSleep(delay, options.signal);
}
}
throw lastError;
}
async function githubJson(path, signal) {
return JSON.parse(
await runGh(["api", `repos/${process.env.GITHUB_REPOSITORY}/${path}`], { signal }),
);
}
async function githubAttemptJobs(runId, runAttempt, signal) {
return (
await runGh(
[
"api",
"--paginate",
`repos/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId}/attempts/${runAttempt}/jobs?per_page=100`,
"--jq",
".jobs[] | @json",
],
{ signal },
)
)
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line));
}
function issue(kind, child, message, extra = {}) {
return {
child: child.key,
kind,
message,
runId: stringValue(child.runId),
url: stringValue(child.url),
...extra,
};
}
export function validateChildBinding(child, run, composite) {
const errors = [];
let provenance = {};
try {
provenance = validateReleaseChildRunProvenance(run, {
...child,
plannedRunAttempt: child.runAttempt,
repository: process.env.GITHUB_REPOSITORY,
});
} catch (error) {
errors.push(
issue("provenance_mismatch", child, error instanceof Error ? error.message : String(error)),
);
}
return {
...child,
...provenance,
compositeJobsSha256: stringValue(composite.sha256),
conclusion: stringValue(run.conclusion),
createdAt: stringValue(run.created_at),
errors,
jobs: composite.jobs,
observedRunAttempts: composite.observedRunAttempts,
plannedRunAttempt: Number(child.runAttempt),
runAttempt: Number(run.run_attempt),
runId: String(run.id),
status: stringValue(run.status),
updatedAt: stringValue(run.updated_at),
url: stringValue(run.html_url, child.url),
workflowRef: stringValue(run.head_branch),
workflowSha: stringValue(run.head_sha),
};
}
export async function readChild(child, previous, signal, options = {}) {
if (!child.selected) {
return { ...child, errors: [], jobs: [], status: "skipped" };
}
if (!child.runId || !child.runAttempt) {
const error = issue("dispatch_missing", child, `${child.key} omitted its exact run identity`);
return { ...child, errors: [error], jobs: [], status: "missing" };
}
try {
const run = options.readRun
? await options.readRun(child.runId, signal)
: await githubJson(`actions/runs/${child.runId}`, signal);
const currentAttempt = positiveInteger(run.run_attempt, `${child.key} run attempt`);
if (options.reuseSelection && currentAttempt !== options.reuseSelection.runAttempt) {
throw new Error(`release child provenance changed: ${child.key} reused attempt is stale`);
}
const plannedAttempt = positiveInteger(child.runAttempt, `${child.key} planned run attempt`);
if (currentAttempt < plannedAttempt) {
return validateChildBinding(child, run, {
jobs: [],
observedRunAttempts: [],
sha256: "",
});
}
const attempts = await Promise.all(
Array.from({ length: currentAttempt - plannedAttempt + 1 }, async (_, index) => {
const runAttempt = plannedAttempt + index;
return {
jobs: options.readAttemptJobs
? await options.readAttemptJobs(child.runId, runAttempt, signal)
: await githubAttemptJobs(child.runId, runAttempt, signal),
runAttempt,
};
}),
);
if (run.status !== "completed" && attempts.at(-1)?.jobs.length === 0) {
if (attempts.slice(0, -1).some((attempt) => attempt.jobs.length === 0)) {
throw new Error(`${child.key} child attempt evidence is gapped`);
}
const partial = validateChildBinding(child, run, {
jobs: [],
observedRunAttempts: [],
sha256: "",
});
return (previous?.compositeJobsSha256 || previous?.transportFailure) &&
partial.errors.length === 0
? {
...previous,
conclusion: stringValue(run.conclusion),
status: stringValue(run.status),
}
: partial;
}
const evidence = composeReleaseChildAttemptEvidence({
attempts,
expected: {
...child,
plannedRunAttempt: plannedAttempt,
repository: process.env.GITHUB_REPOSITORY,
},
run,
});
const snapshot = validateChildBinding(child, run, {
jobs: evidence.jobs,
observedRunAttempts: evidence.observedRunAttempts,
sha256: evidence.compositeJobsSha256,
});
if (snapshot.status === "completed" && snapshot.errors.length === 0) {
Object.assign(
snapshot,
await (options.loadFlakeClassifications ?? loadFlakeClassifications)({
repo: process.env.GITHUB_REPOSITORY,
child: snapshot,
parentRunId: options.parentRunId,
parentRunAttempt: options.parentRunAttempt,
targetSha: options.targetSha,
signal,
}),
);
}
return snapshot;
} catch (error) {
const degraded = classifyReleaseGhTransportError(error) === "transient";
const provenanceMismatch =
error instanceof Error && error.message.startsWith("release child provenance changed:");
const readError = issue(
provenanceMismatch ? "provenance_mismatch" : "api_error",
child,
`${child.key} GitHub read failed: ${error instanceof Error ? error.message : String(error)}`,
);
ghRetryDeadline ??= degraded ? performance.now() + TRANSPORT_UNCERTAINTY_MS : undefined;
return {
...child,
...previous,
errors: degraded
? (previous?.errors ?? []).filter((entry) => entry.kind === "provenance_mismatch")
: [
...(previous?.errors ?? []).filter((entry) => entry.kind === "provenance_mismatch"),
readError,
],
status: degraded ? "transport_uncertain" : stringValue(previous?.status, "unknown"),
transportFailure: degraded ? { errorClass: "transient" } : undefined,
};
}
}
export function updateReleaseTransportEpisode(previous, children, options = {}) {
const monotonicNow = options.monotonicNow ?? performance.now();
const wallNow = options.wallNow ?? Date.now();
const uncertain = children.filter((child) => child.transportFailure?.errorClass === "transient");
const affected = uncertain
.map((child) => ({
child: child.key,
compositeJobsSha256: stringValue(child.compositeJobsSha256),
errorClass: "transient",
lastValidAt: stringValue(child.updatedAt),
runAttempt: child.runAttempt,
runId: String(child.runId),
}))
.toSorted((left, right) => left.child.localeCompare(right.child, "en"));
if (affected.length === 0) {
return { status: "certain" };
}
const deadline = options.deadline ?? ghRetryDeadline ?? monotonicNow + TRANSPORT_UNCERTAINTY_MS;
const wallStart = wallNow + deadline - monotonicNow - TRANSPORT_UNCERTAINTY_MS;
const episode = previous?.deadlineMonotonicMs
? previous
: {
deadlineAt: new Date(wallStart + TRANSPORT_UNCERTAINTY_MS).toISOString(),
deadlineMonotonicMs: deadline,
startedAt: new Date(wallStart).toISOString(),
};
return {
...episode,
affected,
error:
monotonicNow >= episode.deadlineMonotonicMs
? issue(
"transport_deadline_exceeded",
{ key: "<collector>" },
`GitHub transport remained uncertain; affected ${affected.map((child) => `${child.child}:${child.runId}:${child.runAttempt}`).join(",")}`,
)
: undefined,
status: monotonicNow >= episode.deadlineMonotonicMs ? "expired" : "uncertain",
};
}
export function parsePlanInputs(value) {
const parsed = JSON.parse(requiredString(value, "plan inputs JSON"));
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new Error("plan inputs JSON must be an object");
}
return parsed;
}
export function hydrateReusedPlan(plan, evidence) {
if (evidence.childReuse) {
return plan.map((child) => {
const selection = evidence.childReuse[child.key];
return child.selected && selection
? {
...child,
displayTitle: selection.displayTitle,
result: "success",
runAttempt: 1,
runId: selection.runId,
source: "reused",
sourceParentAttempt: selection.sourceParentAttempt,
url: selection.url,
workflowRef: selection.workflowRef,
workflowSha: selection.workflowSha,
}
: child;
});
}
const byRole = new Map((evidence.children ?? []).map((child) => [child.role, child]));
return plan.map((child) => {
if (!child.selected) {
return child;
}
const reused = byRole.get(child.key);
if (!reused) {
return child;
}
return {
...child,
displayTitle: reused.displayTitle,
result: "success",
// Reuse keeps the dispatch origin, so a human rerun still composes earlier jobs.
// Verified manifests predating childEvidence only carry the effective attempt.
runAttempt:
evidence.manifest.childEvidence === undefined
? reused.runAttempt
: evidence.manifest.childEvidence[child.key].plannedRunAttempt,
runId: reused.runId,
url: reused.url,
workflowRef: reused.headBranch,
workflowSha: reused.workflowSha,
};
});
}
function changedPathsValue(value) {
if (Array.isArray(value)) {
return value;
}
try {
const parsed = JSON.parse(stringValue(value, "[]"));
return Array.isArray(parsed) ? parsed : [];
} catch {
return [];
}
}
async function validateReuse(executionPlan, signal) {
const { children: plan, evidenceReuse, trustedWorkflow } = executionPlan;
if (executionPlan.childReuse) {
const results = await Promise.allSettled(
Object.entries(executionPlan.childReuse).map(([role, selection]) =>
validateReusableReleaseChild(selection, {
repository: executionPlan.repository,
targetSha: executionPlan.targetSha,
workflowSha: executionPlan.workflowSha,
role,
inputs: selection.inputs,
}),
),
);
const issues = results.flatMap((result) => {
if (result.status === "fulfilled") {
return [];
}
const message =
result.reason instanceof Error ? result.reason.message : String(result.reason);
return [
{
child: "<evidence>",
kind: API_ERROR_PATTERN.test(message) ? "api_error" : "reused_evidence_invalid",
message,
},
];
});
return {
blockers: issues.filter((entry) => entry.kind !== "api_error"),
children: plan,
errors: issues.filter((entry) => entry.kind === "api_error"),
};
}
if (!evidenceReuse.requested) {
return { blockers: [], children: plan, errors: [] };
}
try {
const args = [
RELEASE_SUMMARY_PATH,
"--validate-run",
requiredString(evidenceReuse.selectedRunId, "evidence selected run ID"),
"--repo",
requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"),
"--trusted-workflow-ref",
requiredString(trustedWorkflow?.ref, "trusted workflow ref"),
"--trusted-workflow-full-ref",
requiredString(trustedWorkflow?.fullRef, "trusted workflow full ref"),
"--trusted-workflow-sha",
requiredString(trustedWorkflow?.sha, "trusted workflow SHA"),
"--verifier-source-sha",
requiredString(executionPlan.workflowSha, "workflow SHA"),
"--verifier-source-file",
RELEASE_SUMMARY_PATH,
"--expected-target-sha",
requiredString(process.env.TARGET_SHA, "target SHA"),
"--expected-evidence-policy",
requiredString(evidenceReuse.policy, "evidence policy"),
"--expected-evidence-sha",
requiredString(evidenceReuse.evidenceSha, "evidence SHA"),
"--expected-root-run-id",
requiredString(evidenceReuse.rootRunId, "evidence root run ID"),
"--expected-selected-run-id",
requiredString(evidenceReuse.selectedRunId, "evidence selected run ID"),
"--expected-changed-paths-json",
JSON.stringify(changedPathsValue(evidenceReuse.changedPaths)),
"--json",
];
const result = await execFileAsync(process.execPath, args, {
encoding: "utf8",
env: process.env,
killSignal: "SIGKILL",
maxBuffer: 64 * 1024 * 1024,
signal,
timeout: GH_TIMEOUT_MS * 6,
});
const evidence = JSON.parse(result.stdout);
if (
evidence.releaseProfile !== process.env.RELEASE_PROFILE ||
evidence.rerunGroup !== process.env.RERUN_GROUP ||
!evidence.manifest ||
typeof evidence.manifest !== "object" ||
Array.isArray(evidence.manifest)
) {
throw new Error("reused release evidence no longer matches the requested validation");
}
validateReleaseTelegramWaiverBinding(executionPlan, evidence.manifest.validationInputs);
validateReleaseCoveragePolicyBinding(executionPlan, evidence.manifest.validationInputs);
const source = validatePublicationSourceBinding(evidence.manifest, {
sourceAdmissionContract: executionPlan.sourceAdmissionContract,
});
if (
JSON.stringify(publicationSourceReuseIdentity(source)) !==
JSON.stringify(publicationSourceReuseIdentity(executionPlan.sourceAdmission))
) {
throw new Error("reused source admission differs from the requested publication source");
}
return {
blockers: [],
children: hydrateReusedPlan(plan, evidence),
errors: [],
sourceManifest: sortJsonValueKeys(evidence.manifest),
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const entry = {
child: "<evidence>",
kind: API_ERROR_PATTERN.test(message) ? "api_error" : "reused_evidence_invalid",
message,
runId: stringValue(evidenceReuse.selectedRunId),
url: stringValue(evidenceReuse.runUrl),
};
return API_ERROR_PATTERN.test(message)
? { blockers: [], children: plan, errors: [entry] }
: { blockers: [entry], children: plan, errors: [] };
}
}
function writeArtifact(path, payload) {
const json = serializeReleaseArtifact(payload);
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, json);
}
function writeResult(path, payload) {
writeArtifact(path, payload);
if (process.env.GITHUB_OUTPUT) {
appendFileSync(process.env.GITHUB_OUTPUT, `state=${payload.state}\n`);
for (const [key, child] of Object.entries(payload.children ?? {})) {
appendFileSync(process.env.GITHUB_OUTPUT, `${key}_conclusion=${child.conclusion ?? ""}\n`);
}
}
}
function writeExecutionPlan(path, payload) {
writeArtifact(path, payload);
if (process.env.GITHUB_OUTPUT) {
appendFileSync(process.env.GITHUB_OUTPUT, `sha256=${payload.sha256}\n`);
appendFileSync(
process.env.GITHUB_OUTPUT,
`source_parent_attempt=${payload.parentRunAttempt}\n`,
);
}
}
function appendSummary(mode, payload) {
if (!process.env.GITHUB_STEP_SUMMARY) {
return;
}
appendFileSync(
process.env.GITHUB_STEP_SUMMARY,
`## ${mode === "decision" ? "Release Decision" : "Diagnostic Drain"}\n\n${formatReleaseStateOutcome(payload)}\n`,
);
}
export function formatReleaseStateHeartbeat(mode, decision) {
return `${mode} heartbeat: state=${decision.state} active=${decision.activeRunIds.length} blockers=${decision.blockers.length} errors=${decision.errors.length}`;
}
async function cancelAffectedChildren(children, blockers, cancelledRunIds, signal) {
const errors = [];
for (const runId of affectedActiveRunIds(children, blockers, cancelledRunIds)) {
const child = children.find((entry) => String(entry.runId) === runId);
try {
await runGh(["run", "cancel", runId], { attempts: 1, signal });
cancelledRunIds.add(runId);
} catch (error) {
errors.push(
issue(
"api_error",
child ?? { key: "<child>", runId },
`exact child cancellation failed: ${
error instanceof Error ? error.message : String(error)
}`,
),
);
}
}
return errors;
}
function readArtifact(path, label) {
try {
if (statSync(path).size > MAX_RELEASE_ARTIFACT_BYTES) {
throw new Error("release artifact exceeds the size limit");
}
return JSON.parse(readFileSync(path, "utf8"));
} catch (error) {
throw new Error(
`${label} artifact is unreadable: ${error instanceof Error ? error.message : String(error)}`,
{ cause: error },
);
}
}
function stateArtifactExpected(attemptLabel = "parent run attempt") {
return {
publicationAdmissionContract:
process.env.FULL_RELEASE_PUBLICATION_ADMISSION_CONTRACT || undefined,
sourceAdmissionContract: process.env.FULL_RELEASE_SOURCE_ADMISSION_CONTRACT || undefined,
maxParentRunAttempt: positiveInteger(process.env.GITHUB_RUN_ATTEMPT, attemptLabel),
parentRunId: requiredString(process.env.GITHUB_RUN_ID, "parent run ID"),
repository: requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"),
releaseProfile: requiredString(process.env.RELEASE_PROFILE, "release profile"),
rerunGroup: requiredString(process.env.RERUN_GROUP, "rerun group"),
targetSha: requiredString(process.env.TARGET_SHA, "target SHA"),
workflowRef: requiredString(process.env.GITHUB_REF_NAME, "workflow ref"),
workflowSha: requiredString(process.env.GITHUB_SHA, "workflow SHA"),
};
}
function verifyMode() {
const expected = stateArtifactExpected();
const verified = verifyReleaseStateArtifacts(
readArtifact(
requiredString(process.env.RELEASE_EXECUTION_PLAN_PATH, "execution plan path"),
"execution plan",
),
readArtifact(requiredString(process.env.RELEASE_DECISION_PATH, "decision path"), "decision"),
readArtifact(requiredString(process.env.DIAGNOSTIC_DRAIN_PATH, "drain path"), "drain"),
expected,
);
appendSummary("decision", verified.decision);
appendSummary("drain", verified.drain);
}
function planExpected() {
return {
publicationAdmissionContract:
process.env.FULL_RELEASE_PUBLICATION_ADMISSION_CONTRACT || undefined,
sourceAdmissionContract: process.env.FULL_RELEASE_SOURCE_ADMISSION_CONTRACT || undefined,
targetContextRef: process.env.TARGET_CONTEXT_REF || undefined,
coveragePolicy: process.env.COVERAGE_POLICY || undefined,
telegramWaiver: process.env.TELEGRAM_WAIVER ?? "",
...(process.env.TARGET_VERSION ? { targetVersion: process.env.TARGET_VERSION } : {}),
parentRunId: requiredString(process.env.GITHUB_RUN_ID, "parent run ID"),
repository: requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"),
releaseProfile: requiredString(process.env.RELEASE_PROFILE, "release profile"),
rerunGroup: requiredString(process.env.RERUN_GROUP, "rerun group"),
targetSha: stringValue(process.env.TARGET_SHA),
workflowSha: requiredString(process.env.GITHUB_SHA, "workflow SHA"),
workflowRef: requiredString(process.env.GITHUB_REF_NAME, "workflow ref"),
};
}
function manifestContextFromEnvironment(source) {
const env = process.env;
const coverage = source?.coverage ?? {};
const inputs = {};
for (const [key, variable, sourceKey] of [
["provider", "PROVIDER", "provider"],
["mode", "MODE", "mode"],
["liveSuiteFilter", "LIVE_SUITE_FILTER", "live_suite_filter"],
["crossOsSuiteFilter", "CROSS_OS_SUITE_FILTER", "cross_os_suite_filter"],
["releasePackageSpec", "RELEASE_PACKAGE_SPEC", "release_package_spec"],
[
"packageAcceptancePackageSpec",
"PACKAGE_ACCEPTANCE_PACKAGE_SPEC",
"package_acceptance_package_spec",
],
["codexPluginSpec", "CODEX_PLUGIN_SPEC", "codex_plugin_spec"],
["npmTelegramPackageSpec", "NPM_TELEGRAM_PACKAGE_SPEC", "npm_telegram_package_spec"],
["npmTelegramProviderMode", "NPM_TELEGRAM_PROVIDER_MODE", "npm_telegram_provider_mode"],
["npmTelegramScenario", "NPM_TELEGRAM_SCENARIO", "npm_telegram_scenario"],
["skipPackageTelegramE2e", "SKIP_PACKAGE_TELEGRAM_E2E", "skip_package_telegram_e2e"],
["allowUnreleasedChangelog", "ALLOW_UNRELEASED_CHANGELOG", "allow_unreleased_changelog"],
[
"pluginPrereleaseNodeExcludePatternsJson",
"PLUGIN_PRERELEASE_NODE_EXCLUDE_PATTERNS_JSON",
"plugin_prerelease_node_exclude_patterns_json",
],
[
"extensionTestExcludePatternsJson",
"EXTENSION_TEST_EXCLUDE_PATTERNS_JSON",
"extension_test_exclude_patterns_json",
],
]) {
inputs[key] = env[variable] ?? coverage[sourceKey] ?? "";
}
inputs.targetContextRef = env.TARGET_CONTEXT_REF ?? source?.targetContextRef ?? "";
inputs.targetVersion = env.TARGET_VERSION ?? source?.projection?.version ?? "";
const waiver = env.TELEGRAM_WAIVER ?? coverage.telegram_waiver ?? "";
if (waiver) {
inputs.telegramWaiver = waiver;
}
return {
runId: env.GITHUB_RUN_ID,
runAttempt: env.GITHUB_RUN_ATTEMPT,
workflowRef: env.GITHUB_REF_NAME,
workflowSha: env.GITHUB_SHA,
workflowFullRef: env.GITHUB_REF,
workflowRefType: env.GITHUB_REF_TYPE,
targetRef: env.TARGET_REF ?? source?.targetContextRef ?? "",
releaseProfile: env.RELEASE_PROFILE ?? coverage.release_profile ?? "",
rerunGroup: env.RERUN_GROUP ?? coverage.rerun_group ?? "",
runReleaseSoak: env.RUN_RELEASE_SOAK ?? coverage.run_release_soak ?? "",
validationInputs: inputs,
publicationArtifacts: {
npmPreflight: JSON.parse(env.QUALIFIED_NPM_BUNDLE_JSON || "null"),
docker: env.PREPARED_DOCKER_MANIFEST_SHA256
? {
preparedRunId: env.PREPARED_DOCKER_RUN_ID,
preparedRunAttempt: env.PREPARED_DOCKER_RUN_ATTEMPT,
preparedArtifactName: env.PREPARED_DOCKER_ARTIFACT_NAME,
preparedManifestSha256: env.PREPARED_DOCKER_MANIFEST_SHA256,
}
: null,
},
};
}
function publicationKnownBudget(record, evidenceReuse) {
serializeReleaseArtifact(record);
const source = record.sourceAdmission;
const context = manifestContextFromEnvironment(source);
const candidateRequest = candidateRequestFromEnvironment();
const inputs = {
childPhaseVersion: 3,
parentRunId: source.runId,
parentRunAttempt: source.runAttempt,
workflowRef: context.workflowRef,
workflowSha: source.workflow.sha,
releaseProfile: context.releaseProfile,
rerunGroup: context.rerunGroup,
targetVersion: context.validationInputs.targetVersion,
runReleaseSoak: context.runReleaseSoak,
coveragePolicy: source.coverage.coverage_policy || undefined,
telegramWaiver: source.coverage.telegram_waiver,
releasePackageSpec: source.coverage.release_package_spec,
npmTelegramPackageSpec: source.coverage.npm_telegram_package_spec,
liveSuiteFilter: source.coverage.live_suite_filter,
candidateRequired: process.env.PUBLICATION_CANDIDATE_REQUIRED === "true",
resolveTargetResult: "success",
evidenceReuse: evidenceReuse?.requested === true,
};
const built = buildReleaseExecutionPlan(inputs);
const plan = buildReleaseExecutionPlanArtifact({
...record,
...inputs,
evidenceReuse,
attemptEvidenceVersion: 3,
children: built.children,
gates: built.gates,
trustedWorkflow: {
fullRef: source.tooling.ref,
ref: source.tooling.ref.replace(/^refs\/(?:heads|tags)\//u, ""),
sha: source.tooling.sha,
},
expected: {
...inputs,
repository: source.repository,
targetSha: source.candidateSha,
candidateRequest,
},
});
assertReleasePublicationKnownBudget(plan, context);
}
async function publicationReuseMode() {
const directory = requiredString(process.env.RUNNER_TEMP, "runner temporary directory");
const record = readArtifact(
join(directory, "full-release-publication-admission/publication-admission.json"),
"publication admission",
);
validatePublicationAdmissionBinding(record, {
publicationAdmissionContract: "1",
parentRunId: process.env.GITHUB_RUN_ID,
sourceParentRunAttempt: 1,
workflowSha: process.env.GITHUB_SHA,
workflowRef: process.env.GITHUB_REF_NAME,
});
let reuse;
let outputs;
if (positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent attempt") > 1) {
const { restoreOriginalPublicationAdmission } = await import("./release-ci-summary.mjs");
const restored = await restoreOriginalPublicationAdmission({
request: { ...record.sourceAdmission, runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT) },
});
reuse = restored.plan.evidenceReuse;
outputs = reuse.requested
? {
reuse: "true",
evidence_run_id: reuse.selectedRunId,
evidence_root_run_id: reuse.rootRunId,
evidence_run_url: reuse.runUrl,
evidence_sha: reuse.evidenceSha,
evidence_policy: reuse.policy,
changed_paths: JSON.stringify(reuse.changedPaths),
changed_path_count: String(reuse.changedPaths.length),
}
: { reuse: "false", reuse_reason: "original admission did not reuse evidence" };
} else {
const path = join(directory, "reusable-evidence.outputs");
if (statSync(path).size > MAX_RELEASE_ARTIFACT_BYTES) {
throw new Error("publication reuse output exceeds the enclosing artifact budget");
}
outputs = {};
for (const line of readFileSync(path, "utf8").trimEnd().split("\n")) {
const separator = line.indexOf("=");
const key = line.slice(0, separator);
if (
separator < 1 ||
Object.hasOwn(outputs, key) ||
![
"reuse",
"reuse_reason",
"evidence_run_id",
"evidence_root_run_id",
"evidence_run_url",
"evidence_sha",
"evidence_policy",
"changed_paths",
"changed_path_count",
"evidence_manifest",
].includes(key)
) {
throw new Error("invalid publication reuse output");
}
outputs[key] = line.slice(separator + 1);
}
if (!["true", "false"].includes(outputs.reuse)) {
throw new Error("publication reuse outcome is missing");
}
if (outputs.reuse === "true") {
reuse = {
requested: true,
selectedRunId: outputs.evidence_run_id,
rootRunId: outputs.evidence_root_run_id,
runUrl: outputs.evidence_run_url,
evidenceSha: outputs.evidence_sha,
policy: outputs.evidence_policy,
changedPaths: JSON.parse(outputs.changed_paths),
sourceManifest: JSON.parse(outputs.evidence_manifest),
};
validatePublicationAdmissionBinding(
reuse.sourceManifest,
record.sourceAdmission.validationPurpose === "publish"
? { publicationAdmissionContract: "1" }
: {},
);
}
delete outputs.evidence_manifest;
}
publicationKnownBudget(record, reuse);
// Bulk root evidence remains in the private file/retained plan, never job outputs.
const lines = [];
for (const [key, value] of Object.entries(outputs)) {
if (typeof value !== "string" || /[\r\n]/u.test(value)) {
throw new Error("publication reuse output is not bounded single-line metadata");
}
lines.push(`${key}=${value}\n`);
}
const output = lines.join("");
if (Buffer.byteLength(output, "utf16le") > MAX_RELEASE_ARTIFACT_BYTES) {
throw new Error("publication reuse job outputs exceed the UTF-16 size limit");
}
appendFileSync(process.env.GITHUB_OUTPUT, output);
}
async function publicationMode(mode) {
const directory = requiredString(process.env.RUNNER_TEMP, "runner temporary directory");
const sourcePath = join(directory, "publication-source-admission.json");
const admissionPath = join(directory, "publication-admission.json");
if (mode === "restore-publication") {
if (positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent attempt") <= 1) {
throw new Error("publication restoration requires a later parent attempt");
}
const request = readArtifact(
join(directory, "publication-source-request.json"),
"publication request",
);
const { restoreOriginalPublicationAdmission } = await import("./release-ci-summary.mjs");
const planPath = requiredString(
process.env.FULL_RELEASE_EXECUTION_PLAN_PATH,
"execution plan path",
);
const restored = await restoreOriginalPublicationAdmission({
request,
cachedPlan: existsSync(planPath) ? readArtifact(planPath, "execution plan") : undefined,
});
writeArtifact(sourcePath, restored.source);
writeArtifact(admissionPath, {
sourceAdmissionContract: "1",
sourceAdmission: restored.source,
publicationAdmissionContract: "1",
publicationAdmission: restored.admission,
});
writeArtifact(planPath, restored.plan);
return;
}
if (positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent attempt") !== 1) {
// Only the preceding authenticated restore creates this file on later attempts.
const restored = readArtifact(admissionPath, "restored publication admission");
validatePublicationAdmissionBinding(restored, {
publicationAdmissionContract: "1",
parentRunId: process.env.GITHUB_RUN_ID,
sourceParentRunAttempt: 1,
workflowSha: process.env.GITHUB_SHA,
workflowRef: process.env.GITHUB_REF_NAME,
});
publicationKnownBudget(restored);
return;
}
const source = readArtifact(sourcePath, "publication source");
let admission = null;
if (source.validationPurpose === "publish") {
if (process.env.PUBLICATION_UPLOAD_OUTCOME !== "success") {
throw new Error("publication observation upload did not succeed");
}
const path = join(directory, "publication-observations.json");
const observations = readArtifact(path, "publication observations");
if (readFileSync(path, "utf8") !== publicationObservationJson(observations)) {
throw new Error("publication observation upload bytes are not canonical");
}
const { createReleaseEvidenceClient, validatePublicationObservationArtifactIdentity } =
await import("./release-ci-summary.mjs");
const uploaded = {
id: requiredString(process.env.PUBLICATION_ARTIFACT_ID, "publication artifact ID"),
digest: requiredString(
process.env.PUBLICATION_ARTIFACT_DIGEST,
"publication artifact digest",
),
};
if (!/^[a-f0-9]{64}$/u.test(uploaded.digest)) {
throw new Error("invalid publication upload action digest");
}
uploaded.digest = `sha256:${uploaded.digest}`;
const metadata = createReleaseEvidenceClient(source.repository).getArtifact(uploaded.id);
const descriptor = validatePublicationObservationArtifactIdentity(metadata, source, uploaded);
admission = createPublicationAdmission(
source,
observations,
descriptor,
new Date().toISOString(),
);
}
const record = {
sourceAdmissionContract: "1",
sourceAdmission: source,
publicationAdmissionContract: "1",
publicationAdmission: admission,
};
validatePublicationAdmissionBinding(record, { publicationAdmissionContract: "1" });
publicationKnownBudget(record);
writeArtifact(admissionPath, record);
}
async function writeManifestMode() {
const plan = readArtifact(process.env.RELEASE_EXECUTION_PLAN_PATH, "execution plan");
const drain = readArtifact(process.env.DIAGNOSTIC_DRAIN_PATH, "diagnostic drain");
const manifest = buildReleaseValidationManifest({
plan,
drain,
context: manifestContextFromEnvironment(plan.sourceAdmission),
});
if (manifest.sourceAdmission?.validationPurpose === "publish" && manifest.rerunGroup === "all") {
const outputDir = mkdtempSync(
join(
requiredString(process.env.RUNNER_TEMP, "runner temporary directory"),
"sealed-npm-preflight-",
),
);
try {
await downloadFullReleaseNpmPreflight({
manifest,
repository: manifest.sourceAdmission.repository,
runId: manifest.runId,
runAttempt: manifest.runAttempt,
sourceSha: manifest.targetSha,
toolingSha: manifest.workflowSha,
outputDir,
token: requiredString(process.env.GH_TOKEN, "GitHub token"),
});
manifest.publishInputs = await createReleasePublishInputs({
manifest,
npmManifest: JSON.parse(readFileSync(join(outputDir, "preflight-manifest.json"), "utf8")),
});
} finally {
rmSync(outputDir, { recursive: true, force: true });
}
}
writeArtifact(
join(
requiredString(process.env.RUNNER_TEMP, "runner temporary directory"),
"full-release-validation/full-release-validation-manifest.json",
),
manifest,
);
}
function candidateRequestFromEnvironment() {
return validateFullReleaseCandidateRequest(
JSON.parse(requiredString(process.env.CANDIDATE_REQUEST_JSON, "candidate request JSON")),
);
}
function evidenceReuseFromInputs(planInputs, sourceManifest = {}) {
return {
changedPaths: changedPathsValue(planInputs.evidenceChangedPaths),
evidenceSha: stringValue(planInputs.evidenceSha),
policy: stringValue(planInputs.evidencePolicy),
requested: planInputs.evidenceReuse === true || planInputs.evidenceReuse === "true",
rootRunId: stringValue(planInputs.evidenceRootRunId),
runUrl: stringValue(planInputs.evidenceRunUrl),
selectedRunId: stringValue(planInputs.evidenceRunId),
sourceManifest,
};
}
function candidateFromInputs(planInputs, gates) {
const candidate = planInputs.candidateEvidence ?? null;
const bindingRequired = gates.some(
(gate) =>
["Acquire full release candidate", "Prepare shared release candidate"].includes(gate.name) &&
gate.required,
);
if (!bindingRequired) {
if (candidate !== null) {
throw new Error("release candidate evidence exists when candidate binding is not required");
}
return null;
}
if (
stringValue(planInputs.candidateAcquisitionResult ?? planInputs.candidateBindingResult) ===
"success"
) {
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) {
throw new Error("successful release candidate binding omitted producer evidence");
}
return candidate;
}
if (candidate !== null) {
throw new Error("release candidate evidence exists without successful binding");
}
return null;
}
async function planMode() {
const outputPath = requiredString(
process.env.FULL_RELEASE_EXECUTION_PLAN_PATH,
"execution plan output path",
);
const expected = planExpected();
const currentAttempt = positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent run attempt");
if (process.env.FULL_RELEASE_RESTORE_PLAN === "true") {
const restoredPayload = readArtifact(outputPath, "execution plan");
const restored = validateReleaseExecutionPlanArtifact(restoredPayload, {
...expected,
...(restoredPayload.attemptEvidenceVersion !== undefined
? {
candidateRequest: validateRecordedFullReleaseCandidateRequest(
JSON.parse(
requiredString(process.env.CANDIDATE_REQUEST_JSON, "candidate request JSON"),
),
),
}
: {}),
sourceParentRunAttempt: 1,
});
if (expected.publicationAdmissionContract) {
const source = JSON.parse(
requiredString(process.env.SOURCE_ADMISSION_JSON, "restored source admission"),
);
const { restoreOriginalPublicationAdmission } = await import("./release-ci-summary.mjs");
await restoreOriginalPublicationAdmission({
request: { ...source, runAttempt: currentAttempt },
cachedPlan: restored,
});
}
writeExecutionPlan(outputPath, restored);
return;
}
if (currentAttempt !== 1) {
throw new Error("collector retry omitted the immutable attempt-one execution plan");
}
const planInputs = {
...parsePlanInputs(process.env.FULL_RELEASE_PLAN_INPUTS_JSON),
releaseProfile: expected.releaseProfile,
};
if (expected.publicationAdmissionContract) {
const publication =
planInputs.resolveTargetResult === "success"
? readArtifact(
requiredString(process.env.PUBLICATION_ADMISSION_PATH, "publication admission path"),
"publication admission",
)
: {
publicationAdmissionContract: expected.publicationAdmissionContract,
publicationAdmission: null,
};
Object.assign(planInputs, publication);
}
const attemptEvidenceVersion = Number(planInputs.childPhaseVersion) === 3 ? 3 : 2;
const built = buildReleaseExecutionPlan(planInputs);
const candidate = candidateFromInputs(planInputs, built.gates);
const candidateRequest = validateFullReleaseCandidateRequest(planInputs.candidateRequestInput);
const abortController = new AbortController();
let finished = false;
const artifactInputs = {
attemptEvidenceVersion,
sourceAdmissionContract: planInputs.sourceAdmissionContract,
sourceAdmission: planInputs.sourceAdmission,
publicationAdmissionContract: planInputs.publicationAdmissionContract,
publicationAdmission: planInputs.publicationAdmission,
candidate,
coveragePolicy: planInputs.coveragePolicy,
children: hydrateReusedPlan(built.children, { childReuse: planInputs.childReuse ?? {} }),
childReuse: planInputs.childReuse,
evidenceReuse: evidenceReuseFromInputs(planInputs),
expected: { ...expected, candidateRequest, parentRunAttempt: currentAttempt },
gates: built.gates,
releaseProfile: expected.releaseProfile,
rerunGroup: expected.rerunGroup,
telegramWaiver: planInputs.telegramWaiver,
targetVersion: planInputs.targetVersion,
trustedWorkflow: planInputs.trustedWorkflow,
};
let plan = buildReleaseExecutionPlanArtifact(artifactInputs);
const stop = () => {
if (finished) {
return;
}
abortController.abort(new Error("execution plan collection cancelled"));
plan = buildReleaseExecutionPlanArtifact({
...plan,
attemptEvidenceVersion,
errors: [
...plan.errors,
{
child: "<collector>",
kind: "collector_cancelled",
message: "execution plan collector received a termination signal",
},
],
expected: {
...expected,
candidateRequest: plan.candidateRequest,
parentRunAttempt: currentAttempt,
},
releaseProfile: expected.releaseProfile,
rerunGroup: expected.rerunGroup,
});
writeExecutionPlan(outputPath, plan);
finished = true;
process.exit(1);
};
process.once("SIGINT", stop);
process.once("SIGTERM", stop);
const reuse = await validateReuse(plan, abortController.signal);
if (finished) {
return;
}
plan = buildReleaseExecutionPlanArtifact({
...artifactInputs,
blockers: reuse.blockers,
children: reuse.children,
errors: reuse.errors,
evidenceReuse: evidenceReuseFromInputs(planInputs, reuse.sourceManifest),
});
writeExecutionPlan(outputPath, plan);
finished = true;
if ((reuse.blockers?.length ?? 0) > 0 || (reuse.errors?.length ?? 0) > 0) {
throw new Error("release execution plan could not bind reusable evidence");
}
}
async function collectMode(mode) {
const expected = {
parentRunAttempt: positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent run attempt"),
parentRunId: requiredString(process.env.GITHUB_RUN_ID, "parent run ID"),
repository: requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"),
targetSha: stringValue(process.env.TARGET_SHA),
workflowRef: requiredString(process.env.GITHUB_REF_NAME, "workflow ref"),
workflowSha: requiredString(process.env.GITHUB_SHA, "workflow SHA"),
};
const releaseProfile = requiredString(process.env.RELEASE_PROFILE, "release profile");
const rerunGroup = requiredString(process.env.RERUN_GROUP, "rerun group");
const outputPath = requiredString(process.env.FULL_RELEASE_STATE_PATH, "state output path");
const executionPlan = validateReleaseExecutionPlanArtifact(
readArtifact(
requiredString(process.env.FULL_RELEASE_EXECUTION_PLAN_PATH, "execution plan path"),
"execution plan",
),
{
publicationAdmissionContract:
process.env.FULL_RELEASE_PUBLICATION_ADMISSION_CONTRACT || undefined,
sourceAdmissionContract: process.env.FULL_RELEASE_SOURCE_ADMISSION_CONTRACT || undefined,
parentRunId: expected.parentRunId,
repository: expected.repository,
releaseProfile,
rerunGroup,
targetSha: expected.targetSha,
workflowSha: expected.workflowSha,
},
);
const plan = executionPlan.children;
const policy = {
releaseProfile,
workflowRef: expected.workflowRef,
};
const gateFailures = releasePlanGateFailures(executionPlan.gates);
const failFast = mode === "decision" && process.env.FAIL_FAST === "true";
const pollIntervalMs =
Number(process.env.FULL_RELEASE_POLL_INTERVAL_MS) || DEFAULT_POLL_INTERVAL_MS;
const heartbeatIntervalMs =
Number(process.env.FULL_RELEASE_HEARTBEAT_INTERVAL_MS) || DEFAULT_HEARTBEAT_INTERVAL_MS;
const cancelledRunIds = new Set();
let snapshots = plan.map((child) => ({
...child,
errors: [],
jobs: [],
status: child.selected && child.runId ? "queued" : child.selected ? "missing" : "skipped",
}));
let finished = false;
const abortController = new AbortController();
let decisionReuse = { blockers: [], children: plan, errors: [] };
let transport = { status: "certain" };
const writePayload = (decision, cancellation = {}) => {
const payload = buildReleaseStateArtifact({
cancellation,
children: snapshots,
decision,
executionPlan,
expected,
mode,
releaseProfile,
rerunGroup,
transport,
});
writeResult(outputPath, payload);
appendSummary(mode, payload);
return payload;
};
const stop = () => {
if (finished) {
return;
}
abortController.abort(new Error(`${mode} collector cancelled`));
const decision = classifyReleaseSnapshot({
cancelled: true,
children: snapshots,
extraBlockers: executionPlan.blockers,
extraErrors: [
...executionPlan.errors,
...(transport.error ? [transport.error] : []),
{
child: "<collector>",
kind: "collector_cancelled",
message: `${mode} collector received a termination signal`,
},
],
localFailures: gateFailures,
...policy,
});
writePayload(decision, { cancelledRunIds, requested: true });
finished = true;
process.exit(1);
};
process.once("SIGINT", stop);
process.once("SIGTERM", stop);
if (executionPlan.childReuse || (mode === "decision" && executionPlan.evidenceReuse.requested)) {
decisionReuse = await validateReuse(executionPlan, abortController.signal);
const exactPlan = JSON.stringify(
plan.map(({ key, runAttempt, runId }) => ({ key, runAttempt, runId })),
);
const revalidatedPlan = JSON.stringify(
decisionReuse.children.map(({ key, runAttempt, runId }) => ({ key, runAttempt, runId })),
);
if (exactPlan !== revalidatedPlan) {
decisionReuse = {
...decisionReuse,
blockers: [
...decisionReuse.blockers,
{
child: "<evidence>",
kind: "provenance_mismatch",
message: "revalidated evidence child identities differ from the immutable plan",
runId: executionPlan.evidenceReuse.rootRunId,
url: executionPlan.evidenceReuse.runUrl,
},
],
};
}
if (
JSON.stringify(sortJsonValueKeys(decisionReuse.sourceManifest)) !==
JSON.stringify(sortJsonValueKeys(executionPlan.evidenceReuse.sourceManifest))
) {
decisionReuse = {
...decisionReuse,
blockers: [
...decisionReuse.blockers,
{
child: "<evidence>",
kind: "provenance_mismatch",
message: "revalidated evidence source manifest differs from the immutable plan",
runId: executionPlan.evidenceReuse.rootRunId,
url: executionPlan.evidenceReuse.runUrl,
},
],
};
}
}
let nextHeartbeat = 0;
while (!finished) {
ghRetryDeadline = transport.deadlineMonotonicMs;
snapshots = await Promise.all(
plan.map((child, index) =>
readChild(child, snapshots[index], abortController.signal, {
reuseSelection: executionPlan.childReuse?.[child.key],
parentRunId: executionPlan.parentRunId,
parentRunAttempt: executionPlan.parentRunAttempt,
targetSha: executionPlan.targetSha,
}),
),
);
transport = updateReleaseTransportEpisode(transport, snapshots);
const transportReadErrors = transport.error ? [transport.error] : [];
let decision = classifyReleaseSnapshot({
children: snapshots,
extraBlockers: [...executionPlan.blockers, ...decisionReuse.blockers],
extraErrors: [...transportReadErrors, ...executionPlan.errors, ...decisionReuse.errors],
localFailures: gateFailures,
...policy,
});
if (Date.now() >= nextHeartbeat) {
console.log(formatReleaseStateHeartbeat(mode, decision));
nextHeartbeat = Date.now() + heartbeatIntervalMs;
}
if (failFast && decision.blockers.length > 0) {
const cancellationErrors = await cancelAffectedChildren(
snapshots,
decision.blockers,
cancelledRunIds,
abortController.signal,
);
if (cancellationErrors.length > 0) {
decision = classifyReleaseSnapshot({
children: snapshots,
extraBlockers: [
...executionPlan.blockers,
...decisionReuse.blockers,
...decision.blockers,
],
extraErrors: [
...transportReadErrors,
...executionPlan.errors,
...decisionReuse.errors,
...cancellationErrors,
],
localFailures: gateFailures,
...policy,
});
}
}
const done =
mode === "decision"
? decision.state !== "qualifying" &&
!(decision.state === "passed" && transport.status === "uncertain")
: transport.status !== "uncertain" &&
(decision.state === "orchestration_error" ||
(decision.state !== "qualifying" && decision.activeRunIds.length === 0));
if (done) {
const payload = writePayload(decision, { cancelledRunIds, requested: false });
finished = true;
process.exitCode =
payload.state === "passed" ? 0 : payload.state === "orchestration_error" ? 2 : 1;
return;
}
await abortableSleep(
deadlineDelayMs(pollIntervalMs, transport.deadlineMonotonicMs, performance.now()),
abortController.signal,
);
}
}
function readStateCandidates(root, prefix, runId, maxParentRunAttempt, filename) {
const pattern = new RegExp(`^${prefix}-${runId}-([1-9][0-9]*)$`, "u");
const candidates = readdirSync(root, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => {
const match = pattern.exec(entry.name);
return match ? { attempt: Number(match[1]), name: entry.name } : undefined;
})
.filter(Boolean)
.filter((entry) => entry.attempt <= maxParentRunAttempt)
.map((entry) => {
const payload = readArtifact(join(root, entry.name, filename), entry.name);
if (Number(payload.parentRunAttempt) !== entry.attempt) {
throw new Error(`${entry.name} payload attempt differs from its artifact name`);
}
return { name: entry.name, payload };
});
const directPath = join(root, filename);
if (existsSync(directPath)) {
const payload = readArtifact(directPath, filename);
const attempt = positiveInteger(payload.parentRunAttempt, `${filename} parent run attempt`);
if (attempt <= maxParentRunAttempt) {
candidates.push({ name: `${prefix}-${runId}-${attempt}`, payload });
}
}
return candidates;
}
async function validateManifestMode() {
const expected = stateArtifactExpected();
const manifestPath = requiredString(
process.env.RELEASE_VALIDATION_MANIFEST_PATH,
"release validation manifest path",
);
const executionPlanPayload = readArtifact(
requiredString(process.env.RELEASE_EXECUTION_PLAN_PATH, "execution plan path"),
"execution plan",
);
const executionPlan = validateReleaseExecutionPlanArtifact(executionPlanPayload, expected);
const verified =
executionPlan.attemptEvidenceVersion !== undefined
? verifyReleaseStateArtifacts(
executionPlanPayload,
readArtifact(
requiredString(process.env.RELEASE_DECISION_PATH, "release decision path"),
"release decision",
),
readArtifact(
requiredString(process.env.DIAGNOSTIC_DRAIN_PATH, "diagnostic drain path"),
"diagnostic drain",
),
expected,
)
: undefined;
const drain = verified?.drain;
const rawManifest = readArtifact(manifestPath, "release validation manifest");
const { validateParentManifest } = await import("./release-ci-summary.mjs");
const manifest = validateParentManifest(rawManifest, {
sourceAdmissionContract: expected.sourceAdmissionContract,
candidateBinding: executionPlan.candidate ?? null,
repository: expected.repository,
runAttempt: positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent run attempt"),
runId: executionPlan.parentRunId,
workflowRef: executionPlan.workflowRef,
workflowSha: executionPlan.workflowSha,
});
validatePublicationAdmissionBinding(rawManifest, expected);
if (
JSON.stringify(rawManifest.sourceAdmission) !== JSON.stringify(executionPlan.sourceAdmission) ||
rawManifest.sourceAdmissionContract !== executionPlan.sourceAdmissionContract ||
(executionPlan.sourceAdmissionContract &&
JSON.stringify(rawManifest.trustedWorkflow) !== JSON.stringify(executionPlan.trustedWorkflow))
) {
throw new Error("release manifest source admission differs from its immutable plan");
}
if (
rawManifest.publicationAdmissionContract !== executionPlan.publicationAdmissionContract ||
JSON.stringify(sortJsonValueKeys(rawManifest.publicationAdmission)) !==
JSON.stringify(sortJsonValueKeys(executionPlan.publicationAdmission))
) {
throw new Error("release manifest publication admission differs from its immutable plan");
}
validateReleaseTelegramWaiverBinding(executionPlan, manifest.validationInputs);
validateReleaseCoveragePolicyBinding(executionPlan, manifest.validationInputs);
const expectedChildRunIds = Object.fromEntries(
executionPlan.children.map((child) => [
child.key,
child.selected ? stringValue(child.runId) : "",
]),
);
const expectedEvidenceReuse = executionPlan.evidenceReuse.requested
? {
changedPaths: executionPlan.evidenceReuse.changedPaths,
evidenceSha: executionPlan.evidenceReuse.evidenceSha,
policy: executionPlan.evidenceReuse.policy,
runId: executionPlan.evidenceReuse.rootRunId,
selectedRunId: executionPlan.evidenceReuse.selectedRunId,
}
: undefined;
const expectedChildEvidence = drain
? Object.fromEntries(
Object.entries(drain.children).map(([key, child]) => [
key,
{
...releaseChildClassificationEvidence(child),
compositeJobsSha256: child.compositeJobsSha256,
dispatchActor: child.dispatchActor,
effectiveRunAttempt: child.runAttempt,
jobs: child.timing.jobs.map((job) => ({
acceptedRunAttempt: job.acceptedRunAttempt,
completedAt: job.completedAt,
conclusion: job.conclusion,
name: job.name,
startedAt: job.startedAt,
status: job.status,
url: job.url,
})),
observedRunAttempts: child.observedRunAttempts,
plannedRunAttempt: child.plannedRunAttempt,
repository: child.repository,
runId: child.runId,
triggeringActor: child.triggeringActor,
},
]),
)
: undefined;
if (
manifest.targetSha !== executionPlan.targetSha ||
manifest.releaseProfile !== executionPlan.releaseProfile ||
manifest.rerunGroup !== executionPlan.rerunGroup ||
JSON.stringify(sortJsonValueKeys(manifest.childRunIds)) !==
JSON.stringify(sortJsonValueKeys(expectedChildRunIds)) ||
JSON.stringify(sortJsonValueKeys(manifest.evidenceReuse)) !==
JSON.stringify(sortJsonValueKeys(expectedEvidenceReuse)) ||
(executionPlan.attemptEvidenceVersion !== undefined &&
JSON.stringify(sortJsonValueKeys(rawManifest.childEvidence)) !==
JSON.stringify(sortJsonValueKeys(expectedChildEvidence))) ||
rawManifest.executionPlanSha256 !== executionPlan.sha256 ||
Number(rawManifest.sourceParentRunAttempt) !== executionPlan.parentRunAttempt
) {
throw new Error("release validation manifest differs from the immutable execution plan");
}
rawManifest.advisoryJobs = manifest.advisoryJobs;
writeArtifact(manifestPath, rawManifest);
}
function selectMode() {
const expected = stateArtifactExpected("current parent run attempt");
const selected = selectReleaseStateArtifacts(
readArtifact(
requiredString(process.env.RELEASE_EXECUTION_PLAN_PATH, "execution plan path"),
"execution plan",
),
readStateCandidates(
requiredString(process.env.RELEASE_DECISION_ATTEMPTS_PATH, "decision attempts path"),
"full-release-decision",
expected.parentRunId,
expected.maxParentRunAttempt,
"full-release-decision.json",
),
readStateCandidates(
requiredString(process.env.DIAGNOSTIC_DRAIN_ATTEMPTS_PATH, "drain attempts path"),
"full-release-diagnostics",
expected.parentRunId,
expected.maxParentRunAttempt,
"full-release-diagnostic-manifest.json",
),
expected,
);
writeArtifact(
requiredString(process.env.RELEASE_DECISION_PATH, "selected decision path"),
selected.decision,
);
writeArtifact(
requiredString(process.env.DIAGNOSTIC_DRAIN_PATH, "selected drain path"),
selected.drain,
);
if (process.env.GITHUB_OUTPUT) {
appendFileSync(
process.env.GITHUB_OUTPUT,
`decision_source_attempt=${selected.sourceAttempts.decision}\n`,
);
appendFileSync(
process.env.GITHUB_OUTPUT,
`drain_source_attempt=${selected.sourceAttempts.drain}\n`,
);
}
}
async function main() {
const mode = process.argv[2];
if (mode === "reuse-publication") {
await publicationReuseMode();
return;
}
if (["restore-publication", "finalize-publication"].includes(mode)) {
await publicationMode(mode);
return;
}
if (mode === "write-manifest") {
await writeManifestMode();
return;
}
if (mode === "plan") {
await planMode();
return;
}
if (mode === "verify") {
verifyMode();
return;
}
if (mode === "select") {
selectMode();
return;
}
if (mode === "validate-manifest") {
await validateManifestMode();
return;
}
if (!["decision", "drain"].includes(mode)) {
throw new Error(
"usage: full-release-validation-state.mjs <plan|decision|drain|select|validate-manifest|verify>",
);
}
await collectMode(mode);
}
if (process.argv[1]?.endsWith("full-release-validation-state.mjs")) {
try {
await main();
} catch (error) {
console.error(error instanceof Error ? error.message : String(error));
process.exit(2);
}
}