From 9dc08b1fac6b88355393c3ed281c4e656d09a912 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 29 Sep 2026 23:53:05 -0700 Subject: [PATCH] fix(release): publish preflight stalls silently, frv continue is silent, and the SDK acknowledgement is reported late (#161633) * fix(release): bound publish preflight observation, narrate frv continue, and surface SDK acknowledgement early * fix(release): replay failed preflight reads with their exact filter * test(release): model the exact-tag release lookup in preflight inventory fixtures --- .../references/regular-release.md | 11 ++ scripts/frv.mjs | 60 ++++++- .../release-publish-preflight-evidence.mts | 106 ++++++++---- scripts/lib/release-publish-state.mts | 107 +++++++----- scripts/plugin-sdk-api-release-evidence.mjs | 11 +- scripts/release-candidate-checklist.mts | 83 +++++++--- test/scripts/frv.test.ts | 75 +++++++++ .../release-candidate-checklist.test.ts | 105 ++++++++++-- ...release-publish-closeout-preflight.test.ts | 4 +- ...release-publish-preflight-evidence.test.ts | 36 +++- test/scripts/release-publish-state.test.ts | 154 +++++++++++++++++- 11 files changed, 632 insertions(+), 120 deletions(-) diff --git a/.agents/skills/release-openclaw-maintainer/references/regular-release.md b/.agents/skills/release-openclaw-maintainer/references/regular-release.md index c67999118f71..3e8398b33aa4 100644 --- a/.agents/skills/release-openclaw-maintainer/references/regular-release.md +++ b/.agents/skills/release-openclaw-maintainer/references/regular-release.md @@ -170,9 +170,14 @@ Manual tag creation remains the fallback. The push may print a tag still exists: verify with `gh api repos/openclaw/openclaw/git/ref/tags/` and, only if missing, create it with `gh api -X POST repos/openclaw/openclaw/git/refs -f ref=refs/tags/ -f sha=`. +Run the candidate from a clean tracked worktree whose HEAD is the Release SHA, +with its frozen dependencies installed. The helper creates and relaunches trusted +Tooling SHA code itself; starting in the tooling checkout fails the target HEAD check. Then consume existing validation against the untagged Release SHA: ```bash +git worktree add --detach /private/tmp/openclaw-candidate- +cd /private/tmp/openclaw-candidate- && pnpm install --frozen-lockfile pnpm release:candidate -- \ --tag \ --target-sha \ @@ -185,6 +190,12 @@ pnpm release:candidate -- \ --skip-dispatch ``` +If `pnpm` stalls on the global store lock, check for another agent running +`pnpm store prune` (`pgrep -fl 'pnpm.*store.*prune'`). With dependencies already +installed, bypass the pnpm launcher using `node --import ./scripts/tsx.mjs scripts/release-candidate-checklist.mts ...` +or `node --import ./scripts/tsx.mjs scripts/release-publish-preflight.mts ...` +with the same helper arguments. A dependency install still needs the lock. + Match channel, route, and profile to the frozen validation selection. The channel and route default to `beta` and `normal`; final versions require stable/full evidence with soak and blocking performance, even on the beta channel. diff --git a/scripts/frv.mjs b/scripts/frv.mjs index 360866371a96..5f304dee9e25 100644 --- a/scripts/frv.mjs +++ b/scripts/frv.mjs @@ -1013,6 +1013,7 @@ async function reconcileAttemptStarts( client, mutationResults, operationDeadline, + onStarted, ) { const reconcileDeadline = Math.min( operationDeadline, @@ -1042,6 +1043,7 @@ async function reconcileAttemptStarts( const expectedAttempt = minimumAttempts.get(runId); const observedAttempt = controllerRunAttempt(run, sourceAttempt, expectedAttempt); if (observedAttempt === expectedAttempt) { + onStarted(run); pending.delete(runId); } } @@ -1457,13 +1459,50 @@ async function verifyRerunAttemptJobs(child, runAttempt, client, operationDeadli } } -export async function continueFailed(plan, rootRunId, client, options = {}) { +export async function continueFailed(plan, rootRunId, reader, options = {}) { const operationDeadline = options.operationDeadline === undefined ? createOperationDeadline() : validateOperationDeadline(options.operationDeadline); const ownedAttempts = new Map(); const target = resolveRerunTarget(plan, options); + const runKeys = new Map(selectedChildren(plan).map((child) => [child.runId, child.key])); + runKeys.set(String(rootRunId), "parent"); + const reported = new Map(); + const log = options.log ?? console.error; + const report = (run, started = false) => { + const key = runKeys.get(String(run.id)); + if (!key) { + return; + } + const message = formatRunProgress(key, run, started ? "started" : undefined); + const previous = reported.get(String(run.id)); + const now = Date.now(); + if ( + started || + previous?.message !== message || + (run.status !== "completed" && now - previous.at >= 5 * 60_000) + ) { + log( + formatProgressEvent("continue", { + message: !started && run.status !== "completed" ? `waiting for ${message}` : message, + url: `https://github.com/${client.repository ?? DEFAULT_REPOSITORY}/actions/runs/${run.id}/attempts/${run.run_attempt}`, + }), + ); + if (!started) { + reported.set(String(run.id), { message, at: now }); + } + } + }; + const getRun = reader.getRun.bind(reader); + const client = { + ...reader, + getRun: async (...args) => { + const run = await getRun(...args); + report(run); + return run; + }, + }; const initial = await preflightContinuation( plan, rootRunId, @@ -1476,6 +1515,9 @@ export async function continueFailed(plan, rootRunId, client, options = {}) { const reruns = []; let status; while (true) { + for (const producer of artifactProducers) { + runKeys.set(producer.runId, `artifact:${producer.request.stage}`); + } status = await inspectRecovery(plan, artifactProducers, client, { operationDeadline }); for (const child of status.children) { const expectedAttempt = ownedAttempts.get(child.runId); @@ -1602,6 +1644,7 @@ export async function continueFailed(plan, rootRunId, client, options = {}) { client, mutationResults.filter((_result, index) => sentRunIds.has(requests[index].child.runId)), operationDeadline, + (run) => report(run, true), ); } const admissionFailure = mutationResults.find( @@ -1730,6 +1773,7 @@ export async function continueFailed(plan, rootRunId, client, options = {}) { client, mutationResults, operationDeadline, + (run) => report(run, true), ); ownedAttempts.set(rootRunId, minimumAttempts.get(rootRunId)); await waitForTerminal([rootRunId], client, operationDeadline, minimumAttempts); @@ -2362,6 +2406,14 @@ function writeWatchState(path, state) { renameSync(temporary, path); } +function formatRunProgress(owner, run, state) { + return `${owner}${owner === "parent" ? "" : " run"} ${run.id} attempt ${run.run_attempt} ${state ?? (run.status === "completed" ? `completed ${run.conclusion}` : run.status)}`; +} + +function formatProgressEvent(command, event) { + return `[frv ${command}] ${new Date().toISOString().slice(11, 19)}Z ${event.message}${event.url ? ` ${event.url}` : ""}`; +} + function failedJobEvent(owner, job, attempt) { if (job.status !== "completed" || !FAILED_JOB_CONCLUSIONS.has(String(job.conclusion))) { return undefined; @@ -2418,7 +2470,7 @@ async function pollRelease(state, client, pending, readOptions) { const parentDone = parent.status === "completed"; report( `run:${parentRunId}:${parent.run_attempt}:${parentDone ? "completed" : "active"}`, - `parent ${parentRunId} attempt ${parent.run_attempt} ${parentDone ? `completed ${parent.conclusion}` : parent.status}`, + formatRunProgress("parent", parent), parent.html_url, ); const dispatchKeys = new Map([ @@ -2476,7 +2528,7 @@ async function pollRelease(state, client, pending, readOptions) { const done = run.status === "completed"; report( `run:${runId}:${current}:${done ? "completed" : "active"}`, - `${child.key} run ${runId} attempt ${current} ${done ? `completed ${run.conclusion}` : run.status}`, + formatRunProgress(child.key, run), run.html_url, ); // Earlier attempts are final; scan each once so a late start still reports them. @@ -2628,7 +2680,7 @@ async function main() { console.log( options.json ? JSON.stringify({ at: new Date().toISOString(), ...event }) - : `[frv watch] ${new Date().toISOString().slice(11, 19)}Z ${event.message}${event.url ? ` ${event.url}` : ""}`, + : formatProgressEvent("watch", event), ), intervalMs: options.intervalMs, once: options.once, diff --git a/scripts/lib/release-publish-preflight-evidence.mts b/scripts/lib/release-publish-preflight-evidence.mts index 06d79db5a104..97e9ecbb97b6 100644 --- a/scripts/lib/release-publish-preflight-evidence.mts +++ b/scripts/lib/release-publish-preflight-evidence.mts @@ -24,7 +24,16 @@ import { import { isRecord, trimString } from "./record-shared.mjs"; import type { ReleasePublishGate } from "./release-publish-gates.mts"; -export type PublishPreflightGh = (args: string[]) => string; +export type PublishPreflightGh = (args: string[], options?: { timeoutMs?: number }) => string; + +export function publishPreflightGhError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + const stderr = + isRecord(error) && (typeof error.stderr === "string" || Buffer.isBuffer(error.stderr)) + ? error.stderr.toString().trim() + : ""; + return [stderr, message].filter(Boolean).join("; ").replace(/\s+/gu, " ").slice(0, 1000); +} export type PublishPreflightRecord = Record; type CoreTarball = { packageName: string; @@ -183,13 +192,14 @@ export function requirePreflightRecord(value: unknown, label: string): PublishPr export function createPublishPreflightGh(): PublishPreflightGh { const responses = new Map(); - return (args) => { + return (args, options) => { const key = JSON.stringify(args); let response = responses.get(key); if (response === undefined) { response = execFileSync("gh", args, { encoding: "utf8", - timeout: 60_000, + timeout: Math.min(60_000, options?.timeoutMs ?? 60_000), + killSignal: "SIGKILL", maxBuffer: 32 * 1024 * 1024, stdio: ["ignore", "pipe", "pipe"], }); @@ -205,11 +215,66 @@ export function preflightApi(runGh: PublishPreflightGh, repo: string, endpoint: ); } -export function readPublishPreflightRelease(runGh: PublishPreflightGh, repo: string, tag: string) { - // The tag endpoint omits drafts, and gh release view can mask a failed draft - // lookup as absence. One bounded list owner preserves errors and shared caching. +export function readPublishPreflightRelease( + runGh: PublishPreflightGh, + repo: string, + tag: string, + onProgress: (message: string) => void = () => {}, +) { + const found = (release: unknown) => { + if ( + !isRecord(release) || + release.tag_name !== tag || + typeof release.id !== "number" || + typeof release.draft !== "boolean" || + typeof release.prerelease !== "boolean" || + typeof release.html_url !== "string" || + typeof release.target_commitish !== "string" + ) { + throw new Error("Invalid GitHub release response."); + } + return { + state: "found" as const, + release: { + id: release.id, + draft: release.draft, + prerelease: release.prerelease, + tag_name: tag, + html_url: release.html_url, + target_commitish: release.target_commitish, + body: release.body, + assets: release.assets, + }, + }; + }; + onProgress(`release lookup exact tag ${tag}`); + let exact: unknown; + try { + exact = preflightApi(runGh, repo, `releases/tags/${encodeURIComponent(tag)}`); + } catch (error) { + // A missing public tag does not establish draft absence; preserve the + // authenticated inventory fallback, but never mask other read failures. + if (!/\b404\b/u.test(publishPreflightGhError(error))) { + throw error; + } + } + if (exact !== undefined) { + return found(exact); + } for (let page = 1; page <= 20; page++) { - const releases = preflightApi(runGh, repo, `releases?per_page=100&page=${page}`); + onProgress(`release lookup page ${page}/20`); + // Retain full evidence only for this tag; other bodies/assets can dominate + // the response by megabytes while contributing only pagination and tag names. + const releases: unknown = JSON.parse( + runGh([ + "api", + `repos/${repo}/releases?per_page=100&page=${page}`, + "--method", + "GET", + "--jq", + `map(if .tag_name == ${JSON.stringify(tag)} then . else {tag_name} end)`, + ]), + ); if ( !Array.isArray(releases) || releases.length > 100 || @@ -222,32 +287,11 @@ export function readPublishPreflightRelease(runGh: PublishPreflightGh, repo: str } const release = releases.find((entry) => entry.tag_name === tag); if (release) { - if ( - typeof release.id !== "number" || - typeof release.draft !== "boolean" || - typeof release.prerelease !== "boolean" || - typeof release.html_url !== "string" || - typeof release.target_commitish !== "string" - ) { - throw new Error("Invalid GitHub release response."); - } - return { - state: "found" as const, - release: { - id: release.id, - draft: release.draft, - prerelease: release.prerelease, - tag_name: tag, - html_url: release.html_url, - target_commitish: release.target_commitish, - body: release.body, - assets: release.assets, - }, - }; + return found(release); } if (releases.length < 100) { - // GitHub includes drafts only for readers with push access. A complete - // public-only list cannot establish that publication has no existing draft. + // GitHub includes drafts only for readers with push access. + onProgress("release lookup verifying draft visibility"); const repository = requirePreflightRecord(preflightApi(runGh, repo, ""), "repository"); if (!isRecord(repository.permissions) || repository.permissions.push !== true) { return { diff --git a/scripts/lib/release-publish-state.mts b/scripts/lib/release-publish-state.mts index ecbb4192a69b..b5ac3fc3ca90 100644 --- a/scripts/lib/release-publish-state.mts +++ b/scripts/lib/release-publish-state.mts @@ -20,7 +20,12 @@ import { import { isRecord } from "./record-shared.mjs"; import type { ReleasePublishGate } from "./release-publish-gates.mts"; import type { ReleaseNpmDecision } from "./release-publish-inputs.mjs"; -import { readPublishPreflightRelease } from "./release-publish-preflight-evidence.mts"; +import { + createPublishPreflightGh, + publishPreflightGhError, + readPublishPreflightRelease, + type PublishPreflightGh, +} from "./release-publish-preflight-evidence.mts"; import { collectReleaseVersionFloorErrors } from "./release-version.mjs"; export function readReleasePublicationPackages(input: { @@ -301,38 +306,50 @@ export function observeReleaseGitHubState(input: { releaseTag: string; sourceSha: string; npmDistTag: string; - runGh?: (args: string[]) => string; + runGh?: PublishPreflightGh; + budgetMs?: number; + now?: () => number; + onProgress?: (message: string) => void; }) { const gates: ReleasePublishGate[] = []; - const runGh = - input.runGh ?? - ((args: string[]) => - execFileSync("gh", args, { - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - timeout: 60_000, - maxBuffer: 8 * 1024 * 1024, - })); - const cache = new Map(); - const raw = (endpoint: string) => { - const cached = cache.get(endpoint); - if (cached !== undefined) { - return cached; + const now = input.now ?? Date.now; + const budgetMs = + input.budgetMs ?? Number(process.env.OPENCLAW_RELEASE_OBSERVATION_BUDGET_MS ?? 180_000); + if (!Number.isSafeInteger(budgetMs) || budgetMs < 1) { + throw new Error("GitHub observation budget must be a positive integer in milliseconds."); + } + const deadline = now() + budgetMs; + const progress = + input.onProgress ?? + ((message: string) => + process.stderr.write(`[release-publish-preflight] github.observation ${message}\n`)); + const execute = input.runGh ?? createPublishPreflightGh(); + let readCommand = ""; + const runGh: PublishPreflightGh = (args) => { + // Replay the exact read, including any --jq filter, so remediation stays cheap. + readCommand = ["gh", ...args] + .map((arg) => (/^[-\w]+$/u.test(arg) ? arg : `'${arg.replaceAll("'", "'\\''")}'`)) + .join(" "); + const remaining = deadline - now(); + if (remaining <= 0) { + throw new Error( + `GitHub observation budget exhausted (${budgetMs} ms); not inspected: ${readCommand}`, + ); } - const value = runGh([ + return execute(args, { timeoutMs: Math.min(60_000, remaining) }); + }; + const raw = (endpoint: string) => + runGh([ "api", `repos/${input.repository}/${endpoint}`, "--method", "GET", ...(endpoint.endsWith("/logs") ? ["--allow-escape-sequences"] : []), ]); - cache.set(endpoint, value); - return value; - }; const api = (endpoint: string): unknown => JSON.parse(raw(endpoint)); let release: Record | undefined; try { - const lookup = readPublishPreflightRelease(runGh, input.repository, input.releaseTag); + const lookup = readPublishPreflightRelease(runGh, input.repository, input.releaseTag, progress); release = lookup.state === "found" ? lookup.release : undefined; gates.push({ id: "github.release", @@ -349,12 +366,12 @@ export function observeReleaseGitHubState(input: { ? "Inspect the release with credentials that can view drafts before dispatch." : "", }); - } catch { + } catch (error) { gates.push({ id: "github.release", status: "WARN", - message: "GitHub release state could not be read.", - remediation: `Inspect authenticated release visibility and exact tag ${input.releaseTag}: gh api 'repos/${input.repository}/releases?per_page=100&page=1' --method GET`, + message: `GitHub release state could not be read: ${publishPreflightGhError(error)}`, + remediation: `Inspect authenticated release visibility and exact tag ${input.releaseTag}: ${readCommand}`, }); } for (const workflow of [ @@ -364,7 +381,8 @@ export function observeReleaseGitHubState(input: { "plugin-clawhub-new.yml", ]) { const runs = new Map(); - let complete = true; + const unresolved: string[] = []; + const unreadCommands: string[] = []; for (const status of [ "in_progress", "queued", @@ -374,6 +392,7 @@ export function observeReleaseGitHubState(input: { "action_required", ]) { const endpoint = `actions/workflows/${workflow}/runs?status=${status}&per_page=100`; + progress(`workflow ${workflow} inventory ${status} (${runs.size} active runs found)`); try { const response = api(endpoint); if ( @@ -398,19 +417,19 @@ export function observeReleaseGitHubState(input: { runs.set(run.id, run as Run); } if (response.total_count > 100) { - complete = false; + throw new Error("Active-run inventory exceeds the bounded first page."); } - } catch { - complete = false; + } catch (error) { + unresolved.push(`${status}: ${publishPreflightGhError(error)}`); + unreadCommands.push(`gh api 'repos/${input.repository}/${endpoint}' --method GET`); } } - if (!complete) { + if (unresolved.length) { gates.push({ id: `concurrency.${workflow}.inventory`, status: "WARN", - message: - "Active-run inventory is unavailable or exceeds the bounded first page; concurrency is unresolved.", - remediation: `gh api 'repos/${input.repository}/actions/workflows/${workflow}/runs?status=waiting&per_page=100' --method GET`, + message: `Active-run inventory unresolved: ${unresolved.join("; ")}`, + remediation: unreadCommands.join("\n"), }); } let candidates = 0; @@ -432,6 +451,9 @@ export function observeReleaseGitHubState(input: { let evidence = match ? "exact workflow run title" : "dispatch inputs unavailable"; let parentRunId: string | undefined; if (!match && candidates++ < 10) { + progress( + `workflow ${workflow}: inspecting candidate run ${run.id} ${candidates}/${runs.size} via job log (attempt ${run.run_attempt})`, + ); try { const jobs = api(`actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100`); const job = @@ -465,19 +487,27 @@ export function observeReleaseGitHubState(input: { : "dispatch inputs incomplete in available job log"; parentRunId = env("RELEASE_PUBLISH_RUN_ID"); } - } catch { - evidence = "dispatch inputs unavailable from completed job logs"; + } catch (error) { + evidence = `dispatch inputs unavailable: ${publishPreflightGhError(error)}; read: ${readCommand}`; } + } else if (!match) { + evidence = "candidate inspection limit (10) reached; dispatch inputs not inspected"; } let orphan = ""; if (match && parentRunId && /^[1-9][0-9]*$/u.test(parentRunId)) { try { + progress(`candidate run ${run.id}: inspecting parent ${parentRunId}`); const producer = api(`actions/runs/${parentRunId}`); if (isRecord(producer) && producer.status === "completed") { orphan = ` Parent ${parentRunId} is terminal (${String(producer.conclusion)}); this may be a detached child or orphan.`; } - } catch { - /* An unavailable parent cannot establish an orphan. */ + } catch (error) { + gates.push({ + id: `concurrency.${workflow}.${run.id}.parent`, + status: "WARN", + message: `Parent ${parentRunId} ownership unresolved: ${publishPreflightGhError(error)}`, + remediation: readCommand, + }); } } gates.push({ @@ -489,7 +519,10 @@ export function observeReleaseGitHubState(input: { : `Resolve the run's exact ${parent ? "npm_dist_tag" : "ref and dry_run"} inputs before dispatch: gh api 'repos/${input.repository}/actions/runs/${run.id}/jobs?per_page=100' --method GET`, }); } - if (complete && !gates.some((gate) => gate.id.startsWith(`concurrency.${workflow}.`))) { + if ( + !unresolved.length && + !gates.some((gate) => gate.id.startsWith(`concurrency.${workflow}.`)) + ) { gates.push({ id: `concurrency.${workflow}.clear`, status: "PASS", diff --git a/scripts/plugin-sdk-api-release-evidence.mjs b/scripts/plugin-sdk-api-release-evidence.mjs index a87e213676cf..ce262d200865 100644 --- a/scripts/plugin-sdk-api-release-evidence.mjs +++ b/scripts/plugin-sdk-api-release-evidence.mjs @@ -204,6 +204,13 @@ export function createPluginSdkApiReleaseEvidenceSet(selectors) { }; } +export class PluginSdkApiAcknowledgementError extends Error { + constructor(digest) { + super(`Plugin SDK API changes require acknowledgement digest ${digest}`); + this.digest = digest; + } +} + export function validatePluginSdkApiReleaseEvidence({ acknowledgement, currentSelectorRef = "", @@ -274,9 +281,7 @@ export function validatePluginSdkApiReleaseEvidence({ } const expectedAcknowledgement = digest.slice(0, 8); if (changed && acknowledgement !== expectedAcknowledgement) { - throw new Error( - `Plugin SDK API changes require acknowledgement digest ${expectedAcknowledgement}`, - ); + throw new PluginSdkApiAcknowledgementError(expectedAcknowledgement); } return { acknowledgement: changed ? expectedAcknowledgement : null, diff --git a/scripts/release-candidate-checklist.mts b/scripts/release-candidate-checklist.mts index 7fa7d99b10a3..7326ffe817c7 100644 --- a/scripts/release-candidate-checklist.mts +++ b/scripts/release-candidate-checklist.mts @@ -50,7 +50,10 @@ import { validateReleasePreflightTagIdentity, } from "./npm-preflight-tooling-identity.mjs"; import { validateNpmPreflightDistTag } from "./openclaw-npm-extended-stable-release.mjs"; -import { validatePluginSdkApiReleaseEvidence } from "./plugin-sdk-api-release-evidence.mjs"; +import { + PluginSdkApiAcknowledgementError, + validatePluginSdkApiReleaseEvidence, +} from "./plugin-sdk-api-release-evidence.mjs"; import { runReleasePublishPreflight } from "./release-publish-preflight.mts"; import { runReleaseToolingGh, verifyReleaseToolingIdentity } from "./release-tooling-identity.mjs"; import { @@ -177,7 +180,7 @@ Options: 8-character digest from the Plugin SDK API diff report. --windows-node-tag Optional exact Windows Node tag for postpublish asset promotion. --skip-dispatch Require Full Release Validation run; separate npm run only for historical recovery. - --skip-local-generated-check Do not run local generated release baseline checks before dispatch. + --skip-local-generated-check Do not run local generated release baseline checks. --run-parallels Force candidate Parallels smoke; beta defaults to postpublish release:beta-smoke. --skip-parallels Force-skip candidate Parallels smoke; stable/full run by default. --parallels-registry-package-artifact @@ -776,6 +779,14 @@ function runFromTrustedTooling( ); } } finally { + // Remove the installed graph before Git walks the checkout to remove it. + try { + rmSync(tempRoot, { force: true, recursive: true }); + } catch (error) { + console.warn( + `could not remove temporary trusted tooling files at ${tempRoot}: ${String(error)}`, + ); + } if (worktreeAdded) { const cleanup = spawnSync("git", ["worktree", "remove", "--force", toolingRoot], { cwd: targetRoot, @@ -788,7 +799,6 @@ function runFromTrustedTooling( ); } } - rmSync(tempRoot, { force: true, recursive: true }); } } @@ -2128,9 +2138,13 @@ async function main() { ), ) : ""; - const localGeneratedCheck = runLocalGeneratedCheckIfNeeded(options); - - if (!options.fullReleaseRunId && !options.skipDispatch) { + // A new dispatch is gated by the local check; consuming existing evidence + // defers it until the SDK acknowledgement has been checked. + const dispatchesValidation = !options.fullReleaseRunId && !options.skipDispatch; + let localGeneratedCheck = dispatchesValidation + ? runLocalGeneratedCheckIfNeeded(options) + : undefined; + if (dispatchesValidation) { const workflowFile = "full-release-validation.yml"; const targetContextRef = releaseBranchForTag(options.tag); const trustedWorkflowFields = fullReleaseTrustedWorkflowFields({ @@ -2225,15 +2239,6 @@ async function main() { if (fullValidationEvidence.source === "direct" && fullRun.headSha !== targetSha) { throw new Error(`run SHA mismatch: tag=${targetSha} full=${fullRun.headSha}`); } - // Only exact historical producers retain local, non-authoritative planning. - // B recovery consumes its original hosted observations without another sweep. - const publicationAdmission = fullValidationEvidence.publicationAdmission; - const pluginNpmPlan = publicationAdmission - ? publicationAdmission.observations.plans.npm - : await collectPluginPlanWithRetry("scripts/plugin-npm-release-plan.ts", options); - const pluginClawHubPlan = publicationAdmission - ? publicationAdmission.observations.plans.clawhub - : await collectPluginPlanWithRetry("scripts/plugin-clawhub-release-plan.ts", options); if (npmUsesFullRun) { rmSync(npmDir, { recursive: true, force: true }); } @@ -2309,13 +2314,47 @@ async function main() { targetSha, npmDistTag: options.npmDistTag, }); - const pluginSdkApiValidation = validatePluginSdkApiReleaseEvidence({ - acknowledgement: options.pluginSdkApiAcknowledgement, - evidence: npmManifest.pluginSdkApi, - expectedHeadSha: targetSha, - expectedWorkflowSha: npmRun.headSha, - npmDistTag: options.npmDistTag, - }); + let pluginSdkApiValidation; + try { + pluginSdkApiValidation = validatePluginSdkApiReleaseEvidence({ + acknowledgement: options.pluginSdkApiAcknowledgement, + evidence: npmManifest.pluginSdkApi, + expectedHeadSha: targetSha, + expectedWorkflowSha: npmRun.headSha, + npmDistTag: options.npmDistTag, + }); + } catch (error) { + if (error instanceof PluginSdkApiAcknowledgementError) { + const argv = stripLeadingPackageManagerSeparator(process.argv.slice(2)); + const end = argv.indexOf("--"); + const flag = "--plugin-sdk-api-acknowledgement"; + let replaced = false; + for (let index = 0; index < (end === -1 ? argv.length : end); index++) { + if (argv[index] === flag) { + argv[++index] = error.digest; + replaced = true; + } + } + if (!replaced) { + argv.unshift(flag, error.digest); + } + throw new Error( + `${error.message}\nReview the Plugin SDK API diff before rerunning.\n${["pnpm", "release:candidate", "--", ...argv].map(shellQuote).join(" ")}`, + { cause: error }, + ); + } + throw error; + } + localGeneratedCheck ??= runLocalGeneratedCheckIfNeeded(options); + // Only exact historical producers retain local, non-authoritative planning. + // B recovery consumes its original hosted observations without another sweep. + const publicationAdmission = fullValidationEvidence.publicationAdmission; + const pluginNpmPlan = publicationAdmission + ? publicationAdmission.observations.plans.npm + : await collectPluginPlanWithRetry("scripts/plugin-npm-release-plan.ts", options); + const pluginClawHubPlan = publicationAdmission + ? publicationAdmission.observations.plans.clawhub + : await collectPluginPlanWithRetry("scripts/plugin-clawhub-release-plan.ts", options); validateFullManifest(fullManifest, { targetSha, releaseProfile: options.releaseProfile, diff --git a/test/scripts/frv.test.ts b/test/scripts/frv.test.ts index 4561361ea96e..3e05b67f7c32 100644 --- a/test/scripts/frv.test.ts +++ b/test/scripts/frv.test.ts @@ -906,6 +906,81 @@ describe("FRV same-parent recovery", () => { expect(parentReruns).toBe(1); }); + it("reports waiting transitions, bounded heartbeats, and exact started attempts on stderr", async () => { + vi.useFakeTimers(); + const started = Date.now(); + const selected = child("normalCi", "101"); + const childRuns = new Map([ + ["101", { attempt: 1, conclusion: null }], + ]); + const parent = { attempt: 1, conclusion: null as string | null }; + const base = controllerClient([selected], childRuns, parent); + const stderr = vi.spyOn(console, "error").mockImplementation(() => undefined); + const stdout = vi.spyOn(console, "log").mockImplementation(() => undefined); + const client = { + ...base, + getRun: async (runId: string) => { + const elapsed = Date.now() - started; + const current = childRuns.get("101")!; + current.conclusion = + elapsed >= 390_000 + ? "success" + : elapsed >= 360_000 && current.attempt === 1 + ? "failure" + : null; + parent.conclusion = + elapsed >= 450_000 + ? "success" + : elapsed >= 420_000 && parent.attempt === 1 + ? "failure" + : null; + const run = await base.getRun(runId); + return runId === "101" && elapsed >= 30_000 && elapsed < 360_000 + ? { ...run, status: "queued" } + : run; + }, + rerunFailed: vi.fn(async () => { + childRuns.get("101")!.attempt = 2; + }), + rerunParent: vi.fn(async () => { + parent.attempt = 2; + }), + verify: vi.fn(async () => "{}"), + }; + try { + const result = continueFailed(plan([selected]), "77", client); + await Promise.all([result, vi.advanceTimersByTimeAsync(480_000)]); + const lines = stderr.mock.calls.map(([line]) => String(line)); + expect( + lines.filter((line) => line.includes("waiting for normalCi run 101 attempt 1 in_progress")), + ).toHaveLength(1); + expect( + lines.filter((line) => line.includes("waiting for normalCi run 101 attempt 1 queued")), + ).toHaveLength(2); + expect(lines).toEqual( + expect.arrayContaining([ + expect.stringContaining("normalCi run 101 attempt 1 completed failure"), + expect.stringContaining( + "normalCi run 101 attempt 2 started https://github.com/openclaw/openclaw/actions/runs/101/attempts/2", + ), + expect.stringContaining("waiting for normalCi run 101 attempt 2 in_progress"), + expect.stringContaining("waiting for parent 77 attempt 1 in_progress"), + expect.stringContaining( + "parent 77 attempt 2 started https://github.com/openclaw/openclaw/actions/runs/77/attempts/2", + ), + expect.stringContaining("parent 77 attempt 2 completed success"), + ]), + ); + expect(lines.filter((line) => line.includes(" started "))).toHaveLength(2); + expect(client.rerunFailed).toHaveBeenCalledExactlyOnceWith("101"); + expect(client.rerunParent).toHaveBeenCalledExactlyOnceWith("77"); + expect(stdout).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + vi.restoreAllMocks(); + } + }); + it("retries each terminal child while the parent and other child attempts are still active", async () => { const first = child("normalCi", "101"); const second = child("pluginPrerelease", "202"); diff --git a/test/scripts/release-candidate-checklist.test.ts b/test/scripts/release-candidate-checklist.test.ts index 4401bc7bafc4..8fac9c51c1d2 100644 --- a/test/scripts/release-candidate-checklist.test.ts +++ b/test/scripts/release-candidate-checklist.test.ts @@ -27,6 +27,7 @@ import { publicationIntentInputs, publicationSourceContract, } from "../../scripts/full-release-publication-contract.mjs"; +import { stripLeadingPackageManagerSeparator } from "../../scripts/lib/arg-utils.mts"; import { parsePluginReleaseSelection } from "../../scripts/lib/plugin-npm-release.ts"; import { splitChangelog } from "../../scripts/lib/release-changelog.mjs"; import { releaseBranchForTag } from "../../scripts/lib/release-context.mjs"; @@ -37,6 +38,11 @@ import { } from "../../scripts/lib/release-publish-preflight-interface.mts"; import { classifyReleaseTrain, parseReleaseVersion } from "../../scripts/lib/release-version.mjs"; import { validateReleaseButtonInputs } from "../../scripts/openclaw-release-ready.mjs"; +import { + PluginSdkApiAcknowledgementError, + createPluginSdkApiReleaseEvidence, + validatePluginSdkApiReleaseEvidence, +} from "../../scripts/plugin-sdk-api-release-evidence.mjs"; import { buildReleaseCandidateState, buildPublishCommand, @@ -240,8 +246,15 @@ describe("release candidate checklist", () => { preflightFailure?: boolean; workflowSha?: string; savedToolingTag?: string; + sdkAcknowledgement?: string; }>([ { tag: "v2026.9.1", pin: "2026.9.1", expected: "passed", failedRegistry: "" }, + ...["", "deadbeef", "f4b495f3"].map((sdkAcknowledgement) => ({ + tag: "v2026.9.1", + pin: "2026.9.1", + expected: "passed", + sdkAcknowledgement, + })), { tag: "v2026.9.1", pin: "2026.9.1", @@ -338,7 +351,7 @@ describe("release candidate checklist", () => { preflightFailure: true, })), ])( - "consumes producer-qualified registry plans ($failedRegistry; $registryAdmission) and records Android evidence for $tag ($pin; $launch; $distTag; $publicationRoute; workflow SHA=$workflowSha; preflight failure=$preflightFailure)", + "consumes producer-qualified registry plans ($failedRegistry; $registryAdmission) and records Android evidence for $tag ($pin; $launch; $distTag; $publicationRoute; workflow SHA=$workflowSha; preflight failure=$preflightFailure; SDK=$sdkAcknowledgement)", async ({ tag, pin, @@ -353,6 +366,7 @@ describe("release candidate checklist", () => { preflightFailure = false, workflowSha, savedToolingTag, + sdkAcknowledgement, }) => { const { root: targetRoot, git } = candidateGitFixture({ "package.json": JSON.stringify({ version: tag.slice(1) }), @@ -365,7 +379,7 @@ describe("release candidate checklist", () => { join(targetRoot, "apps/android/version.json"), JSON.stringify({ version: "2099.1.1" }), ); - const options = parseArgs([ + const candidateArgv = [ "--tag", tag, "--publication-route", @@ -382,7 +396,11 @@ describe("release candidate checklist", () => { ...(workflowSha ? ["--workflow-sha", workflowSha] : ["--publish-workflow-ref", publishWorkflowRef]), - ]); + ...(sdkAcknowledgement ? ["--plugin-sdk-api-acknowledgement", sdkAcknowledgement] : []), + "--output-dir", + `${targetRoot}/evidence 'quoted' $(literal)`, + ]; + const options = parseArgs(candidateArgv); if (failedRegistry) { options.fullReleaseRunId = ""; options.skipDispatch = false; @@ -398,6 +416,7 @@ describe("release candidate checklist", () => { source.match(/^function publicationSelectionForChecklist\([\s\S]*?^\}/mu)?.[0] ?? ""; const savedTagReader = source.match(/^function savedPublishWorkflowRef\([\s\S]*?^\}/mu)?.[0] ?? ""; + const shellQuote = source.match(/^function shellQuote\([\s\S]*?^\}/mu)?.[0] ?? ""; const log = vi.fn(); const stages: string[] = []; const writeState = vi.fn<(path: string, state: unknown) => void>( @@ -435,12 +454,27 @@ describe("release candidate checklist", () => { if (savedState) { writeFileSync(statePath, JSON.stringify(savedState)); } + const sdkPayload = { + entrypointsAdded: [], + entrypointsRemoved: [], + exports: sdkAcknowledgement === undefined ? [] : [{ change: "added", exportName: "send" }], + }; + const sdkEvidence = createPluginSdkApiReleaseEvidence({ + baseRef: "v2026.8.1", + baseSha: "a".repeat(40), + headSha: targetSha, + workflowSha: targetSha, + diff: { + ...sdkPayload, + digest: createHash("sha256").update(JSON.stringify(sdkPayload)).digest("hex"), + }, + }); const npmManifest = { tarballName: "openclaw.tgz", tarballSha256: "fixture-digest", corePackageTarballs: [], dependencyTarballs: [], - pluginSdkApi: {}, + pluginSdkApi: sdkEvidence, }; const fullManifest = { workflowName: "Full Release Validation", @@ -503,10 +537,14 @@ describe("release candidate checklist", () => { const dispatches: Record[] = []; const completion = runInNewContext( stripNodeTypeScriptTypes( - `${android}\n${selectPublication}\n${savedTagReader}\n${main}\nmain();`, + `${android}\n${selectPublication}\n${savedTagReader}\n${shellQuote}\n${main}\nmain();`, ), { - process: { argv: [], cwd: () => targetRoot, env: {} }, + process: { + argv: ["node", "scripts/release-candidate-checklist.mts", ...candidateArgv], + cwd: () => targetRoot, + env: {}, + }, console: { log, warn: log }, TOOLING_ROOT: "/trusted/tooling", PUBLISH_TOOLING_TAG_PATTERN: /^release-publish\/[a-f0-9]{12}-[1-9][0-9]*$/u, @@ -577,7 +615,7 @@ describe("release candidate checklist", () => { ? npmManifest : file.endsWith("full-release-validation-manifest.json") ? fullManifest - : {}, + : sdkEvidence, authenticateFullReleaseValidationEvidence: async () => { stages.push("authenticate"); if (registryAdmission && failedRegistry) { @@ -590,7 +628,9 @@ describe("release candidate checklist", () => { isDeepStrictEqual, sha256: () => "fixture-digest", validatePreflightManifest: () => {}, - validatePluginSdkApiReleaseEvidence: () => ({ status: "passed" }), + validatePluginSdkApiReleaseEvidence, + PluginSdkApiAcknowledgementError, + stripLeadingPackageManagerSeparator, validateFullManifest: () => stages.push("evidence-validated"), preflightCorePackageTarballs, preflightDependencyTarballs, @@ -622,6 +662,31 @@ describe("release candidate checklist", () => { writeFileSync, }, ); + if (sdkAcknowledgement !== undefined && sdkAcknowledgement !== "f4b495f3") { + const error = await completion.catch((cause: Error) => cause); + expect(error.message).toContain( + "Plugin SDK API changes require acknowledgement digest f4b495f3", + ); + expect(error.message).toContain("Review the Plugin SDK API diff before rerunning."); + const command = error.message.split("\n").at(-1); + const decoded = execFileSync( + "bash", + ["-c", `pnpm() { printf '%s\\0' "$@"; }\n${command}`], + { encoding: "utf8", timeout: 10_000 }, + ) + .split("\0") + .filter(Boolean); + expect(decoded.slice(0, 2)).toEqual(["release:candidate", "--"]); + expect(parseArgs(decoded.slice(2))).toEqual({ + ...parseArgs(candidateArgv), + pluginSdkApiAcknowledgement: "f4b495f3", + }); + expect(stages).toEqual(["wait", "wait", "authenticate"]); + expect(generatedChecks).not.toHaveBeenCalled(); + expect(preflight).not.toHaveBeenCalled(); + expect(publishCommand).not.toHaveBeenCalled(); + return; + } if (workflowSha && workflowSha !== toolingSha) { await expect(completion).rejects.toThrow( `--workflow-sha ${workflowSha} does not match tooling checkout ${toolingSha}`, @@ -885,6 +950,9 @@ describe("release candidate checklist", () => { const owner = source.match(/^function runFromTrustedTooling\([\s\S]*?^\}/mu)?.[0]; const jsonReader = source.match(/^function readJson\([\s\S]*?^\}/mu)?.[0]; const ancestry = source.match(/^function gitIsAncestor\([\s\S]*?^\}/mu)?.[0]; + const cleanupWarning = vi.fn(); + let toolingRoot = ""; + let toolingAdminDir = ""; const installs = vi.fn( (_command: string, args: string[], options: Parameters[2]) => { expect(args).toEqual([ @@ -894,6 +962,11 @@ describe("release candidate checklist", () => { "--prefer-offline", ]); const root = options?.cwd ?? ""; + toolingRoot = root; + toolingAdminDir = run("git", ["rev-parse", "--absolute-git-dir"], { + cwd: root, + capture: true, + }).trim(); expect(root).not.toBe(targetRoot); expect(existsSync(join(root, "node_modules"))).toBe(false); expect(run("git", ["rev-parse", "HEAD"], { cwd: root, capture: true }).trim()).toBe( @@ -914,7 +987,6 @@ describe("release candidate checklist", () => { return ""; }, ); - let toolingRoot = ""; let childOutput = ""; const execute = () => runInNewContext( @@ -926,7 +998,12 @@ describe("release candidate checklist", () => { mkdirSync, mkdtempSync, readFileSync, - rmSync, + rmSync: (...args: Parameters) => { + rmSync(...args); + if (scenario === "child failure") { + throw new Error("fixture cleanup failed"); + } + }, symlinkSync, createRequire, pathToFileURL, @@ -934,7 +1011,7 @@ describe("release candidate checklist", () => { join, isRecord, process, - console, + console: { ...console, warn: cleanupWarning }, targetRoot, workflowSha, argv: [scenario === "child failure" ? "--fail" : "--help"], @@ -976,13 +1053,17 @@ describe("release candidate checklist", () => { expect(execute).toThrow("fixture install failed"); } else if (scenario === "child failure") { expect(execute).toThrow("trusted release candidate tooling failed with 7"); + expect(cleanupWarning).toHaveBeenCalledWith( + expect.stringContaining("fixture cleanup failed"), + ); } else { execute(); expect(JSON.parse(childOutput)).toEqual({ parsed: { ready: true }, cwd: targetRoot }); } expect(installs).toHaveBeenCalledTimes(1); if (toolingRoot) { - expect(existsSync(toolingRoot)).toBe(false); + expect(existsSync(dirname(toolingRoot))).toBe(false); + expect(existsSync(toolingAdminDir)).toBe(false); } expect(git("worktree", "list", "--porcelain").match(/^worktree /gmu)).toHaveLength(1); expect(existsSync(join(installedModules, "yaml"))).toBe(true); diff --git a/test/scripts/release-publish-closeout-preflight.test.ts b/test/scripts/release-publish-closeout-preflight.test.ts index 046697e4b9af..7857a0c429ad 100644 --- a/test/scripts/release-publish-closeout-preflight.test.ts +++ b/test/scripts/release-publish-closeout-preflight.test.ts @@ -154,7 +154,9 @@ console.log(JSON.stringify({admission, observation, closeout})); ); } else if (state !== "absent") { expect(result.observation.gates).toContainEqual( - expect.objectContaining({ message: "GitHub release state could not be read." }), + expect.objectContaining({ + message: expect.stringContaining("GitHub release state could not be read:"), + }), ); } }); diff --git a/test/scripts/release-publish-preflight-evidence.test.ts b/test/scripts/release-publish-preflight-evidence.test.ts index 69f4aed466b3..7fd1c2c3fe3b 100644 --- a/test/scripts/release-publish-preflight-evidence.test.ts +++ b/test/scripts/release-publish-preflight-evidence.test.ts @@ -142,7 +142,10 @@ describe("publish preflight release inventory", () => { "refuses absence from an %s inventory", (state) => { let reads = 0; - const runGh = () => { + const runGh = (args: string[]) => { + if (args[1]?.includes("/releases/tags/")) { + throw new Error("HTTP 404: Not Found"); + } reads++; if (state === "malformed") { return JSON.stringify([{ draft: true }]); @@ -178,13 +181,30 @@ describe("publish preflight release inventory", () => { body: "Published release notes", assets: [{ name: "dependency-evidence.zip" }], }; - expect( - readPublishPreflightRelease( - () => JSON.stringify([release]), - "openclaw/openclaw", - `v${version}`, - ), - ).toEqual({ state: "found", release }); + // The exact-tag endpoint serves published releases; drafts need the inventory. + const runGh = (args: string[]) => { + if (args[1]?.includes("/releases/tags/")) { + if (draft) { + throw new Error("HTTP 404: Not Found"); + } + return JSON.stringify(release); + } + return JSON.stringify([release]); + }; + expect(readPublishPreflightRelease(runGh, "openclaw/openclaw", `v${version}`)).toEqual({ + state: "found", + release, + }); + }); + + it("does not mask a failed exact-tag read as absence", () => { + const runGh = vi.fn(() => { + throw new Error("HTTP 502: Bad Gateway"); + }); + expect(() => readPublishPreflightRelease(runGh, "openclaw/openclaw", `v${version}`)).toThrow( + "HTTP 502", + ); + expect(runGh).toHaveBeenCalledTimes(1); }); }); diff --git a/test/scripts/release-publish-state.test.ts b/test/scripts/release-publish-state.test.ts index af8dbf329f30..02dcf308e838 100644 --- a/test/scripts/release-publish-state.test.ts +++ b/test/scripts/release-publish-state.test.ts @@ -253,17 +253,27 @@ describe("release publication state", () => { ); }); -function observeRuns(params: { workflow: string; title?: string; log?: string; count?: number }) { +function observeRuns(params: { + workflow: string; + title?: string; + log?: string; + count?: number; + onProgress?: (message: string) => void; +}) { return observeReleaseGitHubState({ repository: "openclaw/openclaw", releaseTag: `v${version}`, sourceSha, npmDistTag: "latest", + onProgress: params.onProgress, runGh: (args) => { const endpoint = args[1]; if (endpoint === undefined) { throw new Error("Expected a GitHub REST endpoint."); } + if (endpoint.includes("/releases/tags/")) { + throw new Error("HTTP 404: Not Found"); + } if (endpoint.includes("/releases?")) { return "[]"; } @@ -304,6 +314,138 @@ function observeRuns(params: { workflow: string; title?: string; log?: string; c } describe("release concurrency observations", () => { + it("bounds slow GitHub reads and names every uninspected inventory with its exact command", () => { + let now = 0; + const runGh = vi.fn((args: string[], options?: { timeoutMs?: number }) => { + if (args[1]?.includes("/releases/tags/")) { + throw new Error("HTTP 404: Not Found"); + } + expect(options?.timeoutMs).toBe(100); + now += 100; + return "[]"; + }); + const result = observeReleaseGitHubState({ + repository: "openclaw/openclaw", + releaseTag: `v${version}`, + sourceSha, + npmDistTag: "latest", + budgetMs: 100, + now: () => now, + runGh, + }); + expect(result.gates.every((gate) => gate.status === "WARN")).toBe(true); + expect(result.gates[0]?.message).toContain("observation budget exhausted (100 ms)"); + for (const workflow of [ + "openclaw-release-publish.yml", + "plugin-npm-release.yml", + "plugin-clawhub-release.yml", + "plugin-clawhub-new.yml", + ]) { + expect(result.gates).toContainEqual( + expect.objectContaining({ + id: `concurrency.${workflow}.inventory`, + status: "WARN", + message: expect.stringContaining("observation budget exhausted"), + remediation: expect.stringContaining( + `gh api 'repos/openclaw/openclaw/actions/workflows/${workflow}/runs?status=in_progress&per_page=100' --method GET`, + ), + }), + ); + } + expect(runGh).toHaveBeenCalledTimes(2); + }); + + it("retains a gh stderr failure and exact unread command without hiding known blockers", () => { + const result = observeReleaseGitHubState({ + repository: "openclaw/openclaw", + releaseTag: `v${version}`, + sourceSha, + npmDistTag: "latest", + runGh(args) { + if (args[1]?.includes("/plugin-npm-release.yml/") && args[1].includes("status=queued")) { + return JSON.stringify({ + total_count: 1, + workflow_runs: [ + { + id: 321, + run_attempt: 1, + status: "queued", + event: "workflow_dispatch", + display_title: `Plugin NPM Release [default] ${sourceSha}`, + html_url: "https://github.com/openclaw/openclaw/actions/runs/321", + }, + ], + }); + } + throw Object.assign(new Error("spawnSync gh ETIMEDOUT"), { + stderr: Buffer.from("HTTP 503: upstream unavailable"), + }); + }, + }); + expect(result.gates).toContainEqual( + expect.objectContaining({ + id: "concurrency.plugin-npm-release.yml.321", + status: "FAIL", + }), + ); + expect(result.gates).toContainEqual( + expect.objectContaining({ + id: "concurrency.plugin-npm-release.yml.inventory", + status: "WARN", + message: expect.stringContaining("HTTP 503: upstream unavailable"), + remediation: expect.stringContaining("status=in_progress"), + }), + ); + }); + + it("reads a published release by exact tag without transferring the full release inventory", () => { + const release = { + id: 7, + draft: false, + prerelease: false, + tag_name: `v${version}`, + html_url: `https://github.com/openclaw/openclaw/releases/tag/v${version}`, + target_commitish: sourceSha, + }; + const runGh = vi.fn((args: string[]) => + args[1]?.includes("/releases/tags/") + ? JSON.stringify(release) + : JSON.stringify({ total_count: 0, workflow_runs: [] }), + ); + const result = observeReleaseGitHubState({ + repository: "openclaw/openclaw", + releaseTag: `v${version}`, + sourceSha, + npmDistTag: "latest", + runGh, + }); + expect(result.release).toMatchObject(release); + expect(runGh.mock.calls.some(([args]) => args[1]?.includes("/releases?"))).toBe(false); + }); + + it("replays a failed filtered release-inventory read with its exact --jq filter", () => { + const result = observeReleaseGitHubState({ + repository: "openclaw/openclaw", + releaseTag: `v${version}`, + sourceSha, + npmDistTag: "latest", + runGh(args) { + if (args[1]?.includes("/releases/tags/")) { + throw new Error("HTTP 404: Not Found"); + } + if (args[1]?.includes("/releases?")) { + throw new Error("HTTP 502: Bad Gateway"); + } + return JSON.stringify({ total_count: 0, workflow_runs: [] }); + }, + }); + const gate = result.gates.find((entry) => entry.id === "github.release"); + expect(gate).toMatchObject({ status: "WARN", message: expect.stringContaining("HTTP 502") }); + expect(gate?.remediation).toContain( + `gh api 'repos/openclaw/openclaw/releases?per_page=100&page=1' --method GET --jq 'map(if .tag_name == "v${version}" then . else {tag_name} end)'`, + ); + }); + it("recognizes the npm target from the exact preflight title because it shares the publish group", () => { const gates = observeRuns({ workflow: "plugin-npm-release.yml", @@ -320,7 +462,15 @@ describe("release concurrency observations", () => { "2026-09-18T01:02:03Z DRY_RUN: false", "2026-09-18T01:02:03Z RELEASE_PUBLISH_RUN_ID: 789", ].join("\n"); - const gates = observeRuns({ workflow: "plugin-clawhub-release.yml", log }); + const progress: string[] = []; + const gates = observeRuns({ + workflow: "plugin-clawhub-release.yml", + log, + onProgress: (message) => progress.push(message), + }); + expect(progress.join("\n")).toContain("release lookup page 1"); + expect(progress.join("\n")).toContain("workflow plugin-clawhub-release.yml inventory"); + expect(progress.join("\n")).toContain("inspecting candidate run 123 1/1 via job log"); expect(gates).toMatchObject([ { status: "FAIL", message: expect.stringContaining("Parent 789 is terminal (failure)") }, ]);