fix(ci): skip unusable published survivor baselines (#151694)

* fix(ci): skip unusable published survivor baselines

Check published baseline startup before targeted survivor matrix fanout, preserving skipped scenarios and captured errors in release evidence. Probe full CLI setup as version and config reads can bypass missing runtime dependencies. Related: #151657.

* fix(ci): use pinned Node setup for survivor probes

.github/workflows/openclaw-live-and-e2e-checks-reusable.yml

@@ -1770,7 +1770,7 @@ jobs:
     if: (!inputs.prepare_only) && inputs.docker_lanes != ''
     continue-on-error: ${{ inputs.advisory }}
     runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }}
-    timeout-minutes: 5
+    timeout-minutes: 90
     outputs:
       groups_json: ${{ steps.groups.outputs.groups_json }}
     steps:
@@ -1782,6 +1782,12 @@ jobs:
           ref: ${{ needs.validate_selected_ref.outputs.workflow_sha }}
           fetch-depth: 1
 
+      - name: Set up baseline probe Node runtime
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          package-manager-cache: false
+
       - name: Build targeted Docker lane groups
         id: groups
         shell: bash
@@ -1794,12 +1800,25 @@ jobs:
           OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS: ${{ inputs.published_upgrade_survivor_scenarios }}
         run: |
           set -euo pipefail
-          groups_json="$(node scripts/plan-targeted-docker-lane-groups.mjs)"
+          echo "artifact_name=upgrade-baseline-checks-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(node -p 'crypto.randomUUID()')" >> "$GITHUB_OUTPUT"
+          groups_json="$(node scripts/plan-targeted-docker-lane-groups.mjs --check-baselines .artifacts/upgrade-baseline-checks)"
           echo "groups_json=${groups_json}" >> "$GITHUB_OUTPUT"
 
+      - name: Summarize published baseline checks
+        if: always()
+        run: cat .artifacts/upgrade-baseline-checks/summary.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload published baseline evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: ${{ steps.groups.outputs.artifact_name }}
+          path: .artifacts/upgrade-baseline-checks/
+          if-no-files-found: error
+
   validate_docker_lanes:
     needs: [validate_selected_ref, prepare_docker_e2e_image, plan_docker_lane_groups]
-    if: (!inputs.prepare_only) && inputs.docker_lanes != ''
+    if: (!inputs.prepare_only) && inputs.docker_lanes != '' && needs.plan_docker_lane_groups.outputs.groups_json != '[]'
     name: Docker E2E targeted lanes (${{ matrix.group.label }})
     continue-on-error: ${{ inputs.advisory }}
     runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }}

docs/ci/release-validation/package-acceptance.md

@@ -60,6 +60,8 @@ Docker seed CI resolves an exact published stable predecessor of the selected so
 
 The `published-upgrade-survivor` Docker lane validates one published package baseline per scenario. In Package Acceptance, the resolved `package-under-test` tarball is always the candidate and `published_upgrade_survivor_baseline` selects the fallback published baseline, defaulting to `openclaw@latest`; failed-lane rerun commands preserve that baseline. Current source release checks set `published_upgrade_survivor_baselines=supported-lines` for `legacy-operator-state`: npm's current `latest`, the preceding stable version, `extended-stable` when that tag exists, and the documented oldest supported baseline `2026.6.34`. The resolver reads `npm view openclaw versions` and `npm view openclaw dist-tags` at run time, pins exact versions before fanout, and deduplicates overlapping lines. Normal current-source release checks retain `base` and add `legacy-operator-state` and `custom-plugin-siblings`; release soak selects `reported-issues`, including these and the existing issue-shaped fixtures. The sibling-source scenario uses baselines from 2026.9.4 onward and requires actual custom-plugin Doctor contract execution from the private update canary, plus intact source files and plugin loading after the update.
 
+Before targeted Docker fanout, the trusted group planner installs each distinct published baseline in a throwaway npm prefix and checks `openclaw --version` plus `openclaw config set gateway.mode local` against synthetic isolated state. The config write loads the CLI setup path because version/help and config reads can use fast paths. An installed CLI that exits unsuccessfully is recorded as an **unusable published baseline**, with its skipped scenarios and captured error in the job summary and `upgrade-baseline-checks-*` artifact. Skipped scenarios are never counted as successful upgrades. Install errors, probe timeouts, and process-launch failures fail planning. Candidate installs and upgrades retain their existing failure gates.
+
 When a prerelease registry is mounted, the baseline package install and the
 initial manual baseline Gateway start use the configured published upstream.
 Published and candidate packages can share an exact version while containing

scripts/lib/upgrade-survivor-baseline-check.mjs

@@ -0,0 +1,148 @@
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import {
+  parseUpgradeSurvivorBaselineSpecs,
+  parseUpgradeSurvivorScenarios,
+} from "./upgrade-survivor-policy.mjs";
+
+const survivorLanes = new Set(["published-upgrade-survivor", "update-migration"]);
+
+function probeBaseline(baseline) {
+  const root = mkdtempSync(path.join(tmpdir(), "openclaw-baseline-check-"));
+  const prefix = path.join(root, "prefix");
+  const config = path.join(root, "openclaw.json");
+  const npmrc = path.join(root, "npmrc");
+  const checks = [];
+  try {
+    writeFileSync(config, JSON.stringify({ gateway: { mode: "local" } }));
+    writeFileSync(npmrc, "");
+    // Published bytes must never resolve through the candidate registry or the
+    // operator's npm/OpenClaw configuration. Nothing survives this probe's home.
+    const env = {
+      PATH: process.env.PATH,
+      HOME: root,
+      TMPDIR: root,
+      CI: "true",
+      npm_config_prefix: prefix,
+      npm_config_cache: path.join(root, "npm-cache"),
+      npm_config_userconfig: npmrc,
+      npm_config_globalconfig: path.join(root, "global-npmrc"),
+      npm_config_registry: "https://registry.npmjs.org",
+      OPENCLAW_STATE_DIR: path.join(root, "state"),
+      OPENCLAW_CONFIG_PATH: config,
+      OPENCLAW_NO_AUTO_UPDATE: "1",
+    };
+    const cli = path.join(prefix, "bin", "openclaw");
+    for (const [command, args, timeout] of [
+      ["npm", ["install", "-g", "--prefix", prefix, baseline, "--no-fund", "--no-audit"], 600_000],
+      [cli, ["--version"], 60_000],
+      // Version, help, and config reads can bypass full CLI startup. Apply a
+      // synthetic setting to exercise scenario setup without starting a Gateway.
+      [cli, ["config", "set", "gateway.mode", "local"], 60_000],
+    ]) {
+      const result = spawnSync(command, args, {
+        cwd: root,
+        env,
+        encoding: "utf8",
+        timeout,
+        killSignal: "SIGKILL",
+        maxBuffer: 1024 * 1024,
+      });
+      const label = command === "npm" ? "npm install" : `openclaw ${args.join(" ")}`;
+      const error = [result.error?.message, result.stderr, result.stdout]
+        .filter(Boolean)
+        .join("\n")
+        .slice(-16_384);
+      checks.push({
+        command: label,
+        exitCode: result.status,
+        signal: result.signal,
+        output: error,
+      });
+      if (result.status !== 0) {
+        const status = command === "npm" || result.error || result.signal ? "failed" : "skipped";
+        return {
+          status,
+          reason: `${status === "skipped" ? "unusable published baseline" : "baseline precheck failed"}: ${label} (exit ${result.status}, signal ${result.signal})`,
+          error,
+          checks,
+        };
+      }
+    }
+    return { status: "usable", checks };
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+
+export function checkUpgradeSurvivorBaselines(
+  groups,
+  { evidenceDir, baselines, baseline, scenarios },
+) {
+  const selections = new Map();
+  const groupBaselines = new Map();
+  for (const group of groups) {
+    const lanes = group.docker_lanes.split(/\s+/u).filter((lane) => survivorLanes.has(lane));
+    if (lanes.length === 0) {
+      continue;
+    }
+    const specs = parseUpgradeSurvivorBaselineSpecs(
+      group.published_upgrade_survivor_baselines || baselines || baseline || "openclaw@latest",
+    );
+    const requested = parseUpgradeSurvivorScenarios(
+      group.published_upgrade_survivor_scenarios || scenarios || "base",
+    );
+    groupBaselines.set(group, specs);
+    for (const spec of specs) {
+      const entry = selections.get(spec) ?? { baseline: spec, groups: [], scenarios: [] };
+      entry.groups.push(group.label);
+      entry.scenarios = [...new Set([...entry.scenarios, ...requested])];
+      selections.set(spec, entry);
+    }
+  }
+  mkdirSync(evidenceDir, { recursive: true });
+  const results = [...selections.values()].map((selection) => {
+    console.error(`Checking published upgrade baseline ${selection.baseline}`);
+    return Object.assign(selection, probeBaseline(selection.baseline));
+  });
+  writeFileSync(
+    path.join(evidenceDir, "summary.json"),
+    `${JSON.stringify({ baselines: results }, null, 2)}\n`,
+  );
+  const summary = [
+    "### Published upgrade baseline checks",
+    "",
+    "Usable means startup checked; upgrade scenarios still require their own results.",
+  ];
+  for (const result of results) {
+    summary.push(
+      "",
+      `- ${result.baseline}: **${result.status}**${result.reason ? ` — ${result.reason}` : ""}`,
+      `  Scenarios: ${result.scenarios.join(", ")}`,
+    );
+    if (result.error) {
+      summary.push(
+        "",
+        "<pre>",
+        result.error.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;"),
+        "</pre>",
+      );
+    }
+  }
+  writeFileSync(path.join(evidenceDir, "summary.md"), `${summary.join("\n")}\n`);
+  if (results.some((result) => result.status === "failed")) {
+    throw new Error(`Published baseline precheck failed; see ${evidenceDir}/summary.json`);
+  }
+  const unusable = new Set(
+    results.filter((result) => result.status === "skipped").map((result) => result.baseline),
+  );
+  return groups.flatMap((group) => {
+    if (!groupBaselines.get(group)?.some((spec) => unusable.has(spec))) {
+      return [group];
+    }
+    const remaining = group.docker_lanes.split(/\s+/u).filter((lane) => !survivorLanes.has(lane));
+    return remaining.length ? [{ ...group, docker_lanes: remaining.join(" ") }] : [];
+  });
+}

scripts/plan-targeted-docker-lane-groups.mjs

@@ -202,16 +202,29 @@ export function planTargetedDockerLaneGroups({
 const isMain = process.argv[1] ? fileURLToPath(import.meta.url) === process.argv[1] : false;
 
 if (isMain) {
-  process.stdout.write(
-    JSON.stringify(
-      planTargetedDockerLaneGroups({
-        groupSize: process.env.GROUP_SIZE,
-        lanes: process.env.LANES,
-        upgradeSurvivorBaseline: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC,
-        upgradeSurvivorBaselineScope: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SCOPE,
-        upgradeSurvivorBaselines: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS,
-        upgradeSurvivorScenarios: process.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS,
-      }),
-    ),
-  );
+  const options = {
+    groupSize: process.env.GROUP_SIZE,
+    lanes: process.env.LANES,
+    upgradeSurvivorBaseline: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC,
+    upgradeSurvivorBaselineScope: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SCOPE,
+    upgradeSurvivorBaselines: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS,
+    upgradeSurvivorScenarios: process.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS,
+  };
+  let groups = planTargetedDockerLaneGroups(options);
+  if (process.argv.length > 2) {
+    if (process.argv[2] !== "--check-baselines" || !process.argv[3] || process.argv.length !== 4) {
+      throw new Error(
+        "Usage: plan-targeted-docker-lane-groups.mjs [--check-baselines <evidence-dir>]",
+      );
+    }
+    const { checkUpgradeSurvivorBaselines } =
+      await import("./lib/upgrade-survivor-baseline-check.mjs");
+    groups = checkUpgradeSurvivorBaselines(groups, {
+      evidenceDir: process.argv[3],
+      baseline: options.upgradeSurvivorBaseline,
+      baselines: options.upgradeSurvivorBaselines,
+      scenarios: options.upgradeSurvivorScenarios,
+    });
+  }
+  process.stdout.write(JSON.stringify(groups));
 }

test/scripts/upgrade-survivor-baseline-check.test.ts

@@ -0,0 +1,158 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+
+function runFixture(
+  failure: "version" | "runtime" | "install" | "launch",
+  overrides: NodeJS.ProcessEnv = {},
+) {
+  const root = mkdtempSync(path.join(tmpdir(), "survivor-precheck-"));
+  const bin = path.join(root, "bin");
+  const evidence = path.join(root, "evidence");
+  const installs = path.join(root, "installs.jsonl");
+  mkdirSync(bin);
+  writeFileSync(
+    path.join(bin, "npm"),
+    `#!/usr/bin/env node
+const fs = require("node:fs");
+const path = require("node:path");
+const args = process.argv.slice(2);
+const prefix = args[args.indexOf("--prefix") + 1];
+const spec = args.find(arg => arg.startsWith("openclaw@"));
+fs.appendFileSync(${JSON.stringify(installs)}, JSON.stringify({ args, prefix }) + "\\n");
+if (${JSON.stringify(failure)} === "install" && spec.endsWith("8.1")) {
+  console.error("registry unavailable"); process.exit(1);
+}
+if (${JSON.stringify(failure)} === "launch" && spec.endsWith("8.1")) process.exit(0);
+fs.mkdirSync(path.join(prefix, "bin"), { recursive: true });
+fs.writeFileSync(path.join(prefix, "bin", "openclaw"), '#!/usr/bin/env node\\n' +
+  'if (' + JSON.stringify(spec.endsWith("8.1")) + ' && process.argv.includes(' +
+  JSON.stringify(${JSON.stringify(failure)} === "version" ? "--version" : "set") +
+  ')) { console.error("Cannot find package fixture-runtime"); process.exit(1); }\\n' +
+  'console.log(process.argv.includes("--version") ? ' + JSON.stringify(spec) + ' : "local");\\n',
+  { mode: 0o755 });
+`,
+    { mode: 0o755 },
+  );
+  const result = spawnSync(
+    process.execPath,
+    ["scripts/plan-targeted-docker-lane-groups.mjs", "--check-baselines", evidence],
+    {
+      encoding: "utf8",
+      env: {
+        ...process.env,
+        PATH: `${bin}${path.delimiter}${process.env.PATH}`,
+        LANES: "published-upgrade-survivor onboard",
+        GROUP_SIZE: "1",
+        OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS: "2026.8.1 2026.8.2",
+        OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: "openclaw@2026.8.1",
+        OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SCOPE: "all-scenarios",
+        OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS: "legacy-operator-state base",
+        ...overrides,
+      },
+    },
+  );
+  return { root, result, evidence, installs };
+}
+
+describe("published baseline startup admission", () => {
+  it.each(["version", "runtime"] as const)(
+    "skips unusable %s baselines with evidence before scheduling scenarios",
+    (failure) => {
+      const fixture = runFixture(failure);
+      try {
+        expect(fixture.result.status, fixture.result.stderr).toBe(0);
+        const groups = JSON.parse(fixture.result.stdout);
+        expect(groups.map((group: { label: string }) => group.label)).toEqual([
+          "published-upgrade-survivor-2026.8.2",
+          "onboard",
+        ]);
+        const report = JSON.parse(
+          readFileSync(path.join(fixture.evidence, "summary.json"), "utf8"),
+        );
+        expect(report.baselines).toEqual([
+          expect.objectContaining({
+            baseline: "openclaw@2026.8.1",
+            status: "skipped",
+            reason: expect.stringContaining("unusable published baseline"),
+            error: expect.stringContaining("Cannot find package fixture-runtime"),
+            scenarios: ["legacy-operator-state", "base"],
+          }),
+          expect.objectContaining({ baseline: "openclaw@2026.8.2", status: "usable" }),
+        ]);
+        const installs = readFileSync(fixture.installs, "utf8")
+          .trim()
+          .split("\n")
+          .map((line) => JSON.parse(line));
+        expect(installs).toHaveLength(2);
+        for (const install of installs) {
+          expect(install.args).toEqual([
+            "install",
+            "-g",
+            "--prefix",
+            install.prefix,
+            expect.stringMatching(/^openclaw@/),
+            "--no-fund",
+            "--no-audit",
+          ]);
+          expect(() => readFileSync(path.join(install.prefix, "bin", "openclaw"))).toThrow();
+        }
+        const summary = readFileSync(path.join(fixture.evidence, "summary.md"), "utf8");
+        expect(summary).toContain("skipped");
+        expect(summary).toContain("Cannot find package fixture-runtime");
+        expect(summary).not.toContain("passed");
+      } finally {
+        rmSync(fixture.root, { recursive: true, force: true });
+      }
+    },
+  );
+
+  it.each([
+    { lanes: "published-upgrade-survivor", expected: [] },
+    { lanes: "update-migration onboard", expected: ["onboard"] },
+  ])("preserves skip evidence for an inherited baseline in $lanes", ({ lanes, expected }) => {
+    const fixture = runFixture("runtime", {
+      LANES: lanes,
+      GROUP_SIZE: "2",
+      OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS: "",
+    });
+    try {
+      expect(fixture.result.status, fixture.result.stderr).toBe(0);
+      expect(
+        JSON.parse(fixture.result.stdout).map(
+          (group: { docker_lanes: string }) => group.docker_lanes,
+        ),
+      ).toEqual(expected);
+      const report = JSON.parse(readFileSync(path.join(fixture.evidence, "summary.json"), "utf8"));
+      expect(report.baselines).toEqual([
+        expect.objectContaining({ baseline: "openclaw@2026.8.1", status: "skipped" }),
+      ]);
+    } finally {
+      rmSync(fixture.root, { recursive: true, force: true });
+    }
+  });
+
+  it.each([
+    { failure: "install", error: "registry unavailable" },
+    { failure: "launch", error: "ENOENT" },
+  ] as const)(
+    "fails closed on $failure errors, preserving diagnostics instead of skipping coverage",
+    ({ failure, error }) => {
+      const fixture = runFixture(failure);
+      try {
+        expect(fixture.result.status).not.toBe(0);
+        const report = JSON.parse(
+          readFileSync(path.join(fixture.evidence, "summary.json"), "utf8"),
+        );
+        expect(report.baselines[0]).toMatchObject({
+          status: "failed",
+          error: expect.stringContaining(error),
+        });
+      } finally {
+        rmSync(fixture.root, { recursive: true, force: true });
+      }
+    },
+  );
+});