diff --git a/.github/actions/git-owner/owner.py b/.github/actions/git-owner/owner.py index 0b414974ed2a..e9579c03002a 100644 --- a/.github/actions/git-owner/owner.py +++ b/.github/actions/git-owner/owner.py @@ -461,7 +461,7 @@ def checkout_selected_ref(): def checkout_harness(sha): action = ".github/actions/setup-node-env/action.yml" node_setup_scripts = ("scripts/lib/pnpm-lockfile-documents.mjs",) - evidence_scripts = ("scripts/ios-screenshot-evidence.mjs", "scripts/lib/direct-run.mjs") + evidence_scripts = ("scripts/ios-screenshot-evidence.mjs", "scripts/lib/direct-run.mjs", "scripts/ci-static-step.sh") platform_scripts = ("scripts/lib/swift-toolchain.sh",) upgrade_scripts = ("scripts/lib/release-upgrade-baseline.mjs", "scripts/lib/release-version.mjs") npm_lock_scripts = ( diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4cc3c8735d8..9b6723601870 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -743,7 +743,7 @@ jobs: def checkout_harness(sha): action = ".github/actions/setup-node-env/action.yml" node_setup_scripts = ("scripts/lib/pnpm-lockfile-documents.mjs",) - evidence_scripts = ("scripts/ios-screenshot-evidence.mjs", "scripts/lib/direct-run.mjs") + evidence_scripts = ("scripts/ios-screenshot-evidence.mjs", "scripts/lib/direct-run.mjs", "scripts/ci-static-step.sh") platform_scripts = ("scripts/lib/swift-toolchain.sh",) upgrade_scripts = ("scripts/lib/release-upgrade-baseline.mjs", "scripts/lib/release-version.mjs") npm_lock_scripts = ( @@ -2500,6 +2500,8 @@ jobs: hybridHostedBaseRows = Object.values({ "preflight": count(ciQualification), "check-plan": count(hostedControlJobs && runCheckPlan), + "pr-fail-fast": count(workflowEventName === "pull_request" && manifest.run_checks_node_core_nondist && + process.env.OPENCLAW_CI_HEAD_REPOSITORY !== process.env.OPENCLAW_CI_REPOSITORY), "control-ui-performance": count(manifest.run_control_ui_performance), "native-i18n": count(manifest.run_native_i18n), "control-ui-i18n": count(manifest.run_control_ui_i18n), @@ -2886,13 +2888,13 @@ jobs: env: CHECKOUT_REPO: ${{ github.repository }} CHECKOUT_TOKEN: ${{ github.token }} - CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }} + CHECKOUT_SHA: &checkout_sha ${{ needs.preflight.outputs.checkout_revision }} WORKFLOW_SHA: ${{ github.workflow_sha }} run: *owned_checkout_run - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: &cache_mode ${{ needs.preflight.outputs.cache_mode }} install-bun: "true" node-compile-cache: "true" build-all-cache-scope: full @@ -2958,7 +2960,7 @@ jobs: # unavailable historical coverage, never a successful native-host test. if: ${{ needs.preflight.outputs.run_proof_tier == 'true' && (needs.preflight.outputs.frozen_target != 'true' || hashFiles('extensions/browser/src/browser/extension-install.native-host.e2e.test.ts') != '') }} env: - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} + FROZEN_TARGET: &frozen_target ${{ needs.preflight.outputs.frozen_target }} OPENCLAW_E2E_USE_PREBUILT_DIST: "1" OPENCLAW_VITEST_MAX_WORKERS: "1" run: | @@ -3029,7 +3031,7 @@ jobs: RUN_GATEWAY_WATCH: ${{ needs.preflight.outputs.run_proof_tier == 'true' && needs.preflight.outputs.run_check_additional == 'true' }} RUN_SQLITE_SESSION_LIFECYCLE: ${{ needs.preflight.outputs.run_sqlite_session_lifecycle }} RUN_TUI_PTY: ${{ needs.preflight.outputs.run_proof_tier == 'true' && needs.preflight.outputs.run_checks_node_core_dist == 'true' }} - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} + FROZEN_TARGET: *frozen_target shell: bash run: | set -uo pipefail @@ -3288,14 +3290,14 @@ jobs: timeout-minutes: 15 env: CHECKOUT_BASE_SHA: ${{ needs.preflight.outputs.diff_base_revision }} - COMPATIBILITY_TARGET: ${{ needs.preflight.outputs.compatibility_target }} + COMPATIBILITY_TARGET: &historical_target ${{ needs.preflight.outputs.compatibility_target }} steps: - *linux_node_checkout_step - &linux_node_setup_step name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" dependency-cache: *trusted_dependency_cache @@ -3361,17 +3363,17 @@ jobs: # Only the legacy package command is unsharded; exact current targets use the shard runner. shard: ${{ fromJSON(needs.preflight.outputs.compatibility_target == 'true' && '[1]' || '[1,2,3]') }} env: - COMPATIBILITY_TARGET: ${{ needs.preflight.outputs.compatibility_target }} + COMPATIBILITY_TARGET: *historical_target steps: - *linux_node_checkout_step - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode node-version: "24.x" install-bun: "false" dependency-cache: *trusted_dependency_cache - restore-test-caches: ${{ (needs.preflight.outputs.runner_profile == 'github' || needs.preflight.outputs.runner_profile == 'hybrid') && 'true' || 'false' }} + restore-test-caches: &restore_test_caches ${{ (needs.preflight.outputs.runner_profile == 'github' || needs.preflight.outputs.runner_profile == 'hybrid') && 'true' || 'false' }} - name: Setup pinned Bun test runtime if: needs.preflight.outputs.ui_test_runtime_policy == 'bun-compatible' || needs.preflight.outputs.ui_test_runtime_policy == 'dual' @@ -3388,7 +3390,7 @@ jobs: - &install_playwright_chromium name: Install Playwright Chromium env: - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} + FROZEN_TARGET: *frozen_target run: | if [[ "${COMPATIBILITY_TARGET:-false}" == "true" ]]; then # Legacy Vitest configs cannot pass a discovered system browser to Playwright. @@ -3463,11 +3465,11 @@ jobs: - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode node-version: "24.x" install-bun: "false" dependency-cache: &trusted_first_attempt_dependency_cache ${{ ((needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) == 'github' || runner.environment != 'self-hosted' || (github.event_name == 'workflow_dispatch' && ((needs.preflight.outputs.node_runner_backend != 'runson' && needs.preflight.outputs.ci_qualification != 'true') || github.run_attempt != 1)) || (github.event_name == 'pull_request' && github.run_attempt > 1)) && 'false' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'true' || 'false') }} - restore-test-caches: ${{ (needs.preflight.outputs.runner_profile == 'github' || needs.preflight.outputs.runner_profile == 'hybrid') && 'true' || 'false' }} + restore-test-caches: *restore_test_caches - *cache_playwright_chromium - *install_playwright_chromium @@ -3564,7 +3566,7 @@ jobs: - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode node-version: "24.x" install-bun: "false" dependency-cache: *trusted_first_attempt_dependency_cache @@ -3582,8 +3584,8 @@ jobs: - name: Prove desktop resize over node and SSH if: matrix.run_desktop env: - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} - DESKTOP_PROOF_CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }} + FROZEN_TARGET: *frozen_target + DESKTOP_PROOF_CHECKOUT_SHA: *checkout_sha DESKTOP_PROOF_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} DESKTOP_PROOF_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} DESKTOP_PROOF_WORKFLOW_SHA: ${{ github.workflow_sha }} @@ -3601,7 +3603,7 @@ jobs: - name: Test Control UI suites with a real Gateway env: - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} + FROZEN_TARGET: *frozen_target OPENCLAW_CAPTURE_UI_PROOF: ${{ github.event_name == 'workflow_dispatch' && inputs.capture_ui_proof && '1' || '0' }} OPENCLAW_UI_E2E_ARTIFACT_DIR: .artifacts/control-ui-e2e/real-gateway OPENCLAW_UI_E2E_DIAGNOSTIC_DIR: .artifacts/control-ui-e2e-timeouts/real-gateway-attempt-${{ github.run_attempt }} @@ -3741,13 +3743,13 @@ jobs: runs-on: *shared_small_linux_runner timeout-minutes: 10 env: - COMPATIBILITY_TARGET: ${{ needs.preflight.outputs.compatibility_target }} + COMPATIBILITY_TARGET: *historical_target steps: - *linux_node_checkout_step - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode node-version: "24.x" install-bun: "false" dependency-cache: *trusted_dependency_cache @@ -3919,7 +3921,7 @@ jobs: - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: ${{ matrix.task == 'bun-launcher' && 'true' || 'false' }} dependency-cache: *trusted_dependency_cache restore-test-caches: ${{ (needs.preflight.outputs.runner_profile == 'github' || needs.preflight.outputs.runner_profile == 'hybrid') && (matrix.task == 'bundled-protocol' || matrix.task == 'contracts-plugins-ci-routing' || matrix.task == 'ci-routing' || matrix.task == 'bun-launcher') && 'true' || 'false' }} @@ -4066,7 +4068,7 @@ jobs: - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" dependency-cache: *trusted_dependency_cache # The trusted warmer seeds build tooling and Node orchestration together. @@ -4247,10 +4249,10 @@ jobs: name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" dependency-cache: *trusted_dependency_cache - restore-test-caches: ${{ (needs.preflight.outputs.runner_profile == 'github' || needs.preflight.outputs.runner_profile == 'hybrid') && 'true' || 'false' }} + restore-test-caches: *restore_test_caches - name: Run plugin contract shard env: @@ -4327,7 +4329,7 @@ jobs: - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode node-version: "24.16.0" install-bun: "false" build-all-cache-scope: full @@ -4358,7 +4360,8 @@ jobs: runs-on: ${{ (github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '') != '' && fromJSON(format('{{"group":{0},"labels":{1}}}', toJSON((github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '')), toJSON(needs.preflight.outputs.node_runner_backend == 'runson' && matrix.runner == 'runson-c8i-8xlarge' && github.run_attempt == 1 && format('runs-on={0}-{1}/family=c8i.8xlarge/cpu=32/ram=64/spot=true/retry=false/image=ubuntu24-full-x64/volume=80gb', github.run_id, matrix.check_name) || matrix.runner == 'runson-c8i-8xlarge' && 'ubuntu-24.04' || (needs.preflight.outputs.node_runner_backend == 'runson' || needs.preflight.outputs.ci_qualification == 'true') && github.run_attempt == 1 && ((matrix.runner == 'blacksmith-8vcpu-ubuntu-2404' && contains(fromJSON('["checks-node-compact-large-9","checks-node-compact-large-5"]'), matrix.check_name)) && 'blacksmith-16vcpu-ubuntu-2404' || matrix.runner || 'blacksmith-4vcpu-ubuntu-2404') || (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) == 'github' && 'ubuntu-24.04' || (contains(fromJSON('["hybrid","runson"]'), (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND)) && github.run_attempt > 1) && 'ubuntu-24.04' || ((github.event_name == 'workflow_dispatch' && ((needs.preflight.outputs.node_runner_backend != 'runson' && needs.preflight.outputs.ci_qualification != 'true') || github.run_attempt != 1)) || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.run_attempt > 1)) && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || contains(fromJSON('["OWNER","MEMBER","COLLABORATOR","CONTRIBUTOR"]'), github.event.pull_request.author_association)) && ((matrix.runner == 'blacksmith-8vcpu-ubuntu-2404' && contains(fromJSON('["checks-node-compact-large-9","checks-node-compact-large-5"]'), matrix.check_name)) && 'blacksmith-16vcpu-ubuntu-2404' || matrix.runner || 'blacksmith-4vcpu-ubuntu-2404') || 'ubuntu-24.04')))) || (needs.preflight.outputs.node_runner_backend == 'runson' && matrix.runner == 'runson-c8i-8xlarge' && github.run_attempt == 1 && format('runs-on={0}-{1}/family=c8i.8xlarge/cpu=32/ram=64/spot=true/retry=false/image=ubuntu24-full-x64/volume=80gb', github.run_id, matrix.check_name) || matrix.runner == 'runson-c8i-8xlarge' && 'ubuntu-24.04' || (needs.preflight.outputs.node_runner_backend == 'runson' || needs.preflight.outputs.ci_qualification == 'true') && github.run_attempt == 1 && ((matrix.runner == 'blacksmith-8vcpu-ubuntu-2404' && contains(fromJSON('["checks-node-compact-large-9","checks-node-compact-large-5"]'), matrix.check_name)) && 'blacksmith-16vcpu-ubuntu-2404' || matrix.runner || 'blacksmith-4vcpu-ubuntu-2404') || (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) == 'github' && 'ubuntu-24.04' || (contains(fromJSON('["hybrid","runson"]'), (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND)) && github.run_attempt > 1) && 'ubuntu-24.04' || ((github.event_name == 'workflow_dispatch' && ((needs.preflight.outputs.node_runner_backend != 'runson' && needs.preflight.outputs.ci_qualification != 'true') || github.run_attempt != 1)) || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.run_attempt > 1)) && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || contains(fromJSON('["OWNER","MEMBER","COLLABORATOR","CONTRIBUTOR"]'), github.event.pull_request.author_association)) && ((matrix.runner == 'blacksmith-8vcpu-ubuntu-2404' && contains(fromJSON('["checks-node-compact-large-9","checks-node-compact-large-5"]'), matrix.check_name)) && 'blacksmith-16vcpu-ubuntu-2404' || matrix.runner || 'blacksmith-4vcpu-ubuntu-2404') || 'ubuntu-24.04')) }} timeout-minutes: ${{ matrix.timeout_minutes || 60 }} strategy: - fail-fast: ${{ github.event_name == 'pull_request' }} + # The trusted monitor must classify a failure before sibling cancellation. + fail-fast: ${{ github.event_name == 'pull_request' && (github.run_attempt != 1 || github.repository != 'openclaw/openclaw') }} # Compact and PR manifest caps bound registrations; parallelism bounds # active jobs. Canonical main retains two non-canceling parity slots. max-parallel: ${{ github.event_name == 'pull_request' && github.repository == 'openclaw/openclaw' && github.event.pull_request.head.repo.full_name == github.repository && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR","CONTRIBUTOR"]'), github.event.pull_request.author_association) && github.run_attempt == 1 && needs.preflight.outputs.frozen_target != 'true' && needs.preflight.outputs.runner_profile != 'github' && needs.preflight.outputs.node_runner_backend != 'runson' && contains(fromJSON('["","blacksmith","hybrid"]'), vars.OPENCLAW_CI_RUNNER_BACKEND) && 130 || 96 }} @@ -4391,7 +4394,7 @@ jobs: - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode node-version: "${{ (matrix.runner == 'runson-c8i-8xlarge' || startsWith(matrix.check_name, 'checks-node-runson-cron-')) && env.NODE_VERSION || matrix.node_version || '24.x' }}" install-bun: "false" # The trusted warmer seeds Node 24; other versions use the store. @@ -4404,7 +4407,7 @@ jobs: if: needs.preflight.outputs.checkout_revision == 'f773aa06a1a93b36b050f1a3f4b57d3d91311541' uses: ./.ci-harness/.github/actions/frozen-node-test-compat with: - target-sha: ${{ needs.preflight.outputs.checkout_revision }} + target-sha: *checkout_sha - name: Setup pinned Bun test runtime if: matrix.requires_bun == true @@ -4503,7 +4506,7 @@ jobs: env: PREDICTED_TEST_SECONDS: ${{ matrix.predicted_seconds || '' }} SHARD_PLAN_CONCURRENCY: ${{ matrix.plan_concurrency || '' }} - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} + FROZEN_TARGET: *frozen_target RUNNER_ENVIRONMENT: ${{ runner.environment }} RUNSON_JOB: ${{ (matrix.runner == 'runson-c8i-8xlarge' || startsWith(matrix.check_name, 'checks-node-runson-cron-')) && 'true' || 'false' }} run: | @@ -4543,7 +4546,7 @@ jobs: NODE_OPTIONS: --max-old-space-size=8192 OPENCLAW_E2E_USE_PREBUILT_DIST: ${{ steps.node-test-runtime.outcome == 'success' && matrix.pretest_build_mode == 'private-qa' && '1' || '' }} OPENCLAW_CI_TEST_RUNTIME_POLICY: ${{ matrix.test_runtime_policy || 'node' }} - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} + FROZEN_TARGET: *frozen_target RUNNER_ENVIRONMENT: ${{ runner.environment }} OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${{ matrix.groups_gzip_base64 || '' }} OPENCLAW_NODE_TEST_GROUPS_JSON: ${{ matrix.groups && toJson(matrix.groups) || '' }} @@ -4558,6 +4561,7 @@ jobs: OPENCLAW_VITEST_SHARD_NAME: ${{ matrix.shard_name }} OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: ${{ needs.preflight.outputs.compatibility_target == 'true' && '660000' || '300000' }} OPENCLAW_NODE_TEST_PLAN_CONCURRENCY: ${{ matrix.plan_concurrency }} + OPENCLAW_NODE_TEST_PLAN_CONTINUE_ON_FAILURE: ${{ github.event_name == 'pull_request' && github.run_attempt == 1 && github.repository == 'openclaw/openclaw' && '1' || '0' }} shell: bash run: | set -euo pipefail @@ -4608,7 +4612,7 @@ jobs: - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" dependency-cache: ${{ runner.environment == 'self-hosted' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'true' || 'false' }} - name: Check narrow PR global guards @@ -4647,7 +4651,7 @@ jobs: id: npm-lock-scope if: matrix.task == 'npm-lock' env: - HISTORICAL_TARGET: ${{ needs.preflight.outputs.compatibility_target }} + HISTORICAL_TARGET: *historical_target shell: bash run: | # A missing runtime or unfamiliar target must keep the existing check. @@ -4661,7 +4665,7 @@ jobs: if: steps.npm-lock-scope.outputs.skip != 'true' uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" dependency-cache: *trusted_dependency_cache @@ -4816,10 +4820,10 @@ jobs: if: matrix.task != 'lint' || !(needs.preflight.outputs.run_check_plan == 'true' && needs.check-plan.outputs.central_lint_selection_json || needs.preflight.outputs.central_lint_selection_json) env: SKIP_NPM_LOCK: ${{ steps.npm-lock-scope.outputs.skip }} - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} - HISTORICAL_TARGET: ${{ needs.preflight.outputs.compatibility_target }} + FROZEN_TARGET: *frozen_target + HISTORICAL_TARGET: *historical_target FORMAT_CHECK: ${{ needs.preflight.outputs.run_format_check }} - RELEASE_GATE: ${{ inputs.release_gate && (needs.preflight.outputs.node_runner_backend != 'runson' && needs.preflight.outputs.ci_qualification != 'true') && 'true' || 'false' }} + RELEASE_GATE: &release_gate ${{ inputs.release_gate && (needs.preflight.outputs.node_runner_backend != 'runson' && needs.preflight.outputs.ci_qualification != 'true') && 'true' || 'false' }} RUN_CONTROL_UI_I18N: ${{ needs.preflight.outputs.run_control_ui_i18n }} RUN_UI_TESTS: ${{ needs.preflight.outputs.run_ui_tests }} HOSTED_RUNNER_STRIPES: ${{ (needs.preflight.outputs.runner_profile == 'github' || needs.preflight.outputs.runner_profile == 'hybrid') && (needs.preflight.outputs.frozen_target != 'true' || needs.preflight.outputs.hosted_runner_profile_contract == 'true') && 'true' || 'false' }} @@ -4830,6 +4834,7 @@ jobs: CI_TYPE_GRAPHS_JSON: ${{ matrix.type_graph_names_json }} CI_CORE_TYPE_GRAPHS_JSON: ${{ matrix.core_type_graph_names_json }} CI_CORE_TYPE_CONCURRENCY: ${{ matrix.core_type_concurrency }} + OPENCLAW_CI_STATIC_EVIDENCE: &static_evidence ${{ github.event_name == 'pull_request' && github.run_attempt == 1 && github.repository == 'openclaw/openclaw' && needs.preflight.outputs.frozen_target != 'true' && '1' || '0' }} TASK: ${{ matrix.task }} shell: bash run: | @@ -4841,18 +4846,19 @@ jobs: ' "$1" } if [ -n "${NARROW_CHECK_PATHS_JSON:-}" ] && { [ "$TASK" = "test-types" ] || [ "$TASK" = "prod-types" ]; }; then + source .ci-harness/scripts/ci-static-step.sh tsgo if [ -n "${CI_CORE_TYPE_GRAPHS_JSON:-}" ] && [ "$CI_CORE_TYPE_GRAPHS_JSON" != "[]" ]; then core_type_command=(node scripts/run-tsgo-core-test-shards.mjs) if [ "$CI_CORE_TYPE_CONCURRENCY" = "2" ]; then core_type_command=(env -u OPENCLAW_LOCAL_CHECK "${core_type_command[@]}") fi - "${core_type_command[@]}" \ + run_static_check "${core_type_command[@]}" \ --ci-graphs-json "$CI_CORE_TYPE_GRAPHS_JSON" --concurrency "$CI_CORE_TYPE_CONCURRENCY" fi if [ -n "${CI_TYPE_GRAPHS_JSON:-}" ] && [ "$CI_TYPE_GRAPHS_JSON" != "[]" ]; then - node scripts/run-tsgo-core-test-shards.mjs --ci-graphs-json "$CI_TYPE_GRAPHS_JSON" + run_static_check node scripts/run-tsgo-core-test-shards.mjs --ci-graphs-json "$CI_TYPE_GRAPHS_JSON" fi - exit 0 + finish_static_checks fi case "$TASK" in guards) @@ -5166,12 +5172,14 @@ jobs: if: ${{ !matrix.lint_selection_json }} env: CORE_STRIPE: ${{ matrix.stripe }} - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} + FROZEN_TARGET: *frozen_target OPENCLAW_LOCAL_CHECK: "0" - RELEASE_GATE: ${{ inputs.release_gate && (needs.preflight.outputs.node_runner_backend != 'runson' && needs.preflight.outputs.ci_qualification != 'true') && 'true' || 'false' }} + RELEASE_GATE: *release_gate RUNNER_PROFILE: ${{ needs.preflight.outputs.runner_profile }} + OPENCLAW_CI_STATIC_EVIDENCE: *static_evidence run: | set -euo pipefail + source .ci-harness/scripts/ci-static-step.sh oxlint if [ "$FROZEN_TARGET" = "true" ]; then # Older wrappers lack the current constrained-host Go policy. export GOMAXPROCS=2 @@ -5191,14 +5199,15 @@ jobs: fi fi for stripe in "${stripes[@]}"; do - node --import tsx scripts/run-oxlint-shards.mts \ + run_static_check node --import tsx scripts/run-oxlint-shards.mts \ --only=core --split-core --core-stripe="$stripe/5" --threads=1 if [ "$RUNNER_PROFILE" = "github" ] && [ "$RELEASE_GATE" != "true" ] && grep -q -- '--extension-stripe' scripts/run-oxlint-shards.mts 2>/dev/null; then - node --import tsx scripts/run-oxlint-shards.mts \ + run_static_check node --import tsx scripts/run-oxlint-shards.mts \ --only=extensions --extension-stripe="$stripe/6" --threads=1 fi done + finish_static_checks # Keep extension compilation off the already packed hybrid core-lint rows. @@ -5220,7 +5229,7 @@ jobs: name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" - name: Compute extension boundary input fingerprint @@ -5243,11 +5252,14 @@ jobs: EXTENSION_STRIPE: ${{ matrix.stripe }} EXTENSION_STRIPE_COUNT: ${{ matrix.stripe_count || 6 }} OPENCLAW_LOCAL_CHECK: "0" + OPENCLAW_CI_STATIC_EVIDENCE: ${{ github.event_name == 'pull_request' && github.run_attempt == 1 && github.repository == 'openclaw/openclaw' && '1' || '0' }} shell: bash run: | set -euo pipefail - node --import tsx scripts/run-oxlint-shards.mts \ + source .ci-harness/scripts/ci-static-step.sh oxlint + run_static_check node --import tsx scripts/run-oxlint-shards.mts \ --only=extensions --extension-stripe="$EXTENSION_STRIPE/$EXTENSION_STRIPE_COUNT" --threads=1 + finish_static_checks # Run each complete core stripe independently; the central row owns the tail. # Keep two compiler children per row and the frozen target's paired layout. @@ -5283,14 +5295,16 @@ jobs: env: CORE_STRIPE: ${{ matrix.stripe }} CI_TYPE_GRAPHS_JSON: ${{ matrix.type_graph_names_json }} - FROZEN_TARGET: ${{ needs.preflight.outputs.frozen_target }} + FROZEN_TARGET: *frozen_target CHANGED_CORE_TEST_PATHS_JSON: ${{ needs.preflight.outputs.changed_core_test_paths_json }} + OPENCLAW_CI_STATIC_EVIDENCE: *static_evidence shell: bash run: | set -euo pipefail + source .ci-harness/scripts/ci-static-step.sh tsgo if [ -n "${CI_TYPE_GRAPHS_JSON:-}" ]; then - node scripts/run-tsgo-core-test-shards.mjs --ci-graphs-json "$CI_TYPE_GRAPHS_JSON" --concurrency 2 - exit "$?" + run_static_check node scripts/run-tsgo-core-test-shards.mjs --ci-graphs-json "$CI_TYPE_GRAPHS_JSON" --concurrency 2 + finish_static_checks fi # Frozen/historical targets predate stripe support; their whole # test-types lane already runs inside the check-test-types row. @@ -5309,7 +5323,8 @@ jobs: if [ -n "${CHANGED_CORE_TEST_PATHS_JSON:-}" ]; then args+=(--changed-paths-json "$CHANGED_CORE_TEST_PATHS_JSON") fi - node scripts/run-tsgo-core-test-shards.mjs "${args[@]}" + run_static_check node scripts/run-tsgo-core-test-shards.mjs "${args[@]}" + finish_static_checks fi - name: Save core test-type incremental state @@ -5417,7 +5432,7 @@ jobs: ADDITIONAL_CHECK_GROUP: ${{ matrix.group }} TYPE_GRAPH_BOUNDARY_CHECKED: ${{ (needs.preflight.outputs.run_check_plan == 'true' && needs.check-plan.outputs.type_graph_boundary_checked || needs.preflight.outputs.type_graph_boundary_checked) }} CHANGED_CORE_TEST_PATHS_JSON: ${{ needs.preflight.outputs.changed_core_test_paths_json }} - COMPATIBILITY_TARGET: ${{ needs.preflight.outputs.compatibility_target }} + COMPATIBILITY_TARGET: *historical_target RUN_PROMPT_SNAPSHOTS: ${{ needs.preflight.outputs.run_prompt_snapshots }} OPENCLAW_ADDITIONAL_BOUNDARY_SHARD: "" OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY: 4 @@ -5625,7 +5640,7 @@ jobs: CHECKOUT_KIND: skills CHECKOUT_REPO: ${{ github.repository }} CHECKOUT_TOKEN: ${{ github.token }} - CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }} + CHECKOUT_SHA: *checkout_sha run: *owned_checkout_run - name: Setup Python @@ -5672,7 +5687,7 @@ jobs: env: CHECKOUT_REPO: ${{ github.repository }} CHECKOUT_TOKEN: ${{ github.token }} - CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }} + CHECKOUT_SHA: *checkout_sha WORKFLOW_SHA: ${{ github.workflow_sha }} run: *owned_checkout_run @@ -5703,7 +5718,7 @@ jobs: - name: Setup pnpm uses: ./.ci-harness/.github/actions/setup-pnpm-store-cache with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode node-version: ${{ env.NODE_VERSION }} - name: Runtime versions @@ -5842,7 +5857,7 @@ jobs: runs-on: &hosted_xcode_runner ${{ (github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '') != '' && fromJSON(format('{{"group":{0},"labels":{1}}}', toJSON((github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '')), toJSON('xcode-27'))) || ('xcode-27') }} timeout-minutes: 30 env: - HISTORICAL_TARGET: ${{ needs.preflight.outputs.compatibility_target }} + HISTORICAL_TARGET: *historical_target MACOS_PRIMARY_PHASE: ${{ github.event_name == 'workflow_dispatch' && needs.preflight.outputs.validation_tier != 'main' && !inputs.release_gate && needs.preflight.outputs.release_scope == 'full' && 'release' || 'tests' }} OPENCLAWKIT_TEST_EXECUTION: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && needs.preflight.outputs.ci_shape != 'main') || github.run_attempt > 1) && 'serial' || 'parallel' }} steps: @@ -5873,7 +5888,7 @@ jobs: uses: ./.ci-harness/.github/actions/setup-node-env with: node-version: ${{ env.NODE_VERSION }} - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" install-deps: "false" @@ -6274,7 +6289,7 @@ jobs: runs-on: *hosted_xcode_runner timeout-minutes: 150 env: - HISTORICAL_TARGET: ${{ needs.preflight.outputs.compatibility_target }} + HISTORICAL_TARGET: *historical_target IOS_CI_PHASE: ${{ matrix.phase }} IOS_MAIN_TIER: ${{ needs.preflight.outputs.validation_tier == 'main' }} steps: @@ -6615,7 +6630,7 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' && !inputs.release_gate && needs.preflight.outputs.validation_tier == 'full' && needs.preflight.outputs.release_scope == 'full' && needs.preflight.outputs.checkout_revision == github.sha && needs.preflight.outputs.compatibility_target != 'true' && needs.preflight.outputs.run_ios_build == 'true' }} uses: ./.github/workflows/ios-release-e2e.yml with: - target_sha: ${{ needs.preflight.outputs.checkout_revision }} + target_sha: *checkout_sha mode: stock ios-screenshot-shard: @@ -6661,7 +6676,7 @@ jobs: - name: Setup Node environment uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" node-version: ${{ env.IOS_SCREENSHOT_NODE_VERSION }} @@ -6701,7 +6716,7 @@ jobs: DEVICE_FAMILY: ${{ matrix.device_family }} RUN_ATTEMPT: ${{ github.run_attempt }} RUN_ID: ${{ github.run_id }} - TARGET_SHA: ${{ needs.preflight.outputs.checkout_revision }} + TARGET_SHA: *checkout_sha WORKFLOW_SHA: ${{ github.workflow_sha }} run: | set -euo pipefail @@ -6766,7 +6781,7 @@ jobs: - name: Setup screenshot evidence Node uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" install-deps: "false" node-version: ${{ env.IOS_SCREENSHOT_NODE_VERSION }} @@ -6783,7 +6798,7 @@ jobs: env: RUN_ATTEMPT: ${{ github.run_attempt }} RUN_ID: ${{ github.run_id }} - TARGET_SHA: ${{ needs.preflight.outputs.checkout_revision }} + TARGET_SHA: *checkout_sha WORKFLOW_SHA: ${{ github.workflow_sha }} run: | node .ci-harness/scripts/ios-screenshot-evidence.mjs reduce \ @@ -6842,7 +6857,7 @@ jobs: CHECKOUT_KIND: android CHECKOUT_REPO: ${{ github.repository }} CHECKOUT_TOKEN: ${{ github.token }} - CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }} + CHECKOUT_SHA: *checkout_sha run: *owned_checkout_run - &android_toolchain_checkout_step @@ -6866,7 +6881,7 @@ jobs: if: needs.preflight.outputs.use_compatible_android_ci != 'true' uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" install-deps: "false" @@ -7103,12 +7118,12 @@ jobs: - name: Setup Android toolchain uses: ./.ci-harness/.github/actions/setup-android-toolchain with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode - name: Setup Node environment for native resources uses: ./.ci-harness/.github/actions/setup-node-env with: - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode install-bun: "false" - name: Run packaged Access crypto on Android @@ -7198,7 +7213,7 @@ jobs: uses: ./.ci-harness/.github/actions/setup-node-env with: build-all-cache-scope: full - cache-mode: ${{ needs.preflight.outputs.cache_mode }} + cache-mode: *cache_mode node-version: "24.x" install-bun: "false" dependency-cache: *trusted_first_attempt_dependency_cache @@ -7240,8 +7255,8 @@ jobs: OPENCLAW_DOCKER_E2E_ALLOW_UNRELEASED_CHANGELOG: "1" OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS: ${{ needs.preflight.outputs.frozen_target == 'true' && 'base' || 'legacy-operator-state' }} OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE: auto-auth - OPENCLAW_DOCKER_ALL_PARALLELISM: ${{ (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) != 'github' && github.event_name == 'pull_request' && github.run_attempt == 1 && github.event.pull_request.head.repo.full_name == github.repository && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR","CONTRIBUTOR"]'), github.event.pull_request.author_association) && 3 || 1 }} - OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM: ${{ (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) != 'github' && github.event_name == 'pull_request' && github.run_attempt == 1 && github.event.pull_request.head.repo.full_name == github.repository && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR","CONTRIBUTOR"]'), github.event.pull_request.author_association) && 3 || 1 }} + OPENCLAW_DOCKER_ALL_PARALLELISM: &docker_parallelism ${{ (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) != 'github' && github.event_name == 'pull_request' && github.run_attempt == 1 && github.event.pull_request.head.repo.full_name == github.repository && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR","CONTRIBUTOR"]'), github.event.pull_request.author_association) && 3 || 1 }} + OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM: *docker_parallelism run: pnpm test:docker:all - name: Upload Fleet Docker proof @@ -7276,29 +7291,35 @@ jobs: needs: [preflight] # Survive our own cancellation long enough to hand its cause to ci-gate. # Partial reruns reuse prerequisites and do not contain the complete manifest. - if: ${{ always() && github.event_name == 'pull_request' && github.run_attempt == 1 && github.repository == 'openclaw/openclaw' && !github.event.pull_request.draft && github.event.pull_request.head.repo.full_name == github.repository && needs.preflight.result == 'success' && needs.preflight.outputs.run_checks_node_core_nondist == 'true' }} + if: ${{ always() && github.event_name == 'pull_request' && github.run_attempt == 1 && github.repository == 'openclaw/openclaw' && !github.event.pull_request.draft && needs.preflight.result == 'success' && needs.preflight.outputs.run_checks_node_core_nondist == 'true' }} runs-on: ${{ (github.event_name == 'workflow_dispatch' || github.run_attempt > 1 || github.repository != 'openclaw/openclaw' || github.event.pull_request.head.repo.full_name != github.repository || needs.preflight.outputs.runner_profile == 'github') && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 60 outputs: failure_job_id: ${{ steps.monitor.outputs.failure_job_id }} failure_job_name: ${{ steps.monitor.outputs.failure_job_name }} failure_run_attempt: ${{ steps.monitor.outputs.failure_run_attempt }} + known_main_red_attempt: ${{ steps.monitor.outputs.known_main_red_attempt }} steps: - name: Checkout CI monitor if: always() uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.workflow_sha }} + ref: ${{ github.event.pull_request.base.sha }} persist-credentials: false - sparse-checkout: scripts/ci-pr-fail-fast.mjs + sparse-checkout: | + scripts/ci-pr-fail-fast.mjs + scripts/ci-known-main-red.mjs + scripts/lib/ci-node-test-evidence.mjs + scripts/lib/ci-static-check-evidence.mjs sparse-checkout-cone-mode: false - - name: Cancel remaining PR work after a failure + - name: Classify PR failures and cancel eligible same-repository work id: monitor if: always() env: GITHUB_TOKEN: ${{ github.token }} OPENCLAW_CI_PR_NUMBER: ${{ github.event.pull_request.number }} OPENCLAW_CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + OPENCLAW_CI_PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} OPENCLAW_CI_EXPECTED_JOBS: ${{ needs.preflight.outputs.pr_job_count }} OPENCLAW_CI_PREFLIGHT_CHECK_JOBS: ${{ needs.preflight.outputs.pr_check_job_count }} OPENCLAW_CI_CHECK_PLAN_EXPECTED: ${{ needs.preflight.outputs.run_check_plan }} @@ -7371,6 +7392,7 @@ jobs: env: RELEASE_FAST_LANE: ${{ needs.preflight.outputs.release_fast_lane }} ALLOW_COALESCED_IOS: ${{ github.event_name == 'schedule' && github.repository == 'openclaw/openclaw' && github.ref == 'refs/heads/main' }} + ALLOW_KNOWN_MAIN_RED: ${{ github.event_name == 'pull_request' && github.run_attempt == 1 && needs.pr-fail-fast.result == 'success' && needs.pr-fail-fast.outputs.known_main_red_attempt == format('{0}', github.run_attempt) }} JOB_RESULTS: | preflight=${{ needs.preflight.result }}|true security-fast=${{ needs.security-fast.result }}|true @@ -7406,7 +7428,7 @@ jobs: android=${{ needs.android.result }}|${{ needs.preflight.outputs.run_android_job }} android-access-native=${{ needs.android-access-native.result }}|${{ needs.preflight.outputs.run_android_access_native }} docker-seed-e2e=${{ needs.docker-seed-e2e.result }}|${{ needs.preflight.outputs.run_docker_seed_e2e }} - pr-fail-fast=${{ github.run_attempt != 1 && 'skipped' || needs.pr-fail-fast.result }}|${{ github.event_name == 'pull_request' && github.run_attempt == 1 && github.repository == 'openclaw/openclaw' && github.event.pull_request.head.repo.full_name == github.repository && needs.preflight.outputs.run_checks_node_core_nondist == 'true' }} + pr-fail-fast=${{ github.run_attempt != 1 && 'skipped' || needs.pr-fail-fast.result }}|${{ github.event_name == 'pull_request' && github.run_attempt == 1 && github.repository == 'openclaw/openclaw' && needs.preflight.outputs.run_checks_node_core_nondist == 'true' }} run: | set -euo pipefail echo "release fast lane: ${RELEASE_FAST_LANE:-false}" @@ -7421,6 +7443,14 @@ jobs: selected="${result#*|}" result="${result%%|*}" echo "${name}: ${result} (selected=${selected:-missing})" + if [[ "${ALLOW_KNOWN_MAIN_RED:-false}" == "true" && "$selected:$result" == "true:failure" ]]; then + case "$name" in + checks-node-core-test-nondist-shard|check-shard|check-test-types-hosted-core-shard|check-lint-hosted-core-shard|check-lint-hosted-extension-shard) + echo "::notice title=known main red, owned by main::Exact failures match hourly main; failing files and direct subjects are unchanged." + continue + ;; + esac + fi # The scheduled iOS slot finishes its active job and replaces pending # jobs. Only that cancellation delegates proof to the next hourly run. if [[ "${ALLOW_COALESCED_IOS:-false}" == "true" && "$name:$selected:$result" == "ios-build:true:cancelled" ]]; then @@ -7489,4 +7519,4 @@ jobs: uses: ./.github/workflows/full-release-child-evidence.yml with: role: normalCi - target_sha: ${{ needs.preflight.outputs.checkout_revision }} + target_sha: *checkout_sha diff --git a/docs/ci.md b/docs/ci.md index 2b2710a4e067..32dfe7f86e89 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -21,9 +21,11 @@ Full hybrid extension lint packs the same canonical chunks into three existing r no-op events before runner allocation and concurrency, keeping automation on GitHub-hosted runners. -PR Node matrices stop sibling rows on failure. Same-repository PRs also cancel -other job families through a scoped monitor, preserving a failed aggregate that -names the originating job. Main and manual runs retain complete matrices. See +First-attempt PR Node matrices let the scoped monitor classify failures before +cancelling eligible same-repository work. Fork monitoring is read-only. Exact +known hourly-main test and supported static failures can remain advisory when the PR leaves their +subjects unchanged and all remaining checks finish. Retries retain native matrix +fail-fast. Main and manual runs retain complete matrices. See [failure cancellation](/ci/pipeline#fail-fast-order). For the published-upgrade regression gate, see [selection and routing](/ci/scope-and-routing#scope-and-routing), [runner budgets](/ci/capacity#runner-registration-budget), and [Package Acceptance baselines](/ci/release-validation#suite-profiles). Weekly validation is listed under [Update Migration](/ci/scheduled-workflows#update-migration). diff --git a/docs/ci/pipeline.md b/docs/ci/pipeline.md index 3d66494c3ae4..db13bd7455ac 100644 --- a/docs/ci/pipeline.md +++ b/docs/ci/pipeline.md @@ -690,7 +690,7 @@ automation account, and SecOps-owned-path cases before declaring enforcement act 3. `build-artifacts` and the locale checks overlap with the fast Linux lanes. Control UI and native app source PRs exclude generated locale snapshots/resources; their serialized refresh workflows repair and auto-merge isolated generated PRs in the background. Source CI still blocks stale source inventories and unsafe localization calls. Generated PRs, manual CI, and release prep enforce full translated/platform-generated parity. Canonical `release/YYYY.M.PATCH` branches may include release-prep locale repairs with the other generated release output. 4. Baseline ratchets and selected Node test shards start independently after preflight. Node rows consume the manifest, not ratchet outputs. `ci-gate` still requires every selected ratchet to pass, and the PR failure monitor still cancels remaining work after a ratchet failure. Frozen targets retain their existing ratchet selection. 5. Other platform and runtime lanes fan out independently: `checks-fast-core` (including startup corpus), `checks-fast-contracts-plugins`, `checks-fast-contracts-channels`, `checks-windows`, `macos-node`, `macos-swift`, `ios-build`, the screenshot shards, and `android`. -6. PR Node matrices use native fail-fast. For same-repository PRs selecting Node rows, `pr-fail-fast` watches the first attempt and cancels remaining job families after a failure. Only that job has `actions: write`. It starts after preflight and observes failures while the installed check planner queues or runs. Clean completion combines preflight's other job counts with the planner's exact admitted check count, published by its successful `CI check job count v1: N` step. It rechecks the current PR head and newer runs before cancellation; fork PRs retain native matrix fail-fast because their tokens cannot cancel base-repository workflows. The monitor adds one 4-vCPU Blacksmith registration per eligible PR, or uses hosted Ubuntu under the outage override. Main, manual runs, and retries do not start it. Observation ends before the monitor's job limit; ordinary lane verification still owns the result when no failure was observed. Partial reruns retain native fail-fast and ordinary lane verification, ignoring monitor causes and results retained from earlier attempts. +6. For canonical-repository PRs selecting Node rows, `pr-fail-fast` watches the first attempt and classifies failures before cancelling eligible same-repository work. Fork PR monitoring is read-only and never requests cancellation; unknown failures remain blocking through normal lane results. Only that job has `actions: write`. It starts after preflight and observes failures while the installed check planner queues or runs. Clean completion combines preflight's other job counts with the planner's exact admitted check count, published by its successful `CI check job count v1: N` step. It rechecks the current PR head, auto-merge setting, and newer runs before cancellation. Retries retain native matrix fail-fast. The monitor checks out trusted base-revision scripts. It adds one 4-vCPU Blacksmith registration per eligible same-repository PR, or uses hosted Ubuntu for fork PRs and under the outage override. The hybrid hosted admission owner reserves that fork row before spending the unchanged 45-row optional-offload budget. Main, manual runs, and retries do not start it. Observation ends before the monitor's job limit; ordinary lane verification still owns the result when no failure was observed. Partial reruns ignore monitor causes and results retained from earlier attempts. 7. `openclaw/ci-gate` waits for every selected lane. Preflight and security must succeed; downstream jobs may skip only when unselected by the manifest and existing event, runner, and compatibility conditions. An unexpected selected skip or any failed or canceled downstream job fails the aggregate. Failure-triggered cancellation preserves the originating job's identity and runs the gate to report failure, including a cancellation request with an uncertain response. The existing critical-path route already keeps trusted hybrid first attempts on the 4-vCPU Blacksmith class. A first-attempt same-repository failure also uses that class under the default or explicit Blacksmith profile so hosted assignment cannot consume the cancellation grace period. Retries and the GitHub outage override retain hosted aggregation. A superseded run without a recorded failure cause skips final reporting and releases its concurrency slot as before. Bot-authored, same-repository PRs containing only generated native locale data @@ -701,6 +701,29 @@ selection. Add `ci:full` before the next PR run to retain ordinary selection for that generated cohort. Labels alone do not trigger extra CI runs. Main and manual validation keep their normal coverage. +The monitor reads the latest completed hourly main run directly from GitHub. +A supported Node Vitest assertion, compiler diagnostic, or hosted lint diagnostic +is advisory only when its file and complete signature match main, and the PR +changes neither the file nor its direct subjects. Subject ownership conservatively +includes the file directory, directories of relative imports, and verified +workspace package directories; unresolved imports retain blocking behavior. +Workflow, tooling, configuration, and dependency changes cannot use this exception. +The main run must descend from the PR merge base, or validate current main itself. +Later main changes to those subjects retire the exception before the next hourly run. +Both matrix siblings and packed test groups finish before the monitor emits an +attempt-bound receipt. The shard runner records complete plan and process counts +after cleanup; the monitor reconciles every inner invocation with its native test +summaries. Compiler graphs and hosted lint stripes likewise require joined native +diagnostics and complete group and step receipts. These runners finish ordinary +diagnostic failures before reporting coverage. Mixed or narrowed lint commands +without that complete contract retain blocking behavior. The aggregate then +annotates **known main red, owned by main**. +Missing or incomplete evidence, suite/import errors, boundary checks, +and every unmatched or PR-owned failure stay blocking, including tiny lint and +type errors. The separate maintainer landing policy owns any explicit exception +for a PR-caused error with an immediate follow-up fix. The failed job remains visible in +Actions even when the aggregate accepts its main-owned assertion. + Repeated head SHAs are duplicate candidates, not reusable validation: the PR merge tree may have changed. Per-PR concurrency coalesces pending work and cancels superseded work; CI does not turn a previous head-only success into skipped tests @@ -839,4 +862,7 @@ artifacts remain errors in report-only mode. - [Install overview](/install) - [Release channels](/install/development-channels) -When exactly one non-control workload remains, the PR failure monitor exits successfully. The aggregate still waits for and validates that workload; retiring the observer does not admit a failed or unfinished job. +With no observed failures, the PR failure monitor exits when the only remaining +workload cannot qualify for a known-main-red exception. The aggregate still waits +for and validates that workload. Eligible final workloads remain observed so a +late inherited failure can receive complete classification evidence. diff --git a/scripts/ci-known-main-red.mjs b/scripts/ci-known-main-red.mjs new file mode 100644 index 000000000000..a428b9a64b76 --- /dev/null +++ b/scripts/ci-known-main-red.mjs @@ -0,0 +1,362 @@ +import { posix } from "node:path"; +import { isNodeTestEvidencePath, parseNodeFailureReport } from "./lib/ci-node-test-evidence.mjs"; +import { isStaticEvidencePath, parseStaticFailureReport } from "./lib/ci-static-check-evidence.mjs"; + +const SHA = /^[a-f0-9]{40}$/u; + +function staticCheck(jobName, stepName, requireComplete = false) { + if ( + (/^check-test-types-core-\d+$/u.test(jobName) && + stepName === "Run hosted core test-types stripe") || + (["check-prod-types", "check-test-types"].includes(jobName) && stepName === "Run check shard") + ) { + return "tsgo"; + } + if ( + requireComplete && + !( + (/^check-lint-core-\d+$/u.test(jobName) && stepName === "Run hosted core lint stripe") || + (/^check-lint-extensions-\d+$/u.test(jobName) && + stepName === "Run hosted extension lint stripe") + ) + ) { + return null; + } + if ( + (jobName === "check-lint" && ["Run check shard", "Run changed lint"].includes(stepName)) || + (/^check-lint-core-\d+$/u.test(jobName) && + ["Run hosted core lint stripe", "Run changed lint"].includes(stepName)) || + (/^check-lint-extensions-\d+$/u.test(jobName) && + ["Run hosted extension lint stripe", "Run changed lint"].includes(stepName)) + ) { + return "oxlint"; + } + return null; +} + +function client({ repository, token }) { + if (repository !== "openclaw/openclaw" || !token) { + throw new Error("Invalid main CI evidence context"); + } + return async (route, text = false) => { + const response = await fetch(`https://api.github.com/repos/${repository}${route}`, { + headers: { Accept: "application/vnd.github+json", Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(10_000), + }); + if (!response.ok) { + throw new Error(`Main CI evidence unavailable: HTTP ${response.status}`); + } + const body = await response.text(); + if (body.length > 16 * 1024 * 1024) { + throw new Error("Main CI evidence exceeds its bound"); + } + return text ? body : JSON.parse(body); + }; +} + +function scheduledRun(run, repository) { + return ( + Number.isSafeInteger(run?.id) && + Number.isSafeInteger(run.run_attempt) && + run.run_attempt > 0 && + run.event === "schedule" && + run.path === ".github/workflows/ci.yml" && + run.head_branch === "main" && + SHA.test(run.head_sha) && + run.repository?.full_name === repository && + run.head_repository?.full_name === repository && + run.status === "completed" + ); +} + +async function latestMainRuns(api, repository) { + const result = await api( + "/actions/workflows/ci.yml/runs?event=schedule&branch=main&status=completed&per_page=100", + ); + if (!Array.isArray(result.workflow_runs)) { + throw new Error("Missing scheduled CI inventory"); + } + // Never substitute a dispatch, a PR artifact, or an older usable red for the + // latest completed hourly run. Unrecognized evidence keeps the PR blocking. + const runs = result.workflow_runs.toSorted((a, b) => b.run_number - a.run_number); + if (runs.some((run) => !scheduledRun(run, repository))) { + throw new Error("Untrusted scheduled CI inventory"); + } + return runs; +} + +async function runJobs(api, run) { + const jobs = new Map(); + for (let page = 1; page <= 4; page++) { + const body = await api( + `/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`, + ); + if ( + !Number.isSafeInteger(body.total_count) || + body.total_count > 400 || + !Array.isArray(body.jobs) + ) { + throw new Error("Incomplete CI job inventory"); + } + for (const job of body.jobs) { + if ( + job.run_id !== run.id || + job.run_attempt !== run.run_attempt || + !Number.isSafeInteger(job.id) + ) { + throw new Error("CI job identity changed"); + } + jobs.set(job.id, job); + } + if (page * 100 >= body.total_count) { + if (jobs.size !== body.total_count) { + throw new Error("Incomplete CI job inventory"); + } + return [...jobs.values()]; + } + } + throw new Error("Oversized CI job inventory"); +} + +async function jobFailures(api, job, requireComplete = true) { + if (job.status !== "completed" || job.conclusion !== "failure" || !Array.isArray(job.steps)) { + return []; + } + const failed = job.steps.filter((step) => step.conclusion === "failure"); + if (failed.length !== 1) { + return []; + } + const nodeTest = failed[0].name === "Run Node test shard" && job.name.startsWith("checks-node-"); + const kind = staticCheck(job.name, failed[0].name, requireComplete); + if (!nodeTest && !kind) { + return []; + } + const log = await api(`/actions/jobs/${job.id}/logs`, true); + return nodeTest + ? parseNodeFailureReport(log, requireComplete) + : parseStaticFailureReport(log, kind, requireComplete); +} + +async function readMainFailures(api, run) { + const jobs = await runJobs(api, run); + const signatures = []; + for (const job of jobs) { + signatures.push(...(await jobFailures(api, job, false))); + } + return signatures; +} + +async function pullFacts(api, options) { + const pull = await api(`/pulls/${options.pullRequestNumber}`); + if ( + pull.state !== "open" || + pull.draft || + pull.head?.sha !== options.headSha || + pull.head?.repo?.full_name !== (options.headRepository ?? options.repository) || + pull.base?.repo?.full_name !== options.repository || + pull.base?.ref !== "main" + ) { + throw new Error("PR identity changed"); + } + const files = []; + for (let page = 1; page <= 30; page++) { + const batch = await api(`/pulls/${options.pullRequestNumber}/files?per_page=100&page=${page}`); + if (!Array.isArray(batch)) { + throw new Error("Missing PR diff"); + } + files.push(...batch); + if (batch.length < 100) { + if (files.length !== pull.changed_files) { + throw new Error("Incomplete PR diff"); + } + return { + pull, + files: files.flatMap((file) => [file.filename, file.previous_filename].filter(Boolean)), + }; + } + } + throw new Error("PR diff exceeds evidence bound"); +} + +async function untouched(api, ref, files, signatures) { + // Changes to the execution environment can forge output or change behavior + // outside a test's direct subjects. They never receive a main-red exception. + if ( + files.some( + (file) => + /^(?:\.github\/|scripts\/|config\/|test\/vitest\/)/u.test(file) || + !file.includes("/") || + /(?:^|\/)(?:package\.json|[^/]*lock[^/]*|tsconfig[^/]*|vitest[^/]*)$/u.test(file), + ) + ) { + return false; + } + const packages = new Map(); + for (const file of new Set(signatures.map((signature) => signature.file))) { + if ( + signatures.some( + (signature) => + signature.file === file && + !(signature.kind === "vitest" + ? isNodeTestEvidencePath(file) + : ["tsgo", "oxlint"].includes(signature.kind) && isStaticEvidencePath(file)), + ) + ) { + return false; + } + const source = await api(`/contents/${file}?ref=${ref}`); + if ( + source.type !== "file" || + source.encoding !== "base64" || + typeof source.content !== "string" + ) { + return false; + } + const text = Buffer.from(source.content, "base64").toString("utf8"); + // Workspace aliases are direct subjects only after their immutable main + // manifest proves package identity. Guard the whole package, including exports. + const subjects = new Set([`${posix.dirname(file)}/`]); + const aliases = new Map(); + for (const match of text.matchAll(/(["'])(@openclaw\/([a-z0-9-]+)(?:\/[^"'\\\s]+)?)\1/gu)) { + const packageName = `@openclaw/${match[3]}`; + const directory = `packages/${match[3]}/`; + if (match[2].split("/").some((part) => part === "." || part === "..")) { + return false; + } + if (!packages.has(packageName)) { + const manifest = await api(`/contents/${directory}package.json?ref=${ref}`); + packages.set( + packageName, + manifest.type === "file" && + manifest.encoding === "base64" && + typeof manifest.content === "string" && + JSON.parse(Buffer.from(manifest.content, "base64").toString("utf8")).name === + packageName, + ); + } + if (!packages.get(packageName)) { + return false; + } + subjects.add(directory); + aliases.set(match[0], `${match[1]}./workspace-subject${match[1]}`); + } + let checkedText = text; + for (const [alias, resolved] of aliases) { + checkedText = checkedText.replaceAll(alias, resolved); + } + // An unresolved alias or computed import has no proven direct subject. + if ( + /["'](?:@[^"'\s]+\/|openclaw\/|#[^"'\s]+)/u.test(checkedText) || + /(?:from\s*|import\s*(?:\()?|require\s*\(|(?:vi\s*\.\s*)?(?:mock|doMock|unmock|doUnmock|importActual|importMock)\s*\()\s*["'](?!\.{1,2}\/|node:|vitest["'])/u.test( + checkedText, + ) || + /(?:import|require|(?:vi\s*\.\s*)?(?:mock|doMock|unmock|doUnmock|importActual|importMock))\s*\(\s*[^"'\s]/u.test( + text, + ) || + /(?:import|require|(?:vi\s*\.\s*)?(?:mock|doMock|unmock|doUnmock|importActual|importMock))\s*\(\s*(?:"[^"\n]*"|'[^'\n]*')\s*[^,\s)]/u.test( + text, + ) || + /(?:from|import|require|(?:vi\s*\.\s*)?(?:mock|doMock|unmock|doUnmock|importActual|importMock))\s*\/[/*]/u.test( + text, + ) + ) { + return false; + } + // A directory guard deliberately over-approximates direct subjects. Local + // imports and mocks can reach sibling owners outside that directory too. + for (const match of text.matchAll(/["'](\.{1,2}\/[^"'\n]+)["']/gu)) { + const resolved = posix.normalize(posix.join(posix.dirname(file), match[1])); + const subject = posix.dirname(resolved); + if (match[1].includes("\\") || resolved.startsWith("../") || subject === ".") { + return false; + } + subjects.add(`${subject}/`); + } + if (files.some((changed) => [...subjects].some((subject) => changed.startsWith(subject)))) { + return false; + } + } + return true; +} + +export function createKnownMainRed(options) { + const api = client(options); + let evidence; + const baseline = async () => { + const [run] = await latestMainRuns(api, options.repository); + if (!run || run.conclusion === "success") { + return null; + } + const { files } = await pullFacts(api, options); + const main = await api("/git/ref/heads/main"); + if (!SHA.test(main.object?.sha)) { + return null; + } + const comparison = await api(`/compare/${main.object.sha}...${options.headSha}`); + const base = comparison.merge_base_commit?.sha; + if (!SHA.test(base)) { + return null; + } + const age = await api(`/compare/${base}...${run.head_sha}`); + if ( + age.status !== "ahead" && + !(age.status === "identical" && run.head_sha === main.object.sha) + ) { + return null; + } + if (run.head_sha !== main.object.sha) { + const since = await api(`/compare/${run.head_sha}...${main.object.sha}`); + // A newer main fix must retire the exemption before the next hourly run. + // Compare returns at most 300 files; a saturated inventory is unproven. + if ( + since.status !== "ahead" || + !Array.isArray(since.files) || + since.files.length >= 300 || + since.files.some((file) => typeof file.filename !== "string") + ) { + return null; + } + files.push( + ...since.files.flatMap((file) => [file.filename, file.previous_filename].filter(Boolean)), + ); + } + const failures = await readMainFailures(api, run); + return { + run, + files, + signatures: new Set(failures.map((entry) => JSON.stringify(entry))), + }; + }; + return { + canClassifyJob(job) { + return ( + job.name.startsWith("checks-node-") || + [ + "Run check shard", + "Run hosted core test-types stripe", + "Run hosted core lint stripe", + "Run hosted extension lint stripe", + ].some((step) => staticCheck(job.name, step, true) !== null) + ); + }, + async classifyJob(job) { + try { + const signatures = await jobFailures(api, job); + if (signatures.length === 0) { + return { known: false, signatures: [] }; + } + evidence ??= baseline(); + const main = await evidence; + if (!main) { + return { known: false, signatures: [] }; + } + const known = + signatures.every((entry) => main.signatures.has(JSON.stringify(entry))) && + (await untouched(api, main.run.head_sha, main.files, signatures)); + return { known, signatures, mainRunId: main.run.id }; + } catch { + return { known: false, signatures: [] }; + } + }, + }; +} diff --git a/scripts/ci-pr-fail-fast.mjs b/scripts/ci-pr-fail-fast.mjs index 6c4403774e4f..268b8973c9ec 100644 --- a/scripts/ci-pr-fail-fast.mjs +++ b/scripts/ci-pr-fail-fast.mjs @@ -1,5 +1,6 @@ import { appendFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; +import { createKnownMainRed } from "./ci-known-main-red.mjs"; const TERMINAL_FAILURES = new Set(["failure", "timed_out"]); const CONTROL_JOBS = new Set(["pr-fail-fast", "openclaw/ci-gate"]); @@ -51,14 +52,16 @@ function plannedCheckJobCount(planners) { } /** - * @param {{repository: string, runId: number, runAttempt: number, + * @param {{repository: string, headRepository?: string, runId: number, runAttempt: number, * pullRequestNumber: number, headSha: string, expectedJobCount: number, * preflightCheckJobCount: number, checkPlanExpected: boolean, token: string, - * recordFailure: (job: {id: number, name: string, runAttempt: number}) => void}} options + * recordFailure: (job: {id: number, name: string, runAttempt: number}) => void, + * recordKnownMainRed?: (jobs: {id: number, mainRunId: number}[]) => void}} options */ export async function monitorPrFailure(options) { const { repository, + headRepository = repository, runId, runAttempt, pullRequestNumber, @@ -70,6 +73,7 @@ export async function monitorPrFailure(options) { } = options; if ( !/^[\w.-]+\/[\w.-]+$/u.test(repository) || + !/^[\w.-]+\/[\w.-]+$/u.test(headRepository) || !/^[a-f0-9]{40}$/u.test(headSha) || ![runId, runAttempt, pullRequestNumber, expectedJobCount].every( (value) => Number.isSafeInteger(value) && value > 0, @@ -95,6 +99,9 @@ export async function monitorPrFailure(options) { const root = `https://api.github.com/repos/${repository}`; /** @param {string} route @param {string} [method] */ const request = async (route, method = "GET") => { + if (method !== "GET" && headRepository !== repository) { + throw new Error("Fork PR failure observation is read-only"); + } const response = await fetch(`${root}${route}`, { method, headers: { Accept: "application/vnd.github+json", Authorization: `Bearer ${token}` }, @@ -114,7 +121,7 @@ export async function monitorPrFailure(options) { run.head_sha === headSha && run.path === ".github/workflows/ci.yml" && record(run.repository).full_name === repository && - record(run.head_repository).full_name === repository; + record(run.head_repository).full_name === headRepository; const initial = await request(runRoute); if (!matchesRun(initial)) { throw new Error("PR cancellation run identity changed"); @@ -122,6 +129,8 @@ export async function monitorPrFailure(options) { if (initial.status === "completed") { return "completed"; } + let mainRed = createKnownMainRed(options); + const knownJobs = new Map(); const workflowId = initial.workflow_id; const runNumber = initial.run_number; const branch = initial.head_branch; @@ -140,8 +149,9 @@ export async function monitorPrFailure(options) { if ( pull.state !== "open" || pull.draft || + pull.auto_merge || head.sha !== headSha || - record(head.repo).full_name !== repository || + record(head.repo).full_name !== headRepository || record(base.repo).full_name !== repository || head.ref !== branch ) { @@ -158,7 +168,7 @@ export async function monitorPrFailure(options) { return ( run.event === "pull_request" && run.workflow_id === workflowId && - record(run.head_repository).full_name === repository && + record(run.head_repository).full_name === headRepository && run.head_branch === branch && typeof run.run_number === "number" && run.run_number > runNumber @@ -173,7 +183,7 @@ export async function monitorPrFailure(options) { }; while (observationDeadline === undefined || Date.now() < observationDeadline) { - /** @type {Map} */ + /** @type {Map} */ const jobs = new Map(); /** @type {Map>} */ const checkPlanners = new Map(); @@ -220,6 +230,7 @@ export async function monitorPrFailure(options) { status: job.status, conclusion: job.conclusion, completedAt: typeof job.completed_at === "string" ? job.completed_at : null, + steps: job.steps, }); } } @@ -228,18 +239,32 @@ export async function monitorPrFailure(options) { } } const rows = [...jobs.values()]; - const failed = rows + const failures = rows .filter( (job) => job.status === "completed" && job.conclusion !== null && TERMINAL_FAILURES.has(job.conclusion), ) - .toSorted((a, b) => (a.completedAt ?? "").localeCompare(b.completedAt ?? ""))[0]; + .toSorted((a, b) => (a.completedAt ?? "").localeCompare(b.completedAt ?? "")); + let failed; + for (const job of failures) { + if (!knownJobs.has(job.id)) { + knownJobs.set(job.id, await mainRed.classifyJob(job)); + } + if (!knownJobs.get(job.id).known) { + failed = job; + break; + } + } if (failed) { if (!(await isCurrent())) { return "superseded"; } + // Fork observation stays read-only even if repository settings grant more scope. + if (headRepository !== repository) { + return "failure-observed"; + } // Record the verified cause before the one write. An accepted request can // lose its response; ci-gate must still fail rather than skip in that case. options.recordFailure({ id: failed.id, name: failed.name, runAttempt }); @@ -255,6 +280,17 @@ export async function monitorPrFailure(options) { if (rows.some((job) => job.conclusion === "cancelled")) { return "externally-cancelled"; } + if ( + rows.some( + (job) => + job.status === "completed" && + job.conclusion !== "success" && + job.conclusion !== "skipped" && + !knownJobs.get(job.id)?.known, + ) + ) { + return "unclassified-result"; + } // Failure observation starts immediately; only clean completion waits for // the installed planner's successful, attempt-bound inventory publication. const checkCount = checkPlanExpected @@ -267,16 +303,36 @@ export async function monitorPrFailure(options) { checkCount === undefined ? undefined : expectedJobCount - preflightCheckJobCount + checkCount; const selectedRows = rows.filter((job) => job.conclusion !== "skipped"); const completedCount = selectedRows.filter((job) => job.status === "completed").length; - if (finalJobCount !== undefined && completedCount >= finalJobCount) { + if ( + finalJobCount !== undefined && + completedCount >= finalJobCount && + (failures.length === 0 || completedCount === selectedRows.length) + ) { + if (failures.length > 0) { + // The final receipt must use the latest completed main run, not an + // exemption cached while the remaining PR jobs were still executing. + mainRed = createKnownMainRed(options); + const receipts = []; + for (const job of failures) { + const decision = await mainRed.classifyJob(job); + if (!decision.known) { + return "main-evidence-changed"; + } + receipts.push({ id: job.id, mainRunId: decision.mainRunId }); + } + options.recordKnownMainRed?.(receipts); + } return "completed"; } if ( + failures.length === 0 && finalJobCount !== undefined && selectedRows.length === finalJobCount && completedCount === finalJobCount - 1 && - selectedRows.length - completedCount === 1 + selectedRows.length - completedCount === 1 && + !selectedRows.some((job) => job.status !== "completed" && mainRed.canClassifyJob(job)) ) { - // The gate still awaits this job; no sibling workload remains to cancel. + // The gate owns an ineligible final job; eligible failures still need observation. return "last-job-remaining"; } // Keep broad observation cheap; an admitted final tail must not add another 30s to CI. @@ -301,6 +357,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) try { const reason = await monitorPrFailure({ repository: process.env.GITHUB_REPOSITORY ?? "", + headRepository: process.env.OPENCLAW_CI_PR_HEAD_REPOSITORY ?? "", runId: Number(process.env.GITHUB_RUN_ID), runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT), pullRequestNumber: Number(process.env.OPENCLAW_CI_PR_NUMBER), @@ -321,7 +378,28 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) ); recordedFailure = true; }, + recordKnownMainRed(jobs) { + appendFileSync( + process.env.GITHUB_OUTPUT, + `known_main_red_attempt=${process.env.GITHUB_RUN_ATTEMPT}\n`, + ); + for (const job of jobs) { + console.log( + `::notice title=known main red, owned by main::PR job ${job.id} matches main run ${job.mainRunId}; all other selected jobs completed.`, + ); + appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + `Known main red, owned by main: PR job ${job.id}, hourly main run ${job.mainRunId}.\n`, + ); + } + }, }); + if (reason === "unclassified-result") { + console.error( + "::error title=Unclassified CI result::A completed job has an unsupported result; inspect this attempt's job conclusions.", + ); + process.exitCode = 1; + } console.log(`PR failure monitor: ${reason}`); } catch (error) { // Observation failures must not turn otherwise healthy CI red. Tests and diff --git a/scripts/ci-static-step.sh b/scripts/ci-static-step.sh new file mode 100644 index 000000000000..6dd9221ccbdd --- /dev/null +++ b/scripts/ci-static-step.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Sourced by trusted CI steps; native check owners still run every command. +ci_static_kind="$1" +case "$ci_static_kind" in + tsgo) ci_static_failure=2 ;; + oxlint) ci_static_failure=1 ;; + *) echo "Unsupported static check: $ci_static_kind" >&2; exit 64 ;; +esac +ci_static_groups=0 +ci_static_exit=0 + +run_static_check() { + local child_exit + ci_static_groups=$((ci_static_groups + 1)) + if "$@"; then + return 0 + else + child_exit=$? + fi + if [ "${OPENCLAW_CI_STATIC_EVIDENCE:-0}" != "1" ] || [ "$child_exit" != "$ci_static_failure" ]; then + exit "$child_exit" + fi + ci_static_exit="$child_exit" +} + +finish_static_checks() { + if [ "${OPENCLAW_CI_STATIC_EVIDENCE:-0}" = "1" ]; then + printf '[ci-static:%s:step] {"version":1,"groups":%s}\n' "$ci_static_kind" "$ci_static_groups" + fi + exit "$ci_static_exit" +} diff --git a/scripts/lib/ci-node-test-evidence.mjs b/scripts/lib/ci-node-test-evidence.mjs new file mode 100644 index 000000000000..9afa8279566f --- /dev/null +++ b/scripts/lib/ci-node-test-evidence.mjs @@ -0,0 +1,224 @@ +import { stripVTControlCharacters } from "node:util"; + +const FILE = /^(?:src|test|extensions|packages|ui)\/[\w./-]+\.(?:test|spec)\.[cm]?[jt]sx?$/u; + +export function isNodeTestEvidencePath(file) { + return typeof file === "string" && FILE.test(file) && !file.includes(".."); +} + +function completeInvocations(rows) { + const receipts = rows.filter((row) => row.stream === "completion"); + if (receipts.length !== 1) { + return false; + } + let receipt; + try { + receipt = JSON.parse(receipts[0].text); + } catch { + return false; + } + if ( + receipt?.version !== 1 || + ![receipt.planned, receipt.completed, receipt.invocations, receipt.failedInvocations].every( + (count) => Number.isSafeInteger(count) && count > 0 && count <= 800, + ) || + receipt.completed !== receipt.planned || + receipt.invocations < receipt.planned || + receipt.failedInvocations > receipt.invocations + ) { + return false; + } + const streams = new Map(); + for (const { stream, text } of rows) { + if (!stream || stream === "completion") { + continue; + } + // Precise targets use the same label for the outer framing and child output. + if (text === "begin" || /^end \(exit \d+\)$/u.test(text)) { + continue; + } + let state = streams.get(stream); + if (/^\[test\] (?:starting |(?:passed|failed|skipped) \d+ Vitest shards?\b)/u.test(text)) { + if (!state) { + state = { invocations: [], terminal: null, trailer: false }; + streams.set(stream, state); + } + } + if (!state) { + // Before a child starts, only native preparation and outer telemetry are + // evidence. An unaccounted stream must not hide an additional failure. + if ( + text === "" || + /^\[test\] preflight test\/vitest\/vitest\.[\w.-]+\.ts$/u.test(text) || + /^\[test\] running \d+ (?:Vitest shards|exact-target plans) with parallelism \d+(?: and joined exclusive barriers)?$/u.test( + text, + ) || + (stream === "resources" && + /^logicalCpuCount=\d+ totalMemoryBytes=\d+ requested plans=\d+ admitted plans=\d+$/u.test( + text, + )) || + (stream === "resource-snapshot" && /^\{"phase":"(?:start|end)",/u.test(text)) || + (stream === "cache" && + /^(?:cloned restored Vitest seed into \d+ isolated lane\(s\)|(?:vitest|node-compile) \d+ -> \d+ bytes; removed \d+ files)$/u.test( + text, + )) || + (stream.startsWith("node-subset:") && + text === "skipped (native shard has no Node-only files)") + ) { + continue; + } + return false; + } + if (state.terminal) { + if (text === "[test] FAILED (exit 1)" && state.terminal.kind === "failed" && !state.trailer) { + state.trailer = true; + } else if (text !== "") { + return false; + } + continue; + } + const current = state.invocations.at(-1); + if ( + current && + (current.files !== undefined || current.tests !== undefined) && + text !== "" && + !/^(?:Test Files|Tests)\s/u.test(text) && + !/^Start at\s+\d{2}:\d{2}:\d{2}$/u.test(text) && + !/^Duration\s+\d+(?:\.\d+)?(?:ms|s)\b/u.test(text) && + !/^\[test\] (?:starting |(?:passed|failed|skipped) \d+ Vitest shards?\b)/u.test(text) && + text !== "[vitest-workers] verifying completed generation before cleanup" && + text !== "[vitest-workers] retained completed compiler outputs for reuse" + ) { + return false; + } + if (text.startsWith("[test] starting ")) { + state.invocations.push({ files: undefined, tests: undefined }); + } + const summary = /^(Test Files|Tests)\s+.+\(\d+\)$/u.exec(text); + if (summary) { + const active = state.invocations.at(-1); + if (!active) { + return false; + } + const key = summary[1] === "Test Files" ? "files" : "tests"; + const failed = /\b[1-9]\d* failed\b/u.test(text); + if (active[key] !== undefined && active[key] !== failed) { + return false; + } + active[key] = failed; + } + const terminal = /^\[test\] (passed|failed|skipped) (\d+) Vitest shards?\b/u.exec(text); + if (terminal) { + state.terminal = { kind: terminal[1], count: Number(terminal[2]) }; + } + } + let failedStreams = 0; + for (const state of streams.values()) { + if ( + !state.terminal || + state.invocations.some( + (invocation) => + typeof invocation.files !== "boolean" || + typeof invocation.tests !== "boolean" || + invocation.files !== invocation.tests, + ) + ) { + return false; + } + const failed = state.invocations.filter((invocation) => invocation.tests).length; + if (state.terminal.kind === "failed") { + if (!state.trailer || failed === 0 || state.terminal.count !== failed) { + return false; + } + failedStreams++; + } else if (failed !== 0 || state.terminal.count !== state.invocations.length) { + return false; + } + } + return streams.size === receipt.invocations && failedStreams === receipt.failedInvocations; +} + +/** Parse native reports; PR acceptance additionally requires complete execution evidence. */ +export function parseNodeFailureReport(log, requireComplete) { + const rows = stripVTControlCharacters(log) + .split("\n") + .map((line) => { + const raw = line.replace(/^\d{4}-\d{2}-\d{2}T\S+\s/u, ""); + const match = /^\[shard:([^\]]+)\]\s*(.*)$/u.exec(raw); + return { stream: match?.[1] ?? "", text: (match?.[2] ?? raw).trim() }; + }); + if (rows.some(({ stream, text }) => !stream && /^(?:\w*Error:|ERR_[A-Z_]+\b)/u.test(text))) { + return []; + } + if (requireComplete && !completeInvocations(rows)) { + return []; + } + const streams = new Map(); + for (const row of rows) { + const lines = streams.get(row.stream) ?? []; + lines.push(row.text); + streams.set(row.stream, lines); + } + const signatures = []; + const assertionMessages = new Set(); + let failedCount = 0; + let failureHeaders = 0; + for (const lines of streams.values()) { + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; + if (/^(?:\w*Error:|ERR_[A-Z_]+\b)/u.test(line) && !/^FAIL\s/u.test(lines[index - 1] ?? "")) { + return []; + } + if ( + /Unhandled (?:Error|Rejection)|Failed Suites|Worker exited|heap out of memory|Segmentation fault|failed to spawn|error TS\d+/u.test( + line, + ) + ) { + return []; + } + const summary = /^Tests\s+(\d+) failed(?:\s|$)/u.exec(line); + if (summary) { + failedCount += Number(summary[1]); + } + if (!/^FAIL\s/u.test(line)) { + continue; + } + failureHeaders++; + const failure = /^FAIL\s+(?:\S+\s+)?(\S+) > (.+)$/u.exec(line); + const message = lines[index + 1]; + if ( + !failure || + !isNodeTestEvidencePath(failure[1]) || + !/^(?:AssertionError|Error|TypeError|RangeError): .+/u.test(message ?? "") + ) { + return []; + } + let end = index + 2; + while ( + end < lines.length && + !/^(?:FAIL\s|Test Files\s|Tests\s|\[test\]|⎯+(?:\[\d+\/\d+\])?⎯*$)/u.test(lines[end]) + ) { + end++; + } + const assertion = lines + .slice(index + 1, end) + .join("\n") + .trim(); + assertionMessages.add(message); + signatures.push({ kind: "vitest", file: failure[1], test: `${failure[2]} :: ${assertion}` }); + } + } + if ( + rows.some( + ({ text }) => + text.startsWith("##[error]") && + text !== "##[error]Process completed with exit code 1." && + !assertionMessages.has(text.slice(9)), + ) + ) { + return []; + } + return failedCount > 0 && failureHeaders === failedCount && signatures.length === failedCount + ? signatures + : []; +} diff --git a/test/scripts/ci-known-main-red.test.ts b/test/scripts/ci-known-main-red.test.ts new file mode 100644 index 000000000000..ee1976925979 --- /dev/null +++ b/test/scripts/ci-known-main-red.test.ts @@ -0,0 +1,544 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createKnownMainRed } from "../../scripts/ci-known-main-red.mjs"; + +const repository = "openclaw/openclaw"; +const headSha = "a".repeat(40); +const mainSha = "b".repeat(40); +const baseSha = "c".repeat(40); +const file = "src/gateway/example.test.ts"; +const title = "startup > recovers the session"; +const report = (test = title) => ` +2026-09-27T19:34:09.0000000Z [shard:gateway] begin +2026-09-27T19:34:10.0000000Z [shard:gateway] [test] starting test/vitest/vitest.gateway.config.ts +2026-09-27T19:34:16.6951018Z [shard:gateway] FAIL gateway ${file} > ${test} +2026-09-27T19:34:16.6951669Z [shard:gateway] AssertionError: expected true to be false +2026-09-27T19:34:16.6981000Z [shard:gateway] Test Files 1 failed (1) +2026-09-27T19:34:16.6981653Z [shard:gateway] Tests 1 failed | 2 passed (3) +2026-09-27T19:34:17.0000000Z [shard:gateway] [test] failed 1 Vitest shard in 7s +2026-09-27T19:34:17.1000000Z [shard:gateway] [test] FAILED (exit 1) +2026-09-27T19:34:17.1500000Z [shard:gateway] end (exit 1) +2026-09-27T19:34:17.2000000Z [shard:completion] {"version":1,"planned":1,"completed":1,"invocations":1,"failedInvocations":1} +2026-09-27T19:34:16.7040403Z ##[error]AssertionError: expected true to be false +2026-09-27T19:34:18.6571653Z ##[error]Process completed with exit code 1. +`; +const mainRun = { + id: 200, + run_attempt: 1, + run_number: 100, + event: "schedule", + path: ".github/workflows/ci.yml", + head_branch: "main", + head_sha: mainSha, + repository: { full_name: repository }, + head_repository: { full_name: repository }, + status: "completed", + conclusion: "failure", +}; +const job = { + id: 10, + run_id: 100, + run_attempt: 1, + name: "checks-node-compact-small-1", + status: "completed", + conclusion: "failure", + steps: [{ name: "Run Node test shard", conclusion: "failure" }], +}; +const typeFile = "src/acp/control-plane/manager.preactive-cancellation.test.ts"; +const typeDiagnostic = `${typeFile}(92,14): error TS2367: This comparison appears to be unintentional.`; +const typeReport = (diagnostic = typeDiagnostic) => + [ + "##[group]Run node scripts/run-tsgo-core-test-shards.mjs", + "##[endgroup]", + diagnostic, + `[ci-static:tsgo:leaf] ${JSON.stringify({ + version: 1, + id: "batch:0", + config: "test/tsconfig/tsconfig.core.test.agents-root.json", + exitCode: 2, + stdout: `${diagnostic}\n`, + stderr: "", + })}`, + '[ci-static:tsgo:completion] {"version":1,"id":"batch","planned":1,"completed":1,"leaves":["batch:0"]}', + '[ci-static:tsgo:step] {"version":1,"groups":1}', + "##[error]Process completed with exit code 2.", + ].join("\n"); +const typeJob = { + ...job, + name: "check-test-types-core-5", + steps: [{ name: "Run hosted core test-types stripe", conclusion: "failure" }], +}; +const lintFile = "extensions/workboard/browser/lib/workboard/card-alerts.ts"; +const lintReport = `##[group]Run node scripts/run-oxlint.mjs +##[endgroup] +${lintFile}:1:10: error: unused import (eslint(no-unused-vars)) +Found 0 warnings and 1 error. +##[error]Process completed with exit code 1.`; +const lintJob = { + ...job, + name: "check-lint-extensions-3", + steps: [{ name: "Run hosted extension lint stripe", conclusion: "failure" }], +}; +const completeLintReport = (message = "unused import") => { + const diagnostic = `${lintFile}:1:10: error: ${message} (eslint(no-unused-vars))\nFound 0 warnings and 1 error.`; + return [ + "##[group]Run node scripts/run-oxlint.mjs", + "##[endgroup]", + diagnostic, + `[ci-static:oxlint:leaf] ${JSON.stringify({ + version: 1, + id: "batch:0", + config: "config/oxlint/typed.json", + exitCode: 1, + stdout: `${diagnostic}\n`, + stderr: "", + })}`, + '[ci-static:oxlint:completion] {"version":1,"id":"batch","planned":1,"completed":1,"leaves":["batch:0"]}', + '[ci-static:oxlint:step] {"version":1,"groups":1}', + "##[error]Process completed with exit code 1.", + ].join("\n"); +}; + +function fixture( + options: { + changed?: string[]; + headRepository?: string; + mainReport?: string; + prReport?: string; + mainEvent?: string; + age?: string; + source?: string; + changedCount?: number; + mainConclusion?: string; + liveMainSha?: string; + mainChanged?: string[]; + failureJob?: typeof job; + signatureFile?: string; + packageNames?: Record; + } = {}, +) { + const changed = options.changed ?? ["src/channels/unrelated.ts"]; + const failureJob = options.failureJob ?? job; + const signatureFile = options.signatureFile ?? file; + const api = vi.fn(async (url: string) => { + const path = new URL(url).pathname.replace(`/repos/${repository}`, ""); + let body: unknown; + if (path === "/actions/workflows/ci.yml/runs") { + body = { + workflow_runs: [ + { + ...mainRun, + event: options.mainEvent ?? "schedule", + conclusion: options.mainConclusion ?? "failure", + }, + ], + }; + } else if (path === "/pulls/7") { + body = { + state: "open", + draft: false, + changed_files: options.changedCount ?? changed.length, + head: { sha: headSha, repo: { full_name: options.headRepository ?? repository } }, + base: { ref: "main", repo: { full_name: repository } }, + }; + } else if (path === "/pulls/7/files") { + body = changed.map((filename) => ({ filename })); + } else if (path === "/git/ref/heads/main") { + body = { object: { sha: options.liveMainSha ?? mainSha } }; + } else if (path === `/compare/${options.liveMainSha ?? mainSha}...${headSha}`) { + body = { merge_base_commit: { sha: baseSha } }; + } else if (path === `/compare/${baseSha}...${mainSha}`) { + body = { status: options.age ?? "ahead" }; + } else if (options.liveMainSha && path === `/compare/${mainSha}...${options.liveMainSha}`) { + body = { + status: "ahead", + files: (options.mainChanged ?? []).map((filename) => ({ filename })), + }; + } else if (path === "/actions/runs/200/attempts/1/jobs") { + body = { total_count: 1, jobs: [{ ...failureJob, id: 20, run_id: 200 }] }; + } else if (path === "/actions/jobs/20/logs") { + return new Response(options.mainReport ?? report()); + } else if (path === "/actions/jobs/10/logs") { + return new Response(options.prReport ?? report()); + } else if (path === `/contents/${signatureFile}` || path.startsWith("/contents/packages/")) { + if (new URL(url).searchParams.get("ref") !== mainSha) { + throw new Error("Subject source must use the immutable main evidence revision"); + } + body = { + type: "file", + encoding: "base64", + content: Buffer.from( + path.endsWith("/package.json") + ? JSON.stringify({ name: options.packageNames?.[path.split("/")[3]!] }) + : (options.source ?? + 'import { it } from "vitest"; import { start } from "./subject.js";'), + ).toString("base64"), + }; + } else { + throw new Error(`Unexpected evidence request ${path}`); + } + return new Response(JSON.stringify(body)); + }); + vi.stubGlobal("fetch", api); + return { + classify: () => + createKnownMainRed({ + repository, + headRepository: options.headRepository, + token: "synthetic-token", + headSha, + pullRequestNumber: 7, + runId: 100, + runAttempt: 1, + }).classifyJob(failureJob), + }; +} + +afterEach(() => vi.unstubAllGlobals()); + +describe("known hourly main failures", () => { + it("distinguishes different assertion details after the same headline", async () => { + const detailed = report().replace( + "Test Files 1 failed", + "- Expected\n[shard:gateway] + Received\n[shard:gateway] - { value: 1 }\n[shard:gateway] + { value: 2 }\n[shard:gateway] Test Files 1 failed", + ); + expect((await fixture({ mainReport: detailed, prReport: detailed }).classify()).known).toBe( + true, + ); + expect( + ( + await fixture({ + mainReport: detailed, + prReport: detailed.replace("+ { value: 2 }", "+ { value: 3 }"), + }).classify() + ).known, + ).toBe(false); + }); + + it.each([repository, "contributor/openclaw"])( + "accepts the exact file, full test title and assertion from trusted main for %s", + async (headRepository) => { + expect(await fixture({ headRepository }).classify()).toMatchObject({ + known: true, + mainRunId: 200, + }); + }, + ); + + it.each([ + ["different test", { prReport: report("startup > opens a different session") }], + ["different assertion", { prReport: report().replaceAll("true to be false", "42 to be 43") }], + ["changed test", { changed: [file] }], + ["changed direct subject", { changed: ["src/gateway/subject.ts"] }], + [ + "changed external subject", + { changed: ["src/infra/session.ts"], source: 'import { start } from "../infra/session.js";' }, + ], + ["changed workflow", { changed: [".github/workflows/ci.yml"] }], + ["changed dependency", { changed: ["pnpm-lock.yaml"] }], + ["truncated diff", { changedCount: 2 }], + ["non-main event", { mainEvent: "pull_request" }], + ["main predates merge base", { age: "behind" }], + ["unresolved subject alias", { source: 'import { start } from "@openclaw/runtime";' }], + ["unresolved side-effect alias", { source: 'import "@openclaw/runtime";' }], + ["unresolved mock alias", { source: 'vi.mock("@openclaw/runtime", () => ({}));' }], + ["unresolved dynamic mock alias", { source: 'vi.doMock("@openclaw/runtime", () => ({}));' }], + ["unresolved actual-import alias", { source: 'await vi.importActual("@openclaw/runtime");' }], + ["unresolved mock-import alias", { source: 'await vi.importMock("@openclaw/runtime");' }], + ["unresolved bracketed mock alias", { source: 'vi["mock"]("@openclaw/runtime");' }], + [ + "renamed module mock", + { source: 'const {mock: replace} = vi; replace("@openclaw/runtime");' }, + ], + ["escaped relative module", { source: 'await import("./\\u002e\\u002e/subject.js");' }], + ["computed mock subject", { source: "vi.mock(moduleName);" }], + ["concatenated import subject", { source: 'await import("./subject" + suffix);' }], + ["concatenated mock subject", { source: 'vi.mock("./subject" + suffix);' }], + ["template subject", { source: "await import(`./subject${suffix}`);" }], + ["comment-separated alias", { source: 'import /* subject */ "@openclaw/runtime";' }], + [ + "root directory subject", + { changed: ["runtime/index.ts"], source: 'import { start } from "../../runtime";' }, + ], + ["computed subject", { source: "const subject = await import(name);" }], + ["unreported failure", { prReport: report().replace("Tests 1 failed", "Tests 2 failed") }], + ["missing test summary", { prReport: report().replace(/Tests 1 failed[^\n]+/u, "") }], + ["failed suite", { prReport: `${report()}\nFailed Suites 1` }], + ["unhandled rejection", { prReport: `${report()}\nUnhandled Rejection` }], + ["extra failure annotation", { prReport: `${report()}\n##[error]worker crashed` }], + [ + "missing execution receipt", + { prReport: report().replace(/^.*\[shard:completion\].*\n/mu, "") }, + ], + ["unfinished outer plan", { prReport: report().replace('"planned":1', '"planned":2') }], + ["duplicate execution receipt", { prReport: `${report()}\n[shard:completion] {"version":1}` }], + [ + "unknown stream before summary", + { prReport: `[shard:coverage] Error: setup failed\n${report()}` }, + ], + [ + "unknown stream without error header", + { prReport: `[shard:coverage] setup failed\n${report()}` }, + ], + [ + "failure before invocation start", + { prReport: `[shard:gateway] Error: setup failed\n${report()}` }, + ], + [ + "failure before first summary", + { + prReport: report().replace( + "[shard:gateway] Test Files", + "[shard:gateway] Error: coverage setup failed\n[shard:gateway] Test Files", + ), + }, + ], + [ + "unknown later config", + { + prReport: report().replace( + "[test] failed 1 Vitest shard in 7s", + "[test] starting test/vitest/vitest.process.config.ts\n[shard:gateway] failed to load config\n[shard:gateway] [test] failed 2 Vitest shards in 7s", + ), + }, + ], + [ + "unknown second child", + { + prReport: + report().replace( + '"invocations":1,"failedInvocations":1', + '"invocations":2,"failedInvocations":2', + ) + + "\n[shard:other] ERR_PNPM_MISSING_SCRIPT\n[shard:other] [test] failed 0 Vitest shards in 1s\n[shard:other] [test] FAILED (exit 1)", + }, + ], + [ + "unreported failed config", + { prReport: report().replace("failed 1 Vitest shard in", "failed 2 Vitest shards in") }, + ], + [ + "failure after invocation terminal", + { + prReport: report().replace( + "[test] FAILED (exit 1)", + "Error: report close failed\n[shard:gateway] [test] FAILED (exit 1)", + ), + }, + ], + [ + "failure during summaries", + { + prReport: report().replace( + "[shard:gateway] Tests 1 failed", + "[shard:gateway] Error: coverage finalization failed\n[shard:gateway] Tests 1 failed", + ), + }, + ], + [ + "failure after summaries", + { + prReport: report().replace( + "[test] failed 1 Vitest shard in 7s", + "Error: report finalization failed\n[shard:gateway] [test] failed 1 Vitest shard in 7s", + ), + }, + ], + ["outer failure after receipt", { prReport: `${report()}\nError: write failed` }], + ["main has recovered", { mainConclusion: "success" }], + ])("keeps %s blocking", async (_name, options) => { + expect((await fixture(options).classify()).known).toBe(false); + }); + + it("can identify historical main assertions without accepting incomplete PR execution", async () => { + expect( + ( + await fixture({ + mainReport: report().replace(/^.*\[shard:completion\].*\n/mu, ""), + }).classify() + ).known, + ).toBe(true); + }); + + it.each([ + { mainChanged: ["docs/unrelated.md"], known: true }, + { mainChanged: [file], known: false }, + { mainChanged: ["src/gateway/subject.ts"], known: false }, + { mainChanged: ["scripts/run-vitest.mts"], known: false }, + { mainChanged: Array.from({ length: 300 }, (_, i) => `docs/page-${i}.md`), known: false }, + ])( + "retires stale main evidence when later main changes its subjects: %j", + async ({ mainChanged, known }) => { + expect((await fixture({ liveMainSha: "d".repeat(40), mainChanged }).classify()).known).toBe( + known, + ); + }, + ); + + it.each(["acp-core/runtime/types", "normalization-core/record-coerce"])( + "guards the verified workspace package for %s", + async (subject) => { + const name = subject.split("/")[0]!; + const options = { + source: `import { subject } from "@openclaw/${subject}";`, + packageNames: { [name]: `@openclaw/${name}` }, + }; + expect((await fixture(options).classify()).known).toBe(true); + expect( + (await fixture({ ...options, changed: [`packages/${name}/src/other.ts`] }).classify()) + .known, + ).toBe(false); + expect( + ( + await fixture({ + ...options, + changed: ["packages/unrelated/src/other.ts"], + }).classify() + ).known, + ).toBe(true); + expect( + (await fixture({ ...options, packageNames: { [name]: "external-package" } }).classify()) + .known, + ).toBe(false); + }, + ); + + it("rejects unknown failures in legacy main assertion reports", async () => { + const mainReport = report().replace(/^.*\[shard:completion\].*\n/mu, ""); + expect( + (await fixture({ mainReport: `${mainReport}\nError: another unknown failure` }).classify()) + .known, + ).toBe(false); + }); + + it.each([ + typeJob, + { + ...typeJob, + name: "check-prod-types", + steps: [{ name: "Run check shard", conclusion: "failure" }], + }, + { + ...typeJob, + name: "check-test-types", + steps: [{ name: "Run check shard", conclusion: "failure" }], + }, + ])("tolerates only exact complete type diagnostics for $name", async (failureJob) => { + const options = { + failureJob, + signatureFile: typeFile, + mainReport: typeReport(), + prReport: typeReport(), + source: 'import type { Event } from "@openclaw/acp-core/runtime/types";', + packageNames: { "acp-core": "@openclaw/acp-core" }, + }; + expect((await fixture(options).classify()).known).toBe(true); + expect( + ( + await fixture({ + ...options, + prReport: typeReport(typeDiagnostic.replace("TS2367", "TS2554")), + }).classify() + ).known, + ).toBe(false); + expect((await fixture({ ...options, changed: [typeFile] }).classify()).known).toBe(false); + expect( + ( + await fixture({ + ...options, + changed: ["packages/acp-core/src/runtime/types.ts"], + }).classify() + ).known, + ).toBe(false); + }); + + it.each([ + { ...typeJob, name: "other-check" }, + { ...typeJob, steps: [{ name: "Prepare workspace", conclusion: "failure" }] }, + { ...typeJob, steps: [...typeJob.steps, { name: "Cleanup", conclusion: "failure" }] }, + ])("keeps unknown static job or failed step ownership blocking: %j", async (failureJob) => { + const f = fixture({ + failureJob, + signatureFile: typeFile, + mainReport: typeReport(), + prReport: typeReport(), + }); + expect((await f.classify()).known).toBe(false); + }); + + it("accepts complete type reports against legacy main evidence, but never legacy PR evidence", async () => { + const legacy = `##[group]Run typecheck\n##[endgroup]\n${typeDiagnostic}\n##[error]Process completed with exit code 2.`; + const options = { failureJob: typeJob, signatureFile: typeFile, mainReport: legacy }; + expect((await fixture({ ...options, prReport: typeReport() }).classify()).known).toBe(true); + expect((await fixture({ ...options, prReport: legacy }).classify()).known).toBe(false); + }); + + it.each([ + lintJob, + { + ...lintJob, + name: "check-lint-core-1", + steps: [{ name: "Run hosted core lint stripe", conclusion: "failure" }], + }, + ])("requires complete matching diagnostics for hosted $name tolerance", async (failureJob) => { + const options = { + failureJob, + signatureFile: lintFile, + mainReport: lintReport, + prReport: completeLintReport(), + }; + expect((await fixture(options).classify()).known).toBe(true); + expect( + (await fixture({ ...options, prReport: completeLintReport("unused variable") }).classify()) + .known, + ).toBe(false); + expect((await fixture({ ...options, changed: [lintFile] }).classify()).known).toBe(false); + }); + + it.each([ + { ...lintJob, name: "check-lint", steps: [{ name: "Run check shard", conclusion: "failure" }] }, + { ...lintJob, steps: [{ name: "Run changed lint", conclusion: "failure" }] }, + ])("keeps mixed lint execution scopes blocking even with a receipt: %j", async (failureJob) => { + expect( + ( + await fixture({ + failureJob, + signatureFile: lintFile, + mainReport: lintReport, + prReport: completeLintReport(), + }).classify() + ).known, + ).toBe(false); + }); + + it("keeps incomplete lint execution and PR-owned static subjects blocking", async () => { + const options = { + failureJob: lintJob, + signatureFile: lintFile, + mainReport: lintReport, + prReport: completeLintReport(), + source: 'import type { CardSessionState } from "./session-state.ts";', + }; + expect((await fixture({ ...options, prReport: lintReport }).classify()).known).toBe(false); + expect((await fixture(options).classify()).known).toBe(true); + expect((await fixture({ ...options, changed: [lintFile] }).classify()).known).toBe(false); + expect( + ( + await fixture({ + ...options, + changed: ["extensions/workboard/browser/lib/workboard/session-state.ts"], + }).classify() + ).known, + ).toBe(false); + expect( + ( + await fixture({ + ...options, + mainReport: lintReport.replace( + "Found 0 warnings", + "Error: unknown lint failure\nFound 0 warnings", + ), + }).classify() + ).known, + ).toBe(false); + }); +}); diff --git a/test/scripts/ci-pr-fail-fast.test.ts b/test/scripts/ci-pr-fail-fast.test.ts index fb191cb2bbb0..98b629551624 100644 --- a/test/scripts/ci-pr-fail-fast.test.ts +++ b/test/scripts/ci-pr-fail-fast.test.ts @@ -24,8 +24,10 @@ const run = { const pull = { state: "open", draft: false, + auto_merge: null as object | null, + changed_files: 1, head: { sha: headSha, ref: "fixture", repo: { full_name: repository } }, - base: { repo: { full_name: repository } }, + base: { ref: "main", repo: { full_name: repository } }, }; const job = (id: number, conclusion: string | null = "success", name = `row-${id}`) => ({ id, @@ -67,6 +69,7 @@ function plannedChecks(count: number): Job { function fixture( options: { jobs?: Job[]; + headRepository?: string; preflightCheckJobCount?: number; checkPlanExpected?: boolean; currentPull?: typeof pull; @@ -75,6 +78,7 @@ function fixture( recentRuns?: (typeof run)[]; postError?: boolean; monitorStartedAt?: string | null; + evidenceRoutes?: Record; } = {}, ) { let runReads = 0; @@ -111,6 +115,11 @@ function fixture( } else if (route === "/actions/runs/100/attempts/1/jobs") { const page = Number(new URL(url).searchParams.get("page")); body = { total_count: rows.length, jobs: apiRows().slice((page - 1) * 100, page * 100) }; + } else if (route in (options.evidenceRoutes ?? {})) { + body = options.evidenceRoutes![route]; + if (typeof body === "string") { + return new Response(body); + } } else { throw new Error(`Unexpected API route: ${route}`); } @@ -120,14 +129,17 @@ function fixture( const recordFailure = vi.fn((failed: { id: number; name: string }) => { events.push(`cause ${failed.id}`); }); + const recordKnownMainRed = vi.fn(); return { events, rows, fetchMock, recordFailure, + recordKnownMainRed, monitor: (expectedJobCount = 4, runAttempt = 1) => monitorPrFailure({ repository, + headRepository: options.headRepository, headSha, runId: 100, runAttempt, @@ -137,6 +149,7 @@ function fixture( checkPlanExpected: options.checkPlanExpected ?? false, token: "synthetic-test-token", recordFailure, + recordKnownMainRed, }), }; } @@ -240,6 +253,100 @@ describe("PR failure monitor", () => { expect(f.events).toEqual(["cause 3", "POST /actions/runs/100/cancel"]); }); + it.each( + [repository, "contributor/openclaw"].flatMap((headRepository) => + [3, 4] + .flatMap((expectedJobs) => + ["success", "cancelled", "neutral", "action_required", "stale"].map((result) => ({ + headRepository, + expectedJobs, + result, + late: false, + })), + ) + .concat({ headRepository, expectedJobs: 4, result: "success", late: true }), + ), + )( + "publishes complete main-red evidence for $headRepository only after a successful sibling ($result, declared=$expectedJobs, late=$late)", + async ({ headRepository, result, expectedJobs, late }) => { + vi.useFakeTimers(); + const mainSha = "b".repeat(40); + const baseSha = "c".repeat(40); + const file = "src/gateway/example.test.ts"; + const log = `[shard:gateway] [test] starting test/vitest/vitest.gateway.config.ts +[shard:gateway] FAIL gateway ${file} > startup > recovers +[shard:gateway] AssertionError: expected true to be false +[shard:gateway] Test Files 1 failed (1) +[shard:gateway] Tests 1 failed | 2 passed (3) +[shard:gateway] [test] failed 1 Vitest shard in 1s +[shard:gateway] [test] FAILED (exit 1) +[shard:completion] {"version":1,"planned":1,"completed":1,"invocations":1,"failedInvocations":1}`; + const failed = { + ...job(3, late ? null : "failure", "checks-node-compact-small-1"), + steps: [{ name: "Run Node test shard", conclusion: "failure" }], + }; + const mainRun = { + ...run, + id: 200, + head_branch: "main", + head_sha: mainSha, + status: "completed", + conclusion: "failure", + event: "schedule", + }; + const sibling = job(4, late ? "success" : null); + const f = fixture({ + headRepository, + currentRun: { ...run, head_repository: { full_name: headRepository } }, + currentPull: { ...pull, head: { ...pull.head, repo: { full_name: headRepository } } }, + jobs: [job(1), job(2), failed, sibling], + evidenceRoutes: { + "/actions/workflows/ci.yml/runs": { workflow_runs: [mainRun] }, + "/pulls/7/files": [{ filename: "src/channels/unrelated.ts" }], + "/git/ref/heads/main": { object: { sha: mainSha } }, + [`/compare/${mainSha}...${headSha}`]: { merge_base_commit: { sha: baseSha } }, + [`/compare/${baseSha}...${mainSha}`]: { status: "ahead" }, + "/actions/runs/200/attempts/1/jobs": { + total_count: 1, + jobs: [{ ...failed, id: 20, run_id: 200, status: "completed", conclusion: "failure" }], + }, + "/actions/jobs/20/logs": log, + "/actions/jobs/3/logs": log, + [`/contents/${file}`]: { + type: "file", + encoding: "base64", + content: Buffer.from('import "./subject.js"').toString("base64"), + }, + }, + }); + const running = f.monitor(expectedJobs); + await vi.advanceTimersByTimeAsync(0); + expect(f.events).toEqual([]); + expect(f.recordKnownMainRed).not.toHaveBeenCalled(); + Object.assign(failed, job(3, "failure", "checks-node-compact-small-1")); + Object.assign(sibling, job(4, result)); + await vi.advanceTimersByTimeAsync(30_000); + expect(await running).toBe( + result === "success" + ? "completed" + : result === "cancelled" + ? "externally-cancelled" + : "unclassified-result", + ); + expect(f.events).toEqual([]); + if (result === "success") { + expect(f.recordKnownMainRed).toHaveBeenCalledWith([{ id: 3, mainRunId: 200 }]); + } else { + expect(f.recordKnownMainRed).not.toHaveBeenCalled(); + } + }, + ); + + it("never cancels a PR with auto-merge enabled", async () => { + const f = fixture({ currentPull: { ...pull, auto_merge: {} } }); + expect(await f.monitor()).toBe("superseded"); + expect(f.events).toEqual([]); + }); it("replaces the early check reservation with the completed planner's exact count", async () => { vi.useFakeTimers(); const f = fixture({ @@ -410,7 +517,7 @@ describe("PR failure monitor", () => { expect(f.events).toEqual([]); }); - it("never gives a fork run cancellation authority", async () => { + it("rejects an unexpected fork repository before observing its run", async () => { const f = fixture({ currentRun: { ...run, head_repository: { full_name: "contributor/openclaw" } }, }); @@ -418,6 +525,30 @@ describe("PR failure monitor", () => { expect(f.events).toEqual([]); }); + it("observes an unknown fork failure without requesting cancellation or recording a cancel cause", async () => { + const headRepository = "contributor/openclaw"; + const f = fixture({ + headRepository, + currentRun: { ...run, head_repository: { full_name: headRepository } }, + currentPull: { ...pull, head: { ...pull.head, repo: { full_name: headRepository } } }, + }); + expect(await f.monitor()).toBe("failure-observed"); + expect(f.events).toEqual([]); + expect(f.recordFailure).not.toHaveBeenCalled(); + expect(f.recordKnownMainRed).not.toHaveBeenCalled(); + }); + + it("rejects changed live fork ownership without writing", async () => { + const headRepository = "contributor/openclaw"; + const f = fixture({ + headRepository, + currentRun: { ...run, head_repository: { full_name: headRepository } }, + currentPull: { ...pull, head: { ...pull.head, repo: { full_name: "another/openclaw" } } }, + }); + expect(await f.monitor()).toBe("superseded"); + expect(f.events).toEqual([]); + }); + it.each(["new head", "draft", "closed", "same-head newer run", "new attempt"])( "preserves superseding work (%s)", async (change) => { diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 9cd1564d5935..84e7ca42a781 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -9260,6 +9260,11 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" stripeCount?: number; }) => { const root = tempDirs.make("openclaw-hosted-lint-owner-"); + mkdirSync(path.join(root, ".ci-harness/scripts"), { recursive: true }); + copyFileSync( + new URL("../../scripts/ci-static-step.sh", import.meta.url), + path.join(root, ".ci-harness/scripts/ci-static-step.sh"), + ); const binDir = path.join(root, "bin"); const callsPath = path.join(root, "calls.txt"); const goEnvPath = path.join(root, "go-env.txt"); @@ -9329,6 +9334,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" HOSTED_RUNNER_STRIPES: profile === "blacksmith" ? "false" : "true", LINT_CALLS: callsPath, LINT_GO_ENV: goEnvPath, + OPENCLAW_CI_STATIC_EVIDENCE: "0", OPENCLAW_LOCAL_CHECK: "0", PATH: `${binDir}:${process.env.PATH ?? ""}`, RELEASE_GATE: String( @@ -9586,6 +9592,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" CI_TYPE_GRAPHS_JSON: "", CI_CORE_TYPE_GRAPHS_JSON: "", CI_CORE_TYPE_CONCURRENCY: "", + OPENCLAW_CI_STATIC_EVIDENCE: "0", }, }); expect(report.code, report.output).toBe(0); diff --git a/test/scripts/ci-workflow-planning.test.ts b/test/scripts/ci-workflow-planning.test.ts index 3ce5822cf7ed..884c0a78c9a8 100644 --- a/test/scripts/ci-workflow-planning.test.ts +++ b/test/scripts/ci-workflow-planning.test.ts @@ -2614,6 +2614,37 @@ describe("ci workflow guards", () => { }); } + it("reserves the fork observer before admitting optional hosted rows", () => { + const eventName = "pull_request" as const; + const changedPaths = [".github/workflows/ci.yml"]; + const baseline = manifestWithHostedNodeRows(1, { eventName, changedPaths }); + expect(baseline.status, baseline.output).toBe(0); + const nodeRows = 1 + 40 - Number(baseline.outputs.hybrid_hosted_base_rows); + expect(nodeRows).toBeGreaterThan(0); + const sameRepository = manifestWithHostedNodeRows(nodeRows, { eventName, changedPaths }); + const fork = manifestWithHostedNodeRows(nodeRows, { + eventName, + changedPaths, + scopeEnv: { OPENCLAW_CI_HEAD_REPOSITORY: "contributor/openclaw" }, + }); + expect(sameRepository.status, sameRepository.output).toBe(0); + expect(fork.status, fork.output).toBe(0); + expect(Number(sameRepository.outputs.hybrid_hosted_base_rows)).toBe(40); + const sameRows = emittedHostedRows(sameRepository.outputs, { eventName }); + const forkRows = emittedHostedRows(fork.outputs, { + eventName, + headRepository: "contributor/openclaw", + }); + const hostedControls = ["check-plan", "checks-baseline-ratchets"].filter( + (name) => forkRows.includes(name) && !sameRows.includes(name), + ).length; + expect(Number(fork.outputs.hybrid_hosted_base_rows)).toBe(40 + hostedControls + 1); + expect(sameRepository.outputs.hybrid_hosted_offload).toBe("true"); + expect(fork.outputs.hybrid_hosted_offload).toBe("false"); + expect(Number(fork.outputs.hybrid_hosted_total_rows)).toBeLessThanOrEqual(45); + expect(forkRows).toContain("pr-fail-fast"); + }); + it.each([true, false])("bounds hosted rows with Android=%s", (androidSelected) => { const planner = readCiWorkflow().jobs.preflight.steps.find( (step: WorkflowStep) => step.name === "Build CI manifest", diff --git a/test/scripts/ci-workflow-pr-control.test.ts b/test/scripts/ci-workflow-pr-control.test.ts index 0cea40bf74f0..989a75c331c7 100644 --- a/test/scripts/ci-workflow-pr-control.test.ts +++ b/test/scripts/ci-workflow-pr-control.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { readFileSync } from "node:fs"; +import { copyFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; @@ -132,7 +132,7 @@ describe("PR failure cancellation", () => { }, ); - it("limits cancellation authority to the same-repository PR monitor", () => { + it("uses the existing monitor grants for canonical PR observation including forks", () => { const workflow = readCiWorkflow(); expect( Object.entries(workflow.jobs) @@ -144,7 +144,7 @@ describe("PR failure cancellation", () => { ).toEqual(["pr-fail-fast"]); for (const [eventName, headRepository, admitted] of [ ["pull_request", "openclaw/openclaw", true], - ["pull_request", "contributor/openclaw", false], + ["pull_request", "contributor/openclaw", true], ["push", "openclaw/openclaw", false], ["workflow_dispatch", "openclaw/openclaw", false], ] as const) { @@ -181,19 +181,175 @@ describe("PR failure cancellation", () => { }); it.each(["pull_request", "push", "workflow_dispatch"] as const)( - "uses native matrix fail-fast only for PRs (%s)", + "keeps first-attempt continuation within the canonical monitor's scope (%s)", (eventName) => { const workflow = readCiWorkflow(); - const failFast = workflow.jobs["checks-node-core-test-nondist-shard"].strategy["fail-fast"]; - expect( - typeof failFast === "string" - ? evaluateWorkflowExpression(failFast, { - eventName, - repository: "openclaw/openclaw", - runAttempt: 1, - }) - : failFast, - ).toBe(eventName === "pull_request"); + const node = workflow.jobs["checks-node-core-test-nondist-shard"]; + const run = node.steps.find((step: WorkflowStep) => step.name === "Run Node test shard"); + for (const [repository, headRepository, runAttempt, nativeFailFast, continuation] of [ + ["openclaw/openclaw", "openclaw/openclaw", 1, false, "1"], + ["openclaw/openclaw", "contributor/openclaw", 1, false, "1"], + ["openclaw/openclaw", "openclaw/openclaw", 2, true, "0"], + ["fork/openclaw", "fork/openclaw", 1, true, "0"], + ["fork/openclaw", "contributor/openclaw", 1, true, "0"], + ["fork/openclaw", "fork/openclaw", 2, true, "0"], + ] as const) { + const context = { eventName, repository, headRepository, runAttempt }; + expect(evaluateWorkflowExpression(node.strategy["fail-fast"], context)).toBe( + eventName === "pull_request" && nativeFailFast, + ); + expect( + evaluateWorkflowExpression(run.env.OPENCLAW_NODE_TEST_PLAN_CONTINUE_ON_FAILURE, context), + ).toBe(eventName === "pull_request" ? continuation : "0"); + } + }, + ); + + it.skipIf(process.platform === "win32")( + "finishes selected compiler and lint groups only for ordinary diagnostic failures", + () => { + const workflow = readCiWorkflow(); + const central = workflow.jobs["check-shard"].steps.find( + (step: WorkflowStep) => step.name === "Run check shard", + ); + const hosted = workflow.jobs["check-test-types-hosted-core-shard"].steps.find( + (step: WorkflowStep) => step.name === "Run hosted core test-types stripe", + ); + const root = tempDirs.make("ci-type-groups-"); + const calls = path.join(root, "calls"); + mkdirSync(path.join(root, ".ci-harness/scripts"), { recursive: true }); + copyFileSync( + new URL("../../scripts/ci-static-step.sh", import.meta.url), + path.join(root, ".ci-harness/scripts/ci-static-step.sh"), + ); + mkdirSync(path.join(root, "scripts")); + writeFileSync(path.join(root, "scripts/run-oxlint-shards.mts"), "// --extension-stripe\n"); + writeFileSync( + path.join(root, "node"), + '#!/bin/bash\nprintf "%s\\n" "$*" >> "$CALLS_FILE"\nif [[ "$*" == *"${FAIL_MARKER:-first}"* ]]; then exit "$COMPILER_EXIT"; fi\n', + { mode: 0o755 }, + ); + for (const [evidence, compilerExit, expectedCalls, groups] of [ + ["1", "2", 2, 2], + ["0", "2", 1, 0], + ["1", "1", 1, 0], + ] as const) { + writeFileSync(calls, ""); + const run = spawnSync("/bin/bash", ["-c", central.run], { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + PATH: `${root}:${process.env.PATH}`, + CALLS_FILE: calls, + COMPILER_EXIT: compilerExit, + OPENCLAW_CI_STATIC_EVIDENCE: evidence, + NARROW_CHECK_PATHS_JSON: '["src/example.ts"]', + TASK: "test-types", + CI_CORE_TYPE_GRAPHS_JSON: '["first"]', + CI_CORE_TYPE_CONCURRENCY: "1", + CI_TYPE_GRAPHS_JSON: '["second"]', + }, + }); + expect(run.status, run.stderr).toBe(Number(compilerExit)); + expect(readFileSync(calls, "utf8").trim().split("\n")).toHaveLength(expectedCalls); + expect(run.stdout.includes('[ci-static:tsgo:step] {"version":1,"groups":2}')).toBe( + groups === 2, + ); + } + const run = spawnSync("/bin/bash", ["-c", hosted.run], { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + PATH: `${root}:${process.env.PATH}`, + CALLS_FILE: calls, + COMPILER_EXIT: "2", + OPENCLAW_CI_STATIC_EVIDENCE: "1", + CI_TYPE_GRAPHS_JSON: '["first"]', + }, + }); + expect(run.status, run.stderr).toBe(2); + expect(run.stdout).toContain('[ci-static:tsgo:step] {"version":1,"groups":1}'); + + const lint = workflow.jobs["check-lint-hosted-core-shard"].steps.find( + (step: WorkflowStep) => step.name === "Run hosted core lint stripe", + ); + const lintScript = lint.run.replace(/\$\{\{[\s\S]*?\}\}/gu, (expression: string) => + String( + evaluateWorkflowExpression(expression, { + eventName: "pull_request", + runnerProfile: "github", + }), + ), + ); + for (const [evidence, compilerExit, expectedCalls, groups] of [ + ["1", "1", 2, 2], + ["0", "1", 1, 0], + ["1", "2", 1, 0], + ] as const) { + writeFileSync(calls, ""); + const lintRun = spawnSync("/bin/bash", ["-c", lintScript], { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + PATH: `${root}:${process.env.PATH}`, + CALLS_FILE: calls, + FAIL_MARKER: "--only=core", + COMPILER_EXIT: compilerExit, + OPENCLAW_CI_STATIC_EVIDENCE: evidence, + CORE_STRIPE: "1", + FROZEN_TARGET: "false", + RUNNER_PROFILE: "github", + RELEASE_GATE: "false", + }, + }); + expect(lintRun.status, lintRun.stderr).toBe(Number(compilerExit)); + expect(readFileSync(calls, "utf8").trim().split("\n")).toHaveLength(expectedCalls); + expect(lintRun.stdout.includes('[ci-static:oxlint:step] {"version":1,"groups":2}')).toBe( + groups === 2, + ); + } + }, + ); + + it.skipIf(process.platform === "win32")( + "accepts only the completed monitor's supported test and static exceptions", + () => { + const verify = readCiWorkflow().jobs["ci-gate"].steps.find( + (entry: WorkflowStep) => entry.name === "Verify selected CI lanes", + ); + for (const [name, result, receipt, attempt, exit] of [ + ["checks-node-core-test-nondist-shard", "failure", "1", 1, 0], + ["checks-node-core-test-nondist-shard", "failure", "", 1, 1], + ["checks-node-core-test-nondist-shard", "failure", "1", 2, 1], + ["checks-node-core-test-nondist-shard", "cancelled", "1", 1, 1], + ["check-shard", "failure", "1", 1, 0], + ["check-test-types-hosted-core-shard", "failure", "1", 1, 0], + ["check-test-types-hosted-core-shard", "failure", "", 1, 1], + ["check-lint-hosted-core-shard", "failure", "1", 1, 0], + ["check-lint-hosted-extension-shard", "failure", "1", 1, 0], + ["check-lint-hosted-core-shard", "failure", "", 1, 1], + ["check-additional-shard", "failure", "1", 1, 1], + ] as const) { + const allowed = evaluateWorkflowExpression(verify.env.ALLOW_KNOWN_MAIN_RED, { + eventName: "pull_request", + repository: "openclaw/openclaw", + runAttempt: attempt, + failFastResult: "success", + failFastOutputs: { known_main_red_attempt: receipt }, + }); + const run = spawnSync("/bin/bash", ["-c", verify.run], { + encoding: "utf8", + env: { + ...process.env, + ALLOW_KNOWN_MAIN_RED: String(allowed), + JOB_RESULTS: `${name}=${result}|true`, + }, + }); + expect(run.status, run.stdout).toBe(exit); + } }, );