diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index a66d029d605..50267e9c3a0 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -119,6 +119,8 @@ concurrency: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" GH_REPO: ${{ github.repository }} + # Read retries and one-shot dispatch recovery share this classifier; dispatch POSTs never retry. + GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN: "HTTP 5[0-9][0-9]|Server Error|error connecting to|context deadline exceeded|connection reset by peer|connection refused|TLS handshake timeout|i/o timeout|network is unreachable|(^|[^A-Za-z0-9_])EOF([^A-Za-z0-9_]|$)|ETIMEDOUT|ECONNRESET|EAI_AGAIN" NODE_VERSION: "24.15.0" jobs: @@ -290,7 +292,7 @@ jobs: printf '%s\n' "$output" return 0 fi - if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then + if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2 sleep $((attempt * 10)) continue @@ -308,6 +310,11 @@ jobs: set -e printf '%s\n' "$dispatch_output" + if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then + echo "::error::${workflow} dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2 + exit "$dispatch_status" + fi + run_id="" for _ in $(seq 1 60); do if matches_json="$( @@ -450,7 +457,7 @@ jobs: printf '%s\n' "$output" return 0 fi - if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then + if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2 sleep $((attempt * 10)) continue @@ -468,6 +475,11 @@ jobs: set -e printf '%s\n' "$dispatch_output" + if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then + echo "::error::${workflow} dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2 + exit "$dispatch_status" + fi + run_id="" for _ in $(seq 1 60); do if matches_json="$( @@ -620,7 +632,7 @@ jobs: printf '%s\n' "$output" return 0 fi - if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then + if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2 sleep $((attempt * 10)) continue @@ -638,6 +650,11 @@ jobs: set -e printf '%s\n' "$dispatch_output" + if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then + echo "::error::${workflow} dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2 + exit "$dispatch_status" + fi + run_id="" for _ in $(seq 1 60); do if matches_json="$( @@ -883,7 +900,7 @@ jobs: printf '%s\n' "$output" return 0 fi - if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then + if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2 sleep $((attempt * 10)) continue @@ -911,6 +928,11 @@ jobs: set -e printf '%s\n' "$dispatch_output" + if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then + echo "::error::npm-telegram-beta-e2e.yml dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2 + exit "$dispatch_status" + fi + run_id="" for _ in $(seq 1 60); do if matches_json="$( @@ -1026,7 +1048,7 @@ jobs: printf '%s\n' "$output" return 0 fi - if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then + if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2 sleep $((attempt * 10)) continue @@ -1067,6 +1089,11 @@ jobs: set -e printf '%s\n' "$dispatch_output" + if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then + echo "::error::openclaw-performance.yml dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2 + exit "$dispatch_status" + fi + run_id="" for _ in $(seq 1 60); do if matches_json="$( @@ -1181,7 +1208,7 @@ jobs: printf '%s\n' "$output" return 0 fi - if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then + if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2 sleep $((attempt * 10)) continue diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index ce7bf1f3854..ccdec247e82 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -589,26 +589,90 @@ describe("package acceptance workflow", () => { ["npm_telegram", "Dispatch and monitor npm Telegram E2E"], ["performance", "Dispatch and monitor OpenClaw Performance"], ] as const; - - for (const [jobName, stepName] of childDispatches) { + const dispatchScripts = childDispatches.map(([jobName, stepName]) => { const job = workflowJob(FULL_RELEASE_VALIDATION_WORKFLOW, jobName); - const script = workflowStep(job, stepName).run ?? ""; + return workflowStep(job, stepName).run ?? ""; + }); + for (const script of dispatchScripts) { expect(script.match(/gh workflow run/gu)).toHaveLength(1); expect(script).not.toContain("gh_with_retry workflow run"); expectTextToIncludeAll(script, [ "A failed dispatch POST can still create a run. Never retry it", "set +e", "dispatch_status=$?", + 'if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]', + "dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling.", 'DISPATCH_RUN_NAME="$dispatch_run_name" CHILD_WORKFLOW_REF="$CHILD_WORKFLOW_REF"', ".display_title == env.DISPATCH_RUN_NAME and .head_branch == env.CHILD_WORKFLOW_REF", "Multiple runs matched ${dispatch_run_name}; refusing to guess.", "The dispatch was not retried to avoid creating a duplicate child.", "adopted exact run ${run_id}", ]); + expect(script.indexOf("dispatch failed with non-ambiguous status")).toBeLessThan( + script.indexOf('run_id=""'), + ); + } + + const parsedWorkflow = readWorkflow(FULL_RELEASE_VALIDATION_WORKFLOW); + const transientPattern = parsedWorkflow.env?.GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN; + expect(transientPattern).toBeDefined(); + const transientError = new RegExp(transientPattern ?? "", "u"); + for (const message of [ + "could not create workflow dispatch event: HTTP 500: Failed to run workflow dispatch", + "gh: HTTP 502", + "500 Internal Server Error", + "error connecting to api.github.com", + "context deadline exceeded", + "read: connection reset by peer", + "connect: connection refused", + "net/http: TLS handshake timeout", + "read: i/o timeout", + "network is unreachable", + "unexpected EOF", + 'Post "https://api.github.com/repos/openclaw/openclaw/actions/workflows/ci.yml/dispatches": EOF', + "EOF", + "ETIMEDOUT", + "ECONNRESET", + "EAI_AGAIN", + ]) { + expect(transientError.test(message), message).toBe(true); + } + for (const message of [ + "HTTP 400: Bad Request", + "HTTP 401: Bad credentials", + "HTTP 403: Resource not accessible by integration", + "HTTP 404: workflow not found", + "HTTP 422: Validation Failed", + "HTTP 429: too many requests", + "unknown flag --field", + "EOFError while parsing local input", + ]) { + expect(transientError.test(message), message).toBe(false); + } + + const summaryScript = + workflowStep( + workflowJob(FULL_RELEASE_VALIDATION_WORKFLOW, "summary"), + "Verify child workflow results", + ).run ?? ""; + for (const script of [...dispatchScripts, summaryScript]) { + expect(script.match(/gh_with_retry\(\)/gu)).toHaveLength(1); + expectTextToIncludeAll(script, [ + '"$output" == *"HTTP 429"*', + '"$output" == *"abuse detection"*', + '"$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN', + 'return "$status"', + ]); } const workflow = readFileSync(FULL_RELEASE_VALIDATION_WORKFLOW, "utf8"); + const retryCalls = workflow.split("\n").filter((line) => line.includes("gh_with_retry ")); + expect(retryCalls).toHaveLength(28); + for (const call of retryCalls) { + expect(call).toMatch(/gh_with_retry (api|run view)/u); + } + expect(workflow).not.toMatch(/gh_with_retry (workflow run|run cancel)/u); expectTextToIncludeAll(workflow, [ 'dispatch_id="full-release-validation-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-ci"', 'dispatch_id="full-release-validation-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-plugin-prerelease"',