From d9f7e1fbe18e6fde0cf66011fcc07d442f36a509 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Thu, 23 Jul 2026 15:13:42 +0800 Subject: [PATCH] feat(autofix): auto-rerun a check that died on infrastructure, once (#7562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(autofix): auto-rerun a check that died on infrastructure, once A failed check can be red because the machine died, not the code — a self-hosted runner losing the server, the disk filling. #7490's E2E failed with "runner lost communication with the server" and went green on a rerun. The scan now reruns such a check's failed jobs automatically. Detection is a conservative annotation whitelist (INFRA_FAILURE_SIGNATURES) — only unambiguous machine failures, never a test-level timeout, which could be a real regression. The one-shot guard is run_attempt, not a marker: a run already retried to attempt 2 and still infra-failing is persistent, so it is left for a human; after a rerun the attempt increments, so the next scan will not rerun it. Every step is fail-safe (any API error → no rerun), it runs only when the PR actually has a failed check, and the gate carries the same review-address carve-out as the other check selectors so the loop never reruns its own runs. This is the transient-infra sibling of #7554 (stale-base): that merges current main when a check is base-inherited; this reruns when a check died on the runner. Neither touches a check that is a genuine failure. Note: rerun-failed-jobs needs the PAT to hold `actions: write`. * fix(autofix): use POSIX ERE groups in infra-failure regex, cover all signatures in tests (#7562) * fix(autofix): also treat a git fetch/clone transport death as infra #6506's checkout died mid-transfer — "fetch-pack: invalid index-pack output" and "RPC failed; curl 92 ... CANCEL" — which then hung the job into the 20m limit. That is infra, not the PR (it only touches a doc), and a re-run made it green. But the infra-signature whitelist did not cover it, so the auto-rerun did not fire and it waited on a human. Add `invalid index-pack output` and `RPC failed` — the two canonical git-transport-death phrases — to INFRA_FAILURE_SIGNATURES. A co-present job-timeout line does not block the match (one matching line classifies the run), and a BARE timeout with no transport signature is still left alone, since it can be a real regression. Both new signatures are pinned in the test's per-signature loop, plus a case on #6506's real composite annotation and a bare-timeout-is-not-rerun guard. * fix(autofix): paginate annotations and filter Autofix runs in infra-rerun loop (#7562) --------- Co-authored-by: wenshao Co-authored-by: Qwen Code Bot Co-authored-by: qwen-code-ci-bot --- .github/workflows/qwen-autofix.yml | 62 ++++++ scripts/tests/qwen-autofix-workflow.test.js | 212 ++++++++++++++++++++ 2 files changed, 274 insertions(+) diff --git a/.github/workflows/qwen-autofix.yml b/.github/workflows/qwen-autofix.yml index 330612b8b9..215db9ed37 100644 --- a/.github/workflows/qwen-autofix.yml +++ b/.github/workflows/qwen-autofix.yml @@ -129,6 +129,18 @@ env: # invisible to the scan even when it already had unaddressed feedback. # Names must match job ids in qwen-code-pr-review.yml; a test pins that. NON_BLOCKING_CHECKS: '["review-pr"]' + # Failed-check annotation patterns that mean the INFRASTRUCTURE died, not the + # code — a self-hosted runner losing the server, the disk filling, a runner + # shutdown, or a git fetch/clone dying mid-transfer. Such a check is red for a + # reason unrelated to the PR and clears on a re-run (observed: #7490's E2E + # "runner lost communication"; #6506's checkout "RPC failed; curl 92" / + # "fetch-pack: invalid index-pack output" — both green on the rerun). The scan + # auto-reruns those failed jobs ONCE, guarded by run_attempt so a persistent + # infra problem cannot loop. Deliberately conservative — only unambiguous + # machine/transport failures, never a bare test-level timeout, which could be + # a real regression (a co-present timeout does not block a match — one + # matching line classifies the run). Case-insensitive, vs the annotations. + INFRA_FAILURE_SIGNATURES: 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)|invalid index-pack output|RPC failed' # Upper bound on review targets emitted per scan (fan-out defense-in-depth; # excess is logged and deferred to the next scan). MAX_TARGETS_PER_SCAN: '10' @@ -1809,6 +1821,56 @@ jobs: ISSUE="${PR}" fi CHECKS_JSON="$(jq -c '.statusCheckRollup // []' <<< "${PR_META}")" + + # Auto-rerun a check that died on INFRASTRUCTURE, not the code (see + # INFRA_FAILURE_SIGNATURES). Only reached when the PR has a FAILED + # check; then, for each, we read its annotations and — if they carry + # a machine-death signature — rerun that run's failed jobs ONCE. The + # once is enforced by run_attempt: a run already retried to attempt 2 + # and still infra-failing is persistent, so we stop and leave it. No + # marker needed; the attempt counter is the guard, and after a rerun + # the attempt increments so the next scan skips it. Any API failure + # here is fail-safe: it just means no rerun. + PR_HEAD_OID="$(jq -r '.headRefOid // ""' <<< "${PR_META}")" + if [[ -n "${PR_HEAD_OID}" ]] && jq -e 'any(.[]; ((.conclusion // .state // "") | IN("FAILURE","FAILED","ERROR","TIMED_OUT","ACTION_REQUIRED")) and (((.workflowName // "") != "Qwen Autofix") or ((.name // "") | startswith("review-address"))))' <<< "${CHECKS_JSON}" > /dev/null 2>&1; then + RERAN_INFRA=false + # Failed check-runs on this head, with their run id and annotation + # count — fetched once. External statuses (no check-run) are absent + # here, which is fine: only Actions runs can be reran. + FAILED_CRS="$(gh api --paginate "repos/${REPO}/commits/${PR_HEAD_OID}/check-runs" \ + --jq '.check_runs[] | select(.conclusion == "failure") + | select(((.output.annotations_count // 0) > 0)) + | [(.id | tostring), (.details_url // ""), (.name // "")] | @tsv' 2> /dev/null | sort -u || true)" + while IFS=$'\t' read -r CR_ID DETAILS_URL CR_NAME; do + [[ -n "${CR_ID}" && "${DETAILS_URL}" == *"/actions/runs/"* ]] || continue + RUN_ID="${DETAILS_URL#*/actions/runs/}" + RUN_ID="${RUN_ID%%/*}" + [[ "${RUN_ID}" =~ ^[0-9]+$ ]] || continue + ANNS="$(gh api --paginate "repos/${REPO}/check-runs/${CR_ID}/annotations" \ + --jq '[.[].message] | join("\n")' 2> /dev/null || true)" + grep -qiE "${INFRA_FAILURE_SIGNATURES}" <<< "${ANNS}" || continue + RUN_META="$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" --jq '[(.run_attempt // 0), (.name // "")] | @tsv' 2> /dev/null || printf '0\t')" + ATTEMPT="${RUN_META%%$'\t'*}" + WF_NAME="${RUN_META#*$'\t'}" + # Mirror the gate's self-trigger guard: skip Qwen Autofix runs + # unless the check is a review-address job. + if [[ "${WF_NAME}" == "Qwen Autofix" && "${CR_NAME}" != review-address* ]]; then + continue + fi + if [[ "${ATTEMPT}" == '1' ]]; then + if gh api -X POST "repos/${REPO}/actions/runs/${RUN_ID}/rerun-failed-jobs" > /dev/null 2>&1; then + echo "♻️ #${PR}: a check died on infrastructure (run ${RUN_ID}, attempt 1) — reran its failed jobs; not the PR's code" + fleet_row "${PR}" 'infra-reran' "infra failure (run ${RUN_ID}) reran once" + RERAN_INFRA=true + break + fi + else + echo "⚠️ #${PR}: infra failure persisted after a rerun (run ${RUN_ID}, attempt ${ATTEMPT}) — leaving for a human" + fi + done <<< "${FAILED_CRS}" + [[ "${RERAN_INFRA}" == 'true' ]] && continue + fi + # startedAt is the only staleness clock: a check blocks only if it # started within the bound; one with no startedAt (queued, not yet # running) is not blocking (the next scan re-checks once it starts). diff --git a/scripts/tests/qwen-autofix-workflow.test.js b/scripts/tests/qwen-autofix-workflow.test.js index 82df52d7d2..901e24c24d 100644 --- a/scripts/tests/qwen-autofix-workflow.test.js +++ b/scripts/tests/qwen-autofix-workflow.test.js @@ -401,6 +401,218 @@ describe('qwen-autofix workflow', () => { expect(run([{ ...llm, name: 'resolve-pr' }])).toBe('true'); }); + it('auto-reruns a check that died on infrastructure, once, guarded by run_attempt', () => { + // A self-hosted runner losing the server (or the disk filling) reds a check + // for a reason unrelated to the PR; it clears on a rerun (#7490's E2E: + // "runner lost communication" → green on the rerun). The scan reruns such a + // failed job ONCE, and run_attempt is the guard: a run already at attempt 2 + // and still infra-failing is persistent and left alone — no infinite loop. + const block = reviewScanJob.match( + /( {12}PR_HEAD_OID="\$\(jq -r '\.headRefOid[\s\S]*?\n {12}fi\n)\n {12}# startedAt is the only staleness/, + )?.[1]; + expect(block).toBeTruthy(); + const script = block.replace(/^ {12}/gm, ''); + + const run = ({ + checks, + annotations, + attempt = 1, + rerunOk = true, + crName = 'E2E', + wfName = 'CI', + }) => { + const dir = mkdtempSync(join(tmpdir(), 'infra-')); + const bin = join(dir, 'bin'); + mkdirSync(bin); + // Stubbed gh: check-runs → one failed run-1 check-run with annotations; + // annotations → the given message; runs/{id} → run_attempt; POST + // rerun-failed-jobs → success/fail. Records the rerun POST. + // The workflow calls check-runs with a --jq filter that yields, per + // failed check-run WITH annotations, a `\t\t` + // line; the stub emits what that filter would produce (a single line + // when there is an annotation, nothing otherwise) rather than raw JSON + // the stub can't filter. + const crTsv = annotations + ? `42\thttps://github.com/o/r/actions/runs/9001/job/5\t${crName}\n` + : ''; + writeFileSync( + join(bin, 'gh'), + [ + '#!/usr/bin/env bash', + `echo "$*" >> ${JSON.stringify(join(dir, 'calls.log'))}`, + 'args="$*"', + `case "$args" in`, + // %b so the \t/\n in the stubbed tsv become a real tab/newline (the + // filter's @tsv output), which `IFS=$'\\t' read` then splits. + ` *"/commits/"*"/check-runs"*) printf '%b' ${JSON.stringify(crTsv)}; exit 0;;`, + ` *"/check-runs/42/annotations"*) printf '%s' ${JSON.stringify(annotations || '')}; exit 0;;`, + ` *"/actions/runs/9001"*"rerun-failed-jobs"*) exit ${rerunOk ? 0 : 1};;`, + ` *"/actions/runs/9001"*) printf '${attempt}\\t${wfName}'; exit 0;;`, + 'esac', + 'exit 0', + ].join('\n'), + ); + chmodSync(join(bin, 'gh'), 0o755); + const out = execFileSync( + 'bash', + [ + '-c', + `set -uo pipefail\nfleet_row(){ :; }\nfor _ in x; do\n${script}\nprintf 'FELL_THROUGH'\ndone`, + ], + { + env: { + ...process.env, + REPO: 'o/r', + PR: '1', + PR_META: JSON.stringify({ headRefOid: 'headSHA' }), + CHECKS_JSON: JSON.stringify(checks), + INFRA_FAILURE_SIGNATURES: + 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)|invalid index-pack output|RPC failed', + PATH: `${bin}:${process.env.PATH}`, + }, + encoding: 'utf8', + }, + ); + const calls = existsSync(join(dir, 'calls.log')) + ? readFileSync(join(dir, 'calls.log'), 'utf8') + : ''; + rmSync(dir, { recursive: true, force: true }); + return { + reran: /rerun-failed-jobs/.test(calls), + continued: !out.includes('FELL_THROUGH'), + }; + }; + const FAIL = { name: 'E2E', conclusion: 'FAILURE' }; + const OK = { name: 'E2E', conclusion: 'SUCCESS' }; + + // Infra death (runner lost the server) on attempt 1 → rerun & skip. + expect( + run({ + checks: [FAIL], + annotations: + 'The self-hosted runner lost communication with the server', + }), + ).toEqual({ reran: true, continued: true }); + // A REAL failure (no infra signature in the annotation) → never rerun; the + // agent/human handles it. This is the gate that stops masking real bugs. + expect( + run({ + checks: [FAIL], + annotations: 'Expected 1 argument but got 2 — src/foo.ts:10', + }), + ).toEqual({ reran: false, continued: false }); + // Already reran once (attempt 2) and still infra-failing → persistent, do + // not loop. + expect( + run({ + checks: [FAIL], + annotations: 'No space left on device', + attempt: 2, + }), + ).toEqual({ reran: false, continued: false }); + // No failed check at all → the block is skipped entirely. + expect(run({ checks: [OK], annotations: '' })).toEqual({ + reran: false, + continued: false, + }); + // Infra signature but the rerun POST fails (e.g. PAT lacks actions:write) → + // no crash, falls through to normal processing. + expect( + run({ + checks: [FAIL], + annotations: 'No space left on device', + rerunOk: false, + }), + ).toEqual({ reran: true, continued: false }); + // Each remaining production signature also triggers a rerun. + for (const msg of [ + 'ENOSPC', + 'The runner has received a shutdown signal', + 'The runner has received an unexpected signal', + 'Failed to initialize container for job', + 'The runner was lost', + 'The runner was terminated', + 'The runner has been lost', + 'The runner has been terminated', + 'fatal: fetch-pack: invalid index-pack output', + 'error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)', + ]) { + expect(run({ checks: [FAIL], annotations: msg })).toEqual({ + reran: true, + continued: true, + }); + } + // #6506: a git fetch died mid-checkout, then hung the job into the 20m + // limit. The bare timeout line is deliberately NOT a signature (it can be a + // real regression), but the transport death IS — and one matching line + // classifies the whole run, so the co-present timeout does not block it. + expect( + run({ + checks: [FAIL], + annotations: [ + 'The job has exceeded the maximum execution time of 20m0s', + 'fatal: fetch-pack: invalid index-pack output', + 'error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)', + ].join('\n'), + }), + ).toEqual({ reran: true, continued: true }); + // A BARE job timeout with no transport/infra signature is NOT rerun — it + // can be a real regression (a test hanging on the PR's own code). + expect( + run({ + checks: [FAIL], + annotations: 'The job has exceeded the maximum execution time of 20m0s', + }), + ).toEqual({ reran: false, continued: false }); + // Self-trigger guard: a "Qwen Autofix" workflow's own failed check must NOT + // be rerun (prevents the autofix from re-triggering itself), UNLESS the + // check is a review-address job (the exception carved out in the jq filter). + const AUTOFIX_CHECK = { + name: 'E2E', + conclusion: 'FAILURE', + workflowName: 'Qwen Autofix', + }; + expect( + run({ checks: [AUTOFIX_CHECK], annotations: 'No space left on device' }), + ).toEqual({ reran: false, continued: false }); + expect( + run({ + checks: [ + { + name: 'review-address issue-123', + conclusion: 'FAILURE', + workflowName: 'Qwen Autofix', + }, + ], + annotations: 'No space left on device', + }), + ).toEqual({ reran: true, continued: true }); + // In-loop self-trigger guard: the gate above blocks a PR whose ONLY + // failed check is Qwen Autofix, but when a non-Autofix check ALSO failed + // the gate passes and FAILED_CRS returns ALL failed check-runs — the + // in-loop filter must skip the Autofix run so it cannot consume the + // single rerun slot. + expect( + run({ + checks: [FAIL], + annotations: 'No space left on device', + wfName: 'Qwen Autofix', + }), + ).toEqual({ reran: false, continued: false }); + // …but a review-address job from the Autofix workflow IS rerun (the + // exception carved out in both the gate and the in-loop filter). + expect( + run({ + checks: [FAIL], + annotations: 'No space left on device', + crName: 'review-address issue-123', + wfName: 'Qwen Autofix', + }), + ).toEqual({ reran: true, continued: true }); + // Spawn-heavy: each run() forks bash + a stubbed gh. The default 5s per-test + // budget is tight for this many cases, so give it a comfortable margin. + }, 20000); + it('keeps a still-red check visible, but only once per head', () => { // A red check is a STATE, not the instant it turned red. Counting only // "failed since the watermark" made a still-failing PR invisible the