From ca449718157027317e9614ce3905d69bfbc7f38b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E8=89=AF?= <1204183885@qq.com> Date: Thu, 13 Aug 2026 05:13:23 +0000 Subject: [PATCH] fix(ci): cache downloaded linters on ECS runners (#9001) * fix(ci): cache downloaded linters on ECS runners * fix(ci): verify cached linter archives * fix(ci): make linter cache writes optional * test(ci): cover linter cache fail-closed paths * fix(ci): harden linter cache setup --- scripts/lint.js | 105 +++++++++++++++++++++++-- scripts/tests/lint.test.js | 155 ++++++++++++++++++++++++++++++++++++- 2 files changed, 253 insertions(+), 7 deletions(-) diff --git a/scripts/lint.js b/scripts/lint.js index f893cd5e82..db5fd6d31d 100644 --- a/scripts/lint.js +++ b/scripts/lint.js @@ -9,12 +9,28 @@ import { execSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import { mkdirSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { homedir, tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; const ACTIONLINT_VERSION = '1.7.12'; const SHELLCHECK_VERSION = '0.11.0'; const YAMLLINT_VERSION = '1.35.1'; +const ACTIONLINT_SHA256 = { + linux_amd64: + '8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8', + darwin_amd64: + '5b44c3bc2255115c9b69e30efc0fecdf498fdb63c5d58e17084fd5f16324c644', + darwin_arm64: + 'aba9ced2dee8d27fecca3dc7feb1a7f9a52caefa1eb46f3271ea66b6e0e6953f', +}; +const SHELLCHECK_SHA256 = { + 'linux.x86_64': + '8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198', + 'darwin.x86_64': + '3c89db4edcab7cf1c27bff178882e0f6f27f7afdf54e859fa041fca10febe4c6', + 'darwin.aarch64': + '56affdd8de5527894dca6dc3d7e0a99a873b0f004d7aabc30ae407d3f48b0a79', +}; function sanitizePathPart(value) { return value.replace(/[^A-Za-z0-9._-]/g, '_'); @@ -46,7 +62,60 @@ export function getLinterTempDir({ return join(baseDir, 'qwen-code-linters', `local-${workspaceHash}`); } +export function getLinterCacheDir({ + env = process.env, + homeDir = homedir(), +} = {}) { + return join( + env.XDG_CACHE_HOME || join(homeDir, '.cache'), + 'qwen-code', + 'linters', + ); +} + +export function getCachedArchiveInstaller({ + cacheArchive, + localArchive, + expectedSha256, + downloadUrl, + archiveCheck, + extract, + executable, +}) { + if (!expectedSha256) { + throw new Error(`Missing SHA-256 pin for ${downloadUrl}`); + } + return ` + set -e + verify_sha256() { + "${process.execPath}" -e 'const {createHash}=require("node:crypto");const {readFileSync}=require("node:fs");const actual=createHash("sha256").update(readFileSync(process.argv[1])).digest("hex");if(actual!==process.argv[2]){console.error("SHA-256 mismatch for "+process.argv[1]+": expected "+process.argv[2]+", got "+actual);process.exit(1)}' "$1" "$2" + } + mkdir -p "${dirname(cacheArchive)}" || true + if ! cp "${cacheArchive}" "${localArchive}" 2>/dev/null \ + || ! verify_sha256 "${localArchive}" "${expectedSha256}"; then + rm -f "${localArchive}" + curl -fsSL --retry 2 --retry-connrefused --connect-timeout 10 --max-time 90 \ + -o "${localArchive}" "${downloadUrl}" + verify_sha256 "${localArchive}" "${expectedSha256}" + ${archiveCheck} + cache_tmp='' + if cache_tmp="$(mktemp "${cacheArchive}.XXXXXX")" \ + && cp "${localArchive}" "$cache_tmp" \ + && "${process.execPath}" -e 'require("node:fs").renameSync(process.argv[1],process.argv[2])' "$cache_tmp" "${cacheArchive}"; then + : + else + [ -z "$cache_tmp" ] || rm -f "$cache_tmp" + echo "Warning: could not persist linter archive to ${cacheArchive}" >&2 + fi + fi + ${extract} + test -x "${executable}" + `; +} + const TEMP_DIR = getLinterTempDir(); +// Share versioned archives; extracted binaries stay job-scoped in TEMP_DIR. +const CACHE_DIR = getLinterCacheDir(); function getPlatformArch() { const platform = process.platform; @@ -88,13 +157,30 @@ let lintersCache; function getLinters() { if (!lintersCache) { const platformArch = getPlatformArch(); + const actionlintArchive = join( + CACHE_DIR, + `actionlint_${ACTIONLINT_VERSION}_${platformArch.actionlint}.tar.gz`, + ); + const shellcheckArchive = join( + CACHE_DIR, + `shellcheck_${SHELLCHECK_VERSION}_${platformArch.shellcheck}.tar.xz`, + ); + const actionlintLocalArchive = join(TEMP_DIR, '.actionlint.tgz'); + const shellcheckLocalArchive = join(TEMP_DIR, '.shellcheck.txz'); lintersCache = { actionlint: { check: 'command -v actionlint', installer: ` mkdir -p "${TEMP_DIR}/actionlint" - curl -sSLo "${TEMP_DIR}/.actionlint.tgz" "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_${platformArch.actionlint}.tar.gz" - tar -xzf "${TEMP_DIR}/.actionlint.tgz" -C "${TEMP_DIR}/actionlint" + ${getCachedArchiveInstaller({ + cacheArchive: actionlintArchive, + localArchive: actionlintLocalArchive, + expectedSha256: ACTIONLINT_SHA256[platformArch.actionlint], + downloadUrl: `https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_${platformArch.actionlint}.tar.gz`, + archiveCheck: `tar -tzf "${actionlintLocalArchive}" >/dev/null`, + extract: `tar -xzf "${actionlintLocalArchive}" -C "${TEMP_DIR}/actionlint"`, + executable: join(TEMP_DIR, 'actionlint', 'actionlint'), + })} `, run: ` actionlint \ @@ -112,8 +198,15 @@ function getLinters() { check: 'command -v shellcheck', installer: ` mkdir -p "${TEMP_DIR}/shellcheck" - curl -sSLo "${TEMP_DIR}/.shellcheck.txz" "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.${platformArch.shellcheck}.tar.xz" - tar -xf "${TEMP_DIR}/.shellcheck.txz" -C "${TEMP_DIR}/shellcheck" --strip-components=1 + ${getCachedArchiveInstaller({ + cacheArchive: shellcheckArchive, + localArchive: shellcheckLocalArchive, + expectedSha256: SHELLCHECK_SHA256[platformArch.shellcheck], + downloadUrl: `https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.${platformArch.shellcheck}.tar.xz`, + archiveCheck: `tar -tf "${shellcheckLocalArchive}" >/dev/null`, + extract: `tar -xf "${shellcheckLocalArchive}" -C "${TEMP_DIR}/shellcheck" --strip-components=1`, + executable: join(TEMP_DIR, 'shellcheck', 'shellcheck'), + })} `, run: ` git ls-files | grep -v '^integration-tests/terminal-bench/' | grep -E '^([^.]+|.*\\.(sh|zsh|bash))' | xargs file --mime-type \ diff --git a/scripts/tests/lint.test.js b/scripts/tests/lint.test.js index 330fb9f988..39973f30c4 100644 --- a/scripts/tests/lint.test.js +++ b/scripts/tests/lint.test.js @@ -4,6 +4,18 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { execSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -11,7 +23,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; // paths so the suite also passes on the Windows gate. const toPosix = (value) => value.replaceAll(path.sep, '/'); -describe('getLinterTempDir', () => { +describe('linter directories', () => { const originalArgv = process.argv; beforeEach(() => { @@ -69,4 +81,145 @@ describe('getLinterTempDir', () => { expect(toPosix(second)).toMatch(/\/qwen-code-linters\/local-[a-f0-9]{16}$/); expect(first).not.toBe(second); }); + + it('shares cached downloads across GitHub Actions runs', async () => { + const { getLinterCacheDir } = await import('../lint.js'); + + const first = getLinterCacheDir({ + env: { + XDG_CACHE_HOME: '/runner/cache', + GITHUB_RUN_ID: '31583913822', + }, + }); + const second = getLinterCacheDir({ + env: { + XDG_CACHE_HOME: '/runner/cache', + GITHUB_RUN_ID: '31583913823', + }, + }); + + expect(toPosix(first)).toBe('/runner/cache/qwen-code/linters'); + expect(second).toBe(first); + expect( + toPosix(getLinterCacheDir({ env: {}, homeDir: '/home/runner' })), + ).toBe('/home/runner/.cache/qwen-code/linters'); + }); + + it.skipIf(process.platform === 'win32')( + 'verifies and reuses archives without depending on cache writes', + async () => { + const { getCachedArchiveInstaller } = await import('../lint.js'); + const root = mkdtempSync(path.join(tmpdir(), 'linter-cache-')); + + try { + const binDir = path.join(root, 'bin'); + const cacheArchive = path.join(root, 'cache', 'tool.tar'); + const localArchive = path.join(root, 'job', 'tool.tar'); + const executable = path.join(root, 'job', 'tool'); + const fixture = path.join(root, 'official.tar'); + const curlLog = path.join(root, 'curl.log'); + const curl = path.join(binDir, 'curl'); + mkdirSync(binDir, { recursive: true }); + mkdirSync(path.dirname(cacheArchive), { recursive: true }); + mkdirSync(path.dirname(localArchive), { recursive: true }); + writeFileSync(cacheArchive, 'validator-passing plant'); + writeFileSync(fixture, 'official archive'); + writeFileSync( + curl, + '#!/bin/sh\nprintf "download\\n" >> "$CURL_LOG"\nwhile [ "$#" -gt 0 ]; do\n if [ "$1" = "-o" ]; then\n if [ "$CORRUPT_DOWNLOAD" = "1" ]; then printf corrupt > "$2"; else cp "$FIXTURE_ARCHIVE" "$2"; fi\n exit\n fi\n shift\ndone\nexit 1\n', + ); + chmodSync(curl, 0o755); + + const expectedSha256 = createHash('sha256') + .update(readFileSync(fixture)) + .digest('hex'); + expect(() => + getCachedArchiveInstaller({ + cacheArchive, + localArchive, + downloadUrl: 'https://example.invalid/unpinned.tar', + }), + ).toThrow('Missing SHA-256 pin'); + const installer = getCachedArchiveInstaller({ + cacheArchive, + localArchive, + expectedSha256, + downloadUrl: 'https://example.invalid/tool.tar', + archiveCheck: 'true', + extract: `cp "${localArchive}" "${executable}" && chmod +x "${executable}"`, + executable, + }); + const env = { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + CURL_LOG: curlLog, + CORRUPT_DOWNLOAD: '0', + FIXTURE_ARCHIVE: fixture, + }; + + expect(() => + execSync(installer, { + env: { ...env, CORRUPT_DOWNLOAD: '1' }, + }), + ).toThrow(); + expect(readFileSync(cacheArchive, 'utf8')).toBe( + 'validator-passing plant', + ); + + execSync(installer, { env }); + expect(readFileSync(cacheArchive, 'utf8')).toBe('official archive'); + expect(readFileSync(executable, 'utf8')).toBe('official archive'); + expect(readFileSync(curlLog, 'utf8')).toBe('download\ndownload\n'); + + rmSync(localArchive); + rmSync(executable); + const nonExecutableInstaller = getCachedArchiveInstaller({ + cacheArchive, + localArchive, + expectedSha256, + downloadUrl: 'https://example.invalid/tool.tar', + archiveCheck: 'true', + extract: `cp "${localArchive}" "${executable}"`, + executable, + }); + expect(() => execSync(nonExecutableInstaller, { env })).toThrow(); + + rmSync(localArchive); + rmSync(executable); + rmSync(fixture); + execSync(installer, { env }); + expect(readFileSync(executable, 'utf8')).toBe('official archive'); + expect(readFileSync(curlLog, 'utf8')).toBe('download\ndownload\n'); + + writeFileSync(fixture, 'official archive'); + rmSync(localArchive); + rmSync(executable); + rmSync(path.dirname(cacheArchive), { recursive: true }); + execSync(installer, { env }); + expect(readFileSync(cacheArchive, 'utf8')).toBe('official archive'); + expect(readFileSync(curlLog, 'utf8')).toBe( + 'download\ndownload\ndownload\n', + ); + + rmSync(localArchive); + rmSync(executable); + rmSync(cacheArchive); + mkdirSync(cacheArchive); + const result = spawnSync(installer, { env, shell: true }); + expect(result.status).toBe(0); + expect(result.stderr.toString()).toContain('EISDIR'); + expect(result.stderr.toString()).toContain( + 'Warning: could not persist linter archive', + ); + expect(readFileSync(executable, 'utf8')).toBe('official archive'); + expect(statSync(cacheArchive).isDirectory()).toBe(true); + expect(readFileSync(curlLog, 'utf8')).toBe( + 'download\ndownload\ndownload\ndownload\n', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + 15_000, + ); });