mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-08-19 21:55:06 +00:00
fix(ci): cache downloaded linters on ECS runners (#9001)
* fix(ci): cache downloaded linters on ECS runners * fix(ci): verify cached linter archives * fix(ci): make linter cache writes optional * test(ci): cover linter cache fail-closed paths * fix(ci): harden linter cache setup
This commit is contained in:
易良
GitHub
parent
d912f4c6a8
commit
ca44971815
2 changed files with 253 additions and 7 deletions
105
scripts/lint.js
105
scripts/lint.js
|
|
@ -9,12 +9,28 @@
|
|||
import { execSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdirSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { homedir, tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const ACTIONLINT_VERSION = '1.7.12';
|
||||
const SHELLCHECK_VERSION = '0.11.0';
|
||||
const YAMLLINT_VERSION = '1.35.1';
|
||||
const ACTIONLINT_SHA256 = {
|
||||
linux_amd64:
|
||||
'8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8',
|
||||
darwin_amd64:
|
||||
'5b44c3bc2255115c9b69e30efc0fecdf498fdb63c5d58e17084fd5f16324c644',
|
||||
darwin_arm64:
|
||||
'aba9ced2dee8d27fecca3dc7feb1a7f9a52caefa1eb46f3271ea66b6e0e6953f',
|
||||
};
|
||||
const SHELLCHECK_SHA256 = {
|
||||
'linux.x86_64':
|
||||
'8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198',
|
||||
'darwin.x86_64':
|
||||
'3c89db4edcab7cf1c27bff178882e0f6f27f7afdf54e859fa041fca10febe4c6',
|
||||
'darwin.aarch64':
|
||||
'56affdd8de5527894dca6dc3d7e0a99a873b0f004d7aabc30ae407d3f48b0a79',
|
||||
};
|
||||
|
||||
function sanitizePathPart(value) {
|
||||
return value.replace(/[^A-Za-z0-9._-]/g, '_');
|
||||
|
|
@ -46,7 +62,60 @@ export function getLinterTempDir({
|
|||
return join(baseDir, 'qwen-code-linters', `local-${workspaceHash}`);
|
||||
}
|
||||
|
||||
export function getLinterCacheDir({
|
||||
env = process.env,
|
||||
homeDir = homedir(),
|
||||
} = {}) {
|
||||
return join(
|
||||
env.XDG_CACHE_HOME || join(homeDir, '.cache'),
|
||||
'qwen-code',
|
||||
'linters',
|
||||
);
|
||||
}
|
||||
|
||||
export function getCachedArchiveInstaller({
|
||||
cacheArchive,
|
||||
localArchive,
|
||||
expectedSha256,
|
||||
downloadUrl,
|
||||
archiveCheck,
|
||||
extract,
|
||||
executable,
|
||||
}) {
|
||||
if (!expectedSha256) {
|
||||
throw new Error(`Missing SHA-256 pin for ${downloadUrl}`);
|
||||
}
|
||||
return `
|
||||
set -e
|
||||
verify_sha256() {
|
||||
"${process.execPath}" -e 'const {createHash}=require("node:crypto");const {readFileSync}=require("node:fs");const actual=createHash("sha256").update(readFileSync(process.argv[1])).digest("hex");if(actual!==process.argv[2]){console.error("SHA-256 mismatch for "+process.argv[1]+": expected "+process.argv[2]+", got "+actual);process.exit(1)}' "$1" "$2"
|
||||
}
|
||||
mkdir -p "${dirname(cacheArchive)}" || true
|
||||
if ! cp "${cacheArchive}" "${localArchive}" 2>/dev/null \
|
||||
|| ! verify_sha256 "${localArchive}" "${expectedSha256}"; then
|
||||
rm -f "${localArchive}"
|
||||
curl -fsSL --retry 2 --retry-connrefused --connect-timeout 10 --max-time 90 \
|
||||
-o "${localArchive}" "${downloadUrl}"
|
||||
verify_sha256 "${localArchive}" "${expectedSha256}"
|
||||
${archiveCheck}
|
||||
cache_tmp=''
|
||||
if cache_tmp="$(mktemp "${cacheArchive}.XXXXXX")" \
|
||||
&& cp "${localArchive}" "$cache_tmp" \
|
||||
&& "${process.execPath}" -e 'require("node:fs").renameSync(process.argv[1],process.argv[2])' "$cache_tmp" "${cacheArchive}"; then
|
||||
:
|
||||
else
|
||||
[ -z "$cache_tmp" ] || rm -f "$cache_tmp"
|
||||
echo "Warning: could not persist linter archive to ${cacheArchive}" >&2
|
||||
fi
|
||||
fi
|
||||
${extract}
|
||||
test -x "${executable}"
|
||||
`;
|
||||
}
|
||||
|
||||
const TEMP_DIR = getLinterTempDir();
|
||||
// Share versioned archives; extracted binaries stay job-scoped in TEMP_DIR.
|
||||
const CACHE_DIR = getLinterCacheDir();
|
||||
|
||||
function getPlatformArch() {
|
||||
const platform = process.platform;
|
||||
|
|
@ -88,13 +157,30 @@ let lintersCache;
|
|||
function getLinters() {
|
||||
if (!lintersCache) {
|
||||
const platformArch = getPlatformArch();
|
||||
const actionlintArchive = join(
|
||||
CACHE_DIR,
|
||||
`actionlint_${ACTIONLINT_VERSION}_${platformArch.actionlint}.tar.gz`,
|
||||
);
|
||||
const shellcheckArchive = join(
|
||||
CACHE_DIR,
|
||||
`shellcheck_${SHELLCHECK_VERSION}_${platformArch.shellcheck}.tar.xz`,
|
||||
);
|
||||
const actionlintLocalArchive = join(TEMP_DIR, '.actionlint.tgz');
|
||||
const shellcheckLocalArchive = join(TEMP_DIR, '.shellcheck.txz');
|
||||
lintersCache = {
|
||||
actionlint: {
|
||||
check: 'command -v actionlint',
|
||||
installer: `
|
||||
mkdir -p "${TEMP_DIR}/actionlint"
|
||||
curl -sSLo "${TEMP_DIR}/.actionlint.tgz" "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_${platformArch.actionlint}.tar.gz"
|
||||
tar -xzf "${TEMP_DIR}/.actionlint.tgz" -C "${TEMP_DIR}/actionlint"
|
||||
${getCachedArchiveInstaller({
|
||||
cacheArchive: actionlintArchive,
|
||||
localArchive: actionlintLocalArchive,
|
||||
expectedSha256: ACTIONLINT_SHA256[platformArch.actionlint],
|
||||
downloadUrl: `https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_${platformArch.actionlint}.tar.gz`,
|
||||
archiveCheck: `tar -tzf "${actionlintLocalArchive}" >/dev/null`,
|
||||
extract: `tar -xzf "${actionlintLocalArchive}" -C "${TEMP_DIR}/actionlint"`,
|
||||
executable: join(TEMP_DIR, 'actionlint', 'actionlint'),
|
||||
})}
|
||||
`,
|
||||
run: `
|
||||
actionlint \
|
||||
|
|
@ -112,8 +198,15 @@ function getLinters() {
|
|||
check: 'command -v shellcheck',
|
||||
installer: `
|
||||
mkdir -p "${TEMP_DIR}/shellcheck"
|
||||
curl -sSLo "${TEMP_DIR}/.shellcheck.txz" "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.${platformArch.shellcheck}.tar.xz"
|
||||
tar -xf "${TEMP_DIR}/.shellcheck.txz" -C "${TEMP_DIR}/shellcheck" --strip-components=1
|
||||
${getCachedArchiveInstaller({
|
||||
cacheArchive: shellcheckArchive,
|
||||
localArchive: shellcheckLocalArchive,
|
||||
expectedSha256: SHELLCHECK_SHA256[platformArch.shellcheck],
|
||||
downloadUrl: `https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.${platformArch.shellcheck}.tar.xz`,
|
||||
archiveCheck: `tar -tf "${shellcheckLocalArchive}" >/dev/null`,
|
||||
extract: `tar -xf "${shellcheckLocalArchive}" -C "${TEMP_DIR}/shellcheck" --strip-components=1`,
|
||||
executable: join(TEMP_DIR, 'shellcheck', 'shellcheck'),
|
||||
})}
|
||||
`,
|
||||
run: `
|
||||
git ls-files | grep -v '^integration-tests/terminal-bench/' | grep -E '^([^.]+|.*\\.(sh|zsh|bash))' | xargs file --mime-type \
|
||||
|
|
|
|||
|
|
@ -4,6 +4,18 @@
|
|||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { execSync, spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import {
|
||||
chmodSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
|
|
@ -11,7 +23,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|||
// paths so the suite also passes on the Windows gate.
|
||||
const toPosix = (value) => value.replaceAll(path.sep, '/');
|
||||
|
||||
describe('getLinterTempDir', () => {
|
||||
describe('linter directories', () => {
|
||||
const originalArgv = process.argv;
|
||||
|
||||
beforeEach(() => {
|
||||
|
|
@ -69,4 +81,145 @@ describe('getLinterTempDir', () => {
|
|||
expect(toPosix(second)).toMatch(/\/qwen-code-linters\/local-[a-f0-9]{16}$/);
|
||||
expect(first).not.toBe(second);
|
||||
});
|
||||
|
||||
it('shares cached downloads across GitHub Actions runs', async () => {
|
||||
const { getLinterCacheDir } = await import('../lint.js');
|
||||
|
||||
const first = getLinterCacheDir({
|
||||
env: {
|
||||
XDG_CACHE_HOME: '/runner/cache',
|
||||
GITHUB_RUN_ID: '31583913822',
|
||||
},
|
||||
});
|
||||
const second = getLinterCacheDir({
|
||||
env: {
|
||||
XDG_CACHE_HOME: '/runner/cache',
|
||||
GITHUB_RUN_ID: '31583913823',
|
||||
},
|
||||
});
|
||||
|
||||
expect(toPosix(first)).toBe('/runner/cache/qwen-code/linters');
|
||||
expect(second).toBe(first);
|
||||
expect(
|
||||
toPosix(getLinterCacheDir({ env: {}, homeDir: '/home/runner' })),
|
||||
).toBe('/home/runner/.cache/qwen-code/linters');
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === 'win32')(
|
||||
'verifies and reuses archives without depending on cache writes',
|
||||
async () => {
|
||||
const { getCachedArchiveInstaller } = await import('../lint.js');
|
||||
const root = mkdtempSync(path.join(tmpdir(), 'linter-cache-'));
|
||||
|
||||
try {
|
||||
const binDir = path.join(root, 'bin');
|
||||
const cacheArchive = path.join(root, 'cache', 'tool.tar');
|
||||
const localArchive = path.join(root, 'job', 'tool.tar');
|
||||
const executable = path.join(root, 'job', 'tool');
|
||||
const fixture = path.join(root, 'official.tar');
|
||||
const curlLog = path.join(root, 'curl.log');
|
||||
const curl = path.join(binDir, 'curl');
|
||||
mkdirSync(binDir, { recursive: true });
|
||||
mkdirSync(path.dirname(cacheArchive), { recursive: true });
|
||||
mkdirSync(path.dirname(localArchive), { recursive: true });
|
||||
writeFileSync(cacheArchive, 'validator-passing plant');
|
||||
writeFileSync(fixture, 'official archive');
|
||||
writeFileSync(
|
||||
curl,
|
||||
'#!/bin/sh\nprintf "download\\n" >> "$CURL_LOG"\nwhile [ "$#" -gt 0 ]; do\n if [ "$1" = "-o" ]; then\n if [ "$CORRUPT_DOWNLOAD" = "1" ]; then printf corrupt > "$2"; else cp "$FIXTURE_ARCHIVE" "$2"; fi\n exit\n fi\n shift\ndone\nexit 1\n',
|
||||
);
|
||||
chmodSync(curl, 0o755);
|
||||
|
||||
const expectedSha256 = createHash('sha256')
|
||||
.update(readFileSync(fixture))
|
||||
.digest('hex');
|
||||
expect(() =>
|
||||
getCachedArchiveInstaller({
|
||||
cacheArchive,
|
||||
localArchive,
|
||||
downloadUrl: 'https://example.invalid/unpinned.tar',
|
||||
}),
|
||||
).toThrow('Missing SHA-256 pin');
|
||||
const installer = getCachedArchiveInstaller({
|
||||
cacheArchive,
|
||||
localArchive,
|
||||
expectedSha256,
|
||||
downloadUrl: 'https://example.invalid/tool.tar',
|
||||
archiveCheck: 'true',
|
||||
extract: `cp "${localArchive}" "${executable}" && chmod +x "${executable}"`,
|
||||
executable,
|
||||
});
|
||||
const env = {
|
||||
...process.env,
|
||||
PATH: `${binDir}:${process.env.PATH}`,
|
||||
CURL_LOG: curlLog,
|
||||
CORRUPT_DOWNLOAD: '0',
|
||||
FIXTURE_ARCHIVE: fixture,
|
||||
};
|
||||
|
||||
expect(() =>
|
||||
execSync(installer, {
|
||||
env: { ...env, CORRUPT_DOWNLOAD: '1' },
|
||||
}),
|
||||
).toThrow();
|
||||
expect(readFileSync(cacheArchive, 'utf8')).toBe(
|
||||
'validator-passing plant',
|
||||
);
|
||||
|
||||
execSync(installer, { env });
|
||||
expect(readFileSync(cacheArchive, 'utf8')).toBe('official archive');
|
||||
expect(readFileSync(executable, 'utf8')).toBe('official archive');
|
||||
expect(readFileSync(curlLog, 'utf8')).toBe('download\ndownload\n');
|
||||
|
||||
rmSync(localArchive);
|
||||
rmSync(executable);
|
||||
const nonExecutableInstaller = getCachedArchiveInstaller({
|
||||
cacheArchive,
|
||||
localArchive,
|
||||
expectedSha256,
|
||||
downloadUrl: 'https://example.invalid/tool.tar',
|
||||
archiveCheck: 'true',
|
||||
extract: `cp "${localArchive}" "${executable}"`,
|
||||
executable,
|
||||
});
|
||||
expect(() => execSync(nonExecutableInstaller, { env })).toThrow();
|
||||
|
||||
rmSync(localArchive);
|
||||
rmSync(executable);
|
||||
rmSync(fixture);
|
||||
execSync(installer, { env });
|
||||
expect(readFileSync(executable, 'utf8')).toBe('official archive');
|
||||
expect(readFileSync(curlLog, 'utf8')).toBe('download\ndownload\n');
|
||||
|
||||
writeFileSync(fixture, 'official archive');
|
||||
rmSync(localArchive);
|
||||
rmSync(executable);
|
||||
rmSync(path.dirname(cacheArchive), { recursive: true });
|
||||
execSync(installer, { env });
|
||||
expect(readFileSync(cacheArchive, 'utf8')).toBe('official archive');
|
||||
expect(readFileSync(curlLog, 'utf8')).toBe(
|
||||
'download\ndownload\ndownload\n',
|
||||
);
|
||||
|
||||
rmSync(localArchive);
|
||||
rmSync(executable);
|
||||
rmSync(cacheArchive);
|
||||
mkdirSync(cacheArchive);
|
||||
const result = spawnSync(installer, { env, shell: true });
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stderr.toString()).toContain('EISDIR');
|
||||
expect(result.stderr.toString()).toContain(
|
||||
'Warning: could not persist linter archive',
|
||||
);
|
||||
expect(readFileSync(executable, 'utf8')).toBe('official archive');
|
||||
expect(statSync(cacheArchive).isDirectory()).toBe(true);
|
||||
expect(readFileSync(curlLog, 'utf8')).toBe(
|
||||
'download\ndownload\ndownload\ndownload\n',
|
||||
);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
15_000,
|
||||
);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue