diff --git a/integration-tests/globalSetup.test.ts b/integration-tests/globalSetup.test.ts index a3c26f1b16..a1954f4fbd 100644 --- a/integration-tests/globalSetup.test.ts +++ b/integration-tests/globalSetup.test.ts @@ -4,7 +4,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -83,3 +92,104 @@ describe('globalSetup memory-file save/restore', () => { await expect(teardown()).resolves.toBeUndefined(); }); }); + +describe('globalSetup hermetic qwen home', () => { + let tmpRoot: string; + let savedEnv: Map; + + beforeEach(async () => { + tmpRoot = await mkdtemp(join(tmpdir(), 'qwen-globalsetup-tmp-')); + savedEnv = new Map( + [...SETUP_ENV_KEYS, 'QWEN_HOME', 'TMPDIR'].map((key) => [ + key, + process.env[key], + ]), + ); + // The scratch home is only created when nothing has pinned QWEN_HOME, and + // it lands in the OS temp dir — redirect that so the case owns what the + // module picks. Both are read at import time, so they must be set first. + delete process.env['QWEN_HOME']; + process.env['TMPDIR'] = tmpRoot; + process.env['KEEP_OUTPUT'] = 'false'; + }); + + afterEach(async () => { + for (const [key, value] of savedEnv) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + vi.resetModules(); + await rm(tmpRoot, { recursive: true, force: true }); + }); + + async function loadGlobalSetup() { + vi.resetModules(); + return import('./globalSetup.js'); + } + + it('points the run at its own qwen home', async () => { + const { setup, teardown } = await loadGlobalSetup(); + await setup(); + + const home = process.env['QWEN_HOME']; + expect(home?.startsWith(tmpRoot)).toBe(true); + + await expect(teardown()).resolves.toBeUndefined(); + expect(existsSync(home!)).toBe(false); + }); + + it('does not exit an all-green run red when the scratch home cannot be removed', async () => { + // A CLI child that outlives its test keeps writing under `debug/`, so the + // removal walk reaches a directory that refills before the rmdir and + // throws ENOTEMPTY. That is how this cleanup first exited an all-green + // E2E run red, the same way the memory-file restore did in #10325. Stand + // in for that child with a writer that keeps the directory refilling — + // unlike a permission trick, it does not depend on not running as root. + const { setup, teardown } = await loadGlobalSetup(); + await setup(); + const debugDir = join(process.env['QWEN_HOME']!, 'debug'); + await mkdir(debugDir, { recursive: true }); + + const writer = spawn( + process.execPath, + [ + '-e', + // A tight loop, not a timer: the removal walk only fails when a file + // appears between its readdir and its rmdir, and a millisecond timer + // is far too slow to land inside that window. Self-limiting, so a + // child that outlives the kill below cannot spin indefinitely. + `const fs = require('node:fs'); + const dir = ${JSON.stringify(debugDir)}; + const stopAt = Date.now() + 30_000; + while (Date.now() < stopAt) { + try { + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(dir + '/' + process.hrtime.bigint() + '.log', 'x'); + } catch {} + }`, + ], + { stdio: 'ignore' }, + ); + + try { + // Node takes tens of milliseconds to boot, and the removal walk finishes + // in about one — without waiting for the writer to actually be producing, + // teardown would clear an idle directory and the case would pass against + // the very bug it exists to catch. + const deadline = Date.now() + 10_000; + while ((await readdir(debugDir)).length < 50) { + if (Date.now() > deadline) { + throw new Error('writer never started refilling the debug directory'); + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + + await expect(teardown()).resolves.toBeUndefined(); + } finally { + writer.kill('SIGKILL'); + } + }); +}); diff --git a/integration-tests/globalSetup.ts b/integration-tests/globalSetup.ts index bbd5d2b615..48cbdf42c7 100644 --- a/integration-tests/globalSetup.ts +++ b/integration-tests/globalSetup.ts @@ -10,13 +10,16 @@ if (process.env['NO_COLOR'] !== undefined) { } import { + copyFile, mkdir, readdir, rm, readFile, + stat, writeFile, unlink, } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -28,13 +31,116 @@ const integrationTestsDir = join(rootDir, '.integration-tests'); let runDir = ''; // Make runDir accessible in teardown let sdkE2eRunDir = ''; // SDK E2E test run directory +// Resolved before the redirect below, so setup() can still find the host's +// real global qwen dir. +const hostQwenDir = Storage.getGlobalQwenDir(); + +// These suites spawn the real CLI, which reads the global qwen dir for +// settings, saved memories, tool-usage history, and extensions. Inherited +// from the host, that dir is live user state and the run is only as +// reproducible as whatever happens to sit in it. Hosted runners have an empty +// `~/.qwen` and never noticed; the persistent pool (#10085) does not, and +// there a populated `~/.qwen/memories` made managed auto-memory recall issue +// its own model request ahead of the agent's first turn. The SDK suites +// script the fake OpenAI server by `requestIndex`, so that extra request +// shifted every index: each scripted tool call landed on the recall selector +// and the turn under test got the trailing text instead — 42 reds across +// permission-control and tool-control that no hosted runner could reproduce. +// +// Give the run its own global qwen dir, seeded with the host's configuration +// but none of its accumulated files (see carryOverHostConfig). It lives +// outside the worktree so that copy never lands in a tracked tree. A caller +// that pins QWEN_HOME — globalSetup.test.ts, or someone reproducing against +// real state — keeps it, and owns its lifecycle. +const HERMETIC_HOME_PREFIX = 'qwen-e2e-home-'; +const hermeticQwenHome = join( + tmpdir(), + `${HERMETIC_HOME_PREFIX}${process.pid}-${Date.now()}`, +); +const ownsQwenHome = !process.env['QWEN_HOME']; +if (ownsQwenHome) { + process.env['QWEN_HOME'] = hermeticQwenHome; +} + +// Read after the redirect so the save/restore below, the spawned CLIs, and +// the tests all agree on one global qwen dir. const memoryFilePath = join( Storage.getGlobalQwenDir(), DEFAULT_CONTEXT_FILENAME, ); let originalMemoryContent: string | null = null; +/** + * Carries the host's configuration — and only that — into the hermetic global + * qwen dir. + * + * The suites that talk to a real model rely on ambient auth. CI supplies it + * through the environment, but a developer's typically lives in + * `~/.qwen/settings.json`: as credentials under `security.auth`, as provider + * keys in the `env` block, or as routing in `model` / `modelProviders`. There + * is no subset of those that is safe to carry alone, so the file goes across + * whole and a developer's setup keeps working exactly as it does today. + * + * What deliberately does not come across is everything the dir accumulates as + * files: saved memories, tool-usage history, extensions, skills, commands. + * That is the state a run has no business depending on, and the state that + * made these suites fail. Carrying settings.json forward keeps whatever the + * persistent pool relies on for credentials — its `~/.qwen` is populated, + * which is how the memories got there — so this narrows the blast radius to + * the files without gambling on where CI's auth comes from. + */ +async function carryOverHostConfig() { + for (const fileName of ['settings.json', 'oauth_creds.json']) { + await copyFile( + join(hostQwenDir, fileName), + join(hermeticQwenHome, fileName), + ).catch(() => { + // Absent on CI and on a machine that has never run the CLI, and absent + // for whichever auth type the developer is not using. + }); + } +} + +/** + * Removes scratch homes an earlier run could not. Teardown's cleanup is + * best-effort by design, so on a persistent runner the ones it gives up on + * would otherwise pile up forever. The age floor keeps this clear of any run + * in flight on the same host: a full E2E run finishes well inside it. + */ +async function sweepLeakedQwenHomes() { + const cutoff = Date.now() - 24 * 60 * 60 * 1000; + try { + const entries = await readdir(tmpdir()); + await Promise.all( + entries + .filter( + (entry) => + entry.startsWith(HERMETIC_HOME_PREFIX) && + join(tmpdir(), entry) !== hermeticQwenHome, + ) + .map(async (entry) => { + const dir = join(tmpdir(), entry); + try { + if ((await stat(dir)).mtimeMs < cutoff) { + await rm(dir, { recursive: true, force: true, maxRetries: 3 }); + } + } catch { + // Raced with the run that owns it, or still not removable. + } + }), + ); + } catch { + // Housekeeping must never fail a run. + } +} + export async function setup() { + if (ownsQwenHome) { + await mkdir(hermeticQwenHome, { recursive: true }); + await carryOverHostConfig(); + await sweepLeakedQwenHomes(); + } + try { originalMemoryContent = await readFile(memoryFilePath, 'utf-8'); } catch (e) { @@ -118,6 +224,34 @@ export async function teardown() { await rm(sdkE2eRunDir, { recursive: true, force: true }); } + // Only when the memory file is the host's. Under a hermetic home it sits in + // the scratch dir removed just below, so there is nothing to put back. + if (!ownsQwenHome) { + await restoreMemoryFile(); + } + + // Not gated on KEEP_OUTPUT: this is a scratch dir rather than a test + // artifact, and it holds a copy of the developer's credentials. + if (ownsQwenHome) { + try { + // A CLI child outliving its test keeps writing under `debug/`, so the + // walk can reach a directory that refills before the rmdir — retries + // absorb that. The catch is what matters: a cleanup that cannot finish + // must not exit an all-green run red, the way the memory-file restore + // did in #10325. + await rm(hermeticQwenHome, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 200, + }); + } catch (e) { + console.error(`Warning: could not remove ${hermeticQwenHome}:`, e); + } + } +} + +async function restoreMemoryFile() { if (originalMemoryContent !== null) { try { await mkdir(dirname(memoryFilePath), { recursive: true });