mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-08-01 12:24:51 +00:00
* test(integration): migrate flaky E2E tests to fake-openai-server Migrate 39 real-model test cases to use deterministic fake-openai-server scripting, eliminating model output variance as a failure source. - tool-control.test.ts: 23 cases migrated (tool filtering, permission denial, canUseTool routing, priority rules) - abort-and-lifecycle.test.ts: 15 cases migrated (abort mechanics, lifecycle, cleanup, debug output) - list_directory.test.ts: 1 case migrated (CLI TestRig with env injection) channel-plugin.test.ts (3 cases) is intentionally left on real model — it tests the full WebSocket→AcpBridge→model pipeline and belongs in the nightly smoke layer per #7616. Closes #7616 * test(ci): move channel-plugin to nightly + relax assertions channel-plugin.test.ts tests the full WebSocket→AcpBridge→model pipeline with real model inference. Its assertions on specific model output ('4', 'pineapple', '50') are inherently non-deterministic. - Exclude from post-merge E2E jobs (Linux + macOS) - Add dedicated nightly-only job with continue-on-error - Relax assertions: verify response is non-empty rather than matching specific model output content - Add retry: 2 to each test case * fix(test): use streaming chunks for abort test, restore comment - abort-and-lifecycle 'should handle abort during query execution': switch from non-streaming content to contentChunks so the abort signal reliably lands while the response is still streaming - channel-plugin: restore existing comment per AGENTS.md convention * test(integration): address fake server review feedback * test(integration): trim fake server review assertions * test(integration): tighten fake server cleanup * test(integration): fix permanently-failing stdin-close case and harden list_directory against user settings The stdin-close case's fake server needle contained a raw quote that JSON.stringify-escaped transcripts can never match, and the scripted write_file was rejected by prior-read enforcement; the script now keys off a quote-free marker and reads before writing. list_directory now pins auth via CLI flags so a developer's ~/.qwen/settings.json cannot silently route the run to a real model endpoint. * test(integration): tighten fake server review regressions * test(integration): guard abort assertions * test(integration): fix abort timer race and dead coreTools assertions * test(integration): address fake server review findings * test(integration): isolate fake fast model settings * test(e2e): simplify isolated test setup --------- Co-authored-by: qwen-code-dev-bot <qwen-code-dev-bot@users.noreply.github.com> Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> Co-authored-by: Shaojin Wen <shaojin.wensj@alibaba-inc.com>
119 lines
3.7 KiB
TypeScript
119 lines
3.7 KiB
TypeScript
/**
|
|
* @license
|
|
* Copyright 2025 Google LLC
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
import { describe, it, expect, afterEach, vi } from 'vitest';
|
|
import {
|
|
fakeServerHostOptions,
|
|
IS_CONTAINER_SANDBOX,
|
|
CONTAINER_SANDBOX_NO_PROXY,
|
|
TestRig,
|
|
} from '../test-helper.js';
|
|
import { fakeToolCall, startFakeOpenAIServer } from '../fake-openai-server.js';
|
|
import { existsSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
|
|
describe('list_directory', () => {
|
|
afterEach(() => {
|
|
vi.unstubAllEnvs();
|
|
});
|
|
|
|
it('should be able to list a directory', async () => {
|
|
const rig = new TestRig();
|
|
await rig.setup('should be able to list a directory');
|
|
rig.createFile('file1.txt', 'file 1 content');
|
|
rig.mkdir('subdir');
|
|
rig.sync();
|
|
|
|
await rig.poll(
|
|
() => {
|
|
const file1Path = join(rig.testDir!, 'file1.txt');
|
|
const subdirPath = join(rig.testDir!, 'subdir');
|
|
return existsSync(file1Path) && existsSync(subdirPath);
|
|
},
|
|
1000,
|
|
50,
|
|
);
|
|
|
|
const noProxy = IS_CONTAINER_SANDBOX
|
|
? CONTAINER_SANDBOX_NO_PROXY
|
|
: '127.0.0.1,localhost';
|
|
|
|
let streamingRequestIndex = 0;
|
|
const fakeServer = await startFakeOpenAIServer(({ body }) => {
|
|
if (body['stream'] !== true) {
|
|
return { content: '{"selected_memories":[]}' };
|
|
}
|
|
const requestIndex = streamingRequestIndex++;
|
|
if (requestIndex === 0) {
|
|
return {
|
|
toolCalls: [
|
|
fakeToolCall('list_directory', { path: rig.testDir! }, 'list-dir'),
|
|
],
|
|
};
|
|
}
|
|
return { content: 'The directory contains file1.txt and subdir.' };
|
|
}, fakeServerHostOptions());
|
|
|
|
vi.stubEnv('OPENAI_API_KEY', 'fake-key');
|
|
vi.stubEnv('OPENAI_BASE_URL', fakeServer.baseUrl);
|
|
vi.stubEnv('OPENAI_MODEL', 'fake-model');
|
|
vi.stubEnv('QWEN_MODEL', 'fake-model');
|
|
vi.stubEnv('QWEN_HOME', join(rig.testDir!, '.qwen-home'));
|
|
vi.stubEnv('QWEN_RUNTIME_DIR', join(rig.testDir!, '.qwen-home'));
|
|
vi.stubEnv('NO_PROXY', noProxy);
|
|
vi.stubEnv('no_proxy', noProxy);
|
|
|
|
try {
|
|
const prompt = `Call the list_directory tool on the current directory.`;
|
|
// Explicit CLI flags outrank a developer's ~/.qwen/settings.json
|
|
// (settings.model.name beats the OPENAI_MODEL env var and can silently
|
|
// route the run to a real model endpoint instead of the fake server).
|
|
await rig.run(
|
|
prompt,
|
|
'--auth-type',
|
|
'openai',
|
|
'--model',
|
|
'fake-model',
|
|
'--openai-base-url',
|
|
fakeServer.baseUrl,
|
|
'--openai-api-key',
|
|
'fake-key',
|
|
);
|
|
|
|
const foundToolCall = await rig.waitForToolCall('list_directory');
|
|
|
|
expect(foundToolCall, 'Expected a list_directory tool call').toBe(true);
|
|
|
|
const toolResultRequest = fakeServer.requests.find(({ body }) => {
|
|
const messages = body['messages'];
|
|
return (
|
|
Array.isArray(messages) &&
|
|
messages.some(
|
|
(message) =>
|
|
typeof message === 'object' &&
|
|
message !== null &&
|
|
'role' in message &&
|
|
message.role === 'tool',
|
|
)
|
|
);
|
|
});
|
|
expect(
|
|
toolResultRequest,
|
|
'Expected a model request containing the list_directory result',
|
|
).toBeDefined();
|
|
const messages = toolResultRequest?.body['messages'] as
|
|
| Array<{ role?: string; content?: unknown }>
|
|
| undefined;
|
|
const toolResultContent = JSON.stringify(
|
|
messages?.find((message) => message.role === 'tool')?.content ?? '',
|
|
);
|
|
expect(toolResultContent).toContain('file1.txt');
|
|
expect(toolResultContent).toContain('subdir');
|
|
} finally {
|
|
await fakeServer.close();
|
|
}
|
|
});
|
|
});
|