mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-04-28 11:41:04 +00:00
refactor(tests): reorganize integration tests by execution mode
Move non-interactive tests to cli/, interactive tests to interactive/. Add cron-interactive.test.ts wrapping terminal-capture E2E in vitest. Update npm scripts and release workflow for new directory layout.
This commit is contained in:
tanzhenxin
parent
707b06ca48
commit
ded89618ec
28 changed files with 261 additions and 327 deletions
310
integration-tests/cli/json-output.test.ts
Normal file
310
integration-tests/cli/json-output.test.ts
Normal file
|
|
@ -0,0 +1,310 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { expect, describe, it, beforeEach, afterEach } from 'vitest';
|
||||
import { TestRig } from '../test-helper.js';
|
||||
|
||||
describe('JSON output', () => {
|
||||
let rig: TestRig;
|
||||
|
||||
beforeEach(async () => {
|
||||
rig = new TestRig();
|
||||
await rig.setup('json-output-test');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rig.cleanup();
|
||||
});
|
||||
|
||||
it('should return a valid JSON array with result message containing response and stats', async () => {
|
||||
const result = await rig.run(
|
||||
'What is the capital of France?',
|
||||
'--output-format',
|
||||
'json',
|
||||
);
|
||||
const parsed = JSON.parse(result);
|
||||
|
||||
// The output should be an array of messages
|
||||
expect(Array.isArray(parsed)).toBe(true);
|
||||
expect(parsed.length).toBeGreaterThan(0);
|
||||
|
||||
// Find the result message (should be the last message)
|
||||
const resultMessage = parsed.find(
|
||||
(msg: unknown) =>
|
||||
typeof msg === 'object' &&
|
||||
msg !== null &&
|
||||
'type' in msg &&
|
||||
msg.type === 'result',
|
||||
);
|
||||
|
||||
expect(resultMessage).toBeDefined();
|
||||
expect(resultMessage).toHaveProperty('is_error');
|
||||
expect(resultMessage.is_error).toBe(false);
|
||||
expect(resultMessage).toHaveProperty('result');
|
||||
expect(typeof resultMessage.result).toBe('string');
|
||||
expect(resultMessage.result.toLowerCase()).toContain('paris');
|
||||
|
||||
// Stats may be present if available
|
||||
if ('stats' in resultMessage) {
|
||||
expect(typeof resultMessage.stats).toBe('object');
|
||||
}
|
||||
});
|
||||
|
||||
it('should return line-delimited JSON messages for stream-json output format', async () => {
|
||||
const result = await rig.run(
|
||||
'What is the capital of France?',
|
||||
'--output-format',
|
||||
'stream-json',
|
||||
);
|
||||
|
||||
// Stream-json output is line-delimited JSON (one JSON object per line)
|
||||
const lines = result
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter((line) => line.trim());
|
||||
expect(lines.length).toBeGreaterThan(0);
|
||||
|
||||
// Parse each line as a JSON object
|
||||
const messages: unknown[] = [];
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
messages.push(parsed);
|
||||
} catch (parseError) {
|
||||
throw new Error(
|
||||
`Failed to parse JSON line: ${line}. Error: ${parseError}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Should have at least system, assistant, and result messages
|
||||
expect(messages.length).toBeGreaterThanOrEqual(3);
|
||||
|
||||
// Find system message
|
||||
const systemMessage = messages.find(
|
||||
(msg: unknown) =>
|
||||
typeof msg === 'object' &&
|
||||
msg !== null &&
|
||||
'type' in msg &&
|
||||
msg.type === 'system',
|
||||
);
|
||||
expect(systemMessage).toBeDefined();
|
||||
expect(systemMessage).toHaveProperty('subtype');
|
||||
expect(systemMessage).toHaveProperty('session_id');
|
||||
|
||||
// Find assistant message
|
||||
const assistantMessage = messages.find(
|
||||
(msg: unknown) =>
|
||||
typeof msg === 'object' &&
|
||||
msg !== null &&
|
||||
'type' in msg &&
|
||||
msg.type === 'assistant',
|
||||
);
|
||||
expect(assistantMessage).toBeDefined();
|
||||
expect(assistantMessage).toHaveProperty('message');
|
||||
expect(assistantMessage).toHaveProperty('session_id');
|
||||
|
||||
// Find result message (should be the last message)
|
||||
const resultMessage = messages[messages.length - 1] as {
|
||||
type: string;
|
||||
is_error: boolean;
|
||||
result: string;
|
||||
};
|
||||
expect(resultMessage).toBeDefined();
|
||||
expect(
|
||||
typeof resultMessage === 'object' &&
|
||||
resultMessage !== null &&
|
||||
'type' in resultMessage &&
|
||||
resultMessage.type === 'result',
|
||||
).toBe(true);
|
||||
expect(resultMessage).toHaveProperty('is_error');
|
||||
expect(resultMessage.is_error).toBe(false);
|
||||
expect(resultMessage).toHaveProperty('result');
|
||||
expect(typeof resultMessage.result).toBe('string');
|
||||
expect(resultMessage.result.toLowerCase()).toContain('paris');
|
||||
});
|
||||
|
||||
it('should include stream events when using stream-json with include-partial-messages', async () => {
|
||||
const result = await rig.run(
|
||||
'What is the capital of France?',
|
||||
'--output-format',
|
||||
'stream-json',
|
||||
'--include-partial-messages',
|
||||
);
|
||||
|
||||
// Stream-json output is line-delimited JSON (one JSON object per line)
|
||||
const lines = result
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter((line) => line.trim());
|
||||
expect(lines.length).toBeGreaterThan(0);
|
||||
|
||||
// Parse each line as a JSON object
|
||||
const messages: unknown[] = [];
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
messages.push(parsed);
|
||||
} catch (parseError) {
|
||||
throw new Error(
|
||||
`Failed to parse JSON line: ${line}. Error: ${parseError}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Should have more messages than without include-partial-messages
|
||||
// because we're including stream events
|
||||
expect(messages.length).toBeGreaterThan(3);
|
||||
|
||||
// Find stream_event messages
|
||||
const streamEvents = messages.filter(
|
||||
(msg: unknown) =>
|
||||
typeof msg === 'object' &&
|
||||
msg !== null &&
|
||||
'type' in msg &&
|
||||
msg.type === 'stream_event',
|
||||
);
|
||||
expect(streamEvents.length).toBeGreaterThan(0);
|
||||
|
||||
// Verify stream event structure
|
||||
const firstStreamEvent = streamEvents[0];
|
||||
expect(firstStreamEvent).toHaveProperty('event');
|
||||
expect(firstStreamEvent).toHaveProperty('session_id');
|
||||
expect(firstStreamEvent).toHaveProperty('uuid');
|
||||
|
||||
// Check for expected stream event types
|
||||
const eventTypes = streamEvents.map((event: unknown) =>
|
||||
typeof event === 'object' &&
|
||||
event !== null &&
|
||||
'event' in event &&
|
||||
typeof event.event === 'object' &&
|
||||
event.event !== null &&
|
||||
'type' in event.event
|
||||
? event.event.type
|
||||
: null,
|
||||
);
|
||||
|
||||
// Should have message_start event
|
||||
expect(eventTypes).toContain('message_start');
|
||||
|
||||
// Should have content_block_start event
|
||||
expect(eventTypes).toContain('content_block_start');
|
||||
|
||||
// Should have content_block_delta events
|
||||
expect(eventTypes).toContain('content_block_delta');
|
||||
|
||||
// Should have content_block_stop event
|
||||
expect(eventTypes).toContain('content_block_stop');
|
||||
|
||||
// Should have message_stop event
|
||||
expect(eventTypes).toContain('message_stop');
|
||||
|
||||
// Verify that we still have the complete assistant message
|
||||
const assistantMessage = messages.find(
|
||||
(msg: unknown) =>
|
||||
typeof msg === 'object' &&
|
||||
msg !== null &&
|
||||
'type' in msg &&
|
||||
msg.type === 'assistant',
|
||||
);
|
||||
expect(assistantMessage).toBeDefined();
|
||||
expect(assistantMessage).toHaveProperty('message');
|
||||
|
||||
// Verify that we still have the result message
|
||||
const resultMessage = messages[messages.length - 1] as {
|
||||
type: string;
|
||||
is_error: boolean;
|
||||
result: string;
|
||||
};
|
||||
expect(resultMessage).toBeDefined();
|
||||
expect(
|
||||
typeof resultMessage === 'object' &&
|
||||
resultMessage !== null &&
|
||||
'type' in resultMessage &&
|
||||
resultMessage.type === 'result',
|
||||
).toBe(true);
|
||||
expect(resultMessage).toHaveProperty('is_error');
|
||||
expect(resultMessage.is_error).toBe(false);
|
||||
expect(resultMessage).toHaveProperty('result');
|
||||
expect(resultMessage.result.toLowerCase()).toContain('paris');
|
||||
});
|
||||
|
||||
it('should return a JSON error for enforced auth mismatch before running', async () => {
|
||||
const originalOpenaiApiKey = process.env['OPENAI_API_KEY'];
|
||||
process.env['OPENAI_API_KEY'] = 'test-key';
|
||||
await rig.setup('json-output-auth-mismatch', {
|
||||
settings: {
|
||||
security: { auth: { enforcedType: 'qwen-oauth' } },
|
||||
},
|
||||
});
|
||||
|
||||
let thrown: Error | undefined;
|
||||
try {
|
||||
await rig.run('Hello', '--output-format', 'json');
|
||||
expect.fail('Expected process to exit with error');
|
||||
} catch (e) {
|
||||
thrown = e as Error;
|
||||
} finally {
|
||||
process.env['OPENAI_API_KEY'] = originalOpenaiApiKey;
|
||||
}
|
||||
|
||||
expect(thrown).toBeDefined();
|
||||
const message = (thrown as Error).message;
|
||||
|
||||
// The error JSON is written to stdout as a CLIResultMessageError
|
||||
// Extract stdout from the error message
|
||||
const stdoutMatch = message.match(/Stdout:\n([\s\S]*?)(?:\n\nStderr:|$)/);
|
||||
expect(
|
||||
stdoutMatch,
|
||||
'Expected to find stdout in the error message',
|
||||
).toBeTruthy();
|
||||
|
||||
const stdout = stdoutMatch![1];
|
||||
let parsed: unknown[];
|
||||
try {
|
||||
// Parse the JSON array from stdout
|
||||
parsed = JSON.parse(stdout);
|
||||
} catch (parseError) {
|
||||
console.error('Failed to parse the following JSON:', stdout);
|
||||
throw new Error(
|
||||
`Test failed: Could not parse JSON from stdout. Details: ${parseError}`,
|
||||
);
|
||||
}
|
||||
|
||||
// The output should be an array of messages
|
||||
expect(Array.isArray(parsed)).toBe(true);
|
||||
expect(parsed.length).toBeGreaterThan(0);
|
||||
|
||||
// Find the result message with error
|
||||
const resultMessage = parsed.find(
|
||||
(msg: unknown) =>
|
||||
typeof msg === 'object' &&
|
||||
msg !== null &&
|
||||
'type' in msg &&
|
||||
msg.type === 'result' &&
|
||||
'is_error' in msg &&
|
||||
msg.is_error === true,
|
||||
) as {
|
||||
type: string;
|
||||
is_error: boolean;
|
||||
subtype: string;
|
||||
error?: { message: string; type?: string };
|
||||
};
|
||||
|
||||
expect(resultMessage).toBeDefined();
|
||||
expect(resultMessage.is_error).toBe(true);
|
||||
expect(resultMessage).toHaveProperty('subtype');
|
||||
expect(resultMessage.subtype).toBe('error_during_execution');
|
||||
expect(resultMessage).toHaveProperty('error');
|
||||
expect(resultMessage.error).toBeDefined();
|
||||
expect(resultMessage.error?.message).toContain(
|
||||
'configured auth type is qwen-oauth',
|
||||
);
|
||||
expect(resultMessage.error?.message).toContain(
|
||||
'current auth type is openai',
|
||||
);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue