fix(core): stop computer use driver when idle (#5925)
Some checks are pending
E2E Tests / E2E Test (Linux) - sandbox:docker (push) Waiting to run
E2E Tests / E2E Test (Linux) - sandbox:none (push) Waiting to run
E2E Tests / E2E Test - macOS (push) Waiting to run

This commit is contained in:
tt-a1i 2026-06-27 19:18:01 +08:00 committed by GitHub
parent 6a5c041885
commit 8e8ae1ed11
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 395 additions and 51 deletions

View file

@ -1953,6 +1953,7 @@ export async function loadCliConfig(
computerUseEnabled: settings.tools?.computerUse?.enabled ?? true,
computerUseMaxImageDimension:
settings.tools?.computerUse?.maxImageDimension,
computerUseIdleTimeoutMs: settings.tools?.computerUse?.idleTimeoutMs,
emitToolUseSummaries: settings.experimental?.emitToolUseSummaries ?? true,
listExtensions: argv.listExtensions || false,
locale: resolveLocaleForExtensions(settings),

View file

@ -88,6 +88,8 @@ export interface SettingDefinition {
items?: SettingItemDefinition;
/** Minimum value for number-type settings. */
minimum?: number;
/** Maximum value for number-type settings. */
maximum?: number;
/**
* Primitive shapes a field accepted before it was expanded to its current
* type. The exported JSON Schema wraps the field in `anyOf` so values from
@ -2235,9 +2237,21 @@ const SETTINGS_SCHEMA = {
requiresRestart: true,
default: true,
description:
'When enabled (default), the cua-driver computer_use__* tools are registered as deferred built-ins.',
'When enabled (default), the cua-driver computer_use__* tools are registered as deferred built-ins. Set to false to prevent the driver from being downloaded or spawned.',
showInDialog: true,
},
idleTimeoutMs: {
type: 'number',
label: 'Idle Timeout',
category: 'Tools',
requiresRestart: true,
default: 300000,
minimum: 0,
maximum: 2147483647,
description:
'Milliseconds to keep the cua-driver process alive after the last computer_use__* call. The default is 300000 (5 minutes). Set to 0 to keep it running until qwen-code exits.',
showInDialog: false,
},
maxImageDimension: {
type: 'number',
label: 'Max Screenshot Dimension',

View file

@ -90,6 +90,16 @@ describe('SettingsUtils', () => {
description: 'A number field with a minimum.',
showInDialog: true,
},
numberWithMaximum: {
type: 'number',
label: 'Number With Maximum',
category: 'Basic',
requiresRestart: false,
default: 10,
maximum: 10,
description: 'A number field with a maximum.',
showInDialog: true,
},
advanced: {
type: 'object',
label: 'Advanced',
@ -239,6 +249,15 @@ describe('SettingsUtils', () => {
'Value must be >= 0',
);
});
it('rejects numbers above the configured maximum', () => {
const definition = getSettingDefinition('numberWithMaximum');
expect(definition).toBeDefined();
expect(validateSettingValue(definition!, 11)).toBe(
'Value must be <= 10',
);
});
});
describe('requiresRestart', () => {

View file

@ -324,6 +324,8 @@ export function validateSettingValue(
return 'Value must be a finite number';
if (def.minimum !== undefined && value < def.minimum)
return `Value must be >= ${def.minimum}`;
if (def.maximum !== undefined && value > def.maximum)
return `Value must be <= ${def.maximum}`;
break;
case 'string':
if (typeof value !== 'string') return 'Value must be a string';

View file

@ -4676,6 +4676,28 @@ describe('disabledTools runtime sync (#4282 fold-in 5 P2-2 / #4297 fold-in 5)',
});
});
describe('computer use settings', () => {
const baseParams: ConfigParameters = {
targetDir: '.',
debugMode: false,
model: 'test-model',
cwd: '.',
};
it('exposes the configured idle timeout', () => {
const config = new Config({
...baseParams,
computerUseIdleTimeoutMs: 12_345,
});
expect(config.getComputerUseIdleTimeoutMs()).toBe(12_345);
});
it('leaves the idle timeout undefined when not configured', () => {
const config = new Config(baseParams);
expect(config.getComputerUseIdleTimeoutMs()).toBeUndefined();
});
});
describe('BaseLlmClient Lifecycle', () => {
const MODEL = 'gemini-pro';
const SANDBOX: SandboxConfig = {

View file

@ -879,6 +879,7 @@ export interface ConfigParameters {
skipWorkflowUsageWarning?: boolean;
computerUseEnabled?: boolean;
computerUseMaxImageDimension?: number;
computerUseIdleTimeoutMs?: number;
emitToolUseSummaries?: boolean;
listExtensions?: boolean;
overrideExtensions?: string[];
@ -1386,6 +1387,7 @@ export class Config {
private readonly skipWorkflowUsageWarning: boolean = false;
private readonly computerUseEnabled: boolean = true;
private readonly computerUseMaxImageDimension?: number;
private readonly computerUseIdleTimeoutMs?: number;
private readonly emitToolUseSummaries: boolean = true;
private readonly chatRecordingEnabled: boolean;
private readonly loadMemoryFromIncludeDirectories: boolean = false;
@ -1604,6 +1606,7 @@ export class Config {
this.skipWorkflowUsageWarning = params.skipWorkflowUsageWarning ?? false;
this.computerUseEnabled = params.computerUseEnabled ?? true;
this.computerUseMaxImageDimension = params.computerUseMaxImageDimension;
this.computerUseIdleTimeoutMs = params.computerUseIdleTimeoutMs;
this.emitToolUseSummaries = params.emitToolUseSummaries ?? true;
this.listExtensions = params.listExtensions ?? false;
this.overrideExtensions = params.overrideExtensions;
@ -4449,6 +4452,10 @@ export class Config {
return this.computerUseMaxImageDimension;
}
getComputerUseIdleTimeoutMs(): number | undefined {
return this.computerUseIdleTimeoutMs;
}
/**
* Whether the turn loop should fire a fast-model call after each tool batch
* to emit a `tool_use_summary` message. Mirrors Claude Code's

View file

@ -1,5 +1,10 @@
import { describe, it, expect, vi } from 'vitest';
import { ComputerUseClient, isTransportClosedError } from './client.js';
import { afterEach, describe, it, expect, vi } from 'vitest';
import {
ComputerUseClient,
DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS,
isTransportClosedError,
MAX_COMPUTER_USE_IDLE_TIMEOUT_MS,
} from './client.js';
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
describe('ComputerUseClient', () => {
@ -100,6 +105,201 @@ describe('applyRuntimeConfig (set_config on connect)', () => {
});
});
// ---------------------------------------------------------------------------
// Idle shutdown
// ---------------------------------------------------------------------------
describe('idle shutdown', () => {
type FakeInner = {
callTool: ReturnType<typeof vi.fn>;
close: ReturnType<typeof vi.fn>;
};
afterEach(() => {
vi.useRealTimers();
});
const installInner = (c: ComputerUseClient, inner: FakeInner): void => {
(c as unknown as { client: FakeInner }).client = inner;
};
const makeInner = (): FakeInner => ({
callTool: vi.fn().mockResolvedValue(successResult),
close: vi.fn().mockResolvedValue(undefined),
});
it('defaults to a 5 minute idle timeout', async () => {
vi.useFakeTimers();
const c = new ComputerUseClient({ binary: '/fake/cua-driver' });
const inner = makeInner();
installInner(c, inner);
await c.callTool('list_windows', {});
await vi.advanceTimersByTimeAsync(DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS - 1);
expect(inner.close).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
expect(inner.close).toHaveBeenCalledTimes(1);
expect(c.isStarted()).toBe(false);
});
it('treats negative idle timeouts as invalid and falls back to the default', async () => {
vi.useFakeTimers();
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
idleTimeoutMs: -1,
});
const inner = makeInner();
installInner(c, inner);
await c.callTool('list_windows', {});
await vi.advanceTimersByTimeAsync(DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS - 1);
expect(inner.close).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
expect(inner.close).toHaveBeenCalledTimes(1);
});
it('uses the configured idle timeout after the last tool call', async () => {
vi.useFakeTimers();
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
idleTimeoutMs: 25,
});
const inner = makeInner();
installInner(c, inner);
await c.callTool('get_app_state', {});
await vi.advanceTimersByTimeAsync(24);
expect(inner.close).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
expect(inner.close).toHaveBeenCalledTimes(1);
});
it('does not stop while another tool call is still active', async () => {
vi.useFakeTimers();
let releaseFirst!: (result: CallToolResult) => void;
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
idleTimeoutMs: 25,
});
const inner: FakeInner = {
callTool: vi
.fn()
.mockImplementationOnce(
() =>
new Promise<CallToolResult>((resolve) => {
releaseFirst = resolve;
}),
)
.mockResolvedValueOnce(successResult),
close: vi.fn().mockResolvedValue(undefined),
};
installInner(c, inner);
const first = c.callTool('get_window_state', {});
await Promise.resolve();
const second = c.callTool('click', {});
await second;
await vi.advanceTimersByTimeAsync(25);
expect(inner.close).not.toHaveBeenCalled();
releaseFirst(successResult);
await first;
await vi.advanceTimersByTimeAsync(25);
expect(inner.close).toHaveBeenCalledTimes(1);
});
it('disables idle shutdown when configured as 0', async () => {
vi.useFakeTimers();
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
idleTimeoutMs: 0,
});
const inner = makeInner();
installInner(c, inner);
await c.callTool('list_windows', {});
await vi.advanceTimersByTimeAsync(60 * 60 * 1000);
expect(inner.close).not.toHaveBeenCalled();
expect(c.isStarted()).toBe(true);
});
it('reschedules the idle timer when setIdleTimeoutMs is called while pending', async () => {
vi.useFakeTimers();
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
idleTimeoutMs: 25,
});
const inner = makeInner();
installInner(c, inner);
await c.callTool('list_windows', {});
await vi.advanceTimersByTimeAsync(20);
c.setIdleTimeoutMs(100);
await vi.advanceTimersByTimeAsync(5);
expect(inner.close).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(95);
expect(inner.close).toHaveBeenCalledTimes(1);
});
it('cancels the pending idle timer when stop() is called explicitly', async () => {
vi.useFakeTimers();
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
idleTimeoutMs: 25,
});
const inner = makeInner();
installInner(c, inner);
await c.callTool('list_windows', {});
await c.stop();
expect(inner.close).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(100);
expect(inner.close).toHaveBeenCalledTimes(1);
});
it.each([NaN, Infinity, -Infinity])(
'treats %p as invalid and falls back to the default',
async (value) => {
vi.useFakeTimers();
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
idleTimeoutMs: value,
});
const inner = makeInner();
installInner(c, inner);
await c.callTool('list_windows', {});
await vi.advanceTimersByTimeAsync(
DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS - 1,
);
expect(inner.close).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
expect(inner.close).toHaveBeenCalledTimes(1);
},
);
it('treats idle timeouts above the setTimeout safe range as invalid', async () => {
vi.useFakeTimers();
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
idleTimeoutMs: MAX_COMPUTER_USE_IDLE_TIMEOUT_MS + 1,
});
const inner = makeInner();
installInner(c, inner);
await c.callTool('list_windows', {});
await vi.advanceTimersByTimeAsync(DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS - 1);
expect(inner.close).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
expect(inner.close).toHaveBeenCalledTimes(1);
});
});
// ---------------------------------------------------------------------------
// isTransportClosedError unit tests
// ---------------------------------------------------------------------------

View file

@ -13,6 +13,9 @@ import type {
import { homedir } from 'node:os';
import { binaryPath } from './constants.js';
export const DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS = 5 * 60 * 1000;
export const MAX_COMPUTER_USE_IDLE_TIMEOUT_MS = 2_147_483_647;
/**
* Singleton stdio MCP client for the cua-driver binary.
*
@ -21,11 +24,11 @@ import { binaryPath } from './constants.js';
* downloads + verifies it before the first spawn). Spawns are sub-second
* there is no npx/download cost on this path anymore.
*
* Lifecycle: lazy spawn on first `callTool` invocation. The process
* stays alive until `stop()` or qwen-code exits. State (element_index
* map per window) lives in the process if the process restarts, the
* model must call `get_window_state` again before any element-targeted
* action.
* Lifecycle: lazy spawn on first `callTool` invocation. The process is
* stopped by `stop()`, qwen-code exit, or the idle timeout after the last
* tool call. State (element_index map per window) lives in the process if
* the process restarts, the model must call `get_window_state` again before
* any element-targeted action.
*/
export interface ComputerUseClientOptions {
/** Absolute path to the spawnable `cua-driver` binary. */
@ -38,6 +41,11 @@ export interface ComputerUseClientOptions {
* (1568) untouched; `0` disables resizing. See {@link resolveMaxImageDimension}.
*/
maxImageDimension?: number;
/**
* How long an idle cua-driver client stays alive after the last tool call.
* `0` disables automatic idle shutdown.
*/
idleTimeoutMs?: number;
}
export class ComputerUseClient {
@ -46,13 +54,17 @@ export class ComputerUseClient {
private readonly binary: string;
private readonly onProgress: (message: string) => void;
private maxImageDimension: number | undefined;
private idleTimeoutMs: number;
private client: Client | undefined;
private startPromise: Promise<void> | undefined;
private activeCalls = 0;
private idleStopTimer: ReturnType<typeof setTimeout> | undefined;
constructor(options: ComputerUseClientOptions) {
this.binary = options.binary;
this.onProgress = options.onProgress ?? (() => {});
this.maxImageDimension = options.maxImageDimension;
this.idleTimeoutMs = normalizeIdleTimeoutMs(options.idleTimeoutMs);
}
/**
@ -65,6 +77,11 @@ export class ComputerUseClient {
this.maxImageDimension = value;
}
setIdleTimeoutMs(value: number | undefined): void {
this.idleTimeoutMs = normalizeIdleTimeoutMs(value);
this.scheduleIdleStop();
}
/**
* Shared singleton instance, created with default options on first
* access. Tests can replace it via `setSharedForTest()`.
@ -104,12 +121,20 @@ export class ComputerUseClient {
* responsible for mapping the throw into user-facing UX.
*/
async start(onProgress?: (message: string) => void): Promise<void> {
if (this.client) return;
this.clearIdleStopTimer();
if (this.client) {
this.scheduleIdleStop();
return;
}
if (this.startPromise) return this.startPromise;
this.startPromise = this.doStart(onProgress).finally(() => {
this.startPromise = undefined;
});
this.startPromise = this.doStart(onProgress)
.then(() => {
this.scheduleIdleStop();
})
.finally(() => {
this.startPromise = undefined;
});
return this.startPromise;
}
@ -178,58 +203,67 @@ export class ComputerUseClient {
*
* On transport-closed errors (e.g. macOS kills the upstream binary after
* the user grants Screen Recording permission), this method transparently
* tears down the stale connection, reconnects, and retries the call once.
* If the retry also fails, the error is re-thrown without further
* reconnect attempts.
* tears down the stale connection, reconnects, and retries with a bounded
* backoff loop. If the retries also fail, the last error is re-thrown.
*/
async callTool(
name: string,
args: Record<string, unknown>,
): Promise<CallToolResult> {
if (!this.client) throw new Error('ComputerUseClient not started');
this.activeCalls++;
this.clearIdleStopTimer();
try {
return (await this.client.callTool({
name,
arguments: args,
})) as CallToolResult;
} catch (err) {
if (!isTransportClosedError(err)) throw err;
// The connection died. Two recoverable causes, both fixed by respawning
// the proxy (which relaunches the cua-driver daemon):
// 1. stdio "Connection closed" — the `cua-driver mcp` child was killed.
// 2. "daemon transport error … Connection refused" — the CuaDriver
// DAEMON behind the proxy restarted. macOS forces a restart right
// after the Screen Recording grant, so the proxy's Unix socket to
// the daemon goes dead and every subsequent tool fails. This is the
// first-use failure mode (grant SR → restart → all tools error).
//
// Respawn + retry, with a few attempts to absorb the daemon's restart /
// startup window (a single retry can land before the new daemon is up).
// Element-index state is lost across the restart; the model re-snapshots
// via get_window_state on a stale-index error.
let lastErr: unknown = err;
for (let attempt = 0; attempt < 3; attempt++) {
await this.stop();
await this.start();
if (!this.client) throw new Error('ComputerUseClient reconnect failed');
try {
return (await this.client.callTool({
name,
arguments: args,
})) as CallToolResult;
} catch (retryErr) {
if (!isTransportClosedError(retryErr)) throw retryErr;
lastErr = retryErr;
// Daemon may still be coming up after a restart — back off, retry.
await new Promise((r) => setTimeout(r, 1000));
try {
return (await this.client.callTool({
name,
arguments: args,
})) as CallToolResult;
} catch (err) {
if (!isTransportClosedError(err)) throw err;
// The connection died. Two recoverable causes, both fixed by respawning
// the proxy (which relaunches the cua-driver daemon):
// 1. stdio "Connection closed" — the `cua-driver mcp` child was killed.
// 2. "daemon transport error … Connection refused" — the CuaDriver
// DAEMON behind the proxy restarted. macOS forces a restart right
// after the Screen Recording grant, so the proxy's Unix socket to
// the daemon goes dead and every subsequent tool fails. This is the
// first-use failure mode (grant SR → restart → all tools error).
//
// Respawn + retry, with a few attempts to absorb the daemon's restart /
// startup window (a single retry can land before the new daemon is up).
// Element-index state is lost across the restart; the model re-snapshots
// via get_window_state on a stale-index error.
let lastErr: unknown = err;
for (let attempt = 0; attempt < 3; attempt++) {
await this.stop();
await this.start();
if (!this.client) {
throw new Error('ComputerUseClient reconnect failed');
}
try {
return (await this.client.callTool({
name,
arguments: args,
})) as CallToolResult;
} catch (retryErr) {
if (!isTransportClosedError(retryErr)) throw retryErr;
lastErr = retryErr;
// Daemon may still be coming up after a restart — back off, retry.
await new Promise((r) => setTimeout(r, 1000));
}
}
throw lastErr;
}
throw lastErr;
} finally {
this.activeCalls--;
this.scheduleIdleStop();
}
}
/** Tear down the child process. Safe to call multiple times. */
async stop(): Promise<void> {
this.clearIdleStopTimer();
const client = this.client;
this.client = undefined;
if (client) {
@ -240,6 +274,36 @@ export class ComputerUseClient {
}
}
}
private scheduleIdleStop(): void {
this.clearIdleStopTimer();
if (!this.client || this.activeCalls > 0 || this.idleTimeoutMs <= 0) {
return;
}
this.idleStopTimer = setTimeout(() => {
this.idleStopTimer = undefined;
if (this.activeCalls > 0) return;
void this.stop();
}, this.idleTimeoutMs);
this.idleStopTimer.unref?.();
}
private clearIdleStopTimer(): void {
if (!this.idleStopTimer) return;
clearTimeout(this.idleStopTimer);
this.idleStopTimer = undefined;
}
}
function normalizeIdleTimeoutMs(value: number | undefined): number {
if (value === undefined) return DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS;
if (!Number.isFinite(value)) return DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS;
if (value < 0) return DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS;
if (value > MAX_COMPUTER_USE_IDLE_TIMEOUT_MS) {
return DEFAULT_COMPUTER_USE_IDLE_TIMEOUT_MS;
}
return value;
}
/**

View file

@ -30,6 +30,7 @@ function makeFakeClient(
callTool: vi.fn(callToolImpl),
stop: vi.fn(async () => {}),
setMaxImageDimension: vi.fn(),
setIdleTimeoutMs: vi.fn(),
};
return fake as unknown as ComputerUseClient;
}
@ -92,6 +93,7 @@ describe('ComputerUseTool', () => {
const config = {
getComputerUseMaxImageDimension: () => 1280,
getComputerUseIdleTimeoutMs: () => 30_000,
} as unknown as Config;
const tool = new ComputerUseTool(
'list_apps',
@ -101,6 +103,7 @@ describe('ComputerUseTool', () => {
await tool.build({}).execute(new AbortController().signal);
expect(fake.setMaxImageDimension).toHaveBeenCalledWith(1280);
expect(fake.setIdleTimeoutMs).toHaveBeenCalledWith(30_000);
} finally {
if (prev === undefined) {
delete process.env['QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION'];

View file

@ -221,6 +221,7 @@ class ComputerUseInvocation extends BaseToolInvocation<
client.setMaxImageDimension(
resolveMaxImageDimension(this.config?.getComputerUseMaxImageDimension()),
);
client.setIdleTimeoutMs(this.config?.getComputerUseIdleTimeoutMs());
// If the user confirmed through the pre-execution dialog, the install state
// was already written by onConfirm — runBootstrap will skip promptInstallApproval.

View file

@ -1047,10 +1047,17 @@
"type": "object",
"properties": {
"enabled": {
"description": "When enabled (default), the cua-driver computer_use__* tools are registered as deferred built-ins.",
"description": "When enabled (default), the cua-driver computer_use__* tools are registered as deferred built-ins. Set to false to prevent the driver from being downloaded or spawned.",
"type": "boolean",
"default": true
},
"idleTimeoutMs": {
"description": "Milliseconds to keep the cua-driver process alive after the last computer_use__* call. The default is 300000 (5 minutes). Set to 0 to keep it running until qwen-code exits.",
"type": "number",
"default": 300000,
"minimum": 0,
"maximum": 2147483647
},
"maxImageDimension": {
"description": "Longest-edge pixel cap applied to cua-driver screenshots (via set_config's max_image_dimension). -1 (default) keeps cua-driver's built-in default (1568); 0 disables resizing (full resolution); a positive value caps the longest edge. Lower caps cut vision-token cost at the expense of fine detail. Overridable via the QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION env var.",
"type": "number",

View file

@ -39,6 +39,7 @@ interface JsonSchemaProperty {
enum?: (string | number)[];
default?: unknown;
minimum?: number;
maximum?: number;
additionalProperties?: boolean | JsonSchemaProperty;
required?: string[];
oneOf?: JsonSchemaProperty[];
@ -188,6 +189,9 @@ function convertSettingToJsonSchema(
if (setting.type === 'number' && setting.minimum !== undefined) {
schema.minimum = setting.minimum;
}
if (setting.type === 'number' && setting.maximum !== undefined) {
schema.maximum = setting.maximum;
}
// If the field accepts a legacy primitive shape (e.g. a boolean that was
// later expanded into an object), wrap with `anyOf` so existing values