mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-10 00:11:19 +00:00
fix(qa-lab): surface gateway child stream failures (#102104)
Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
wangmiao0668000666
GitHub
parent
fd34019b4d
commit
6433b82723
4 changed files with 258 additions and 55 deletions
|
|
@ -1,5 +1,6 @@
|
|||
import { spawn } from "node:child_process";
|
||||
// Qa Lab tests cover gateway child plugin behavior.
|
||||
import { EventEmitter } from "node:events";
|
||||
import { EventEmitter, once } from "node:events";
|
||||
import { lstat, mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
|
@ -142,6 +143,51 @@ describe("runQaGatewayCliCommand", () => {
|
|||
}),
|
||||
).rejects.toThrow("OpenClaw CLI exited 7: fixture failure");
|
||||
});
|
||||
|
||||
it.each(["stdout", "stderr"] as const)(
|
||||
"rejects and stops the CLI child when its %s pipe fails",
|
||||
async (streamName) => {
|
||||
const child = spawn(process.execPath, ["--eval", "setInterval(() => {}, 1000)"], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const close = once(child, "close");
|
||||
const result = testing.readQaGatewayCliCommand(child);
|
||||
const message = `synthetic ${streamName} read failure`;
|
||||
|
||||
child[streamName]?.destroy(new Error(message));
|
||||
|
||||
await expect(result).rejects.toThrow(
|
||||
`qa gateway cli ${streamName} stream failed: ${message}`,
|
||||
);
|
||||
await close;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("monitorQaGatewayChildFailure", () => {
|
||||
it("records the first pipe failure and stops the detached Gateway child", async () => {
|
||||
const child = spawn(process.execPath, ["--eval", "setInterval(() => {}, 1000)"], {
|
||||
detached: process.platform !== "win32",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const close = once(child, "close");
|
||||
const output = testing.createQaGatewayChildLogCollector();
|
||||
const getFailure = testing.monitorQaGatewayChildFailure(child, output);
|
||||
const error = new Error("synthetic gateway stdout read failure");
|
||||
|
||||
child.stdout?.destroy(error);
|
||||
child.stderr?.destroy(new Error("later stderr read failure"));
|
||||
|
||||
await vi.waitFor(() => expect(getFailure()).toEqual({ source: "stdout", error }));
|
||||
await close;
|
||||
expect(output.text()).toContain(
|
||||
"gateway child stdout stream failed: synthetic gateway stdout read failure",
|
||||
);
|
||||
expect(output.text()).not.toContain("later stderr read failure");
|
||||
expect(() => testing.throwQaGatewayChildFailure(getFailure, () => output.text())).toThrow(
|
||||
"gateway child stdout stream failed: synthetic gateway stdout read failure",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Gateway child fixture helpers", () => {
|
||||
|
|
|
|||
|
|
@ -133,17 +133,58 @@ async function runQaGatewayCliCommand(params: {
|
|||
cwd: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}): Promise<string> {
|
||||
const stdout = createQaChildOutputCapture();
|
||||
const stderr = createQaChildOutputTail();
|
||||
const child = spawn(params.executablePath, [...params.argsPrefix, ...params.args], {
|
||||
cwd: params.cwd,
|
||||
env: { ...params.env, OPENCLAW_CLI: "1" },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
child.stdout.on("data", (chunk) => appendQaChildOutput(stdout, chunk));
|
||||
child.stderr.on("data", (chunk) => appendQaChildOutputTail(stderr, chunk));
|
||||
return await readQaGatewayCliCommand(child);
|
||||
}
|
||||
|
||||
type QaChildFailure = {
|
||||
source: "process" | "stdout" | "stderr";
|
||||
error: unknown;
|
||||
};
|
||||
|
||||
function monitorQaChildFailure(child: ChildProcess, onFailure: (failure: QaChildFailure) => void) {
|
||||
let reported = false;
|
||||
const report = (source: QaChildFailure["source"]) => (error: unknown) => {
|
||||
if (reported) {
|
||||
return;
|
||||
}
|
||||
reported = true;
|
||||
onFailure({ source, error });
|
||||
};
|
||||
child.once("error", report("process"));
|
||||
child.stdout?.once("error", report("stdout"));
|
||||
child.stderr?.once("error", report("stderr"));
|
||||
}
|
||||
|
||||
async function readQaGatewayCliCommand(child: ChildProcess): Promise<string> {
|
||||
const stdout = createQaChildOutputCapture();
|
||||
const stderr = createQaChildOutputTail();
|
||||
child.stdout?.on("data", (chunk) => appendQaChildOutput(stdout, chunk));
|
||||
child.stderr?.on("data", (chunk) => appendQaChildOutputTail(stderr, chunk));
|
||||
const exitCode = await new Promise<number>((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
monitorQaChildFailure(child, (failure) => {
|
||||
if (failure.source === "process") {
|
||||
reject(toQaErrorObject(failure.error, "OpenClaw CLI process failed"));
|
||||
return;
|
||||
}
|
||||
if (!hasChildExited(child) && !child.killed) {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// The child exited between the state check and signal.
|
||||
}
|
||||
}
|
||||
reject(
|
||||
new Error(
|
||||
`qa gateway cli ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`,
|
||||
{ cause: failure.error },
|
||||
),
|
||||
);
|
||||
});
|
||||
child.once("close", (code) => resolve(code ?? 1));
|
||||
});
|
||||
const stdoutText = readQaChildOutput(stdout);
|
||||
|
|
@ -358,18 +399,43 @@ function createQaGatewayChildLogCollector() {
|
|||
};
|
||||
}
|
||||
|
||||
function monitorQaGatewayChildSpawnError(
|
||||
child: ChildProcess,
|
||||
output: { push(chunk: Buffer): void },
|
||||
function formatQaGatewayChildFailure(failure: QaChildFailure) {
|
||||
return failure.source === "process"
|
||||
? `gateway failed to spawn: ${formatErrorMessage(failure.error)}`
|
||||
: `gateway child ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`;
|
||||
}
|
||||
|
||||
function throwQaGatewayChildFailure(
|
||||
getChildFailure: (() => QaChildFailure | null) | undefined,
|
||||
logs: () => string,
|
||||
) {
|
||||
let spawnError: unknown = null;
|
||||
child.once("error", (error) => {
|
||||
spawnError = error;
|
||||
output.push(
|
||||
Buffer.from(`[qa-lab] gateway child process error: ${formatErrorMessage(error)}\n`),
|
||||
);
|
||||
const failure = getChildFailure?.();
|
||||
if (!failure) {
|
||||
return;
|
||||
}
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`${formatQaGatewayChildFailure(failure)}\n${logs()}`,
|
||||
{ cause: failure.error },
|
||||
);
|
||||
}
|
||||
|
||||
function monitorQaGatewayChildFailure(child: ChildProcess, output: { push(chunk: Buffer): void }) {
|
||||
let childFailure: QaChildFailure | null = null;
|
||||
monitorQaChildFailure(child, (failure) => {
|
||||
childFailure = failure;
|
||||
const description =
|
||||
failure.source === "process"
|
||||
? `gateway child process error: ${formatErrorMessage(failure.error)}`
|
||||
: formatQaGatewayChildFailure(failure);
|
||||
output.push(Buffer.from(`[qa-lab] ${description}\n`));
|
||||
if (failure.source !== "process" && !hasChildExited(child)) {
|
||||
// A broken parent-side pipe means QA can no longer observe the Gateway.
|
||||
// Stop the detached process tree so the existing lifecycle reports the failure.
|
||||
signalQaGatewayChildProcessTree(child, "SIGTERM");
|
||||
}
|
||||
});
|
||||
return () => spawnError;
|
||||
return () => childFailure;
|
||||
}
|
||||
|
||||
async function fetchLocalGatewayHealth(params: {
|
||||
|
|
@ -458,7 +524,10 @@ export const testing = {
|
|||
resolveQaBundledPluginSourceDir,
|
||||
resolveQaRuntimeHostVersion,
|
||||
runQaGatewayCliCommand,
|
||||
readQaGatewayCliCommand,
|
||||
createQaGatewayChildLogCollector,
|
||||
monitorQaGatewayChildFailure,
|
||||
throwQaGatewayChildFailure,
|
||||
createQaBundledPluginsDir,
|
||||
signalQaGatewayChildProcessTree,
|
||||
stopQaGatewayChildProcessTree,
|
||||
|
|
@ -640,19 +709,12 @@ async function waitForGatewayReady(params: {
|
|||
exitCode: number | null;
|
||||
signalCode: NodeJS.Signals | null;
|
||||
};
|
||||
getSpawnError?: () => unknown;
|
||||
getChildFailure?: () => QaChildFailure | null;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) {
|
||||
const spawnError = params.getSpawnError?.();
|
||||
if (spawnError) {
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`gateway failed to spawn: ${formatErrorMessage(spawnError)}\n${params.logs()}`,
|
||||
{ cause: spawnError },
|
||||
);
|
||||
}
|
||||
throwQaGatewayChildFailure(params.getChildFailure, params.logs);
|
||||
if (params.child.exitCode !== null || params.child.signalCode !== null) {
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
|
|
@ -683,19 +745,12 @@ async function waitForGatewayListening(params: {
|
|||
exitCode: number | null;
|
||||
signalCode: NodeJS.Signals | null;
|
||||
};
|
||||
getSpawnError?: () => unknown;
|
||||
getChildFailure?: () => QaChildFailure | null;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) {
|
||||
const spawnError = params.getSpawnError?.();
|
||||
if (spawnError) {
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`gateway failed to spawn: ${formatErrorMessage(spawnError)}\n${params.logs()}`,
|
||||
{ cause: spawnError },
|
||||
);
|
||||
}
|
||||
throwQaGatewayChildFailure(params.getChildFailure, params.logs);
|
||||
if (params.child.exitCode !== null || params.child.signalCode !== null) {
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
|
|
@ -879,6 +934,7 @@ export async function startQaGatewayChild(params: {
|
|||
let child: ReturnType<typeof spawn> | null = null;
|
||||
let cfg!: OpenClawConfig;
|
||||
let rpcClient: Awaited<ReturnType<typeof startQaGatewayRpcClient>> | null = null;
|
||||
let getChildFailure: (() => QaChildFailure | null) | null = null;
|
||||
let stagedBundledPluginsRoot: string | null = null;
|
||||
let env: NodeJS.ProcessEnv | null = null;
|
||||
|
||||
|
|
@ -981,14 +1037,14 @@ export async function startQaGatewayChild(params: {
|
|||
stderrLog.write(buffer);
|
||||
});
|
||||
child = attemptChild;
|
||||
const getAttemptSpawnError = monitorQaGatewayChildSpawnError(attemptChild, output);
|
||||
const getAttemptChildFailure = monitorQaGatewayChildFailure(attemptChild, output);
|
||||
|
||||
try {
|
||||
await waitForGatewayListening({
|
||||
baseUrl,
|
||||
logs,
|
||||
child: attemptChild,
|
||||
getSpawnError: getAttemptSpawnError,
|
||||
getChildFailure: getAttemptChildFailure,
|
||||
timeoutMs: 120_000,
|
||||
});
|
||||
await params.onListening?.({
|
||||
|
|
@ -1003,7 +1059,7 @@ export async function startQaGatewayChild(params: {
|
|||
baseUrl,
|
||||
logs,
|
||||
child: attemptChild,
|
||||
getSpawnError: getAttemptSpawnError,
|
||||
getChildFailure: getAttemptChildFailure,
|
||||
timeoutMs: 120_000,
|
||||
});
|
||||
const attemptRpcClient = await startQaGatewayRpcClient({
|
||||
|
|
@ -1035,7 +1091,7 @@ export async function startQaGatewayChild(params: {
|
|||
baseUrl,
|
||||
logs,
|
||||
child: attemptChild,
|
||||
getSpawnError: getAttemptSpawnError,
|
||||
getChildFailure: getAttemptChildFailure,
|
||||
timeoutMs: QA_GATEWAY_CHILD_RPC_RETRY_HEALTH_TIMEOUT_MS,
|
||||
});
|
||||
}
|
||||
|
|
@ -1046,11 +1102,13 @@ export async function startQaGatewayChild(params: {
|
|||
"Non-Error thrown",
|
||||
);
|
||||
}
|
||||
throwQaGatewayChildFailure(getAttemptChildFailure, logs);
|
||||
} catch (error) {
|
||||
await attemptRpcClient.stop().catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
rpcClient = attemptRpcClient;
|
||||
getChildFailure = getAttemptChildFailure;
|
||||
break;
|
||||
} catch (error) {
|
||||
const details = formatErrorMessage(error);
|
||||
|
|
@ -1076,12 +1134,14 @@ export async function startQaGatewayChild(params: {
|
|||
}
|
||||
}
|
||||
|
||||
if (!child || !cfg || !baseUrl || !wsUrl || !rpcClient || !env) {
|
||||
if (!child || !cfg || !baseUrl || !wsUrl || !rpcClient || !getChildFailure || !env) {
|
||||
throw new Error("qa gateway child failed to start");
|
||||
}
|
||||
let activeChild = child;
|
||||
let activeRpcClient = rpcClient;
|
||||
let activeGetChildFailure = getChildFailure;
|
||||
const runningEnv = env;
|
||||
const throwActiveChildFailure = () => throwQaGatewayChildFailure(activeGetChildFailure, logs);
|
||||
|
||||
const spawnReplacementGatewayChild = async () => {
|
||||
const nextChild = spawn(nodeExecPath, buildGatewayArgs(), {
|
||||
|
|
@ -1102,14 +1162,14 @@ export async function startQaGatewayChild(params: {
|
|||
output.push(buffer);
|
||||
stderrLog.write(buffer);
|
||||
});
|
||||
const getNextSpawnError = monitorQaGatewayChildSpawnError(nextChild, output);
|
||||
const getNextChildFailure = monitorQaGatewayChildFailure(nextChild, output);
|
||||
|
||||
try {
|
||||
await waitForGatewayReady({
|
||||
baseUrl,
|
||||
logs,
|
||||
child: nextChild,
|
||||
getSpawnError: getNextSpawnError,
|
||||
getChildFailure: getNextChildFailure,
|
||||
timeoutMs: 120_000,
|
||||
});
|
||||
const nextRpcClient = await startQaGatewayRpcClient({
|
||||
|
|
@ -1141,7 +1201,7 @@ export async function startQaGatewayChild(params: {
|
|||
baseUrl,
|
||||
logs,
|
||||
child: nextChild,
|
||||
getSpawnError: getNextSpawnError,
|
||||
getChildFailure: getNextChildFailure,
|
||||
timeoutMs: 15_000,
|
||||
});
|
||||
}
|
||||
|
|
@ -1152,6 +1212,7 @@ export async function startQaGatewayChild(params: {
|
|||
"Non-Error thrown",
|
||||
);
|
||||
}
|
||||
throwQaGatewayChildFailure(getNextChildFailure, logs);
|
||||
} catch (error) {
|
||||
await nextRpcClient.stop().catch(() => {});
|
||||
throw error;
|
||||
|
|
@ -1159,6 +1220,7 @@ export async function startQaGatewayChild(params: {
|
|||
return {
|
||||
child: nextChild,
|
||||
rpcClient: nextRpcClient,
|
||||
getChildFailure: getNextChildFailure,
|
||||
};
|
||||
} catch (error) {
|
||||
await stopQaGatewayChildProcessTree(nextChild, {
|
||||
|
|
@ -1183,6 +1245,7 @@ export async function startQaGatewayChild(params: {
|
|||
runtimeEnv: runningEnv,
|
||||
logs,
|
||||
runCli(args: readonly string[]) {
|
||||
throwActiveChildFailure();
|
||||
return runQaGatewayCliCommand({
|
||||
executablePath: nodeExecPath,
|
||||
argsPrefix: cliArgsPrefix,
|
||||
|
|
@ -1192,12 +1255,14 @@ export async function startQaGatewayChild(params: {
|
|||
});
|
||||
},
|
||||
signalProcess(signal: NodeJS.Signals) {
|
||||
throwActiveChildFailure();
|
||||
if (!activeChild.pid) {
|
||||
throw new Error("qa gateway child has no pid");
|
||||
}
|
||||
process.kill(activeChild.pid, signal);
|
||||
},
|
||||
async restart(signal: NodeJS.Signals = "SIGUSR1") {
|
||||
throwActiveChildFailure();
|
||||
if (!activeChild.pid) {
|
||||
throw new Error("qa gateway child has no pid");
|
||||
}
|
||||
|
|
@ -1212,6 +1277,7 @@ export async function startQaGatewayChild(params: {
|
|||
baseUrl,
|
||||
logs,
|
||||
child: activeChild,
|
||||
getChildFailure: activeGetChildFailure,
|
||||
timeoutMs: 120_000,
|
||||
});
|
||||
}
|
||||
|
|
@ -1219,6 +1285,7 @@ export async function startQaGatewayChild(params: {
|
|||
async restartAfterStateMutation(
|
||||
mutateState: (context: QaGatewayChildStateMutationContext) => Promise<void>,
|
||||
) {
|
||||
throwActiveChildFailure();
|
||||
await activeRpcClient.stop().catch(() => {});
|
||||
await stopQaGatewayChildProcessTree(activeChild);
|
||||
await mutateState({
|
||||
|
|
@ -1230,6 +1297,7 @@ export async function startQaGatewayChild(params: {
|
|||
const restarted = await spawnReplacementGatewayChild();
|
||||
activeChild = restarted.child;
|
||||
activeRpcClient = restarted.rpcClient;
|
||||
activeGetChildFailure = restarted.getChildFailure;
|
||||
child = activeChild;
|
||||
rpcClient = activeRpcClient;
|
||||
},
|
||||
|
|
@ -1241,12 +1309,14 @@ export async function startQaGatewayChild(params: {
|
|||
const timeoutMs = opts?.timeoutMs ?? 20_000;
|
||||
let lastDetails = "";
|
||||
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||||
throwActiveChildFailure();
|
||||
try {
|
||||
return await activeRpcClient.request(method, rpcParams, {
|
||||
...opts,
|
||||
timeoutMs,
|
||||
});
|
||||
} catch (error) {
|
||||
throwActiveChildFailure();
|
||||
const details = formatErrorMessage(error);
|
||||
lastDetails = details;
|
||||
if (attempt >= 3 || !isRetryableGatewayCallError(details)) {
|
||||
|
|
@ -1256,6 +1326,7 @@ export async function startQaGatewayChild(params: {
|
|||
baseUrl,
|
||||
logs,
|
||||
child: activeChild,
|
||||
getChildFailure: activeGetChildFailure,
|
||||
timeoutMs: Math.max(10_000, timeoutMs),
|
||||
});
|
||||
}
|
||||
|
|
@ -1282,6 +1353,7 @@ export async function startQaGatewayChild(params: {
|
|||
stagedBundledPluginsRoot,
|
||||
});
|
||||
}
|
||||
throwActiveChildFailure();
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -37,6 +37,39 @@ function makeQaSuiteTestLabHandle(): QaLabServerHandle {
|
|||
}
|
||||
|
||||
describe("qa suite", () => {
|
||||
it("continues ordered cleanup after a resource reports failure", async () => {
|
||||
const calls: string[] = [];
|
||||
const failure = new Error("gateway pipe failed");
|
||||
|
||||
const errors = await qaSuiteProgressTesting.runQaSuiteCleanupSteps([
|
||||
async () => {
|
||||
calls.push("gateway");
|
||||
throw failure;
|
||||
},
|
||||
async () => {
|
||||
calls.push("transport");
|
||||
},
|
||||
async () => {
|
||||
calls.push("lab");
|
||||
},
|
||||
]);
|
||||
|
||||
expect(calls).toEqual(["gateway", "transport", "lab"]);
|
||||
expect(errors).toEqual([failure]);
|
||||
});
|
||||
|
||||
it("keeps the primary suite error as the cause of aggregated cleanup failures", () => {
|
||||
const runError = new Error("gateway infrastructure failed");
|
||||
|
||||
expect(() =>
|
||||
qaSuiteProgressTesting.throwQaSuiteCleanupErrors({
|
||||
cleanupErrors: [new Error("transport cleanup failed")],
|
||||
runFailed: true,
|
||||
runError,
|
||||
}),
|
||||
).toThrow(expect.objectContaining({ cause: runError }));
|
||||
});
|
||||
|
||||
it("rejects unsupported transport ids before starting the lab", async () => {
|
||||
const startLab = vi.fn();
|
||||
|
||||
|
|
|
|||
|
|
@ -295,6 +295,36 @@ async function waitForQaLabReadyOrStopOwned(params: {
|
|||
}
|
||||
}
|
||||
|
||||
async function runQaSuiteCleanupSteps(steps: ReadonlyArray<() => Promise<void>>) {
|
||||
const errors: unknown[] = [];
|
||||
for (const step of steps) {
|
||||
try {
|
||||
await step();
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function throwQaSuiteCleanupErrors(params: {
|
||||
cleanupErrors: unknown[];
|
||||
runFailed: boolean;
|
||||
runError: unknown;
|
||||
}) {
|
||||
if (params.cleanupErrors.length === 0) {
|
||||
return;
|
||||
}
|
||||
if (params.cleanupErrors.length === 1 && !params.runFailed) {
|
||||
throw params.cleanupErrors[0];
|
||||
}
|
||||
throw new AggregateError(
|
||||
params.runFailed ? [params.runError, ...params.cleanupErrors] : params.cleanupErrors,
|
||||
params.runFailed ? "QA suite and cleanup failed" : "QA suite cleanup failed",
|
||||
params.runFailed ? { cause: params.runError } : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
function requireQaSuiteStartLab(startLab: QaSuiteStartLabFn | undefined): QaSuiteStartLabFn {
|
||||
if (startLab) {
|
||||
return startLab;
|
||||
|
|
@ -1626,6 +1656,8 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
|
|||
let gateway: Awaited<ReturnType<typeof startQaGatewayChild>> | undefined;
|
||||
let env: QaSuiteEnvironment | undefined;
|
||||
let preserveGatewayRuntimeDir: string | undefined;
|
||||
let runFailed = false;
|
||||
let runError: unknown;
|
||||
try {
|
||||
writeQaSuiteProgress(progressEnabled, `provider start: ${providerMode}`);
|
||||
const activeMock = await startQaProviderServer(providerMode);
|
||||
|
|
@ -1883,28 +1915,46 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
|
|||
...(runtimeParityCell ? { runtimeParityCell } : {}),
|
||||
} satisfies QaSuiteResult;
|
||||
} catch (error) {
|
||||
runFailed = true;
|
||||
runError = error;
|
||||
preserveGatewayRuntimeDir = path.join(outputDir, "artifacts", "gateway-runtime");
|
||||
throw error;
|
||||
} finally {
|
||||
if (env) {
|
||||
await closeQaWebSessions(env.webSessionIds);
|
||||
const cleanupSteps: Array<() => Promise<void>> = [];
|
||||
const activeEnv = env;
|
||||
if (activeEnv) {
|
||||
cleanupSteps.push(() => closeQaWebSessions(activeEnv.webSessionIds));
|
||||
}
|
||||
const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1" || false;
|
||||
await gateway?.stop({
|
||||
keepTemp,
|
||||
preserveToDir: keepTemp ? undefined : preserveGatewayRuntimeDir,
|
||||
});
|
||||
await transportFactoryResult.cleanup();
|
||||
await disposeRegisteredAgentHarnesses();
|
||||
await mock?.stop();
|
||||
const activeGateway = gateway;
|
||||
if (activeGateway) {
|
||||
cleanupSteps.push(() =>
|
||||
activeGateway.stop({
|
||||
keepTemp,
|
||||
preserveToDir: keepTemp ? undefined : preserveGatewayRuntimeDir,
|
||||
}),
|
||||
);
|
||||
}
|
||||
cleanupSteps.push(
|
||||
() => transportFactoryResult.cleanup(),
|
||||
() => disposeRegisteredAgentHarnesses(),
|
||||
);
|
||||
const activeMock = mock;
|
||||
if (activeMock) {
|
||||
cleanupSteps.push(() => activeMock.stop());
|
||||
}
|
||||
if (ownsLab) {
|
||||
await lab.stop();
|
||||
cleanupSteps.push(() => lab.stop());
|
||||
} else {
|
||||
lab.setControlUi({
|
||||
controlUiUrl: null,
|
||||
controlUiProxyTarget: null,
|
||||
cleanupSteps.push(async () => {
|
||||
lab.setControlUi({
|
||||
controlUiUrl: null,
|
||||
controlUiProxyTarget: null,
|
||||
});
|
||||
});
|
||||
}
|
||||
const cleanupErrors = await runQaSuiteCleanupSteps(cleanupSteps);
|
||||
throwQaSuiteCleanupErrors({ cleanupErrors, runFailed, runError });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1919,6 +1969,8 @@ export const qaSuiteProgressTesting = {
|
|||
mergeQaRuntimeEnvPatches,
|
||||
parseQaSuiteBooleanEnv,
|
||||
remapModelRefForForcedRuntime,
|
||||
runQaSuiteCleanupSteps,
|
||||
throwQaSuiteCleanupErrors,
|
||||
resolveQaSuiteControlUiEnabled,
|
||||
scenarioRequiresControlUi,
|
||||
resolveQaSuiteTransportReadyTimeoutMs,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue