fix(qa): avoid startup prewarm contention (#99368)

This commit is contained in:
Dallin Romney 2026-07-03 20:14:58 -07:00 committed by GitHub
parent e7aea60542
commit 19035bdca1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 81 additions and 3 deletions

View file

@ -167,6 +167,8 @@ describe("buildQaRuntimeEnv", () => {
});
expect(env.OPENCLAW_TEST_FAST).toBe("1");
expect(env.OPENCLAW_SKIP_STARTUP_MODEL_PREWARM).toBe("1");
expect(env.OPENCLAW_SKIP_PROVIDER_AUTH_PREWARM).toBe("1");
expect(env.OPENCLAW_EMBEDDED_ABORT_SETTLE_TIMEOUT_MS).toBe("2000");
expect(env.OPENCLAW_QA_PARENT_PID).toBe(String(process.pid));
expect(env.OPENCLAW_QA_TEMP_ROOT).toBe("/tmp/openclaw-qa");

View file

@ -228,6 +228,8 @@ export function buildQaRuntimeEnv(params: {
OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1",
OPENCLAW_SKIP_GMAIL_WATCHER: "1",
OPENCLAW_SKIP_CANVAS_HOST: "1",
OPENCLAW_SKIP_STARTUP_MODEL_PREWARM: "1",
OPENCLAW_SKIP_PROVIDER_AUTH_PREWARM: "1",
OPENCLAW_NO_RESPAWN: "1",
OPENCLAW_TEST_FAST: "1",
OPENCLAW_EMBEDDED_ABORT_SETTLE_TIMEOUT_MS: "2000",

View file

@ -1045,6 +1045,40 @@ describe("startGatewayPostAttachRuntime", () => {
}
});
it("skips provider auth startup prewarm without disabling failure rewarm", async () => {
vi.useFakeTimers();
const onGatewayLifetimeSidecars = vi.fn();
try {
await startGatewayPostAttachRuntime({
...createPostAttachParams(),
deferSidecars: true,
providerAuthPrewarm: { startupEnabled: false },
onGatewayLifetimeSidecars,
});
await vi.dynamicImportSettled();
await vi.waitFor(() => {
expect(hoisted.setAuthProfileFailureHook).toHaveBeenCalledTimes(1);
});
expect(onGatewayLifetimeSidecars.mock.calls[0]?.[0]).toHaveLength(2);
await vi.advanceTimersByTimeAsync(10_000);
expect(hoisted.warmCurrentProviderAuthStateOffMainThread).not.toHaveBeenCalled();
const hook = hoisted.setAuthProfileFailureHook.mock.calls[0]?.[0] as (() => void) | undefined;
hook?.();
expect(hoisted.clearCurrentProviderAuthState).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(1_000);
await vi.waitFor(() => {
expect(hoisted.warmCurrentProviderAuthStateOffMainThread).toHaveBeenCalledTimes(1);
});
} finally {
vi.useRealTimers();
}
});
it("uses current config when agent runtime plugin prewarm runs", async () => {
const startupConfig = { marker: "startup" } as never;
const currentConfig = { marker: "current" } as never;

View file

@ -36,6 +36,7 @@ const AGENT_RUNTIME_PLUGIN_PREWARM_START_DELAY_MS = 10_000;
const DEFERRED_SIDECAR_START_DELAY_MS = 100;
const SESSION_LOCK_CLEANUP_CONCURRENCY = 4;
const SKIP_STARTUP_MODEL_PREWARM_ENV = "OPENCLAW_SKIP_STARTUP_MODEL_PREWARM";
const SKIP_PROVIDER_AUTH_PREWARM_ENV = "OPENCLAW_SKIP_PROVIDER_AUTH_PREWARM";
const QMD_STARTUP_IDLE_DELAY_MS = 120_000;
type Awaitable<T> = T | Promise<T>;
@ -128,6 +129,12 @@ function shouldSkipStartupModelPrewarm(env: NodeJS.ProcessEnv = process.env): bo
return raw === "1" || raw === "true" || raw === "yes" || raw === "on";
}
export function shouldSkipProviderAuthStartupPrewarm(
env: NodeJS.ProcessEnv = process.env,
): boolean {
return isTruthyEnvValue(env[SKIP_PROVIDER_AUTH_PREWARM_ENV]);
}
function resolveGatewayMemoryStartupPolicy(cfg: OpenClawConfig): GatewayMemoryStartupPolicy {
if (cfg.memory?.backend !== "qmd") {
return { mode: "off" };
@ -200,6 +207,7 @@ function scheduleProviderAuthStatePrewarm(params: {
warn: (msg: string) => void;
};
delayMs?: number;
startupEnabled?: boolean;
}): GatewayPostReadySidecarHandle {
let stopped = false;
let startupTimer: ReturnType<typeof setTimeout> | undefined;
@ -270,6 +278,9 @@ function scheduleProviderAuthStatePrewarm(params: {
clearCurrentProviderAuthState();
scheduleAuthMapRewarm("auth-profile-failure");
});
if (params.startupEnabled === false) {
return;
}
startupTimer = setTimeout(
() => {
void (async () => {
@ -1109,6 +1120,7 @@ export async function startGatewayPostAttachRuntime(
logReadyOnSidecars?: boolean;
providerAuthPrewarm?: {
enabled?: boolean;
startupEnabled?: boolean;
delayMs?: number;
getConfig?: () => OpenClawConfig;
};
@ -1288,6 +1300,7 @@ export async function startGatewayPostAttachRuntime(
getConfig: params.providerAuthPrewarm?.getConfig ?? (() => params.cfgAtStart),
log: params.log,
delayMs: params.providerAuthPrewarm?.delayMs,
startupEnabled: params.providerAuthPrewarm?.startupEnabled,
}),
);
}
@ -1391,6 +1404,7 @@ export const testing = {
cleanupStaleSessionLocks,
scheduleProviderAuthStatePrewarm,
schedulePrimaryModelPrewarm,
shouldSkipProviderAuthStartupPrewarm,
shouldSkipStartupModelPrewarm,
stopPostReadySidecarsAfterCloseStarted,
};

View file

@ -29,6 +29,7 @@ vi.mock("../agents/embedded-agent-runner/model.js", () => ({
}));
let prewarmConfiguredPrimaryModel: typeof import("./server-startup-post-attach.js").testing.prewarmConfiguredPrimaryModel;
let shouldSkipProviderAuthStartupPrewarm: typeof import("./server-startup-post-attach.js").testing.shouldSkipProviderAuthStartupPrewarm;
let shouldSkipStartupModelPrewarm: typeof import("./server-startup-post-attach.js").testing.shouldSkipStartupModelPrewarm;
function expectModelsJsonPrewarmCall(cfg: OpenClawConfig) {
@ -47,7 +48,11 @@ function expectModelsJsonPrewarmCall(cfg: OpenClawConfig) {
describe("gateway startup primary model warmup", () => {
beforeAll(async () => {
({
testing: { prewarmConfiguredPrimaryModel, shouldSkipStartupModelPrewarm },
testing: {
prewarmConfiguredPrimaryModel,
shouldSkipProviderAuthStartupPrewarm,
shouldSkipStartupModelPrewarm,
},
} = await import("./server-startup-post-attach.js"));
});
@ -100,6 +105,20 @@ describe("gateway startup primary model warmup", () => {
).toBe(true);
});
it("honors the provider auth prewarm skip env", () => {
expect(shouldSkipProviderAuthStartupPrewarm({})).toBe(false);
expect(
shouldSkipProviderAuthStartupPrewarm({
OPENCLAW_SKIP_PROVIDER_AUTH_PREWARM: "1",
}),
).toBe(true);
expect(
shouldSkipProviderAuthStartupPrewarm({
OPENCLAW_SKIP_PROVIDER_AUTH_PREWARM: "true",
}),
).toBe(true);
});
it("skips static warmup for configured CLI backends", async () => {
await prewarmConfiguredPrimaryModel({
cfg: {

View file

@ -1599,7 +1599,11 @@ export async function startGatewayServer(
pluginServices: runtimeState.pluginServices,
} = await startupTrace.measure("runtime.post-attach", () =>
loadGatewayStartupPostAttachModule().then(
({ startGatewayPostAttachRuntime, stopPostReadySidecarsAfterCloseStarted }) =>
({
shouldSkipProviderAuthStartupPrewarm,
startGatewayPostAttachRuntime,
stopPostReadySidecarsAfterCloseStarted,
}) =>
startGatewayPostAttachRuntime({
minimalTestGateway,
cfgAtStart,
@ -1687,7 +1691,10 @@ export async function startGatewayServer(
startupTrace,
deferSidecars: deferStartupSidecars,
logReadyOnSidecars: !deferStartupSidecars,
providerAuthPrewarm: { getConfig: getRuntimeConfig },
providerAuthPrewarm: {
startupEnabled: !shouldSkipProviderAuthStartupPrewarm(),
getConfig: getRuntimeConfig,
},
}),
),
));