mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
fix(agents): keep finite LLM idle watchdog under unlimited run budgets
This commit is contained in:
Ayaan Zaidi
parent
c2789b52a7
commit
215be4e4ee
5 changed files with 135 additions and 18 deletions
|
|
@ -127,13 +127,13 @@ Assistant deltas buffer into chat `delta` messages. A chat `final` is emitted on
|
|||
|
||||
## Timeouts
|
||||
|
||||
| Timeout | Default | Notes |
|
||||
| ------------------------------------------------ | ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `agent.wait` | 30s | Wait-only; `timeoutMs` param overrides. Does not stop the underlying run. |
|
||||
| Agent runtime (`agents.defaults.timeoutSeconds`) | 172800s (48h) | Enforced by `runEmbeddedAgent`'s abort timer. |
|
||||
| Cron isolated agent turn | owned by cron | The scheduler starts its own timer when execution begins, aborts the run at the configured deadline, then runs bounded cleanup before recording the timeout so a stale child session cannot keep the lane stuck. |
|
||||
| Model idle timeout | `agents.defaults.timeoutSeconds`, capped at 120s by default | OpenClaw aborts a model request when no response chunks arrive before the idle window. `models.providers.<id>.timeoutSeconds` extends this idle watchdog for slow local/self-hosted providers, but stays bounded by any lower `agents.defaults.timeoutSeconds` or run-specific timeout, since those govern the whole agent run. Cron-triggered cloud model runs with no explicit model/agent timeout use the same default; with an explicit cron run timeout, cloud model stream stalls cap at 60s so configured model fallbacks can still run before the outer cron deadline. Cron-triggered local/self-hosted model runs disable the implicit watchdog unless an explicit timeout is configured; set `models.providers.<id>.timeoutSeconds` for slow local providers. |
|
||||
| Provider HTTP request timeout | `models.providers.<id>.timeoutSeconds` | Covers connect, headers, body, SDK request timeout, guarded-fetch abort handling, and the model stream idle watchdog for that provider. Use for slow local/self-hosted providers (for example Ollama) before raising the whole agent runtime timeout; keep the agent/runtime timeout at least as high when the model request needs to run longer. |
|
||||
| Timeout | Default | Notes |
|
||||
| ------------------------------------------------ | -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `agent.wait` | 30s | Wait-only; `timeoutMs` param overrides. Does not stop the underlying run. |
|
||||
| Agent runtime (`agents.defaults.timeoutSeconds`) | 172800s (48h) | Enforced by `runEmbeddedAgent`'s abort timer. Set `0` for an unlimited run budget; model stream liveness watchdogs still apply. |
|
||||
| Cron isolated agent turn | owned by cron | The scheduler starts its own timer when execution begins, aborts the run at the configured deadline, then runs bounded cleanup before recording the timeout so a stale child session cannot keep the lane stuck. |
|
||||
| Model idle timeout | Cloud 120s; self-hosted 300s | OpenClaw aborts a model request when no response chunks arrive before the idle window. `models.providers.<id>.timeoutSeconds` extends this idle watchdog for slow local/self-hosted providers, but stays bounded by any lower finite `agents.defaults.timeoutSeconds` or run-specific timeout, since those govern the whole agent run. Unlimited run budgets still keep the provider-class idle watchdog. Cron-triggered cloud model runs with no explicit model/agent timeout use the same default; with an explicit cron run timeout, cloud model stream stalls cap at 60s so configured model fallbacks can still run before the outer cron deadline. Cron-triggered local/self-hosted model runs disable the implicit watchdog unless an explicit timeout is configured; set `models.providers.<id>.timeoutSeconds` for slow local providers. |
|
||||
| Provider HTTP request timeout | `models.providers.<id>.timeoutSeconds` | Covers connect, headers, body, SDK request timeout, guarded-fetch abort handling, and the model stream idle watchdog for that provider. Use for slow local/self-hosted providers (for example Ollama) before raising the whole agent runtime timeout; keep the agent/runtime timeout at least as high when the model request needs to run longer. |
|
||||
|
||||
### Stuck session diagnostics
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../../../config/config.js";
|
||||
import type { StreamFn } from "../../runtime/index.js";
|
||||
import { resolveAgentTimeoutMs } from "../../timeout.js";
|
||||
import {
|
||||
resolveLlmFirstEventTimeoutMs,
|
||||
resolveLlmIdleTimeoutMs,
|
||||
|
|
@ -16,6 +17,7 @@ import {
|
|||
} from "./llm-idle-timeout.js";
|
||||
|
||||
const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000;
|
||||
const SELF_HOSTED_LLM_IDLE_TIMEOUT_MS = 300_000;
|
||||
const CRON_LLM_IDLE_TIMEOUT_MS = 60_000;
|
||||
const CLOUD_LLM_FIRST_EVENT_TIMEOUT_MS = DEFAULT_LLM_IDLE_TIMEOUT_MS;
|
||||
const LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS = 300_000;
|
||||
|
|
@ -48,6 +50,30 @@ describe("resolveLlmIdleTimeoutMs", () => {
|
|||
expect(resolveLlmIdleTimeoutMs({ runTimeoutMs: 30_000 })).toBe(30_000);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"cloud",
|
||||
{ provider: "openai", baseUrl: "https://api.openai.com/v1" },
|
||||
DEFAULT_LLM_IDLE_TIMEOUT_MS,
|
||||
],
|
||||
[
|
||||
"self-hosted",
|
||||
{ provider: "vllm", baseUrl: "https://gpu.example.com/v1" },
|
||||
SELF_HOSTED_LLM_IDLE_TIMEOUT_MS,
|
||||
],
|
||||
])("uses the provider-class idle default for no-timeout %s models", (_label, model, expected) => {
|
||||
expect(resolveLlmIdleTimeoutMs({ runTimeoutMs: MAX_TIMER_TIMEOUT_MS, model })).toBe(expected);
|
||||
});
|
||||
|
||||
it("keeps local base URLs opted out of the implicit idle watchdog under no-timeout runs", () => {
|
||||
expect(
|
||||
resolveLlmIdleTimeoutMs({
|
||||
runTimeoutMs: MAX_TIMER_TIMEOUT_MS,
|
||||
model: { baseUrl: "http://127.0.0.1:11434" },
|
||||
}),
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it("caps explicit cron run timeouts so stream stalls can reach model fallbacks", () => {
|
||||
expect(resolveLlmIdleTimeoutMs({ trigger: "cron", runTimeoutMs: 600_000 })).toBe(
|
||||
CRON_LLM_IDLE_TIMEOUT_MS,
|
||||
|
|
@ -197,10 +223,6 @@ describe("resolveLlmIdleTimeoutMs", () => {
|
|||
).toBe(CRON_LLM_IDLE_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
it("disables the idle watchdog when an explicit run timeout disables timeouts", () => {
|
||||
expect(resolveLlmIdleTimeoutMs({ runTimeoutMs: MAX_TIMER_TIMEOUT_MS })).toBe(0);
|
||||
});
|
||||
|
||||
it("honors an explicit models.providers.<id>.timeoutSeconds for cloud providers (#77744, #78361)", () => {
|
||||
// models.providers.<id>.timeoutSeconds is documented as the user-facing
|
||||
// knob to extend slow model responses. The idle watchdog must respect it
|
||||
|
|
@ -288,6 +310,38 @@ describe("resolveLlmIdleTimeoutMs", () => {
|
|||
).toBe(180_000);
|
||||
});
|
||||
|
||||
it("keeps the cloud idle watchdog finite when config timeoutSeconds is unlimited", () => {
|
||||
const cfg = { agents: { defaults: { timeoutSeconds: 0 } } } as OpenClawConfig;
|
||||
const runTimeoutMs = resolveAgentTimeoutMs({ cfg });
|
||||
|
||||
expect(runTimeoutMs).toBe(MAX_TIMER_TIMEOUT_MS);
|
||||
expect(
|
||||
resolveLlmIdleTimeoutMs({
|
||||
cfg,
|
||||
runTimeoutMs,
|
||||
model: { provider: "openai", baseUrl: "https://api.openai.com/v1" },
|
||||
}),
|
||||
).toBe(DEFAULT_LLM_IDLE_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["vllm", "https://gpu.example.com/v1"],
|
||||
["sglang-rig", "https://llm.example.net/v1"],
|
||||
["lmstudio", "http://llm.example.net/v1"],
|
||||
])("uses the self-hosted idle default for provider %s at %s", (provider, baseUrl) => {
|
||||
expect(resolveLlmIdleTimeoutMs({ model: { provider, baseUrl } })).toBe(
|
||||
SELF_HOSTED_LLM_IDLE_TIMEOUT_MS,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the cloud provider idle default unchanged", () => {
|
||||
expect(
|
||||
resolveLlmIdleTimeoutMs({
|
||||
model: { provider: "openai", baseUrl: "https://api.openai.com/v1" },
|
||||
}),
|
||||
).toBe(DEFAULT_LLM_IDLE_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
it("uses provider request timeout for cron model calls", () => {
|
||||
expect(resolveLlmIdleTimeoutMs({ trigger: "cron", modelRequestTimeoutMs: 300_000 })).toBe(
|
||||
300_000,
|
||||
|
|
@ -510,6 +564,36 @@ describe("resolveLlmFirstEventTimeoutMs", () => {
|
|||
).toBe(LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"cloud",
|
||||
{ provider: "openai", baseUrl: "https://api.openai.com/v1" },
|
||||
CLOUD_LLM_FIRST_EVENT_TIMEOUT_MS,
|
||||
],
|
||||
[
|
||||
"self-hosted",
|
||||
{ provider: "vllm", baseUrl: "https://gpu.example.com/v1" },
|
||||
LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS,
|
||||
],
|
||||
])(
|
||||
"uses the provider-class first-event default for no-timeout %s models",
|
||||
(_label, model, expected) => {
|
||||
expect(resolveLlmFirstEventTimeoutMs({ runTimeoutMs: MAX_TIMER_TIMEOUT_MS, model })).toBe(
|
||||
expected,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("honors explicit first-event provider request timeouts under no-timeout runs", () => {
|
||||
expect(
|
||||
resolveLlmFirstEventTimeoutMs({
|
||||
runTimeoutMs: MAX_TIMER_TIMEOUT_MS,
|
||||
modelRequestTimeoutMs: 600_000,
|
||||
model: { provider: "openai", baseUrl: "https://api.openai.com/v1" },
|
||||
}),
|
||||
).toBe(600_000);
|
||||
});
|
||||
|
||||
it("caps first-event timeout by agents.defaults.timeoutSeconds when no explicit run timeout exists", () => {
|
||||
const cfg = { agents: { defaults: { timeoutSeconds: 20 } } } as OpenClawConfig;
|
||||
expect(
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import type { EmbeddedRunTrigger } from "./params.js";
|
|||
* Default idle timeout for LLM streaming responses in milliseconds.
|
||||
*/
|
||||
const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000;
|
||||
const SELF_HOSTED_LLM_IDLE_TIMEOUT_MS = 300_000;
|
||||
const CLOUD_LLM_FIRST_EVENT_TIMEOUT_MS = DEFAULT_LLM_IDLE_TIMEOUT_MS;
|
||||
const LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS = 300_000;
|
||||
// Cron has its own outer watchdog; stream stalls must fail early enough for
|
||||
|
|
@ -262,6 +263,8 @@ export function resolveLlmIdleTimeoutMs(params?: {
|
|||
isExplicitLocalHostnameRuntimeModel,
|
||||
isSelfHostedHostnameRuntimeModel,
|
||||
} = resolveRuntimeModelLocality(params);
|
||||
const isSelfHostedRuntimeModel =
|
||||
isSelfHostedProviderId(params?.model?.provider) && !isOllamaCloudModel(params?.model);
|
||||
const timeoutBounds = [
|
||||
runTimeoutIsNoTimeout ? undefined : runTimeoutMs,
|
||||
hasExplicitRunTimeout ? undefined : agentTimeoutMs,
|
||||
|
|
@ -297,10 +300,9 @@ export function resolveLlmIdleTimeoutMs(params?: {
|
|||
return clampTimeoutMs(boundedTimeoutMs);
|
||||
}
|
||||
|
||||
if (typeof runTimeoutMs === "number" && Number.isFinite(runTimeoutMs) && runTimeoutMs > 0) {
|
||||
if (runTimeoutMs >= MAX_TIMER_TIMEOUT_MS) {
|
||||
return 0;
|
||||
}
|
||||
// Unlimited run budget bounds total cost, not stream liveness. Only finite
|
||||
// explicit run budgets cap the idle watchdog.
|
||||
if (hasExplicitRunTimeout && runTimeoutMs < MAX_TIMER_TIMEOUT_MS) {
|
||||
if (params?.trigger === "cron") {
|
||||
if (
|
||||
isLocalRuntimeModel ||
|
||||
|
|
@ -329,6 +331,14 @@ export function resolveLlmIdleTimeoutMs(params?: {
|
|||
return 0;
|
||||
}
|
||||
|
||||
if (
|
||||
isSelfHostedRuntimeModel ||
|
||||
isExplicitLocalHostnameRuntimeModel ||
|
||||
isSelfHostedHostnameRuntimeModel
|
||||
) {
|
||||
return SELF_HOSTED_LLM_IDLE_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
return DEFAULT_LLM_IDLE_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
|
|
@ -351,7 +361,11 @@ export function resolveLlmFirstEventTimeoutMs(params?: {
|
|||
isExplicitLocalHostnameRuntimeModel,
|
||||
isSelfHostedHostnameRuntimeModel,
|
||||
} = resolveRuntimeModelLocality(params);
|
||||
const isSelfHostedRuntimeModel =
|
||||
isSelfHostedProviderId(params?.model?.provider) && !isOllamaCloudModel(params?.model);
|
||||
const timeoutBounds = [
|
||||
// Unlimited run budget bounds total cost, not first-token liveness. Omit
|
||||
// the sentinel from bounds so provider-class defaults still apply.
|
||||
runTimeoutIsBounded ? runTimeoutMs : undefined,
|
||||
hasExplicitRunTimeout ? undefined : agentTimeoutMs,
|
||||
].filter(
|
||||
|
|
@ -372,7 +386,10 @@ export function resolveLlmFirstEventTimeoutMs(params?: {
|
|||
}
|
||||
|
||||
const defaultTimeoutMs =
|
||||
isLocalRuntimeModel || isExplicitLocalHostnameRuntimeModel || isSelfHostedHostnameRuntimeModel
|
||||
isLocalRuntimeModel ||
|
||||
isExplicitLocalHostnameRuntimeModel ||
|
||||
isSelfHostedHostnameRuntimeModel ||
|
||||
isSelfHostedRuntimeModel
|
||||
? LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS
|
||||
: CLOUD_LLM_FIRST_EVENT_TIMEOUT_MS;
|
||||
return clampTimeoutMs(Math.min(defaultTimeoutMs, ...timeoutBounds));
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import os from "node:os";
|
|||
import path from "node:path";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { getSubagentDepthFromSessionStore } from "./subagent-depth.js";
|
||||
import { resolveAgentTimeoutMs } from "./timeout.js";
|
||||
|
||||
|
|
@ -146,6 +147,16 @@ describe("resolveAgentTimeoutMs", () => {
|
|||
expect(resolveAgentTimeoutMs({})).toBe(48 * 60 * 60 * 1000);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["unlimited", 0, MAX_TIMER_TIMEOUT_MS],
|
||||
["finite", 30, 30_000],
|
||||
["negative", -1, 1_000],
|
||||
["NaN", Number.NaN, 48 * 60 * 60 * 1000],
|
||||
])("resolves config timeoutSeconds %s", (_label, timeoutSeconds, expected) => {
|
||||
const cfg = { agents: { defaults: { timeoutSeconds } } } as OpenClawConfig;
|
||||
expect(resolveAgentTimeoutMs({ cfg })).toBe(expected);
|
||||
});
|
||||
|
||||
it("uses a timer-safe sentinel for no-timeout overrides", () => {
|
||||
expect(resolveAgentTimeoutMs({ overrideSeconds: 0 })).toBe(MAX_TIMER_TIMEOUT_MS);
|
||||
expect(resolveAgentTimeoutMs({ overrideMs: 0 })).toBe(MAX_TIMER_TIMEOUT_MS);
|
||||
|
|
|
|||
|
|
@ -11,12 +11,19 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
|
|||
|
||||
const DEFAULT_AGENT_TIMEOUT_SECONDS = 48 * 60 * 60;
|
||||
export const DEFAULT_AGENT_TIMEOUT_MS = DEFAULT_AGENT_TIMEOUT_SECONDS * 1000;
|
||||
const NO_TIMEOUT_MS = MAX_TIMER_TIMEOUT_MS;
|
||||
const NO_TIMEOUT_SECONDS = Math.floor(NO_TIMEOUT_MS / 1000);
|
||||
|
||||
const normalizeNumber = (value: unknown): number | undefined =>
|
||||
typeof value === "number" && Number.isFinite(value) ? Math.floor(value) : undefined;
|
||||
|
||||
function resolveAgentTimeoutSeconds(cfg?: OpenClawConfig): number {
|
||||
const raw = normalizeNumber(cfg?.agents?.defaults?.timeoutSeconds);
|
||||
// Config 0 uses the same unlimited-run sentinel as per-run overrides. The
|
||||
// LLM idle watchdog still enforces liveness under that sentinel.
|
||||
if (raw === 0) {
|
||||
return NO_TIMEOUT_SECONDS;
|
||||
}
|
||||
const seconds = raw ?? DEFAULT_AGENT_TIMEOUT_SECONDS;
|
||||
return Math.max(seconds, 1);
|
||||
}
|
||||
|
|
@ -30,8 +37,6 @@ export function resolveAgentTimeoutMs(opts: {
|
|||
const minMs = Math.max(normalizeNumber(opts.minMs) ?? 1, 1);
|
||||
const clampTimeoutMs = (valueMs: number) => clampTimerTimeoutMs(valueMs, minMs) ?? minMs;
|
||||
const defaultMs = clampTimeoutMs(resolveAgentTimeoutSeconds(opts.cfg) * 1000);
|
||||
// Use the maximum timer-safe timeout to represent "no timeout" when explicitly set to 0.
|
||||
const NO_TIMEOUT_MS = MAX_TIMER_TIMEOUT_MS;
|
||||
const overrideMs = normalizeNumber(opts.overrideMs);
|
||||
if (overrideMs !== undefined) {
|
||||
if (overrideMs === 0) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue