From 215be4e4ee20a9b0010dd1f9c8529f8b6019046b Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Wed, 8 Jul 2026 08:12:27 +0530 Subject: [PATCH] fix(agents): keep finite LLM idle watchdog under unlimited run budgets --- docs/concepts/agent-loop.md | 14 +-- .../run/llm-idle-timeout.test.ts | 92 ++++++++++++++++++- .../run/llm-idle-timeout.ts | 27 +++++- src/agents/subagent-depth.test.ts | 11 +++ src/agents/timeout.ts | 9 +- 5 files changed, 135 insertions(+), 18 deletions(-) diff --git a/docs/concepts/agent-loop.md b/docs/concepts/agent-loop.md index 870b2906bfb..12c5273f970 100644 --- a/docs/concepts/agent-loop.md +++ b/docs/concepts/agent-loop.md @@ -127,13 +127,13 @@ Assistant deltas buffer into chat `delta` messages. A chat `final` is emitted on ## Timeouts -| Timeout | Default | Notes | -| ------------------------------------------------ | ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `agent.wait` | 30s | Wait-only; `timeoutMs` param overrides. Does not stop the underlying run. | -| Agent runtime (`agents.defaults.timeoutSeconds`) | 172800s (48h) | Enforced by `runEmbeddedAgent`'s abort timer. | -| Cron isolated agent turn | owned by cron | The scheduler starts its own timer when execution begins, aborts the run at the configured deadline, then runs bounded cleanup before recording the timeout so a stale child session cannot keep the lane stuck. | -| Model idle timeout | `agents.defaults.timeoutSeconds`, capped at 120s by default | OpenClaw aborts a model request when no response chunks arrive before the idle window. `models.providers..timeoutSeconds` extends this idle watchdog for slow local/self-hosted providers, but stays bounded by any lower `agents.defaults.timeoutSeconds` or run-specific timeout, since those govern the whole agent run. Cron-triggered cloud model runs with no explicit model/agent timeout use the same default; with an explicit cron run timeout, cloud model stream stalls cap at 60s so configured model fallbacks can still run before the outer cron deadline. Cron-triggered local/self-hosted model runs disable the implicit watchdog unless an explicit timeout is configured; set `models.providers..timeoutSeconds` for slow local providers. | -| Provider HTTP request timeout | `models.providers..timeoutSeconds` | Covers connect, headers, body, SDK request timeout, guarded-fetch abort handling, and the model stream idle watchdog for that provider. Use for slow local/self-hosted providers (for example Ollama) before raising the whole agent runtime timeout; keep the agent/runtime timeout at least as high when the model request needs to run longer. | +| Timeout | Default | Notes | +| ------------------------------------------------ | -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `agent.wait` | 30s | Wait-only; `timeoutMs` param overrides. Does not stop the underlying run. | +| Agent runtime (`agents.defaults.timeoutSeconds`) | 172800s (48h) | Enforced by `runEmbeddedAgent`'s abort timer. Set `0` for an unlimited run budget; model stream liveness watchdogs still apply. | +| Cron isolated agent turn | owned by cron | The scheduler starts its own timer when execution begins, aborts the run at the configured deadline, then runs bounded cleanup before recording the timeout so a stale child session cannot keep the lane stuck. | +| Model idle timeout | Cloud 120s; self-hosted 300s | OpenClaw aborts a model request when no response chunks arrive before the idle window. `models.providers..timeoutSeconds` extends this idle watchdog for slow local/self-hosted providers, but stays bounded by any lower finite `agents.defaults.timeoutSeconds` or run-specific timeout, since those govern the whole agent run. Unlimited run budgets still keep the provider-class idle watchdog. Cron-triggered cloud model runs with no explicit model/agent timeout use the same default; with an explicit cron run timeout, cloud model stream stalls cap at 60s so configured model fallbacks can still run before the outer cron deadline. Cron-triggered local/self-hosted model runs disable the implicit watchdog unless an explicit timeout is configured; set `models.providers..timeoutSeconds` for slow local providers. | +| Provider HTTP request timeout | `models.providers..timeoutSeconds` | Covers connect, headers, body, SDK request timeout, guarded-fetch abort handling, and the model stream idle watchdog for that provider. Use for slow local/self-hosted providers (for example Ollama) before raising the whole agent runtime timeout; keep the agent/runtime timeout at least as high when the model request needs to run longer. | ### Stuck session diagnostics diff --git a/src/agents/embedded-agent-runner/run/llm-idle-timeout.test.ts b/src/agents/embedded-agent-runner/run/llm-idle-timeout.test.ts index 0d150af1dc5..4f400eb5eeb 100644 --- a/src/agents/embedded-agent-runner/run/llm-idle-timeout.test.ts +++ b/src/agents/embedded-agent-runner/run/llm-idle-timeout.test.ts @@ -9,6 +9,7 @@ import { import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../../config/config.js"; import type { StreamFn } from "../../runtime/index.js"; +import { resolveAgentTimeoutMs } from "../../timeout.js"; import { resolveLlmFirstEventTimeoutMs, resolveLlmIdleTimeoutMs, @@ -16,6 +17,7 @@ import { } from "./llm-idle-timeout.js"; const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000; +const SELF_HOSTED_LLM_IDLE_TIMEOUT_MS = 300_000; const CRON_LLM_IDLE_TIMEOUT_MS = 60_000; const CLOUD_LLM_FIRST_EVENT_TIMEOUT_MS = DEFAULT_LLM_IDLE_TIMEOUT_MS; const LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS = 300_000; @@ -48,6 +50,30 @@ describe("resolveLlmIdleTimeoutMs", () => { expect(resolveLlmIdleTimeoutMs({ runTimeoutMs: 30_000 })).toBe(30_000); }); + it.each([ + [ + "cloud", + { provider: "openai", baseUrl: "https://api.openai.com/v1" }, + DEFAULT_LLM_IDLE_TIMEOUT_MS, + ], + [ + "self-hosted", + { provider: "vllm", baseUrl: "https://gpu.example.com/v1" }, + SELF_HOSTED_LLM_IDLE_TIMEOUT_MS, + ], + ])("uses the provider-class idle default for no-timeout %s models", (_label, model, expected) => { + expect(resolveLlmIdleTimeoutMs({ runTimeoutMs: MAX_TIMER_TIMEOUT_MS, model })).toBe(expected); + }); + + it("keeps local base URLs opted out of the implicit idle watchdog under no-timeout runs", () => { + expect( + resolveLlmIdleTimeoutMs({ + runTimeoutMs: MAX_TIMER_TIMEOUT_MS, + model: { baseUrl: "http://127.0.0.1:11434" }, + }), + ).toBe(0); + }); + it("caps explicit cron run timeouts so stream stalls can reach model fallbacks", () => { expect(resolveLlmIdleTimeoutMs({ trigger: "cron", runTimeoutMs: 600_000 })).toBe( CRON_LLM_IDLE_TIMEOUT_MS, @@ -197,10 +223,6 @@ describe("resolveLlmIdleTimeoutMs", () => { ).toBe(CRON_LLM_IDLE_TIMEOUT_MS); }); - it("disables the idle watchdog when an explicit run timeout disables timeouts", () => { - expect(resolveLlmIdleTimeoutMs({ runTimeoutMs: MAX_TIMER_TIMEOUT_MS })).toBe(0); - }); - it("honors an explicit models.providers..timeoutSeconds for cloud providers (#77744, #78361)", () => { // models.providers..timeoutSeconds is documented as the user-facing // knob to extend slow model responses. The idle watchdog must respect it @@ -288,6 +310,38 @@ describe("resolveLlmIdleTimeoutMs", () => { ).toBe(180_000); }); + it("keeps the cloud idle watchdog finite when config timeoutSeconds is unlimited", () => { + const cfg = { agents: { defaults: { timeoutSeconds: 0 } } } as OpenClawConfig; + const runTimeoutMs = resolveAgentTimeoutMs({ cfg }); + + expect(runTimeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); + expect( + resolveLlmIdleTimeoutMs({ + cfg, + runTimeoutMs, + model: { provider: "openai", baseUrl: "https://api.openai.com/v1" }, + }), + ).toBe(DEFAULT_LLM_IDLE_TIMEOUT_MS); + }); + + it.each([ + ["vllm", "https://gpu.example.com/v1"], + ["sglang-rig", "https://llm.example.net/v1"], + ["lmstudio", "http://llm.example.net/v1"], + ])("uses the self-hosted idle default for provider %s at %s", (provider, baseUrl) => { + expect(resolveLlmIdleTimeoutMs({ model: { provider, baseUrl } })).toBe( + SELF_HOSTED_LLM_IDLE_TIMEOUT_MS, + ); + }); + + it("keeps the cloud provider idle default unchanged", () => { + expect( + resolveLlmIdleTimeoutMs({ + model: { provider: "openai", baseUrl: "https://api.openai.com/v1" }, + }), + ).toBe(DEFAULT_LLM_IDLE_TIMEOUT_MS); + }); + it("uses provider request timeout for cron model calls", () => { expect(resolveLlmIdleTimeoutMs({ trigger: "cron", modelRequestTimeoutMs: 300_000 })).toBe( 300_000, @@ -510,6 +564,36 @@ describe("resolveLlmFirstEventTimeoutMs", () => { ).toBe(LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS); }); + it.each([ + [ + "cloud", + { provider: "openai", baseUrl: "https://api.openai.com/v1" }, + CLOUD_LLM_FIRST_EVENT_TIMEOUT_MS, + ], + [ + "self-hosted", + { provider: "vllm", baseUrl: "https://gpu.example.com/v1" }, + LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS, + ], + ])( + "uses the provider-class first-event default for no-timeout %s models", + (_label, model, expected) => { + expect(resolveLlmFirstEventTimeoutMs({ runTimeoutMs: MAX_TIMER_TIMEOUT_MS, model })).toBe( + expected, + ); + }, + ); + + it("honors explicit first-event provider request timeouts under no-timeout runs", () => { + expect( + resolveLlmFirstEventTimeoutMs({ + runTimeoutMs: MAX_TIMER_TIMEOUT_MS, + modelRequestTimeoutMs: 600_000, + model: { provider: "openai", baseUrl: "https://api.openai.com/v1" }, + }), + ).toBe(600_000); + }); + it("caps first-event timeout by agents.defaults.timeoutSeconds when no explicit run timeout exists", () => { const cfg = { agents: { defaults: { timeoutSeconds: 20 } } } as OpenClawConfig; expect( diff --git a/src/agents/embedded-agent-runner/run/llm-idle-timeout.ts b/src/agents/embedded-agent-runner/run/llm-idle-timeout.ts index d39daf0feb6..c07c2d463f0 100644 --- a/src/agents/embedded-agent-runner/run/llm-idle-timeout.ts +++ b/src/agents/embedded-agent-runner/run/llm-idle-timeout.ts @@ -18,6 +18,7 @@ import type { EmbeddedRunTrigger } from "./params.js"; * Default idle timeout for LLM streaming responses in milliseconds. */ const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000; +const SELF_HOSTED_LLM_IDLE_TIMEOUT_MS = 300_000; const CLOUD_LLM_FIRST_EVENT_TIMEOUT_MS = DEFAULT_LLM_IDLE_TIMEOUT_MS; const LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS = 300_000; // Cron has its own outer watchdog; stream stalls must fail early enough for @@ -262,6 +263,8 @@ export function resolveLlmIdleTimeoutMs(params?: { isExplicitLocalHostnameRuntimeModel, isSelfHostedHostnameRuntimeModel, } = resolveRuntimeModelLocality(params); + const isSelfHostedRuntimeModel = + isSelfHostedProviderId(params?.model?.provider) && !isOllamaCloudModel(params?.model); const timeoutBounds = [ runTimeoutIsNoTimeout ? undefined : runTimeoutMs, hasExplicitRunTimeout ? undefined : agentTimeoutMs, @@ -297,10 +300,9 @@ export function resolveLlmIdleTimeoutMs(params?: { return clampTimeoutMs(boundedTimeoutMs); } - if (typeof runTimeoutMs === "number" && Number.isFinite(runTimeoutMs) && runTimeoutMs > 0) { - if (runTimeoutMs >= MAX_TIMER_TIMEOUT_MS) { - return 0; - } + // Unlimited run budget bounds total cost, not stream liveness. Only finite + // explicit run budgets cap the idle watchdog. + if (hasExplicitRunTimeout && runTimeoutMs < MAX_TIMER_TIMEOUT_MS) { if (params?.trigger === "cron") { if ( isLocalRuntimeModel || @@ -329,6 +331,14 @@ export function resolveLlmIdleTimeoutMs(params?: { return 0; } + if ( + isSelfHostedRuntimeModel || + isExplicitLocalHostnameRuntimeModel || + isSelfHostedHostnameRuntimeModel + ) { + return SELF_HOSTED_LLM_IDLE_TIMEOUT_MS; + } + return DEFAULT_LLM_IDLE_TIMEOUT_MS; } @@ -351,7 +361,11 @@ export function resolveLlmFirstEventTimeoutMs(params?: { isExplicitLocalHostnameRuntimeModel, isSelfHostedHostnameRuntimeModel, } = resolveRuntimeModelLocality(params); + const isSelfHostedRuntimeModel = + isSelfHostedProviderId(params?.model?.provider) && !isOllamaCloudModel(params?.model); const timeoutBounds = [ + // Unlimited run budget bounds total cost, not first-token liveness. Omit + // the sentinel from bounds so provider-class defaults still apply. runTimeoutIsBounded ? runTimeoutMs : undefined, hasExplicitRunTimeout ? undefined : agentTimeoutMs, ].filter( @@ -372,7 +386,10 @@ export function resolveLlmFirstEventTimeoutMs(params?: { } const defaultTimeoutMs = - isLocalRuntimeModel || isExplicitLocalHostnameRuntimeModel || isSelfHostedHostnameRuntimeModel + isLocalRuntimeModel || + isExplicitLocalHostnameRuntimeModel || + isSelfHostedHostnameRuntimeModel || + isSelfHostedRuntimeModel ? LOCAL_LLM_FIRST_EVENT_TIMEOUT_MS : CLOUD_LLM_FIRST_EVENT_TIMEOUT_MS; return clampTimeoutMs(Math.min(defaultTimeoutMs, ...timeoutBounds)); diff --git a/src/agents/subagent-depth.test.ts b/src/agents/subagent-depth.test.ts index 609109c7ae3..f4b7b99f6de 100644 --- a/src/agents/subagent-depth.test.ts +++ b/src/agents/subagent-depth.test.ts @@ -5,6 +5,7 @@ import os from "node:os"; import path from "node:path"; import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; import { describe, expect, it } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; import { getSubagentDepthFromSessionStore } from "./subagent-depth.js"; import { resolveAgentTimeoutMs } from "./timeout.js"; @@ -146,6 +147,16 @@ describe("resolveAgentTimeoutMs", () => { expect(resolveAgentTimeoutMs({})).toBe(48 * 60 * 60 * 1000); }); + it.each([ + ["unlimited", 0, MAX_TIMER_TIMEOUT_MS], + ["finite", 30, 30_000], + ["negative", -1, 1_000], + ["NaN", Number.NaN, 48 * 60 * 60 * 1000], + ])("resolves config timeoutSeconds %s", (_label, timeoutSeconds, expected) => { + const cfg = { agents: { defaults: { timeoutSeconds } } } as OpenClawConfig; + expect(resolveAgentTimeoutMs({ cfg })).toBe(expected); + }); + it("uses a timer-safe sentinel for no-timeout overrides", () => { expect(resolveAgentTimeoutMs({ overrideSeconds: 0 })).toBe(MAX_TIMER_TIMEOUT_MS); expect(resolveAgentTimeoutMs({ overrideMs: 0 })).toBe(MAX_TIMER_TIMEOUT_MS); diff --git a/src/agents/timeout.ts b/src/agents/timeout.ts index 9f7a9bdccde..e569ceda1aa 100644 --- a/src/agents/timeout.ts +++ b/src/agents/timeout.ts @@ -11,12 +11,19 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; const DEFAULT_AGENT_TIMEOUT_SECONDS = 48 * 60 * 60; export const DEFAULT_AGENT_TIMEOUT_MS = DEFAULT_AGENT_TIMEOUT_SECONDS * 1000; +const NO_TIMEOUT_MS = MAX_TIMER_TIMEOUT_MS; +const NO_TIMEOUT_SECONDS = Math.floor(NO_TIMEOUT_MS / 1000); const normalizeNumber = (value: unknown): number | undefined => typeof value === "number" && Number.isFinite(value) ? Math.floor(value) : undefined; function resolveAgentTimeoutSeconds(cfg?: OpenClawConfig): number { const raw = normalizeNumber(cfg?.agents?.defaults?.timeoutSeconds); + // Config 0 uses the same unlimited-run sentinel as per-run overrides. The + // LLM idle watchdog still enforces liveness under that sentinel. + if (raw === 0) { + return NO_TIMEOUT_SECONDS; + } const seconds = raw ?? DEFAULT_AGENT_TIMEOUT_SECONDS; return Math.max(seconds, 1); } @@ -30,8 +37,6 @@ export function resolveAgentTimeoutMs(opts: { const minMs = Math.max(normalizeNumber(opts.minMs) ?? 1, 1); const clampTimeoutMs = (valueMs: number) => clampTimerTimeoutMs(valueMs, minMs) ?? minMs; const defaultMs = clampTimeoutMs(resolveAgentTimeoutSeconds(opts.cfg) * 1000); - // Use the maximum timer-safe timeout to represent "no timeout" when explicitly set to 0. - const NO_TIMEOUT_MS = MAX_TIMER_TIMEOUT_MS; const overrideMs = normalizeNumber(opts.overrideMs); if (overrideMs !== undefined) { if (overrideMs === 0) {