mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-28 06:31:11 +00:00
perf(gateway): skip cold startup sidecars until needed
This commit is contained in:
parent
6d3ce088da
commit
99b933f160
8 changed files with 325 additions and 170 deletions
|
|
@ -323,7 +323,7 @@ async function waitForProbe(params: {
|
|||
function requestStatus(port: number, pathname: string): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = request(
|
||||
{ host: "127.0.0.1", method: "GET", path: pathname, port, timeout: 1000 },
|
||||
{ host: "127.0.0.1", method: "GET", path: pathname, port, timeout: 100 },
|
||||
(res) => {
|
||||
res.resume();
|
||||
res.on("end", () => resolve(res.statusCode ?? 0));
|
||||
|
|
@ -511,20 +511,22 @@ async function runGatewaySample(options: {
|
|||
child.stdout.on("data", onChunk);
|
||||
child.stderr.on("data", onChunk);
|
||||
|
||||
const healthz = await waitForProbe({
|
||||
deadlineAt,
|
||||
isDone: () => childExited,
|
||||
path: "/healthz",
|
||||
port,
|
||||
startAt,
|
||||
});
|
||||
const readyz = await waitForProbe({
|
||||
deadlineAt,
|
||||
isDone: () => childExited,
|
||||
path: "/readyz",
|
||||
port,
|
||||
startAt,
|
||||
});
|
||||
const [healthz, readyz] = await Promise.all([
|
||||
waitForProbe({
|
||||
deadlineAt,
|
||||
isDone: () => childExited,
|
||||
path: "/healthz",
|
||||
port,
|
||||
startAt,
|
||||
}),
|
||||
waitForProbe({
|
||||
deadlineAt,
|
||||
isDone: () => childExited,
|
||||
path: "/readyz",
|
||||
port,
|
||||
startAt,
|
||||
}),
|
||||
]);
|
||||
const exit = await stopChild(child);
|
||||
await childExitPromise.catch(() => null);
|
||||
rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 });
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import type { GatewayWsLogStyle } from "../../gateway/ws-logging.js";
|
|||
import { setGatewayWsLogStyle } from "../../gateway/ws-logging.js";
|
||||
import { setVerbose } from "../../globals.js";
|
||||
import { resolveControlUiRootSync } from "../../infra/control-ui-assets.js";
|
||||
import { isTruthyEnvValue } from "../../infra/env.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import { GatewayLockError } from "../../infra/gateway-lock.js";
|
||||
import { formatPortDiagnostics, inspectPortUsage } from "../../infra/ports.js";
|
||||
|
|
@ -98,6 +99,8 @@ const GATEWAY_RUN_BOOLEAN_KEYS = [
|
|||
|
||||
const SUPERVISED_GATEWAY_LOCK_RETRY_MS = 5000;
|
||||
|
||||
type Awaitable<T> = T | Promise<T>;
|
||||
|
||||
/**
|
||||
* EX_CONFIG (78) from sysexits.h — used for configuration errors so systemd
|
||||
* (via RestartPreventExitStatus=78) stops restarting instead of entering a
|
||||
|
|
@ -113,6 +116,36 @@ const GATEWAY_AUTH_MODES: readonly GatewayAuthMode[] = [
|
|||
];
|
||||
const GATEWAY_TAILSCALE_MODES: readonly GatewayTailscaleMode[] = ["off", "serve", "funnel"];
|
||||
|
||||
function createGatewayCliStartupTrace() {
|
||||
const enabled = isTruthyEnvValue(process.env.OPENCLAW_GATEWAY_STARTUP_TRACE);
|
||||
const started = performance.now();
|
||||
let last = started;
|
||||
const emit = (name: string, durationMs: number, totalMs: number) => {
|
||||
if (enabled) {
|
||||
gatewayLog.info(
|
||||
`startup trace: ${name} ${durationMs.toFixed(1)}ms total=${totalMs.toFixed(1)}ms`,
|
||||
);
|
||||
}
|
||||
};
|
||||
return {
|
||||
mark(name: string) {
|
||||
const now = performance.now();
|
||||
emit(name, now - last, now - started);
|
||||
last = now;
|
||||
},
|
||||
async measure<T>(name: string, run: () => Awaitable<T>): Promise<T> {
|
||||
const before = performance.now();
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
const now = performance.now();
|
||||
emit(name, now - before, now - started);
|
||||
last = now;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function warnInlinePasswordFlag() {
|
||||
defaultRuntime.error(
|
||||
"Warning: --password can be exposed via process listings. Prefer --password-file or OPENCLAW_GATEWAY_PASSWORD.",
|
||||
|
|
@ -284,22 +317,28 @@ async function runGatewayCommand(opts: GatewayRunOpts) {
|
|||
process.env.OPENCLAW_RAW_STREAM_PATH = rawStreamPath;
|
||||
}
|
||||
|
||||
const startupTrace = createGatewayCliStartupTrace();
|
||||
|
||||
// The heaviest part of gateway startup is loading the server module tree
|
||||
// (channels, plugins, HTTP stack, etc.). Show a spinner so the user sees
|
||||
// progress instead of a silent 15-20 s pause (especially on Windows/NTFS).
|
||||
const { startGatewayServer } = await withProgress(
|
||||
{ label: "Loading gateway modules…", indeterminate: true },
|
||||
async () => import("../../gateway/server.js"),
|
||||
const { startGatewayServer } = await startupTrace.measure("cli.server-import", () =>
|
||||
withProgress(
|
||||
{ label: "Loading gateway modules…", indeterminate: true },
|
||||
async () => import("../../gateway/server.js"),
|
||||
),
|
||||
);
|
||||
|
||||
setConsoleTimestampPrefix(true);
|
||||
|
||||
if (devMode) {
|
||||
await ensureDevGatewayConfig({ reset: Boolean(opts.reset) });
|
||||
await startupTrace.measure("cli.dev-config", () =>
|
||||
ensureDevGatewayConfig({ reset: Boolean(opts.reset) }),
|
||||
);
|
||||
}
|
||||
|
||||
gatewayLog.info("loading configuration…");
|
||||
const cfg = loadConfig();
|
||||
const cfg = await startupTrace.measure("cli.config-load", () => loadConfig());
|
||||
maybeLogPendingControlUiBuild(cfg);
|
||||
const portOverride = parsePort(opts.port);
|
||||
if (opts.port !== undefined && portOverride === null) {
|
||||
|
|
@ -422,7 +461,9 @@ async function runGatewayCommand(opts: GatewayRunOpts) {
|
|||
const tokenRaw = toOptionString(opts.token);
|
||||
|
||||
gatewayLog.info("resolving authentication…");
|
||||
const snapshot = await readConfigFileSnapshot().catch(() => null);
|
||||
const snapshot = await startupTrace.measure("cli.config-snapshot", () =>
|
||||
readConfigFileSnapshot().catch(() => null),
|
||||
);
|
||||
const configExists = snapshot?.exists ?? fs.existsSync(CONFIG_PATH);
|
||||
const configAuditPath = path.join(resolveStateDir(process.env), "logs", "config-audit.jsonl");
|
||||
const effectiveCfg = snapshot?.valid ? snapshot.config : cfg;
|
||||
|
|
@ -449,12 +490,14 @@ async function runGatewayCommand(opts: GatewayRunOpts) {
|
|||
...(passwordRaw ? { password: passwordRaw } : {}),
|
||||
}
|
||||
: undefined;
|
||||
const resolvedAuth = resolveGatewayAuth({
|
||||
authConfig: cfg.gateway?.auth,
|
||||
authOverride,
|
||||
env: process.env,
|
||||
tailscaleMode: tailscaleMode ?? cfg.gateway?.tailscale?.mode ?? "off",
|
||||
});
|
||||
const resolvedAuth = await startupTrace.measure("cli.auth-resolve", () =>
|
||||
resolveGatewayAuth({
|
||||
authConfig: cfg.gateway?.auth,
|
||||
authOverride,
|
||||
env: process.env,
|
||||
tailscaleMode: tailscaleMode ?? cfg.gateway?.tailscale?.mode ?? "off",
|
||||
}),
|
||||
);
|
||||
const resolvedAuthMode = resolvedAuth.mode;
|
||||
const tokenValue = resolvedAuth.token;
|
||||
const passwordValue = resolvedAuth.password;
|
||||
|
|
@ -537,6 +580,7 @@ async function runGatewayCommand(opts: GatewayRunOpts) {
|
|||
: undefined;
|
||||
|
||||
gatewayLog.info("starting...");
|
||||
startupTrace.mark("cli.gateway-loop");
|
||||
const startLoop = async () =>
|
||||
await runGatewayLoop({
|
||||
runtime: defaultRuntime,
|
||||
|
|
|
|||
|
|
@ -148,6 +148,21 @@ describe("startGatewayPostAttachRuntime", () => {
|
|||
expect(hoisted.logGatewayStartup).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ loadedPluginIds: ["beta", "alpha"] }),
|
||||
);
|
||||
expect(hoisted.startGatewayMemoryBackend).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("starts the qmd memory backend only when configured", async () => {
|
||||
await startGatewayPostAttachRuntime({
|
||||
...createPostAttachParams(),
|
||||
gatewayPluginConfigAtStart: {
|
||||
hooks: { internal: { enabled: false } },
|
||||
memory: { backend: "qmd" },
|
||||
} as never,
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(hoisted.startGatewayMemoryBackend).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps startup-gated methods unavailable while sidecars are still resuming", async () => {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,29 @@ import type { startGatewayTailscaleExposure } from "./server-tailscale.js";
|
|||
|
||||
const SESSION_LOCK_STALE_MS = 30 * 60 * 1000;
|
||||
|
||||
type Awaitable<T> = T | Promise<T>;
|
||||
|
||||
type GatewayStartupTrace = {
|
||||
mark: (name: string) => void;
|
||||
measure: <T>(name: string, run: () => Awaitable<T>) => Promise<T>;
|
||||
};
|
||||
|
||||
async function measureStartup<T>(
|
||||
startupTrace: GatewayStartupTrace | undefined,
|
||||
name: string,
|
||||
run: () => Awaitable<T>,
|
||||
): Promise<T> {
|
||||
return startupTrace ? startupTrace.measure(name, run) : await run();
|
||||
}
|
||||
|
||||
function shouldCheckRestartSentinel(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
return !env.VITEST && env.NODE_ENV !== "test";
|
||||
}
|
||||
|
||||
function shouldStartGatewayMemoryBackend(cfg: OpenClawConfig): boolean {
|
||||
return cfg.memory?.backend === "qmd";
|
||||
}
|
||||
|
||||
async function prewarmConfiguredPrimaryModel(params: {
|
||||
cfg: OpenClawConfig;
|
||||
log: { warn: (msg: string) => void };
|
||||
|
|
@ -88,114 +111,126 @@ export async function startGatewaySidecars(params: {
|
|||
error: (msg: string) => void;
|
||||
};
|
||||
logChannels: { info: (msg: string) => void; error: (msg: string) => void };
|
||||
startupTrace?: GatewayStartupTrace;
|
||||
}) {
|
||||
try {
|
||||
const [{ resolveStateDir }, { resolveAgentSessionDirs }, { cleanStaleLockFiles }] =
|
||||
await Promise.all([
|
||||
import("../config/paths.js"),
|
||||
import("../agents/session-dirs.js"),
|
||||
import("../agents/session-write-lock.js"),
|
||||
]);
|
||||
const stateDir = resolveStateDir(process.env);
|
||||
const sessionDirs = await resolveAgentSessionDirs(stateDir);
|
||||
for (const sessionsDir of sessionDirs) {
|
||||
await cleanStaleLockFiles({
|
||||
sessionsDir,
|
||||
staleMs: SESSION_LOCK_STALE_MS,
|
||||
removeStale: true,
|
||||
log: { warn: (message) => params.log.warn(message) },
|
||||
await measureStartup(params.startupTrace, "sidecars.session-locks", async () => {
|
||||
try {
|
||||
const [{ resolveStateDir }, { resolveAgentSessionDirs }, { cleanStaleLockFiles }] =
|
||||
await Promise.all([
|
||||
import("../config/paths.js"),
|
||||
import("../agents/session-dirs.js"),
|
||||
import("../agents/session-write-lock.js"),
|
||||
]);
|
||||
const stateDir = resolveStateDir(process.env);
|
||||
const sessionDirs = await resolveAgentSessionDirs(stateDir);
|
||||
for (const sessionsDir of sessionDirs) {
|
||||
await cleanStaleLockFiles({
|
||||
sessionsDir,
|
||||
staleMs: SESSION_LOCK_STALE_MS,
|
||||
removeStale: true,
|
||||
log: { warn: (message) => params.log.warn(message) },
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
params.log.warn(`session lock cleanup failed on startup: ${String(err)}`);
|
||||
}
|
||||
});
|
||||
|
||||
await measureStartup(params.startupTrace, "sidecars.gmail-watch", async () => {
|
||||
if (params.cfg.hooks?.enabled && params.cfg.hooks.gmail?.account) {
|
||||
const { startGmailWatcherWithLogs } = await import("../hooks/gmail-watcher-lifecycle.js");
|
||||
await startGmailWatcherWithLogs({
|
||||
cfg: params.cfg,
|
||||
log: params.logHooks,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
params.log.warn(`session lock cleanup failed on startup: ${String(err)}`);
|
||||
}
|
||||
});
|
||||
|
||||
if (params.cfg.hooks?.enabled && params.cfg.hooks.gmail?.account) {
|
||||
const { startGmailWatcherWithLogs } = await import("../hooks/gmail-watcher-lifecycle.js");
|
||||
await startGmailWatcherWithLogs({
|
||||
cfg: params.cfg,
|
||||
log: params.logHooks,
|
||||
});
|
||||
}
|
||||
|
||||
if (params.cfg.hooks?.gmail?.model) {
|
||||
const [
|
||||
{ DEFAULT_MODEL, DEFAULT_PROVIDER },
|
||||
{ loadModelCatalog },
|
||||
{ getModelRefStatus, resolveConfiguredModelRef, resolveHooksGmailModel },
|
||||
] = await Promise.all([
|
||||
import("../agents/defaults.js"),
|
||||
import("../agents/model-catalog.js"),
|
||||
import("../agents/model-selection.js"),
|
||||
]);
|
||||
const hooksModelRef = resolveHooksGmailModel({
|
||||
cfg: params.cfg,
|
||||
defaultProvider: DEFAULT_PROVIDER,
|
||||
});
|
||||
if (hooksModelRef) {
|
||||
const { provider: resolvedDefaultProvider, model: defaultModel } = resolveConfiguredModelRef({
|
||||
await measureStartup(params.startupTrace, "sidecars.gmail-model", async () => {
|
||||
if (params.cfg.hooks?.gmail?.model) {
|
||||
const [
|
||||
{ DEFAULT_MODEL, DEFAULT_PROVIDER },
|
||||
{ loadModelCatalog },
|
||||
{ getModelRefStatus, resolveConfiguredModelRef, resolveHooksGmailModel },
|
||||
] = await Promise.all([
|
||||
import("../agents/defaults.js"),
|
||||
import("../agents/model-catalog.js"),
|
||||
import("../agents/model-selection.js"),
|
||||
]);
|
||||
const hooksModelRef = resolveHooksGmailModel({
|
||||
cfg: params.cfg,
|
||||
defaultProvider: DEFAULT_PROVIDER,
|
||||
defaultModel: DEFAULT_MODEL,
|
||||
});
|
||||
const catalog = await loadModelCatalog({ config: params.cfg });
|
||||
const status = getModelRefStatus({
|
||||
cfg: params.cfg,
|
||||
catalog,
|
||||
ref: hooksModelRef,
|
||||
defaultProvider: resolvedDefaultProvider,
|
||||
defaultModel,
|
||||
});
|
||||
if (!status.allowed) {
|
||||
params.logHooks.warn(
|
||||
`hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`,
|
||||
);
|
||||
}
|
||||
if (!status.inCatalog) {
|
||||
params.logHooks.warn(
|
||||
`hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`,
|
||||
);
|
||||
if (hooksModelRef) {
|
||||
const { provider: resolvedDefaultProvider, model: defaultModel } =
|
||||
resolveConfiguredModelRef({
|
||||
cfg: params.cfg,
|
||||
defaultProvider: DEFAULT_PROVIDER,
|
||||
defaultModel: DEFAULT_MODEL,
|
||||
});
|
||||
const catalog = await loadModelCatalog({ config: params.cfg });
|
||||
const status = getModelRefStatus({
|
||||
cfg: params.cfg,
|
||||
catalog,
|
||||
ref: hooksModelRef,
|
||||
defaultProvider: resolvedDefaultProvider,
|
||||
defaultModel,
|
||||
});
|
||||
if (!status.allowed) {
|
||||
params.logHooks.warn(
|
||||
`hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`,
|
||||
);
|
||||
}
|
||||
if (!status.inCatalog) {
|
||||
params.logHooks.warn(
|
||||
`hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const internalHooksConfigured = hasConfiguredInternalHooks(params.cfg);
|
||||
try {
|
||||
if (internalHooksConfigured) {
|
||||
const [{ setInternalHooksEnabled }, { loadInternalHooks }] = await Promise.all([
|
||||
import("../hooks/internal-hooks.js"),
|
||||
import("../hooks/loader.js"),
|
||||
]);
|
||||
setInternalHooksEnabled(params.cfg.hooks?.internal?.enabled !== false);
|
||||
const loadedCount = await loadInternalHooks(params.cfg, params.defaultWorkspaceDir);
|
||||
if (loadedCount > 0) {
|
||||
params.logHooks.info(
|
||||
`loaded ${loadedCount} internal hook handler${loadedCount > 1 ? "s" : ""}`,
|
||||
);
|
||||
await measureStartup(params.startupTrace, "sidecars.internal-hooks", async () => {
|
||||
try {
|
||||
if (internalHooksConfigured) {
|
||||
const [{ setInternalHooksEnabled }, { loadInternalHooks }] = await Promise.all([
|
||||
import("../hooks/internal-hooks.js"),
|
||||
import("../hooks/loader.js"),
|
||||
]);
|
||||
setInternalHooksEnabled(params.cfg.hooks?.internal?.enabled !== false);
|
||||
const loadedCount = await loadInternalHooks(params.cfg, params.defaultWorkspaceDir);
|
||||
if (loadedCount > 0) {
|
||||
params.logHooks.info(
|
||||
`loaded ${loadedCount} internal hook handler${loadedCount > 1 ? "s" : ""}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
params.logHooks.error(`failed to load hooks: ${String(err)}`);
|
||||
}
|
||||
} catch (err) {
|
||||
params.logHooks.error(`failed to load hooks: ${String(err)}`);
|
||||
}
|
||||
});
|
||||
|
||||
const skipChannels =
|
||||
isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) ||
|
||||
isTruthyEnvValue(process.env.OPENCLAW_SKIP_PROVIDERS);
|
||||
if (!skipChannels) {
|
||||
try {
|
||||
await prewarmConfiguredPrimaryModel({
|
||||
cfg: params.cfg,
|
||||
log: params.log,
|
||||
});
|
||||
await params.startChannels();
|
||||
} catch (err) {
|
||||
params.logChannels.error(`channel startup failed: ${String(err)}`);
|
||||
await measureStartup(params.startupTrace, "sidecars.channels", async () => {
|
||||
if (!skipChannels) {
|
||||
try {
|
||||
await prewarmConfiguredPrimaryModel({
|
||||
cfg: params.cfg,
|
||||
log: params.log,
|
||||
});
|
||||
await params.startChannels();
|
||||
} catch (err) {
|
||||
params.logChannels.error(`channel startup failed: ${String(err)}`);
|
||||
}
|
||||
} else {
|
||||
params.logChannels.info(
|
||||
"skipping channel start (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)",
|
||||
);
|
||||
}
|
||||
} else {
|
||||
params.logChannels.info(
|
||||
"skipping channel start (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
if (internalHooksConfigured) {
|
||||
setTimeout(() => {
|
||||
|
|
@ -213,16 +248,18 @@ export async function startGatewaySidecars(params: {
|
|||
}
|
||||
|
||||
let pluginServices: PluginServicesHandle | null = null;
|
||||
try {
|
||||
const { startPluginServices } = await import("../plugins/services.js");
|
||||
pluginServices = await startPluginServices({
|
||||
registry: params.pluginRegistry,
|
||||
config: params.cfg,
|
||||
workspaceDir: params.defaultWorkspaceDir,
|
||||
});
|
||||
} catch (err) {
|
||||
params.log.warn(`plugin services failed to start: ${String(err)}`);
|
||||
}
|
||||
await measureStartup(params.startupTrace, "sidecars.plugin-services", async () => {
|
||||
try {
|
||||
const { startPluginServices } = await import("../plugins/services.js");
|
||||
pluginServices = await startPluginServices({
|
||||
registry: params.pluginRegistry,
|
||||
config: params.cfg,
|
||||
workspaceDir: params.defaultWorkspaceDir,
|
||||
});
|
||||
} catch (err) {
|
||||
params.log.warn(`plugin services failed to start: ${String(err)}`);
|
||||
}
|
||||
});
|
||||
|
||||
if (params.cfg.acp?.enabled) {
|
||||
const [{ getAcpSessionManager }, { ACP_SESSION_IDENTITY_RENDERER_VERSION }] = await Promise.all(
|
||||
|
|
@ -243,30 +280,48 @@ export async function startGatewaySidecars(params: {
|
|||
});
|
||||
}
|
||||
|
||||
void import("./server-startup-memory.js")
|
||||
.then(({ startGatewayMemoryBackend }) =>
|
||||
startGatewayMemoryBackend({ cfg: params.cfg, log: params.log }),
|
||||
)
|
||||
.catch((err) => {
|
||||
params.log.warn(`qmd memory startup initialization failed: ${String(err)}`);
|
||||
await measureStartup(params.startupTrace, "sidecars.memory", async () => {
|
||||
if (!shouldStartGatewayMemoryBackend(params.cfg)) {
|
||||
return;
|
||||
}
|
||||
setImmediate(() => {
|
||||
void import("./server-startup-memory.js")
|
||||
.then(({ startGatewayMemoryBackend }) =>
|
||||
startGatewayMemoryBackend({ cfg: params.cfg, log: params.log }),
|
||||
)
|
||||
.catch((err) => {
|
||||
params.log.warn(`qmd memory startup initialization failed: ${String(err)}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const { shouldWakeFromRestartSentinel, scheduleRestartSentinelWake } =
|
||||
await import("./server-restart-sentinel.js");
|
||||
if (shouldWakeFromRestartSentinel()) {
|
||||
await measureStartup(params.startupTrace, "sidecars.restart-sentinel", async () => {
|
||||
if (!shouldCheckRestartSentinel()) {
|
||||
return;
|
||||
}
|
||||
const { hasRestartSentinel } = await import("../infra/restart-sentinel.js");
|
||||
if (!(await hasRestartSentinel())) {
|
||||
return;
|
||||
}
|
||||
setTimeout(() => {
|
||||
void scheduleRestartSentinelWake({ deps: params.deps });
|
||||
void import("./server-restart-sentinel.js")
|
||||
.then(({ scheduleRestartSentinelWake }) =>
|
||||
scheduleRestartSentinelWake({ deps: params.deps }),
|
||||
)
|
||||
.catch((err) => {
|
||||
params.log.warn(`restart sentinel wake failed to schedule: ${String(err)}`);
|
||||
});
|
||||
}, 750);
|
||||
}
|
||||
});
|
||||
|
||||
const { scheduleSubagentOrphanRecovery } = await import("../agents/subagent-registry.js");
|
||||
scheduleSubagentOrphanRecovery();
|
||||
await measureStartup(params.startupTrace, "sidecars.subagent-recovery", async () => {
|
||||
const { scheduleSubagentOrphanRecovery } = await import("../agents/subagent-registry.js");
|
||||
scheduleSubagentOrphanRecovery();
|
||||
});
|
||||
|
||||
return { pluginServices };
|
||||
}
|
||||
|
||||
type Awaitable<T> = T | Promise<T>;
|
||||
|
||||
type GatewayPostAttachRuntimeDeps = {
|
||||
getGlobalHookRunner: () => Awaitable<ReturnType<typeof getGlobalHookRunner>>;
|
||||
logGatewayStartup: (params: Parameters<typeof logGatewayStartup>[0]) => Awaitable<void>;
|
||||
|
|
@ -329,26 +384,29 @@ export async function startGatewayPostAttachRuntime(
|
|||
unavailableGatewayMethods: Set<string>;
|
||||
onPluginServices?: (pluginServices: PluginServicesHandle | null) => void;
|
||||
onSidecarsReady?: () => void;
|
||||
startupTrace?: GatewayStartupTrace;
|
||||
},
|
||||
runtimeDeps: GatewayPostAttachRuntimeDeps = defaultGatewayPostAttachRuntimeDeps,
|
||||
) {
|
||||
await runtimeDeps.logGatewayStartup({
|
||||
cfg: params.cfgAtStart,
|
||||
bindHost: params.bindHost,
|
||||
bindHosts: params.bindHosts,
|
||||
port: params.port,
|
||||
tlsEnabled: params.tlsEnabled,
|
||||
loadedPluginIds: params.pluginRegistry.plugins
|
||||
.filter((plugin) => plugin.status === "loaded")
|
||||
.map((plugin) => plugin.id),
|
||||
log: params.log,
|
||||
isNixMode: params.isNixMode,
|
||||
startupStartedAt: params.startupStartedAt,
|
||||
});
|
||||
await measureStartup(params.startupTrace, "post-attach.log", () =>
|
||||
runtimeDeps.logGatewayStartup({
|
||||
cfg: params.cfgAtStart,
|
||||
bindHost: params.bindHost,
|
||||
bindHosts: params.bindHosts,
|
||||
port: params.port,
|
||||
tlsEnabled: params.tlsEnabled,
|
||||
loadedPluginIds: params.pluginRegistry.plugins
|
||||
.filter((plugin) => plugin.status === "loaded")
|
||||
.map((plugin) => plugin.id),
|
||||
log: params.log,
|
||||
isNixMode: params.isNixMode,
|
||||
startupStartedAt: params.startupStartedAt,
|
||||
}),
|
||||
);
|
||||
|
||||
const stopGatewayUpdateCheckPromise = params.minimalTestGateway
|
||||
? Promise.resolve(() => {})
|
||||
: Promise.resolve(
|
||||
: measureStartup(params.startupTrace, "post-attach.update-check", () =>
|
||||
runtimeDeps.scheduleGatewayUpdateCheck({
|
||||
cfg: params.cfgAtStart,
|
||||
log: params.log,
|
||||
|
|
@ -364,7 +422,7 @@ export async function startGatewayPostAttachRuntime(
|
|||
? Promise.resolve(null)
|
||||
: params.tailscaleMode === "off" && !params.resetOnExit
|
||||
? Promise.resolve(null)
|
||||
: Promise.resolve(
|
||||
: measureStartup(params.startupTrace, "post-attach.tailscale", () =>
|
||||
runtimeDeps.startGatewayTailscaleExposure({
|
||||
tailscaleMode: params.tailscaleMode,
|
||||
resetOnExit: params.resetOnExit,
|
||||
|
|
@ -378,21 +436,25 @@ export async function startGatewayPostAttachRuntime(
|
|||
? Promise.resolve({ pluginServices: null })
|
||||
: new Promise<void>((resolve) => setImmediate(resolve)).then(async () => {
|
||||
params.log.info("starting channels and sidecars...");
|
||||
const result = await runtimeDeps.startGatewaySidecars({
|
||||
cfg: params.gatewayPluginConfigAtStart,
|
||||
pluginRegistry: params.pluginRegistry,
|
||||
defaultWorkspaceDir: params.defaultWorkspaceDir,
|
||||
deps: params.deps,
|
||||
startChannels: params.startChannels,
|
||||
log: params.log,
|
||||
logHooks: params.logHooks,
|
||||
logChannels: params.logChannels,
|
||||
});
|
||||
const result = await measureStartup(params.startupTrace, "sidecars.total", () =>
|
||||
runtimeDeps.startGatewaySidecars({
|
||||
cfg: params.gatewayPluginConfigAtStart,
|
||||
pluginRegistry: params.pluginRegistry,
|
||||
defaultWorkspaceDir: params.defaultWorkspaceDir,
|
||||
deps: params.deps,
|
||||
startChannels: params.startChannels,
|
||||
log: params.log,
|
||||
logHooks: params.logHooks,
|
||||
logChannels: params.logChannels,
|
||||
startupTrace: params.startupTrace,
|
||||
}),
|
||||
);
|
||||
for (const method of STARTUP_UNAVAILABLE_GATEWAY_METHODS) {
|
||||
params.unavailableGatewayMethods.delete(method);
|
||||
}
|
||||
params.onPluginServices?.(result.pluginServices);
|
||||
params.onSidecarsReady?.();
|
||||
params.startupTrace?.mark("sidecars.ready");
|
||||
return result;
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -830,6 +830,7 @@ export async function startGatewayServer(
|
|||
onSidecarsReady: () => {
|
||||
startupSidecarsReady = true;
|
||||
},
|
||||
startupTrace,
|
||||
}),
|
||||
));
|
||||
startupTrace.mark("ready");
|
||||
|
|
|
|||
|
|
@ -102,6 +102,28 @@ describe("createReadinessChecker", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("does not cache startup-pending readiness", () => {
|
||||
withReadinessClock(() => {
|
||||
let startupPending = true;
|
||||
const { manager, readiness } = createReadinessHarness({
|
||||
startedAgoMs: 5 * 60_000,
|
||||
accounts: {},
|
||||
getStartupPending: () => startupPending,
|
||||
cacheTtlMs: 1_000,
|
||||
});
|
||||
expect(readiness()).toEqual({
|
||||
ready: false,
|
||||
failing: ["startup-sidecars"],
|
||||
uptimeMs: 300_000,
|
||||
});
|
||||
expect(manager.getRuntimeSnapshot).not.toHaveBeenCalled();
|
||||
|
||||
startupPending = false;
|
||||
expect(readiness()).toEqual({ ready: true, failing: [], uptimeMs: 300_000 });
|
||||
expect(manager.getRuntimeSnapshot).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores disabled and unconfigured channels", () => {
|
||||
withReadinessClock(() => {
|
||||
const { readiness } = createReadinessHarness({
|
||||
|
|
|
|||
|
|
@ -46,15 +46,15 @@ export function createReadinessChecker(deps: {
|
|||
return (): ReadinessResult => {
|
||||
const now = Date.now();
|
||||
const uptimeMs = now - startedAt;
|
||||
if (deps.getStartupPending?.()) {
|
||||
return { ready: false, failing: ["startup-sidecars"], uptimeMs };
|
||||
}
|
||||
if (cachedState && now - cachedAt < cacheTtlMs) {
|
||||
return { ...cachedState, uptimeMs };
|
||||
}
|
||||
|
||||
const snapshot = channelManager.getRuntimeSnapshot();
|
||||
const failing: string[] = [];
|
||||
if (deps.getStartupPending?.()) {
|
||||
failing.push("startup-sidecars");
|
||||
}
|
||||
|
||||
for (const [channelId, accounts] of Object.entries(snapshot.channelAccounts)) {
|
||||
if (!accounts) {
|
||||
|
|
|
|||
|
|
@ -96,6 +96,15 @@ export async function readRestartSentinel(
|
|||
}
|
||||
}
|
||||
|
||||
export async function hasRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(resolveRestartSentinelPath(env));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function consumeRestartSentinel(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<RestartSentinel | null> {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue