mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
2291fe823e
Share repeated tooling parsing, projections, and fixture transforms while preserving command and generated-output contracts. Repair the OpenGrep help range so bootstrap code no longer replaces documented usage.
1537 lines
48 KiB
TypeScript
1537 lines
48 KiB
TypeScript
import { execFileSync, spawn } from "node:child_process";
|
|
import {
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
readdirSync,
|
|
realpathSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
import { expectDefined } from "../packages/normalization-core/src/expect.js";
|
|
import {
|
|
assertCompatibleCliStartupExecutionModes,
|
|
assertCompatibleCliStartupMemoryMetrics,
|
|
CLI_RUNTIME_MEMORY_METRIC,
|
|
type CliStartupExecutionMode,
|
|
cliStartupMemoryMetric,
|
|
} from "./lib/cli-startup-memory-contract.mts";
|
|
import {
|
|
inspectManagedProcessGroup,
|
|
terminateManagedChild,
|
|
waitForManagedProcessGroupExit,
|
|
} from "./lib/managed-child-process.mts";
|
|
|
|
type CommandCase = {
|
|
id: string;
|
|
name: string;
|
|
args: string[];
|
|
presets: readonly string[];
|
|
stateScope?: "case" | "sample";
|
|
expectedExitCodes?: readonly number[];
|
|
expectedNonzeroOutputIncludes?: readonly string[];
|
|
firstOutputBudgetMs?: number;
|
|
exitBudgetMs?: number;
|
|
};
|
|
|
|
type Sample = {
|
|
ms: number;
|
|
firstOutputMs: number | null;
|
|
maxRssMb: number | null;
|
|
memory?: SampleMemory;
|
|
exitCode: number | null;
|
|
signal: string | null;
|
|
startedAt?: string;
|
|
endedAt?: string;
|
|
timedOut?: boolean;
|
|
stdoutTail?: string;
|
|
stderrTail?: string;
|
|
};
|
|
|
|
type RssObservation = {
|
|
pid: number;
|
|
parentPid: number;
|
|
matchesArguments: boolean;
|
|
matchesInvocation: boolean;
|
|
maxRssBytes: number | null;
|
|
};
|
|
|
|
type SampleMemory = {
|
|
runtimePid: number | null;
|
|
processes: Array<{
|
|
pid: number;
|
|
parentPid: number;
|
|
role: "runtime" | "launcher" | "auxiliary" | "unresolved";
|
|
metricKind: "process-high-water-rss";
|
|
maxRssBytes: number | null;
|
|
}>;
|
|
error?: string;
|
|
};
|
|
|
|
type CaseRuns = {
|
|
warmupSamples: Sample[];
|
|
samples: Sample[];
|
|
};
|
|
|
|
type SummaryStats = {
|
|
avg: number;
|
|
p50: number;
|
|
p95: number;
|
|
min: number;
|
|
max: number;
|
|
};
|
|
|
|
type CaseSummary = {
|
|
sampleCount: number;
|
|
durationMs: SummaryStats;
|
|
firstOutputMs: SummaryStats | null;
|
|
maxRssMb: SummaryStats | null;
|
|
exitSummary: string;
|
|
};
|
|
|
|
type SuiteResult = {
|
|
entry: string;
|
|
executionMode?: CliStartupExecutionMode;
|
|
memoryMetric?: string;
|
|
cases: Array<{
|
|
id: string;
|
|
name: string;
|
|
args: string[];
|
|
expectedExitCodes?: number[];
|
|
expectedNonzeroOutputIncludes?: string[];
|
|
contract: {
|
|
firstOutputBudgetMs: number | null;
|
|
exitBudgetMs: number | null;
|
|
} | null;
|
|
warmupSamples?: Sample[];
|
|
samples: Sample[];
|
|
summary: CaseSummary;
|
|
}>;
|
|
};
|
|
|
|
type BenchmarkReport = {
|
|
primary: SuiteResult;
|
|
secondary?: SuiteResult | null;
|
|
};
|
|
|
|
type CaseDelta = {
|
|
id: string;
|
|
name: string;
|
|
durationAvgDeltaMs: number;
|
|
durationAvgDeltaPct: number;
|
|
maxRssAvgDeltaMb: number | null;
|
|
maxRssAvgDeltaPct: number | null;
|
|
};
|
|
|
|
type BenchmarkComparison = {
|
|
baseline: string;
|
|
candidate: string;
|
|
deltas: CaseDelta[];
|
|
};
|
|
|
|
type BenchmarkComparisonResult = {
|
|
baseline: SuiteResult;
|
|
candidate: SuiteResult;
|
|
comparison: BenchmarkComparison;
|
|
};
|
|
|
|
type CliOptions = {
|
|
cases: CommandCase[];
|
|
compareBaseline?: string;
|
|
compareCandidate?: string;
|
|
entryPrimary: string;
|
|
entrySecondary?: string;
|
|
runs: number;
|
|
warmup: number;
|
|
timeoutMs: number;
|
|
runtimeRss: boolean;
|
|
json: boolean;
|
|
output?: string;
|
|
cpuProfDir?: string;
|
|
heapProfDir?: string;
|
|
};
|
|
|
|
const DEFAULT_RUNS = 5;
|
|
const DEFAULT_WARMUP = 1;
|
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
const DEFAULT_TIMEOUT_KILL_GRACE_MS = 1_000;
|
|
const TIMEOUT_KILL_GRACE_MS = resolveTimeoutKillGraceMs(process.env);
|
|
const DEFAULT_ENTRY = "openclaw.mjs";
|
|
const MAX_RSS_MARKER = "__OPENCLAW_MAX_RSS_KB__=";
|
|
|
|
type SampleTransport = {
|
|
prefix: string[];
|
|
binary: string;
|
|
env: Record<string, string>;
|
|
cwd: string;
|
|
};
|
|
|
|
function sampleTransport(): SampleTransport | undefined {
|
|
const raw = process.env.OPENCLAW_BENCH_TRANSPORT_JSON;
|
|
if (raw === undefined) {
|
|
return undefined;
|
|
}
|
|
const value: unknown = JSON.parse(raw);
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
throw new Error("Invalid benchmark transport");
|
|
}
|
|
const prefix = "prefix" in value ? value.prefix : undefined;
|
|
const binary = "binary" in value ? value.binary : undefined;
|
|
const env = "env" in value ? value.env : undefined;
|
|
const cwd = "cwd" in value ? value.cwd : undefined;
|
|
if (
|
|
!Array.isArray(prefix) ||
|
|
prefix.length === 0 ||
|
|
!prefix.every(
|
|
(part: unknown) => typeof part === "string" && part.length > 0 && !part.includes("\0"),
|
|
) ||
|
|
typeof prefix[0] !== "string" ||
|
|
!path.isAbsolute(prefix[0]) ||
|
|
typeof binary !== "string" ||
|
|
!path.isAbsolute(binary) ||
|
|
binary.includes("\0") ||
|
|
!env ||
|
|
typeof env !== "object" ||
|
|
Array.isArray(env)
|
|
) {
|
|
throw new Error("Invalid benchmark transport");
|
|
}
|
|
const fixedEnv: Record<string, string> = {};
|
|
for (const [key, entry] of Object.entries(env)) {
|
|
if (!/^[A-Z_][A-Z0-9_]*$/u.test(key) || typeof entry !== "string" || entry.includes("\0")) {
|
|
throw new Error("Invalid benchmark transport environment");
|
|
}
|
|
fixedEnv[key] = entry;
|
|
}
|
|
if (!fixedEnv.HOME || !path.isAbsolute(fixedEnv.HOME) || !fixedEnv.PATH) {
|
|
throw new Error("Benchmark transport requires fixed HOME and PATH");
|
|
}
|
|
const directory = cwd === undefined ? fixedEnv.HOME : cwd;
|
|
if (typeof directory !== "string" || !path.isAbsolute(directory) || directory.includes("\0")) {
|
|
throw new Error("Invalid benchmark transport cwd");
|
|
}
|
|
return { prefix, binary, env: fixedEnv, cwd: directory };
|
|
}
|
|
|
|
function transportedCommand(
|
|
transport: SampleTransport,
|
|
args: string[],
|
|
env: Record<string, string> = {},
|
|
timeoutMs?: number,
|
|
): { command: string; args: string[] } {
|
|
return {
|
|
command: expectDefined(transport.prefix[0], "benchmark transport command"),
|
|
args: [
|
|
...transport.prefix.slice(1),
|
|
"/usr/bin/env",
|
|
"-C",
|
|
transport.cwd,
|
|
"-i",
|
|
...Object.entries({ ...transport.env, ...env }).map(([key, value]) => `${key}=${value}`),
|
|
...(timeoutMs === undefined
|
|
? []
|
|
: ["/usr/bin/timeout", "--signal=TERM", "--kill-after=1s", `${timeoutMs / 1000}s`]),
|
|
transport.binary,
|
|
...args,
|
|
],
|
|
};
|
|
}
|
|
|
|
function sampleFilesystem(
|
|
transport: SampleTransport,
|
|
operation: "create" | "prepare" | "remove",
|
|
root?: string,
|
|
config?: Record<string, unknown> | null,
|
|
hook?: string,
|
|
): string {
|
|
// Only the SUT opens these paths. Its replies are never read as paths on the runner.
|
|
const launch = transportedCommand(transport, [
|
|
"--input-type=module",
|
|
"-e",
|
|
`import fs from "node:fs"; import os from "node:os"; import path from "node:path";
|
|
const {operation,root,config,hook} = JSON.parse(fs.readFileSync(0,"utf8"));
|
|
if (operation === "create") process.stdout.write(fs.mkdtempSync(path.join(os.tmpdir(),"openclaw-cli-bench-home-")));
|
|
else if (operation === "prepare") {
|
|
fs.mkdirSync(path.join(root,".openclaw"),{recursive:true});
|
|
if (config) fs.writeFileSync(path.join(root,".openclaw/openclaw.json"),JSON.stringify(config)+"\\n");
|
|
fs.writeFileSync(path.join(root,"measure-rss.mjs"),hook);
|
|
} else if (operation === "remove") fs.rmSync(root,{recursive:true,force:true});
|
|
else throw new Error("Invalid sample filesystem operation");`,
|
|
]);
|
|
return execFileSync(launch.command, launch.args, {
|
|
input: JSON.stringify({ operation, root, config, hook }),
|
|
encoding: "utf8",
|
|
timeout: 10_000,
|
|
maxBuffer: 64 * 1024,
|
|
env: { PATH: process.env.PATH },
|
|
});
|
|
}
|
|
|
|
function createSampleRoot(transport?: SampleTransport): string {
|
|
if (!transport) {
|
|
return mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-home-"));
|
|
}
|
|
const root = sampleFilesystem(transport, "create");
|
|
if (!path.isAbsolute(root) || /[\0\r\n]/u.test(root)) {
|
|
throw new Error("Invalid SUT sample directory");
|
|
}
|
|
return root;
|
|
}
|
|
|
|
function removeSampleRoot(root: string, transport?: SampleTransport): void {
|
|
if (transport) {
|
|
sampleFilesystem(transport, "remove", root);
|
|
} else {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function resolveTimeoutKillGraceMs(env: NodeJS.ProcessEnv): number {
|
|
const raw = env.VITEST ? env.OPENCLAW_TEST_CLI_STARTUP_TIMEOUT_KILL_GRACE_MS : undefined;
|
|
if (!raw || !/^\d+$/u.test(raw)) {
|
|
return DEFAULT_TIMEOUT_KILL_GRACE_MS;
|
|
}
|
|
const parsed = Number(raw);
|
|
return Number.isSafeInteger(parsed) ? parsed : DEFAULT_TIMEOUT_KILL_GRACE_MS;
|
|
}
|
|
const VALUE_FLAGS = new Set([
|
|
"--case",
|
|
"--compare-baseline",
|
|
"--compare-candidate",
|
|
"--cpu-prof-dir",
|
|
"--entry",
|
|
"--entry-primary",
|
|
"--entry-secondary",
|
|
"--heap-prof-dir",
|
|
"--output",
|
|
"--preset",
|
|
"--runs",
|
|
"--timeout-ms",
|
|
"--warmup",
|
|
]);
|
|
const BOOLEAN_FLAGS = new Set(["--help", "--json", "--runtime-rss"]);
|
|
|
|
function responseCase(
|
|
id: string,
|
|
args: string[],
|
|
options: Partial<Pick<CommandCase, "presets" | "firstOutputBudgetMs" | "exitBudgetMs">> = {},
|
|
): CommandCase {
|
|
return {
|
|
id,
|
|
name: args.join(" "),
|
|
args,
|
|
presets: ["response"],
|
|
firstOutputBudgetMs: 2_500,
|
|
exitBudgetMs: 6_000,
|
|
...options,
|
|
};
|
|
}
|
|
|
|
const COMMAND_CASES: readonly CommandCase[] = [
|
|
responseCase("version", ["--version"], {
|
|
presets: ["startup", "response"],
|
|
firstOutputBudgetMs: 1_000,
|
|
exitBudgetMs: 2_000,
|
|
}),
|
|
responseCase("help", ["--help"], {
|
|
presets: ["startup", "response"],
|
|
firstOutputBudgetMs: 1_000,
|
|
exitBudgetMs: 2_000,
|
|
}),
|
|
responseCase("onboardHelp", ["onboard", "--help"]),
|
|
responseCase("setupHelp", ["setup", "--help"]),
|
|
responseCase("configureHelp", ["configure", "--help"]),
|
|
responseCase("channelsAddHelp", ["channels", "add", "--help"]),
|
|
responseCase("channelsParent", ["channels"]),
|
|
responseCase("doctorHelp", ["doctor", "--help"]),
|
|
responseCase("modelsHelp", ["models", "--help"]),
|
|
responseCase("pluginsHelp", ["plugins", "--help"]),
|
|
responseCase("pluginsParent", ["plugins"]),
|
|
responseCase("pluginsListJson", ["plugins", "list", "--json"], { presets: ["response", "real"] }),
|
|
responseCase("gatewayHelp", ["gateway", "--help"]),
|
|
responseCase("agentsHelp", ["agents", "--help"], {
|
|
firstOutputBudgetMs: 3_500,
|
|
exitBudgetMs: 8_000,
|
|
}),
|
|
responseCase("sessionsHelp", ["sessions", "--help"]),
|
|
responseCase("messageHelp", ["message", "--help"]),
|
|
responseCase("pairingHelp", ["pairing", "--help"]),
|
|
responseCase("authHelp", ["auth", "--help"]),
|
|
responseCase("configHelp", ["config", "--help"]),
|
|
responseCase("secretsHelp", ["secrets", "--help"]),
|
|
responseCase("skillsHelp", ["skills", "--help"]),
|
|
responseCase("nodesHelp", ["nodes", "--help"], {
|
|
firstOutputBudgetMs: 3_500,
|
|
exitBudgetMs: 8_000,
|
|
}),
|
|
responseCase("directoryHelp", ["directory", "--help"]),
|
|
responseCase("sandboxHelp", ["sandbox", "--help"]),
|
|
responseCase("devicesParent", ["devices"]),
|
|
responseCase("mcpParent", ["mcp"]),
|
|
responseCase("browserHelp", ["browser", "--help"], {
|
|
firstOutputBudgetMs: 1_500,
|
|
exitBudgetMs: 3_000,
|
|
}),
|
|
{
|
|
id: "health",
|
|
name: "health",
|
|
args: ["health"],
|
|
presets: ["startup", "real"],
|
|
expectedExitCodes: [0, 1],
|
|
expectedNonzeroOutputIncludes: ["Gateway target:"],
|
|
},
|
|
{
|
|
id: "healthJson",
|
|
name: "health --json",
|
|
args: ["health", "--json"],
|
|
presets: ["startup"],
|
|
expectedExitCodes: [0, 1],
|
|
expectedNonzeroOutputIncludes: ['"ok"', '"gateway_transport_error"'],
|
|
},
|
|
{
|
|
id: "statusJson",
|
|
name: "status --json",
|
|
args: ["status", "--json"],
|
|
presets: ["startup", "real"],
|
|
},
|
|
{ id: "status", name: "status", args: ["status"], presets: ["startup", "real"] },
|
|
{ id: "sessions", name: "sessions", args: ["sessions"], presets: ["real"] },
|
|
{
|
|
id: "sessionsJson",
|
|
name: "sessions --json",
|
|
args: ["sessions", "--json"],
|
|
presets: ["real"],
|
|
},
|
|
{
|
|
id: "agentsListJson",
|
|
name: "agents list --json",
|
|
args: ["agents", "list", "--json"],
|
|
presets: ["real"],
|
|
},
|
|
{
|
|
id: "gatewayStatus",
|
|
name: "gateway status",
|
|
args: ["gateway", "status"],
|
|
presets: ["real"],
|
|
},
|
|
{
|
|
id: "gatewayStatusJson",
|
|
name: "gateway status --json",
|
|
args: ["gateway", "status", "--json"],
|
|
presets: ["real"],
|
|
},
|
|
{
|
|
id: "gatewayHealthJson",
|
|
name: "gateway health --json",
|
|
args: ["gateway", "health", "--json"],
|
|
presets: ["real"],
|
|
expectedExitCodes: [0, 1],
|
|
expectedNonzeroOutputIncludes: ['"ok"', '"gateway_transport_error"'],
|
|
},
|
|
{
|
|
id: "gatewayHealthJsonWarmState",
|
|
name: "gateway health --json (warm state)",
|
|
args: ["gateway", "health", "--json"],
|
|
presets: [],
|
|
stateScope: "case",
|
|
},
|
|
{
|
|
id: "gatewayHealthJsonFreshState",
|
|
name: "gateway health --json (fresh state)",
|
|
args: ["gateway", "health", "--json"],
|
|
presets: [],
|
|
stateScope: "sample",
|
|
},
|
|
{
|
|
id: "configGetGatewayPort",
|
|
name: "config get gateway.port",
|
|
args: ["config", "get", "gateway.port"],
|
|
presets: ["real"],
|
|
expectedExitCodes: [0, 1],
|
|
expectedNonzeroOutputIncludes: ["Config path not found: gateway.port"],
|
|
},
|
|
] as const;
|
|
|
|
function parseFlagValue(flag: string): string | undefined {
|
|
const idx = process.argv.indexOf(flag);
|
|
if (idx === -1) {
|
|
return undefined;
|
|
}
|
|
const value = process.argv[idx + 1];
|
|
if (!value || value.startsWith("-")) {
|
|
throw new Error(`${flag} requires a value`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function hasFlag(flag: string): boolean {
|
|
return process.argv.includes(flag);
|
|
}
|
|
|
|
function parseRepeatableFlag(flag: string): string[] {
|
|
const values: string[] = [];
|
|
for (let i = 0; i < process.argv.length; i += 1) {
|
|
const value = process.argv[i + 1];
|
|
if (process.argv[i] === flag && value && !value.startsWith("-")) {
|
|
values.push(value);
|
|
}
|
|
}
|
|
return values;
|
|
}
|
|
|
|
function validateCliArgs(argv: readonly string[] = process.argv.slice(2)): void {
|
|
const seenSingleValueFlags = new Set<string>();
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = expectDefined(argv[index], `CLI benchmark argument at index ${index}`);
|
|
if (VALUE_FLAGS.has(arg)) {
|
|
if (arg !== "--case") {
|
|
if (seenSingleValueFlags.has(arg)) {
|
|
throw new Error(`${arg} was provided more than once`);
|
|
}
|
|
seenSingleValueFlags.add(arg);
|
|
}
|
|
const value = argv[index + 1];
|
|
if (!value || value.startsWith("-")) {
|
|
throw new Error(`${arg} requires a value`);
|
|
}
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (BOOLEAN_FLAGS.has(arg)) {
|
|
continue;
|
|
}
|
|
throw new Error(`Unknown argument: ${arg}`);
|
|
}
|
|
}
|
|
|
|
function parsePositiveInt(raw: string | undefined, fallback: number, label = "value"): number {
|
|
return parseIntegerOption(raw, fallback, label, 1);
|
|
}
|
|
|
|
function parseNonNegativeInt(raw: string | undefined, fallback: number, label = "value"): number {
|
|
return parseIntegerOption(raw, fallback, label, 0);
|
|
}
|
|
|
|
// This runner is checked out from trusted main beside frozen candidates, whose
|
|
// root dependencies need not include current workspace packages.
|
|
function parseIntegerOption(
|
|
raw: string | undefined,
|
|
fallback: number,
|
|
label: string,
|
|
min: number,
|
|
): number {
|
|
const value = raw?.trim();
|
|
if (!value) {
|
|
return fallback;
|
|
}
|
|
if (!/^\d+$/u.test(value)) {
|
|
throw new Error(`${label} must be an integer >= ${min}; got ${JSON.stringify(raw)}`);
|
|
}
|
|
const parsed = Number(value);
|
|
if (!Number.isSafeInteger(parsed) || parsed < min) {
|
|
throw new Error(`${label} must be an integer >= ${min}; got ${JSON.stringify(raw)}`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function parseGatewayPortEnv(raw: string | undefined): number {
|
|
const value = raw?.trim();
|
|
if (!value) {
|
|
return 32123;
|
|
}
|
|
const bracketHostMatch = /^\[[^\]]+\]:(\d+)$/u.exec(value);
|
|
if (bracketHostMatch) {
|
|
return parsePositiveInt(bracketHostMatch[1], 32123, "OPENCLAW_GATEWAY_PORT");
|
|
}
|
|
if (value.startsWith("[") && value.endsWith("]")) {
|
|
return 32123;
|
|
}
|
|
const colonCount = value.split(":").length - 1;
|
|
if (colonCount > 1) {
|
|
return 32123;
|
|
}
|
|
const portRaw = colonCount === 1 ? value.split(":")[1] : value;
|
|
return parsePositiveInt(portRaw, 32123, "OPENCLAW_GATEWAY_PORT");
|
|
}
|
|
|
|
function parsePresets(raw: string | undefined): string[] {
|
|
if (!raw) {
|
|
return ["startup"];
|
|
}
|
|
const values = raw
|
|
.split(",")
|
|
.map((value) => value.trim())
|
|
.filter(Boolean);
|
|
if (values.includes("all")) {
|
|
return ["startup", "real", "response"];
|
|
}
|
|
return values.length > 0 ? values : ["startup"];
|
|
}
|
|
|
|
function resolveCases(options: { presets: string[]; caseIds: string[] }): CommandCase[] {
|
|
const byId = new Map(COMMAND_CASES.map((commandCase) => [commandCase.id, commandCase]));
|
|
if (options.caseIds.length > 0) {
|
|
const seenIds = new Set<string>();
|
|
return options.caseIds.map((id) => {
|
|
if (seenIds.has(id)) {
|
|
throw new Error(`Duplicate --case "${id}"`);
|
|
}
|
|
seenIds.add(id);
|
|
const commandCase = byId.get(id);
|
|
if (!commandCase) {
|
|
throw new Error(`Unknown --case "${id}"`);
|
|
}
|
|
return commandCase;
|
|
});
|
|
}
|
|
return COMMAND_CASES.filter((commandCase) =>
|
|
commandCase.presets.some((preset) => options.presets.includes(preset)),
|
|
);
|
|
}
|
|
|
|
function summarizeNumbers(values: number[]): SummaryStats {
|
|
const sorted = values.toSorted((a, b) => a - b);
|
|
const mid = Math.floor(sorted.length / 2);
|
|
const total = values.reduce((sum, value) => sum + value, 0);
|
|
return {
|
|
avg: values.length > 0 ? total / values.length : 0,
|
|
p50:
|
|
sorted.length === 0
|
|
? 0
|
|
: sorted.length % 2 === 0
|
|
? (expectDefined(sorted[mid - 1], "lower middle CLI benchmark sample") +
|
|
expectDefined(sorted[mid], "upper middle CLI benchmark sample")) /
|
|
2
|
|
: expectDefined(sorted[mid], "middle CLI benchmark sample"),
|
|
p95: sorted[Math.min(sorted.length - 1, Math.floor(0.95 * sorted.length))] ?? 0,
|
|
min: values.length > 0 ? Math.min(...values) : 0,
|
|
max: values.length > 0 ? Math.max(...values) : 0,
|
|
};
|
|
}
|
|
|
|
function summarizeSamples(samples: Sample[]): CaseSummary {
|
|
const durations = summarizeNumbers(samples.map((sample) => sample.ms));
|
|
const firstOutputValues = samples
|
|
.map((sample) => sample.firstOutputMs)
|
|
.filter((value): value is number => typeof value === "number" && Number.isFinite(value));
|
|
const rssValues = samples
|
|
.map((sample) => sample.maxRssMb)
|
|
.filter((value): value is number => typeof value === "number" && Number.isFinite(value));
|
|
return {
|
|
sampleCount: samples.length,
|
|
durationMs: durations,
|
|
firstOutputMs: firstOutputValues.length > 0 ? summarizeNumbers(firstOutputValues) : null,
|
|
maxRssMb: rssValues.length > 0 ? summarizeNumbers(rssValues) : null,
|
|
exitSummary: collectExitSummary(samples),
|
|
};
|
|
}
|
|
|
|
function formatMs(value: number): string {
|
|
return `${value.toFixed(1)}ms`;
|
|
}
|
|
|
|
function formatMb(value: number): string {
|
|
return `${value.toFixed(1)}MiB`;
|
|
}
|
|
|
|
function collectExitSummary(samples: Sample[]): string {
|
|
const buckets = new Map<string, number>();
|
|
for (const sample of samples) {
|
|
const key =
|
|
sample.signal != null
|
|
? `signal:${sample.signal}`
|
|
: `code:${sample.exitCode == null ? "null" : String(sample.exitCode)}`;
|
|
buckets.set(key, (buckets.get(key) ?? 0) + 1);
|
|
}
|
|
return [...buckets.entries()].map(([key, count]) => `${key}x${count}`).join(", ");
|
|
}
|
|
|
|
function buildConfigFixture(commandCase: CommandCase): Record<string, unknown> | null {
|
|
const usesSharedToken =
|
|
commandCase.id === "gatewayHealthJsonWarmState" ||
|
|
commandCase.id === "gatewayHealthJsonFreshState";
|
|
if (
|
|
commandCase.id !== "configGetGatewayPort" &&
|
|
commandCase.id !== "gatewayHealthJson" &&
|
|
!usesSharedToken &&
|
|
commandCase.id !== "health" &&
|
|
commandCase.id !== "healthJson"
|
|
) {
|
|
return null;
|
|
}
|
|
const port = parseGatewayPortEnv(process.env.OPENCLAW_GATEWAY_PORT);
|
|
return {
|
|
gateway: {
|
|
auth: { mode: usesSharedToken ? "token" : "none" },
|
|
bind: "loopback",
|
|
mode: "local",
|
|
port,
|
|
},
|
|
};
|
|
}
|
|
|
|
function buildRssHook(tmpDir: string): string {
|
|
const rssHookPath = path.join(tmpDir, "measure-rss.mjs");
|
|
writeFileSync(
|
|
rssHookPath,
|
|
[
|
|
"process.on('exit', () => {",
|
|
" const usage = typeof process.resourceUsage === 'function' ? process.resourceUsage() : null;",
|
|
` if (usage && typeof usage.maxRSS === 'number') console.error('${MAX_RSS_MARKER}' + String(usage.maxRSS));`,
|
|
"});",
|
|
"",
|
|
].join("\n"),
|
|
"utf8",
|
|
);
|
|
return rssHookPath;
|
|
}
|
|
|
|
function parseMaxRssMb(stderr: string): number | null {
|
|
const matches = [...stderr.matchAll(new RegExp(`^${MAX_RSS_MARKER}(\\d+)\\s*$`, "gm"))];
|
|
const lastMatch = matches.at(-1);
|
|
if (!lastMatch?.[1]) {
|
|
return null;
|
|
}
|
|
return Number(lastMatch[1]) / 1024;
|
|
}
|
|
|
|
function buildRuntimeRssHook(tmpDir: string): string {
|
|
const rssHookPath = path.join(tmpDir, "measure-rss.mjs");
|
|
writeFileSync(
|
|
rssHookPath,
|
|
`import { writeFileSync, realpathSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { isMainThread } from "node:worker_threads";
|
|
if (isMainThread) {
|
|
const { directory, entries, args } = JSON.parse(process.env.OPENCLAW_BENCH_MEMORY);
|
|
let entry;
|
|
try {
|
|
if (process.argv[1]) entry = realpathSync(process.argv[1]);
|
|
} catch (error) {
|
|
if (error.code !== "ENOENT") throw error;
|
|
}
|
|
const matchesArguments = JSON.stringify(process.argv.slice(2)) === JSON.stringify(args);
|
|
const record = {
|
|
pid: process.pid,
|
|
parentPid: process.ppid,
|
|
matchesArguments,
|
|
matchesInvocation: entries.includes(entry) && matchesArguments,
|
|
maxRssBytes: null,
|
|
};
|
|
const file = join(directory, String(process.pid) + ".json");
|
|
writeFileSync(file, JSON.stringify(record));
|
|
process.on("exit", () => {
|
|
record.maxRssBytes = process.resourceUsage().maxRSS * 1024;
|
|
writeFileSync(file, JSON.stringify(record));
|
|
});
|
|
}
|
|
`,
|
|
"utf8",
|
|
);
|
|
return rssHookPath;
|
|
}
|
|
|
|
function readSampleMemory(directory: string, entryPid: number | undefined): SampleMemory {
|
|
const memory: SampleMemory = { runtimePid: null, processes: [] };
|
|
try {
|
|
const observations: RssObservation[] = readdirSync(directory).map((file) => {
|
|
const value: unknown = JSON.parse(readFileSync(path.join(directory, file), "utf8"));
|
|
if (
|
|
typeof value !== "object" ||
|
|
value === null ||
|
|
!("pid" in value) ||
|
|
typeof value.pid !== "number" ||
|
|
!Number.isSafeInteger(value.pid) ||
|
|
value.pid <= 0 ||
|
|
file !== `${value.pid}.json` ||
|
|
!("parentPid" in value) ||
|
|
typeof value.parentPid !== "number" ||
|
|
!Number.isSafeInteger(value.parentPid) ||
|
|
value.parentPid < 0 ||
|
|
!("matchesArguments" in value) ||
|
|
typeof value.matchesArguments !== "boolean" ||
|
|
!("matchesInvocation" in value) ||
|
|
typeof value.matchesInvocation !== "boolean" ||
|
|
!("maxRssBytes" in value) ||
|
|
!(
|
|
value.maxRssBytes === null ||
|
|
(typeof value.maxRssBytes === "number" &&
|
|
Number.isSafeInteger(value.maxRssBytes) &&
|
|
value.maxRssBytes > 0)
|
|
)
|
|
) {
|
|
throw new Error("invalid process RSS observation");
|
|
}
|
|
return {
|
|
pid: value.pid,
|
|
parentPid: value.parentPid,
|
|
matchesArguments: value.matchesArguments,
|
|
matchesInvocation: value.matchesInvocation,
|
|
maxRssBytes: value.maxRssBytes,
|
|
};
|
|
});
|
|
memory.processes = observations.map((record) => ({
|
|
pid: record.pid,
|
|
parentPid: record.parentPid,
|
|
role: record.matchesInvocation ? "unresolved" : "auxiliary",
|
|
metricKind: "process-high-water-rss",
|
|
maxRssBytes: record.maxRssBytes,
|
|
}));
|
|
if (observations.some((record) => record.matchesArguments && !record.matchesInvocation)) {
|
|
throw new Error("unrecognized CLI entry with matching command arguments");
|
|
}
|
|
const invocation = observations.filter((record) => record.matchesInvocation);
|
|
const entryObservation = invocation.find((record) => record.pid === entryPid);
|
|
if (!entryObservation) {
|
|
throw new Error("missing CLI entry process identity");
|
|
}
|
|
let current: RssObservation = entryObservation;
|
|
const lineage = new Set<number>();
|
|
// Respawns preserve CLI argv and the preload. Follow the unique invocation
|
|
// chain, not exit order, RSS magnitude, or platform-specific ready flags.
|
|
while (true) {
|
|
if (lineage.has(current.pid)) {
|
|
throw new Error("cyclic CLI process identity");
|
|
}
|
|
lineage.add(current.pid);
|
|
if (current.maxRssBytes === null) {
|
|
throw new Error(`missing process high-water RSS for CLI PID ${current.pid}`);
|
|
}
|
|
const parentPid = current.pid;
|
|
const children = invocation.filter((record) => record.parentPid === parentPid);
|
|
if (children.length > 1) {
|
|
throw new Error("ambiguous CLI runtime identity: multiple matching children");
|
|
}
|
|
const child = children[0];
|
|
if (!child) {
|
|
break;
|
|
}
|
|
current = child;
|
|
}
|
|
if (lineage.size !== invocation.length) {
|
|
throw new Error("disconnected CLI runtime identity");
|
|
}
|
|
memory.runtimePid = current.pid;
|
|
for (const record of memory.processes) {
|
|
if (lineage.has(record.pid)) {
|
|
record.role = record.pid === current.pid ? "runtime" : "launcher";
|
|
}
|
|
}
|
|
} catch (error) {
|
|
memory.error = error instanceof Error ? error.message : String(error);
|
|
}
|
|
return memory;
|
|
}
|
|
|
|
function memoryInvocationEntries(entry: string): string[] {
|
|
const resolvedEntry = realpathSync(entry);
|
|
const entries = [resolvedEntry];
|
|
// The wrapper's compile-cache handoff can enter dist directly.
|
|
if (path.basename(resolvedEntry) === "openclaw.mjs") {
|
|
for (const name of ["entry.js", "entry.mjs"]) {
|
|
try {
|
|
entries.push(realpathSync(path.join(path.dirname(resolvedEntry), "dist", name)));
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return entries;
|
|
}
|
|
|
|
function nodeImportSpecifierForPath(filePath: string): string {
|
|
return pathToFileURL(filePath).href;
|
|
}
|
|
|
|
function buildCpuOrHeapFlags(options: { cpuProfDir?: string; heapProfDir?: string }): string[] {
|
|
const flags: string[] = [];
|
|
if (options.cpuProfDir) {
|
|
flags.push("--cpu-prof", "--cpu-prof-dir", options.cpuProfDir);
|
|
}
|
|
if (options.heapProfDir) {
|
|
flags.push("--heap-prof", "--heap-prof-dir", options.heapProfDir);
|
|
}
|
|
return flags;
|
|
}
|
|
|
|
function appendLimited(current: string, chunk: Buffer | string, maxLength: number): string {
|
|
const next = current + String(chunk);
|
|
return next.length > maxLength ? next.slice(next.length - maxLength) : next;
|
|
}
|
|
|
|
async function runSample(params: {
|
|
entry: string;
|
|
commandCase: CommandCase;
|
|
timeoutMs: number;
|
|
runtimeRss: boolean;
|
|
cpuProfDir?: string;
|
|
heapProfDir?: string;
|
|
rssHookPath: string;
|
|
runRoot?: string;
|
|
}): Promise<Sample> {
|
|
const transport = sampleTransport();
|
|
const runRoot = params.runRoot ?? createSampleRoot(transport);
|
|
const ownsRunRoot = params.runRoot == null;
|
|
const stateDir = path.join(runRoot, ".openclaw");
|
|
const configPath = path.join(stateDir, "openclaw.json");
|
|
const configFixture = buildConfigFixture(params.commandCase);
|
|
let rssHookPath = params.rssHookPath;
|
|
if (transport) {
|
|
sampleFilesystem(
|
|
transport,
|
|
"prepare",
|
|
runRoot,
|
|
configFixture,
|
|
readFileSync(rssHookPath, "utf8"),
|
|
);
|
|
rssHookPath = path.join(runRoot, "measure-rss.mjs");
|
|
} else if (configFixture) {
|
|
mkdirSync(stateDir, { recursive: true });
|
|
writeFileSync(configPath, `${JSON.stringify(configFixture, null, 2)}\n`, "utf8");
|
|
}
|
|
const nodeArgs = [
|
|
...(transport
|
|
? [
|
|
"--import",
|
|
`data:text/javascript,${encodeURIComponent(`process.chdir(${JSON.stringify(path.dirname(params.entry))});`)}`,
|
|
]
|
|
: []),
|
|
"--import",
|
|
nodeImportSpecifierForPath(rssHookPath),
|
|
...buildCpuOrHeapFlags({
|
|
cpuProfDir: params.cpuProfDir,
|
|
heapProfDir: params.heapProfDir,
|
|
}),
|
|
params.entry,
|
|
...params.commandCase.args,
|
|
];
|
|
const startedAt = new Date();
|
|
const started = process.hrtime.bigint();
|
|
let firstOutputMs: number | null = null;
|
|
let stdout = "";
|
|
let stderr = "";
|
|
let settled = false;
|
|
let timedOut = false;
|
|
let forceKillAt: number | null = null;
|
|
let forceKillTimer: ReturnType<typeof setTimeout> | null = null;
|
|
const maxOutputLength = 32 * 1024 * 1024;
|
|
const memoryDirectory = params.runtimeRss
|
|
? mkdtempSync(path.join(path.dirname(params.rssHookPath), "sample-"))
|
|
: undefined;
|
|
|
|
try {
|
|
return await new Promise<Sample>((resolve) => {
|
|
const sampleEnv = {
|
|
HOME: runRoot,
|
|
USERPROFILE: runRoot,
|
|
OPENCLAW_HOME: runRoot,
|
|
OPENCLAW_STATE_DIR: stateDir,
|
|
OPENCLAW_CONFIG_PATH: configPath,
|
|
OPENCLAW_HIDE_BANNER: "1",
|
|
NO_COLOR: "1",
|
|
FORCE_COLOR: "0",
|
|
...(memoryDirectory
|
|
? {
|
|
OPENCLAW_BENCH_MEMORY: JSON.stringify({
|
|
directory: memoryDirectory,
|
|
entries: memoryInvocationEntries(params.entry),
|
|
args: params.commandCase.args,
|
|
}),
|
|
}
|
|
: {}),
|
|
};
|
|
const launch = transport
|
|
? transportedCommand(
|
|
transport,
|
|
nodeArgs,
|
|
{
|
|
...sampleEnv,
|
|
...(process.env.OPENCLAW_GATEWAY_TOKEN
|
|
? { OPENCLAW_GATEWAY_TOKEN: process.env.OPENCLAW_GATEWAY_TOKEN }
|
|
: {}),
|
|
...(process.env.OPENCLAW_GATEWAY_PORT
|
|
? { OPENCLAW_GATEWAY_PORT: process.env.OPENCLAW_GATEWAY_PORT }
|
|
: {}),
|
|
},
|
|
params.timeoutMs,
|
|
)
|
|
: { command: process.execPath, args: nodeArgs };
|
|
const proc = spawn(launch.command, launch.args, {
|
|
cwd: process.cwd(),
|
|
detached: process.platform !== "win32",
|
|
env: transport ? { PATH: process.env.PATH } : { ...process.env, ...sampleEnv },
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
|
|
const finish = (sample: Omit<Sample, "ms" | "firstOutputMs" | "maxRssMb">) => {
|
|
if (settled) {
|
|
return;
|
|
}
|
|
settled = true;
|
|
if (forceKillTimer) {
|
|
clearTimeout(forceKillTimer);
|
|
forceKillTimer = null;
|
|
}
|
|
const ms = Number(process.hrtime.bigint() - started) / 1e6;
|
|
const memory = memoryDirectory ? readSampleMemory(memoryDirectory, proc.pid) : undefined;
|
|
const runtimeRss = memory?.processes.find(
|
|
(record) => record.role === "runtime",
|
|
)?.maxRssBytes;
|
|
resolve({
|
|
ms,
|
|
firstOutputMs,
|
|
maxRssMb: memory
|
|
? runtimeRss == null
|
|
? null
|
|
: runtimeRss / 1024 / 1024
|
|
: parseMaxRssMb(stderr),
|
|
...(memory ? { memory } : {}),
|
|
startedAt: startedAt.toISOString(),
|
|
endedAt: new Date().toISOString(),
|
|
...(timedOut ? { timedOut } : {}),
|
|
...sample,
|
|
});
|
|
};
|
|
|
|
const markFirstOutput = () => {
|
|
if (firstOutputMs == null) {
|
|
firstOutputMs = Number(process.hrtime.bigint() - started) / 1e6;
|
|
}
|
|
};
|
|
|
|
const timeout = setTimeout(() => {
|
|
timedOut = true;
|
|
signalSampleProcess(proc, "SIGTERM");
|
|
forceKillAt = Date.now() + TIMEOUT_KILL_GRACE_MS;
|
|
forceKillTimer = setTimeout(() => {
|
|
signalSampleProcess(proc, "SIGKILL");
|
|
}, TIMEOUT_KILL_GRACE_MS).unref?.();
|
|
}, params.timeoutMs);
|
|
timeout.unref?.();
|
|
|
|
proc.stdout?.on("data", (chunk) => {
|
|
markFirstOutput();
|
|
stdout = appendLimited(stdout, chunk, maxOutputLength);
|
|
});
|
|
proc.stderr?.on("data", (chunk) => {
|
|
markFirstOutput();
|
|
stderr = appendLimited(stderr, chunk, maxOutputLength);
|
|
});
|
|
proc.once("error", (error) => {
|
|
clearTimeout(timeout);
|
|
stderr = appendLimited(
|
|
stderr,
|
|
error instanceof Error ? error.message : String(error),
|
|
maxOutputLength,
|
|
);
|
|
finish({
|
|
exitCode: null,
|
|
signal: null,
|
|
stdoutTail: tailLines(stdout, 20),
|
|
stderrTail: tailLines(stderr, 20),
|
|
});
|
|
});
|
|
proc.once("close", (code, signal) => {
|
|
clearTimeout(timeout);
|
|
const complete = () =>
|
|
finish({
|
|
exitCode: code,
|
|
signal,
|
|
...(code === 0 && signal == null
|
|
? {}
|
|
: {
|
|
stdoutTail: tailLines(stdout, 20),
|
|
stderrTail: tailLines(stderr, 20),
|
|
}),
|
|
});
|
|
if (timedOut && isSampleProcessGroupAlive(proc)) {
|
|
void finishAfterTimeoutCleanup({
|
|
complete,
|
|
forceKillAt,
|
|
proc,
|
|
});
|
|
return;
|
|
}
|
|
complete();
|
|
});
|
|
});
|
|
} finally {
|
|
if (memoryDirectory) {
|
|
rmSync(memoryDirectory, { recursive: true, force: true });
|
|
}
|
|
if (ownsRunRoot) {
|
|
removeSampleRoot(runRoot, transport);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function finishAfterTimeoutCleanup(params: {
|
|
complete: () => void;
|
|
forceKillAt: number | null;
|
|
proc: ReturnType<typeof spawn>;
|
|
}): Promise<void> {
|
|
const graceRemainingMs =
|
|
params.forceKillAt === null
|
|
? TIMEOUT_KILL_GRACE_MS
|
|
: Math.max(0, params.forceKillAt - Date.now());
|
|
if (graceRemainingMs > 0) {
|
|
await waitForSampleProcessGroupExit(params.proc, graceRemainingMs);
|
|
}
|
|
if (isSampleProcessGroupAlive(params.proc)) {
|
|
signalSampleProcess(params.proc, "SIGKILL");
|
|
}
|
|
await waitForSampleProcessGroupExit(params.proc, TIMEOUT_KILL_GRACE_MS);
|
|
params.complete();
|
|
}
|
|
|
|
function signalSampleProcess(proc: ReturnType<typeof spawn>, signal: NodeJS.Signals): void {
|
|
if (!proc.pid) {
|
|
return;
|
|
}
|
|
const handleSignalError = (error: unknown) => {
|
|
const code = (error as NodeJS.ErrnoException | undefined)?.code;
|
|
if (code !== "ESRCH" && code !== "EPERM") {
|
|
throw error;
|
|
}
|
|
};
|
|
terminateManagedChild(proc, signal, {
|
|
onChildSignalError: handleSignalError,
|
|
onProcessGroupSignalError: handleSignalError,
|
|
processGroupFallback: "never",
|
|
useWindowsTaskkill: false,
|
|
});
|
|
}
|
|
|
|
function isSampleProcessGroupAlive(proc: ReturnType<typeof spawn>): boolean {
|
|
return inspectManagedProcessGroup(proc, { errorPolicy: "alive-on-eperm" }) === "live";
|
|
}
|
|
|
|
function waitForSampleProcessGroupExit(
|
|
proc: ReturnType<typeof spawn>,
|
|
timeoutMs: number,
|
|
): Promise<boolean> {
|
|
return waitForManagedProcessGroupExit(proc, timeoutMs, { errorPolicy: "alive-on-eperm" });
|
|
}
|
|
|
|
async function runCase(params: {
|
|
entry: string;
|
|
commandCase: CommandCase;
|
|
runs: number;
|
|
warmup: number;
|
|
timeoutMs: number;
|
|
runtimeRss: boolean;
|
|
cpuProfDir?: string;
|
|
heapProfDir?: string;
|
|
rssHookPath: string;
|
|
}): Promise<CaseRuns> {
|
|
const warmupSamples: Sample[] = [];
|
|
const samples: Sample[] = [];
|
|
const totalRuns = params.warmup + params.runs;
|
|
const transport = sampleTransport();
|
|
const caseRunRoot =
|
|
params.commandCase.stateScope === "case" ? createSampleRoot(transport) : undefined;
|
|
try {
|
|
for (let i = 0; i < totalRuns; i += 1) {
|
|
const sample = await runSample({ ...params, runRoot: caseRunRoot });
|
|
if (i < params.warmup) {
|
|
warmupSamples.push(sample);
|
|
continue;
|
|
}
|
|
samples.push(sample);
|
|
}
|
|
return { warmupSamples, samples };
|
|
} finally {
|
|
if (caseRunRoot) {
|
|
removeSampleRoot(caseRunRoot, transport);
|
|
}
|
|
}
|
|
}
|
|
|
|
function tailLines(value: string, maxLines: number): string {
|
|
return value.split(/\r?\n/).filter(Boolean).slice(-maxLines).join("\n");
|
|
}
|
|
|
|
function printSuite(result: SuiteResult): void {
|
|
console.log(`Entry: ${result.entry}`);
|
|
console.log(`RSS metric: ${cliStartupMemoryMetric(result)}`);
|
|
for (const commandCase of result.cases) {
|
|
const { durationMs, firstOutputMs, maxRssMb, exitSummary } = commandCase.summary;
|
|
const rssSummary =
|
|
maxRssMb == null
|
|
? "rss=n/a"
|
|
: `rss(avg=${formatMb(maxRssMb.avg)} p50=${formatMb(maxRssMb.p50)} p95=${formatMb(maxRssMb.p95)})`;
|
|
const firstOutputSummary =
|
|
firstOutputMs == null
|
|
? "first-output=n/a"
|
|
: `first-output(avg=${formatMs(firstOutputMs.avg)} p50=${formatMs(
|
|
firstOutputMs.p50,
|
|
)} p95=${formatMs(firstOutputMs.p95)})`;
|
|
console.log(
|
|
`${commandCase.name.padEnd(24)} avg=${formatMs(durationMs.avg)} p50=${formatMs(
|
|
durationMs.p50,
|
|
)} p95=${formatMs(durationMs.p95)} min=${formatMs(durationMs.min)} max=${formatMs(
|
|
durationMs.max,
|
|
)} ${firstOutputSummary} ${rssSummary} exits=[${exitSummary}]`,
|
|
);
|
|
}
|
|
console.log("");
|
|
}
|
|
|
|
function printDelta(primary: SuiteResult, secondary: SuiteResult): void {
|
|
const deltas = buildCaseDeltas(primary, secondary);
|
|
console.log("Delta (secondary - primary, avg)");
|
|
for (const delta of deltas) {
|
|
const durationDelta = delta.durationAvgDeltaMs;
|
|
const durationPct = delta.durationAvgDeltaPct;
|
|
const durationSign = durationDelta > 0 ? "+" : "";
|
|
let line = `${delta.name.padEnd(24)} ${durationSign}${formatMs(durationDelta)} (${durationSign}${durationPct.toFixed(1)}%)`;
|
|
if (delta.maxRssAvgDeltaMb != null && delta.maxRssAvgDeltaPct != null) {
|
|
const rssDelta = delta.maxRssAvgDeltaMb;
|
|
const rssPct = delta.maxRssAvgDeltaPct;
|
|
const rssSign = rssDelta > 0 ? "+" : "";
|
|
line += ` rss ${rssSign}${formatMb(rssDelta)} (${rssSign}${rssPct.toFixed(1)}%)`;
|
|
}
|
|
console.log(line);
|
|
}
|
|
}
|
|
|
|
function buildCaseDeltas(primary: SuiteResult, secondary: SuiteResult): CaseDelta[] {
|
|
assertCompatibleCliStartupExecutionModes(primary, secondary);
|
|
assertCompatibleCliStartupMemoryMetrics(primary, secondary);
|
|
const primaryById = new Map(primary.cases.map((commandCase) => [commandCase.id, commandCase]));
|
|
const deltas: CaseDelta[] = [];
|
|
for (const commandCase of secondary.cases) {
|
|
const baseline = primaryById.get(commandCase.id);
|
|
if (!baseline) {
|
|
continue;
|
|
}
|
|
const durationDelta = commandCase.summary.durationMs.avg - baseline.summary.durationMs.avg;
|
|
const durationPct =
|
|
baseline.summary.durationMs.avg > 0
|
|
? (durationDelta / baseline.summary.durationMs.avg) * 100
|
|
: 0;
|
|
const rssDelta =
|
|
baseline.summary.maxRssMb && commandCase.summary.maxRssMb
|
|
? commandCase.summary.maxRssMb.avg - baseline.summary.maxRssMb.avg
|
|
: null;
|
|
const rssPct =
|
|
rssDelta != null && baseline.summary.maxRssMb && baseline.summary.maxRssMb.avg > 0
|
|
? (rssDelta / baseline.summary.maxRssMb.avg) * 100
|
|
: null;
|
|
deltas.push({
|
|
id: commandCase.id,
|
|
name: commandCase.name,
|
|
durationAvgDeltaMs: durationDelta,
|
|
durationAvgDeltaPct: durationPct,
|
|
maxRssAvgDeltaMb: rssDelta,
|
|
maxRssAvgDeltaPct: rssPct,
|
|
});
|
|
}
|
|
return deltas;
|
|
}
|
|
|
|
export function collectFailedSamples(result: SuiteResult): string[] {
|
|
const failures: string[] = [];
|
|
for (const commandCase of result.cases) {
|
|
if (commandCase.samples.length === 0) {
|
|
failures.push(`${result.entry} ${commandCase.id}: no measured samples`);
|
|
}
|
|
for (const [sampleKind, samples] of [
|
|
["warmup", commandCase.warmupSamples ?? []],
|
|
["sample", commandCase.samples],
|
|
] as const) {
|
|
for (const [sampleIndex, sample] of samples.entries()) {
|
|
const label = `${result.entry} ${commandCase.id} ${sampleKind} ${sampleIndex + 1}`;
|
|
const expectedExitCodes = new Set(commandCase.expectedExitCodes ?? [0]);
|
|
if (sample.timedOut === true) {
|
|
failures.push(`${label}: timed out`);
|
|
} else if (sample.signal !== null) {
|
|
failures.push(`${label}: exited via signal ${sample.signal}`);
|
|
} else if (!expectedExitCodes.has(sample.exitCode ?? -1)) {
|
|
failures.push(`${label}: exited with code ${String(sample.exitCode)}`);
|
|
} else if (sample.maxRssMb === null) {
|
|
failures.push(
|
|
`${label}: did not report max RSS${sample.memory?.error ? ` (${sample.memory.error})` : ""}`,
|
|
);
|
|
} else if (sample.exitCode !== 0) {
|
|
const output = `${sample.stdoutTail ?? ""}\n${sample.stderrTail ?? ""}`;
|
|
const missing = (commandCase.expectedNonzeroOutputIncludes ?? []).filter(
|
|
(snippet) => !output.includes(snippet),
|
|
);
|
|
if (missing.length > 0) {
|
|
failures.push(
|
|
`${label}: exited with expected code ${String(
|
|
sample.exitCode,
|
|
)} but output did not match expected clean-state markers (${missing.join(", ")})`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return failures;
|
|
}
|
|
|
|
async function buildSuiteResult(params: {
|
|
entry: string;
|
|
executionMode: CliStartupExecutionMode;
|
|
options: CliOptions;
|
|
rssHookPath: string;
|
|
}): Promise<SuiteResult> {
|
|
const cases = [];
|
|
for (const commandCase of params.options.cases) {
|
|
const { warmupSamples, samples } = await runCase({
|
|
entry: params.entry,
|
|
commandCase,
|
|
runs: params.options.runs,
|
|
warmup: params.options.warmup,
|
|
timeoutMs: params.options.timeoutMs,
|
|
runtimeRss: params.options.runtimeRss,
|
|
cpuProfDir: params.options.cpuProfDir,
|
|
heapProfDir: params.options.heapProfDir,
|
|
rssHookPath: params.rssHookPath,
|
|
});
|
|
cases.push({
|
|
id: commandCase.id,
|
|
name: commandCase.name,
|
|
args: commandCase.args,
|
|
...(commandCase.expectedExitCodes && commandCase.expectedExitCodes.some((code) => code !== 0)
|
|
? { expectedExitCodes: [...commandCase.expectedExitCodes] }
|
|
: {}),
|
|
...(commandCase.expectedNonzeroOutputIncludes
|
|
? { expectedNonzeroOutputIncludes: [...commandCase.expectedNonzeroOutputIncludes] }
|
|
: {}),
|
|
contract:
|
|
commandCase.firstOutputBudgetMs != null || commandCase.exitBudgetMs != null
|
|
? {
|
|
firstOutputBudgetMs: commandCase.firstOutputBudgetMs ?? null,
|
|
exitBudgetMs: commandCase.exitBudgetMs ?? null,
|
|
}
|
|
: null,
|
|
warmupSamples,
|
|
samples,
|
|
summary: summarizeSamples(samples),
|
|
});
|
|
}
|
|
return {
|
|
entry: params.entry,
|
|
executionMode: params.executionMode,
|
|
...(params.options.runtimeRss ? { memoryMetric: CLI_RUNTIME_MEMORY_METRIC } : {}),
|
|
cases,
|
|
};
|
|
}
|
|
|
|
function parseOptions(): CliOptions {
|
|
const presets = parsePresets(parseFlagValue("--preset"));
|
|
const cases = resolveCases({
|
|
presets,
|
|
caseIds: parseRepeatableFlag("--case"),
|
|
});
|
|
return {
|
|
cases,
|
|
compareBaseline: parseFlagValue("--compare-baseline"),
|
|
compareCandidate: parseFlagValue("--compare-candidate"),
|
|
entryPrimary: parseFlagValue("--entry-primary") ?? parseFlagValue("--entry") ?? DEFAULT_ENTRY,
|
|
entrySecondary: parseFlagValue("--entry-secondary"),
|
|
runs: parsePositiveInt(parseFlagValue("--runs"), DEFAULT_RUNS, "--runs"),
|
|
warmup: parseNonNegativeInt(parseFlagValue("--warmup"), DEFAULT_WARMUP, "--warmup"),
|
|
timeoutMs: parsePositiveInt(parseFlagValue("--timeout-ms"), DEFAULT_TIMEOUT_MS, "--timeout-ms"),
|
|
runtimeRss: hasFlag("--runtime-rss"),
|
|
json: hasFlag("--json"),
|
|
output: parseFlagValue("--output"),
|
|
cpuProfDir: parseFlagValue("--cpu-prof-dir"),
|
|
heapProfDir: parseFlagValue("--heap-prof-dir"),
|
|
};
|
|
}
|
|
|
|
function printUsage(): void {
|
|
console.log(`OpenClaw CLI benchmark
|
|
|
|
Usage:
|
|
pnpm tsx scripts/bench-cli-startup.ts [options]
|
|
|
|
Options:
|
|
--preset <startup|real|response|all>
|
|
Command preset to run (default: startup)
|
|
--case <id> Specific case id to run; repeatable
|
|
--entry <path> Primary entry file (default: openclaw.mjs)
|
|
--entry-secondary <path> Secondary entry file for avg delta comparison
|
|
--runs <n> Measured runs per case (default: ${DEFAULT_RUNS})
|
|
--warmup <n> Warmup runs per case (default: ${DEFAULT_WARMUP})
|
|
--timeout-ms <ms> Per-run timeout (default: ${DEFAULT_TIMEOUT_MS})
|
|
--output <path> Write machine-readable JSON to a file
|
|
--compare-baseline <path> Read a saved JSON report as the baseline
|
|
--compare-candidate <path> Read a saved JSON report as the candidate and print deltas
|
|
--cpu-prof-dir <dir> Write V8 CPU profiles for each run
|
|
--heap-prof-dir <dir> Write V8 heap profiles for each run
|
|
--runtime-rss Attribute RSS to the CLI runtime (default: legacy last marker)
|
|
--json Emit machine-readable JSON
|
|
--help Show this text
|
|
|
|
Case ids:
|
|
${COMMAND_CASES.map((commandCase) => `${commandCase.id} (${commandCase.name})`).join("\n ")}
|
|
`);
|
|
}
|
|
|
|
function readBenchmarkReport(filePath: string): BenchmarkReport {
|
|
return JSON.parse(readFileSync(filePath, "utf8")) as BenchmarkReport;
|
|
}
|
|
|
|
function writeJsonOutput(filePath: string, value: unknown): void {
|
|
mkdirSync(path.dirname(filePath), { recursive: true });
|
|
writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
}
|
|
|
|
function readBenchmarkComparison(
|
|
baselinePath: string,
|
|
candidatePath: string,
|
|
): BenchmarkComparisonResult {
|
|
const baseline = readBenchmarkReport(baselinePath);
|
|
const candidate = readBenchmarkReport(candidatePath);
|
|
return {
|
|
baseline: baseline.primary,
|
|
candidate: candidate.primary,
|
|
comparison: {
|
|
baseline: baselinePath,
|
|
candidate: candidatePath,
|
|
deltas: buildCaseDeltas(baseline.primary, candidate.primary),
|
|
},
|
|
};
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
validateCliArgs();
|
|
if (hasFlag("--help")) {
|
|
printUsage();
|
|
return;
|
|
}
|
|
|
|
const options = parseOptions();
|
|
const transport = sampleTransport();
|
|
if (transport && options.runtimeRss) {
|
|
throw new Error("Cross-user runtime RSS sampling is not supported");
|
|
}
|
|
if (transport && (options.cpuProfDir || options.heapProfDir)) {
|
|
throw new Error(
|
|
"Cross-user CLI profiles must be collected through the SUT diagnostic exporter",
|
|
);
|
|
}
|
|
if (options.compareBaseline || options.compareCandidate) {
|
|
if (!options.compareBaseline || !options.compareCandidate) {
|
|
throw new Error("--compare-baseline and --compare-candidate must be provided together");
|
|
}
|
|
const { baseline, candidate, comparison } = readBenchmarkComparison(
|
|
options.compareBaseline,
|
|
options.compareCandidate,
|
|
);
|
|
if (options.output) {
|
|
writeJsonOutput(options.output, comparison);
|
|
}
|
|
if (options.json) {
|
|
console.log(JSON.stringify(comparison, null, 2));
|
|
return;
|
|
}
|
|
printDelta(baseline, candidate);
|
|
return;
|
|
}
|
|
const tmpDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-"));
|
|
const rssHookPath = options.runtimeRss ? buildRuntimeRssHook(tmpDir) : buildRssHook(tmpDir);
|
|
try {
|
|
const primary = await buildSuiteResult({
|
|
entry: options.entryPrimary,
|
|
executionMode: transport ? "transport" : "native",
|
|
options,
|
|
rssHookPath,
|
|
});
|
|
const secondary = options.entrySecondary
|
|
? await buildSuiteResult({
|
|
entry: options.entrySecondary,
|
|
executionMode: transport ? "transport" : "native",
|
|
options,
|
|
rssHookPath,
|
|
})
|
|
: undefined;
|
|
|
|
const report = {
|
|
node: process.version,
|
|
runs: options.runs,
|
|
warmup: options.warmup,
|
|
timeoutMs: options.timeoutMs,
|
|
cpuProfDir: options.cpuProfDir ?? null,
|
|
heapProfDir: options.heapProfDir ?? null,
|
|
primary,
|
|
secondary: secondary ?? null,
|
|
};
|
|
const failures = [
|
|
...collectFailedSamples(primary),
|
|
...(secondary ? collectFailedSamples(secondary) : []),
|
|
];
|
|
|
|
if (options.output) {
|
|
writeJsonOutput(options.output, report);
|
|
}
|
|
|
|
if (options.json) {
|
|
console.log(JSON.stringify(report, null, 2));
|
|
if (failures.length > 0) {
|
|
process.exitCode = 1;
|
|
for (const failure of failures) {
|
|
console.error(`[startup-bench] ${failure}`);
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
|
|
console.log(`Node: ${process.version}`);
|
|
console.log(`Runs per case: ${options.runs}`);
|
|
console.log(`Warmup runs per case: ${options.warmup}`);
|
|
console.log(`Timeout: ${options.timeoutMs}ms`);
|
|
if (options.cpuProfDir) {
|
|
console.log(`CPU profiles: ${options.cpuProfDir}`);
|
|
}
|
|
if (options.heapProfDir) {
|
|
console.log(`Heap profiles: ${options.heapProfDir}`);
|
|
}
|
|
console.log("");
|
|
|
|
console.log("Primary entry");
|
|
printSuite(primary);
|
|
if (secondary) {
|
|
console.log("Secondary entry");
|
|
printSuite(secondary);
|
|
printDelta(primary, secondary);
|
|
}
|
|
|
|
if (failures.length > 0) {
|
|
process.exitCode = 1;
|
|
console.error("\nFailed startup benchmark samples:");
|
|
for (const failure of failures) {
|
|
console.error(`- ${failure}`);
|
|
}
|
|
}
|
|
} finally {
|
|
rmSync(tmpDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
export const testing = {
|
|
buildConfigFixture,
|
|
collectFailedSamples,
|
|
nodeImportSpecifierForPath,
|
|
parseGatewayPortEnv,
|
|
parseNonNegativeInt,
|
|
parsePositiveInt,
|
|
validateCliArgs,
|
|
};
|
|
|
|
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
|
await main().catch((error: unknown) => {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exit(1);
|
|
});
|
|
}
|