From e6823c3d16d405a5ea82d32ee06fea57cead2608 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 20 Jun 2026 03:06:22 +0200 Subject: [PATCH] fix(test): guard model benchmark cli args --- scripts/bench-model.ts | 127 +++++++++++++++++++++++++------ test/scripts/bench-model.test.ts | 57 ++++++++++++++ 2 files changed, 162 insertions(+), 22 deletions(-) create mode 100644 test/scripts/bench-model.test.ts diff --git a/scripts/bench-model.ts b/scripts/bench-model.ts index 1003711f1ef..4424b6aab37 100644 --- a/scripts/bench-model.ts +++ b/scripts/bench-model.ts @@ -1,5 +1,7 @@ // Bench Model script supports OpenClaw repository automation. +import { pathToFileURL } from "node:url"; import { completeSimple, type Model } from "openclaw/plugin-sdk/llm"; +import { parseStrictIntegerOption } from "./lib/dev-tooling-safety.ts"; type Usage = { input?: number; @@ -14,26 +16,87 @@ type RunResult = { usage?: Usage; }; +type CliOptions = { + help: boolean; + prompt: string; + runs: number; +}; + const DEFAULT_PROMPT = "Reply with a single word: ok. No punctuation or extra text."; const DEFAULT_RUNS = 10; +const BOOLEAN_FLAGS = new Set(["--help", "-h"]); +const VALUE_FLAGS = new Set(["--prompt", "--runs"]); -function parseArg(flag: string): string | undefined { - const idx = process.argv.indexOf(flag); - if (idx === -1) { +class CliArgumentError extends Error { + override name = "CliArgumentError"; +} + +function readValue(argv: string[], index: number, flag: string): string { + const value = argv[index + 1]?.trim() ?? ""; + if (!value || value.startsWith("--")) { + throw new CliArgumentError(`${flag} requires a value`); + } + return value; +} + +function validateCliArgs(argv: string[]): void { + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] ?? ""; + if (BOOLEAN_FLAGS.has(arg)) { + continue; + } + if (VALUE_FLAGS.has(arg)) { + readValue(argv, index, arg); + index += 1; + continue; + } + throw new CliArgumentError(`Unknown argument: ${arg}`); + } +} + +function parseArg(argv: string[], flag: string): string | undefined { + const index = argv.indexOf(flag); + if (index === -1) { return undefined; } - return process.argv[idx + 1]; + return readValue(argv, index, flag); } function parseRuns(raw: string | undefined): number { - if (!raw) { - return DEFAULT_RUNS; - } - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) { - return DEFAULT_RUNS; - } - return Math.floor(parsed); + return parseStrictIntegerOption({ + fallback: DEFAULT_RUNS, + label: "--runs", + min: 1, + raw, + }); +} + +function parseArgs(argv = process.argv.slice(2)): CliOptions { + validateCliArgs(argv); + return { + help: argv.includes("--help") || argv.includes("-h"), + prompt: parseArg(argv, "--prompt") ?? DEFAULT_PROMPT, + runs: parseRuns(parseArg(argv, "--runs")), + }; +} + +function printUsage(): void { + console.log(`OpenClaw model latency benchmark + +Usage: + node --import tsx scripts/bench-model.ts [options] + +Options: + --runs Runs per model (default: ${DEFAULT_RUNS}) + --prompt Prompt to send to each model + --help, -h Show this text + +Environment: + ANTHROPIC_API_KEY + MINIMAX_API_KEY + MINIMAX_BASE_URL + MINIMAX_MODEL +`); } function median(values: number[]): number { @@ -78,9 +141,12 @@ async function runModel(opts: { return results; } -async function main(): Promise { - const runs = parseRuns(parseArg("--runs")); - const prompt = parseArg("--prompt") ?? DEFAULT_PROMPT; +async function main(argv = process.argv.slice(2)): Promise { + const options = parseArgs(argv); + if (options.help) { + printUsage(); + return; + } const anthropicKey = process.env.ANTHROPIC_API_KEY?.trim(); const minimaxKey = process.env.MINIMAX_API_KEY?.trim(); @@ -118,23 +184,23 @@ async function main(): Promise { maxTokens: 32000, }; - console.log(`Prompt: ${prompt}`); - console.log(`Runs: ${runs}`); + console.log(`Prompt: ${options.prompt}`); + console.log(`Runs: ${options.runs}`); console.log(""); const minimaxResults = await runModel({ label: "minimax", model: minimaxModel, apiKey: minimaxKey, - runs, - prompt, + runs: options.runs, + prompt: options.prompt, }); const opusResults = await runModel({ label: "opus", model: opusModel, apiKey: anthropicKey, - runs, - prompt, + runs: options.runs, + prompt: options.prompt, }); const summarize = (label: string, results: RunResult[]) => { @@ -153,4 +219,21 @@ async function main(): Promise { } } -await main(); +export const testing = { + median, + parseArgs, + parseRuns, + validateCliArgs, +}; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main().catch((err: unknown) => { + if (err instanceof CliArgumentError) { + console.error(err.message); + process.exitCode = 1; + return; + } + console.error(err instanceof Error ? err.stack : String(err)); + process.exitCode = 1; + }); +} diff --git a/test/scripts/bench-model.test.ts b/test/scripts/bench-model.test.ts new file mode 100644 index 00000000000..51f31a33437 --- /dev/null +++ b/test/scripts/bench-model.test.ts @@ -0,0 +1,57 @@ +// Bench Model tests cover live model benchmark CLI safety. +import { spawnSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; +import { testing } from "../../scripts/bench-model.ts"; + +function runBenchModel(args: string[]) { + return spawnSync(process.execPath, ["--import", "tsx", "scripts/bench-model.ts", ...args], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + ANTHROPIC_API_KEY: "", + MINIMAX_API_KEY: "", + }, + }); +} + +describe("scripts/bench-model", () => { + it("parses benchmark options without importing live credentials", () => { + expect(testing.parseArgs(["--runs", "2", "--prompt", "ping"])).toMatchObject({ + help: false, + prompt: "ping", + runs: 2, + }); + }); + + it("rejects unknown args before checking provider credentials", () => { + expect(() => testing.parseArgs(["--wat"])).toThrow("Unknown argument: --wat"); + + const result = runBenchModel(["--wat"]); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr.trim()).toBe("Unknown argument: --wat"); + expect(result.stderr).not.toContain("Missing ANTHROPIC_API_KEY"); + expect(result.stderr).not.toContain("\n at "); + }); + + it("rejects malformed run counts instead of silently using defaults", () => { + expect(() => testing.parseArgs(["--runs", "1e3"])).toThrow("--runs must be an integer"); + + const result = runBenchModel(["--runs", "1e3"]); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("--runs must be an integer"); + expect(result.stderr).not.toContain("Missing ANTHROPIC_API_KEY"); + }); + + it("prints help without checking provider credentials", () => { + const result = runBenchModel(["--help"]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("OpenClaw model latency benchmark"); + expect(result.stderr).toBe(""); + }); +});