mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
7cbaa6b36f
Consolidate script lifecycle and helper ownership while preserving CLI, generated-output, cleanup, and guard contracts. Reject inherited object keys in model-matrix option admission with the existing unknown-argument diagnostic. Validation: 90 full sibling suites; original-fails parser regression; byte-identical native outputs; changed checks; SDK surface and API comparison; both zero-cycle checks; independent isolated review.
1954 lines
68 KiB
JavaScript
1954 lines
68 KiB
JavaScript
#!/usr/bin/env node
|
||
|
||
import { execFile } from "node:child_process";
|
||
import { createHash } from "node:crypto";
|
||
import fs from "node:fs/promises";
|
||
import os from "node:os";
|
||
import path from "node:path";
|
||
import process from "node:process";
|
||
import { pathToFileURL } from "node:url";
|
||
import { promisify } from "node:util";
|
||
import {
|
||
buildScriptEvidenceSummary,
|
||
captureQaEvidenceRuntimeIdentity,
|
||
captureQaEvidenceSourceIdentity as readSourceIdentity,
|
||
createQaEvidenceInvocation,
|
||
QA_EVIDENCE_FILENAME,
|
||
QA_FRONTIER_PROVIDER_IDS,
|
||
validateQaEvidenceSummaryJson,
|
||
type QaEvidenceIdentity,
|
||
type QaEvidenceStatus,
|
||
type QaEvidenceSummaryJson,
|
||
} from "../extensions/qa-lab/test-api.js";
|
||
import type { CodeModeExecutorId } from "../src/agents/code-mode-executor-types.js";
|
||
import type { AgentExecEnvelope } from "../src/commands/agent-exec-result.ts";
|
||
import type { OpenClawConfig } from "../src/config/types.openclaw.js";
|
||
import { mergeDeep } from "../src/infra/deep-merge.js";
|
||
import { requireOptionArgument } from "./lib/arg-utils.mts";
|
||
import { summarizeGatewayMatrixOutcomes } from "./lib/code-mode-matrix-comparison.ts";
|
||
import {
|
||
GATEWAY_MATRIX_TASKS,
|
||
type GatewayMatrixTask,
|
||
} from "./lib/code-mode-matrix-gateway-fixtures.ts";
|
||
import type {
|
||
GatewayMatrixEvidence,
|
||
GatewayMatrixWorkload,
|
||
} from "./lib/code-mode-matrix-gateway.ts";
|
||
import {
|
||
MATRIX_PERFORMANCE_TASKS,
|
||
isMatrixPerformanceTask,
|
||
type MatrixPerformanceTask,
|
||
} from "./lib/code-mode-matrix-performance-fixtures.ts";
|
||
import {
|
||
classifyCodeModeMatrixProviderFailure,
|
||
matrixModelConfig,
|
||
matrixProviderEnv,
|
||
matrixProviderAuthSelection,
|
||
type MatrixProviderFailureCategory,
|
||
} from "./lib/code-mode-matrix-provider.ts";
|
||
import type { MatrixUsageAccounting } from "./lib/code-mode-matrix-usage.ts";
|
||
import { previewForDevToolLog, redactJsonValueForDevToolLog } from "./lib/dev-tooling-safety.ts";
|
||
import { groupBy } from "./lib/group-by.mts";
|
||
|
||
export { validateQaEvidenceSummaryJson };
|
||
|
||
const execFileAsync = promisify(execFile);
|
||
const SOURCE_PATH = "scripts/code-mode-model-matrix.ts";
|
||
const MATRIX_SCHEMA_VERSION = 1;
|
||
const DEFAULT_REPETITIONS = 3;
|
||
const DEFAULT_TIMEOUT_SECONDS = 180;
|
||
const MAX_REPETITIONS = 10;
|
||
const MAX_DIAGNOSTIC_CHARS = 8_000;
|
||
const STRICT_SOURCE_IDENTITY_OPTIONS = { gitTimeoutMs: 60_000 };
|
||
const DEFAULT_ADMISSION = {
|
||
concurrency: 2,
|
||
maxCells: 36,
|
||
maxTokens: 1_000_000,
|
||
maxKnownCostUsd: 25,
|
||
maxWallSeconds: 3_600,
|
||
};
|
||
|
||
export type CodeModeMatrixMode = "direct" | "auto" | "code";
|
||
const MATRIX_TASKS = [
|
||
"read",
|
||
"dependent-read-write",
|
||
"large-result-reduction",
|
||
"parallel-independent-reads",
|
||
"dependent-chain",
|
||
...GATEWAY_MATRIX_TASKS,
|
||
...MATRIX_PERFORMANCE_TASKS,
|
||
] as const;
|
||
export type CodeModeMatrixTask = (typeof MATRIX_TASKS)[number];
|
||
|
||
export type CodeModeMatrixOptions = {
|
||
allowFailures: boolean;
|
||
dryRun: boolean;
|
||
gatewayExecutor?: CodeModeExecutorId;
|
||
keepState: boolean;
|
||
models: string[];
|
||
modes: CodeModeMatrixMode[];
|
||
outputDir?: string;
|
||
repetitions: number;
|
||
repoRoot: string;
|
||
runtimeDir?: string;
|
||
baselineResults?: string;
|
||
tasks: CodeModeMatrixTask[];
|
||
thinking: string;
|
||
timeoutSeconds: number;
|
||
concurrency?: number;
|
||
maxCells?: number;
|
||
maxTokens?: number;
|
||
maxKnownCostUsd?: number;
|
||
maxWallSeconds?: number;
|
||
schedulePath?: string;
|
||
};
|
||
|
||
export type MatrixCell = {
|
||
id: string;
|
||
mode: CodeModeMatrixMode;
|
||
model: string;
|
||
repetition: number;
|
||
task: CodeModeMatrixTask;
|
||
};
|
||
|
||
type MatrixTaskFixture = {
|
||
expected: string;
|
||
prompt: string;
|
||
resultPath?: string;
|
||
};
|
||
|
||
export type MatrixRuntimeEntrypoint = {
|
||
args: string[];
|
||
cwd: string;
|
||
};
|
||
|
||
type CellFailureCategory =
|
||
| MatrixProviderFailureCategory
|
||
| "activation"
|
||
| "agent_error"
|
||
| "answer_mismatch"
|
||
| "effect_mismatch"
|
||
| "harness_error"
|
||
| "interview_mismatch"
|
||
| "model_mismatch"
|
||
| "timeout"
|
||
| "tool_execution";
|
||
|
||
export type CodeModeMatrixCellResult = {
|
||
accounting?: MatrixUsageAccounting;
|
||
assistantTurns?: number;
|
||
bridgeCalls?: AgentExecEnvelope["bridgeCalls"];
|
||
buildSha256: string;
|
||
codeModeEngaged: boolean | null;
|
||
costUsd?: number;
|
||
diagnostics?: string;
|
||
elapsedMs: number;
|
||
evidenceOccurrenceId?: string;
|
||
executor?: CodeModeExecutorId;
|
||
error?: AgentExecEnvelope["error"];
|
||
expected: string;
|
||
failureCategory: CellFailureCategory | null;
|
||
final: string;
|
||
gitSha: string;
|
||
id: string;
|
||
mode: CodeModeMatrixMode;
|
||
model: string;
|
||
observedModel: string | null;
|
||
observedProvider: string | null;
|
||
oracle: {
|
||
answer: boolean | null;
|
||
effect: boolean;
|
||
engagement: boolean;
|
||
identity: boolean;
|
||
toolExecution: boolean;
|
||
};
|
||
passed: boolean;
|
||
repetition: number;
|
||
sourceDirty: boolean;
|
||
sourcePatchSha256: string | null;
|
||
status: AgentExecEnvelope["status"];
|
||
task: CodeModeMatrixTask;
|
||
timestamp: string;
|
||
toolSummary?: AgentExecEnvelope["toolSummary"];
|
||
usage?: AgentExecEnvelope["usage"];
|
||
gateway?: GatewayMatrixEvidence;
|
||
workload?: GatewayMatrixWorkload;
|
||
};
|
||
|
||
export type RunCellParams = {
|
||
abortSignal?: AbortSignal;
|
||
buildSha256: string;
|
||
cell: MatrixCell;
|
||
executor?: CodeModeExecutorId;
|
||
gitSha: string;
|
||
keepState: boolean;
|
||
outputDir: string;
|
||
repoRoot: string;
|
||
runtime?: MatrixRuntimeEntrypoint;
|
||
sourceDirty: boolean;
|
||
sourcePatchSha256: string | null;
|
||
thinking: string;
|
||
timeoutSeconds: number;
|
||
};
|
||
|
||
type MatrixRunDependencies = {
|
||
buildCliArtifacts?: (repoRoot: string) => Promise<void>;
|
||
now?: () => Date;
|
||
readBuildSha256?: (repoRoot: string) => Promise<string>;
|
||
readGitSha?: (repoRoot: string) => Promise<string>;
|
||
readSourceIdentity?: (repoRoot: string) => Promise<SourceIdentity>;
|
||
runCell?: (params: RunCellParams) => Promise<CodeModeMatrixCellResult>;
|
||
};
|
||
|
||
type SourceIdentity = {
|
||
gitSha: string;
|
||
sourceDirty: boolean;
|
||
sourcePatchSha256: string | null;
|
||
};
|
||
|
||
function usage() {
|
||
return `Usage: pnpm qa:code-mode-models --model <provider/model> [options]
|
||
|
||
Runs repeated Code Mode acceptance cells through the normal embedded agent path.
|
||
|
||
Options:
|
||
--model <provider/model> Model reference; repeat for multiple models
|
||
--mode <mode> direct | auto | code; repeat to select modes
|
||
--executor <executor> node | quickjs for Gateway tasks only (default: node)
|
||
--task <task> ${MATRIX_TASKS.join(" | ")}; repeat to select tasks
|
||
(default: read, dependent-read-write)
|
||
--repetitions <n> Runs per model/mode/task cell (default: ${DEFAULT_REPETITIONS}, max: ${MAX_REPETITIONS})
|
||
--timeout <seconds> Per-run agent deadline (default: ${DEFAULT_TIMEOUT_SECONDS})
|
||
--thinking <level> Agent thinking level (default: low)
|
||
--concurrency <n> Concurrent root cells per wave, 1–3 (default: 2)
|
||
--schedule <path> JSON array of {model,task,repetition,firstMode} paired waves
|
||
--max-cells <n> Admit complete waves up to this count (default: 36)
|
||
--max-tokens <n> Stop new waves at observed token total (default: 1000000)
|
||
--max-known-cost-usd <n> Stop new waves at known cost (default: 25)
|
||
--max-wall-seconds <n> Stop new waves after elapsed time (default: 3600)
|
||
--output-dir <path> Repo-relative artifact directory
|
||
--runtime-dir <path> Use a clean, already-built checkout without rebuilding it
|
||
--baseline-results <path> Compare matching cells from an earlier results.jsonl
|
||
--keep-state Retain per-cell state and workspace directories
|
||
--allow-failures Exit zero after writing evidence even when cells fail
|
||
--dry-run Write the manifest without calling models
|
||
-h, --help Show this help
|
||
|
||
Provider credentials are read from the environment and are never written to artifacts.
|
||
Token, known-cost, and wall limits stop admission; already admitted waves settle.
|
||
Missing usage and prices remain unknown. These limits are not hard spending caps.
|
||
`;
|
||
}
|
||
|
||
function parseIntegerOption(raw: string, flag: string, max?: number): number {
|
||
if (!/^\d+$/u.test(raw)) {
|
||
throw new Error(`${flag} must be a positive integer`);
|
||
}
|
||
const value = Number(raw);
|
||
if (!Number.isSafeInteger(value) || value < 1 || (max !== undefined && value > max)) {
|
||
const suffix = max === undefined ? "" : ` from 1 to ${max}`;
|
||
throw new Error(`${flag} must be an integer${suffix}`);
|
||
}
|
||
return value;
|
||
}
|
||
|
||
function collectUnique<T extends string>(values: T[], value: T, flag: string): void {
|
||
if (values.includes(value)) {
|
||
throw new Error(`Duplicate ${flag} value: ${value}`);
|
||
}
|
||
values.push(value);
|
||
}
|
||
|
||
function parseMode(raw: string): CodeModeMatrixMode {
|
||
if (raw === "direct" || raw === "auto" || raw === "code") {
|
||
return raw;
|
||
}
|
||
throw new Error(`--mode must be one of direct, auto, code; got ${JSON.stringify(raw)}`);
|
||
}
|
||
|
||
function parseTask(raw: string): CodeModeMatrixTask {
|
||
const task = MATRIX_TASKS.find((candidate) => candidate === raw);
|
||
if (task) {
|
||
return task;
|
||
}
|
||
throw new Error(`--task must be one of ${MATRIX_TASKS.join(", ")}; got ${JSON.stringify(raw)}`);
|
||
}
|
||
|
||
function isGatewayTask(
|
||
task: CodeModeMatrixTask,
|
||
): task is GatewayMatrixTask | MatrixPerformanceTask {
|
||
return isMatrixPerformanceTask(task) || isGatewayContractTask(task);
|
||
}
|
||
|
||
function isGatewayContractTask(task: CodeModeMatrixTask): task is GatewayMatrixTask {
|
||
return GATEWAY_MATRIX_TASKS.some((candidate) => candidate === task);
|
||
}
|
||
|
||
export function parseCodeModeMatrixOptions(
|
||
argv: readonly string[],
|
||
cwd = process.cwd(),
|
||
): CodeModeMatrixOptions {
|
||
const models: string[] = [];
|
||
const modes: CodeModeMatrixMode[] = [];
|
||
const tasks: CodeModeMatrixTask[] = [];
|
||
const options: CodeModeMatrixOptions = {
|
||
allowFailures: false,
|
||
...DEFAULT_ADMISSION,
|
||
dryRun: false,
|
||
gatewayExecutor: "node",
|
||
keepState: false,
|
||
models,
|
||
modes,
|
||
outputDir: undefined,
|
||
repetitions: DEFAULT_REPETITIONS,
|
||
repoRoot: path.resolve(cwd),
|
||
tasks,
|
||
thinking: "low",
|
||
timeoutSeconds: DEFAULT_TIMEOUT_SECONDS,
|
||
};
|
||
const seen = new Set<string>();
|
||
const recordOnce = (flag: string) => {
|
||
if (seen.has(flag)) {
|
||
throw new Error(`${flag} was provided more than once`);
|
||
}
|
||
seen.add(flag);
|
||
};
|
||
|
||
for (let index = 0; index < argv.length; index += 1) {
|
||
const arg = argv[index] ?? "";
|
||
if (arg === "--model") {
|
||
const value = requireOptionArgument(argv, index, arg).trim();
|
||
if (!value.includes("/")) {
|
||
throw new Error(
|
||
`--model must use a provider/model reference; got ${JSON.stringify(value)}`,
|
||
);
|
||
}
|
||
collectUnique(models, value, arg);
|
||
index += 1;
|
||
continue;
|
||
}
|
||
const admissionKeys: Record<string, keyof typeof DEFAULT_ADMISSION> = {
|
||
"--concurrency": "concurrency",
|
||
"--max-cells": "maxCells",
|
||
"--max-tokens": "maxTokens",
|
||
"--max-known-cost-usd": "maxKnownCostUsd",
|
||
"--max-wall-seconds": "maxWallSeconds",
|
||
};
|
||
const admissionKey = Object.hasOwn(admissionKeys, arg) ? admissionKeys[arg] : undefined;
|
||
if (admissionKey) {
|
||
recordOnce(arg);
|
||
const raw = requireOptionArgument(argv, index, arg);
|
||
const value =
|
||
admissionKey === "maxKnownCostUsd"
|
||
? Number(raw)
|
||
: parseIntegerOption(raw, arg, admissionKey === "concurrency" ? 3 : undefined);
|
||
if (!Number.isFinite(value) || value <= 0) {
|
||
throw new Error(`${arg} must be a positive number`);
|
||
}
|
||
options[admissionKey] = value;
|
||
index += 1;
|
||
continue;
|
||
}
|
||
switch (arg) {
|
||
case "--schedule":
|
||
recordOnce(arg);
|
||
options.schedulePath = path.resolve(cwd, requireOptionArgument(argv, index, arg));
|
||
break;
|
||
case "--mode":
|
||
collectUnique(modes, parseMode(requireOptionArgument(argv, index, arg)), arg);
|
||
break;
|
||
case "--executor": {
|
||
recordOnce(arg);
|
||
const value = requireOptionArgument(argv, index, arg);
|
||
if (value !== "node" && value !== "quickjs") {
|
||
throw new Error(`--executor must be node or quickjs; got ${JSON.stringify(value)}`);
|
||
}
|
||
options.gatewayExecutor = value;
|
||
break;
|
||
}
|
||
case "--task":
|
||
collectUnique(tasks, parseTask(requireOptionArgument(argv, index, arg)), arg);
|
||
break;
|
||
case "--repetitions":
|
||
recordOnce(arg);
|
||
options.repetitions = parseIntegerOption(
|
||
requireOptionArgument(argv, index, arg),
|
||
arg,
|
||
MAX_REPETITIONS,
|
||
);
|
||
break;
|
||
case "--timeout":
|
||
recordOnce(arg);
|
||
options.timeoutSeconds = parseIntegerOption(requireOptionArgument(argv, index, arg), arg);
|
||
break;
|
||
case "--thinking":
|
||
recordOnce(arg);
|
||
options.thinking = requireOptionArgument(argv, index, arg).trim();
|
||
break;
|
||
case "--output-dir":
|
||
recordOnce(arg);
|
||
options.outputDir = requireOptionArgument(argv, index, arg);
|
||
break;
|
||
case "--runtime-dir":
|
||
case "--baseline-results": {
|
||
recordOnce(arg);
|
||
const value = path.resolve(cwd, requireOptionArgument(argv, index, arg));
|
||
if (arg === "--runtime-dir") {
|
||
options.runtimeDir = value;
|
||
} else {
|
||
options.baselineResults = value;
|
||
}
|
||
break;
|
||
}
|
||
case "--allow-failures":
|
||
recordOnce(arg);
|
||
options.allowFailures = true;
|
||
continue;
|
||
case "--keep-state":
|
||
recordOnce(arg);
|
||
options.keepState = true;
|
||
continue;
|
||
case "--dry-run":
|
||
recordOnce(arg);
|
||
options.dryRun = true;
|
||
continue;
|
||
case "--help":
|
||
case "-h":
|
||
throw Object.assign(new Error(usage()), { code: "HELP" });
|
||
default:
|
||
throw new Error(`Unknown argument: ${arg}`);
|
||
}
|
||
index += 1;
|
||
}
|
||
|
||
if (models.length === 0) {
|
||
throw new Error("At least one --model <provider/model> is required");
|
||
}
|
||
if (
|
||
seen.has("--executor") &&
|
||
(tasks.length === 0 || tasks.some((task) => !isGatewayTask(task)))
|
||
) {
|
||
throw new Error("--executor requires only Gateway-backed tasks.");
|
||
}
|
||
if (tasks.some(isGatewayContractTask)) {
|
||
if (modes.length !== 1 || modes[0] !== "code") {
|
||
throw new Error("Gateway interview tasks require --mode code.");
|
||
}
|
||
}
|
||
if (
|
||
tasks.some(isGatewayTask) &&
|
||
models.some(
|
||
(model) => !QA_FRONTIER_PROVIDER_IDS.some((provider) => model.startsWith(`${provider}/`)),
|
||
)
|
||
) {
|
||
throw new Error("Gateway tasks require explicit OpenAI, Anthropic, or Google models.");
|
||
}
|
||
if (
|
||
tasks.some(isMatrixPerformanceTask) &&
|
||
modes.length > 0 &&
|
||
(modes.length !== 2 || !modes.includes("direct") || !modes.includes("code"))
|
||
) {
|
||
throw new Error("Performance tasks require both explicit direct/code treatment arms.");
|
||
}
|
||
if (
|
||
options.baselineResults &&
|
||
(tasks.length === 0 || tasks.some((task) => !isGatewayTask(task)))
|
||
) {
|
||
throw new Error(
|
||
"--baseline-results requires Gateway interview tasks with fixed workload fingerprints.",
|
||
);
|
||
}
|
||
options.modes =
|
||
modes.length > 0
|
||
? modes
|
||
: tasks.some(isMatrixPerformanceTask)
|
||
? ["direct", "code"]
|
||
: ["direct", "auto", "code"];
|
||
options.tasks = tasks.length > 0 ? tasks : ["read", "dependent-read-write"];
|
||
return options;
|
||
}
|
||
|
||
function slug(value: string): string {
|
||
return value
|
||
.toLowerCase()
|
||
.replace(/[^a-z0-9]+/gu, "-")
|
||
.replace(/^-+|-+$/gu, "")
|
||
.slice(0, 80);
|
||
}
|
||
|
||
function defaultOutputDir(now: Date): string {
|
||
return path.join(
|
||
".artifacts",
|
||
"qa-e2e",
|
||
"code-mode-model-matrix",
|
||
now.toISOString().replaceAll(":", "-"),
|
||
);
|
||
}
|
||
|
||
export function resolveCodeModeMatrixOutputDir(
|
||
repoRoot: string,
|
||
configured: string | undefined,
|
||
now = new Date(),
|
||
): string {
|
||
const raw = configured?.trim() || defaultOutputDir(now);
|
||
if (path.isAbsolute(raw)) {
|
||
throw new Error("--output-dir must be repo-relative");
|
||
}
|
||
const resolvedRoot = path.resolve(repoRoot);
|
||
const resolved = path.resolve(resolvedRoot, raw);
|
||
if (resolved === resolvedRoot || !resolved.startsWith(`${resolvedRoot}${path.sep}`)) {
|
||
throw new Error("--output-dir must stay within the repository");
|
||
}
|
||
return resolved;
|
||
}
|
||
|
||
function pathsOverlap(left: string, right: string, caseInsensitive: boolean): boolean {
|
||
const normalize = (value: string) => {
|
||
const resolved = path.resolve(value);
|
||
return caseInsensitive ? resolved.toLowerCase() : resolved;
|
||
};
|
||
const resolvedLeft = normalize(left);
|
||
const resolvedRight = normalize(right);
|
||
return (
|
||
resolvedLeft === resolvedRight ||
|
||
resolvedLeft.startsWith(`${resolvedRight}${path.sep}`) ||
|
||
resolvedRight.startsWith(`${resolvedLeft}${path.sep}`)
|
||
);
|
||
}
|
||
|
||
async function filesystemUsesCaseInsensitivePaths(repoRoot: string): Promise<boolean> {
|
||
const canonicalRoot = await fs.realpath(repoRoot);
|
||
const rootName = path.basename(canonicalRoot);
|
||
const letterIndex = rootName.search(/[a-z]/iu);
|
||
if (letterIndex < 0) {
|
||
return process.platform === "win32";
|
||
}
|
||
const letter = rootName[letterIndex] ?? "";
|
||
const alternateLetter =
|
||
letter === letter.toLowerCase() ? letter.toUpperCase() : letter.toLowerCase();
|
||
const alternateRoot = path.join(
|
||
path.dirname(canonicalRoot),
|
||
`${rootName.slice(0, letterIndex)}${alternateLetter}${rootName.slice(letterIndex + 1)}`,
|
||
);
|
||
return await fs.realpath(alternateRoot).then(
|
||
(resolved) => resolved === canonicalRoot,
|
||
() => false,
|
||
);
|
||
}
|
||
|
||
async function canonicalizeExistingPathPrefix(value: string): Promise<string> {
|
||
let current = path.resolve(value);
|
||
const missingSegments: string[] = [];
|
||
for (;;) {
|
||
try {
|
||
return path.join(await fs.realpath(current), ...missingSegments.toReversed());
|
||
} catch (error) {
|
||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||
throw error;
|
||
}
|
||
const parent = path.dirname(current);
|
||
if (parent === current) {
|
||
throw error;
|
||
}
|
||
missingSegments.push(path.basename(current));
|
||
current = parent;
|
||
}
|
||
}
|
||
}
|
||
|
||
async function runtimeArtifactDirectories(repoRoot: string, outputDir: string): Promise<string[]> {
|
||
const packagesRoot = path.join(repoRoot, "packages");
|
||
const packageEntries = await fs
|
||
.readdir(packagesRoot, { withFileTypes: true })
|
||
.catch((error: unknown) => {
|
||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||
return [];
|
||
}
|
||
throw error;
|
||
});
|
||
const artifacts = [
|
||
path.join(repoRoot, "dist"),
|
||
...packageEntries
|
||
.filter((entry) => entry.isDirectory())
|
||
.map((entry) => path.join(packagesRoot, entry.name, "dist")),
|
||
];
|
||
const outputSegments = path
|
||
.relative(path.resolve(repoRoot), path.resolve(outputDir))
|
||
.split(path.sep);
|
||
const outputPackage = outputSegments[0]?.toLowerCase() === "packages" && outputSegments[1];
|
||
if (outputPackage) {
|
||
artifacts.push(path.join(packagesRoot, outputPackage, "dist"));
|
||
}
|
||
return artifacts;
|
||
}
|
||
|
||
async function assertOutputOutsideRuntimeArtifacts(
|
||
repoRoot: string,
|
||
outputDir: string,
|
||
): Promise<void> {
|
||
const caseInsensitive = await filesystemUsesCaseInsensitivePaths(repoRoot);
|
||
const canonicalOutput = await canonicalizeExistingPathPrefix(outputDir);
|
||
for (const artifactDir of await runtimeArtifactDirectories(repoRoot, outputDir)) {
|
||
const canonicalArtifact = await canonicalizeExistingPathPrefix(artifactDir);
|
||
if (pathsOverlap(canonicalOutput, canonicalArtifact, caseInsensitive)) {
|
||
throw new Error(
|
||
`--output-dir must not overlap runtime artifacts: ${path.relative(repoRoot, artifactDir)}`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
async function assertOutputOutsideGitMetadata(repoRoot: string, outputDir: string): Promise<void> {
|
||
const gitDirectories = [path.join(repoRoot, ".git")];
|
||
const discovered = await execFileAsync(
|
||
"git",
|
||
["rev-parse", "--path-format=absolute", "--git-dir", "--git-common-dir"],
|
||
{
|
||
cwd: repoRoot,
|
||
encoding: "utf8",
|
||
},
|
||
).catch(() => null);
|
||
if (discovered) {
|
||
gitDirectories.push(
|
||
...discovered.stdout
|
||
.split(/\r?\n/u)
|
||
.map((value) => value.trim())
|
||
.filter(Boolean),
|
||
);
|
||
}
|
||
|
||
const caseInsensitive = await filesystemUsesCaseInsensitivePaths(repoRoot);
|
||
const canonicalOutput = await canonicalizeExistingPathPrefix(outputDir);
|
||
for (const gitDirectory of new Set(gitDirectories)) {
|
||
const canonicalGitDirectory = await canonicalizeExistingPathPrefix(gitDirectory);
|
||
if (pathsOverlap(canonicalOutput, canonicalGitDirectory, caseInsensitive)) {
|
||
throw new Error("--output-dir must not overlap Git metadata");
|
||
}
|
||
}
|
||
}
|
||
|
||
export async function reserveCodeModeMatrixOutputDir(
|
||
repoRoot: string,
|
||
outputDir: string,
|
||
): Promise<void> {
|
||
const resolvedRoot = path.resolve(repoRoot);
|
||
const resolvedOutput = path.resolve(outputDir);
|
||
const relative = path.relative(resolvedRoot, resolvedOutput);
|
||
if (
|
||
!relative ||
|
||
relative === ".." ||
|
||
relative.startsWith(`..${path.sep}`) ||
|
||
path.isAbsolute(relative)
|
||
) {
|
||
throw new Error("--output-dir must stay within the repository");
|
||
}
|
||
let current = resolvedRoot;
|
||
for (const segment of relative.split(path.sep)) {
|
||
current = path.join(current, segment);
|
||
for (;;) {
|
||
try {
|
||
const stats = await fs.lstat(current);
|
||
if (stats.isSymbolicLink()) {
|
||
throw new Error(`--output-dir must not traverse symlinks: ${relative}`);
|
||
}
|
||
if (current === resolvedOutput) {
|
||
throw new Error(`--output-dir must not already exist: ${relative}`);
|
||
}
|
||
if (!stats.isDirectory()) {
|
||
throw new Error(
|
||
`--output-dir parent must be a directory: ${path.relative(repoRoot, current)}`,
|
||
);
|
||
}
|
||
break;
|
||
} catch (error) {
|
||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||
throw error;
|
||
}
|
||
try {
|
||
await fs.mkdir(current);
|
||
break;
|
||
} catch (mkdirError) {
|
||
if ((mkdirError as NodeJS.ErrnoException).code !== "EEXIST") {
|
||
throw mkdirError;
|
||
}
|
||
// A concurrent creator won the race. Inspect its path before proceeding.
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
type MatrixScheduleEntry = {
|
||
model: string;
|
||
task: CodeModeMatrixTask;
|
||
repetition: number;
|
||
firstMode: "direct" | "code";
|
||
};
|
||
|
||
async function buildCellWaves(options: CodeModeMatrixOptions): Promise<MatrixCell[][]> {
|
||
let schedule: MatrixScheduleEntry[];
|
||
if (options.schedulePath) {
|
||
const value: unknown = JSON.parse(await fs.readFile(options.schedulePath, "utf8"));
|
||
if (!Array.isArray(value) || value.length === 0) {
|
||
throw new Error("--schedule must contain a nonempty JSON array of paired waves.");
|
||
}
|
||
if (
|
||
options.modes.length !== 2 ||
|
||
!options.modes.includes("direct") ||
|
||
!options.modes.includes("code")
|
||
) {
|
||
throw new Error("--schedule requires exactly --mode direct and --mode code.");
|
||
}
|
||
schedule = value.map((row: unknown): MatrixScheduleEntry => {
|
||
if (
|
||
!row ||
|
||
typeof row !== "object" ||
|
||
!("model" in row) ||
|
||
!("task" in row) ||
|
||
!("repetition" in row) ||
|
||
typeof row.model !== "string" ||
|
||
!options.models.includes(row.model) ||
|
||
typeof row.task !== "string" ||
|
||
!options.tasks.some((task) => task === row.task) ||
|
||
typeof row.repetition !== "number" ||
|
||
!Number.isInteger(row.repetition) ||
|
||
row.repetition < 1 ||
|
||
row.repetition > options.repetitions ||
|
||
!("firstMode" in row) ||
|
||
(row.firstMode !== "direct" && row.firstMode !== "code")
|
||
) {
|
||
throw new Error(
|
||
"Each schedule wave must select an admitted model, task, repetition, and direct/code firstMode.",
|
||
);
|
||
}
|
||
return {
|
||
model: row.model,
|
||
task: parseTask(row.task),
|
||
repetition: row.repetition,
|
||
firstMode: row.firstMode,
|
||
};
|
||
});
|
||
} else {
|
||
schedule = options.models.flatMap((model, modelIndex) =>
|
||
options.tasks.flatMap((task, taskIndex) =>
|
||
Array.from({ length: options.repetitions }, (_, index) => ({
|
||
model,
|
||
task,
|
||
repetition: index + 1,
|
||
firstMode:
|
||
(modelIndex + taskIndex + index) % 2 === 0 ? ("direct" as const) : ("code" as const),
|
||
})),
|
||
),
|
||
);
|
||
}
|
||
const seen = new Set<string>();
|
||
return schedule.map(({ model, task, repetition, firstMode }) => {
|
||
const key = `${model}\0${task}\0${repetition}`;
|
||
if (seen.has(key)) {
|
||
throw new Error(`Duplicate scheduled wave: ${model} ${task} ${repetition}`);
|
||
}
|
||
seen.add(key);
|
||
const modes: CodeModeMatrixMode[] =
|
||
options.modes.includes("direct") && options.modes.includes("code")
|
||
? [
|
||
firstMode,
|
||
firstMode === "direct" ? ("code" as const) : ("direct" as const),
|
||
...options.modes.filter((mode) => mode === "auto"),
|
||
]
|
||
: options.modes;
|
||
return modes.map((mode) => ({
|
||
id: `${modelCellPrefix(model)}-${mode}-${task}-${repetition}`,
|
||
mode,
|
||
model,
|
||
repetition,
|
||
task,
|
||
}));
|
||
});
|
||
}
|
||
|
||
export function modelCellPrefix(model: string): string {
|
||
const modelHash = createHash("sha256").update(model).digest("hex").slice(0, 10);
|
||
return `${slug(model)}-${modelHash}`;
|
||
}
|
||
|
||
function verificationCode(cell: MatrixCell): string {
|
||
return `CM-${createHash("sha256").update(cell.id).digest("hex").slice(0, 12).toUpperCase()}`;
|
||
}
|
||
|
||
function taskFixture(cell: MatrixCell): MatrixTaskFixture & { files: Record<string, string> } {
|
||
// Every model/mode gets identical inputs for a task repetition.
|
||
const code = verificationCode({ ...cell, id: `${cell.task}-${cell.repetition}` });
|
||
const facts = { "facts.txt": `project=openclaw\nverification_code=${code}\n` };
|
||
if (cell.task === "read") {
|
||
return {
|
||
expected: code,
|
||
files: facts,
|
||
prompt:
|
||
"Read facts.txt using tools. Reply with only the verification_code value, with no prose or formatting.",
|
||
};
|
||
}
|
||
if (cell.task === "dependent-read-write") {
|
||
return {
|
||
expected: code,
|
||
files: facts,
|
||
prompt:
|
||
"Read facts.txt using tools. Write only its verification_code value to result.txt, then read result.txt and reply with only that value. Do not guess or skip verification.",
|
||
resultPath: "result.txt",
|
||
};
|
||
}
|
||
const finish =
|
||
" Write only the answer to result.txt, read it back, and reply with only that answer, with no prose or formatting. Use file tools, not shell commands.";
|
||
if (cell.task === "large-result-reduction") {
|
||
const rows = Array.from({ length: 512 }, (_, index) => ({
|
||
id: index + 1,
|
||
region: index % 3 === 0 ? "west" : "east",
|
||
units: (index % 7) + 1,
|
||
unitPriceCents: 100 + (index % 23),
|
||
note: "irrelevant-detail-".repeat(4),
|
||
}));
|
||
const selected = rows.filter((row) => row.region === "east" && row.units >= 4);
|
||
return {
|
||
expected: `${code}:${selected.length}:${selected.reduce((total, row) => total + row.units * row.unitPriceCents, 0)}`,
|
||
files: {
|
||
"rules.json": JSON.stringify({ region: "east", minUnits: 4, verificationCode: code }),
|
||
"orders.jsonl": rows.map((row) => JSON.stringify(row)).join("\n") + "\n",
|
||
},
|
||
prompt:
|
||
"Read rules.json and all of orders.jsonl using tools, following read continuations when truncated. Select orders matching rules.region with units >= rules.minUnits. Compute their count and sum of units * unitPriceCents. The answer is verificationCode:count:sum, using rules.verificationCode and integer decimal numbers. Do not emit the raw orders." +
|
||
finish,
|
||
resultPath: "result.txt",
|
||
};
|
||
}
|
||
if (cell.task === "parallel-independent-reads") {
|
||
const names = ["north", "south", "west"];
|
||
const values = names.map((name, index) => `${code}-${index + 1}-${name}`);
|
||
return {
|
||
expected: values.join("|"),
|
||
files: Object.fromEntries(
|
||
names.map((name, index) => [`${name}.json`, JSON.stringify({ value: values[index] })]),
|
||
),
|
||
prompt:
|
||
"Read north.json, south.json, and west.json using independent tool calls. Run the reads in parallel when the tool surface supports it; in Code Mode use Promise.all. Join their value fields in north, south, west order with | (not completion order)." +
|
||
finish,
|
||
resultPath: "result.txt",
|
||
};
|
||
}
|
||
const route = `route-${code.slice(3, 9)}.json`;
|
||
const payload = `payload-${code.slice(9)}.json`;
|
||
return {
|
||
expected: code,
|
||
files: {
|
||
"start.json": JSON.stringify({ next: route }),
|
||
[route]: JSON.stringify({ next: payload }),
|
||
[payload]: JSON.stringify({ value: code }),
|
||
},
|
||
prompt:
|
||
"Read start.json using tools. Its next field names the next file; read that file and follow its next field to the payload file. The answer is the payload's value. Await each read before choosing the next path; do not list the directory or guess paths." +
|
||
finish,
|
||
resultPath: "result.txt",
|
||
};
|
||
}
|
||
|
||
async function prepareTaskFixture(workspace: string, cell: MatrixCell): Promise<MatrixTaskFixture> {
|
||
const { files, ...fixture } = taskFixture(cell);
|
||
await fs.mkdir(workspace, { recursive: true });
|
||
for (const [name, content] of Object.entries(files)) {
|
||
await fs.writeFile(path.join(workspace, name), content, "utf8");
|
||
}
|
||
if (fixture.resultPath) {
|
||
fixture.resultPath = path.join(workspace, fixture.resultPath);
|
||
await fs.rm(fixture.resultPath, { force: true });
|
||
}
|
||
return fixture;
|
||
}
|
||
|
||
async function hashDirectory(root: string): Promise<string> {
|
||
const hash = createHash("sha256");
|
||
const visit = async (directory: string): Promise<void> => {
|
||
const entries = (await fs.readdir(directory, { withFileTypes: true })).toSorted((a, b) =>
|
||
a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
|
||
);
|
||
for (const entry of entries) {
|
||
const filePath = path.join(directory, entry.name);
|
||
const relativePath = path.relative(root, filePath);
|
||
hash.update(`\0${relativePath}\0`);
|
||
if (entry.isDirectory()) {
|
||
await visit(filePath);
|
||
} else if (entry.isSymbolicLink()) {
|
||
hash.update(await fs.readlink(filePath));
|
||
} else if (entry.isFile()) {
|
||
hash.update(await fs.readFile(filePath));
|
||
}
|
||
}
|
||
};
|
||
await visit(root);
|
||
return hash.digest("hex");
|
||
}
|
||
|
||
async function hashRuntimeArtifacts(repoRoot: string): Promise<string> {
|
||
const artifacts = [{ label: "dist", root: path.join(repoRoot, "dist") }];
|
||
const packagesRoot = path.join(repoRoot, "packages");
|
||
const packageEntries = await fs.readdir(packagesRoot, { withFileTypes: true });
|
||
for (const entry of packageEntries.toSorted((a, b) =>
|
||
a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
|
||
)) {
|
||
if (!entry.isDirectory()) {
|
||
continue;
|
||
}
|
||
const packageDist = path.join(packagesRoot, entry.name, "dist");
|
||
const stat = await fs.stat(packageDist).catch(() => null);
|
||
if (stat?.isDirectory()) {
|
||
artifacts.push({ label: `packages/${entry.name}/dist`, root: packageDist });
|
||
}
|
||
}
|
||
|
||
const hash = createHash("sha256");
|
||
for (const artifact of artifacts) {
|
||
hash.update(`\0${artifact.label}\0${await hashDirectory(artifact.root)}`);
|
||
}
|
||
return hash.digest("hex");
|
||
}
|
||
|
||
async function buildMatrixCliArtifacts(repoRoot: string): Promise<void> {
|
||
for (const args of [
|
||
["--import", "tsx", "scripts/bundled-plugin-assets.mts", "--phase", "build"],
|
||
["--import", "tsx", "scripts/tsdown-build.mts", "--no-clean"],
|
||
["scripts/runtime-postbuild.mjs"],
|
||
]) {
|
||
const { stderr, stdout } = await execFileAsync(process.execPath, args, {
|
||
cwd: repoRoot,
|
||
encoding: "utf8",
|
||
env: {
|
||
...process.env,
|
||
OPENCLAW_BUILD_ALL_NO_PNPM: "1",
|
||
OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1",
|
||
},
|
||
maxBuffer: 8 * 1024 * 1024,
|
||
timeout: 10 * 60 * 1_000,
|
||
});
|
||
const output = `${stdout}\n${stderr}`.trim();
|
||
if (output) {
|
||
console.log(output);
|
||
}
|
||
}
|
||
}
|
||
|
||
function expectedEngagement(mode: CodeModeMatrixMode, engaged: boolean | undefined): boolean {
|
||
if (mode === "auto") {
|
||
return typeof engaged === "boolean";
|
||
}
|
||
return engaged === (mode === "code");
|
||
}
|
||
|
||
export function classifyCodeModeMatrixCell(params: {
|
||
diagnostics: string;
|
||
effectPassed: boolean;
|
||
envelope: Readonly<AgentExecEnvelope>;
|
||
expected: string;
|
||
mode: CodeModeMatrixMode;
|
||
model: string;
|
||
stdoutContractValid?: boolean;
|
||
task: CodeModeMatrixTask;
|
||
}): {
|
||
failureCategory: CellFailureCategory | null;
|
||
oracle: CodeModeMatrixCellResult["oracle"];
|
||
passed: boolean;
|
||
} {
|
||
const engagement = expectedEngagement(params.mode, params.envelope.codeModeEngaged);
|
||
const answer = params.envelope.final.trim() === params.expected;
|
||
const effect = params.effectPassed;
|
||
const separator = params.model.indexOf("/");
|
||
const requestedProvider = params.model.slice(0, separator);
|
||
const requestedModel = params.model.slice(separator + 1);
|
||
const identity =
|
||
params.envelope.provider === requestedProvider && params.envelope.model === requestedModel;
|
||
const outerToolExecution = (params.envelope.toolSummary?.calls ?? 0) > 0;
|
||
// Direct and auto evaluate the model-visible outer tool surface. Forced Code
|
||
// Mode additionally proves that the exec cell reached a nested catalog tool.
|
||
const toolExecution =
|
||
outerToolExecution && (params.mode !== "code" || (params.envelope.bridgeCalls?.call ?? 0) > 0);
|
||
const oracle = { answer, effect, engagement, identity, toolExecution };
|
||
let failureCategory: CellFailureCategory | null = null;
|
||
if (params.stdoutContractValid === false) {
|
||
failureCategory = "harness_error";
|
||
} else if (params.envelope.status === "timeout") {
|
||
failureCategory = "timeout";
|
||
} else if (!params.envelope.ok) {
|
||
failureCategory =
|
||
classifyCodeModeMatrixProviderFailure(
|
||
`${params.envelope.error?.message ?? ""}\n${params.diagnostics}`,
|
||
) ?? "agent_error";
|
||
} else if (!identity) {
|
||
failureCategory = "model_mismatch";
|
||
} else if (!engagement) {
|
||
failureCategory = "activation";
|
||
} else if (!toolExecution) {
|
||
failureCategory = "tool_execution";
|
||
} else if (!effect) {
|
||
failureCategory = "effect_mismatch";
|
||
} else if (!answer) {
|
||
failureCategory = "answer_mismatch";
|
||
}
|
||
return { failureCategory, oracle, passed: failureCategory === null };
|
||
}
|
||
|
||
function parseAgentExecOutput(stdout: string): {
|
||
envelope: AgentExecEnvelope;
|
||
trailing: string;
|
||
} {
|
||
const value = stdout.trimStart();
|
||
if (!value) {
|
||
throw new Error("agent exec produced no JSON envelope");
|
||
}
|
||
if (value[0] !== "{") {
|
||
throw new Error("agent exec stdout did not begin with a JSON envelope");
|
||
}
|
||
let depth = 0;
|
||
let escaped = false;
|
||
let inString = false;
|
||
for (let index = 0; index < value.length; index += 1) {
|
||
const character = value[index];
|
||
if (inString) {
|
||
if (escaped) {
|
||
escaped = false;
|
||
} else if (character === "\\") {
|
||
escaped = true;
|
||
} else if (character === '"') {
|
||
inString = false;
|
||
}
|
||
continue;
|
||
}
|
||
if (character === '"') {
|
||
inString = true;
|
||
} else if (character === "{") {
|
||
depth += 1;
|
||
} else if (character === "}") {
|
||
depth -= 1;
|
||
if (depth === 0) {
|
||
const envelope = JSON.parse(value.slice(0, index + 1)) as AgentExecEnvelope;
|
||
return { envelope, trailing: value.slice(index + 1).trim() };
|
||
}
|
||
}
|
||
}
|
||
throw new Error("agent exec produced an incomplete JSON envelope");
|
||
}
|
||
|
||
async function pathIsDirectory(value: string): Promise<boolean> {
|
||
return await fs
|
||
.stat(value)
|
||
.then((stats) => stats.isDirectory())
|
||
.catch(() => false);
|
||
}
|
||
|
||
async function cloneTreeWithHardlinks(source: string, destination: string): Promise<void> {
|
||
await fs.mkdir(destination, { recursive: true });
|
||
for (const entry of await fs.readdir(source, { withFileTypes: true })) {
|
||
const sourcePath = path.join(source, entry.name);
|
||
const destinationPath = path.join(destination, entry.name);
|
||
if (entry.isDirectory()) {
|
||
await cloneTreeWithHardlinks(sourcePath, destinationPath);
|
||
} else if (entry.isSymbolicLink()) {
|
||
await fs.symlink(await fs.readlink(sourcePath), destinationPath);
|
||
} else {
|
||
try {
|
||
await fs.link(sourcePath, destinationPath);
|
||
} catch (error) {
|
||
if ((error as NodeJS.ErrnoException).code !== "EXDEV") {
|
||
throw error;
|
||
}
|
||
await fs.copyFile(sourcePath, destinationPath);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
async function prepareRuntimeEntrypoint(
|
||
repoRoot: string,
|
||
runtimeRoot: string,
|
||
): Promise<MatrixRuntimeEntrypoint> {
|
||
const entrypoint = path.join(repoRoot, "dist", "entry.js");
|
||
try {
|
||
await fs.access(entrypoint);
|
||
} catch (error) {
|
||
throw new Error("dist/entry.js is missing; run pnpm build before the matrix", { cause: error });
|
||
}
|
||
|
||
const nodeModules = path.join(repoRoot, "node_modules");
|
||
const physicalNodeModules = await fs.realpath(nodeModules);
|
||
if (physicalNodeModules === path.resolve(nodeModules)) {
|
||
return { args: [entrypoint], cwd: repoRoot };
|
||
}
|
||
|
||
const overlayDist = path.join(runtimeRoot, "dist");
|
||
const overlayNodeModules = path.join(runtimeRoot, "node_modules");
|
||
await cloneTreeWithHardlinks(path.join(repoRoot, "dist"), overlayDist);
|
||
await fs.mkdir(overlayNodeModules);
|
||
await fs.copyFile(path.join(repoRoot, "package.json"), path.join(runtimeRoot, "package.json"));
|
||
|
||
const entries = await fs.readdir(physicalNodeModules, { withFileTypes: true });
|
||
for (const entry of entries) {
|
||
if (entry.name.startsWith(".") || entry.name === "@openclaw") {
|
||
continue;
|
||
}
|
||
await fs.symlink(
|
||
path.join(physicalNodeModules, entry.name),
|
||
path.join(overlayNodeModules, entry.name),
|
||
process.platform === "win32" ? "junction" : "dir",
|
||
);
|
||
}
|
||
|
||
const overlayOpenClaw = path.join(overlayNodeModules, "@openclaw");
|
||
const physicalOpenClaw = path.join(physicalNodeModules, "@openclaw");
|
||
await fs.mkdir(overlayOpenClaw);
|
||
for (const entry of await fs.readdir(physicalOpenClaw, { withFileTypes: true })) {
|
||
const worktreePackage = path.join(repoRoot, "packages", entry.name);
|
||
const target = (await pathIsDirectory(path.join(worktreePackage, "dist")))
|
||
? worktreePackage
|
||
: path.join(physicalOpenClaw, entry.name);
|
||
await fs.symlink(
|
||
target,
|
||
path.join(overlayOpenClaw, entry.name),
|
||
process.platform === "win32" ? "junction" : "dir",
|
||
);
|
||
}
|
||
|
||
return {
|
||
args: [path.join(runtimeRoot, "dist", "entry.js")],
|
||
cwd: runtimeRoot,
|
||
};
|
||
}
|
||
|
||
export function buildCodeModeMatrixAgentEnv(
|
||
model: string,
|
||
runtimeCwd: string,
|
||
baseEnv: NodeJS.ProcessEnv = process.env,
|
||
): NodeJS.ProcessEnv {
|
||
const provider = model.slice(0, model.indexOf("/"));
|
||
const config: OpenClawConfig = {
|
||
plugins: { allow: [provider], entries: { [provider]: { enabled: true } } },
|
||
};
|
||
return {
|
||
PATH: baseEnv.PATH,
|
||
SystemRoot: baseEnv.SystemRoot,
|
||
...matrixProviderEnv(model, config, baseEnv),
|
||
NODE_DISABLE_COMPILE_CACHE: "1",
|
||
OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(runtimeCwd, "dist", "extensions"),
|
||
};
|
||
}
|
||
|
||
async function executeAgentExec(params: {
|
||
fixture: MatrixTaskFixture;
|
||
matrix: RunCellParams;
|
||
stateDir: string;
|
||
workspace: string;
|
||
}): Promise<{
|
||
diagnostics: string;
|
||
envelope: AgentExecEnvelope;
|
||
stdoutContractValid: boolean;
|
||
}> {
|
||
const runtime = params.matrix.runtime;
|
||
if (!runtime) {
|
||
throw new Error("matrix runtime entrypoint was not prepared");
|
||
}
|
||
const provider = params.matrix.cell.model.split("/")[0]!;
|
||
const home = path.join(params.stateDir, "home");
|
||
const tmp = path.join(params.stateDir, "tmp");
|
||
await Promise.all([fs.mkdir(home, { recursive: true }), fs.mkdir(tmp, { recursive: true })]);
|
||
const config = mergeDeep(
|
||
{
|
||
agents: { entries: { main: {} } },
|
||
plugins: { allow: [provider], entries: { [provider]: { enabled: true } } },
|
||
tools: { codeMode: { executor: "node" } },
|
||
},
|
||
{ agents: matrixModelConfig(params.matrix.cell.model, params.matrix.thinking) },
|
||
);
|
||
const configPath = path.join(params.stateDir, "matrix-config.json");
|
||
await fs.writeFile(configPath, JSON.stringify(config), { mode: 0o600 });
|
||
const args = [
|
||
...runtime.args,
|
||
"agent",
|
||
"exec",
|
||
params.fixture.prompt,
|
||
"--cwd",
|
||
params.workspace,
|
||
"--state-dir",
|
||
params.stateDir,
|
||
"--model",
|
||
params.matrix.cell.model,
|
||
"--code-mode",
|
||
params.matrix.cell.mode,
|
||
"--config",
|
||
configPath,
|
||
"--thinking",
|
||
params.matrix.thinking,
|
||
"--timeout",
|
||
String(params.matrix.timeoutSeconds),
|
||
"--json",
|
||
];
|
||
let output: { stdout: string; stderr: string };
|
||
try {
|
||
const env: NodeJS.ProcessEnv = {
|
||
...buildCodeModeMatrixAgentEnv(params.matrix.cell.model, runtime.cwd),
|
||
HOME: home,
|
||
USERPROFILE: home,
|
||
OPENCLAW_HOME: home,
|
||
OPENCLAW_STATE_DIR: params.stateDir,
|
||
TMPDIR: tmp,
|
||
TEMP: tmp,
|
||
TMP: tmp,
|
||
};
|
||
output = await execFileAsync(process.execPath, args, {
|
||
cwd: runtime.cwd,
|
||
encoding: "utf8",
|
||
env,
|
||
maxBuffer: 4 * 1024 * 1024,
|
||
timeout: (params.matrix.timeoutSeconds + 30) * 1_000,
|
||
signal: params.matrix.abortSignal,
|
||
});
|
||
} catch (error) {
|
||
const commandError = error as Error & {
|
||
code?: string;
|
||
killed?: boolean;
|
||
stderr?: string;
|
||
stdout?: string;
|
||
};
|
||
if (commandError.killed || commandError.code === "ETIMEDOUT") {
|
||
return {
|
||
diagnostics: previewForDevToolLog(commandError.stderr ?? commandError.message, 2_000),
|
||
envelope: {
|
||
ok: false,
|
||
status: "timeout",
|
||
final: "",
|
||
payloads: [],
|
||
model: null,
|
||
provider: null,
|
||
sessionId: "",
|
||
error: { kind: "timeout", message: "agent exec process deadline elapsed" },
|
||
},
|
||
stdoutContractValid: true,
|
||
};
|
||
}
|
||
if (commandError.stdout?.trim()) {
|
||
output = { stdout: commandError.stdout, stderr: commandError.stderr ?? "" };
|
||
} else {
|
||
throw error;
|
||
}
|
||
}
|
||
const parsed = parseAgentExecOutput(output.stdout);
|
||
return {
|
||
diagnostics:
|
||
`${output.stderr}\n${parsed.trailing ? `unexpected stdout after JSON: ${parsed.trailing}` : ""}`
|
||
.trim()
|
||
.slice(-MAX_DIAGNOSTIC_CHARS),
|
||
envelope: parsed.envelope,
|
||
stdoutContractValid: parsed.trailing.length === 0,
|
||
};
|
||
}
|
||
|
||
async function runMatrixCell(params: RunCellParams): Promise<CodeModeMatrixCellResult> {
|
||
if (isGatewayTask(params.cell.task)) {
|
||
const { runGatewayMatrixCell } = await import("./lib/code-mode-matrix-gateway.ts");
|
||
return await runGatewayMatrixCell({
|
||
...params,
|
||
cell: { ...params.cell, task: params.cell.task },
|
||
});
|
||
}
|
||
const retainedRoot = path.join(params.outputDir, "state", params.cell.id);
|
||
if (params.keepState) {
|
||
await fs.rm(retainedRoot, { force: true, recursive: true });
|
||
}
|
||
const root = params.keepState
|
||
? retainedRoot
|
||
: await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-code-mode-matrix-"));
|
||
const stateDir = path.join(root, "state");
|
||
const workspace = path.join(root, "workspace");
|
||
await fs.mkdir(stateDir, { recursive: true });
|
||
const fixture = await prepareTaskFixture(workspace, params.cell);
|
||
const startedAt = Date.now();
|
||
try {
|
||
const command = await executeAgentExec({
|
||
fixture,
|
||
matrix: params,
|
||
stateDir,
|
||
workspace,
|
||
});
|
||
const effectPassed = fixture.resultPath
|
||
? (await fs.readFile(fixture.resultPath, "utf8").catch(() => "")).trim() === fixture.expected
|
||
: true;
|
||
const diagnosticText = command.diagnostics;
|
||
const classification = classifyCodeModeMatrixCell({
|
||
diagnostics: diagnosticText,
|
||
effectPassed,
|
||
envelope: command.envelope,
|
||
expected: fixture.expected,
|
||
mode: params.cell.mode,
|
||
model: params.cell.model,
|
||
stdoutContractValid: command.stdoutContractValid,
|
||
task: params.cell.task,
|
||
});
|
||
return {
|
||
...(command.envelope.assistantTurns !== undefined
|
||
? { assistantTurns: command.envelope.assistantTurns }
|
||
: {}),
|
||
...(command.envelope.bridgeCalls ? { bridgeCalls: command.envelope.bridgeCalls } : {}),
|
||
buildSha256: params.buildSha256,
|
||
codeModeEngaged: command.envelope.codeModeEngaged ?? null,
|
||
...(command.envelope.costUsd !== undefined ? { costUsd: command.envelope.costUsd } : {}),
|
||
...(diagnosticText ? { diagnostics: diagnosticText } : {}),
|
||
elapsedMs: Date.now() - startedAt,
|
||
...(command.envelope.error ? { error: command.envelope.error } : {}),
|
||
expected: fixture.expected,
|
||
failureCategory: classification.failureCategory,
|
||
final: command.envelope.final,
|
||
gitSha: params.gitSha,
|
||
id: params.cell.id,
|
||
mode: params.cell.mode,
|
||
model: params.cell.model,
|
||
observedModel: command.envelope.model,
|
||
observedProvider: command.envelope.provider,
|
||
oracle: classification.oracle,
|
||
passed: classification.passed,
|
||
repetition: params.cell.repetition,
|
||
sourceDirty: params.sourceDirty,
|
||
sourcePatchSha256: params.sourcePatchSha256,
|
||
status: command.envelope.status,
|
||
task: params.cell.task,
|
||
timestamp: new Date().toISOString(),
|
||
...(command.envelope.toolSummary ? { toolSummary: command.envelope.toolSummary } : {}),
|
||
...(command.envelope.usage ? { usage: command.envelope.usage } : {}),
|
||
};
|
||
} finally {
|
||
if (!params.keepState) {
|
||
await fs.rm(root, { force: true, recursive: true });
|
||
}
|
||
}
|
||
}
|
||
|
||
function harnessFailureResult(
|
||
cell: MatrixCell,
|
||
provenance: Pick<RunCellParams, "buildSha256" | "gitSha" | "sourceDirty" | "sourcePatchSha256">,
|
||
elapsedMs: number,
|
||
error: unknown,
|
||
): CodeModeMatrixCellResult {
|
||
const message = previewForDevToolLog(
|
||
error instanceof Error ? error.message : String(error),
|
||
2_000,
|
||
);
|
||
return {
|
||
buildSha256: provenance.buildSha256,
|
||
codeModeEngaged: null,
|
||
diagnostics: message,
|
||
elapsedMs,
|
||
error: { kind: "harness_error", message },
|
||
expected: isGatewayTask(cell.task) ? "Gateway task result" : taskFixture(cell).expected,
|
||
failureCategory: classifyCodeModeMatrixProviderFailure(message) ?? "harness_error",
|
||
final: "",
|
||
gitSha: provenance.gitSha,
|
||
id: cell.id,
|
||
mode: cell.mode,
|
||
model: cell.model,
|
||
observedModel: null,
|
||
observedProvider: null,
|
||
oracle: {
|
||
answer: isMatrixPerformanceTask(cell.task) ? null : false,
|
||
effect: false,
|
||
engagement: false,
|
||
identity: false,
|
||
toolExecution: false,
|
||
},
|
||
passed: false,
|
||
repetition: cell.repetition,
|
||
sourceDirty: provenance.sourceDirty,
|
||
sourcePatchSha256: provenance.sourcePatchSha256,
|
||
status: "error",
|
||
task: cell.task,
|
||
timestamp: new Date().toISOString(),
|
||
};
|
||
}
|
||
|
||
function summarizeMetric(values: (number | undefined)[]) {
|
||
const samples = values
|
||
.filter((value): value is number => value !== undefined)
|
||
.toSorted((a, b) => a - b);
|
||
return {
|
||
samples: samples.length,
|
||
total: samples.length > 0 ? samples.reduce((total, value) => total + value, 0) : null,
|
||
p50: samples[Math.floor(samples.length / 2)] ?? null,
|
||
};
|
||
}
|
||
|
||
function summarizeResults(results: CodeModeMatrixCellResult[]) {
|
||
const groups = groupBy(results, (result) => `${result.model}\0${result.mode}\0${result.task}`);
|
||
return [...groups.entries()].map(([key, group]) => {
|
||
const [model, mode, task] = key.split("\0");
|
||
const passed = group.filter((result) => result.passed);
|
||
const failures: Record<string, number> = {};
|
||
for (const result of group) {
|
||
if (!result.passed) {
|
||
const category = result.failureCategory ?? "unknown";
|
||
failures[category] = (failures[category] ?? 0) + 1;
|
||
}
|
||
}
|
||
const summary = {
|
||
codeModeEngaged: group.filter((result) => result.codeModeEngaged === true).length,
|
||
failed: group.length - passed.length,
|
||
failures,
|
||
firstPassPassed: passed.some((result) => result.repetition === 1),
|
||
mode,
|
||
model,
|
||
eventualPassed: passed.length > 0,
|
||
p50WallMs: summarizeMetric(group.map((result) => result.elapsedMs)).p50 ?? 0,
|
||
metrics: {
|
||
assistantTurns: summarizeMetric(group.map((result) => result.assistantTurns)),
|
||
outerToolCalls: summarizeMetric(
|
||
group.map((result) => result.toolSummary?.calls ?? result.gateway?.outerCalls),
|
||
),
|
||
bridgeSearchCalls: summarizeMetric(group.map((result) => result.bridgeCalls?.search)),
|
||
bridgeDescribeCalls: summarizeMetric(group.map((result) => result.bridgeCalls?.describe)),
|
||
bridgeToolCalls: summarizeMetric(group.map((result) => result.bridgeCalls?.call)),
|
||
costUsd: summarizeMetric(group.map((result) => result.costUsd)),
|
||
gatewayUpstreamCalls: summarizeMetric(group.map((result) => result.gateway?.upstreamCalls)),
|
||
gatewayTaskElapsedMs: summarizeMetric(group.map((result) => result.gateway?.taskElapsedMs)),
|
||
inputTokens: summarizeMetric(group.map((result) => result.usage?.input)),
|
||
outputTokens: summarizeMetric(group.map((result) => result.usage?.output)),
|
||
},
|
||
passRate: passed.length / group.length,
|
||
passed: passed.length,
|
||
task,
|
||
total: group.length,
|
||
};
|
||
return group.some((result) => result.workload)
|
||
? Object.assign(summary, { gatewayOutcomes: summarizeGatewayMatrixOutcomes(group) })
|
||
: summary;
|
||
});
|
||
}
|
||
|
||
async function writeJson(filePath: string, value: unknown): Promise<void> {
|
||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||
await fs.writeFile(
|
||
filePath,
|
||
`${JSON.stringify(redactJsonValueForDevToolLog(value), null, 2)}\n`,
|
||
"utf8",
|
||
);
|
||
}
|
||
|
||
function evidenceStatus(result: CodeModeMatrixCellResult): QaEvidenceStatus {
|
||
if (result.passed) {
|
||
return "pass";
|
||
}
|
||
if (result.failureCategory === "provider_auth" || result.failureCategory === "provider_billing") {
|
||
return "blocked";
|
||
}
|
||
return "fail";
|
||
}
|
||
|
||
function observedModelRef(result: CodeModeMatrixCellResult): string {
|
||
if (result.observedProvider && result.observedModel) {
|
||
return `${result.observedProvider}/${result.observedModel}`;
|
||
}
|
||
return result.model;
|
||
}
|
||
|
||
function buildCodeModeMatrixEvidence(params: {
|
||
generatedAt: string;
|
||
repoRoot: string;
|
||
results: readonly CodeModeMatrixCellResult[];
|
||
}): QaEvidenceSummaryJson {
|
||
const evidenceOptions = {
|
||
artifactPaths: [
|
||
{ kind: "manifest", path: "manifest.json" },
|
||
{ kind: "summary", path: "summary.json" },
|
||
{ kind: "results", path: "results.jsonl" },
|
||
],
|
||
evidenceMode: "full" as const,
|
||
providerMode: "live-frontier" as const,
|
||
repoRoot: params.repoRoot,
|
||
runner: "code-mode-model-matrix",
|
||
};
|
||
const entries = params.results.flatMap((result) => {
|
||
const summary = buildScriptEvidenceSummary({
|
||
...evidenceOptions,
|
||
generatedAt: result.timestamp,
|
||
packageSource: { kind: "source-checkout", sha: result.gitSha },
|
||
primaryModel: observedModelRef(result),
|
||
targets: [
|
||
{
|
||
id: result.id,
|
||
title: `${result.model} ${result.mode} ${result.task} repetition ${result.repetition}`,
|
||
sourcePath: SOURCE_PATH,
|
||
},
|
||
],
|
||
results: [
|
||
{
|
||
id: result.id,
|
||
status: evidenceStatus(result),
|
||
durationMs: Math.max(1, result.elapsedMs),
|
||
failureMessage: result.failureCategory ?? undefined,
|
||
},
|
||
],
|
||
});
|
||
const entry = summary.entries[0];
|
||
if (!entry) {
|
||
return [];
|
||
}
|
||
if (entry.result.failure && result.failureCategory) {
|
||
entry.result.failure.class = result.failureCategory;
|
||
}
|
||
return [entry];
|
||
});
|
||
const base = buildScriptEvidenceSummary({
|
||
...evidenceOptions,
|
||
generatedAt: params.generatedAt,
|
||
packageSource: { kind: "source-checkout" },
|
||
primaryModel: "unknown/unknown",
|
||
targets: [],
|
||
results: [],
|
||
});
|
||
return validateQaEvidenceSummaryJson({
|
||
...base,
|
||
entries,
|
||
});
|
||
}
|
||
|
||
export async function runCodeModeModelMatrix(
|
||
options: CodeModeMatrixOptions,
|
||
deps: MatrixRunDependencies = {},
|
||
): Promise<{ exitCode: number; outputDir: string; summary: unknown }> {
|
||
const now = deps.now?.() ?? new Date();
|
||
const outputDir = resolveCodeModeMatrixOutputDir(options.repoRoot, options.outputDir, now);
|
||
const runtimeRepoRoot = options.runtimeDir ?? options.repoRoot;
|
||
const sourceIdentity = deps.readSourceIdentity
|
||
? await deps.readSourceIdentity(runtimeRepoRoot)
|
||
: deps.readGitSha
|
||
? {
|
||
gitSha: await deps.readGitSha(runtimeRepoRoot),
|
||
sourceDirty: false,
|
||
sourcePatchSha256: null,
|
||
}
|
||
: await readSourceIdentity(runtimeRepoRoot, STRICT_SOURCE_IDENTITY_OPTIONS);
|
||
if (options.runtimeDir && sourceIdentity.sourceDirty) {
|
||
throw new Error("--runtime-dir must identify a clean committed checkout.");
|
||
}
|
||
const waves = await buildCellWaves(options);
|
||
const cells = waves.flat();
|
||
const admission = {
|
||
concurrency: options.concurrency ?? DEFAULT_ADMISSION.concurrency,
|
||
maxCells: options.maxCells ?? DEFAULT_ADMISSION.maxCells,
|
||
maxTokens: options.maxTokens ?? DEFAULT_ADMISSION.maxTokens,
|
||
maxKnownCostUsd: options.maxKnownCostUsd ?? DEFAULT_ADMISSION.maxKnownCostUsd,
|
||
maxWallSeconds: options.maxWallSeconds ?? DEFAULT_ADMISSION.maxWallSeconds,
|
||
};
|
||
if (
|
||
!Number.isInteger(admission.concurrency) ||
|
||
admission.concurrency < 1 ||
|
||
admission.concurrency > 3
|
||
) {
|
||
throw new Error("concurrency must be an integer from 1 to 3");
|
||
}
|
||
await assertOutputOutsideGitMetadata(options.repoRoot, outputDir);
|
||
if (!options.dryRun && !options.runtimeDir) {
|
||
await (deps.buildCliArtifacts ?? buildMatrixCliArtifacts)(options.repoRoot);
|
||
}
|
||
if (!options.dryRun && options.runtimeDir) {
|
||
// Current source can be clean after reverting edits that produced the retained artifacts.
|
||
for (const stamp of [".buildstamp", ".runtime-postbuildstamp"]) {
|
||
const value: unknown = JSON.parse(
|
||
await fs.readFile(path.join(runtimeRepoRoot, "dist", stamp), "utf8"),
|
||
);
|
||
if (
|
||
!value ||
|
||
typeof value !== "object" ||
|
||
!("head" in value) ||
|
||
value.head !== sourceIdentity.gitSha ||
|
||
!("inputsClean" in value) ||
|
||
value.inputsClean !== true
|
||
) {
|
||
throw new Error(
|
||
`Frozen runtime ${stamp} must match its clean committed source and record clean build inputs. Choose a revision with provenance-capable stamp writers and run pnpm build; rebuilding older source without those writers cannot satisfy this check.`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
// Build first so its output set is complete, then reserve evidence storage
|
||
// before hashing. Dry runs also write evidence, so every run needs isolation.
|
||
await assertOutputOutsideRuntimeArtifacts(options.repoRoot, outputDir);
|
||
if (options.runtimeDir) {
|
||
await assertOutputOutsideRuntimeArtifacts(runtimeRepoRoot, outputDir);
|
||
}
|
||
await reserveCodeModeMatrixOutputDir(options.repoRoot, outputDir);
|
||
const buildSha256 = options.dryRun
|
||
? null
|
||
: await (deps.readBuildSha256 ?? hashRuntimeArtifacts)(runtimeRepoRoot);
|
||
const launch: QaEvidenceIdentity = {
|
||
source: {
|
||
ref: sourceIdentity.gitSha,
|
||
integrity: `git:${sourceIdentity.gitSha}${sourceIdentity.sourcePatchSha256 ? `+sha256:${sourceIdentity.sourcePatchSha256}` : ""}`,
|
||
},
|
||
runtime: captureQaEvidenceRuntimeIdentity(),
|
||
package: null,
|
||
protocol: null,
|
||
accountRef: null,
|
||
proofClass: null,
|
||
};
|
||
const invocation = createQaEvidenceInvocation({
|
||
scenarios: cells.map((cell) => ({ id: cell.id, execution: { kind: "script" } })),
|
||
channel: null,
|
||
launch,
|
||
});
|
||
const writeEvidence = async (generatedAt: string) =>
|
||
await writeJson(
|
||
path.join(outputDir, QA_EVIDENCE_FILENAME),
|
||
invocation.snapshot({ generatedAt, evidenceMode: "full" }),
|
||
);
|
||
const manifest = {
|
||
schemaVersion: MATRIX_SCHEMA_VERSION,
|
||
generatedAt: now.toISOString(),
|
||
source: SOURCE_PATH,
|
||
...(options.runtimeDir
|
||
? { harness: await readSourceIdentity(options.repoRoot, STRICT_SOURCE_IDENTITY_OPTIONS) }
|
||
: {}),
|
||
...(options.runtimeDir ? { runtimeDir: runtimeRepoRoot } : {}),
|
||
...sourceIdentity,
|
||
buildSha256,
|
||
models: options.models,
|
||
authentication: options.models.map((model) => ({
|
||
model,
|
||
...matrixProviderAuthSelection(model),
|
||
})),
|
||
modes: options.modes,
|
||
...(options.tasks.some(isGatewayTask)
|
||
? { gatewayExecutor: options.gatewayExecutor ?? "node" }
|
||
: {}),
|
||
tasks: options.tasks,
|
||
repetitions: options.repetitions,
|
||
timeoutSeconds: options.timeoutSeconds,
|
||
thinking: options.thinking,
|
||
keepState: options.keepState,
|
||
cells: cells.map((cell) => cell.id),
|
||
waves: waves.map((wave) => wave.map((cell) => cell.id)),
|
||
admission,
|
||
admissionContract:
|
||
"Whole waves are admitted. Token, known-cost, and wall limits are soft; already admitted cells settle. Missing usage and price evidence are not zero.",
|
||
};
|
||
await writeJson(path.join(outputDir, "manifest.json"), manifest);
|
||
// Persist the complete schedule before any cell starts. An interrupted or dry
|
||
// run retains null selections instead of inventing successful observations.
|
||
await writeEvidence(now.toISOString());
|
||
if (options.dryRun) {
|
||
const summary = { status: "dry-run", total: cells.length };
|
||
await writeJson(path.join(outputDir, "summary.json"), summary);
|
||
return { exitCode: 0, outputDir, summary };
|
||
}
|
||
|
||
const runtimeRoot = deps.runCell
|
||
? undefined
|
||
: await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-code-mode-runtime-"));
|
||
const abortController = new AbortController();
|
||
const interrupt = () => abortController.abort(new Error("Code Mode matrix interrupted"));
|
||
process.once("SIGINT", interrupt);
|
||
process.once("SIGTERM", interrupt);
|
||
try {
|
||
const runtime = runtimeRoot
|
||
? await prepareRuntimeEntrypoint(runtimeRepoRoot, runtimeRoot)
|
||
: undefined;
|
||
const results: CodeModeMatrixCellResult[] = [];
|
||
const resultsPath = path.join(outputDir, "results.jsonl");
|
||
await fs.writeFile(resultsPath, "", "utf8");
|
||
const executeCell = deps.runCell ?? runMatrixCell;
|
||
const executionStartedAt = Date.now();
|
||
let admittedCells = 0;
|
||
let stopReason: string | null = null;
|
||
const stoppedProviders = new Map<string, string>();
|
||
const stoppedModels = new Map<string, string>();
|
||
const unstarted: Array<{ id: string; reason: string }> = [];
|
||
let persistence = Promise.resolve();
|
||
const observedBudget = () => ({
|
||
tokens: results.reduce(
|
||
(total, result) =>
|
||
total + (result.accounting?.knownTotalTokens ?? result.usage?.total ?? 0),
|
||
0,
|
||
),
|
||
knownCostUsd: results.reduce(
|
||
(total, result) => total + (result.accounting?.knownCostUsd ?? result.costUsd ?? 0),
|
||
0,
|
||
),
|
||
missingTokenCells: results.filter((result) =>
|
||
result.accounting ? !result.accounting.complete : result.usage?.total === undefined,
|
||
).length,
|
||
missingCostCells: results.filter((result) =>
|
||
result.accounting ? !result.accounting.costComplete : result.costUsd === undefined,
|
||
).length,
|
||
wallSeconds: (Date.now() - executionStartedAt) / 1_000,
|
||
});
|
||
const stoppedReason = (cell: MatrixCell) =>
|
||
stoppedModels.get(cell.model) ?? stoppedProviders.get(cell.model.split("/")[0]!);
|
||
const runCell = async (cell: MatrixCell) => {
|
||
const index = cells.indexOf(cell);
|
||
abortController.signal.throwIfAborted();
|
||
const workload = isGatewayTask(cell.task)
|
||
? (await import("./lib/code-mode-matrix-gateway.ts")).createGatewayMatrixWorkload(
|
||
cell.task,
|
||
cell.repetition,
|
||
options.thinking,
|
||
options.timeoutSeconds,
|
||
options.gatewayExecutor ?? "node",
|
||
)
|
||
: undefined;
|
||
// Repetitions are independent scheduled cells, not retries whose eventual
|
||
// success could hide an earlier failure.
|
||
const occurrenceId = invocation.begin(index, null);
|
||
let result: CodeModeMatrixCellResult;
|
||
const cellStartedAt = Date.now();
|
||
try {
|
||
result = await executeCell({
|
||
abortSignal: abortController.signal,
|
||
buildSha256: buildSha256 ?? "dry-run",
|
||
cell,
|
||
...(workload ? { executor: workload.settings.executor } : {}),
|
||
gitSha: sourceIdentity.gitSha,
|
||
keepState: options.keepState,
|
||
outputDir,
|
||
repoRoot: runtimeRepoRoot,
|
||
runtime,
|
||
sourceDirty: sourceIdentity.sourceDirty,
|
||
sourcePatchSha256: sourceIdentity.sourcePatchSha256,
|
||
thinking: options.thinking,
|
||
timeoutSeconds: options.timeoutSeconds,
|
||
});
|
||
} catch (error) {
|
||
result = harnessFailureResult(
|
||
cell,
|
||
{
|
||
buildSha256: buildSha256 ?? "dry-run",
|
||
...sourceIdentity,
|
||
},
|
||
Date.now() - cellStartedAt,
|
||
error,
|
||
);
|
||
}
|
||
if (workload) {
|
||
result.workload = workload;
|
||
result.executor = workload.settings.executor;
|
||
}
|
||
if (
|
||
result.failureCategory === "provider_auth" ||
|
||
result.failureCategory === "provider_billing"
|
||
) {
|
||
stoppedProviders.set(cell.model.split("/")[0]!, result.failureCategory);
|
||
} else if (result.failureCategory === "model_unavailable") {
|
||
stoppedModels.set(cell.model, result.failureCategory);
|
||
}
|
||
const persist = async () => {
|
||
result.evidenceOccurrenceId = occurrenceId;
|
||
const artifactPath = path.posix.join("observations", `${occurrenceId}.json`);
|
||
const artifactBytes = `${JSON.stringify(redactJsonValueForDevToolLog({ result, launch, buildSha256 }), null, 2)}\n`;
|
||
await fs.mkdir(path.join(outputDir, "observations"), { recursive: true });
|
||
await fs.writeFile(path.join(outputDir, artifactPath), artifactBytes, {
|
||
flag: "wx",
|
||
mode: 0o600,
|
||
});
|
||
const artifact = {
|
||
kind: "matrix-observation",
|
||
path: artifactPath,
|
||
source: "code-mode-model-matrix",
|
||
sha256: createHash("sha256").update(artifactBytes).digest("hex"),
|
||
};
|
||
// The envelope observes provider/model responses, not a target runtime's
|
||
// package, protocol, account, or proof class. Keep this receipt prepared-only.
|
||
invocation.complete(occurrenceId, {
|
||
status: evidenceStatus(result),
|
||
entries: buildCodeModeMatrixEvidence({
|
||
generatedAt: result.timestamp,
|
||
repoRoot: options.repoRoot,
|
||
results: [result],
|
||
}).entries.map((entry) =>
|
||
Object.assign({}, entry, {
|
||
execution: entry.execution
|
||
? {
|
||
...entry.execution,
|
||
artifacts: [
|
||
...entry.execution.artifacts,
|
||
{
|
||
kind: artifact.kind,
|
||
path: artifact.path,
|
||
source: artifact.source,
|
||
},
|
||
],
|
||
}
|
||
: undefined,
|
||
}),
|
||
),
|
||
receipts: [
|
||
{ id: `${occurrenceId}:prepared`, phase: "prepared", identity: launch, artifact },
|
||
],
|
||
});
|
||
invocation.select(index, occurrenceId);
|
||
results.push(result);
|
||
await fs.appendFile(
|
||
resultsPath,
|
||
`${JSON.stringify(redactJsonValueForDevToolLog(result))}\n`,
|
||
"utf8",
|
||
);
|
||
await writeEvidence(result.timestamp);
|
||
console.log(
|
||
`[code-mode-matrix-result] ${JSON.stringify(redactJsonValueForDevToolLog(result))}`,
|
||
);
|
||
const label = result.passed ? "PASS" : `FAIL ${result.failureCategory ?? "unknown"}`;
|
||
console.log(`[code-mode-matrix] ${label} ${result.id} ${result.elapsedMs}ms`);
|
||
};
|
||
const written = persistence.then(persist);
|
||
persistence = written.catch(() => {});
|
||
await written;
|
||
};
|
||
for (const wave of waves) {
|
||
abortController.signal.throwIfAborted();
|
||
const budget = observedBudget();
|
||
const stopped = stoppedReason(wave[0]!);
|
||
stopReason ??=
|
||
admittedCells + wave.length > admission.maxCells
|
||
? "max_cells"
|
||
: budget.tokens >= admission.maxTokens
|
||
? "max_tokens"
|
||
: budget.knownCostUsd >= admission.maxKnownCostUsd
|
||
? "max_known_cost_usd"
|
||
: budget.wallSeconds >= admission.maxWallSeconds
|
||
? "max_wall_seconds"
|
||
: null;
|
||
if (stopReason || stopped) {
|
||
unstarted.push(...wave.map((cell) => ({ id: cell.id, reason: stopReason ?? stopped! })));
|
||
continue;
|
||
}
|
||
admittedCells += wave.length;
|
||
let next = 0;
|
||
const workers = Array.from(
|
||
{ length: Math.min(admission.concurrency, wave.length) },
|
||
async () => {
|
||
while (next < wave.length) {
|
||
const cell = wave[next++]!;
|
||
await runCell(cell);
|
||
}
|
||
},
|
||
);
|
||
const outcomes = await Promise.allSettled(workers);
|
||
const rejected = outcomes.find((outcome) => outcome.status === "rejected");
|
||
if (rejected?.status === "rejected") {
|
||
throw rejected.reason;
|
||
}
|
||
if ((await (deps.readBuildSha256 ?? hashRuntimeArtifacts)(runtimeRepoRoot)) !== buildSha256) {
|
||
throw new Error(
|
||
"Runtime build changed during the benchmark; per-cell evidence is retained but cannot establish a fixed-build comparison.",
|
||
);
|
||
}
|
||
}
|
||
abortController.signal.throwIfAborted();
|
||
if (options.runtimeDir && !deps.runCell) {
|
||
const finalIdentity = await readSourceIdentity(
|
||
runtimeRepoRoot,
|
||
STRICT_SOURCE_IDENTITY_OPTIONS,
|
||
);
|
||
if (finalIdentity.sourceDirty || finalIdentity.gitSha !== sourceIdentity.gitSha) {
|
||
throw new Error(
|
||
"Frozen runtime changed during the benchmark; per-cell evidence is retained but cannot establish a fixed-source comparison.",
|
||
);
|
||
}
|
||
}
|
||
|
||
const groups = summarizeResults(results);
|
||
const failed = results.filter((result) => !result.passed).length;
|
||
const firstPassPassed = groups.filter((group) => group.firstPassPassed).length;
|
||
const eventualPassed = groups.filter((group) => group.eventualPassed).length;
|
||
const summary = {
|
||
schemaVersion: MATRIX_SCHEMA_VERSION,
|
||
finishedAt: new Date().toISOString(),
|
||
...(options.tasks.some(isGatewayTask)
|
||
? { gatewayExecutor: options.gatewayExecutor ?? "node" }
|
||
: {}),
|
||
...sourceIdentity,
|
||
buildSha256,
|
||
counts: {
|
||
total: results.length,
|
||
passed: results.length - failed,
|
||
failed,
|
||
},
|
||
groupCounts: {
|
||
total: groups.length,
|
||
firstPassPassed,
|
||
eventualPassed,
|
||
},
|
||
...(results.some((result) => result.workload)
|
||
? { gatewayOutcomes: summarizeGatewayMatrixOutcomes(results) }
|
||
: {}),
|
||
groups,
|
||
admission: {
|
||
...admission,
|
||
admittedCells,
|
||
plannedCells: cells.length,
|
||
completedCells: results.length,
|
||
stoppedProviders: Object.fromEntries(stoppedProviders),
|
||
stoppedModels: Object.fromEntries(stoppedModels),
|
||
stopReason,
|
||
unstarted,
|
||
observed: observedBudget(),
|
||
overshoot: {
|
||
tokens: Math.max(0, observedBudget().tokens - admission.maxTokens),
|
||
knownCostUsd: Math.max(0, observedBudget().knownCostUsd - admission.maxKnownCostUsd),
|
||
wallSeconds: Math.max(0, observedBudget().wallSeconds - admission.maxWallSeconds),
|
||
contract:
|
||
"Admission limits are soft. An admitted wave finishes before another starts, including its queued arm at concurrency 1. Missing usage/prices make observed totals lower bounds, not hard spend limits.",
|
||
},
|
||
},
|
||
};
|
||
await writeJson(path.join(outputDir, "summary.json"), summary);
|
||
const performanceResults = results.filter((result) => isMatrixPerformanceTask(result.task));
|
||
if (performanceResults.length > 0) {
|
||
const { compareCodeModeMatrixModes } = await import("./lib/code-mode-matrix-comparison.ts");
|
||
await writeJson(
|
||
path.join(outputDir, "mode-comparison.json"),
|
||
compareCodeModeMatrixModes(performanceResults),
|
||
);
|
||
}
|
||
if (options.baselineResults) {
|
||
const { compareCodeModeMatrixResultsFile } =
|
||
await import("./lib/code-mode-matrix-comparison.ts");
|
||
await writeJson(
|
||
path.join(outputDir, "comparison.json"),
|
||
await compareCodeModeMatrixResultsFile(options.baselineResults, results),
|
||
);
|
||
}
|
||
await writeEvidence(summary.finishedAt);
|
||
return {
|
||
exitCode: (failed > 0 || unstarted.length > 0) && !options.allowFailures ? 1 : 0,
|
||
outputDir,
|
||
summary,
|
||
};
|
||
} finally {
|
||
process.off("SIGINT", interrupt);
|
||
process.off("SIGTERM", interrupt);
|
||
if (runtimeRoot) {
|
||
await fs.rm(runtimeRoot, { force: true, recursive: true });
|
||
}
|
||
}
|
||
}
|
||
|
||
async function main(): Promise<void> {
|
||
try {
|
||
const options = parseCodeModeMatrixOptions(process.argv.slice(2));
|
||
const result = await runCodeModeModelMatrix(options);
|
||
console.log(
|
||
`[code-mode-matrix] artifacts ${path.relative(options.repoRoot, result.outputDir)}`,
|
||
);
|
||
process.exitCode = result.exitCode;
|
||
} catch (error) {
|
||
if ((error as { code?: unknown }).code === "HELP") {
|
||
console.log(error instanceof Error ? error.message : String(error));
|
||
return;
|
||
}
|
||
console.error(error instanceof Error ? error.message : String(error));
|
||
process.exitCode = 1;
|
||
}
|
||
}
|
||
|
||
const entrypoint = process.argv[1];
|
||
if (entrypoint && import.meta.url === pathToFileURL(path.resolve(entrypoint)).href) {
|
||
await main();
|
||
}
|