refactor: dedupe tooling helpers

This commit is contained in:
Peter Steinberger 2026-04-23 18:06:49 +01:00
parent f98f93c29a
commit 2045c0977e
No known key found for this signature in database
11 changed files with 244 additions and 347 deletions

View file

@ -1,8 +1,13 @@
#!/usr/bin/env node
import { readdirSync, readFileSync, statSync } from "node:fs";
import { readFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import ts from "typescript";
import {
collectSourceFiles,
collectStronglyConnectedComponents,
formatCycle,
} from "./lib/import-cycle-graph.ts";
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const scanRoots = ["src", "extensions", "scripts"] as const;
@ -13,14 +18,6 @@ const declarationSourcePattern = /\.d\.[cm]?ts$/;
const ignoredPathPartPattern =
/(^|\/)(node_modules|dist|build|coverage|\.artifacts|\.git|assets)(\/|$)/;
function normalizeRepoPath(filePath: string): string {
return path.relative(repoRoot, filePath).split(path.sep).join("/");
}
function cycleSignature(files: readonly string[]): string {
return files.toSorted((left, right) => left.localeCompare(right)).join("\n");
}
function shouldSkipRepoPath(repoPath: string): boolean {
return (
ignoredPathPartPattern.test(repoPath) ||
@ -30,23 +27,6 @@ function shouldSkipRepoPath(repoPath: string): boolean {
);
}
function collectSourceFiles(root: string): string[] {
const repoPath = normalizeRepoPath(root);
if (shouldSkipRepoPath(repoPath)) {
return [];
}
const stats = statSync(root);
if (stats.isFile()) {
return sourceExtensions.some((extension) => repoPath.endsWith(extension)) ? [repoPath] : [];
}
if (!stats.isDirectory()) {
return [];
}
return readdirSync(root, { withFileTypes: true })
.flatMap((entry) => collectSourceFiles(path.join(root, entry.name)))
.toSorted((left, right) => left.localeCompare(right));
}
function createSourceResolver(files: readonly string[]) {
const fileSet = new Set(files);
const pathMap = new Map<string, string>();
@ -152,106 +132,14 @@ function collectRuntimeStaticImports(
return imports.toSorted((left, right) => left.localeCompare(right));
}
function collectStronglyConnectedComponents(
graph: ReadonlyMap<string, readonly string[]>,
): string[][] {
let nextIndex = 0;
const stack: string[] = [];
const onStack = new Set<string>();
const indexByNode = new Map<string, number>();
const lowLinkByNode = new Map<string, number>();
const components: string[][] = [];
const visit = (node: string) => {
indexByNode.set(node, nextIndex);
lowLinkByNode.set(node, nextIndex);
nextIndex += 1;
stack.push(node);
onStack.add(node);
for (const next of graph.get(node) ?? []) {
if (!indexByNode.has(next)) {
visit(next);
lowLinkByNode.set(node, Math.min(lowLinkByNode.get(node)!, lowLinkByNode.get(next)!));
} else if (onStack.has(next)) {
lowLinkByNode.set(node, Math.min(lowLinkByNode.get(node)!, indexByNode.get(next)!));
}
}
if (lowLinkByNode.get(node) !== indexByNode.get(node)) {
return;
}
const component: string[] = [];
let current: string | undefined;
do {
current = stack.pop();
if (!current) {
throw new Error("Import cycle stack underflow");
}
onStack.delete(current);
component.push(current);
} while (current !== node);
if (component.length > 1 || (graph.get(node) ?? []).includes(node)) {
components.push(component.toSorted((left, right) => left.localeCompare(right)));
}
};
for (const node of graph.keys()) {
if (!indexByNode.has(node)) {
visit(node);
}
}
return components.toSorted(
(left, right) =>
right.length - left.length || cycleSignature(left).localeCompare(cycleSignature(right)),
);
}
function findCycleWitness(
component: readonly string[],
graph: ReadonlyMap<string, readonly string[]>,
): string[] {
const componentSet = new Set(component);
const start = component[0];
if (!start) {
return [];
}
const activePath: string[] = [];
const visited = new Set<string>();
const visit = (node: string): string[] | null => {
activePath.push(node);
visited.add(node);
for (const next of graph.get(node) ?? []) {
if (!componentSet.has(next)) {
continue;
}
const existingIndex = activePath.indexOf(next);
if (existingIndex >= 0) {
return [...activePath.slice(existingIndex), next];
}
if (!visited.has(next)) {
const result = visit(next);
if (result) {
return result;
}
}
}
activePath.pop();
return null;
};
return visit(start) ?? component;
}
function formatCycle(
component: readonly string[],
graph: ReadonlyMap<string, readonly string[]>,
): string {
const witness = findCycleWitness(component, graph);
return witness.map((file, index) => `${index === 0 ? " " : " -> "}${file}`).join("\n");
}
function main(): number {
const files = scanRoots.flatMap((root) => collectSourceFiles(path.join(repoRoot, root)));
const files = scanRoots.flatMap((root) =>
collectSourceFiles(path.join(repoRoot, root), {
repoRoot,
sourceExtensions,
shouldSkipRepoPath,
}),
);
const resolveSource = createSourceResolver(files);
const graph = new Map(
files.map((file): [string, string[]] => [

View file

@ -1,8 +1,12 @@
#!/usr/bin/env node
import { readdirSync, readFileSync, statSync } from "node:fs";
import { readFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import ts from "typescript";
import {
collectSourceFiles,
collectStronglyConnectedComponents,
} from "./lib/import-cycle-graph.ts";
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const scanRoots = ["src", "extensions", "ui"] as const;
@ -10,35 +14,10 @@ const sourceExtensions = [".ts"] as const;
const ignoredPathPartPattern =
/(^|\/)(node_modules|dist|build|coverage|\.artifacts|\.git|assets)(\/|$)/;
function normalizeRepoPath(filePath: string): string {
return path.relative(repoRoot, filePath).split(path.sep).join("/");
}
function cycleSignature(files: readonly string[]): string {
return files.toSorted((left, right) => left.localeCompare(right)).join("\n");
}
function shouldSkipRepoPath(repoPath: string): boolean {
return ignoredPathPartPattern.test(repoPath);
}
function collectSourceFiles(root: string): string[] {
const repoPath = normalizeRepoPath(root);
if (shouldSkipRepoPath(repoPath)) {
return [];
}
const stats = statSync(root);
if (stats.isFile()) {
return sourceExtensions.some((extension) => repoPath.endsWith(extension)) ? [repoPath] : [];
}
if (!stats.isDirectory()) {
return [];
}
return readdirSync(root, { withFileTypes: true })
.flatMap((entry) => collectSourceFiles(path.join(root, entry.name)))
.toSorted((left, right) => left.localeCompare(right));
}
function loadCompilerOptions(): ts.CompilerOptions {
const configPath = path.join(repoRoot, "tsconfig.json");
const config = ts.readConfigFile(configPath, (filePath) => ts.sys.readFile(filePath));
@ -110,64 +89,14 @@ function createImportGraph(files: readonly string[]): Map<string, string[]> {
return graph;
}
function collectStronglyConnectedComponents(
graph: ReadonlyMap<string, readonly string[]>,
): string[][] {
let nextIndex = 0;
const stack: string[] = [];
const onStack = new Set<string>();
const indexByNode = new Map<string, number>();
const lowLinkByNode = new Map<string, number>();
const components: string[][] = [];
const visit = (node: string) => {
indexByNode.set(node, nextIndex);
lowLinkByNode.set(node, nextIndex);
nextIndex += 1;
stack.push(node);
onStack.add(node);
for (const next of graph.get(node) ?? []) {
if (!indexByNode.has(next)) {
visit(next);
lowLinkByNode.set(node, Math.min(lowLinkByNode.get(node)!, lowLinkByNode.get(next)!));
} else if (onStack.has(next)) {
lowLinkByNode.set(node, Math.min(lowLinkByNode.get(node)!, indexByNode.get(next)!));
}
}
if (lowLinkByNode.get(node) !== indexByNode.get(node)) {
return;
}
const component: string[] = [];
let current: string | undefined;
do {
current = stack.pop();
if (!current) {
throw new Error("Import cycle stack underflow");
}
onStack.delete(current);
component.push(current);
} while (current !== node);
if (component.length > 1 || (graph.get(node) ?? []).includes(node)) {
components.push(component.toSorted((left, right) => left.localeCompare(right)));
}
};
for (const node of graph.keys()) {
if (!indexByNode.has(node)) {
visit(node);
}
}
return components.toSorted(
(left, right) =>
right.length - left.length || cycleSignature(left).localeCompare(cycleSignature(right)),
);
}
function main(): number {
const files = scanRoots.flatMap((root) => collectSourceFiles(path.join(repoRoot, root)));
const files = scanRoots.flatMap((root) =>
collectSourceFiles(path.join(repoRoot, root), {
repoRoot,
sourceExtensions,
shouldSkipRepoPath,
}),
);
const graph = createImportGraph(files);
const cycles = collectStronglyConnectedComponents(graph);

View file

@ -18,7 +18,7 @@ function formatSeconds(value) {
return value === null ? "" : `${value}s`;
}
export function summarizeRunTimings(run, limit = 15) {
function collectRunTimingContext(run) {
const created = parseTime(run.createdAt);
const updated = parseTime(run.updatedAt);
const jobs = (run.jobs ?? [])
@ -31,9 +31,17 @@ export function summarizeRunTimings(run, limit = 15) {
durationSeconds: secondsBetween(started, completed),
name: job.name,
queueSeconds: secondsBetween(created, started),
started,
completed,
status: job.status,
};
});
return { created, jobs, updated };
}
export function summarizeRunTimings(run, limit = 15) {
const { created, jobs, updated } = collectRunTimingContext(run);
const byDuration = [...jobs]
.filter((job) => job.durationSeconds !== null)
.toSorted((left, right) => right.durationSeconds - left.durationSeconds)
@ -105,30 +113,14 @@ function loadRun(runId) {
}
function summarizeJobs(run) {
const created = parseTime(run.createdAt);
const updated = parseTime(run.updatedAt);
const jobs = (run.jobs ?? [])
.filter((job) => !job.name?.startsWith("matrix."))
.map((job) => {
const started = parseTime(job.startedAt);
const completed = parseTime(job.completedAt);
return {
conclusion: job.conclusion ?? "",
durationSeconds: secondsBetween(started, completed),
name: job.name,
queueSeconds: secondsBetween(created, started),
started,
completed,
status: job.status,
};
})
.filter((job) => job.started !== null && job.completed !== null);
const { created, jobs, updated } = collectRunTimingContext(run);
const completedJobs = jobs.filter((job) => job.started !== null && job.completed !== null);
const successfulDurations = jobs
.filter((job) => job.status === "completed" && job.conclusion === "success")
.map((job) => job.durationSeconds)
.filter((duration) => duration !== null);
const firstStart = Math.min(...jobs.map((job) => job.started));
const lastComplete = Math.max(...jobs.map((job) => job.completed));
const firstStart = Math.min(...completedJobs.map((job) => job.started));
const lastComplete = Math.max(...completedJobs.map((job) => job.completed));
return {
avgDurationSeconds:

View file

@ -2,31 +2,10 @@ import fs from "node:fs/promises";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import {
applyProviderConfigWithDefaultModelPreset,
type ModelDefinitionConfig,
type OpenClawConfig,
} from "../../src/plugin-sdk/provider-onboard.ts";
import { applyDockerOpenAiProviderConfig, type OpenClawConfig } from "./docker-openai-seed.ts";
const require = createRequire(import.meta.url);
const DOCKER_OPENAI_MODEL_REF = "openai/gpt-5.4";
const DOCKER_OPENAI_MODEL: ModelDefinitionConfig = {
id: "gpt-5.4",
name: "gpt-5.4",
api: "openai-responses",
reasoning: true,
input: ["text", "image"],
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 1_050_000,
maxTokens: 128_000,
};
async function writeProbeServer(params: {
serverPath: string;
pidPath: string;
@ -88,7 +67,7 @@ async function main() {
await fs.rm(exitPath, { force: true });
await writeProbeServer({ serverPath, pidPath, pidsPath, exitPath });
const seededConfig = applyProviderConfigWithDefaultModelPreset(
const seededConfig = applyDockerOpenAiProviderConfig(
{
gateway: {
controlUi: {
@ -123,21 +102,8 @@ async function main() {
},
},
} satisfies OpenClawConfig,
{
providerId: "openai",
api: "openai-responses",
baseUrl: "http://127.0.0.1:9/v1",
defaultModel: DOCKER_OPENAI_MODEL,
defaultModelId: DOCKER_OPENAI_MODEL.id,
aliases: [{ modelRef: DOCKER_OPENAI_MODEL_REF, alias: "GPT" }],
primaryModelRef: DOCKER_OPENAI_MODEL_REF,
},
"sk-docker-cron-mcp-cleanup-test",
);
const openAiProvider = seededConfig.models?.providers?.openai;
if (!openAiProvider) {
throw new Error("failed to seed OpenAI provider config");
}
openAiProvider.apiKey = "sk-docker-cron-mcp-cleanup-test";
await fs.writeFile(configPath, `${JSON.stringify(seededConfig, null, 2)}\n`, "utf-8");

View file

@ -0,0 +1,45 @@
import {
applyProviderConfigWithDefaultModelPreset,
type ModelDefinitionConfig,
type OpenClawConfig,
} from "../../src/plugin-sdk/provider-onboard.ts";
export type { OpenClawConfig };
const DOCKER_OPENAI_MODEL_REF = "openai/gpt-5.4";
const DOCKER_OPENAI_MODEL: ModelDefinitionConfig = {
id: "gpt-5.4",
name: "gpt-5.4",
api: "openai-responses",
reasoning: true,
input: ["text", "image"],
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 1_050_000,
maxTokens: 128_000,
};
export function applyDockerOpenAiProviderConfig(
config: OpenClawConfig,
apiKey: string,
): OpenClawConfig {
const seededConfig = applyProviderConfigWithDefaultModelPreset(config, {
providerId: "openai",
api: "openai-responses",
baseUrl: "http://127.0.0.1:9/v1",
defaultModel: DOCKER_OPENAI_MODEL,
defaultModelId: DOCKER_OPENAI_MODEL.id,
aliases: [{ modelRef: DOCKER_OPENAI_MODEL_REF, alias: "GPT" }],
primaryModelRef: DOCKER_OPENAI_MODEL_REF,
});
const openAiProvider = seededConfig.models?.providers?.openai;
if (!openAiProvider) {
throw new Error("failed to seed OpenAI provider config");
}
openAiProvider.apiKey = apiKey;
return seededConfig;
}

View file

@ -1,28 +1,7 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import {
applyProviderConfigWithDefaultModelPreset,
type ModelDefinitionConfig,
type OpenClawConfig,
} from "../../src/plugin-sdk/provider-onboard.ts";
const DOCKER_OPENAI_MODEL_REF = "openai/gpt-5.4";
const DOCKER_OPENAI_MODEL: ModelDefinitionConfig = {
id: "gpt-5.4",
name: "gpt-5.4",
api: "openai-responses",
reasoning: true,
input: ["text", "image"],
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 1_050_000,
maxTokens: 128_000,
};
import { applyDockerOpenAiProviderConfig, type OpenClawConfig } from "./docker-openai-seed.ts";
async function main() {
const stateDir = process.env.OPENCLAW_STATE_DIR?.trim() || path.join(os.homedir(), ".openclaw");
@ -36,7 +15,7 @@ async function main() {
await fs.mkdir(sessionsDir, { recursive: true });
await fs.mkdir(path.dirname(configPath), { recursive: true });
const seededConfig = applyProviderConfigWithDefaultModelPreset(
const seededConfig = applyDockerOpenAiProviderConfig(
{
gateway: {
controlUi: {
@ -45,21 +24,8 @@ async function main() {
},
},
} satisfies OpenClawConfig,
{
providerId: "openai",
api: "openai-responses",
baseUrl: "http://127.0.0.1:9/v1",
defaultModel: DOCKER_OPENAI_MODEL,
defaultModelId: DOCKER_OPENAI_MODEL.id,
aliases: [{ modelRef: DOCKER_OPENAI_MODEL_REF, alias: "GPT" }],
primaryModelRef: DOCKER_OPENAI_MODEL_REF,
},
"sk-docker-smoke-test",
);
const openAiProvider = seededConfig.models?.providers?.openai;
if (!openAiProvider) {
throw new Error("failed to seed OpenAI provider config");
}
openAiProvider.apiKey = "sk-docker-smoke-test";
await fs.writeFile(configPath, JSON.stringify(seededConfig, null, 2), "utf-8");

View file

@ -7,6 +7,19 @@ export function readEnvNumber(name, env = process.env) {
return Number.isFinite(parsed) ? parsed : null;
}
export function readFlagValue(args, name) {
for (let index = 0; index < args.length; index += 1) {
const arg = args[index];
if (arg === name) {
return args[index + 1];
}
if (arg.startsWith(`${name}=`)) {
return arg.slice(name.length + 1);
}
}
return undefined;
}
export function consumeStringFlag(argv, index, flag, currentValue) {
if (argv[index] !== flag) {
return null;

View file

@ -0,0 +1,134 @@
import { readdirSync, statSync } from "node:fs";
import path from "node:path";
type SourceFileCollectionOptions = {
repoRoot: string;
sourceExtensions: readonly string[];
shouldSkipRepoPath?: (repoPath: string) => boolean;
};
export function normalizeRepoPath(filePath: string, repoRoot: string): string {
return path.relative(repoRoot, filePath).split(path.sep).join("/");
}
export function cycleSignature(files: readonly string[]): string {
return files.toSorted((left, right) => left.localeCompare(right)).join("\n");
}
export function collectSourceFiles(root: string, options: SourceFileCollectionOptions): string[] {
const repoPath = normalizeRepoPath(root, options.repoRoot);
if (options.shouldSkipRepoPath?.(repoPath)) {
return [];
}
const stats = statSync(root);
if (stats.isFile()) {
return options.sourceExtensions.some((extension) => repoPath.endsWith(extension))
? [repoPath]
: [];
}
if (!stats.isDirectory()) {
return [];
}
return readdirSync(root, { withFileTypes: true })
.flatMap((entry) => collectSourceFiles(path.join(root, entry.name), options))
.toSorted((left, right) => left.localeCompare(right));
}
export function collectStronglyConnectedComponents(
graph: ReadonlyMap<string, readonly string[]>,
): string[][] {
let nextIndex = 0;
const stack: string[] = [];
const onStack = new Set<string>();
const indexByNode = new Map<string, number>();
const lowLinkByNode = new Map<string, number>();
const components: string[][] = [];
const visit = (node: string) => {
indexByNode.set(node, nextIndex);
lowLinkByNode.set(node, nextIndex);
nextIndex += 1;
stack.push(node);
onStack.add(node);
for (const next of graph.get(node) ?? []) {
if (!indexByNode.has(next)) {
visit(next);
lowLinkByNode.set(node, Math.min(lowLinkByNode.get(node)!, lowLinkByNode.get(next)!));
} else if (onStack.has(next)) {
lowLinkByNode.set(node, Math.min(lowLinkByNode.get(node)!, indexByNode.get(next)!));
}
}
if (lowLinkByNode.get(node) !== indexByNode.get(node)) {
return;
}
const component: string[] = [];
let current: string | undefined;
do {
current = stack.pop();
if (!current) {
throw new Error("Import cycle stack underflow");
}
onStack.delete(current);
component.push(current);
} while (current !== node);
if (component.length > 1 || (graph.get(node) ?? []).includes(node)) {
components.push(component.toSorted((left, right) => left.localeCompare(right)));
}
};
for (const node of graph.keys()) {
if (!indexByNode.has(node)) {
visit(node);
}
}
return components.toSorted(
(left, right) =>
right.length - left.length || cycleSignature(left).localeCompare(cycleSignature(right)),
);
}
export function findCycleWitness(
component: readonly string[],
graph: ReadonlyMap<string, readonly string[]>,
): string[] {
const componentSet = new Set(component);
const start = component[0];
if (!start) {
return [];
}
const activePath: string[] = [];
const visited = new Set<string>();
const visit = (node: string): string[] | null => {
activePath.push(node);
visited.add(node);
for (const next of graph.get(node) ?? []) {
if (!componentSet.has(next)) {
continue;
}
const existingIndex = activePath.indexOf(next);
if (existingIndex >= 0) {
return [...activePath.slice(existingIndex), next];
}
if (!visited.has(next)) {
const result = visit(next);
if (result) {
return result;
}
}
}
activePath.pop();
return null;
};
return visit(start) ?? [...component];
}
export function formatCycle(
component: readonly string[],
graph: ReadonlyMap<string, readonly string[]>,
): string {
const witness = findCycleWitness(component, graph);
return witness.map((file, index) => `${index === 0 ? " " : " -> "}${file}`).join("\n");
}

View file

@ -1,6 +1,7 @@
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { readFlagValue } from "./arg-utils.mjs";
const CORE_TEST_CONFIGS = new Set([
"tsconfig.core.test.json",
@ -73,16 +74,3 @@ function isMetadataOnlyCommand(args) {
["--help", "-h", "--version", "-v", "--init", "--showConfig"].includes(arg),
);
}
function readFlagValue(args, name) {
for (let index = 0; index < args.length; index++) {
const arg = args[index];
if (arg === name) {
return args[index + 1];
}
if (arg.startsWith(`${name}=`)) {
return arg.slice(name.length + 1);
}
}
return undefined;
}

View file

@ -219,13 +219,13 @@ const hasDirtyRuntimePostBuildInputs = (deps) => {
return parseGitStatusPaths(output).some((repoPath) => isRuntimePostBuildRelevantPath(repoPath));
};
const readBuildStamp = (deps) => {
const mtime = statMtime(deps.buildStampPath, deps.fs);
const readJsonStamp = (filePath, deps) => {
const mtime = statMtime(filePath, deps.fs);
if (mtime == null) {
return { mtime: null, head: null };
}
try {
const raw = deps.fs.readFileSync(deps.buildStampPath, "utf8").trim();
const raw = deps.fs.readFileSync(filePath, "utf8").trim();
if (!raw.startsWith("{")) {
return { mtime, head: null };
}
@ -237,22 +237,10 @@ const readBuildStamp = (deps) => {
}
};
const readBuildStamp = (deps) => readJsonStamp(deps.buildStampPath, deps);
const readRuntimePostBuildStamp = (deps) => {
const mtime = statMtime(deps.runtimePostBuildStampPath, deps.fs);
if (mtime == null) {
return { mtime: null, head: null };
}
try {
const raw = deps.fs.readFileSync(deps.runtimePostBuildStampPath, "utf8").trim();
if (!raw.startsWith("{")) {
return { mtime, head: null };
}
const parsed = JSON.parse(raw);
const head = typeof parsed?.head === "string" && parsed.head.trim() ? parsed.head.trim() : null;
return { mtime, head };
} catch {
return { mtime, head: null };
}
return readJsonStamp(deps.runtimePostBuildStampPath, deps);
};
const hasSourceMtimeChanged = (stampMtime, deps) => {

View file

@ -1,6 +1,7 @@
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { readFlagValue } from "./lib/arg-utils.mjs";
import {
acquireLocalHeavyCheckLockSync,
applyLocalTsgoPolicy,
@ -45,16 +46,3 @@ try {
} finally {
releaseLock();
}
function readFlagValue(args, name) {
for (let index = 0; index < args.length; index++) {
const arg = args[index];
if (arg === name) {
return args[index + 1];
}
if (arg.startsWith(`${name}=`)) {
return arg.slice(name.length + 1);
}
}
return undefined;
}