perf(build): canonicalize plugin SDK declarations (#98758)

This commit is contained in:
Dallin Romney 2026-07-02 00:21:41 -07:00 committed by GitHub
parent cb44f40474
commit d32c4cf300
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 109 additions and 134 deletions

View file

@ -1181,8 +1181,6 @@ jobs:
pnpm lint:auth:no-pairing-store-group
pnpm lint:auth:pairing-account-scope
pnpm check:import-cycles
# build-artifacts already runs the tsdown/runtime build for the same Node-relevant changes.
NODE_OPTIONS=--max-old-space-size=8192 pnpm build:plugin-sdk:strict-smoke
;;
shrinkwrap)
pnpm deps:shrinkwrap:check

View file

@ -1514,8 +1514,8 @@
"build:ci-artifacts": "node scripts/build-all.mjs ciArtifacts",
"build:docker": "node scripts/tsdown-build.mjs && node scripts/check-cli-bootstrap-imports.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && node scripts/runtime-postbuild-stamp.mjs && pnpm plugins:assets:build && pnpm plugins:assets:copy && node --experimental-strip-types scripts/copy-hook-metadata.ts && node --experimental-strip-types scripts/copy-export-html-templates.ts && node --experimental-strip-types scripts/write-build-info.ts && node --experimental-strip-types scripts/write-cli-startup-metadata.ts && node --experimental-strip-types scripts/write-cli-compat.ts",
"build:plugin-sdk:dts": "node scripts/run-tsgo.mjs -p tsconfig.plugin-sdk.dts.json --declaration true",
"build:plugin-sdk:strict-smoke": "pnpm build:plugin-sdk:dts && node --experimental-strip-types scripts/write-plugin-sdk-entry-dts.ts",
"build:strict-smoke": "pnpm plugins:assets:build && node scripts/tsdown-build.mjs && node scripts/check-cli-bootstrap-imports.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && node scripts/runtime-postbuild-stamp.mjs && pnpm build:plugin-sdk:dts && node --experimental-strip-types scripts/write-plugin-sdk-entry-dts.ts && node scripts/check-plugin-sdk-exports.mjs",
"build:plugin-sdk:strict-smoke": "node scripts/tsdown-build.mjs && node scripts/run-with-env.mjs OPENCLAW_PLUGIN_SDK_CANONICAL_DTS=1 -- node --experimental-strip-types scripts/write-plugin-sdk-entry-dts.ts && node scripts/check-plugin-sdk-exports.mjs",
"build:strict-smoke": "pnpm plugins:assets:build && node scripts/tsdown-build.mjs && node scripts/check-cli-bootstrap-imports.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && node scripts/runtime-postbuild-stamp.mjs && node scripts/run-with-env.mjs OPENCLAW_PLUGIN_SDK_CANONICAL_DTS=1 -- node --experimental-strip-types scripts/write-plugin-sdk-entry-dts.ts && node scripts/check-plugin-sdk-exports.mjs",
"canvas:a2ui:bundle": "node scripts/bundle-a2ui.mjs",
"canvas:a2ui:native:check": "node scripts/sync-native-a2ui.mjs --check",
"canvas:a2ui:native:sync": "node scripts/sync-native-a2ui.mjs --write",

View file

@ -11,42 +11,11 @@ import { pluginSdkEntrypoints } from "./lib/plugin-sdk-entries.mjs";
import { resolvePnpmRunner } from "./pnpm-runner.mjs";
const nodeBin = process.execPath;
const WINDOWS_BUILD_MAX_OLD_SPACE_MB = 8192;
const BUILD_CACHE_VERSION = 3;
const PLUGIN_SDK_DTS_CACHE_INPUTS = [
"package.json",
"pnpm-lock.yaml",
"npm-shrinkwrap.json",
"packages/plugin-sdk/package.json",
"packages/llm-core/package.json",
"packages/markdown-core/package.json",
"packages/media-core/package.json",
"packages/media-understanding-common/package.json",
"packages/terminal-core/package.json",
"packages/acp-core/package.json",
"packages/model-catalog-core/package.json",
"packages/normalization-core/package.json",
"packages/web-content-core/package.json",
"packages/memory-host-sdk/package.json",
"tsconfig.json",
"tsconfig.plugin-sdk.dts.json",
"src/plugin-sdk",
"packages/llm-core/src",
"packages/markdown-core/src",
"packages/media-core/src",
"packages/media-generation-core/src",
"packages/model-catalog-core/src",
"packages/memory-host-sdk/src",
"packages/normalization-core/src",
"packages/acp-core/src",
"packages/media-understanding-common/src",
"packages/terminal-core/src",
"packages/web-content-core/src",
"src/types",
"src/video-generation/dashscope-compatible.ts",
"src/video-generation/types.ts",
const PLUGIN_SDK_ENTRY_DTS_CACHE_ENV = [
"OPENCLAW_BUILD_PRIVATE_QA",
"OPENCLAW_PLUGIN_SDK_CANONICAL_DTS",
];
const PLUGIN_SDK_ENTRY_DTS_CACHE_ENV = ["OPENCLAW_BUILD_PRIVATE_QA"];
const PLUGIN_SDK_ENTRY_DTS_CACHE_INPUTS = [
"scripts/write-plugin-sdk-entry-dts.ts",
"scripts/lib/plugin-sdk-entries.mjs",
@ -54,20 +23,14 @@ const PLUGIN_SDK_ENTRY_DTS_CACHE_INPUTS = [
"scripts/lib/plugin-sdk-private-local-only-subpaths.json",
"scripts/lib/plugin-sdk-deprecated-public-subpaths.json",
"scripts/lib/plugin-sdk-deprecated-barrel-subpaths.json",
...PLUGIN_SDK_DTS_CACHE_INPUTS,
];
const PLUGIN_SDK_ENTRY_DTS_CACHE_OUTPUTS = [
{ path: "dist/plugin-sdk", extensions: [".d.ts"], recursive: false },
"dist/plugin-sdk/webhook-path.js",
"dist/plugin-sdk/.boundary-entry-shims.stamp",
...pluginSdkEntrypoints.map((entry) => `packages/plugin-sdk/dist/src/plugin-sdk/${entry}.d.ts`),
];
const PNPM_STEP_NODE_FALLBACKS = new Map([
["plugins:assets:build", ["scripts/bundled-plugin-assets.mjs", "--phase", "build"]],
[
"build:plugin-sdk:dts",
["scripts/run-tsgo.mjs", "-p", "tsconfig.plugin-sdk.dts.json", "--declaration", "true"],
],
["plugins:assets:copy", ["scripts/bundled-plugin-assets.mjs", "--phase", "copy"]],
["ui:build", ["scripts/ui.js", "build"]],
]);
@ -86,20 +49,13 @@ export const BUILD_ALL_STEPS = [
kind: "node",
args: ["scripts/runtime-postbuild-stamp.mjs"],
},
{
label: "build:plugin-sdk:dts",
kind: "pnpm",
pnpmArgs: ["build:plugin-sdk:dts"],
windowsNodeOptions: `--max-old-space-size=${WINDOWS_BUILD_MAX_OLD_SPACE_MB}`,
cache: {
inputs: PLUGIN_SDK_DTS_CACHE_INPUTS,
outputs: ["dist/plugin-sdk/.tsbuildinfo", "dist/plugin-sdk/packages", "dist/plugin-sdk/src"],
},
},
{
label: "write-plugin-sdk-entry-dts",
kind: "node",
args: ["--experimental-strip-types", "scripts/write-plugin-sdk-entry-dts.ts"],
env: {
OPENCLAW_PLUGIN_SDK_CANONICAL_DTS: "1",
},
cache: {
env: PLUGIN_SDK_ENTRY_DTS_CACHE_ENV,
inputs: PLUGIN_SDK_ENTRY_DTS_CACHE_INPUTS,
@ -171,7 +127,6 @@ export const BUILD_ALL_PROFILES = {
"runtime-postbuild",
"build-stamp",
"runtime-postbuild-stamp",
"build:plugin-sdk:dts",
"write-plugin-sdk-entry-dts",
"check-plugin-sdk-exports",
"plugins:assets:copy",
@ -216,7 +171,6 @@ export const BUILD_ALL_PROFILE_STEP_ENV = {
},
ciArtifacts: {
tsdown: {
OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1",
OPENCLAW_PRESERVE_CLI_STARTUP_METADATA: "1",
},
},

View file

@ -8,10 +8,10 @@
* aliases before release.
*/
import { readFileSync, existsSync, readdirSync } from "node:fs";
import { resolve, dirname } from "node:path";
import { readFileSync, existsSync, statSync } from "node:fs";
import { resolve, dirname, relative, sep } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { publicPluginSdkSubpaths } from "./lib/plugin-sdk-entries.mjs";
import { publicPluginSdkEntrypoints, publicPluginSdkSubpaths } from "./lib/plugin-sdk-entries.mjs";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const distFile = resolve(scriptDir, "..", "dist", "plugin-sdk", "index.js");
@ -43,6 +43,8 @@ const exportSet = new Set(exportedNames);
const requiredRuntimeShimEntries = ["compat.js", "root-alias.cjs"];
const forbiddenPublicDeclarationSpecifiers = ["@openclaw/llm-core"];
const MAX_PLUGIN_SDK_DECLARATION_BYTES = 5_000_000;
const RELATIVE_DECLARATION_SPECIFIER_RE = /\b(?:from|import)\s*(?:\(\s*)?["']([^"']+)["']/gu;
const requiredSubpathExports = {
"secret-input-runtime": [
"coerceSecretRef",
@ -114,24 +116,58 @@ for (const [entry, names] of Object.entries(requiredSubpathExports)) {
}
}
for (const entry of readdirSync(resolve(scriptDir, "..", "dist", "plugin-sdk"), {
withFileTypes: true,
})) {
if (!entry.isFile() || !entry.name.endsWith(".d.ts")) {
const distDir = resolve(scriptDir, "..", "dist");
const declarationPaths = new Set();
const declarationQueue = publicPluginSdkEntrypoints.map((entry) =>
resolve(distDir, "plugin-sdk", `${entry}.d.ts`),
);
while (declarationQueue.length > 0) {
const dtsPath = declarationQueue.pop();
if (!dtsPath || declarationPaths.has(dtsPath)) {
continue;
}
const dtsPath = resolve(scriptDir, "..", "dist", "plugin-sdk", entry.name);
if (!existsSync(dtsPath)) {
console.error(`MISSING PUBLIC DTS DEPENDENCY: ${relative(resolve(scriptDir, ".."), dtsPath)}`);
missing += 1;
continue;
}
declarationPaths.add(dtsPath);
const dtsContent = readFileSync(dtsPath, "utf8");
for (const match of dtsContent.matchAll(RELATIVE_DECLARATION_SPECIFIER_RE)) {
const specifier = match[1];
if (!specifier?.startsWith(".")) {
continue;
}
const declarationSpecifier = specifier.endsWith(".js")
? `${specifier.slice(0, -3)}.d.ts`
: `${specifier}.d.ts`;
const importedPath = resolve(dirname(dtsPath), declarationSpecifier);
if (importedPath.startsWith(`${distDir}${sep}`)) {
declarationQueue.push(importedPath);
}
}
for (const specifier of forbiddenPublicDeclarationSpecifiers) {
if (dtsContent.includes(`"${specifier}`) || dtsContent.includes(`'${specifier}`)) {
console.error(
`FORBIDDEN PUBLIC DTS SPECIFIER: dist/plugin-sdk/${entry.name} imports ${specifier}`,
`FORBIDDEN PUBLIC DTS SPECIFIER: ${relative(resolve(scriptDir, ".."), dtsPath)} imports ${specifier}`,
);
missing += 1;
}
}
}
const declarationBytes = Array.from(declarationPaths).reduce(
(total, dtsPath) => total + statSync(dtsPath).size,
0,
);
if (declarationBytes > MAX_PLUGIN_SDK_DECLARATION_BYTES) {
console.error(
`PLUGIN SDK DTS TOO LARGE: ${declarationBytes} bytes exceeds ${MAX_PLUGIN_SDK_DECLARATION_BYTES} bytes.`,
);
console.error("Keep plugin SDK declarations in the canonical unified tsdown graph.");
missing += 1;
}
if (missing > 0) {
console.error(
`\nERROR: ${missing} required plugin-sdk artifact(s) missing (named exports or subpath files).`,

View file

@ -44,6 +44,8 @@ const RUNTIME_SHIMS: Partial<Record<string, string>> = {
].join("\n"),
};
const USE_CANONICAL_DECLARATIONS = process.env.OPENCLAW_PLUGIN_SDK_CANONICAL_DTS === "1";
function isBareImportSpecifier(id: string): boolean {
if (
id === "@openclaw/llm-core" ||
@ -81,35 +83,46 @@ function copyFlatDeclarations(fromDir: string, toDir: string): void {
}
const distPluginSdkDir = path.join(process.cwd(), "dist/plugin-sdk");
const flatDeclarationTempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-sdk-dts-"));
const shouldBuildPrivateQaEntries = process.env.OPENCLAW_BUILD_PRIVATE_QA === "1";
const flatDeclarationEntrypoints = shouldBuildPrivateQaEntries
? pluginSdkEntrypoints
: publicPluginSdkEntrypoints;
const flatDeclarationEntrypointSet = new Set(flatDeclarationEntrypoints);
try {
await build({
clean: true,
config: false,
deps: { neverBundle: (id) => isBareImportSpecifier(id) },
dts: true,
entry: buildPluginSdkEntrySources(flatDeclarationEntrypoints),
failOnWarn: false,
fixedExtension: false,
format: "esm",
logLevel: "error",
outDir: flatDeclarationTempDir,
outExtensions: () => ({ js: ".js", dts: ".d.ts" }),
platform: "node",
report: false,
tsconfig: "tsconfig.plugin-sdk.dts.json",
});
if (USE_CANONICAL_DECLARATIONS) {
for (const entry of flatDeclarationEntrypoints) {
const declarationPath = path.join(distPluginSdkDir, `${entry}.d.ts`);
if (!fs.existsSync(declarationPath)) {
throw new Error(
`Missing canonical plugin SDK declaration: ${path.relative(process.cwd(), declarationPath)}`,
);
}
}
} else {
const flatDeclarationTempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-sdk-dts-"));
try {
await build({
clean: true,
config: false,
deps: { neverBundle: (id) => isBareImportSpecifier(id) },
dts: true,
entry: buildPluginSdkEntrySources(flatDeclarationEntrypoints),
failOnWarn: false,
fixedExtension: false,
format: "esm",
logLevel: "error",
outDir: flatDeclarationTempDir,
outExtensions: () => ({ js: ".js", dts: ".d.ts" }),
platform: "node",
report: false,
tsconfig: "tsconfig.plugin-sdk.dts.json",
});
removeExistingFlatDeclarations(distPluginSdkDir);
copyFlatDeclarations(flatDeclarationTempDir, distPluginSdkDir);
} finally {
fs.rmSync(flatDeclarationTempDir, { recursive: true, force: true });
removeExistingFlatDeclarations(distPluginSdkDir);
copyFlatDeclarations(flatDeclarationTempDir, distPluginSdkDir);
} finally {
fs.rmSync(flatDeclarationTempDir, { recursive: true, force: true });
}
}
// The root npm package ships flat bundled declarations under `dist/plugin-sdk`.

View file

@ -134,46 +134,6 @@ describe("resolveBuildAllStep", () => {
});
});
it("adds heap headroom for plugin-sdk dts on Windows", () => {
const step = getBuildAllStep("build:plugin-sdk:dts");
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-pnpm-runner-"));
const npmExecPath = path.join(tempDir, "pnpm.cjs");
fs.writeFileSync(npmExecPath, "console.log('pnpm');\n");
try {
const result = resolveBuildAllStep(step, {
platform: "win32",
nodeExecPath: "C:\\Program Files\\nodejs\\node.exe",
npmExecPath,
env: { FOO: "bar" },
});
expect(result).toEqual({
command: "C:\\Program Files\\nodejs\\node.exe",
args: [npmExecPath, "build:plugin-sdk:dts"],
options: {
stdio: "inherit",
env: {
FOO: "bar",
NODE_OPTIONS: "--max-old-space-size=8192",
},
shell: false,
windowsVerbatimArguments: undefined,
},
});
} finally {
fs.rmSync(tempDir, { force: true, recursive: true });
}
});
it("keeps plugin-sdk dts cache metadata aligned with declaration inputs", () => {
const step = getBuildAllStep("build:plugin-sdk:dts");
expect(step.cache?.inputs).toEqual(expect.arrayContaining(["packages/memory-host-sdk/src"]));
expect(step.cache?.inputs).toEqual(expect.arrayContaining(["npm-shrinkwrap.json"]));
expect(step.cache?.outputs).toEqual(expect.arrayContaining(["dist/plugin-sdk/packages"]));
});
it("keeps export-html build output aligned with runtime template lookup", () => {
const step = getBuildAllStep("copy-export-html-templates");
@ -237,7 +197,6 @@ describe("resolveBuildAllSteps", () => {
"runtime-postbuild",
"build-stamp",
"runtime-postbuild-stamp",
"build:plugin-sdk:dts",
"write-plugin-sdk-entry-dts",
"check-plugin-sdk-exports",
"plugins:assets:copy",
@ -251,7 +210,7 @@ describe("resolveBuildAllSteps", () => {
});
it("skips bundled tsdown declarations for runtime-only profiles", () => {
for (const profile of ["ciArtifacts", "gatewayWatch", "qaRuntime", "cliStartup"]) {
for (const profile of ["gatewayWatch", "qaRuntime", "cliStartup"]) {
const tsdown = resolveBuildAllSteps(profile).find((step) => step.label === "tsdown");
if (!tsdown) {
throw new Error(`Missing ${profile} tsdown step`);
@ -268,6 +227,17 @@ describe("resolveBuildAllSteps", () => {
}
});
it("keeps canonical declarations enabled for package artifact builds", () => {
const tsdown = resolveBuildAllSteps("ciArtifacts").find((step) => step.label === "tsdown");
if (!tsdown) {
throw new Error("Missing ciArtifacts tsdown step");
}
expect(resolveBuildAllStep(tsdown, { env: {} }).options.env).not.toHaveProperty(
"OPENCLAW_RUN_NODE_SKIP_DTS_BUILD",
);
});
it("preserves startup metadata only for profiles that regenerate it", () => {
for (const profile of ["full", "ciArtifacts", "cliStartup"]) {
const tsdown = resolveBuildAllSteps(profile).find((step) => step.label === "tsdown");
@ -410,24 +380,28 @@ describe("resolveBuildAllSteps", () => {
it("caches plugin-sdk entry declarations without restoring compiled JS", () => {
const step = getBuildAllStep("write-plugin-sdk-entry-dts");
expect(step.cache?.env).toEqual(["OPENCLAW_BUILD_PRIVATE_QA"]);
expect(step.env).toEqual({ OPENCLAW_PLUGIN_SDK_CANONICAL_DTS: "1" });
expect(step.cache?.env).toEqual([
"OPENCLAW_BUILD_PRIVATE_QA",
"OPENCLAW_PLUGIN_SDK_CANONICAL_DTS",
]);
expect(step.cache?.inputs).toEqual(
expect.arrayContaining([
"scripts/write-plugin-sdk-entry-dts.ts",
"scripts/lib/plugin-sdk-entrypoints.json",
"src/plugin-sdk",
"packages/model-catalog-core/src",
]),
);
expect(step.cache?.inputs).not.toContain("src/plugin-sdk");
expect(step.cache?.outputs).toEqual(
expect.arrayContaining([
{ path: "dist/plugin-sdk", extensions: [".d.ts"], recursive: false },
"dist/plugin-sdk/webhook-path.js",
"dist/plugin-sdk/.boundary-entry-shims.stamp",
"packages/plugin-sdk/dist/src/plugin-sdk/provider-entry.d.ts",
]),
);
expect(step.cache?.outputs).not.toContain("dist/plugin-sdk");
expect(step.cache?.outputs).not.toContainEqual(
expect.objectContaining({ path: "dist/plugin-sdk" }),
);
expect(step.cache?.restore).toBe("always");
});