From d32c4cf3007fab55cbf7fbdd7c082e4a175a2cb7 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 2 Jul 2026 00:21:41 -0700 Subject: [PATCH] perf(build): canonicalize plugin SDK declarations (#98758) --- .github/workflows/ci.yml | 2 - package.json | 4 +- scripts/build-all.mjs | 58 +++-------------------- scripts/check-plugin-sdk-exports.mjs | 54 +++++++++++++++++---- scripts/write-plugin-sdk-entry-dts.ts | 57 +++++++++++++--------- test/scripts/build-all.test.ts | 68 +++++++++------------------ 6 files changed, 109 insertions(+), 134 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 625da2013dc..ae1f6fb798d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1181,8 +1181,6 @@ jobs: pnpm lint:auth:no-pairing-store-group pnpm lint:auth:pairing-account-scope pnpm check:import-cycles - # build-artifacts already runs the tsdown/runtime build for the same Node-relevant changes. - NODE_OPTIONS=--max-old-space-size=8192 pnpm build:plugin-sdk:strict-smoke ;; shrinkwrap) pnpm deps:shrinkwrap:check diff --git a/package.json b/package.json index f44e0b299c3..5892d93f1e0 100644 --- a/package.json +++ b/package.json @@ -1514,8 +1514,8 @@ "build:ci-artifacts": "node scripts/build-all.mjs ciArtifacts", "build:docker": "node scripts/tsdown-build.mjs && node scripts/check-cli-bootstrap-imports.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && node scripts/runtime-postbuild-stamp.mjs && pnpm plugins:assets:build && pnpm plugins:assets:copy && node --experimental-strip-types scripts/copy-hook-metadata.ts && node --experimental-strip-types scripts/copy-export-html-templates.ts && node --experimental-strip-types scripts/write-build-info.ts && node --experimental-strip-types scripts/write-cli-startup-metadata.ts && node --experimental-strip-types scripts/write-cli-compat.ts", "build:plugin-sdk:dts": "node scripts/run-tsgo.mjs -p tsconfig.plugin-sdk.dts.json --declaration true", - "build:plugin-sdk:strict-smoke": "pnpm build:plugin-sdk:dts && node --experimental-strip-types scripts/write-plugin-sdk-entry-dts.ts", - "build:strict-smoke": "pnpm plugins:assets:build && node scripts/tsdown-build.mjs && node scripts/check-cli-bootstrap-imports.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && node scripts/runtime-postbuild-stamp.mjs && pnpm build:plugin-sdk:dts && node --experimental-strip-types scripts/write-plugin-sdk-entry-dts.ts && node scripts/check-plugin-sdk-exports.mjs", + "build:plugin-sdk:strict-smoke": "node scripts/tsdown-build.mjs && node scripts/run-with-env.mjs OPENCLAW_PLUGIN_SDK_CANONICAL_DTS=1 -- node --experimental-strip-types scripts/write-plugin-sdk-entry-dts.ts && node scripts/check-plugin-sdk-exports.mjs", + "build:strict-smoke": "pnpm plugins:assets:build && node scripts/tsdown-build.mjs && node scripts/check-cli-bootstrap-imports.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && node scripts/runtime-postbuild-stamp.mjs && node scripts/run-with-env.mjs OPENCLAW_PLUGIN_SDK_CANONICAL_DTS=1 -- node --experimental-strip-types scripts/write-plugin-sdk-entry-dts.ts && node scripts/check-plugin-sdk-exports.mjs", "canvas:a2ui:bundle": "node scripts/bundle-a2ui.mjs", "canvas:a2ui:native:check": "node scripts/sync-native-a2ui.mjs --check", "canvas:a2ui:native:sync": "node scripts/sync-native-a2ui.mjs --write", diff --git a/scripts/build-all.mjs b/scripts/build-all.mjs index 905e59e8fea..68e71f4643c 100644 --- a/scripts/build-all.mjs +++ b/scripts/build-all.mjs @@ -11,42 +11,11 @@ import { pluginSdkEntrypoints } from "./lib/plugin-sdk-entries.mjs"; import { resolvePnpmRunner } from "./pnpm-runner.mjs"; const nodeBin = process.execPath; -const WINDOWS_BUILD_MAX_OLD_SPACE_MB = 8192; const BUILD_CACHE_VERSION = 3; -const PLUGIN_SDK_DTS_CACHE_INPUTS = [ - "package.json", - "pnpm-lock.yaml", - "npm-shrinkwrap.json", - "packages/plugin-sdk/package.json", - "packages/llm-core/package.json", - "packages/markdown-core/package.json", - "packages/media-core/package.json", - "packages/media-understanding-common/package.json", - "packages/terminal-core/package.json", - "packages/acp-core/package.json", - "packages/model-catalog-core/package.json", - "packages/normalization-core/package.json", - "packages/web-content-core/package.json", - "packages/memory-host-sdk/package.json", - "tsconfig.json", - "tsconfig.plugin-sdk.dts.json", - "src/plugin-sdk", - "packages/llm-core/src", - "packages/markdown-core/src", - "packages/media-core/src", - "packages/media-generation-core/src", - "packages/model-catalog-core/src", - "packages/memory-host-sdk/src", - "packages/normalization-core/src", - "packages/acp-core/src", - "packages/media-understanding-common/src", - "packages/terminal-core/src", - "packages/web-content-core/src", - "src/types", - "src/video-generation/dashscope-compatible.ts", - "src/video-generation/types.ts", +const PLUGIN_SDK_ENTRY_DTS_CACHE_ENV = [ + "OPENCLAW_BUILD_PRIVATE_QA", + "OPENCLAW_PLUGIN_SDK_CANONICAL_DTS", ]; -const PLUGIN_SDK_ENTRY_DTS_CACHE_ENV = ["OPENCLAW_BUILD_PRIVATE_QA"]; const PLUGIN_SDK_ENTRY_DTS_CACHE_INPUTS = [ "scripts/write-plugin-sdk-entry-dts.ts", "scripts/lib/plugin-sdk-entries.mjs", @@ -54,20 +23,14 @@ const PLUGIN_SDK_ENTRY_DTS_CACHE_INPUTS = [ "scripts/lib/plugin-sdk-private-local-only-subpaths.json", "scripts/lib/plugin-sdk-deprecated-public-subpaths.json", "scripts/lib/plugin-sdk-deprecated-barrel-subpaths.json", - ...PLUGIN_SDK_DTS_CACHE_INPUTS, ]; const PLUGIN_SDK_ENTRY_DTS_CACHE_OUTPUTS = [ - { path: "dist/plugin-sdk", extensions: [".d.ts"], recursive: false }, "dist/plugin-sdk/webhook-path.js", "dist/plugin-sdk/.boundary-entry-shims.stamp", ...pluginSdkEntrypoints.map((entry) => `packages/plugin-sdk/dist/src/plugin-sdk/${entry}.d.ts`), ]; const PNPM_STEP_NODE_FALLBACKS = new Map([ ["plugins:assets:build", ["scripts/bundled-plugin-assets.mjs", "--phase", "build"]], - [ - "build:plugin-sdk:dts", - ["scripts/run-tsgo.mjs", "-p", "tsconfig.plugin-sdk.dts.json", "--declaration", "true"], - ], ["plugins:assets:copy", ["scripts/bundled-plugin-assets.mjs", "--phase", "copy"]], ["ui:build", ["scripts/ui.js", "build"]], ]); @@ -86,20 +49,13 @@ export const BUILD_ALL_STEPS = [ kind: "node", args: ["scripts/runtime-postbuild-stamp.mjs"], }, - { - label: "build:plugin-sdk:dts", - kind: "pnpm", - pnpmArgs: ["build:plugin-sdk:dts"], - windowsNodeOptions: `--max-old-space-size=${WINDOWS_BUILD_MAX_OLD_SPACE_MB}`, - cache: { - inputs: PLUGIN_SDK_DTS_CACHE_INPUTS, - outputs: ["dist/plugin-sdk/.tsbuildinfo", "dist/plugin-sdk/packages", "dist/plugin-sdk/src"], - }, - }, { label: "write-plugin-sdk-entry-dts", kind: "node", args: ["--experimental-strip-types", "scripts/write-plugin-sdk-entry-dts.ts"], + env: { + OPENCLAW_PLUGIN_SDK_CANONICAL_DTS: "1", + }, cache: { env: PLUGIN_SDK_ENTRY_DTS_CACHE_ENV, inputs: PLUGIN_SDK_ENTRY_DTS_CACHE_INPUTS, @@ -171,7 +127,6 @@ export const BUILD_ALL_PROFILES = { "runtime-postbuild", "build-stamp", "runtime-postbuild-stamp", - "build:plugin-sdk:dts", "write-plugin-sdk-entry-dts", "check-plugin-sdk-exports", "plugins:assets:copy", @@ -216,7 +171,6 @@ export const BUILD_ALL_PROFILE_STEP_ENV = { }, ciArtifacts: { tsdown: { - OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1", OPENCLAW_PRESERVE_CLI_STARTUP_METADATA: "1", }, }, diff --git a/scripts/check-plugin-sdk-exports.mjs b/scripts/check-plugin-sdk-exports.mjs index 90b28c99eef..a8041172bc9 100755 --- a/scripts/check-plugin-sdk-exports.mjs +++ b/scripts/check-plugin-sdk-exports.mjs @@ -8,10 +8,10 @@ * aliases before release. */ -import { readFileSync, existsSync, readdirSync } from "node:fs"; -import { resolve, dirname } from "node:path"; +import { readFileSync, existsSync, statSync } from "node:fs"; +import { resolve, dirname, relative, sep } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { publicPluginSdkSubpaths } from "./lib/plugin-sdk-entries.mjs"; +import { publicPluginSdkEntrypoints, publicPluginSdkSubpaths } from "./lib/plugin-sdk-entries.mjs"; const scriptDir = dirname(fileURLToPath(import.meta.url)); const distFile = resolve(scriptDir, "..", "dist", "plugin-sdk", "index.js"); @@ -43,6 +43,8 @@ const exportSet = new Set(exportedNames); const requiredRuntimeShimEntries = ["compat.js", "root-alias.cjs"]; const forbiddenPublicDeclarationSpecifiers = ["@openclaw/llm-core"]; +const MAX_PLUGIN_SDK_DECLARATION_BYTES = 5_000_000; +const RELATIVE_DECLARATION_SPECIFIER_RE = /\b(?:from|import)\s*(?:\(\s*)?["']([^"']+)["']/gu; const requiredSubpathExports = { "secret-input-runtime": [ "coerceSecretRef", @@ -114,24 +116,58 @@ for (const [entry, names] of Object.entries(requiredSubpathExports)) { } } -for (const entry of readdirSync(resolve(scriptDir, "..", "dist", "plugin-sdk"), { - withFileTypes: true, -})) { - if (!entry.isFile() || !entry.name.endsWith(".d.ts")) { +const distDir = resolve(scriptDir, "..", "dist"); +const declarationPaths = new Set(); +const declarationQueue = publicPluginSdkEntrypoints.map((entry) => + resolve(distDir, "plugin-sdk", `${entry}.d.ts`), +); +while (declarationQueue.length > 0) { + const dtsPath = declarationQueue.pop(); + if (!dtsPath || declarationPaths.has(dtsPath)) { continue; } - const dtsPath = resolve(scriptDir, "..", "dist", "plugin-sdk", entry.name); + if (!existsSync(dtsPath)) { + console.error(`MISSING PUBLIC DTS DEPENDENCY: ${relative(resolve(scriptDir, ".."), dtsPath)}`); + missing += 1; + continue; + } + declarationPaths.add(dtsPath); const dtsContent = readFileSync(dtsPath, "utf8"); + for (const match of dtsContent.matchAll(RELATIVE_DECLARATION_SPECIFIER_RE)) { + const specifier = match[1]; + if (!specifier?.startsWith(".")) { + continue; + } + const declarationSpecifier = specifier.endsWith(".js") + ? `${specifier.slice(0, -3)}.d.ts` + : `${specifier}.d.ts`; + const importedPath = resolve(dirname(dtsPath), declarationSpecifier); + if (importedPath.startsWith(`${distDir}${sep}`)) { + declarationQueue.push(importedPath); + } + } for (const specifier of forbiddenPublicDeclarationSpecifiers) { if (dtsContent.includes(`"${specifier}`) || dtsContent.includes(`'${specifier}`)) { console.error( - `FORBIDDEN PUBLIC DTS SPECIFIER: dist/plugin-sdk/${entry.name} imports ${specifier}`, + `FORBIDDEN PUBLIC DTS SPECIFIER: ${relative(resolve(scriptDir, ".."), dtsPath)} imports ${specifier}`, ); missing += 1; } } } +const declarationBytes = Array.from(declarationPaths).reduce( + (total, dtsPath) => total + statSync(dtsPath).size, + 0, +); +if (declarationBytes > MAX_PLUGIN_SDK_DECLARATION_BYTES) { + console.error( + `PLUGIN SDK DTS TOO LARGE: ${declarationBytes} bytes exceeds ${MAX_PLUGIN_SDK_DECLARATION_BYTES} bytes.`, + ); + console.error("Keep plugin SDK declarations in the canonical unified tsdown graph."); + missing += 1; +} + if (missing > 0) { console.error( `\nERROR: ${missing} required plugin-sdk artifact(s) missing (named exports or subpath files).`, diff --git a/scripts/write-plugin-sdk-entry-dts.ts b/scripts/write-plugin-sdk-entry-dts.ts index 14148c87878..2a1624afac9 100644 --- a/scripts/write-plugin-sdk-entry-dts.ts +++ b/scripts/write-plugin-sdk-entry-dts.ts @@ -44,6 +44,8 @@ const RUNTIME_SHIMS: Partial> = { ].join("\n"), }; +const USE_CANONICAL_DECLARATIONS = process.env.OPENCLAW_PLUGIN_SDK_CANONICAL_DTS === "1"; + function isBareImportSpecifier(id: string): boolean { if ( id === "@openclaw/llm-core" || @@ -81,35 +83,46 @@ function copyFlatDeclarations(fromDir: string, toDir: string): void { } const distPluginSdkDir = path.join(process.cwd(), "dist/plugin-sdk"); -const flatDeclarationTempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-sdk-dts-")); const shouldBuildPrivateQaEntries = process.env.OPENCLAW_BUILD_PRIVATE_QA === "1"; const flatDeclarationEntrypoints = shouldBuildPrivateQaEntries ? pluginSdkEntrypoints : publicPluginSdkEntrypoints; const flatDeclarationEntrypointSet = new Set(flatDeclarationEntrypoints); -try { - await build({ - clean: true, - config: false, - deps: { neverBundle: (id) => isBareImportSpecifier(id) }, - dts: true, - entry: buildPluginSdkEntrySources(flatDeclarationEntrypoints), - failOnWarn: false, - fixedExtension: false, - format: "esm", - logLevel: "error", - outDir: flatDeclarationTempDir, - outExtensions: () => ({ js: ".js", dts: ".d.ts" }), - platform: "node", - report: false, - tsconfig: "tsconfig.plugin-sdk.dts.json", - }); +if (USE_CANONICAL_DECLARATIONS) { + for (const entry of flatDeclarationEntrypoints) { + const declarationPath = path.join(distPluginSdkDir, `${entry}.d.ts`); + if (!fs.existsSync(declarationPath)) { + throw new Error( + `Missing canonical plugin SDK declaration: ${path.relative(process.cwd(), declarationPath)}`, + ); + } + } +} else { + const flatDeclarationTempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-sdk-dts-")); + try { + await build({ + clean: true, + config: false, + deps: { neverBundle: (id) => isBareImportSpecifier(id) }, + dts: true, + entry: buildPluginSdkEntrySources(flatDeclarationEntrypoints), + failOnWarn: false, + fixedExtension: false, + format: "esm", + logLevel: "error", + outDir: flatDeclarationTempDir, + outExtensions: () => ({ js: ".js", dts: ".d.ts" }), + platform: "node", + report: false, + tsconfig: "tsconfig.plugin-sdk.dts.json", + }); - removeExistingFlatDeclarations(distPluginSdkDir); - copyFlatDeclarations(flatDeclarationTempDir, distPluginSdkDir); -} finally { - fs.rmSync(flatDeclarationTempDir, { recursive: true, force: true }); + removeExistingFlatDeclarations(distPluginSdkDir); + copyFlatDeclarations(flatDeclarationTempDir, distPluginSdkDir); + } finally { + fs.rmSync(flatDeclarationTempDir, { recursive: true, force: true }); + } } // The root npm package ships flat bundled declarations under `dist/plugin-sdk`. diff --git a/test/scripts/build-all.test.ts b/test/scripts/build-all.test.ts index 1f314c4d365..b04a2eda1dc 100644 --- a/test/scripts/build-all.test.ts +++ b/test/scripts/build-all.test.ts @@ -134,46 +134,6 @@ describe("resolveBuildAllStep", () => { }); }); - it("adds heap headroom for plugin-sdk dts on Windows", () => { - const step = getBuildAllStep("build:plugin-sdk:dts"); - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-pnpm-runner-")); - const npmExecPath = path.join(tempDir, "pnpm.cjs"); - fs.writeFileSync(npmExecPath, "console.log('pnpm');\n"); - - try { - const result = resolveBuildAllStep(step, { - platform: "win32", - nodeExecPath: "C:\\Program Files\\nodejs\\node.exe", - npmExecPath, - env: { FOO: "bar" }, - }); - - expect(result).toEqual({ - command: "C:\\Program Files\\nodejs\\node.exe", - args: [npmExecPath, "build:plugin-sdk:dts"], - options: { - stdio: "inherit", - env: { - FOO: "bar", - NODE_OPTIONS: "--max-old-space-size=8192", - }, - shell: false, - windowsVerbatimArguments: undefined, - }, - }); - } finally { - fs.rmSync(tempDir, { force: true, recursive: true }); - } - }); - - it("keeps plugin-sdk dts cache metadata aligned with declaration inputs", () => { - const step = getBuildAllStep("build:plugin-sdk:dts"); - - expect(step.cache?.inputs).toEqual(expect.arrayContaining(["packages/memory-host-sdk/src"])); - expect(step.cache?.inputs).toEqual(expect.arrayContaining(["npm-shrinkwrap.json"])); - expect(step.cache?.outputs).toEqual(expect.arrayContaining(["dist/plugin-sdk/packages"])); - }); - it("keeps export-html build output aligned with runtime template lookup", () => { const step = getBuildAllStep("copy-export-html-templates"); @@ -237,7 +197,6 @@ describe("resolveBuildAllSteps", () => { "runtime-postbuild", "build-stamp", "runtime-postbuild-stamp", - "build:plugin-sdk:dts", "write-plugin-sdk-entry-dts", "check-plugin-sdk-exports", "plugins:assets:copy", @@ -251,7 +210,7 @@ describe("resolveBuildAllSteps", () => { }); it("skips bundled tsdown declarations for runtime-only profiles", () => { - for (const profile of ["ciArtifacts", "gatewayWatch", "qaRuntime", "cliStartup"]) { + for (const profile of ["gatewayWatch", "qaRuntime", "cliStartup"]) { const tsdown = resolveBuildAllSteps(profile).find((step) => step.label === "tsdown"); if (!tsdown) { throw new Error(`Missing ${profile} tsdown step`); @@ -268,6 +227,17 @@ describe("resolveBuildAllSteps", () => { } }); + it("keeps canonical declarations enabled for package artifact builds", () => { + const tsdown = resolveBuildAllSteps("ciArtifacts").find((step) => step.label === "tsdown"); + if (!tsdown) { + throw new Error("Missing ciArtifacts tsdown step"); + } + + expect(resolveBuildAllStep(tsdown, { env: {} }).options.env).not.toHaveProperty( + "OPENCLAW_RUN_NODE_SKIP_DTS_BUILD", + ); + }); + it("preserves startup metadata only for profiles that regenerate it", () => { for (const profile of ["full", "ciArtifacts", "cliStartup"]) { const tsdown = resolveBuildAllSteps(profile).find((step) => step.label === "tsdown"); @@ -410,24 +380,28 @@ describe("resolveBuildAllSteps", () => { it("caches plugin-sdk entry declarations without restoring compiled JS", () => { const step = getBuildAllStep("write-plugin-sdk-entry-dts"); - expect(step.cache?.env).toEqual(["OPENCLAW_BUILD_PRIVATE_QA"]); + expect(step.env).toEqual({ OPENCLAW_PLUGIN_SDK_CANONICAL_DTS: "1" }); + expect(step.cache?.env).toEqual([ + "OPENCLAW_BUILD_PRIVATE_QA", + "OPENCLAW_PLUGIN_SDK_CANONICAL_DTS", + ]); expect(step.cache?.inputs).toEqual( expect.arrayContaining([ "scripts/write-plugin-sdk-entry-dts.ts", "scripts/lib/plugin-sdk-entrypoints.json", - "src/plugin-sdk", - "packages/model-catalog-core/src", ]), ); + expect(step.cache?.inputs).not.toContain("src/plugin-sdk"); expect(step.cache?.outputs).toEqual( expect.arrayContaining([ - { path: "dist/plugin-sdk", extensions: [".d.ts"], recursive: false }, "dist/plugin-sdk/webhook-path.js", "dist/plugin-sdk/.boundary-entry-shims.stamp", "packages/plugin-sdk/dist/src/plugin-sdk/provider-entry.d.ts", ]), ); - expect(step.cache?.outputs).not.toContain("dist/plugin-sdk"); + expect(step.cache?.outputs).not.toContainEqual( + expect.objectContaining({ path: "dist/plugin-sdk" }), + ); expect(step.cache?.restore).toBe("always"); });