mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-10 00:11:19 +00:00
fix(postinstall): bound packaged dist scans
This commit is contained in:
Vincent Koc
parent
1ee788189a
commit
147e979713
2 changed files with 239 additions and 34 deletions
|
|
@ -8,6 +8,7 @@ import {
|
|||
closeSync,
|
||||
existsSync,
|
||||
lstatSync,
|
||||
opendirSync,
|
||||
openSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
|
|
@ -29,6 +30,7 @@ const DEFAULT_PACKAGE_ROOT = join(scriptDir, "..");
|
|||
const DISABLE_POSTINSTALL_ENV = "OPENCLAW_DISABLE_BUNDLED_PLUGIN_POSTINSTALL";
|
||||
const DISABLE_PLUGIN_REGISTRY_MIGRATION_ENV = "OPENCLAW_DISABLE_PLUGIN_REGISTRY_MIGRATION";
|
||||
const DIST_INVENTORY_PATH = "dist/postinstall-inventory.json";
|
||||
export const MAX_INSTALLED_DIST_SCAN_ENTRIES = 25_000;
|
||||
const LEGACY_PLUGIN_RUNTIME_DEPS_DIR = "plugin-runtime-deps";
|
||||
const BAILEYS_MEDIA_FILE = join("node_modules", "baileys", "lib", "Utils", "messages-media.js");
|
||||
const BAILEYS_MEDIA_HOTFIX_NEEDLE = [
|
||||
|
|
@ -107,6 +109,8 @@ const BAILEYS_MEDIA_ASYNC_CONTEXT_RE =
|
|||
/async\s+function\s+encryptedStream|encryptedStream\s*=\s*async/u;
|
||||
const NODE_COMPILE_CACHE_VERSION_DIR_RE = /^v\d+\.\d+\.\d+-/u;
|
||||
|
||||
class InstalledDistScanLimitError extends Error {}
|
||||
|
||||
function hasEnvFlag(env, key) {
|
||||
const value = env?.[key]?.trim().toLowerCase();
|
||||
return Boolean(value && value !== "0" && value !== "false" && value !== "no");
|
||||
|
|
@ -197,8 +201,57 @@ function assertSafeInstalledDistPath(relativePath, params) {
|
|||
return candidatePath;
|
||||
}
|
||||
|
||||
function createInstalledDistScanBudget(params = {}) {
|
||||
return {
|
||||
entries: 0,
|
||||
limit: params.maxDistScanEntries ?? MAX_INSTALLED_DIST_SCAN_ENTRIES,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveInstalledDistScanBudget(params = {}) {
|
||||
return params.distScanBudget ?? createInstalledDistScanBudget(params);
|
||||
}
|
||||
|
||||
function countInstalledDistScanEntry(budget) {
|
||||
budget.entries += 1;
|
||||
if (budget.entries > budget.limit) {
|
||||
throw new InstalledDistScanLimitError(
|
||||
`installed dist scan exceeded ${budget.limit} filesystem entries; refusing to scan unbounded package contents`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function* iterateInstalledDistEntries(currentDir, params = {}) {
|
||||
if (params.readdirSync) {
|
||||
yield* params.readdirSync(currentDir, { withFileTypes: true });
|
||||
return;
|
||||
}
|
||||
|
||||
const dir = opendirSync(currentDir);
|
||||
try {
|
||||
while (true) {
|
||||
const entry = dir.readSync();
|
||||
if (!entry) {
|
||||
break;
|
||||
}
|
||||
yield entry;
|
||||
}
|
||||
} finally {
|
||||
dir.closeSync();
|
||||
}
|
||||
}
|
||||
|
||||
function* iterateOptionalInstalledDistEntries(currentDir, params = {}) {
|
||||
try {
|
||||
yield* iterateInstalledDistEntries(currentDir, params);
|
||||
} catch (error) {
|
||||
if (error instanceof InstalledDistScanLimitError) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function listInstalledDistFiles(params = {}) {
|
||||
const readDir = params.readdirSync ?? readdirSync;
|
||||
const distRoot = resolveInstalledDistRoot(params);
|
||||
if (distRoot === null) {
|
||||
return [];
|
||||
|
|
@ -206,12 +259,14 @@ function listInstalledDistFiles(params = {}) {
|
|||
const packageRoot = params.packageRoot ?? DEFAULT_PACKAGE_ROOT;
|
||||
const pending = [distRoot.distDir];
|
||||
const files = [];
|
||||
const budget = resolveInstalledDistScanBudget(params);
|
||||
while (pending.length > 0) {
|
||||
const currentDir = pending.pop();
|
||||
if (!currentDir) {
|
||||
continue;
|
||||
}
|
||||
for (const entry of readDir(currentDir, { withFileTypes: true })) {
|
||||
for (const entry of iterateInstalledDistEntries(currentDir, params)) {
|
||||
countInstalledDistScanEntry(budget);
|
||||
const entryPath = join(currentDir, entry.name);
|
||||
if (entry.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
|
|
@ -236,7 +291,6 @@ function listInstalledDistFiles(params = {}) {
|
|||
}
|
||||
|
||||
function pruneEmptyDistDirectories(params = {}) {
|
||||
const readDir = params.readdirSync ?? readdirSync;
|
||||
const removeDirectory = params.rmdirSync ?? rmdirSync;
|
||||
const distRoot = resolveInstalledDistRoot(params);
|
||||
if (distRoot === null) {
|
||||
|
|
@ -244,9 +298,21 @@ function pruneEmptyDistDirectories(params = {}) {
|
|||
}
|
||||
const packageRoot = params.packageRoot ?? DEFAULT_PACKAGE_ROOT;
|
||||
const pathLstat = params.lstatSync ?? lstatSync;
|
||||
const budget = resolveInstalledDistScanBudget(params);
|
||||
|
||||
function isDirectoryEmpty(currentDir) {
|
||||
for (const entry of iterateInstalledDistEntries(currentDir, params)) {
|
||||
void entry;
|
||||
countInstalledDistScanEntry(budget);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function prune(currentDir) {
|
||||
for (const entry of readDir(currentDir, { withFileTypes: true })) {
|
||||
const childDirs = [];
|
||||
for (const entry of iterateInstalledDistEntries(currentDir, params)) {
|
||||
countInstalledDistScanEntry(budget);
|
||||
if (entry.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
`unsafe dist entry: ${normalizeRelativePath(relative(packageRoot, join(currentDir, entry.name)))}`,
|
||||
|
|
@ -255,7 +321,10 @@ function pruneEmptyDistDirectories(params = {}) {
|
|||
if (!entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
prune(join(currentDir, entry.name));
|
||||
childDirs.push(join(currentDir, entry.name));
|
||||
}
|
||||
for (const childDir of childDirs) {
|
||||
prune(childDir);
|
||||
}
|
||||
if (currentDir === distRoot.distDir) {
|
||||
return;
|
||||
|
|
@ -266,7 +335,7 @@ function pruneEmptyDistDirectories(params = {}) {
|
|||
`unsafe dist directory: ${normalizeRelativePath(relative(packageRoot, currentDir))}`,
|
||||
);
|
||||
}
|
||||
if (readDir(currentDir).length === 0) {
|
||||
if (isDirectoryEmpty(currentDir)) {
|
||||
removeDirectory(
|
||||
assertSafeInstalledDistPath(normalizeRelativePath(relative(packageRoot, currentDir)), {
|
||||
packageRoot,
|
||||
|
|
@ -285,45 +354,42 @@ function isLegacyInstalledPluginDependencyDirName(name) {
|
|||
}
|
||||
|
||||
function pruneLegacyInstalledPluginDependencyDirs(params) {
|
||||
const readDir = params.readdirSync ?? readdirSync;
|
||||
const removePath = params.rmSync ?? rmSync;
|
||||
const packageRoot = params.packageRoot ?? DEFAULT_PACKAGE_ROOT;
|
||||
const extensionsDir = join(packageRoot, "dist", "extensions");
|
||||
const budget = resolveInstalledDistScanBudget(params);
|
||||
const removed = [];
|
||||
let pluginEntries;
|
||||
try {
|
||||
pluginEntries = readDir(extensionsDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return removed;
|
||||
}
|
||||
|
||||
for (const pluginEntry of pluginEntries) {
|
||||
for (const pluginEntry of iterateOptionalInstalledDistEntries(extensionsDir, params)) {
|
||||
countInstalledDistScanEntry(budget);
|
||||
if (!pluginEntry.isDirectory() || pluginEntry.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
const pluginDir = join(extensionsDir, pluginEntry.name);
|
||||
let pluginChildren;
|
||||
try {
|
||||
pluginChildren = readDir(pluginDir, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const childEntry of pluginChildren) {
|
||||
const dependencyDirNames = [];
|
||||
for (const childEntry of iterateOptionalInstalledDistEntries(pluginDir, params)) {
|
||||
countInstalledDistScanEntry(budget);
|
||||
if (!isLegacyInstalledPluginDependencyDirName(childEntry.name)) {
|
||||
continue;
|
||||
}
|
||||
const safePluginDir = assertSafeInstalledDistPath(
|
||||
normalizeRelativePath(relative(packageRoot, pluginDir)),
|
||||
{
|
||||
packageRoot,
|
||||
distDirReal: params.distDirReal,
|
||||
realpathSync: params.realpathSync,
|
||||
},
|
||||
);
|
||||
dependencyDirNames.push(childEntry.name);
|
||||
}
|
||||
if (dependencyDirNames.length === 0) {
|
||||
continue;
|
||||
}
|
||||
const safePluginDir = assertSafeInstalledDistPath(
|
||||
normalizeRelativePath(relative(packageRoot, pluginDir)),
|
||||
{
|
||||
packageRoot,
|
||||
distDirReal: params.distDirReal,
|
||||
realpathSync: params.realpathSync,
|
||||
},
|
||||
);
|
||||
for (const dependencyDirName of dependencyDirNames) {
|
||||
const relativePath = normalizeRelativePath(
|
||||
relative(packageRoot, join(pluginDir, childEntry.name)),
|
||||
relative(packageRoot, join(pluginDir, dependencyDirName)),
|
||||
);
|
||||
removePath(join(safePluginDir, childEntry.name), { recursive: true, force: true });
|
||||
removePath(join(safePluginDir, dependencyDirName), { recursive: true, force: true });
|
||||
removed.push(relativePath);
|
||||
}
|
||||
}
|
||||
|
|
@ -502,11 +568,13 @@ export function pruneInstalledPackageDist(params = {}) {
|
|||
if (distRoot === null) {
|
||||
return [];
|
||||
}
|
||||
const distScanBudget = createInstalledDistScanBudget(params);
|
||||
const distScanParams = { ...params, distScanBudget };
|
||||
const removedLegacyDependencyDirs = pruneLegacyInstalledPluginDependencyDirs({
|
||||
...distScanParams,
|
||||
packageRoot,
|
||||
distDirReal: distRoot.distDirReal,
|
||||
realpathSync: params.realpathSync,
|
||||
readdirSync: params.readdirSync,
|
||||
rmSync: params.rmSync,
|
||||
});
|
||||
let expectedFiles = params.expectedFiles ?? null;
|
||||
|
|
@ -521,7 +589,7 @@ export function pruneInstalledPackageDist(params = {}) {
|
|||
return [];
|
||||
}
|
||||
}
|
||||
const installedFiles = listInstalledDistFiles(params);
|
||||
const installedFiles = listInstalledDistFiles(distScanParams);
|
||||
const readFile = params.readFileSync ?? readFileSync;
|
||||
expectedFiles = new Set(
|
||||
expandPackageDistImportClosure({
|
||||
|
|
@ -555,7 +623,7 @@ export function pruneInstalledPackageDist(params = {}) {
|
|||
removed.push(relativePath);
|
||||
}
|
||||
|
||||
pruneEmptyDistDirectories(params);
|
||||
pruneEmptyDistDirectories(distScanParams);
|
||||
|
||||
if (removed.length > 0) {
|
||||
log.log(`[postinstall] pruned stale dist files: ${removed.join(", ")}`);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
collectLegacyPluginRuntimeDepsStateRoots,
|
||||
isSourceCheckoutRoot,
|
||||
isDirectPostinstallInvocation,
|
||||
MAX_INSTALLED_DIST_SCAN_ENTRIES,
|
||||
pruneOpenClawCompileCache,
|
||||
pruneInstalledPackageDist,
|
||||
pruneLegacyPluginRuntimeDepsState,
|
||||
|
|
@ -953,6 +954,142 @@ describe("bundled plugin postinstall", () => {
|
|||
).toThrow("unsafe dist entry: dist/escape");
|
||||
});
|
||||
|
||||
it("rejects packaged dist scans that exceed the filesystem entry limit", () => {
|
||||
expect(() =>
|
||||
pruneInstalledPackageDist({
|
||||
packageRoot: "/pkg",
|
||||
expectedFiles: new Set(),
|
||||
existsSync: vi.fn(() => true),
|
||||
lstatSync: vi.fn(() => ({
|
||||
isDirectory: () => true,
|
||||
isSymbolicLink: () => false,
|
||||
})),
|
||||
maxDistScanEntries: 1,
|
||||
realpathSync: vi.fn((filePath) => filePath),
|
||||
readdirSync: vi.fn((filePath, options) => {
|
||||
if (filePath === "/pkg/dist" && options?.withFileTypes) {
|
||||
return [
|
||||
{
|
||||
name: "first.js",
|
||||
isDirectory: () => false,
|
||||
isFile: () => true,
|
||||
isSymbolicLink: () => false,
|
||||
},
|
||||
{
|
||||
name: "second.js",
|
||||
isDirectory: () => false,
|
||||
isFile: () => true,
|
||||
isSymbolicLink: () => false,
|
||||
},
|
||||
];
|
||||
}
|
||||
return [];
|
||||
}),
|
||||
rmSync: vi.fn(),
|
||||
log: { log: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).toThrow(
|
||||
"installed dist scan exceeded 1 filesystem entries; refusing to scan unbounded package contents",
|
||||
);
|
||||
expect(MAX_INSTALLED_DIST_SCAN_ENTRIES).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("uses one packaged dist scan budget across listing and pruning phases", () => {
|
||||
expect(() =>
|
||||
pruneInstalledPackageDist({
|
||||
packageRoot: "/pkg",
|
||||
expectedFiles: new Set(["dist/kept.js"]),
|
||||
existsSync: vi.fn(() => true),
|
||||
lstatSync: vi.fn(() => ({
|
||||
isDirectory: () => true,
|
||||
isSymbolicLink: () => false,
|
||||
})),
|
||||
maxDistScanEntries: 1,
|
||||
readFileSync: vi.fn(() => "export {};\n"),
|
||||
realpathSync: vi.fn((filePath) => filePath),
|
||||
readdirSync: vi.fn((filePath, options) => {
|
||||
if (filePath === "/pkg/dist" && options?.withFileTypes) {
|
||||
return [
|
||||
{
|
||||
name: "kept.js",
|
||||
isDirectory: () => false,
|
||||
isFile: () => true,
|
||||
isSymbolicLink: () => false,
|
||||
},
|
||||
];
|
||||
}
|
||||
return [];
|
||||
}),
|
||||
rmSync: vi.fn(),
|
||||
log: { log: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).toThrow(
|
||||
"installed dist scan exceeded 1 filesystem entries; refusing to scan unbounded package contents",
|
||||
);
|
||||
});
|
||||
|
||||
it("applies the packaged dist scan budget to legacy dependency debris prepass", () => {
|
||||
expect(() =>
|
||||
pruneInstalledPackageDist({
|
||||
packageRoot: "/pkg",
|
||||
expectedFiles: new Set(),
|
||||
existsSync: vi.fn(() => true),
|
||||
lstatSync: vi.fn(() => ({
|
||||
isDirectory: () => true,
|
||||
isSymbolicLink: () => false,
|
||||
})),
|
||||
maxDistScanEntries: 1,
|
||||
realpathSync: vi.fn((filePath) => filePath),
|
||||
readdirSync: vi.fn((filePath, options) => {
|
||||
if (filePath === "/pkg/dist/extensions" && options?.withFileTypes) {
|
||||
return [
|
||||
{
|
||||
name: "slack",
|
||||
isDirectory: () => true,
|
||||
isFile: () => false,
|
||||
isSymbolicLink: () => false,
|
||||
},
|
||||
];
|
||||
}
|
||||
if (filePath === "/pkg/dist/extensions/slack" && options?.withFileTypes) {
|
||||
return [
|
||||
{
|
||||
name: "node_modules",
|
||||
isDirectory: () => true,
|
||||
isFile: () => false,
|
||||
isSymbolicLink: () => false,
|
||||
},
|
||||
];
|
||||
}
|
||||
return [];
|
||||
}),
|
||||
rmSync: vi.fn(),
|
||||
log: { log: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).toThrow(
|
||||
"installed dist scan exceeded 1 filesystem entries; refusing to scan unbounded package contents",
|
||||
);
|
||||
});
|
||||
|
||||
it("prunes sibling empty dist directories after closing parent scans", async () => {
|
||||
const packageRoot = await createTempDirAsync("openclaw-packaged-install-empty-dirs-");
|
||||
const firstEmptyDir = path.join(packageRoot, "dist", "empty-a");
|
||||
const secondEmptyDir = path.join(packageRoot, "dist", "empty-b");
|
||||
await fs.mkdir(firstEmptyDir, { recursive: true });
|
||||
await fs.mkdir(secondEmptyDir, { recursive: true });
|
||||
|
||||
expect(
|
||||
pruneInstalledPackageDist({
|
||||
packageRoot,
|
||||
expectedFiles: new Set(),
|
||||
log: { log: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).toEqual([]);
|
||||
|
||||
await expectPathMissing(firstEmptyDir);
|
||||
await expectPathMissing(secondEmptyDir);
|
||||
});
|
||||
|
||||
it("prunes stale bundled plugin dependency debris from packaged dist", async () => {
|
||||
const packageRoot = await createTempDirAsync("openclaw-packaged-install-dist-prune-");
|
||||
const staleFile = path.join(packageRoot, "dist", "stale-runtime.js");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue