openclaw/test/scripts/dependency-vulnerability-gate.test.ts

// Dependency Vulnerability Gate tests cover dependency vulnerability gate script behavior.
import { spawnSync } from "node:child_process";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { describe, expect, it, vi } from "vitest";
import {
  main,
  renderDependencyVulnerabilityGateMarkdownReport,
  runDependencyVulnerabilityGate,
} from "../../scripts/dependency-vulnerability-gate.mts";

vi.mock("node:timers/promises", () => ({ setTimeout: vi.fn(async () => {}) }));

const releaseLocks = [
  ".github/release/clawhub-cli/package-lock.json",
  ".github/release/vercel-cli/package-lock.json",
] as const;

function requestUrl(input: string | URL | Request): string {
  return typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
}

function advisory(severity: string, title = "Fixture vulnerability") {
  return {
    id: "GHSA-fixture",
    severity,
    title,
    vulnerable_versions: "<1.0.0",
    url: "https://github.com/advisories/GHSA-fixture",
  };
}

async function writeNpmLock(
  rootDir: string,
  lockfile: string,
  packages: Record<string, unknown> = { "node_modules/fixture-tool": { version: "2.0.0" } },
) {
  await mkdir(path.dirname(path.join(rootDir, lockfile)), { recursive: true });
  if (lockfile !== "pnpm-lock.yaml") {
    await writeFile(
      path.join(rootDir, path.dirname(lockfile), "package.json"),
      '{"name":"fixture"}',
    );
  }
  await writeFile(
    path.join(rootDir, lockfile),
    JSON.stringify({
      lockfileVersion: 3,
      packages: { "": { name: "fixture", version: "1.0.0" }, ...packages },
    }),
  );
}

async function withLockfiles(run: (rootDir: string) => Promise<void>) {
  const rootDir = await mkdtemp(path.join(tmpdir(), "openclaw-vuln-gate-"));
  try {
    await writeFile(
      path.join(rootDir, "pnpm-lock.yaml"),
      `lockfileVersion: '9.0'
importers:
  .:
    dependencies:
      runtime-high:
        version: 1.0.0
    devDependencies:
      dev-high:
        version: 1.0.0
snapshots:
  runtime-high@1.0.0: {}
  dev-high@1.0.0: {}
  transitive-critical@1.0.0: {}
`,
    );
    for (const lockfile of releaseLocks) {
      await writeNpmLock(rootDir, lockfile);
    }
    await run(rootDir);
  } finally {
    await rm(rootDir, { force: true, recursive: true });
  }
}

function publishedAdvisory(range = ">= 0.8.0, < 1.0.0") {
  return {
    ghsa_id: "GHSA-2222-3333-4444",
    state: "published",
    withdrawn_at: null,
    severity: "high",
    summary: "Upstream fixture vulnerability",
    vulnerabilities: [
      { package: { ecosystem: "npm", name: "runtime-high" }, vulnerable_version_range: range },
    ],
  };
}

function withPublicUpstream(npmFetch: typeof fetch, advisories: unknown[] = []): typeof fetch {
  return async (input, init) => {
    const url = new URL(requestUrl(input));
    if (url.pathname.endsWith("/advisories/bulk")) {
      return npmFetch(input, init);
    }
    if (url.hostname === "registry.npmjs.org") {
      const [name, version] = url.pathname.slice(1).split("/").map(decodeURIComponent);
      return Response.json({ name, version, repository: "https://github.com/fixture/packages" });
    }
    return Response.json(
      url.pathname.endsWith("/security-advisories") ? advisories : { id: 42, private: false },
    );
  };
}

function requestPayload(init: RequestInit | undefined): Record<string, string[]> {
  if (typeof init?.body !== "string") {
    throw new Error("expected a JSON request body");
  }
  return JSON.parse(init.body);
}

describe("dependency-vulnerability-gate", () => {
  it("reports only target-owned graphs when release-tool manifests are absent", async () => {
    await withLockfiles(async (rootDir) => {
      for (const lockfile of releaseLocks) {
        await rm(path.join(rootDir, path.dirname(lockfile)), { force: true, recursive: true });
      }
      const report = await runDependencyVulnerabilityGate({
        rootDir,
        fetchImpl: withPublicUpstream(async () => Response.json({})),
      });

      expect(report.graphs.map(({ lockfile }) => lockfile)).toEqual(["pnpm-lock.yaml"]);
      const markdown = renderDependencyVulnerabilityGateMarkdownReport(report);
      expect(markdown).toContain(
        "the target's pnpm lock and each release-tool npm lock declared by that target",
      );
      expect(markdown).toContain("### pnpm-lock.yaml");
      for (const lockfile of releaseLocks) {
        expect(markdown).not.toContain(`### ${lockfile}`);
      }
    });
  });

  it("reports CLI argument errors without a Node stack trace", () => {
    const result = spawnSync(
      process.execPath,
      ["--import", "tsx", "scripts/dependency-vulnerability-gate.mts", "--wat"],
      {
        cwd: path.resolve("."),
        encoding: "utf8",
      },
    );
    expect(result.status).toBe(1);
    expect(result.stdout).toBe("");
    expect(result.stderr.trim()).toBe(
      "Unsupported argument: --wat\n[dependency-vulnerability-gate] FAILED (exit 1)",
    );
    expect(result.stderr).not.toContain("Node.js");
    expect(result.stderr).not.toContain("\n    at ");
  });

  it.each([false, true])(
    "keeps pnpm toolchain and production graphs separate (metadata %s)",
    async (withToolchain) => {
      await withLockfiles(async (rootDir) => {
        if (withToolchain) {
          const lockPath = path.join(rootDir, "pnpm-lock.yaml");
          await writeFile(
            lockPath,
            "---\nlockfileVersion: '9.0'\nsnapshots:\n  toolchain@1.0.0: {}\n---\n" +
              (await readFile(lockPath, "utf8")),
          );
        }
        const payloads: Record<string, string[]>[] = [];
        const report = await runDependencyVulnerabilityGate({
          rootDir,
          fetchImpl: withPublicUpstream(async (_url, init) => {
            const payload = requestPayload(init);
            payloads.push(payload);
            const advisories = {
              "runtime-high": advisory("high"),
              "dev-high": advisory("high"),
              "transitive-critical": advisory("critical"),
              toolchain: advisory("low", "Malware in toolchain"),
            };
            const body = Object.fromEntries(
              Object.entries(advisories)
                .filter(([name]) => payload[name]?.includes("1.0.0"))
                .map(([name, finding]) => [name, [{ ...finding, vulnerable_versions: "<=1.0.0" }]]),
            );
            return new Response(JSON.stringify(body));
          }),
        });
        expect(payloads.filter((payload) => "runtime-high" in payload)).toEqual([
          {
            "dev-high": ["1.0.0"],
            "runtime-high": ["1.0.0"],
            "transitive-critical": ["1.0.0"],
            ...(withToolchain ? { toolchain: ["1.0.0"] } : {}),
          },
          { "runtime-high": ["1.0.0"] },
        ]);
        expect(report.blockers.map(({ packageName }) => packageName)).toEqual([
          "transitive-critical",
          "runtime-high",
          ...(withToolchain ? ["toolchain"] : []),
        ]);
        expect(report.findings.every(({ lockfile }) => lockfile === "pnpm-lock.yaml")).toBe(true);
        expect(report.graphs).toContainEqual({
          lockfile: "pnpm-lock.yaml",
          all: { packages: withToolchain ? 4 : 3, packageVersions: withToolchain ? 4 : 3 },
          production: { packages: 1, packageVersions: 1 },
        });
      });
    },
  );

  const npmCases = [
    {
      name: "runtime high",
      location: "node_modules/runtime-high",
      metadata: {},
      severity: "high",
      graph: "production",
      blocks: true,
    },
    {
      name: "nested scoped alias",
      location: "node_modules/@scope/parent/node_modules/@scope/alias",
      metadata: { name: "runtime-high" },
      severity: "high",
      graph: "production",
      blocks: true,
    },
    {
      name: "dev-only high",
      location: "node_modules/runtime-high",
      metadata: { dev: true },
      severity: "high",
      graph: "all",
      blocks: false,
    },
    {
      name: "optional runtime high",
      location: "node_modules/runtime-high",
      metadata: { optional: true },
      severity: "high",
      graph: "production",
      blocks: true,
    },
    {
      name: "dev and optional high",
      location: "node_modules/runtime-high",
      metadata: { dev: true, optional: true },
      severity: "high",
      graph: "all",
      blocks: false,
    },
    {
      name: "shared devOptional high",
      location: "node_modules/runtime-high",
      metadata: { devOptional: true },
      severity: "high",
      graph: "production",
      blocks: true,
    },
    {
      name: "dev-only critical",
      location: "node_modules/runtime-high",
      metadata: { dev: true },
      severity: "critical",
      graph: "all",
      blocks: true,
    },
    {
      name: "dev-only malware",
      location: "node_modules/runtime-high",
      metadata: { dev: true },
      severity: "low",
      title: "Malware in dependency",
      graph: "all",
      blocks: true,
    },
    {
      name: "runtime moderate",
      location: "node_modules/runtime-high",
      metadata: {},
      severity: "moderate",
      graph: "production",
      blocks: false,
    },
  ];
  it.each(
    releaseLocks.flatMap((lockfile) =>
      npmCases.map((entry) => Object.assign({}, entry, { lockfile })),
    ),
  )(
    "$lockfile: $name stays isolated from the safe product version",
    async ({ lockfile, location, metadata, severity, title, graph, blocks }) => {
      await withLockfiles(async (rootDir) => {
        await writeNpmLock(rootDir, lockfile, { [location]: { version: "0.9.0", ...metadata } });
        const report = await runDependencyVulnerabilityGate({
          rootDir,
          fetchImpl: withPublicUpstream(
            async (_url, init) =>
              new Response(
                JSON.stringify(
                  requestPayload(init)["runtime-high"]?.includes("0.9.0")
                    ? { "runtime-high": [advisory(severity, title)] }
                    : {},
                ),
              ),
          ),
        });
        expect(report.findings).toMatchObject([
          { lockfile, packageName: "runtime-high", graph, severity },
        ]);
        expect(report.findings).toHaveLength(1);
        expect(report.blockers).toEqual(blocks ? report.findings : []);
        const markdown = renderDependencyVulnerabilityGateMarkdownReport(report);
        expect(markdown).toContain(`runtime-high (${lockfile}; ${graph})`);
        expect(markdown).not.toContain("runtime-high (pnpm-lock.yaml;");
      });
    },
  );

  it("blocks a published upstream advisory missing from npm without tainting the patched product", async () => {
    await withLockfiles(async (rootDir) => {
      await writeNpmLock(rootDir, releaseLocks[1], {
        "node_modules/runtime-high": { version: "0.9.0" },
      });
      const report = await runDependencyVulnerabilityGate({
        rootDir,
        fetchImpl: withPublicUpstream(async () => Response.json({}), [publishedAdvisory()]),
      });

      expect(report.blockers).toMatchObject([
        {
          lockfile: releaseLocks[1],
          packageName: "runtime-high",
          graph: "production",
          id: "GHSA-2222-3333-4444",
          source: "github-repository",
          matchedVersions: ["0.9.0"],
        },
      ]);
      expect(report.findings).toHaveLength(1);
    });
  });

  it.each([false, true])(
    "correlates npm GHSA IDs only inside the same graph (runtime reported %s)",
    async (runtimeReported) => {
      await withLockfiles(async (rootDir) => {
        const lockfile = path.join(rootDir, "pnpm-lock.yaml");
        await writeFile(lockfile, `${await readFile(lockfile, "utf8")}  runtime-high@0.9.0: {}\n`);
        const report = await runDependencyVulnerabilityGate({
          rootDir,
          fetchImpl: withPublicUpstream(
            async (_url, init) => {
              const versions = requestPayload(init)["runtime-high"];
              return Response.json(
                versions?.includes("0.9.0") || (runtimeReported && versions?.includes("1.0.0"))
                  ? {
                      "runtime-high": [
                        {
                          ...advisory("high"),
                          id: 123456,
                          url: "https://github.com/advisories/GHSA-2222-3333-4444",
                        },
                      ],
                    }
                  : {},
              );
            },
            [publishedAdvisory(">= 0.8.0, <= 1.0.0")],
          ),
        });
        expect(report.blockers).toHaveLength(1);
        expect(report.blockers[0]).toMatchObject({
          lockfile: "pnpm-lock.yaml",
          graph: "production",
          source: runtimeReported ? "npm-bulk" : "github-repository",
        });
        expect(report.findings).toHaveLength(runtimeReported ? 1 : 2);
      });
    },
  );

  it("writes partial coverage without claiming an unavailable upstream check passed", async () => {
    await withLockfiles(async (rootDir) => {
      const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
        const url = new URL(requestUrl(input));
        return url.pathname.endsWith("/advisories/bulk")
          ? Response.json({})
          : new Response("unavailable", { status: 503 });
      });
      const stdout = vi.spyOn(process.stdout, "write").mockReturnValue(true);
      const stderr = vi.spyOn(process.stderr, "write").mockReturnValue(true);
      try {
        const jsonPath = path.join(rootDir, "report.json");
        const markdownPath = path.join(rootDir, "report.md");
        expect(
          await main(["--root", rootDir, "--json", jsonPath, "--markdown", markdownPath]),
        ).toBe(0);
        const report = JSON.parse(await readFile(jsonPath, "utf8"));
        expect(report.coverage.upstream).toMatchObject({
          status: "partial",
          checkedRepositories: 0,
        });
        expect(report.coverage.upstream.issues).toHaveLength(4);
        expect(stdout.mock.calls.flat().join("")).toContain("public upstream partial");
        expect(stdout.mock.calls.flat().join("")).toContain(
          "not comprehensive vulnerability clearance",
        );
        expect(stderr.mock.calls.flat().join("")).toContain(
          "WARN incomplete upstream advisory coverage",
        );
        expect(await readFile(markdownPath, "utf8")).toContain("Incomplete Upstream Coverage");
      } finally {
        fetchSpy.mockRestore();
        stdout.mockRestore();
        stderr.mockRestore();
      }
    });
  });

  it("retains nested versions and findings in both release locks without auditing the root or links", async () => {
    await withLockfiles(async (rootDir) => {
      for (const lockfile of releaseLocks) {
        await writeNpmLock(rootDir, lockfile, {
          "": { name: "root-only", version: "0.9.0" },
          "node_modules/@scope/leaf": { version: "0.8.0" },
          "node_modules/parent/node_modules/@scope/leaf": { version: "0.9.0" },
          "node_modules/other/node_modules/@scope/leaf": { version: "0.9.0" },
          "node_modules/link-only": { link: true, resolved: "../local" },
        });
      }
      const payloads: Record<string, string[]>[] = [];
      const report = await runDependencyVulnerabilityGate({
        rootDir,
        fetchImpl: withPublicUpstream(async (_url, init) => {
          const payload = requestPayload(init);
          payloads.push(payload);
          return new Response(
            JSON.stringify(payload["@scope/leaf"] ? { "@scope/leaf": [advisory("high")] } : {}),
          );
        }),
      });
      expect(payloads.filter((payload) => "@scope/leaf" in payload)).toEqual(
        Array.from({ length: 4 }, () => ({ "@scope/leaf": ["0.8.0", "0.9.0"] })),
      );
      expect(
        payloads.every((payload) => !("root-only" in payload) && !("link-only" in payload)),
      ).toBe(true);
      expect(report.blockers.map(({ lockfile }) => lockfile)).toEqual(releaseLocks);
      expect(report.findings).toHaveLength(2);
      for (const lockfile of releaseLocks) {
        expect(report.graphs).toContainEqual({
          lockfile,
          all: { packages: 1, packageVersions: 2 },
          production: { packages: 1, packageVersions: 2 },
        });
      }
    });
  });

  it.each(
    releaseLocks.flatMap((lockfile) => [
      { lockfile, name: "missing", contents: null },
      { lockfile, name: "invalid JSON", contents: "{" },
      { lockfile, name: "missing packages", contents: '{"lockfileVersion":3}' },
      {
        lockfile,
        name: "empty dependency graph",
        contents: '{"lockfileVersion":3,"packages":{"":{}}}',
      },
      {
        lockfile,
        name: "invalid package entry",
        contents: '{"lockfileVersion":3,"packages":{"":{},"node_modules/broken":{}}}',
      },
    ]),
  )("rejects $name lock $lockfile before requesting advisories", async ({ lockfile, contents }) => {
    await withLockfiles(async (rootDir) => {
      if (contents === null) {
        await rm(path.join(rootDir, lockfile));
      } else {
        await writeFile(path.join(rootDir, lockfile), contents);
      }
      const fetchImpl = vi.fn(async () => new Response("{}"));
      await expect(runDependencyVulnerabilityGate({ rootDir, fetchImpl })).rejects.toThrow(
        lockfile,
      );
      expect(fetchImpl).not.toHaveBeenCalled();
    });
  });

  it.each([false, true])(
    "handles mixed release-tool advisory failures (persistent %s)",
    async (persistent) => {
      await withLockfiles(async (rootDir) => {
        await writeNpmLock(rootDir, releaseLocks[0], {
          "node_modules/tool-only": { version: "1.0.0", dev: true },
        });
        vi.stubEnv("OPENCLAW_PNPM_AUDIT_BULK_TIMEOUT_MS", "5");
        let calls = 0;
        const events: string[] = [];
        vi.mocked(delay).mockImplementation(async () => {
          events.push("wait");
        });
        try {
          const gate = runDependencyVulnerabilityGate({
            rootDir,
            fetchImpl: withPublicUpstream(async (_url, init) => {
              if (!requestPayload(init)["tool-only"]) {
                return new Response("{}");
              }
              calls += 1;
              if (calls === 4 && !persistent) {
                events.push("success");
                return new Response("{}");
              }
              if (calls % 2 === 1) {
                events.push("503");
                return new Response("unavailable", { status: 503 });
              }
              events.push("timeout");
              return await new Promise<Response>((_resolve, reject) => {
                const signal = init?.signal;
                signal?.addEventListener("abort", () => reject(signal.reason as Error), {
                  once: true,
                });
              });
            }),
          });
          if (persistent) {
            await expect(gate).rejects.toThrow(/failed after 4 attempts.*no clearance/u);
          } else {
            expect((await gate).blockers).toEqual([]);
          }
          expect(calls).toBe(4);
          expect(events).toEqual([
            "503",
            "wait",
            "timeout",
            "wait",
            "503",
            "wait",
            persistent ? "timeout" : "success",
          ]);
        } finally {
          vi.mocked(delay).mockImplementation(async () => {});
          vi.unstubAllEnvs();
        }
      });
    },
  );

  it("submits a complete release graph and propagates HTTP client failures", async () => {
    await withLockfiles(async (rootDir) => {
      const packageNames = Array.from({ length: 401 }, (_, index) => `boundary-${index}`);
      await writeNpmLock(
        rootDir,
        releaseLocks[0],
        Object.fromEntries(
          packageNames.map((name) => [`node_modules/${name}`, { version: "1.0.0", dev: true }]),
        ),
      );
      const payloads: Record<string, string[]>[] = [];
      let calls = 0;
      await expect(
        runDependencyVulnerabilityGate({
          rootDir,
          fetchImpl: withPublicUpstream(async (_url, init) => {
            const payload = requestPayload(init);
            if (!Object.keys(payload).some((name) => name.startsWith("boundary-"))) {
              return new Response("{}");
            }
            payloads.push(payload);
            calls += 1;
            return new Response("fixture forbidden", { status: 403 });
          }),
        }),
      ).rejects.toThrow("Bulk advisory request failed (403");
      expect(payloads).toEqual([Object.fromEntries(packageNames.map((name) => [name, ["1.0.0"]]))]);
      expect(calls).toBe(1);
    });
  });

  it("rejects malformed npm data before reporting release clearance", async () => {
    await withLockfiles(async (rootDir) => {
      await expect(
        runDependencyVulnerabilityGate({
          rootDir,
          fetchImpl: withPublicUpstream(async () => Response.json(null)),
        }),
      ).rejects.toThrow("Invalid bulk advisory response");
    });
  });

  it.each([false, true])(
    "writes attributed CLI artifacts and the gate exit code (tool blocker %s)",
    async (blocked) => {
      await withLockfiles(async (rootDir) => {
        const lockfile = ".github/release/vercel-cli/package-lock.json";
        await writeNpmLock(rootDir, lockfile, {
          "node_modules/runtime-high": { version: "0.9.0" },
        });
        const fetchSpy = vi
          .spyOn(globalThis, "fetch")
          .mockImplementation(
            withPublicUpstream(
              async (_url, init) =>
                new Response(
                  JSON.stringify(
                    blocked && requestPayload(init)["runtime-high"]?.includes("0.9.0")
                      ? { "runtime-high": [advisory("critical")] }
                      : {},
                  ),
                ),
            ),
          );
        const stderr = vi.spyOn(process.stderr, "write").mockReturnValue(true);
        const stdout = vi.spyOn(process.stdout, "write").mockReturnValue(true);
        try {
          const jsonPath = path.join(rootDir, "report.json");
          const markdownPath = path.join(rootDir, "report.md");
          expect(
            await main(["--root", rootDir, "--json", jsonPath, "--markdown", markdownPath]),
          ).toBe(blocked ? 1 : 0);
          const report = JSON.parse(await readFile(jsonPath, "utf8"));
          expect(report.blockers).toHaveLength(blocked ? 1 : 0);
          const markdown = await readFile(markdownPath, "utf8");
          expect(markdown).toContain("### pnpm-lock.yaml");
          expect(markdown).toContain("### .github/release/clawhub-cli/package-lock.json");
          if (blocked) {
            expect(report.blockers).toMatchObject([{ lockfile }]);
            expect(markdown).toContain(
              "runtime-high (.github/release/vercel-cli/package-lock.json; production)",
            );
            expect(stderr.mock.calls.flat().join("")).toContain(lockfile);
            expect(stdout).not.toHaveBeenCalled();
          } else {
            expect(markdown).toContain("No matching advisories returned by the checked sources.");
            expect(markdown).toContain("not comprehensive vulnerability clearance");
            expect(report.coverage).toMatchObject({
              npm: "checked",
              upstream: { status: "checked" },
            });
            expect(stdout.mock.calls.flat().join("")).toContain(
              "checked 5 resolved package versions across 3 separate lockfile graphs; 0 hard blockers",
            );
            expect(stderr).not.toHaveBeenCalled();
          }
        } finally {
          fetchSpy.mockRestore();
          stderr.mockRestore();
          stdout.mockRestore();
        }
      });
    },
  );
});