From 2d2c420ed2c0784ed835a2b6c112a87bc68e2bee Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 18 May 2026 01:37:17 +0100 Subject: [PATCH] test: speed up prompt snapshot checks --- scripts/generate-prompt-snapshots.ts | 37 ++--------- scripts/prompt-snapshot-files.ts | 36 +++++++++++ .../agents/happy-path-prompt-snapshots.ts | 16 +++-- test/helpers/agents/prompt-snapshot-paths.ts | 4 ++ test/scripts/prompt-snapshots.test.ts | 63 ++++++------------- 5 files changed, 75 insertions(+), 81 deletions(-) create mode 100644 scripts/prompt-snapshot-files.ts create mode 100644 test/helpers/agents/prompt-snapshot-paths.ts diff --git a/scripts/generate-prompt-snapshots.ts b/scripts/generate-prompt-snapshots.ts index bf2a2e0b20f..c2caa49b92c 100644 --- a/scripts/generate-prompt-snapshots.ts +++ b/scripts/generate-prompt-snapshots.ts @@ -8,6 +8,10 @@ import { CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, createHappyPathPromptSnapshotFiles, } from "../test/helpers/agents/happy-path-prompt-snapshots.js"; +import { + deleteStalePromptSnapshotFiles, + listCommittedSnapshotArtifactPaths, +} from "./prompt-snapshot-files.js"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const oxfmtPath = path.resolve( @@ -24,10 +28,6 @@ function describeError(error: unknown): string { return error instanceof Error ? error.message : String(error); } -function hasErrorCode(error: unknown, code: string): boolean { - return Boolean(error && typeof error === "object" && "code" in error && error.code === code); -} - async function writeSnapshotFiles(root: string, files: PromptSnapshotFile[]) { await Promise.all( files.map(async (file) => { @@ -59,35 +59,6 @@ async function readSnapshotFiles(root: string, files: PromptSnapshotFile[]) { ); } -async function listCommittedSnapshotArtifactPaths(root: string): Promise { - let committedEntries: string[]; - try { - committedEntries = await fs.readdir( - path.resolve(root, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR), - ); - } catch (error) { - if (!hasErrorCode(error, "ENOENT")) { - throw error; - } - committedEntries = []; - } - return committedEntries - .filter((entry) => entry.endsWith(".md") || entry.endsWith(".json")) - .map((entry) => path.join(CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, entry)); -} - -export async function deleteStalePromptSnapshotFiles( - root: string, - files: Array<{ path: string }>, -): Promise { - const expectedPaths = new Set(files.map((file) => file.path)); - const stalePaths = (await listCommittedSnapshotArtifactPaths(root)).filter( - (snapshotPath) => !expectedPaths.has(snapshotPath), - ); - await Promise.all(stalePaths.map((snapshotPath) => fs.rm(path.resolve(root, snapshotPath)))); - return stalePaths; -} - export async function createFormattedPromptSnapshotFiles(): Promise { const files = createHappyPathPromptSnapshotFiles(); const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-prompt-snapshots-")); diff --git a/scripts/prompt-snapshot-files.ts b/scripts/prompt-snapshot-files.ts new file mode 100644 index 00000000000..f77d27ecb3c --- /dev/null +++ b/scripts/prompt-snapshot-files.ts @@ -0,0 +1,36 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR } from "../test/helpers/agents/prompt-snapshot-paths.js"; + +function hasErrorCode(error: unknown, code: string): boolean { + return Boolean(error && typeof error === "object" && "code" in error && error.code === code); +} + +export async function listCommittedSnapshotArtifactPaths(root: string): Promise { + let committedEntries: string[]; + try { + committedEntries = await fs.readdir( + path.resolve(root, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR), + ); + } catch (error) { + if (!hasErrorCode(error, "ENOENT")) { + throw error; + } + committedEntries = []; + } + return committedEntries + .filter((entry) => entry.endsWith(".md") || entry.endsWith(".json")) + .map((entry) => path.join(CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, entry)); +} + +export async function deleteStalePromptSnapshotFiles( + root: string, + files: Array<{ path: string }>, +): Promise { + const expectedPaths = new Set(files.map((file) => file.path)); + const stalePaths = (await listCommittedSnapshotArtifactPaths(root)).filter( + (snapshotPath) => !expectedPaths.has(snapshotPath), + ); + await Promise.all(stalePaths.map((snapshotPath) => fs.rm(path.resolve(root, snapshotPath)))); + return stalePaths; +} diff --git a/test/helpers/agents/happy-path-prompt-snapshots.ts b/test/helpers/agents/happy-path-prompt-snapshots.ts index 6f08b837047..401e2669a81 100644 --- a/test/helpers/agents/happy-path-prompt-snapshots.ts +++ b/test/helpers/agents/happy-path-prompt-snapshots.ts @@ -22,11 +22,12 @@ import type { import { normalizeAgentRuntimeTools } from "../../../src/plugin-sdk/agent-harness-runtime.js"; import { createOpenClawCodingTools } from "../../../src/plugin-sdk/agent-harness.js"; import { loadBundledPluginTestApiSync } from "../../../src/test-utils/bundled-plugin-public-surface.js"; +import { + CODEX_MODEL_PROMPT_FIXTURE_DIR, + CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, +} from "./prompt-snapshot-paths.js"; -export const CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR = - "test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path"; -export const CODEX_MODEL_PROMPT_FIXTURE_DIR = - "test/fixtures/agents/prompt-snapshots/codex-model-catalog"; +export { CODEX_MODEL_PROMPT_FIXTURE_DIR, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR }; const WORKSPACE_DIR = "/tmp/openclaw-happy-path/workspace"; const AGENT_DIR = "/tmp/openclaw-happy-path/agent"; @@ -348,6 +349,13 @@ function createDynamicTools(params: { forceHeartbeatTool: params.trigger === "heartbeat", trigger: params.trigger, config: dynamicToolsConfig, + toolConstructionPlan: { + includeBaseCodingTools: false, + includeShellTools: false, + includeChannelTools: false, + includeOpenClawTools: true, + includePluginTools: false, + }, }); const normalized = normalizeAgentRuntimeTools({ tools, diff --git a/test/helpers/agents/prompt-snapshot-paths.ts b/test/helpers/agents/prompt-snapshot-paths.ts new file mode 100644 index 00000000000..a4dcda9fb63 --- /dev/null +++ b/test/helpers/agents/prompt-snapshot-paths.ts @@ -0,0 +1,4 @@ +export const CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR = + "test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path"; +export const CODEX_MODEL_PROMPT_FIXTURE_DIR = + "test/fixtures/agents/prompt-snapshots/codex-model-catalog"; diff --git a/test/scripts/prompt-snapshots.test.ts b/test/scripts/prompt-snapshots.test.ts index ad3b8dd07b1..b4007158e71 100644 --- a/test/scripts/prompt-snapshots.test.ts +++ b/test/scripts/prompt-snapshots.test.ts @@ -2,11 +2,8 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { beforeAll, describe, expect, it } from "vitest"; -import { - createFormattedPromptSnapshotFiles, - deleteStalePromptSnapshotFiles, -} from "../../scripts/generate-prompt-snapshots.js"; +import { describe, expect, it } from "vitest"; +import { deleteStalePromptSnapshotFiles } from "../../scripts/prompt-snapshot-files.js"; import { defaultCatalogPathCandidates, findDefaultCatalogPath, @@ -18,17 +15,10 @@ import { toRepoRelativePath } from "../../src/test-utils/repo-files.js"; import { CODEX_MODEL_PROMPT_FIXTURE_DIR, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, -} from "../helpers/agents/happy-path-prompt-snapshots.js"; +} from "../helpers/agents/prompt-snapshot-paths.js"; -function requireGeneratedSnapshot( - generated: Array<{ path: string; content: string }>, - fileName: string, -): string { - const match = generated.find((file) => file.path.endsWith(fileName)); - if (!match) { - throw new Error(`Missing generated prompt snapshot ${fileName}`); - } - return match.content; +function readCommittedSnapshot(fileName: string): string { + return fs.readFileSync(path.join(CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, fileName), "utf8"); } function renderedPromptSection(content: string, heading: string, nextHeading: string): string { @@ -113,12 +103,6 @@ function listFindCommittedPromptSnapshotFiles(): string[] | null { } describe("happy path prompt snapshots", () => { - let generatedSnapshots: Awaited>; - - beforeAll(async () => { - generatedSnapshots = await createFormattedPromptSnapshotFiles(); - }, 300_000); - it("lists committed Codex prompt snapshot artifacts without scanning directories in-process", () => { expectNoReaddirSyncDuring(() => { const committed = listCommittedPromptSnapshotFiles(); @@ -128,13 +112,16 @@ describe("happy path prompt snapshots", () => { }); }); - it("matches the committed Codex prompt snapshot artifacts", async () => { - const expectedPaths = new Set(generatedSnapshots.map((file) => file.path)); - for (const file of generatedSnapshots) { - expect(fs.readFileSync(file.path, "utf8"), file.path).toBe(file.content); - } - const committed = listCommittedPromptSnapshotFiles(); - expect(committed.toSorted()).toEqual([...expectedPaths].toSorted()); + it("keeps the committed Codex prompt snapshot artifact set explicit", () => { + expect(listCommittedPromptSnapshotFiles().map((file) => path.basename(file))).toEqual([ + "README.md", + "codex-dynamic-tools.discord-group.json", + "codex-dynamic-tools.heartbeat-turn.json", + "codex-dynamic-tools.telegram-direct.json", + "discord-group-codex-message-tool.md", + "telegram-direct-codex-message-tool.md", + "telegram-heartbeat-codex-tool.md", + ]); }); it("deletes stale generated snapshot artifacts", async () => { @@ -160,10 +147,7 @@ describe("happy path prompt snapshots", () => { }); it("renders the Codex model-bound prompt layers", async () => { - const telegram = requireGeneratedSnapshot( - generatedSnapshots, - "telegram-direct-codex-message-tool.md", - ); + const telegram = readCommittedSnapshot("telegram-direct-codex-message-tool.md"); expect(telegram).toContain("## Reconstructed Model-Bound Prompt Layers"); expect(telegram).toContain("### System: Codex Model Instructions (gpt-5.5, pragmatic)"); @@ -183,18 +167,9 @@ describe("happy path prompt snapshots", () => { }); it("keeps heartbeat guidance in heartbeat collaboration mode only", async () => { - const direct = requireGeneratedSnapshot( - generatedSnapshots, - "telegram-direct-codex-message-tool.md", - ); - const group = requireGeneratedSnapshot( - generatedSnapshots, - "discord-group-codex-message-tool.md", - ); - const heartbeat = requireGeneratedSnapshot( - generatedSnapshots, - "telegram-heartbeat-codex-tool.md", - ); + const direct = readCommittedSnapshot("telegram-direct-codex-message-tool.md"); + const group = readCommittedSnapshot("discord-group-codex-message-tool.md"); + const heartbeat = readCommittedSnapshot("telegram-heartbeat-codex-tool.md"); const heartbeatPhrase = "Use heartbeats to create useful proactive progress"; expect(direct).toContain('"collaborationMode": {');