mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-19 07:42:04 +00:00
test: speed up prompt snapshot checks
This commit is contained in:
Peter Steinberger
parent
3d85e84df3
commit
2d2c420ed2
5 changed files with 75 additions and 81 deletions
|
|
@ -8,6 +8,10 @@ import {
|
|||
CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR,
|
||||
createHappyPathPromptSnapshotFiles,
|
||||
} from "../test/helpers/agents/happy-path-prompt-snapshots.js";
|
||||
import {
|
||||
deleteStalePromptSnapshotFiles,
|
||||
listCommittedSnapshotArtifactPaths,
|
||||
} from "./prompt-snapshot-files.js";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const oxfmtPath = path.resolve(
|
||||
|
|
@ -24,10 +28,6 @@ function describeError(error: unknown): string {
|
|||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function hasErrorCode(error: unknown, code: string): boolean {
|
||||
return Boolean(error && typeof error === "object" && "code" in error && error.code === code);
|
||||
}
|
||||
|
||||
async function writeSnapshotFiles(root: string, files: PromptSnapshotFile[]) {
|
||||
await Promise.all(
|
||||
files.map(async (file) => {
|
||||
|
|
@ -59,35 +59,6 @@ async function readSnapshotFiles(root: string, files: PromptSnapshotFile[]) {
|
|||
);
|
||||
}
|
||||
|
||||
async function listCommittedSnapshotArtifactPaths(root: string): Promise<string[]> {
|
||||
let committedEntries: string[];
|
||||
try {
|
||||
committedEntries = await fs.readdir(
|
||||
path.resolve(root, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR),
|
||||
);
|
||||
} catch (error) {
|
||||
if (!hasErrorCode(error, "ENOENT")) {
|
||||
throw error;
|
||||
}
|
||||
committedEntries = [];
|
||||
}
|
||||
return committedEntries
|
||||
.filter((entry) => entry.endsWith(".md") || entry.endsWith(".json"))
|
||||
.map((entry) => path.join(CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, entry));
|
||||
}
|
||||
|
||||
export async function deleteStalePromptSnapshotFiles(
|
||||
root: string,
|
||||
files: Array<{ path: string }>,
|
||||
): Promise<string[]> {
|
||||
const expectedPaths = new Set(files.map((file) => file.path));
|
||||
const stalePaths = (await listCommittedSnapshotArtifactPaths(root)).filter(
|
||||
(snapshotPath) => !expectedPaths.has(snapshotPath),
|
||||
);
|
||||
await Promise.all(stalePaths.map((snapshotPath) => fs.rm(path.resolve(root, snapshotPath))));
|
||||
return stalePaths;
|
||||
}
|
||||
|
||||
export async function createFormattedPromptSnapshotFiles(): Promise<PromptSnapshotFile[]> {
|
||||
const files = createHappyPathPromptSnapshotFiles();
|
||||
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-prompt-snapshots-"));
|
||||
|
|
|
|||
36
scripts/prompt-snapshot-files.ts
Normal file
36
scripts/prompt-snapshot-files.ts
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR } from "../test/helpers/agents/prompt-snapshot-paths.js";
|
||||
|
||||
function hasErrorCode(error: unknown, code: string): boolean {
|
||||
return Boolean(error && typeof error === "object" && "code" in error && error.code === code);
|
||||
}
|
||||
|
||||
export async function listCommittedSnapshotArtifactPaths(root: string): Promise<string[]> {
|
||||
let committedEntries: string[];
|
||||
try {
|
||||
committedEntries = await fs.readdir(
|
||||
path.resolve(root, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR),
|
||||
);
|
||||
} catch (error) {
|
||||
if (!hasErrorCode(error, "ENOENT")) {
|
||||
throw error;
|
||||
}
|
||||
committedEntries = [];
|
||||
}
|
||||
return committedEntries
|
||||
.filter((entry) => entry.endsWith(".md") || entry.endsWith(".json"))
|
||||
.map((entry) => path.join(CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, entry));
|
||||
}
|
||||
|
||||
export async function deleteStalePromptSnapshotFiles(
|
||||
root: string,
|
||||
files: Array<{ path: string }>,
|
||||
): Promise<string[]> {
|
||||
const expectedPaths = new Set(files.map((file) => file.path));
|
||||
const stalePaths = (await listCommittedSnapshotArtifactPaths(root)).filter(
|
||||
(snapshotPath) => !expectedPaths.has(snapshotPath),
|
||||
);
|
||||
await Promise.all(stalePaths.map((snapshotPath) => fs.rm(path.resolve(root, snapshotPath))));
|
||||
return stalePaths;
|
||||
}
|
||||
|
|
@ -22,11 +22,12 @@ import type {
|
|||
import { normalizeAgentRuntimeTools } from "../../../src/plugin-sdk/agent-harness-runtime.js";
|
||||
import { createOpenClawCodingTools } from "../../../src/plugin-sdk/agent-harness.js";
|
||||
import { loadBundledPluginTestApiSync } from "../../../src/test-utils/bundled-plugin-public-surface.js";
|
||||
import {
|
||||
CODEX_MODEL_PROMPT_FIXTURE_DIR,
|
||||
CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR,
|
||||
} from "./prompt-snapshot-paths.js";
|
||||
|
||||
export const CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR =
|
||||
"test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path";
|
||||
export const CODEX_MODEL_PROMPT_FIXTURE_DIR =
|
||||
"test/fixtures/agents/prompt-snapshots/codex-model-catalog";
|
||||
export { CODEX_MODEL_PROMPT_FIXTURE_DIR, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR };
|
||||
|
||||
const WORKSPACE_DIR = "/tmp/openclaw-happy-path/workspace";
|
||||
const AGENT_DIR = "/tmp/openclaw-happy-path/agent";
|
||||
|
|
@ -348,6 +349,13 @@ function createDynamicTools(params: {
|
|||
forceHeartbeatTool: params.trigger === "heartbeat",
|
||||
trigger: params.trigger,
|
||||
config: dynamicToolsConfig,
|
||||
toolConstructionPlan: {
|
||||
includeBaseCodingTools: false,
|
||||
includeShellTools: false,
|
||||
includeChannelTools: false,
|
||||
includeOpenClawTools: true,
|
||||
includePluginTools: false,
|
||||
},
|
||||
});
|
||||
const normalized = normalizeAgentRuntimeTools({
|
||||
tools,
|
||||
|
|
|
|||
4
test/helpers/agents/prompt-snapshot-paths.ts
Normal file
4
test/helpers/agents/prompt-snapshot-paths.ts
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
export const CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR =
|
||||
"test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path";
|
||||
export const CODEX_MODEL_PROMPT_FIXTURE_DIR =
|
||||
"test/fixtures/agents/prompt-snapshots/codex-model-catalog";
|
||||
|
|
@ -2,11 +2,8 @@ import { spawnSync } from "node:child_process";
|
|||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
createFormattedPromptSnapshotFiles,
|
||||
deleteStalePromptSnapshotFiles,
|
||||
} from "../../scripts/generate-prompt-snapshots.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { deleteStalePromptSnapshotFiles } from "../../scripts/prompt-snapshot-files.js";
|
||||
import {
|
||||
defaultCatalogPathCandidates,
|
||||
findDefaultCatalogPath,
|
||||
|
|
@ -18,17 +15,10 @@ import { toRepoRelativePath } from "../../src/test-utils/repo-files.js";
|
|||
import {
|
||||
CODEX_MODEL_PROMPT_FIXTURE_DIR,
|
||||
CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR,
|
||||
} from "../helpers/agents/happy-path-prompt-snapshots.js";
|
||||
} from "../helpers/agents/prompt-snapshot-paths.js";
|
||||
|
||||
function requireGeneratedSnapshot(
|
||||
generated: Array<{ path: string; content: string }>,
|
||||
fileName: string,
|
||||
): string {
|
||||
const match = generated.find((file) => file.path.endsWith(fileName));
|
||||
if (!match) {
|
||||
throw new Error(`Missing generated prompt snapshot ${fileName}`);
|
||||
}
|
||||
return match.content;
|
||||
function readCommittedSnapshot(fileName: string): string {
|
||||
return fs.readFileSync(path.join(CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, fileName), "utf8");
|
||||
}
|
||||
|
||||
function renderedPromptSection(content: string, heading: string, nextHeading: string): string {
|
||||
|
|
@ -113,12 +103,6 @@ function listFindCommittedPromptSnapshotFiles(): string[] | null {
|
|||
}
|
||||
|
||||
describe("happy path prompt snapshots", () => {
|
||||
let generatedSnapshots: Awaited<ReturnType<typeof createFormattedPromptSnapshotFiles>>;
|
||||
|
||||
beforeAll(async () => {
|
||||
generatedSnapshots = await createFormattedPromptSnapshotFiles();
|
||||
}, 300_000);
|
||||
|
||||
it("lists committed Codex prompt snapshot artifacts without scanning directories in-process", () => {
|
||||
expectNoReaddirSyncDuring(() => {
|
||||
const committed = listCommittedPromptSnapshotFiles();
|
||||
|
|
@ -128,13 +112,16 @@ describe("happy path prompt snapshots", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("matches the committed Codex prompt snapshot artifacts", async () => {
|
||||
const expectedPaths = new Set(generatedSnapshots.map((file) => file.path));
|
||||
for (const file of generatedSnapshots) {
|
||||
expect(fs.readFileSync(file.path, "utf8"), file.path).toBe(file.content);
|
||||
}
|
||||
const committed = listCommittedPromptSnapshotFiles();
|
||||
expect(committed.toSorted()).toEqual([...expectedPaths].toSorted());
|
||||
it("keeps the committed Codex prompt snapshot artifact set explicit", () => {
|
||||
expect(listCommittedPromptSnapshotFiles().map((file) => path.basename(file))).toEqual([
|
||||
"README.md",
|
||||
"codex-dynamic-tools.discord-group.json",
|
||||
"codex-dynamic-tools.heartbeat-turn.json",
|
||||
"codex-dynamic-tools.telegram-direct.json",
|
||||
"discord-group-codex-message-tool.md",
|
||||
"telegram-direct-codex-message-tool.md",
|
||||
"telegram-heartbeat-codex-tool.md",
|
||||
]);
|
||||
});
|
||||
|
||||
it("deletes stale generated snapshot artifacts", async () => {
|
||||
|
|
@ -160,10 +147,7 @@ describe("happy path prompt snapshots", () => {
|
|||
});
|
||||
|
||||
it("renders the Codex model-bound prompt layers", async () => {
|
||||
const telegram = requireGeneratedSnapshot(
|
||||
generatedSnapshots,
|
||||
"telegram-direct-codex-message-tool.md",
|
||||
);
|
||||
const telegram = readCommittedSnapshot("telegram-direct-codex-message-tool.md");
|
||||
|
||||
expect(telegram).toContain("## Reconstructed Model-Bound Prompt Layers");
|
||||
expect(telegram).toContain("### System: Codex Model Instructions (gpt-5.5, pragmatic)");
|
||||
|
|
@ -183,18 +167,9 @@ describe("happy path prompt snapshots", () => {
|
|||
});
|
||||
|
||||
it("keeps heartbeat guidance in heartbeat collaboration mode only", async () => {
|
||||
const direct = requireGeneratedSnapshot(
|
||||
generatedSnapshots,
|
||||
"telegram-direct-codex-message-tool.md",
|
||||
);
|
||||
const group = requireGeneratedSnapshot(
|
||||
generatedSnapshots,
|
||||
"discord-group-codex-message-tool.md",
|
||||
);
|
||||
const heartbeat = requireGeneratedSnapshot(
|
||||
generatedSnapshots,
|
||||
"telegram-heartbeat-codex-tool.md",
|
||||
);
|
||||
const direct = readCommittedSnapshot("telegram-direct-codex-message-tool.md");
|
||||
const group = readCommittedSnapshot("discord-group-codex-message-tool.md");
|
||||
const heartbeat = readCommittedSnapshot("telegram-heartbeat-codex-tool.md");
|
||||
const heartbeatPhrase = "Use heartbeats to create useful proactive progress";
|
||||
|
||||
expect(direct).toContain('"collaborationMode": {');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue