From 3aab4de286861a3a7b7beaceacd33a87c05d9793 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 29 Sep 2026 20:26:21 -0700 Subject: [PATCH] refactor(scripts): deslop tooling scripts fourth pass (#161525) Share documentation translation response cleanup, simplify sorted translation-memory JSONL writing, use canonical Node inspector types, and remove redundant benchmark and preview-fixture state. Preserve CLI flags, exit codes, output formats, generated bytes, fixture responses, and benchmark workloads. Focused tests, exact before/after JSONL and mock-response checks, and both import-cycle gates passed. The optional cron-reaper smoke reproduced the same existing worker timeout before and after the refactor. Release notes: internal tooling cleanup only. Hosted CI exception: run 36661925129 failed before tests because the trusted CI harness omitted ci-build-manifest.mjs. Current unrelated run 36663040449 failed identically. Security/dependency review and security-fast passed; skipped CI lanes remain unrun. Merged under the maintainer-authorized pre-existing-failure exception. --- scripts/bench-cron-session-reaper.ts | 6 +-- .../bench-session-manager-hydration-memory.ts | 30 +++++-------- scripts/control-ui-mock-skill-library.ts | 20 ++++----- scripts/control-ui-mock-skill-workshop.ts | 45 ++++++------------- scripts/docs-i18n/tm.go | 29 +++--------- scripts/docs-i18n/tm_test.go | 40 +++++++++++++++++ scripts/docs-i18n/translator.go | 43 ++++++------------ scripts/embedded-run-abort-leak.ts | 45 ++++++------------- 8 files changed, 110 insertions(+), 148 deletions(-) create mode 100644 scripts/docs-i18n/tm_test.go diff --git a/scripts/bench-cron-session-reaper.ts b/scripts/bench-cron-session-reaper.ts index 9da348be3392..f4fddffd9685 100644 --- a/scripts/bench-cron-session-reaper.ts +++ b/scripts/bench-cron-session-reaper.ts @@ -22,7 +22,7 @@ const { sweepCronRunSessions } = await import("../src/cron/session-reaper.js"); const now = Date.now(); const hour = 3_600_000; -const targets: Array<{ agentId: string; storePath: string; databasePath: string }> = []; +const targets: Array<{ agentId: string; storePath: string }> = []; const warnings: unknown[][] = []; const log = { debug: () => {}, @@ -116,12 +116,12 @@ try { for (const { sessionKey, entry } of expectedEntries(agentId)) { await replaceSessionEntry({ agentId, storePath, sessionKey }, entry); } - targets.push({ agentId, storePath, databasePath: database.path }); + targets.push({ agentId, storePath }); } closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); const databaseBytes = targets.reduce( - (sum, target) => sum + fs.statSync(target.databasePath).size, + (sum, target) => sum + fs.statSync(target.storePath).size, 0, ); const discovery = await sweep(now, 0); diff --git a/scripts/bench-session-manager-hydration-memory.ts b/scripts/bench-session-manager-hydration-memory.ts index 41a81c8c12c6..68e7d639a727 100644 --- a/scripts/bench-session-manager-hydration-memory.ts +++ b/scripts/bench-session-manager-hydration-memory.ts @@ -1,4 +1,5 @@ import { spawnSync } from "node:child_process"; +import type { HeapProfiler } from "node:inspector"; import { Session as InspectorSession } from "node:inspector/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -27,14 +28,8 @@ type ReaderResult = { }>; }; -type HeapProfileNode = { - callFrame: { functionName: string; lineNumber: number; url: string }; - children?: HeapProfileNode[]; - selfSize: number; -}; - function forceGc(): void { - const gc = (globalThis as typeof globalThis & { gc?: () => void }).gc; + const gc = globalThis.gc; if (!gc) { throw new Error("reader requires --expose-gc"); } @@ -51,11 +46,12 @@ function readArg(name: string): string { return value; } -function summarizeHeapProfile(profile: { - head: HeapProfileNode; -}): ReaderResult["topAllocationSites"] { - const samples: Array<{ node: HeapProfileNode; stack: HeapProfileNode[] }> = []; - const visit = (node: HeapProfileNode, stack: HeapProfileNode[]): void => { +function summarizeHeapProfile( + profile: HeapProfiler.SamplingHeapProfile, +): ReaderResult["topAllocationSites"] { + type ProfileNode = HeapProfiler.SamplingHeapProfileNode; + const samples: Array<{ node: ProfileNode; stack: ProfileNode[] }> = []; + const visit = (node: ProfileNode, stack: ProfileNode[]): void => { const nextStack = [...stack, node]; if (node.selfSize > 0) { samples.push({ node, stack: nextStack }); @@ -64,9 +60,7 @@ function summarizeHeapProfile(profile: { visit(child, nextStack); } }; - const formatFrame = ( - node: HeapProfileNode, - ): { functionName: string; line: number; url: string } => { + const formatFrame = (node: ProfileNode): { functionName: string; line: number; url: string } => { const rawUrl = node.callFrame.url; const filePath = rawUrl.startsWith("file://") ? fileURLToPath(rawUrl) : rawUrl; const relativeUrl = path.isAbsolute(filePath) @@ -98,7 +92,7 @@ function summarizeHeapProfile(profile: { } async function runReader(): Promise { - const mode = readArg("--mode") as ReaderResult["mode"]; + const mode = readArg("--mode"); if (mode !== "full" && mode !== "bounded") { throw new Error("--mode must be full or bounded"); } @@ -119,9 +113,7 @@ async function runReader(): Promise { : SessionManager.open(target); forceGc(); const after = process.memoryUsage(); - const { profile } = (await inspector.post("HeapProfiler.stopSampling")) as { - profile: { head: HeapProfileNode }; - }; + const { profile } = await inspector.post("HeapProfiler.stopSampling"); inspector.disconnect(); const result: ReaderResult = { mode, diff --git a/scripts/control-ui-mock-skill-library.ts b/scripts/control-ui-mock-skill-library.ts index 6a8e4914a7f7..3d8df7dcd531 100644 --- a/scripts/control-ui-mock-skill-library.ts +++ b/scripts/control-ui-mock-skill-library.ts @@ -132,14 +132,12 @@ function installSkillLibraryMock( return; } if (method === "skills.library.read") { - const selected = params.sessionKey - ? pins(params.sessionKey).find( - (read) => - read.entry.skillId === params.skillId && read.entry.revision === params.revision, - ) - : undefined; const read = params.sessionKey - ? selected + ? pins(params.sessionKey).find( + (candidate) => + candidate.entry.skillId === params.skillId && + candidate.entry.revision === params.revision, + ) : params.skillId && visible().some((entry) => entry.skillId === params.skillId) ? params.revision ? histories.get(params.skillId)?.get(params.revision) @@ -182,10 +180,10 @@ function installSkillLibraryMock( "Refresh requires current library access. The existing session pin remains unchanged.", ); } - let next = selected.filter((read) => read.entry.skillId !== params.skillId); - if (params.action === "refresh" && !params.skillId) { - next = []; - } + const next = + params.action === "refresh" && !params.skillId + ? [] + : selected.filter((read) => read.entry.skillId !== params.skillId); if (params.action !== "detach") { for (const id of targets) { const read = id diff --git a/scripts/control-ui-mock-skill-workshop.ts b/scripts/control-ui-mock-skill-workshop.ts index cc3985eedd62..dcb70751e125 100644 --- a/scripts/control-ui-mock-skill-workshop.ts +++ b/scripts/control-ui-mock-skill-workshop.ts @@ -110,7 +110,7 @@ export function buildSkillWorkshopMocks(baseTime: number) { skillKey: proposal.skillKey, description: proposal.description, })), - }, + } satisfies SkillsProposalsListResult, inspect: { cases: proposals.map((proposal) => ({ match: { proposalId: proposal.id }, @@ -165,15 +165,12 @@ function installSkillWorkshopMock(seed: ReturnType { + const reject = (message: string) => + respond({ __mockError: { code: "INVALID_REQUEST", message } }); if (method === "historyStatus" || method === "historyScan") { - respond({ - __mockError: { - code: "INVALID_REQUEST", - message: - "Historical batch scans are retired. Start a learning session from Workshop to review past conversations.", - }, - }); - return; + return reject( + "Historical batch scans are retired. Start a learning session from Workshop to review past conversations.", + ); } const params = (input ?? {}) as { agentId?: string; @@ -185,10 +182,7 @@ function installSkillWorkshopMock(seed: ReturnType [ entry.match.proposalId, @@ -211,36 +205,25 @@ function installSkillWorkshopMock(seed: ReturnType - respond({ __mockError: { code: "INVALID_REQUEST", message } }); if (!detail) { - reject("Unknown mock proposal; refresh the Workshop."); - return; + return reject("Unknown mock proposal; refresh the Workshop."); } if (method === "inspect") { respond(detail); return; } if (detail.record.status !== "pending") { - reject("Only pending proposals can be evaluated, applied, or rejected."); - return; + return reject("Only pending proposals can be evaluated, applied, or rejected."); } if (params.expectedRevisionHash && params.expectedRevisionHash !== detail.revisionHash) { - reject("The proposal revision changed; refresh the proposal before retrying."); - return; + return reject("The proposal revision changed; refresh the proposal before retrying."); } const now = new Date().toISOString(); detail.record.updatedAt = now; diff --git a/scripts/docs-i18n/tm.go b/scripts/docs-i18n/tm.go index a9a3aa3059e0..2bffb8e67b80 100644 --- a/scripts/docs-i18n/tm.go +++ b/scripts/docs-i18n/tm.go @@ -6,9 +6,10 @@ import ( "errors" "fmt" "io" + "maps" "os" "path/filepath" - "sort" + "slices" "strings" ) @@ -69,10 +70,7 @@ func LoadTranslationMemory(path string) (*TranslationMemory, error) { func (tm *TranslationMemory) Get(cacheKey string) (TMEntry, bool) { entry, ok := tm.entries[cacheKey] - if !ok { - return TMEntry{}, false - } - if strings.TrimSpace(entry.Translated) == "" { + if !ok || strings.TrimSpace(entry.Translated) == "" { return TMEntry{}, false } return entry, true @@ -98,25 +96,10 @@ func (tm *TranslationMemory) Save() error { return err } - keys := make([]string, 0, len(tm.entries)) - for key := range tm.entries { - keys = append(keys, key) - } - sort.Strings(keys) - writer := bufio.NewWriter(file) - for _, key := range keys { - entry := tm.entries[key] - payload, err := json.Marshal(entry) - if err != nil { - _ = file.Close() - return err - } - if _, err := writer.Write(payload); err != nil { - _ = file.Close() - return err - } - if _, err := writer.WriteString("\n"); err != nil { + encoder := json.NewEncoder(writer) + for _, key := range slices.Sorted(maps.Keys(tm.entries)) { + if err := encoder.Encode(tm.entries[key]); err != nil { _ = file.Close() return err } diff --git a/scripts/docs-i18n/tm_test.go b/scripts/docs-i18n/tm_test.go new file mode 100644 index 000000000000..8d8b7aae729e --- /dev/null +++ b/scripts/docs-i18n/tm_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestTranslationMemorySavePreservesJSONL(t *testing.T) { + t.Parallel() + file := filepath.Join(t.TempDir(), "memory.jsonl") + memory, err := LoadTranslationMemory(file) + if err != nil { + t.Fatal(err) + } + memory.Put(TMEntry{CacheKey: "z", Text: "\n&", Translated: "日本語\u2028"}) + memory.Put(TMEntry{CacheKey: "a", Translated: "first"}) + if err := memory.Save(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + want := `{"cache_key":"a","segment_id":"","source_path":"","text_hash":"","text":"","translated":"first","src_lang":"","tgt_lang":"","updated_at":""}` + "\n" + + `{"cache_key":"z","segment_id":"","source_path":"","text_hash":"","text":"\u003ctag\u003e\n\u0026","translated":"日本語\u2028","src_lang":"","tgt_lang":"","updated_at":""}` + "\n" + if string(data) != want { + t.Fatalf("translation memory bytes changed:\n%s", data) + } + reloaded, err := LoadTranslationMemory(file) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"a", "z"} { + want, _ := memory.Get(key) + if got, ok := reloaded.Get(key); !ok || got != want { + t.Fatalf("translation memory entry %q did not round trip: %+v", key, got) + } + } +} diff --git a/scripts/docs-i18n/translator.go b/scripts/docs-i18n/translator.go index 51a083b0211e..218826b2ee60 100644 --- a/scripts/docs-i18n/translator.go +++ b/scripts/docs-i18n/translator.go @@ -133,14 +133,10 @@ func (t *CodexTranslator) translateMasked(ctx context.Context, core string) (str placeholders := make([]string, 0, 8) mapping := map[string]string{} masked := maskMarkdown(core, state.Next, &placeholders, mapping) - resText, err := t.prompt(ctx, masked) + translated, err := t.translateRaw(ctx, masked) if err != nil { return "", err } - translated := stripCodexI18nInputWrappers(strings.TrimSpace(resText)) - if translated == "" { - return "", errEmptyTranslation - } if err := validatePlaceholders(translated, placeholders); err != nil { return "", err } @@ -202,10 +198,7 @@ func (t *CodexTranslator) prompt(ctx context.Context, message string) (string, e } func isRetryableTranslateError(err error) bool { - if err == nil { - return false - } - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + if err == nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { return false } if errors.Is(err, errEmptyTranslation) { @@ -401,37 +394,27 @@ func sleepWithContext(ctx context.Context, delay time.Duration) error { func (t *CodexTranslator) Close() {} func normalizeThinking(value string) string { - switch strings.ToLower(strings.TrimSpace(value)) { - case "low", "medium", "high", "xhigh": - return strings.ToLower(strings.TrimSpace(value)) - case "max": - // Preserve an explicit maximum effort while leaving the default unchanged. - return "max" + value = strings.ToLower(strings.TrimSpace(value)) + switch value { + case "low", "medium", "high", "xhigh", "max": + return value default: return "xhigh" } } func docsI18nPromptTimeout() time.Duration { - value := strings.TrimSpace(os.Getenv(envDocsI18nPromptTimeout)) - if value == "" { - return defaultPromptTimeout - } - parsed, err := time.ParseDuration(value) - if err != nil || parsed <= 0 { - return defaultPromptTimeout - } - return parsed + return docsI18nDuration(envDocsI18nPromptTimeout, defaultPromptTimeout) } func docsI18nCommandWaitDelay() time.Duration { - value := strings.TrimSpace(os.Getenv(envDocsI18nCommandWaitDelay)) - if value == "" { - return defaultCommandWaitDelay - } - parsed, err := time.ParseDuration(value) + return docsI18nDuration(envDocsI18nCommandWaitDelay, defaultCommandWaitDelay) +} + +func docsI18nDuration(name string, fallback time.Duration) time.Duration { + parsed, err := time.ParseDuration(strings.TrimSpace(os.Getenv(name))) if err != nil || parsed <= 0 { - return defaultCommandWaitDelay + return fallback } return parsed } diff --git a/scripts/embedded-run-abort-leak.ts b/scripts/embedded-run-abort-leak.ts index c565008848dd..c6c8eef9e295 100644 --- a/scripts/embedded-run-abort-leak.ts +++ b/scripts/embedded-run-abort-leak.ts @@ -156,13 +156,6 @@ const finalizer = new FinalizationRegistry(() => { }); let productionAbortable: Abortable | null = null; -async function loadProductionAbortable(): Promise { - const module = (await import("../src/agents/embedded-agent-runner/run/abortable.js")) as { - abortable: Abortable; - }; - productionAbortable = module.abortable; -} - function abortableExtracted(signal: AbortSignal, promise: Promise): Promise { if (signal.aborted) { return Promise.reject(new Error("aborted")); @@ -254,7 +247,6 @@ async function settleAndGc(): Promise { } type SampleRow = { - label: string; rssBytes: number; heapUsedBytes: number; totalIters: number; @@ -262,11 +254,17 @@ type SampleRow = { snapshotPath: string; }; -function takeSnapshot(snapDir: string, label: string): string { +function takeSnapshot(snapDir: string, label: string, totalIters: number): SampleRow { fs.mkdirSync(snapDir, { recursive: true }); - const filename = path.join(snapDir, `${label}-${process.pid}-${Date.now()}.heapsnapshot`); - v8.writeHeapSnapshot(filename); - return filename; + const snapshotPath = path.join(snapDir, `${label}-${process.pid}-${Date.now()}.heapsnapshot`); + v8.writeHeapSnapshot(snapshotPath); + return { + rssBytes: process.memoryUsage().rss, + heapUsedBytes: process.memoryUsage().heapUsed, + totalIters, + trackedFinalized: FINALIZED.count, + snapshotPath, + }; } function fmtBytes(bytes: number): string { @@ -281,7 +279,8 @@ async function main(): Promise { fail(error instanceof Error ? error.message : String(error)); } if (opts.mode === "production") { - await loadProductionAbortable(); + productionAbortable = (await import("../src/agents/embedded-agent-runner/run/abortable.js")) + .abortable; } if (typeof globalThis.gc !== "function") { fail("--expose-gc is required (run with: node --expose-gc ...)"); @@ -297,15 +296,7 @@ async function main(): Promise { } await settleAndGc(); - const baselinePath = takeSnapshot(opts.snapDir, "baseline"); - const baseline: SampleRow = { - label: "baseline", - rssBytes: process.memoryUsage().rss, - heapUsedBytes: process.memoryUsage().heapUsed, - totalIters: 0, - trackedFinalized: FINALIZED.count, - snapshotPath: baselinePath, - }; + const baseline = takeSnapshot(opts.snapDir, "baseline", 0); let final = baseline; if (!opts.quiet) { process.stdout.write( @@ -320,15 +311,7 @@ async function main(): Promise { totalIters += 1; } await settleAndGc(); - const snapshotPath = takeSnapshot(opts.snapDir, `batch-${b}`); - const row: SampleRow = { - label: `batch-${b}`, - rssBytes: process.memoryUsage().rss, - heapUsedBytes: process.memoryUsage().heapUsed, - totalIters, - trackedFinalized: FINALIZED.count, - snapshotPath, - }; + const row = takeSnapshot(opts.snapDir, `batch-${b}`, totalIters); final = row; if (!opts.quiet) { process.stdout.write(