From d9bf2a328ebed9db90912baf868b52da65dae3fb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 23 Sep 2026 22:40:00 +0800 Subject: [PATCH] improve(i18n): reuse source hashes in bulk catalog verification (#152657) * improve(i18n): reduce locale catalog hashing overhead * improve(i18n): reuse source hashes in bulk catalog verification * Merge branch 'main' into improve/i18n-prepared-catalog-source Co-authored-by: Vincent Koc --- scripts/control-ui-i18n-verify.ts | 44 +-- scripts/lib/control-ui-i18n-catalog-values.ts | 40 +++ .../scripts/control-ui-i18n-sync-plan.test.ts | 99 +++++- test/scripts/control-ui-i18n-verify.test.ts | 290 ++++++++++++++++++ 4 files changed, 438 insertions(+), 35 deletions(-) create mode 100644 test/scripts/control-ui-i18n-verify.test.ts diff --git a/scripts/control-ui-i18n-verify.ts b/scripts/control-ui-i18n-verify.ts index f21edbefe116..c7c2bb089f56 100644 --- a/scripts/control-ui-i18n-verify.ts +++ b/scripts/control-ui-i18n-verify.ts @@ -6,7 +6,9 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import * as ts from "typescript"; import { loadControlUiTranslationMemory, - materializeControlUiLocaleCatalog, + materializePreparedControlUiLocaleCatalog, + prepareControlUiCatalogSource, + type PreparedControlUiCatalogSource, } from "./lib/control-ui-i18n-catalog-values.ts"; import { loadControlUiSourceCatalog, @@ -197,24 +199,30 @@ async function buildCatalogFallbackBaseline( const sourceMap = loadControlUiSourceCatalog(); const sourceFlat = flattenControlUiCatalog(sourceMap, "en"); const localeFlats = new Map>(); - for (const entry of CONTROL_UI_LOCALE_ENTRIES) { - const memoryPath = path.join(I18N_ASSETS_DIR, `${entry.locale}.tm.jsonl`); - if (!existsSync(memoryPath)) { - throw new Error(`${toRepoPath(memoryPath)} does not contain ${entry.locale} translations`); + { + let prepared: PreparedControlUiCatalogSource | undefined; + for (const [index, entry] of CONTROL_UI_LOCALE_ENTRIES.entries()) { + const memoryPath = path.join(I18N_ASSETS_DIR, `${entry.locale}.tm.jsonl`); + if (!existsSync(memoryPath)) { + throw new Error(`${toRepoPath(memoryPath)} does not contain ${entry.locale} translations`); + } + const memory = loadControlUiTranslationMemory(memoryPath); + prepared ??= prepareControlUiCatalogSource(sourceFlat); + // Match the source + translation-memory materialization served by the runtime Vite module. + const localeMap = materializePreparedControlUiLocaleCatalog(prepared, memory); + if (index === CONTROL_UI_LOCALE_ENTRIES.length - 1) { + // Analysis retains locale flats, but no longer needs the prepared hashes. + prepared = undefined; + } + const localeFlat = flattenControlUiCatalog(localeMap, entry.locale); + const invalid = AUTOMATIONS_FEATURE_KEYS.slice(1, 3).filter((key) => + /\bcron\b/i.test(localeFlat.get(key) ?? ""), + ); + if (invalid.length > 0) { + throw new Error(`${entry.locale}: ${invalid.join(", ")}`); + } + localeFlats.set(entry.locale, localeFlat); } - // Match the source + translation-memory materialization served by the runtime Vite module. - const localeMap = materializeControlUiLocaleCatalog( - sourceFlat, - loadControlUiTranslationMemory(memoryPath), - ); - const localeFlat = flattenControlUiCatalog(localeMap, entry.locale); - const invalid = AUTOMATIONS_FEATURE_KEYS.slice(1, 3).filter((key) => - /\bcron\b/i.test(localeFlat.get(key) ?? ""), - ); - if (invalid.length > 0) { - throw new Error(`${entry.locale}: ${invalid.join(", ")}`); - } - localeFlats.set(entry.locale, localeFlat); } const analysis = analyzeControlUiCatalogs(sourceFlat, localeFlats); diff --git a/scripts/lib/control-ui-i18n-catalog-values.ts b/scripts/lib/control-ui-i18n-catalog-values.ts index fa8c807ca48a..519acf839d3d 100644 --- a/scripts/lib/control-ui-i18n-catalog-values.ts +++ b/scripts/lib/control-ui-i18n-catalog-values.ts @@ -89,3 +89,43 @@ export function materializeControlUiLocaleCatalog( } return catalog; } + +export type PreparedControlUiCatalogSource = { + readonly hashesByKey: ReadonlyMap; +}; + +export function prepareControlUiCatalogSource( + sourceFlat: ReadonlyMap, +): PreparedControlUiCatalogSource { + const hashesByKey = new Map(); + for (const [key, text] of sourceFlat) { + hashesByKey.set(key, hashControlUiTranslationText(text)); + } + return { hashesByKey }; +} + +export function materializePreparedControlUiLocaleCatalog( + source: PreparedControlUiCatalogSource, + memory: ReadonlyMap, +): TranslationMap { + const translations = new Map(); + + for (const entry of memory.values()) { + for (const key of [entry.segment_id, ...(entry.segment_ids ?? [])]) { + const expectedHash = source.hashesByKey.get(key); + if (expectedHash === undefined || entry.text_hash !== expectedHash) { + continue; + } + translations.set(key, entry.translated); + } + } + + const catalog: TranslationMap = {}; + for (const key of source.hashesByKey.keys()) { + const translated = translations.get(key); + if (translated !== undefined) { + setControlUiCatalogValue(catalog, key, translated); + } + } + return catalog; +} diff --git a/test/scripts/control-ui-i18n-sync-plan.test.ts b/test/scripts/control-ui-i18n-sync-plan.test.ts index 869a41915c11..883a504217b7 100644 --- a/test/scripts/control-ui-i18n-sync-plan.test.ts +++ b/test/scripts/control-ui-i18n-sync-plan.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest"; import { hashControlUiTranslationText, materializeControlUiLocaleCatalog, + materializePreparedControlUiLocaleCatalog, mergeControlUiTranslationMaps, + prepareControlUiCatalogSource, } from "../../scripts/lib/control-ui-i18n-catalog-values.ts"; import { createControlUiLocaleSyncPlan, @@ -306,26 +308,89 @@ describe("createControlUiLocaleSyncPlan", () => { expect(createPlan(new Map([["group.alias", "Changed"]])).pending).toHaveLength(1); }); - it("materializes grouped aliases in source order and discards stale or retired segments", () => { - const grouped = memoryEntry({ - segment_id: "group.first", - segment_ids: ["group.second", "removed"], - text_hash: hashControlUiTranslationText("Shared"), - translated: "Partagé", + describe.each([ + { name: "raw", materialize: materializeControlUiLocaleCatalog }, + { + name: "prepared", + materialize: ( + source: ReadonlyMap, + memory: ReadonlyMap, + ) => materializePreparedControlUiLocaleCatalog(prepareControlUiCatalogSource(source), memory), + }, + ])("$name materialization", ({ materialize }) => { + it("keeps aliases independent, source order, and the last valid write", () => { + const abcHash = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + const grouped = memoryEntry({ + segment_id: "group.second", + segment_ids: ["group.first", "removed", "changed"], + text: "Stored text is not the freshness authority", + text_hash: abcHash, + translated: "Partagé", + }); + const replacement = memoryEntry({ + cache_key: "replacement", + segment_id: "group.first", + text_hash: abcHash, + translated: "Dernier", + }); + const stale = memoryEntry({ + cache_key: "stale", + segment_id: "group.first", + text_hash: "stale", + translated: "Obsolète", + }); + const memory = new Map([ + [grouped.cache_key, grouped], + [replacement.cache_key, replacement], + [stale.cache_key, stale], + ]); + const source = flattenTranslations({ + group: { first: "abc", second: "abc" }, + changed: "Changed", + unused: "abc", + }); + const catalog = materialize(source, memory); + expect(catalog).toEqual({ group: { first: "Dernier", second: "Partagé" } }); + expect([...flattenTranslations(catalog)]).toEqual([ + ["group.first", "Dernier"], + ["group.second", "Partagé"], + ]); + expect(materialize(new Map([["group.first", "abc"]]), new Map())).toEqual({}); + expect(materialize(new Map(), memory)).toEqual({}); + expect(materialize(new Map([["group.first", "Changed"]]), memory)).toEqual({}); + expect( + materialize(new Map([["group.first", "abc"]]), new Map([[grouped.cache_key, grouped]])), + ).toEqual({ group: { first: "Partagé" } }); }); - const source = flattenTranslations({ group: { first: "Shared", second: "Shared" } }); + }); - expect( - materializeControlUiLocaleCatalog(source, new Map([[grouped.cache_key, grouped]])), - ).toEqual({ - group: { first: "Partagé", second: "Partagé" }, + it("keeps independently prepared source snapshots after the raw map changes", () => { + const source = new Map([["title", "abc"]]); + const first = prepareControlUiCatalogSource(source); + source.set("title", ""); + source.set("alias", "abc"); + const second = prepareControlUiCatalogSource(source); + const original = memoryEntry({ + cache_key: "original", + segment_id: "title", + segment_ids: ["alias"], + text_hash: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + translated: "Original", }); - expect( - materializeControlUiLocaleCatalog( - new Map([["group.first", "Changed"]]), - new Map([[grouped.cache_key, grouped]]), - ), - ).toEqual({}); + const changed = memoryEntry({ + cache_key: "changed", + segment_id: "title", + text_hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + translated: "Changed", + }); + const memory = new Map([ + [original.cache_key, original], + [changed.cache_key, changed], + ]); + expect(materializePreparedControlUiLocaleCatalog(first, memory)).toEqual({ title: "Original" }); + const next = materializePreparedControlUiLocaleCatalog(second, memory); + expect(next).toEqual({ title: "Changed", alias: "Original" }); + expect(Object.keys(next)).toEqual(["title", "alias"]); }); it("refreshes recorded fallbacks and records translated replacements", () => { diff --git a/test/scripts/control-ui-i18n-verify.test.ts b/test/scripts/control-ui-i18n-verify.test.ts new file mode 100644 index 000000000000..48821cdfea58 --- /dev/null +++ b/test/scripts/control-ui-i18n-verify.test.ts @@ -0,0 +1,290 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { syncControlUiCatalogFallbackBaseline } from "../../scripts/control-ui-i18n-verify.ts"; +import type { + LocaleEntry, + TranslationMap, + TranslationMemoryEntry, +} from "../../scripts/lib/control-ui-i18n-sync-plan.ts"; + +const fixture = vi.hoisted(() => { + // Tooling shares a serial module cache; load this owner with this file's input mocks. + vi.resetModules(); + const writes: { path: string; data: string }[] = []; + return { + files: new Map(), + paths: new Set(), + assetsDir: "", + baselinePath: "", + writes, + loadSource: vi.fn<() => TranslationMap>(), + readSource: vi.fn<() => Promise>(), + }; +}); + +vi.mock("../../scripts/lib/control-ui-i18n-catalog.ts", () => ({ + loadControlUiSourceCatalog: fixture.loadSource, + readControlUiSourceCatalog: fixture.readSource, +})); + +vi.mock("../../scripts/lib/control-ui-i18n-config.ts", () => ({ + CONTROL_UI_LOCALE_ENTRIES: [ + { locale: "fr", fileName: "fr.ts", exportName: "fr", languageKey: "fr" }, + { locale: "de", fileName: "de.ts", exportName: "de", languageKey: "de" }, + ] satisfies LocaleEntry[], +})); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + existsSync: vi.fn((file) => + typeof file === "string" && fixture.paths.has(file) + ? fixture.files.has(file) + : actual.existsSync(file), + ), + readFileSync: ( + file: Parameters[0], + options?: Parameters[1] | BufferEncoding | null, + ) => { + if (typeof file === "string" && fixture.paths.has(file) && options === "utf8") { + const content = fixture.files.get(file); + if (content === undefined) { + throw new Error("Missing in-memory catalog fixture"); + } + return content; + } + if (typeof options === "string") { + return actual.readFileSync(file, options); + } + if (options === null || options === undefined) { + return actual.readFileSync(file, options); + } + return actual.readFileSync(file, options); + }, + }; +}); + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readFile: async ( + file: Parameters[0], + options?: Parameters[1], + ) => { + if (file === fixture.baselinePath && options === "utf8") { + const content = fixture.files.get(fixture.baselinePath); + if (content === undefined) { + throw new Error("Missing in-memory baseline fixture"); + } + return content; + } + if (options === undefined) { + return actual.readFile(file); + } + return actual.readFile(file, options); + }, + writeFile: vi.fn(async (file, data, options) => { + if (file === fixture.baselinePath) { + if (typeof data !== "string" || options !== "utf8") { + throw new Error("Expected UTF-8 baseline text"); + } + fixture.files.set(fixture.baselinePath, data); + fixture.writes.push({ path: fixture.baselinePath, data }); + return; + } + return actual.writeFile(file, data, options); + }), + mkdir: async (...[dir, options]: Parameters) => + dir === fixture.assetsDir ? undefined : actual.mkdir(dir, options), + }; +}); + +const assetsDir = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../ui/src/i18n/.i18n", +); +const baselinePath = path.join(assetsDir, "catalog-fallbacks.json"); +const memoryPath = (locale: string) => path.join(assetsDir, `${locale}.tm.jsonl`); +const abcHash = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; +const emptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; +const writeOptions = { checkOnly: false, write: true }; + +function row(overrides: Partial = {}): TranslationMemoryEntry { + return { + cache_key: "shared", + segment_id: "group.first", + segment_ids: ["group.second", "retired"], + source_path: "ui/src/i18n/locales/fr.ts", + src_lang: "en", + text: "abc", + text_hash: abcHash, + tgt_lang: "fr", + translated: "Partage", + updated_at: "2026-01-01T00:00:00.000Z", + ...overrides, + }; +} + +function setMemory(locale: string, rows: readonly TranslationMemoryEntry[]) { + fixture.files.set( + memoryPath(locale), + `${rows.map((entry) => JSON.stringify(entry)).join("\n")}\n`, + ); +} + +function baseline(fallbacks: Record, sourceHash = abcHash) { + return `${JSON.stringify({ fallbacks, sourceHash, version: 1 }, null, 2)}\n`; +} + +beforeEach(() => { + fixture.assetsDir = assetsDir; + fixture.baselinePath = baselinePath; + fixture.paths.clear(); + for (const file of [baselinePath, memoryPath("fr"), memoryPath("de")]) { + fixture.paths.add(file); + } + fixture.files.clear(); + fixture.writes.length = 0; + fixture.readSource.mockResolvedValue("abc"); + fixture.loadSource.mockReturnValue({ group: { first: "abc", second: "abc" }, missing: "abc" }); + setMemory("fr", [ + row(), + row({ cache_key: "stale", segment_id: "missing", segment_ids: [], text_hash: emptyHash }), + ]); + setMemory("de", [row({ segment_ids: [], translated: "Geteilt" })]); +}); + +afterEach(() => { + vi.restoreAllMocks(); + fixture.loadSource.mockReset(); + fixture.readSource.mockReset(); + fixture.files.clear(); + fixture.paths.clear(); + fixture.writes.length = 0; +}); + +describe("syncControlUiCatalogFallbackBaseline", () => { + it("checks and writes ordered fallback bytes without rewriting a matching baseline", async () => { + const stdout = vi.spyOn(process.stdout, "write").mockReturnValue(true); + const expected = baseline({ "group.second": ["de"], missing: ["de", "fr"] }); + + await expect( + syncControlUiCatalogFallbackBaseline({ checkOnly: true, write: true }), + ).rejects.toThrow("control-ui catalog fallback baseline drift detected."); + expect(fixture.writes).toEqual([]); + await syncControlUiCatalogFallbackBaseline(writeOptions); + expect(fixture.writes).toEqual([{ path: baselinePath, data: expected }]); + expect(fixture.files.get(baselinePath)).toBe(expected); + expect(stdout).toHaveBeenCalledWith( + "control-ui-i18n: catalog: fallback_keys=2 fallback_pairs=3\n", + ); + + fixture.writes.length = 0; + await syncControlUiCatalogFallbackBaseline(writeOptions); + await syncControlUiCatalogFallbackBaseline({ checkOnly: true, write: false }); + expect(fixture.writes).toEqual([]); + }); + + it("uses a fresh English snapshot and hashes raw source bytes on every invocation", async () => { + await syncControlUiCatalogFallbackBaseline(writeOptions); + fixture.loadSource.mockReturnValue({ group: { first: "", second: "abc" }, missing: "abc" }); + fixture.readSource.mockResolvedValue(""); + await syncControlUiCatalogFallbackBaseline(writeOptions); + expect(fixture.files.get(baselinePath)).toBe( + baseline( + { "group.first": ["de", "fr"], "group.second": ["de"], missing: ["de", "fr"] }, + emptyHash, + ), + ); + + fixture.readSource.mockResolvedValue("abc\n"); + await syncControlUiCatalogFallbackBaseline(writeOptions); + expect(fixture.files.get(baselinePath)).toBe( + baseline( + { "group.first": ["de", "fr"], "group.second": ["de"], missing: ["de", "fr"] }, + "edeaaff3f1774ad2888673770c6d64097e391bc362d7d6fb34982ddf0efd18cb", + ), + ); + }); + + it("keeps an empty first memory as fallbacks and rejects missing hashes from parsed rows", async () => { + fixture.files.set(memoryPath("fr"), "\n"); + fixture.files.set( + memoryPath("de"), + [ + JSON.stringify(row({ segment_ids: [], translated: "Geteilt" })), + '{"cache_key":"no-hash","segment_id":"missing","segment_ids":["retired"],"translated":"Invalid"}', + '{"cache_key":"retired-no-hash","segment_id":"retired","translated":"Invalid"}', + ].join("\n"), + ); + await syncControlUiCatalogFallbackBaseline(writeOptions); + expect(fixture.files.get(baselinePath)).toBe( + baseline({ + "group.first": ["fr"], + "group.second": ["de", "fr"], + missing: ["de", "fr"], + }), + ); + }); + + it.each([false, true])( + "preserves the first missing/malformed memory error (allowCatalogDrift=%s)", + async (allowCatalogDrift) => { + fixture.files.delete(memoryPath("fr")); + fixture.files.set(memoryPath("de"), "{"); + await expect( + syncControlUiCatalogFallbackBaseline({ ...writeOptions, allowCatalogDrift }), + ).rejects.toThrow("ui/src/i18n/.i18n/fr.tm.jsonl does not contain fr translations"); + + fixture.files.set(memoryPath("fr"), "{"); + fixture.files.delete(memoryPath("de")); + await expect( + syncControlUiCatalogFallbackBaseline({ ...writeOptions, allowCatalogDrift }), + ).rejects.toBeInstanceOf(SyntaxError); + expect(fixture.writes).toEqual([]); + }, + ); + + it("only tolerates analyzer drift during scoped sync", async () => { + const stdout = vi.spyOn(process.stdout, "write").mockReturnValue(true); + fixture.loadSource.mockReturnValue({ greeting: "Hello {count}" }); + setMemory("fr", [ + row({ + segment_id: "greeting", + segment_ids: [], + text_hash: createHash("sha256").update("Hello {count}", "utf8").digest("hex"), + translated: "Bonjour", + }), + ]); + setMemory("de", []); + + await expect(syncControlUiCatalogFallbackBaseline(writeOptions)).rejects.toThrow( + "fr:greeting expected {count} got {}", + ); + expect(fixture.writes).toEqual([]); + await syncControlUiCatalogFallbackBaseline({ ...writeOptions, allowCatalogDrift: true }); + expect(fixture.files.get(baselinePath)).toBe(baseline({ greeting: ["de"] })); + expect(stdout).toHaveBeenCalledWith( + "control-ui-i18n: catalog: tolerated_errors=1 during scoped sync\n", + ); + }); + + it.each([false, true])( + "rejects terminology errors before later locale errors (allowCatalogDrift=%s)", + async (allowCatalogDrift) => { + fixture.loadSource.mockReturnValue({ sessionsView: { subagentPrefix: "abc" } }); + setMemory("fr", [ + row({ segment_id: "sessionsView.subagentPrefix", segment_ids: [], translated: "Cron" }), + ]); + fixture.files.set(memoryPath("de"), "{"); + await expect( + syncControlUiCatalogFallbackBaseline({ ...writeOptions, allowCatalogDrift }), + ).rejects.toThrow("fr: sessionsView.subagentPrefix"); + expect(fixture.writes).toEqual([]); + }, + ); +});