From a87f01c46ccbc62b8195b74c629a6dfa39ce70ed Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 23 Sep 2026 22:37:23 +0800 Subject: [PATCH] feat(models): publish v2 catalog beside unchanged v1 (#156530) * feat(models): publish v2 catalog beside unchanged v1 * test(models): satisfy catalog publisher fixture lint --- scripts/publish-model-catalog.mts | 136 +++++++++- test/scripts/publish-model-catalog-v2.test.ts | 243 ++++++++++++++++++ 2 files changed, 373 insertions(+), 6 deletions(-) create mode 100644 test/scripts/publish-model-catalog-v2.test.ts diff --git a/scripts/publish-model-catalog.mts b/scripts/publish-model-catalog.mts index 1c2eb87df721..b21c36ffb555 100644 --- a/scripts/publish-model-catalog.mts +++ b/scripts/publish-model-catalog.mts @@ -18,7 +18,9 @@ import { isRecord } from "@openclaw/normalization-core/record-coerce"; import type { ModelCatalogModel } from "../packages/model-catalog-core/src/model-catalog-types.js"; import type { RemoteModelCatalogBundle, + RemoteModelCatalogBundleV2, RemoteModelCatalogPricing, + RemoteModelCatalogPricingV2, } from "../packages/model-catalog-core/src/remote-catalog-bundle.js"; import { importToolingTypeScript } from "./lib/import-tooling-typescript.mts"; import { resolveRepoRoot } from "./lib/repo-root.mjs"; @@ -55,6 +57,7 @@ type ModelsDevModel = Record & { type ModelCatalogHydrationCounts = { added: number; filled: number; skipped: number }; type ModelCatalogHydrationResult = Record; type ModelCatalogSourceLoader = (url: string, label: string) => Promise; +type PricingSelection = Pick; const MODEL_CATALOG_MIN_VERSION = "2026.7.0"; export const MODEL_CATALOG_MIN_MODELS = 200; @@ -87,6 +90,7 @@ export function parsePublishModelCatalogArgs(args: string[]) { let dryRun = false; let pricing = false; let out: string | undefined; + let outV2: string | undefined; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (arg === "--dry-run") { @@ -102,12 +106,17 @@ export function parsePublishModelCatalogArgs(args: string[]) { index += 1; continue; } + if (arg === "--out-v2") { + outV2 = requireOptionValue(args, index, arg); + index += 1; + continue; + } throw new Error(`unknown argument: ${arg}`); } if (!dryRun && !out) { throw new Error("provide --out or --dry-run"); } - return { dryRun, pricing, ...(out ? { out } : {}) }; + return { dryRun, pricing, ...(out ? { out } : {}), ...(outV2 ? { outV2 } : {}) }; } export function readModelCatalogManifests( @@ -131,16 +140,20 @@ export function readModelCatalogManifests( .toSorted((left, right) => left.pluginId.localeCompare(right.pluginId)); } -async function loadClientBundleValidator() { +async function loadClientBundleValidator(version: 1 | 2 = 1) { const modulePath = path.join( defaultRootDir, "packages/model-catalog-core/src/remote-catalog-bundle.ts", ); const module = await importToolingTypeScript(pathToFileURL(modulePath).href, import.meta.url); - if (typeof module.validateAndSanitizeRemoteModelCatalogBundle !== "function") { + const name = + version === 1 + ? "validateAndSanitizeRemoteModelCatalogBundle" + : "validateAndSanitizeRemoteModelCatalogBundleV2"; + if (typeof module[name] !== "function") { throw new Error("remote catalog bundle validator export is unavailable"); } - return module.validateAndSanitizeRemoteModelCatalogBundle; + return module[name]; } export async function assembleModelCatalogBundle(options: { @@ -732,6 +745,7 @@ export async function enrichModelCatalogPricing(options: { manifests: ModelCatalogManifestInput[]; fetchImpl?: typeof fetch; loadSource?: ModelCatalogSourceLoader; + pricingSelections?: WeakMap; }): Promise<{ modelsEnriched: number; pricingEntries: number }> { const policies = readPricingPolicies(options.manifests); const sources = await fetchPricingSources( @@ -763,10 +777,12 @@ export async function enrichModelCatalogPricing(options: { ); if (chosen?.pricing) { model.cost = chosen.pricing; + options.pricingSelections?.set(model, { status: "known", source: chosen.source.id }); enriched += 1; } else if (chosen) { // Keep the metadata row: removing it would revive the bundled seed's stale price. delete model.cost; + options.pricingSelections?.set(model, { status: "unavailable", source: chosen.source.id }); process.stderr.write( `[${SCRIPT_LABEL}] warning: ${chosen.source.label} pricing unavailable for ${providerId}/${model.id}; preserving metadata without cost\n`, ); @@ -845,6 +861,79 @@ export function serializeModelCatalogBundle(bundle: PublishedModelCatalogBundle) return `${JSON.stringify(sortCatalogValue({ ...bundle, providers }), null, 2)}\n`; } +export async function assembleModelCatalogBundleV2( + bundle: PublishedModelCatalogBundle, + pricingSelections: WeakMap, +): Promise { + const providers: RemoteModelCatalogBundleV2["providers"] = {}; + const models: RemoteModelCatalogBundleV2["models"] = []; + for (const [providerId, provider] of Object.entries(bundle.providers)) { + providers[providerId] = { + api: provider.api, + defaultModel: provider.defaultModel, + defaultUtilityModel: provider.defaultUtilityModel, + }; + for (const model of provider.models) { + const { cost, ...metadata } = model; + delete metadata.baseUrl; + delete metadata.headers; + delete metadata.upstreamModel; + const selection = pricingSelections.get(model); + const pricing: RemoteModelCatalogPricingV2 = + cost && (selection?.status === "known" || hasKnownPricing(cost)) + ? { + status: "known", + currency: "USD", + unit: "million_tokens", + ...cost, + ...(selection?.source ? { source: selection.source } : {}), + } + : { + status: selection?.status === "unavailable" ? "unavailable" : "unknown", + ...(selection?.source ? { source: selection.source } : {}), + }; + models.push({ ...metadata, provider: providerId, pricing }); + } + } + const validateBundle = await loadClientBundleValidator(2); + // The first supporting release is not assigned yet. schemaVersion gates v2; + // never copy v1's older client floor onto a new wire contract. + return validateBundle({ + schemaVersion: 2, + generatedAt: bundle.generatedAt, + sourceCommit: bundle.sourceCommit, + providers, + models, + }); +} + +export function serializeModelCatalogBundleV2(bundle: RemoteModelCatalogBundleV2): string { + const models = bundle.models + .toSorted( + (left, right) => + left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id), + ) + .map(({ id, provider, ...metadata }) => + Object.assign( + { id, provider }, + Object.fromEntries( + Object.entries(metadata) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => [key, sortCatalogValue(value)]), + ), + ), + ); + return `${JSON.stringify( + Object.fromEntries( + Object.entries(bundle) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => [key, key === "models" ? models : sortCatalogValue(value)]), + ), + null, + 2, + )}\n`; +} + function resolveSourceCommit(rootDir: string): string { return execFileSync("git", ["rev-parse", "HEAD"], { cwd: rootDir, @@ -853,7 +942,7 @@ function resolveSourceCommit(rootDir: string): string { }).trim(); } -async function runPublishModelCatalog( +export async function runPublishModelCatalog( options: { args?: string[]; fetchImpl?: typeof fetch; @@ -864,22 +953,44 @@ async function runPublishModelCatalog( ) { const rootDir = options.rootDir ?? defaultRootDir; const args = parsePublishModelCatalogArgs(options.args ?? process.argv.slice(2)); + if ( + args.out && + args.outV2 && + path.resolve(rootDir, args.out) === path.resolve(rootDir, args.outV2) + ) { + throw new Error("--out and --out-v2 must name different files"); + } const generatedAt = (options.now ?? Date.now)(); const sourceCommit = options.sourceCommit ?? resolveSourceCommit(rootDir); const manifests = readModelCatalogManifests({ rootDir }); let bundle = await assembleModelCatalogBundle({ manifests, generatedAt, sourceCommit }); + const pricingSelections = new WeakMap(); + // Capture seed ownership before hydration/enrichment can replace its cost. + for (const provider of Object.values(bundle.providers)) { + for (const model of provider.models) { + if (model.cost && hasKnownPricing(model.cost)) { + pricingSelections.set(model, { status: "known", source: "manifest" }); + } + } + } const loadSource = createModelCatalogSourceLoader(options.fetchImpl); const hydrationResult = await hydrateModelCatalogFromModelsDev({ bundle, manifests, loadSource }); const pricingResult = args.pricing - ? await enrichModelCatalogPricing({ bundle, manifests, loadSource }) + ? await enrichModelCatalogPricing({ bundle, manifests, loadSource, pricingSelections }) : { modelsEnriched: 0, pricingEntries: 0 }; // Validate after all enrichment so metadata-only and dry-run output obey the // same client contract as priced catalogs. const validateBundle = await loadClientBundleValidator(); + // Project while selection facts still refer to the assembled model objects. + const bundleV2 = args.outV2 + ? await assembleModelCatalogBundleV2(bundle, pricingSelections) + : undefined; bundle = validateBundle(bundle); const summary = summarizeModelCatalogBundle(bundle); const serialized = serializeModelCatalogBundle(bundle); const bundleBytes = Buffer.byteLength(serialized); + const serializedV2 = bundleV2 ? serializeModelCatalogBundleV2(bundleV2) : undefined; + const bundleV2Bytes = serializedV2 ? Buffer.byteLength(serializedV2) : 0; if (bundleBytes > BUNDLE_SIZE_WARNING_BYTES) { process.stderr.write( `[${SCRIPT_LABEL}] warning: bundle size ${bundleBytes} bytes exceeds ${BUNDLE_SIZE_WARNING_BYTES} bytes\n`, @@ -890,6 +1001,11 @@ async function runPublishModelCatalog( `catalog bundle ${bundleBytes} bytes exceeds client limit ${CLIENT_BUNDLE_LIMIT_BYTES} bytes`, ); } + if (bundleV2Bytes > CLIENT_BUNDLE_LIMIT_BYTES) { + throw new Error( + `catalog v2 bundle ${bundleV2Bytes} bytes exceeds client limit ${CLIENT_BUNDLE_LIMIT_BYTES} bytes`, + ); + } const hydrationSummary = Object.entries(hydrationResult) .toSorted(([left], [right]) => left.localeCompare(right)) .map( @@ -908,6 +1024,14 @@ async function runPublishModelCatalog( const outputFile = path.resolve(rootDir, args.out); fs.mkdirSync(path.dirname(outputFile), { recursive: true }); fs.writeFileSync(outputFile, serialized); + if (args.outV2 && serializedV2) { + const outputV2File = path.resolve(rootDir, args.outV2); + fs.mkdirSync(path.dirname(outputV2File), { recursive: true }); + fs.writeFileSync(outputV2File, serializedV2); + process.stdout.write( + `[${SCRIPT_LABEL}] published schemaVersion=2 models=${summary.models} bundleBytes=${bundleV2Bytes} out=${args.outV2}\n`, + ); + } process.stdout.write(`[${SCRIPT_LABEL}] published ${stats} out=${args.out}\n${hydrationSummary}`); return { bundle, summary, pricingEnriched: pricingResult.modelsEnriched, wrote: true }; } diff --git a/test/scripts/publish-model-catalog-v2.test.ts b/test/scripts/publish-model-catalog-v2.test.ts new file mode 100644 index 000000000000..f172bb463a48 --- /dev/null +++ b/test/scripts/publish-model-catalog-v2.test.ts @@ -0,0 +1,243 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + parseRemoteModelCatalogBundle, + parseRemoteModelCatalogBundleV2, + type RemoteModelCatalogBundle, +} from "@openclaw/model-catalog-core"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + assembleModelCatalogBundleV2, + parsePublishModelCatalogArgs, + runPublishModelCatalog, + serializeModelCatalogBundle, + serializeModelCatalogBundleV2, +} from "../../scripts/publish-model-catalog.mts"; + +const roots: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function fixtureRoot() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-catalog-v2-")); + roots.push(root); + const dir = path.join(root, "extensions", "fixture"); + fs.mkdirSync(dir, { recursive: true }); + const seeds = Array.from({ length: 100 }, (_, index) => ({ id: `seed-${index}` })); + fs.writeFileSync( + path.join(dir, "openclaw.plugin.json"), + JSON.stringify({ + providers: ["anthropic", "openai", "fixture-native"], + modelCatalog: { + modelsDev: { "fixture-native": "upstream" }, + providers: { + anthropic: { + models: seeds.map((model, index) => + index === 0 ? { ...model, cost: { input: 0.5 } } : model, + ), + }, + openai: { defaultModel: "seed-0", models: seeds }, + "fixture-native": { + models: [ + { id: "free", cost: { input: 9, output: 9 } }, + { id: "paid" }, + { id: "withdrawn", cost: { input: 8, output: 8 } }, + ], + }, + }, + }, + modelPricing: { providers: { "fixture-native": { openCode: { provider: "upstream" } } } }, + }), + ); + return root; +} + +function fixtureFetch() { + return vi.fn(async (url) => { + if (url === "https://models.opencode.ai/api.json") { + return Response.json({ + upstream: { + id: "upstream", + models: { + free: { id: "free", cost: { input: 0, output: 0 } }, + paid: { id: "paid", cost: { input: 2, output: 3 } }, + extra: { id: "extra", cost: { input: 5, output: 6 } }, + }, + }, + }); + } + return Response.json({ data: [] }); + }); +} + +describe("publish model catalog v2", () => { + it("adds an explicit second output while keeping --out as v1", () => { + expect(parsePublishModelCatalogArgs(["--out", "v1.json", "--out-v2", "v2.json"])).toEqual({ + dryRun: false, + pricing: false, + out: "v1.json", + outV2: "v2.json", + }); + expect(() => parsePublishModelCatalogArgs(["--out-v2", "v2.json"])).toThrow("provide --out"); + expect(() => parsePublishModelCatalogArgs(["--out", "v1.json", "--out-v2"])).toThrow( + "requires a value", + ); + }); + + it("projects only metadata rows with partial costs, tiers, and native tuple identity", async () => { + const bundle: RemoteModelCatalogBundle = { + schemaVersion: 1, + generatedAt: 1, + sourceCommit: "fixture", + minVersion: "2026.7.0", + providers: { + alpha: { + models: [ + { id: "vendor/model", cost: { input: 2 } }, + { id: "zero", cost: { input: 0, output: 0 } }, + { + id: "tier", + cost: { + tieredPricing: [{ input: 3, output: 4, cacheRead: 0, cacheWrite: 0, range: [0] }], + }, + }, + ], + }, + beta: { models: [{ id: "vendor/model" }] }, + }, + pricing: { "alpha/extra": { input: 9, output: 9 } }, + }; + const before = serializeModelCatalogBundle(bundle); + const v2 = await assembleModelCatalogBundleV2(bundle, new WeakMap()); + expect(v2.models).toHaveLength(4); + expect(v2.models[0]).toMatchObject({ + id: "vendor/model", + provider: "alpha", + pricing: { status: "known", input: 2 }, + }); + expect(v2.models[0]?.pricing).not.toHaveProperty("source"); + expect(v2.models[0]?.pricing).not.toHaveProperty("output"); + expect(v2.models[1]?.pricing).toEqual({ status: "unknown" }); + expect(v2.models[2]?.pricing).toMatchObject({ + status: "known", + tieredPricing: bundle.providers.alpha?.models[2]?.cost?.tieredPricing, + }); + expect(v2.models[3]).toMatchObject({ + id: "vendor/model", + provider: "beta", + pricing: { status: "unknown" }, + }); + expect(v2).not.toHaveProperty("pricing"); + expect(v2).not.toHaveProperty("minVersion"); + expect(serializeModelCatalogBundle(bundle)).toBe(before); + const serialized = serializeModelCatalogBundleV2(v2); + const parsed = parseRemoteModelCatalogBundleV2(JSON.parse(serialized)); + expect(parsed.models).toHaveLength(4); + expect(Object.keys(JSON.parse(serialized).models[0]).slice(0, 2)).toEqual(["id", "provider"]); + expect(serializeModelCatalogBundleV2({ ...v2, models: v2.models.toReversed() })).toBe( + serialized, + ); + }); + + it("writes paired catalogs from one source snapshot, preserving authoritative zero and withdrawal", async () => { + const rootDir = fixtureRoot(); + const fetchImpl = fixtureFetch(); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + await runPublishModelCatalog({ + rootDir, + fetchImpl, + now: () => 42, + sourceCommit: "fixture", + args: ["--pricing", "--out", "v1/catalog.json", "--out-v2", "v2/catalog.json"], + }); + const v1 = parseRemoteModelCatalogBundle( + JSON.parse(fs.readFileSync(path.join(rootDir, "v1/catalog.json"), "utf8")), + ); + const v2 = parseRemoteModelCatalogBundleV2( + JSON.parse(fs.readFileSync(path.join(rootDir, "v2/catalog.json"), "utf8")), + ); + expect(v1.generatedAt).toBe(v2.generatedAt); + expect(v1.sourceCommit).toBe(v2.sourceCommit); + expect(v1.minVersion).toBe("2026.7.0"); + expect(v2.models).toHaveLength(203); + expect(v2.providers.openai?.defaultModel).toBe("seed-0"); + expect( + v2.models.find((model) => model.provider === "anthropic" && model.id === "seed-0")?.pricing, + ).toEqual({ + status: "known", + currency: "USD", + unit: "million_tokens", + input: 0.5, + source: "manifest", + }); + expect(v2.models.find((model) => model.id === "free")?.pricing).toMatchObject({ + status: "known", + input: 0, + output: 0, + source: "openCode", + }); + expect(v2.models.find((model) => model.id === "paid")?.pricing).toMatchObject({ + status: "known", + input: 2, + output: 3, + source: "openCode", + }); + expect(v2.models.find((model) => model.id === "withdrawn")?.pricing).toEqual({ + status: "unavailable", + source: "openCode", + }); + expect(v1.pricing?.["fixture-native/extra"]).toBeDefined(); + expect(v2.models.some((model) => model.id === "extra")).toBe(false); + expect( + fetchImpl.mock.calls.filter(([url]) => url === "https://models.opencode.ai/api.json"), + ).toHaveLength(1); + await runPublishModelCatalog({ + rootDir, + fetchImpl: fixtureFetch(), + now: () => 42, + sourceCommit: "fixture", + args: ["--pricing", "--out", "v1-only.json"], + }); + expect(fs.readFileSync(path.join(rootDir, "v1-only.json"), "utf8")).toBe( + fs.readFileSync(path.join(rootDir, "v1/catalog.json"), "utf8"), + ); + }); + + it("keeps both prior files on source failure and never writes in dry-run mode", async () => { + const rootDir = fixtureRoot(); + const out = path.join(rootDir, "v1.json"); + const outV2 = path.join(rootDir, "v2.json"); + fs.writeFileSync(out, "previous v1"); + fs.writeFileSync(outV2, "previous v2"); + const args = ["--pricing", "--out", out, "--out-v2", outV2]; + vi.spyOn(process.stdout, "write").mockImplementation(() => true); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + await expect( + runPublishModelCatalog({ + rootDir, + sourceCommit: "fixture", + args, + fetchImpl: async () => { + throw new Error("fixture outage"); + }, + }), + ).rejects.toThrow("fixture outage"); + await runPublishModelCatalog({ + rootDir, + sourceCommit: "fixture", + args: [...args, "--dry-run"], + fetchImpl: fixtureFetch(), + }); + expect(fs.readFileSync(out, "utf8")).toBe("previous v1"); + expect(fs.readFileSync(outV2, "utf8")).toBe("previous v2"); + await expect( + runPublishModelCatalog({ rootDir, args: ["--out", "same.json", "--out-v2", "./same.json"] }), + ).rejects.toThrow("different files"); + }); +});