diff --git a/packages/stats/core/src/domain/inference.test.ts b/packages/stats/core/src/domain/inference.test.ts index 081613bc08..ca5c31314c 100644 --- a/packages/stats/core/src/domain/inference.test.ts +++ b/packages/stats/core/src/domain/inference.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test" -import { toModelAggregate } from "./inference" -import { modelAuthor, normalizeInferenceModel } from "./model-normalization" +import { toModelAggregate, toProviderAggregate } from "./inference" +import { modelAuthor, normalizeInferenceModel, statModel, statProvider } from "./model-normalization" describe("inference stat normalization", () => { test("normalizes model suffixes used by router/provider variants", () => { @@ -12,7 +12,7 @@ describe("inference stat normalization", () => { }) test("maps normalized model ids to public authors", () => { - expect(modelAuthor("big-pickle")).toBe("opencode") + expect(modelAuthor("big-pickle")).toBe("unknown") expect(modelAuthor("claude-sonnet-4-5")).toBe("anthropic") expect(modelAuthor("deepseek-v4-pro")).toBe("deepseek") expect(modelAuthor("gemini-3.5-flash")).toBe("google") @@ -28,7 +28,15 @@ describe("inference stat normalization", () => { expect(modelAuthor("alpha-gpt-next")).toBeUndefined() }) - test("model aggregates ignore datalake provider and use normalized author/model", () => { + test("uses provider.model to resolve opencode route providers", () => { + expect(statModel("big-pickle", "claude-sonnet-4-5")).toBe("claude-sonnet-4-5") + expect(statModel("big-pickle", "gpt-5-free")).toBe("gpt-5") + expect(statProvider("big-pickle", "claude-sonnet-4-5", "opencode")).toBe("anthropic") + expect(statProvider("big-pickle", "gpt-5", "opencode")).toBe("openai") + expect(statProvider("big-pickle", "", "opencode")).toBe("unknown") + }) + + test("model aggregates prefer provider.model and use normalized model", () => { expect(toModelAggregate(aggregate("alpha-gpt-next", "openai"))).toEqual([]) expect(toModelAggregate(aggregate("deepseek-v4-flash-free", "not-public-provider"))).toMatchObject([ @@ -38,6 +46,23 @@ describe("inference stat normalization", () => { model: "deepseek-v4-flash", }, ]) + + expect( + toModelAggregate({ ...aggregate("big-pickle", "opencode"), provider_model: "claude-sonnet-4-5" }), + ).toMatchObject([ + { + provider: "anthropic", + model: "claude-sonnet-4-5", + provider_model: "claude-sonnet-4-5", + }, + ]) + }) + + test("provider aggregates never keep opencode as the provider", () => { + expect(toProviderAggregate({ ...aggregate("big-pickle", "opencode"), provider_model: "gpt-5" })).toMatchObject([ + { provider: "openai" }, + ]) + expect(toProviderAggregate(aggregate("big-pickle", "opencode"))).toMatchObject([{ provider: "unknown" }]) }) test("model aggregates use ISO week period keys", () => { diff --git a/packages/stats/core/src/domain/inference.ts b/packages/stats/core/src/domain/inference.ts index 83a614f351..02bb46e356 100644 --- a/packages/stats/core/src/domain/inference.ts +++ b/packages/stats/core/src/domain/inference.ts @@ -4,10 +4,9 @@ import type { GeoStatAggregate } from "./geo" import type { ModelStatAggregate } from "./model" import { EXCLUDED_MODELS, - MODEL_AUTHOR_OVERRIDES, MODEL_AUTHOR_RULES, - modelAuthor, - normalizeInferenceModel, + statModel, + statProvider, } from "./model-normalization" import type { ProviderStatAggregate } from "./provider" import { normalizeCountry, normalizeTier, type StatBaseAggregate } from "./stat" @@ -64,7 +63,7 @@ WITH normalized AS ( SELECT from_iso8601_timestamp(event_timestamp) AS event_time, model AS raw_model, - COALESCE(NULLIF(regexp_replace(model, '(-free|:global)+$', ''), ''), 'unknown') AS model, + ${statModelSql("model", "provider_model")} AS model, COALESCE(NULLIF(provider_model, ''), '') AS provider_model, UPPER(COALESCE(NULLIF(cf_country, ''), 'ZZ')) AS country, COALESCE(NULLIF(cf_continent, ''), '') AS continent, @@ -98,10 +97,10 @@ WITH normalized AS ( event_time, CASE WHEN source = 'lite' THEN 'Go' - WHEN model IN ('gpt-5-nano', 'grok-code', 'big-pickle') OR regexp_like(raw_model, '-free(:global)?$') THEN 'Free' + WHEN raw_model IN ('gpt-5-nano', 'grok-code', 'big-pickle') OR regexp_like(raw_model, '-free(:global)?$') THEN 'Free' ELSE 'Paid' END AS tier, - ${modelAuthorSql("model")} AS provider, + ${statProviderSql("model", "provider_model")} AS provider, provider_model, model, country, @@ -157,25 +156,27 @@ ORDER BY grain, period_key, total_tokens DESC } export function toModelAggregate(data: AthenaData): ModelStatAggregate[] { - const model = normalizeInferenceModel(data.model) - const author = modelAuthor(model) - if (!author) return [] + const model = statModel(data.model, data.provider_model) + const provider = statProvider(model, data.provider_model, data.provider) + if (!provider) return [] return toStatBaseAggregate(data).flatMap((base) => [ - { ...base, provider: author, model, provider_model: data.provider_model || "" }, + { ...base, provider, model, provider_model: data.provider_model || "" }, ]) } export function toProviderAggregate(data: AthenaData): ProviderStatAggregate[] { - return toStatBaseAggregate(data).flatMap((base) => [{ ...base, provider: data.provider || "unknown" }]) + return toStatBaseAggregate(data).flatMap((base) => [ + { ...base, provider: statProvider(data.model, data.provider_model, data.provider) || "unknown" }, + ]) } export function toGeoAggregate(data: AthenaData): GeoStatAggregate[] { return toStatBaseAggregate(data).flatMap((base) => [ { ...base, - provider: data.provider || "all", - model: normalizeInferenceModel(data.model || "all"), + provider: statProvider(data.model, data.provider_model, data.provider) || "all", + model: statModel(data.model || "all", data.provider_model), country: normalizeCountry(data.country), continent: data.continent || "", }, @@ -243,9 +244,16 @@ function sqlString(value: string) { return `'${value.replace(/'/g, "''")}'` } -function modelAuthorSql(model: string) { +function statModelSql(model: string, providerModel: string) { + return `COALESCE(NULLIF(regexp_replace(CASE + WHEN ${model} = 'big-pickle' THEN COALESCE(NULLIF(${providerModel}, ''), ${model}) + ELSE ${model} + END, '(-free|:global)+$', ''), ''), 'unknown')` +} + +function statProviderSql(model: string, providerModel: string) { return `CASE -${MODEL_AUTHOR_OVERRIDES.map((item) => ` WHEN lower(${model}) = ${sqlString(item.model)} THEN ${sqlString(item.author)}`).join("\n")} +${MODEL_AUTHOR_RULES.map((item) => ` WHEN strpos(lower(${providerModel}), ${sqlString(item.match)}) > 0 THEN ${sqlString(item.author)}`).join("\n")} ${MODEL_AUTHOR_RULES.map((item) => ` WHEN strpos(lower(${model}), ${sqlString(item.match)}) > 0 THEN ${sqlString(item.author)}`).join("\n")} ELSE 'unknown' END` diff --git a/packages/stats/core/src/domain/model-normalization.ts b/packages/stats/core/src/domain/model-normalization.ts index 29cb585e33..9e6d534dfe 100644 --- a/packages/stats/core/src/domain/model-normalization.ts +++ b/packages/stats/core/src/domain/model-normalization.ts @@ -1,4 +1,3 @@ -export const MODEL_AUTHOR_OVERRIDES = [{ model: "big-pickle", author: "opencode" }] as const export const MODEL_AUTHOR_RULES = [ { match: "claude", author: "anthropic" }, { match: "gemini", author: "google" }, @@ -23,8 +22,26 @@ export function modelAuthor(value: string | undefined) { const model = normalizeInferenceModel(value).toLowerCase() if (EXCLUDED_MODELS.has(model)) return undefined - const override = MODEL_AUTHOR_OVERRIDES.find((item) => item.model === model) - if (override) return override.author - return MODEL_AUTHOR_RULES.find((item) => model.includes(item.match))?.author ?? "unknown" } + +export function statModel(model: string | undefined, providerModel: string | undefined) { + const normalized = normalizeInferenceModel(model) + if (normalized.toLowerCase() === "big-pickle") return normalizeInferenceModel(providerModel) + return normalized +} + +export function statProvider( + model: string | undefined, + providerModel: string | undefined, + provider: string | undefined, +) { + const modelAuthorValue = modelAuthor(statModel(model, providerModel)) + if (!modelAuthorValue) return undefined + + const providerModelAuthor = modelAuthor(providerModel) + if (providerModelAuthor && providerModelAuthor !== "unknown") return providerModelAuthor + if (modelAuthorValue !== "unknown") return modelAuthorValue + if (provider && provider.toLowerCase() !== "opencode") return provider + return modelAuthorValue +} diff --git a/packages/stats/core/src/honeycomb-backfill.ts b/packages/stats/core/src/honeycomb-backfill.ts index ff40a9f386..b3cf73a9e9 100644 --- a/packages/stats/core/src/honeycomb-backfill.ts +++ b/packages/stats/core/src/honeycomb-backfill.ts @@ -3,7 +3,7 @@ import { readdir } from "node:fs/promises" import path from "node:path" import { drizzle } from "drizzle-orm/planetscale-serverless" import { geoStat, modelStat, providerStat } from "./database/schema" -import { modelAuthor, normalizeInferenceModel } from "./domain/model-normalization" +import { statModel, statProvider } from "./domain/model-normalization" import { chunks, collapseRows, @@ -190,28 +190,44 @@ function buildQueries(limit: number, tiers: string[]): QuerySpec[] { querySpec( "model-day", tier, - metricQuery(["date", "tier", "stat_provider", "stat_model"], limit, tierFilters(tier)), + metricQuery(["date", "tier", "stat_provider_2", "stat_model_2"], limit, tierFilters(tier)), + ), + querySpec( + "provider-day", + tier, + metricQuery(["date", "tier", "stat_provider_2"], limit, tierFilters(tier)), ), - querySpec("provider-day", tier, metricQuery(["date", "tier", "stat_provider"], limit, tierFilters(tier))), querySpec("geo-day", tier, metricQuery(["date", "tier", "country", "continent"], limit, tierFilters(tier))), querySpec( "geo-model-day", tier, - metricQuery(["date", "tier", "stat_provider", "stat_model", "country", "continent"], limit, tierFilters(tier)), + metricQuery( + ["date", "tier", "stat_provider_2", "stat_model_2", "country", "continent"], + limit, + tierFilters(tier), + ), ), ]) const weekly = tiers.flatMap((tier) => [ querySpec( "model-week", tier, - metricQuery(["week", "tier", "stat_provider", "stat_model"], limit, tierFilters(tier)), + metricQuery(["week", "tier", "stat_provider_2", "stat_model_2"], limit, tierFilters(tier)), + ), + querySpec( + "provider-week", + tier, + metricQuery(["week", "tier", "stat_provider_2"], limit, tierFilters(tier)), ), - querySpec("provider-week", tier, metricQuery(["week", "tier", "stat_provider"], limit, tierFilters(tier))), querySpec("geo-week", tier, metricQuery(["week", "tier", "country", "continent"], limit, tierFilters(tier))), querySpec( "geo-model-week", tier, - metricQuery(["week", "tier", "stat_provider", "stat_model", "country", "continent"], limit, tierFilters(tier)), + metricQuery( + ["week", "tier", "stat_provider_2", "stat_model_2", "country", "continent"], + limit, + tierFilters(tier), + ), ), ]) @@ -345,12 +361,23 @@ function classifyRows(file: string, rows: RawRow[]): ImportKey { const headers = new Set(rows.flatMap((row) => Object.keys(row).map(normalizeHeader))) const grain: Grain = headers.has("date") ? "day" : "week" if (hasHeader(headers, ["country", "cf.country"])) { - if (hasHeader(headers, ["model", "stat_model"]) && hasMetricHeaders(headers)) return `geo-model-${grain}` + if (hasHeader(headers, ["model", "stat_model", "stat_model_2"]) && hasMetricHeaders(headers)) + return `geo-model-${grain}` return hasMetricHeaders(headers) ? `geo-${grain}` : `geo-continent-${grain}` } - if (hasHeader(headers, ["model", "stat_model"])) + if (hasHeader(headers, ["model", "stat_model", "stat_model_2"])) return hasMetricHeaders(headers) ? `model-${grain}` : `model-provider-model-${grain}` - if (hasHeader(headers, ["provider", "provider.normalized", "stat_provider"])) return `provider-${grain}` + if ( + hasHeader(headers, [ + "provider", + "provider.normalized", + "stat_provider", + "stat_provider_2", + "provider.model", + "provider_model", + ]) + ) + return `provider-${grain}` fail(`Cannot classify export from columns in ${file}`) } @@ -588,11 +615,11 @@ function deriveTier(row: RawRow) { } function provider(row: RawRow) { - return cell(row, ["stat_provider"]) || modelAuthor(model(row)) + return statProvider(model(row), providerModel(row), cell(row, ["stat_provider_2", "stat_provider"])) } function model(row: RawRow) { - return normalizeInferenceModel(cell(row, ["stat_model"]) || rawModel(row)) + return statModel(cell(row, ["stat_model_2", "stat_model"]) || rawModel(row), providerModel(row)) } function rawModel(row: RawRow) {