vrr/opencode
2
0
Fork 0
mirror of https://github.com/anomalyco/opencode.git synced 2026-07-10 04:38:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

fix(stats): align big-pickle provider resolution (#30274)

Browse source
This commit is contained in:
Adam 2026-06-01 14:09:07 -05:00 • committed by GitHub
parent 1813256d8e
commit 2bf85b8479
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 112 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

33
packages/stats/core/src/domain/inference.test.ts
View file

@ -1,6 +1,6 @@
import { describe, expect, test } from "bun:test"
import { toModelAggregate } from "./inference"
import { modelAuthor, normalizeInferenceModel } from "./model-normalization"
import { toModelAggregate, toProviderAggregate } from "./inference"
import { modelAuthor, normalizeInferenceModel, statModel, statProvider } from "./model-normalization"
describe("inference stat normalization", () => {
test("normalizes model suffixes used by router/provider variants", () => {
@ -12,7 +12,7 @@ describe("inference stat normalization", () => {
})
test("maps normalized model ids to public authors", () => {
expect(modelAuthor("big-pickle")).toBe("opencode")
expect(modelAuthor("big-pickle")).toBe("unknown")
expect(modelAuthor("claude-sonnet-4-5")).toBe("anthropic")
expect(modelAuthor("deepseek-v4-pro")).toBe("deepseek")
expect(modelAuthor("gemini-3.5-flash")).toBe("google")
@ -28,7 +28,15 @@ describe("inference stat normalization", () => {
expect(modelAuthor("alpha-gpt-next")).toBeUndefined()
})
test("model aggregates ignore datalake provider and use normalized author/model", () => {
test("uses provider.model to resolve opencode route providers", () => {
expect(statModel("big-pickle", "claude-sonnet-4-5")).toBe("claude-sonnet-4-5")
expect(statModel("big-pickle", "gpt-5-free")).toBe("gpt-5")
expect(statProvider("big-pickle", "claude-sonnet-4-5", "opencode")).toBe("anthropic")
expect(statProvider("big-pickle", "gpt-5", "opencode")).toBe("openai")
expect(statProvider("big-pickle", "", "opencode")).toBe("unknown")
})
test("model aggregates prefer provider.model and use normalized model", () => {
expect(toModelAggregate(aggregate("alpha-gpt-next", "openai"))).toEqual([])
expect(toModelAggregate(aggregate("deepseek-v4-flash-free", "not-public-provider"))).toMatchObject([
@ -38,6 +46,23 @@ describe("inference stat normalization", () => {
model: "deepseek-v4-flash",
},
])
expect(
toModelAggregate({ ...aggregate("big-pickle", "opencode"), provider_model: "claude-sonnet-4-5" }),
).toMatchObject([
{
provider: "anthropic",
model: "claude-sonnet-4-5",
provider_model: "claude-sonnet-4-5",
},
])
})
test("provider aggregates never keep opencode as the provider", () => {
expect(toProviderAggregate({ ...aggregate("big-pickle", "opencode"), provider_model: "gpt-5" })).toMatchObject([
{ provider: "openai" },
])
expect(toProviderAggregate(aggregate("big-pickle", "opencode"))).toMatchObject([{ provider: "unknown" }])
})
test("model aggregates use ISO week period keys", () => {

38
packages/stats/core/src/domain/inference.ts
View file

@ -4,10 +4,9 @@ import type { GeoStatAggregate } from "./geo"
import type { ModelStatAggregate } from "./model"
import {
EXCLUDED_MODELS,
MODEL_AUTHOR_OVERRIDES,
MODEL_AUTHOR_RULES,
modelAuthor,
normalizeInferenceModel,
statModel,
statProvider,
} from "./model-normalization"
import type { ProviderStatAggregate } from "./provider"
import { normalizeCountry, normalizeTier, type StatBaseAggregate } from "./stat"
@ -64,7 +63,7 @@ WITH normalized AS (
SELECT
from_iso8601_timestamp(event_timestamp) AS event_time,
model AS raw_model,
COALESCE(NULLIF(regexp_replace(model, '(-free|:global)+$', ''), ''), 'unknown') AS model,
${statModelSql("model", "provider_model")} AS model,
COALESCE(NULLIF(provider_model, ''), '') AS provider_model,
UPPER(COALESCE(NULLIF(cf_country, ''), 'ZZ')) AS country,
COALESCE(NULLIF(cf_continent, ''), '') AS continent,
@ -98,10 +97,10 @@ WITH normalized AS (
event_time,
CASE
WHEN source = 'lite' THEN 'Go'
WHEN model IN ('gpt-5-nano', 'grok-code', 'big-pickle') OR regexp_like(raw_model, '-free(:global)?$') THEN 'Free'
WHEN raw_model IN ('gpt-5-nano', 'grok-code', 'big-pickle') OR regexp_like(raw_model, '-free(:global)?$') THEN 'Free'
ELSE 'Paid'
END AS tier,
${modelAuthorSql("model")} AS provider,
${statProviderSql("model", "provider_model")} AS provider,
provider_model,
model,
country,
@ -157,25 +156,27 @@ ORDER BY grain, period_key, total_tokens DESC
}
export function toModelAggregate(data: AthenaData): ModelStatAggregate[] {
const model = normalizeInferenceModel(data.model)
const author = modelAuthor(model)
if (!author) return []
const model = statModel(data.model, data.provider_model)
const provider = statProvider(model, data.provider_model, data.provider)
if (!provider) return []
return toStatBaseAggregate(data).flatMap((base) => [
{ ...base, provider: author, model, provider_model: data.provider_model || "" },
{ ...base, provider, model, provider_model: data.provider_model || "" },
])
}
export function toProviderAggregate(data: AthenaData): ProviderStatAggregate[] {
return toStatBaseAggregate(data).flatMap((base) => [{ ...base, provider: data.provider || "unknown" }])
return toStatBaseAggregate(data).flatMap((base) => [
{ ...base, provider: statProvider(data.model, data.provider_model, data.provider) || "unknown" },
])
}
export function toGeoAggregate(data: AthenaData): GeoStatAggregate[] {
return toStatBaseAggregate(data).flatMap((base) => [
{
...base,
provider: data.provider || "all",
model: normalizeInferenceModel(data.model || "all"),
provider: statProvider(data.model, data.provider_model, data.provider) || "all",
model: statModel(data.model || "all", data.provider_model),
country: normalizeCountry(data.country),
continent: data.continent || "",
},
@ -243,9 +244,16 @@ function sqlString(value: string) {
return `'${value.replace(/'/g, "''")}'`
}
function modelAuthorSql(model: string) {
function statModelSql(model: string, providerModel: string) {
return `COALESCE(NULLIF(regexp_replace(CASE
WHEN ${model} = 'big-pickle' THEN COALESCE(NULLIF(${providerModel}, ''), ${model})
ELSE ${model}
END, '(-free|:global)+$', ''), ''), 'unknown')`
}
function statProviderSql(model: string, providerModel: string) {
return `CASE
${MODEL_AUTHOR_OVERRIDES.map((item) => ` WHEN lower(${model}) = ${sqlString(item.model)} THEN ${sqlString(item.author)}`).join("\n")}
${MODEL_AUTHOR_RULES.map((item) => ` WHEN strpos(lower(${providerModel}), ${sqlString(item.match)}) > 0 THEN ${sqlString(item.author)}`).join("\n")}
${MODEL_AUTHOR_RULES.map((item) => ` WHEN strpos(lower(${model}), ${sqlString(item.match)}) > 0 THEN ${sqlString(item.author)}`).join("\n")}
ELSE 'unknown'
END`

25
packages/stats/core/src/domain/model-normalization.ts
View file

@ -1,4 +1,3 @@
export const MODEL_AUTHOR_OVERRIDES = [{ model: "big-pickle", author: "opencode" }] as const
export const MODEL_AUTHOR_RULES = [
{ match: "claude", author: "anthropic" },
{ match: "gemini", author: "google" },
@ -23,8 +22,26 @@ export function modelAuthor(value: string | undefined) {
const model = normalizeInferenceModel(value).toLowerCase()
if (EXCLUDED_MODELS.has(model)) return undefined
const override = MODEL_AUTHOR_OVERRIDES.find((item) => item.model === model)
if (override) return override.author
return MODEL_AUTHOR_RULES.find((item) => model.includes(item.match))?.author ?? "unknown"
}
export function statModel(model: string | undefined, providerModel: string | undefined) {
const normalized = normalizeInferenceModel(model)
if (normalized.toLowerCase() === "big-pickle") return normalizeInferenceModel(providerModel)
return normalized
}
export function statProvider(
model: string | undefined,
providerModel: string | undefined,
provider: string | undefined,
) {
const modelAuthorValue = modelAuthor(statModel(model, providerModel))
if (!modelAuthorValue) return undefined
const providerModelAuthor = modelAuthor(providerModel)
if (providerModelAuthor && providerModelAuthor !== "unknown") return providerModelAuthor
if (modelAuthorValue !== "unknown") return modelAuthorValue
if (provider && provider.toLowerCase() !== "opencode") return provider
return modelAuthorValue
}

51
packages/stats/core/src/honeycomb-backfill.ts
View file

@ -3,7 +3,7 @@ import { readdir } from "node:fs/promises"
import path from "node:path"
import { drizzle } from "drizzle-orm/planetscale-serverless"
import { geoStat, modelStat, providerStat } from "./database/schema"
import { modelAuthor, normalizeInferenceModel } from "./domain/model-normalization"
import { statModel, statProvider } from "./domain/model-normalization"
import {
chunks,
collapseRows,
@ -190,28 +190,44 @@ function buildQueries(limit: number, tiers: string[]): QuerySpec[] {
querySpec(
"model-day",
tier,
metricQuery(["date", "tier", "stat_provider", "stat_model"], limit, tierFilters(tier)),
metricQuery(["date", "tier", "stat_provider_2", "stat_model_2"], limit, tierFilters(tier)),
),
querySpec(
"provider-day",
tier,
metricQuery(["date", "tier", "stat_provider_2"], limit, tierFilters(tier)),
),
querySpec("provider-day", tier, metricQuery(["date", "tier", "stat_provider"], limit, tierFilters(tier))),
querySpec("geo-day", tier, metricQuery(["date", "tier", "country", "continent"], limit, tierFilters(tier))),
querySpec(
"geo-model-day",
tier,
metricQuery(["date", "tier", "stat_provider", "stat_model", "country", "continent"], limit, tierFilters(tier)),
metricQuery(
["date", "tier", "stat_provider_2", "stat_model_2", "country", "continent"],
limit,
tierFilters(tier),
),
),
])
const weekly = tiers.flatMap((tier) => [
querySpec(
"model-week",
tier,
metricQuery(["week", "tier", "stat_provider", "stat_model"], limit, tierFilters(tier)),
metricQuery(["week", "tier", "stat_provider_2", "stat_model_2"], limit, tierFilters(tier)),
),
querySpec(
"provider-week",
tier,
metricQuery(["week", "tier", "stat_provider_2"], limit, tierFilters(tier)),
),
querySpec("provider-week", tier, metricQuery(["week", "tier", "stat_provider"], limit, tierFilters(tier))),
querySpec("geo-week", tier, metricQuery(["week", "tier", "country", "continent"], limit, tierFilters(tier))),
querySpec(
"geo-model-week",
tier,
metricQuery(["week", "tier", "stat_provider", "stat_model", "country", "continent"], limit, tierFilters(tier)),
metricQuery(
["week", "tier", "stat_provider_2", "stat_model_2", "country", "continent"],
limit,
tierFilters(tier),
),
),
])
@ -345,12 +361,23 @@ function classifyRows(file: string, rows: RawRow[]): ImportKey {
const headers = new Set(rows.flatMap((row) => Object.keys(row).map(normalizeHeader)))
const grain: Grain = headers.has("date") ? "day" : "week"
if (hasHeader(headers, ["country", "cf.country"])) {
if (hasHeader(headers, ["model", "stat_model"]) && hasMetricHeaders(headers)) return `geo-model-${grain}`
if (hasHeader(headers, ["model", "stat_model", "stat_model_2"]) && hasMetricHeaders(headers))
return `geo-model-${grain}`
return hasMetricHeaders(headers) ? `geo-${grain}` : `geo-continent-${grain}`
}
if (hasHeader(headers, ["model", "stat_model"]))
if (hasHeader(headers, ["model", "stat_model", "stat_model_2"]))
return hasMetricHeaders(headers) ? `model-${grain}` : `model-provider-model-${grain}`
if (hasHeader(headers, ["provider", "provider.normalized", "stat_provider"])) return `provider-${grain}`
if (
hasHeader(headers, [
"provider",
"provider.normalized",
"stat_provider",
"stat_provider_2",
"provider.model",
"provider_model",
])
)
return `provider-${grain}`
fail(`Cannot classify export from columns in ${file}`)
}
@ -588,11 +615,11 @@ function deriveTier(row: RawRow) {
}
function provider(row: RawRow) {
return cell(row, ["stat_provider"]) || modelAuthor(model(row))
return statProvider(model(row), providerModel(row), cell(row, ["stat_provider_2", "stat_provider"]))
}
function model(row: RawRow) {
return normalizeInferenceModel(cell(row, ["stat_model"]) || rawModel(row))
return statModel(cell(row, ["stat_model_2", "stat_model"]) || rawModel(row), providerModel(row))
}
function rawModel(row: RawRow) {