diff --git a/packages/stats/app/src/component/model-compare-detail.tsx b/packages/stats/app/src/component/model-compare-detail.tsx index 3790789b58b..8fc61d1e930 100644 --- a/packages/stats/app/src/component/model-compare-detail.tsx +++ b/packages/stats/app/src/component/model-compare-detail.tsx @@ -3,6 +3,7 @@ import { ProviderIcon } from "@opencode-ai/ui/provider-icon" import { getStatsModelsComparisonData, type ModelUsagePoint, + type RetentionEntry, type StatsModelComparisonInput, type StatsModelComparisonEntry, } from "@opencode-ai/stats-core/domain/home" @@ -949,6 +950,17 @@ function buildComparisonDetailSections(models: readonly ComparisonModel[]): Comp ], usage: models.map((model) => model.stats?.usage ?? []), }, + { + title: "Retention", + badge: "Week 1", + rows: [ + comparisonDetailRow( + "Returning users", + models.map((model) => retentionCell(model.stats?.weeklyRetention)), + "higher", + ), + ], + }, ] } @@ -1031,6 +1043,15 @@ function percentCell(value: number | undefined): ComparisonDetailCell { return value === undefined ? { value: "No usage" } : { value: formatPercent(value), score: value } } +function retentionCell(value: RetentionEntry | null | undefined): ComparisonDetailCell { + if (!value || value.rank === null) return { value: "Pending" } + return { + value: formatPercent(value.rate), + unit: `${formatTokens(value.eligibleUserWeeks)} user-weeks`, + score: value.rate, + } +} + function tokenCell(value: number | undefined, trend: number | undefined): ComparisonDetailCell { if (value === undefined) return { value: "No usage" } return { value: formatTokens(value), score: value, trend } diff --git a/packages/stats/app/src/routes/[lab]/[model].tsx b/packages/stats/app/src/routes/[lab]/[model].tsx index c9a3e6ef6d7..d2fbfdf7c5e 100644 --- a/packages/stats/app/src/routes/[lab]/[model].tsx +++ b/packages/stats/app/src/routes/[lab]/[model].tsx @@ -470,7 +470,7 @@ function ModelMomentumSection(props: { data: StatsModelPageData | null }) { value={formatInteger(data().totals.sessions)} /> - + - + 0} fallback={ } > @@ -660,13 +656,13 @@ function RetentionSection(props: { data: RetentionEntry[] }) { onPointerEnter={() => setActiveIndex(index())} onFocus={() => setActiveIndex(index())} onClick={() => setActiveIndex(index())} - aria-label={`${item.model}, ${formatRetentionRate(item.rate)} seven-day retention, ${formatUsers(item.eligibleUserDays)} eligible user-days`} + aria-label={`${item.model}, ${formatRetentionRate(item.rate)} weekly retention, ${formatUsers(item.eligibleUserWeeks)} eligible user-weeks`} > {item.rank === null ? "–" : String(item.rank).padStart(2, "0")} {item.model} {formatRetentionRate(item.rate)} - {formatUsers(item.eligibleUserDays)} + {formatUsers(item.eligibleUserWeeks)} )} diff --git a/packages/stats/core/src/domain/home.test.ts b/packages/stats/core/src/domain/home.test.ts index c8b665048c3..3608d56d6a7 100644 --- a/packages/stats/core/src/domain/home.test.ts +++ b/packages/stats/core/src/domain/home.test.ts @@ -7,7 +7,7 @@ process.env.SST_RESOURCE_StatsDatabase = JSON.stringify({ url: "mysql://localhos const { buildRetentionEntries } = await import("./home") describe("retention aggregates", () => { - test("pools the latest seven cohorts and ranks models above the sample floor", () => { + test("pools the latest seven weekly cohorts and ranks models above the sample floor", () => { const rows = [ ...cohorts("model-a", "provider-a", 8, 20, 10), ...cohorts("model-b", "provider-b", 8, 20, 12), @@ -16,20 +16,20 @@ describe("retention aggregates", () => { const entries = buildRetentionEntries(rows) expect(entries.find((item) => item.model === "model-a")).toMatchObject({ - eligibleUserDays: 140, - retainedUserDays: 70, + eligibleUserWeeks: 140, + retainedUserWeeks: 70, rate: 50, rank: 2, }) expect(entries.find((item) => item.model === "model-b")).toMatchObject({ - eligibleUserDays: 140, - retainedUserDays: 84, + eligibleUserWeeks: 140, + retainedUserWeeks: 84, rate: 60, rank: 1, }) expect(entries.find((item) => item.model === "small-model")).toMatchObject({ - eligibleUserDays: 70, - retainedUserDays: 63, + eligibleUserWeeks: 70, + retainedUserWeeks: 63, rate: 90, rank: null, }) diff --git a/packages/stats/core/src/domain/home.ts b/packages/stats/core/src/domain/home.ts index c1784ed18a4..d5ce1b9c86f 100644 --- a/packages/stats/core/src/domain/home.ts +++ b/packages/stats/core/src/domain/home.ts @@ -29,8 +29,8 @@ export type RetentionEntry = { provider: string author: string rate: number - eligibleUserDays: number - retainedUserDays: number + eligibleUserWeeks: number + retainedUserWeeks: number rank: number | null } export type CountryEntry = { country: string; continent: string; tokens: number; share: number; rank: number } @@ -64,7 +64,7 @@ export type StatsModelData = { totalModels: number tokenShare: number tokenChange: number - retention7d: RetentionEntry | null + weeklyRetention: RetentionEntry | null totals: { sessions: number uniqueUsers: number @@ -105,6 +105,7 @@ export type StatsModelComparisonEntry = { totalModels: number tokenShare: number tokenChange: number + weeklyRetention: RetentionEntry | null totals: StatsModelData["totals"] usage: ModelUsagePoint[] } @@ -142,8 +143,8 @@ const TOKEN_SCALE = 1_000_000 const DOLLARS_PER_MICROCENT = 1 / 100_000_000 const METRIC_MODEL_LIMIT = 10 const RETENTION_MODEL_LIMIT = 15 -const RETENTION_MIN_ELIGIBLE_USER_DAYS = 100 -const RETENTION_COHORT_DAYS = 7 +const RETENTION_MIN_ELIGIBLE_USER_WEEKS = 100 +const RETENTION_COHORT_WEEKS = 7 const TOP_MODEL_SEGMENT_LIMIT = 9 // Preserve the response shape while the public site presents Go and Free as one cohort. const SITE_PRODUCT = "Go" @@ -193,7 +194,7 @@ export function getStatsHomeData(): Effect.Effect const [modelRows, geoRows, retentionRows] = await Promise.all([ listModelDaily(), listGeoDaily(), - listRetentionDaily(), + listRetentionWeekly(), ]) return buildStatsHomeData(modelRows, geoRows, retentionRows) }, @@ -207,7 +208,7 @@ export function getStatsModelData( ): Effect.Effect { return Effect.tryPromise({ try: async () => { - const [modelRows, retentionRows] = await Promise.all([listModelDaily(), listRetentionDaily()]) + const [modelRows, retentionRows] = await Promise.all([listModelDaily(), listRetentionWeekly()]) const normalized = modelRows.flatMap(normalizeStatRow) const resolvedModel = resolveModelName(model, normalized, provider) if (!resolvedModel) return null @@ -288,12 +289,12 @@ async function listGeoDaily(opts?: { provider?: string; model?: string }): Promi })) } -async function listRetentionDaily(): Promise { +async function listRetentionWeekly(): Promise { try { return ( await queryRows( `select cohort_date, updated_at, provider, model, eligible_users, retained_users - from model_retention where dataset = 'zen' and tier = 'all' order by cohort_date`, + from model_retention where dataset = 'zen' and tier = 'Go' order by cohort_date`, ) ).map((row) => ({ cohortDate: stringValue(row.cohort_date), @@ -334,8 +335,16 @@ export const getStatsModelsComparisonData: ( ) => Effect.Effect = Effect.fn("StatsModelsComparison.getData")( function* (models) { const modelStats = yield* ModelStatRepo - const rows = yield* modelStats.listDaily() - const entries = models.map((model) => toComparisonEntry(buildStatsModelData(model.model, rows, [], model.provider))) + const [rows, retentionRows] = yield* Effect.all([ + modelStats.listDaily(), + Effect.tryPromise({ + try: listRetentionWeekly, + catch: (cause) => DatabaseError.make({ cause }), + }), + ]) + const entries = models.map((model) => + toComparisonEntry(buildStatsModelData(model.model, rows, [], model.provider, retentionRows)), + ) const latest = entries .map((model) => model?.updatedAt) .flatMap((value) => (value ? [dateTime(value)] : [])) @@ -458,7 +467,7 @@ function buildStatsModelData( const peerRank = rankIndex >= 0 ? rankIndex + 1 : 1 const totalTokens = windowPeers.reduce((sum, item) => sum + item.totalTokens, 0) const peerTokens = rankPeers.reduce((sum, item) => sum + item.totalTokens, 0) - const retention7d = buildRetentionEntries(retentionRows).find((item) => item.model === model) ?? null + const weeklyRetention = buildRetentionEntries(retentionRows).find((item) => item.model === model) ?? null return { updatedAt: Number.isFinite(latestUpdate) ? new Date(latestUpdate).toISOString() : null, @@ -471,7 +480,7 @@ function buildStatsModelData( totalModels: windowPeers.length, tokenShare: totalTokens > 0 ? round((current.totalTokens / totalTokens) * 100, 2) : 0, tokenChange: percentChange(current.totalTokens, previous.totalTokens), - retention7d, + weeklyRetention, totals: { sessions: current.sessions, uniqueUsers: current.uniqueUsers, @@ -553,6 +562,7 @@ function toComparisonEntry(data: StatsModelData | null): StatsModelComparisonEnt totalModels: data.totalModels, tokenShare: data.tokenShare, tokenChange: data.tokenChange, + weeklyRetention: data.weeklyRetention, totals: data.totals, usage: data.usage, } @@ -574,7 +584,7 @@ function emptyStatsHomeData(): StatsHomeData { } export function buildRetentionEntries(rows: RetentionMetricRow[]): RetentionEntry[] { - const cohortDates = [...new Set(rows.map((row) => row.cohortDate))].toSorted().slice(-RETENTION_COHORT_DAYS) + const cohortDates = [...new Set(rows.map((row) => row.cohortDate))].toSorted().slice(-RETENTION_COHORT_WEEKS) const aggregate = rows .filter((row) => cohortDates.includes(row.cohortDate)) .reduce>>((result, row) => { @@ -582,20 +592,22 @@ export function buildRetentionEntries(rows: RetentionMetricRow[]): RetentionEntr result.set(row.model, { model: row.model, provider: current?.provider ?? row.provider, - eligibleUserDays: (current?.eligibleUserDays ?? 0) + row.eligibleUsers, - retainedUserDays: (current?.retainedUserDays ?? 0) + row.retainedUsers, + eligibleUserWeeks: (current?.eligibleUserWeeks ?? 0) + row.eligibleUsers, + retainedUserWeeks: (current?.retainedUserWeeks ?? 0) + row.retainedUsers, }) return result }, new Map()) const entries = [...aggregate.values()].map((item) => ({ ...item, author: formatProvider(item.provider), - rate: item.eligibleUserDays > 0 ? round((item.retainedUserDays / item.eligibleUserDays) * 100, 1) : 0, + rate: item.eligibleUserWeeks > 0 ? round((item.retainedUserWeeks / item.eligibleUserWeeks) * 100, 1) : 0, })) const ranks = new Map( entries - .filter((item) => item.eligibleUserDays >= RETENTION_MIN_ELIGIBLE_USER_DAYS) - .toSorted((a, b) => b.rate - a.rate || b.eligibleUserDays - a.eligibleUserDays || a.model.localeCompare(b.model)) + .filter((item) => item.eligibleUserWeeks >= RETENTION_MIN_ELIGIBLE_USER_WEEKS) + .toSorted( + (a, b) => b.rate - a.rate || b.eligibleUserWeeks - a.eligibleUserWeeks || a.model.localeCompare(b.model), + ) .map((item, index) => [item.model, index + 1]), ) return entries diff --git a/packages/stats/core/src/domain/inference.test.ts b/packages/stats/core/src/domain/inference.test.ts index c534ac2b7a3..8eaf1f91896 100644 --- a/packages/stats/core/src/domain/inference.test.ts +++ b/packages/stats/core/src/domain/inference.test.ts @@ -163,24 +163,32 @@ describe("inference stat normalization", () => { expect(query).toContain("(source = 'inference' AND started_at >= '2026-08-11T10:57:48.186Z')") }) - test("builds complete seven-day cohort retention queries", () => { - const queries = buildRetentionQueries(new Date("2026-08-10T00:00:00.000Z"), new Date("2026-08-20T00:00:00.000Z"), { + test("builds complete week-over-week retention queries", () => { + const queries = buildRetentionQueries(new Date("2026-08-10T00:00:00.000Z"), new Date("2026-08-31T00:00:00.000Z"), { namespace: "inference", table: "generation", dataset: "zen", }) expect(queries).toHaveLength(1) - expect(queries[0]?.cohortDates).toEqual(["2026-08-10", "2026-08-11", "2026-08-12"]) + expect(queries[0]?.cohortDates).toEqual(["2026-08-10", "2026-08-17"]) + expect(queries[0]?.query).toContain("AND product = 'go'") + expect(queries[0]?.query).toContain("COUNT(*) AS model_requests") + expect(queries[0]?.query).toContain( + "SUM(model_requests) OVER (PARTITION BY cohort_date, user_key) AS total_requests", + ) expect(queries[0]?.query).toContain("ROW_NUMBER() OVER") expect(queries[0]?.query).toContain("PARTITION BY cohort_date, user_key") - expect(queries[0]?.query).toContain("ORDER BY total_tokens DESC, requests DESC, model ASC") + expect(queries[0]?.query).toContain("ORDER BY model_requests DESC, model ASC") + expect(queries[0]?.query).toContain("total_requests >= 10") + expect(queries[0]?.query).toContain("CAST(model_requests AS double) / NULLIF(total_requests, 0) >= 0.8") expect(queries[0]?.query).toContain("WHEN '2026-08-17' THEN '2026-08-10'") - expect(queries[0]?.query).toContain("WHEN '2026-08-19' THEN '2026-08-12'") + expect(queries[0]?.query).toContain("WHEN '2026-08-24' THEN '2026-08-17'") expect(queries[0]?.query).toContain("started_at >= '2026-08-10T00:00:00.000Z'") - expect(queries[0]?.query).toContain("started_at < '2026-08-20T00:00:00.000Z'") + expect(queries[0]?.query).toContain("started_at < '2026-08-31T00:00:00.000Z'") expect(queries[0]?.query).toContain("LEFT JOIN returned ON primary_models.user_key = returned.user_key") expect(queries[0]?.query).toContain("primary_models.cohort_date = returned.cohort_date") + expect(queries[0]?.query).toContain("'Go' AS tier") expect(queries[0]?.query).toContain("COUNT(*) AS eligible_users") expect(queries[0]?.query).toContain("LIMIT 10000") }) diff --git a/packages/stats/core/src/domain/inference.ts b/packages/stats/core/src/domain/inference.ts index cf475d2809b..bf770844462 100644 --- a/packages/stats/core/src/domain/inference.ts +++ b/packages/stats/core/src/domain/inference.ts @@ -74,23 +74,23 @@ function buildRetentionQuery( const scanEndValue = sqlString(last.returnEnd.toISOString()) const ingestEndValue = sqlString(new Date(last.returnEnd.getTime() + DAY_MS).toISOString()) const sourceTable = [source.namespace, source.table].map(sqlIdentifier).join(".") - const activityDates = [ + const activityWeeks = [ ...new Map( periods.flatMap((period) => [period.start, period.returnStart]).map((date) => [date.toISOString(), date]), ).values(), ].toSorted((a, b) => a.getTime() - b.getTime()) - const activityDateSql = `CASE -${activityDates + const activityWeekSql = `CASE +${activityWeeks .map( (date) => - ` WHEN started_at >= ${sqlString(date.toISOString())} AND started_at < ${sqlString(new Date(date.getTime() + DAY_MS).toISOString())} THEN ${sqlString(date.toISOString().slice(0, 10))}`, + ` WHEN started_at >= ${sqlString(date.toISOString())} AND started_at < ${sqlString(new Date(date.getTime() + WEEK_MS).toISOString())} THEN ${sqlString(date.toISOString().slice(0, 10))}`, ) .join("\n")} ELSE null END` const cohortDates = periods.map((period) => sqlString(period.start.toISOString().slice(0, 10))).join(", ") const returnDates = periods.map((period) => sqlString(period.returnStart.toISOString().slice(0, 10))).join(", ") - const returnCohortSql = `CASE activity_date + const returnCohortSql = `CASE activity_week ${periods .map( (period) => @@ -102,12 +102,11 @@ ${periods return ` WITH normalized AS ( SELECT - ${activityDateSql} AS activity_date, + ${activityWeekSql} AS activity_week, ${statModelSql("model_requested", "route_model")} AS model, COALESCE(NULLIF(route_model, ''), '') AS provider_model, COALESCE(NULLIF(provider_id, ''), '') AS raw_provider, - COALESCE(NULLIF(user_id, ''), NULLIF(workspace_id, ''), NULLIF(service_api_key_id, '')) AS user_key, - COALESCE(tokens_cache_read, 0) + COALESCE(tokens_cache_write, 0) + COALESCE(tokens_input, 0) + COALESCE(tokens_output, 0) AS tokens_total + COALESCE(NULLIF(user_id, ''), NULLIF(workspace_id, ''), NULLIF(service_api_key_id, '')) AS user_key FROM ${sourceTable} WHERE event_type = 'generation.completed' AND source IN ('inference', 'inference-legacy') @@ -115,7 +114,7 @@ WITH normalized AS ( (source = 'inference-legacy' AND started_at < ${sqlString(LIVE_SOURCE_START)}) OR (source = 'inference' AND started_at >= ${sqlString(LIVE_SOURCE_START)}) ) - AND (product = 'go' OR (${freeTierSql("model_tier", "model_requested")})) + AND product = 'go' AND model_requested IS NOT NULL AND model_requested <> '' AND __ingest_ts >= ${scanStartValue} @@ -124,53 +123,55 @@ WITH normalized AS ( AND started_at < ${scanEndValue} ), filtered AS ( SELECT - activity_date, + activity_week, ${statProviderSql("model", "provider_model", "raw_provider")} AS provider, model, - user_key, - tokens_total + user_key FROM normalized - WHERE activity_date IS NOT NULL + WHERE activity_week IS NOT NULL AND user_key <> '' AND lower(model) NOT IN (${[...EXCLUDED_MODELS].map(sqlString).join(", ")}) ), model_usage AS ( SELECT - activity_date AS cohort_date, + activity_week AS cohort_date, user_key, provider, model, - SUM(tokens_total) AS total_tokens, - COUNT(*) AS requests + COUNT(*) AS model_requests FROM filtered - WHERE activity_date IN (${cohortDates}) - GROUP BY activity_date, user_key, provider, model + WHERE activity_week IN (${cohortDates}) + GROUP BY activity_week, user_key, provider, model ), ranked_models AS ( SELECT cohort_date, user_key, provider, model, + model_requests, + SUM(model_requests) OVER (PARTITION BY cohort_date, user_key) AS total_requests, ROW_NUMBER() OVER ( PARTITION BY cohort_date, user_key - ORDER BY total_tokens DESC, requests DESC, model ASC + ORDER BY model_requests DESC, model ASC ) AS model_rank FROM model_usage ), primary_models AS ( SELECT cohort_date, user_key, provider, model FROM ranked_models WHERE model_rank = 1 + AND total_requests >= 10 + AND CAST(model_requests AS double) / NULLIF(total_requests, 0) >= 0.8 ), returned AS ( SELECT ${returnCohortSql} AS cohort_date, user_key FROM filtered - WHERE activity_date IN (${returnDates}) + WHERE activity_week IN (${returnDates}) GROUP BY ${returnCohortSql}, user_key ) SELECT primary_models.cohort_date, ${sqlString(source.dataset)} AS dataset, - 'all' AS tier, + 'Go' AS tier, primary_models.provider, primary_models.model, COUNT(*) AS eligible_users, @@ -453,14 +454,13 @@ function statPeriods(grain: "day" | "week", periodStart: Date, periodEnd: Date) } function retentionPeriods(periodStart: Date, periodEnd: Date) { - const first = startOfUtcDay(periodStart) - const last = new Date(startOfUtcDay(periodEnd).getTime() - WEEK_MS) - const count = Math.max(0, Math.floor((last.getTime() - first.getTime()) / DAY_MS)) + const first = startOfIsoWeek(periodStart) + const completeEnd = startOfIsoWeek(periodEnd) + const count = Math.max(0, Math.floor((completeEnd.getTime() - first.getTime()) / WEEK_MS) - 1) return Array.from({ length: count }, (_, index) => { - const start = new Date(first.getTime() + index * DAY_MS) - const end = new Date(start.getTime() + DAY_MS) - const returnStart = new Date(start.getTime() + WEEK_MS) - return { start, end, returnStart, returnEnd: new Date(returnStart.getTime() + DAY_MS) } + const start = new Date(first.getTime() + index * WEEK_MS) + const end = new Date(start.getTime() + WEEK_MS) + return { start, end, returnStart: end, returnEnd: new Date(end.getTime() + WEEK_MS) } }) } diff --git a/packages/stats/core/src/stat-sync.ts b/packages/stats/core/src/stat-sync.ts index cd8cdf35b66..aca7fbc6a4a 100644 --- a/packages/stats/core/src/stat-sync.ts +++ b/packages/stats/core/src/stat-sync.ts @@ -20,7 +20,9 @@ const DATALAKE_INGESTION_LAG_MS = 5 * 60_000 const STATS_DATA_START_MS = new Date("2026-05-28T00:00:00.000Z").getTime() const WEEK_MS = 7 * 86_400_000 const DISPLAY_WINDOW_MS = 56 * 86_400_000 -const RETENTION_INCREMENTAL_LOOKBACK_MS = 9 * 86_400_000 +// A retention result needs one complete activity week plus its complete return +// week. Keep another partial week of slack around the ISO-week boundary. +const RETENTION_INCREMENTAL_LOOKBACK_MS = 16 * 86_400_000 // Anchor incremental passes to the ISO week containing this lookback, so the pass // after a week boundary still recomputes the previous week's final aggregates even // if the boundary pass itself failed. @@ -80,7 +82,7 @@ export const syncStats: (options?: { retentionStats.replace(retentionRows, { cohortDates: retentionQueries.flatMap((item) => item.cohortDates), dataset: Resource.StatsSyncConfig.dataset, - tier: "all", + tier: "Go", }), ], {