feat(stats): add comparison seo pages

packages/console/app/public/robots.txt

@@ -3,4 +3,7 @@ Allow: /
 
 # Disallow shared content pages
 Disallow: /s/
-Disallow: /share/
+Disallow: /share/
+
+Sitemap: https://opencode.ai/sitemap.xml
+Sitemap: https://opencode.ai/data/sitemap.xml

packages/stats/app/src/lib/comparison-pages.ts

@@ -0,0 +1,286 @@
+import { catalogSlug, findModelCatalogEntry, type ModelCatalog, type ModelCatalogEntry } from "../routes/model-catalog"
+
+type ComparisonFamilyDefinition = {
+  slug: string
+  name: string
+  lab: string
+  prefixes: string[]
+  aliases?: string[]
+  preferredFamilies?: string[]
+}
+
+export type ResolvedComparisonFamily = ComparisonFamilyDefinition & {
+  model: ModelCatalogEntry
+}
+
+export const comparisonFamilies: ComparisonFamilyDefinition[] = [
+  {
+    slug: "gpt",
+    name: "GPT",
+    lab: "openai",
+    prefixes: ["gpt", "o"],
+    aliases: ["openai"],
+    preferredFamilies: ["gpt", "o"],
+  },
+  {
+    slug: "claude",
+    name: "Claude",
+    lab: "anthropic",
+    prefixes: ["claude"],
+    aliases: ["anthropic"],
+    preferredFamilies: ["claude-sonnet", "claude-opus"],
+  },
+  {
+    slug: "gemini",
+    name: "Gemini",
+    lab: "google",
+    prefixes: ["gemini"],
+    aliases: ["google"],
+    preferredFamilies: ["gemini-pro", "gemini-flash", "gemini"],
+  },
+  {
+    slug: "deepseek",
+    name: "DeepSeek",
+    lab: "deepseek",
+    prefixes: ["deepseek"],
+    preferredFamilies: ["deepseek-thinking", "deepseek"],
+  },
+  {
+    slug: "qwen",
+    name: "Qwen",
+    lab: "alibaba",
+    prefixes: ["qwen"],
+    aliases: ["alibaba"],
+    preferredFamilies: ["qwen"],
+  },
+  {
+    slug: "glm",
+    name: "GLM",
+    lab: "zhipuai",
+    prefixes: ["glm"],
+    aliases: ["zhipu", "zhipuai", "zai"],
+    preferredFamilies: ["glm"],
+  },
+  {
+    slug: "kimi",
+    name: "Kimi",
+    lab: "moonshotai",
+    prefixes: ["kimi"],
+    aliases: ["moonshot", "moonshotai"],
+    preferredFamilies: ["kimi-k2", "kimi-thinking"],
+  },
+  {
+    slug: "minimax",
+    name: "MiniMax",
+    lab: "minimax",
+    prefixes: ["minimax"],
+  },
+  {
+    slug: "grok",
+    name: "Grok",
+    lab: "xai",
+    prefixes: ["grok"],
+    aliases: ["xai"],
+    preferredFamilies: ["grok"],
+  },
+  {
+    slug: "mistral",
+    name: "Mistral",
+    lab: "mistral",
+    prefixes: ["mistral", "magistral", "devstral", "codestral"],
+    preferredFamilies: ["mistral-large", "mistral-medium", "mistral-small"],
+  },
+  {
+    slug: "llama",
+    name: "Llama",
+    lab: "meta",
+    prefixes: ["llama"],
+    aliases: ["meta"],
+  },
+  {
+    slug: "nemotron",
+    name: "Nemotron",
+    lab: "nvidia",
+    prefixes: ["nemotron", "llama-nemotron"],
+    aliases: ["nvidia"],
+  },
+  {
+    slug: "mimo",
+    name: "MiMo",
+    lab: "xiaomi",
+    prefixes: ["mimo"],
+    aliases: ["xiaomi"],
+  },
+  {
+    slug: "command",
+    name: "Command",
+    lab: "cohere",
+    prefixes: ["command"],
+    aliases: ["cohere"],
+    preferredFamilies: ["command-a", "command-r"],
+  },
+  {
+    slug: "sonar",
+    name: "Sonar",
+    lab: "perplexity",
+    prefixes: ["sonar"],
+    aliases: ["perplexity"],
+    preferredFamilies: ["sonar-pro", "sonar-reasoning", "sonar"],
+  },
+  {
+    slug: "longcat",
+    name: "LongCat",
+    lab: "meituan",
+    prefixes: ["longcat"],
+    aliases: ["meituan"],
+  },
+  {
+    slug: "step",
+    name: "Step",
+    lab: "stepfun",
+    prefixes: ["step"],
+    aliases: ["stepfun"],
+  },
+  {
+    slug: "mai",
+    name: "MAI",
+    lab: "microsoft",
+    prefixes: ["mai"],
+    aliases: ["microsoft"],
+  },
+]
+
+export function resolveComparisonFamily(catalog: ModelCatalog, value: string) {
+  const family = findComparisonFamily(value)
+  if (!family) return undefined
+  const model = comparisonFamilyCandidates(catalog, family.slug)[0]
+  if (!model) return undefined
+  return { ...family, model } satisfies ResolvedComparisonFamily
+}
+
+export function findComparisonFamily(value: string) {
+  const slug = catalogSlug(value)
+  return comparisonFamilies.find((family) => family.slug === slug || family.aliases?.includes(slug))
+}
+
+export function comparisonFamilyCandidates(catalog: ModelCatalog, value: string) {
+  const family = findComparisonFamily(value)
+  if (!family) return []
+  const matches = catalog.models
+    .filter((model) => model.lab === family.lab && isFamilyModel(model, family) && isGeneralComparisonModel(model))
+    .toSorted((a, b) => comparisonFamilyModelSort(a, b, family))
+  return matches.filter((model) => !isDuplicateAliasModel(model, matches))
+}
+
+export function comparisonSitemapModels(
+  catalog: ModelCatalog,
+  leaderboard: { model: string; provider: string }[] = [],
+) {
+  return uniqueModels([
+    ...comparisonFamilies.flatMap((family) => comparisonFamilyCandidates(catalog, family.slug).slice(0, 2)),
+    ...leaderboard.flatMap((entry) => {
+      const model =
+        findModelCatalogEntry(catalog, entry.model, entry.provider) ?? findModelCatalogEntry(catalog, entry.model)
+      return model && isGeneralComparisonModel(model) ? [model] : []
+    }),
+  ]).toSorted((a, b) => a.id.localeCompare(b.id))
+}
+
+export function canonicalModelComparisonPath(first: ModelCatalogEntry, second: ModelCatalogEntry) {
+  const models = [first, second].toSorted((a, b) => a.id.localeCompare(b.id))
+  return `/data/compare/${models[0].lab}/${models[0].slug}/${models[1].lab}/${models[1].slug}`
+}
+
+export function canonicalFamilyComparisonPath(first: ResolvedComparisonFamily, second: ResolvedComparisonFamily) {
+  const families = [first, second].toSorted((a, b) => a.slug.localeCompare(b.slug))
+  return `/data/compare/${families[0].slug}/${families[1].slug}`
+}
+
+export function latestFamilyComparisonPath(catalog: ModelCatalog, first: ModelCatalogEntry, second: ModelCatalogEntry) {
+  const firstFamily = comparisonFamilyForModel(catalog, first)
+  const secondFamily = comparisonFamilyForModel(catalog, second)
+  if (!firstFamily || !secondFamily || firstFamily.slug === secondFamily.slug) return undefined
+  if (firstFamily.model.id !== first.id || secondFamily.model.id !== second.id) return undefined
+  return canonicalFamilyComparisonPath(firstFamily, secondFamily)
+}
+
+export function comparisonFamilyForModel(catalog: ModelCatalog, model: ModelCatalogEntry) {
+  const family = comparisonFamilies.find(
+    (candidate) => candidate.lab === model.lab && isFamilyModel(model, candidate) && isGeneralComparisonModel(model),
+  )
+  if (!family) return undefined
+  const latest = comparisonFamilyCandidates(catalog, family.slug)[0]
+  if (!latest) return undefined
+  return { ...family, model: latest } satisfies ResolvedComparisonFamily
+}
+
+function isFamilyModel(model: ModelCatalogEntry, family: ComparisonFamilyDefinition) {
+  const values = [model.family, model.slug, model.name]
+    .filter((value): value is string => Boolean(value))
+    .map(catalogSlug)
+  return family.prefixes.some((prefix) => values.some((value) => value === prefix || value.startsWith(`${prefix}-`)))
+}
+
+function isGeneralComparisonModel(model: ModelCatalogEntry) {
+  const input = model.modalities.input.map(catalogSlug)
+  const output = model.modalities.output.map(catalogSlug)
+  if (!input.includes("text") || !output.includes("text")) return false
+  return !/(?:^|-)(?:audio|embedding|guard|image|moderation|omni|rerank|safety|speech|transcribe|tts|vision)(?:-|$)/.test(
+    model.slug,
+  )
+}
+
+function comparisonFamilyModelSort(
+  first: ModelCatalogEntry,
+  second: ModelCatalogEntry,
+  family: ComparisonFamilyDefinition,
+) {
+  return (
+    displayDateTime(second.releaseDate ?? second.lastUpdated) -
+      displayDateTime(first.releaseDate ?? first.lastUpdated) ||
+    preferredFamilyIndex(first, family) - preferredFamilyIndex(second, family) ||
+    modelVariantPenalty(first) - modelVariantPenalty(second) ||
+    first.slug.length - second.slug.length ||
+    first.name.localeCompare(second.name)
+  )
+}
+
+function preferredFamilyIndex(model: ModelCatalogEntry, family: ComparisonFamilyDefinition) {
+  const index = family.preferredFamilies?.indexOf(catalogSlug(model.family ?? "")) ?? -1
+  return index === -1 ? (family.preferredFamilies?.length ?? 0) : index
+}
+
+function modelVariantPenalty(model: ModelCatalogEntry) {
+  return /(?:highspeed|latest|preview|turbo|ultraspeed)/.test(model.slug) ? 1 : 0
+}
+
+function isDuplicateAliasModel(model: ModelCatalogEntry, models: ModelCatalogEntry[]) {
+  if (!/(?:-latest|-highspeed|-ultraspeed)$/.test(model.slug)) return false
+  return models.some(
+    (candidate) =>
+      candidate.id !== model.id &&
+      candidate.releaseDate === model.releaseDate &&
+      candidate.family === model.family &&
+      !/(?:-latest|-highspeed|-ultraspeed)$/.test(candidate.slug),
+  )
+}
+
+function uniqueModels(models: ModelCatalogEntry[]) {
+  return models.reduce<{ ids: Set<string>; models: ModelCatalogEntry[] }>(
+    (result, model) => {
+      if (result.ids.has(model.id)) return result
+      result.ids.add(model.id)
+      result.models.push(model)
+      return result
+    },
+    { ids: new Set(), models: [] },
+  ).models
+}
+
+function displayDateTime(value: string | undefined) {
+  if (!value) return 0
+  const date = new Date(value)
+  if (!Number.isNaN(date.getTime())) return date.getTime()
+  const year = Number(value.match(/\d{4}/)?.[0] ?? 0)
+  return Number.isFinite(year) ? year : 0
+}

packages/stats/app/src/routes/compare-cards.tsx

@@ -32,6 +32,11 @@ export function comparisonHref(first: ComparisonModelRef, second: ComparisonMode
   )}/${catalogSlug(second.slug)}`
 }
 
+export function canonicalComparisonHref(first: ComparisonModelRef, second: ComparisonModelRef) {
+  const models = [first, second].toSorted((a, b) => modelKey(a).localeCompare(modelKey(b)))
+  return comparisonHref(models[0], models[1])
+}
+
 export function uniqueComparisonPairs(pairs: ComparisonPair[]) {
   return pairs.reduce<{ keys: Set<string>; pairs: ComparisonPair[] }>(
     (result, pair) => {
@@ -80,7 +85,7 @@ function FeaturedComparisonCard(props: { pair: ComparisonPair }) {
   return (
     <a
       data-component="compare-home-card"
-      href={comparisonHref(props.pair.first, props.pair.second)}
+      href={canonicalComparisonHref(props.pair.first, props.pair.second)}
       aria-label={`${props.pair.detail}: ${props.pair.first.name} vs ${props.pair.second.name}`}
     >
       <span data-slot="compare-home-card-head">
@@ -106,7 +111,7 @@ function FeaturedComparisonCard(props: { pair: ComparisonPair }) {
 
 function ComparisonPanelCard(props: { pair: ComparisonPair }) {
   return (
-    <a data-component="comparison-card" href={comparisonHref(props.pair.first, props.pair.second)}>
+    <a data-component="comparison-card" href={canonicalComparisonHref(props.pair.first, props.pair.second)}>
       <span>{props.pair.detail}</span>
       <strong>
         {props.pair.first.name} <em>vs</em> {props.pair.second.name}

packages/stats/app/src/routes/compare/[firstFamily]/[secondFamily].tsx

@@ -0,0 +1,47 @@
+import { Meta, Title } from "@solidjs/meta"
+import { createAsync, useParams } from "@solidjs/router"
+import { createMemo, Show } from "solid-js"
+import ModelCompareDetailPage from "../../../component/model-compare-detail"
+import { resolveComparisonFamily } from "../../../lib/comparison-pages"
+import { getModelCatalog } from "../../model-catalog"
+
+export default function ModelCompareFamily() {
+  const params = useParams()
+  const catalog = createAsync(() => getModelCatalog())
+  const comparison = createMemo(() => {
+    const source = catalog()
+    if (!source) return undefined
+    const first = resolveComparisonFamily(source, params.firstFamily ?? "")
+    const second = resolveComparisonFamily(source, params.secondFamily ?? "")
+    if (!first || !second || first.slug === second.slug) return null
+    return { first, second }
+  })
+
+  return (
+    <Show
+      when={comparison()}
+      fallback={
+        <Show when={comparison() === null}>
+          <Title>Model comparison not found</Title>
+          <Meta name="robots" content="noindex,follow" />
+          <main data-page="stats">
+            <div data-component="empty-state">
+              <strong>Comparison not found</strong>
+              <p>Choose two model families to compare.</p>
+              <a href={`${import.meta.env.BASE_URL}compare`}>Compare models</a>
+            </div>
+          </main>
+        </Show>
+      }
+    >
+      {(resolved) => (
+        <ModelCompareDetailPage
+          first={{ lab: resolved().first.model.lab, slug: resolved().first.model.slug }}
+          second={{ lab: resolved().second.model.lab, slug: resolved().second.model.slug }}
+          family={resolved()}
+          catalog={catalog()}
+        />
+      )}
+    </Show>
+  )
+}

packages/stats/app/src/routes/model-catalog.ts

@@ -56,14 +56,18 @@ export type ModelCatalog = {
   labs: ModelCatalogLab[]
 }
 
-export const getModelCatalog = query(async () => {
-  "use server"
+export async function loadModelCatalog() {
   const [models, pricing, labs] = await Promise.all([
     fetchCatalogPayload(modelCatalogSourceUrl),
     fetchCatalogPayload(modelCatalogPricingUrl),
     fetchLabCatalogPayload(modelCatalogLabSourceUrl),
   ])
   return buildModelCatalog(models, pricing, labs)
+}
+
+export const getModelCatalog = query(async () => {
+  "use server"
+  return loadModelCatalog()
 }, "getModelCatalog")
 
 export function findModelCatalogEntry(catalog: ModelCatalog, model: string, lab?: string) {

packages/stats/app/src/routes/sitemap.xml.ts

@@ -0,0 +1,113 @@
+import { getStatsHomeData } from "@opencode-ai/stats-core/domain/home"
+import { runtime } from "@opencode-ai/stats-core/runtime"
+import {
+  canonicalFamilyComparisonPath,
+  canonicalModelComparisonPath,
+  comparisonFamilies,
+  comparisonSitemapModels,
+  latestFamilyComparisonPath,
+  resolveComparisonFamily,
+} from "../lib/comparison-pages"
+import { baseUrl } from "../lib/language"
+import { loadModelCatalog } from "./model-catalog"
+
+type SitemapEntry = {
+  path: string
+  lastmod?: string
+}
+
+export async function GET() {
+  const [catalog, stats] = await Promise.all([
+    loadModelCatalog(),
+    runtime.runPromise(getStatsHomeData()).catch(() => undefined),
+  ])
+  const lastmod = sitemapDate(
+    stats?.updatedAt,
+    ...catalog.models.map((model) => model.lastUpdated ?? model.releaseDate),
+  )
+  const families = comparisonFamilies.flatMap((family) => {
+    const resolved = resolveComparisonFamily(catalog, family.slug)
+    return resolved ? [resolved] : []
+  })
+  const familyComparisons = families.flatMap((first, index) =>
+    families.slice(index + 1).map((second) => ({
+      path: canonicalFamilyComparisonPath(first, second),
+      lastmod: sitemapDate(
+        stats?.updatedAt,
+        first.model.lastUpdated ?? first.model.releaseDate,
+        second.model.lastUpdated ?? second.model.releaseDate,
+      ),
+    })),
+  )
+  const models = comparisonSitemapModels(catalog, stats?.leaderboard["All Users"]["2M"])
+  const modelComparisons = models.flatMap((first, index) =>
+    models.slice(index + 1).flatMap((second) => {
+      if (latestFamilyComparisonPath(catalog, first, second)) return []
+      return [
+        {
+          path: canonicalModelComparisonPath(first, second),
+          lastmod: sitemapDate(
+            stats?.updatedAt,
+            first.lastUpdated ?? first.releaseDate,
+            second.lastUpdated ?? second.releaseDate,
+          ),
+        },
+      ]
+    }),
+  )
+  const entries = uniqueSitemapEntries([{ path: "/data/compare", lastmod }, ...familyComparisons, ...modelComparisons])
+
+  return new Response(sitemapXml(entries), {
+    headers: {
+      "Cache-Control": "public, max-age=300, s-maxage=3600, stale-while-revalidate=86400",
+      "Content-Type": "application/xml; charset=utf-8",
+    },
+  })
+}
+
+function uniqueSitemapEntries(entries: SitemapEntry[]) {
+  return Object.values(
+    entries.reduce<Record<string, SitemapEntry>>((result, entry) => {
+      result[entry.path] = entry
+      return result
+    }, {}),
+  ).toSorted((a, b) => a.path.localeCompare(b.path))
+}
+
+function sitemapXml(entries: SitemapEntry[]) {
+  const urls = entries
+    .map(
+      (entry) => `  <url>
+    <loc>${escapeXml(new URL(entry.path, baseUrl).toString())}</loc>${
+      entry.lastmod
+        ? `
+    <lastmod>${entry.lastmod}</lastmod>`
+        : ""
+    }
+  </url>`,
+    )
+    .join("\n")
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls}
+</urlset>`
+}
+
+function sitemapDate(...values: (string | undefined | null)[]) {
+  const dates = values.flatMap((value) => {
+    if (!value) return []
+    const date = new Date(value)
+    return Number.isNaN(date.getTime()) ? [] : [date]
+  })
+  if (dates.length === 0) return undefined
+  return new Date(Math.min(Date.now(), Math.max(...dates.map((date) => date.getTime())))).toISOString().slice(0, 10)
+}
+
+function escapeXml(value: string) {
+  return value
+    .replaceAll("&", "&amp;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&apos;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+}