opencode/packages/app/e2e/performance/tab-switch-reporter.ts

import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from "@playwright/test/reporter"
import { mkdir, writeFile } from "node:fs/promises"
import path from "node:path"

type BenchmarkRecord = {
  status?: string
  metrics?: { firstCorrectObservedMs?: unknown; stableObservedMs?: unknown } | null
}

export default class TabSwitchReporter implements Reporter {
  private output = ""
  private tests: TestCase[] = []
  private results: { test: TestCase; status: TestResult["status"]; records: string[] }[] = []

  onBegin(config: FullConfig, suite: Suite) {
    this.output = config.projects[0].outputDir
    this.tests = suite.allTests()
  }

  onTestEnd(test: TestCase, result: TestResult) {
    this.results.push({
      test,
      status: result.status,
      records: Buffer.concat(result.stdout.map((chunk) => (typeof chunk === "string" ? Buffer.from(chunk) : chunk)))
        .toString("utf8")
        .split(/\r?\n/)
        .filter((line) => line.startsWith("BENCHMARK "))
        .map((line) => line.slice("BENCHMARK ".length)),
    })
  }

  async onEnd(result: FullResult) {
    const file = path.join(this.output, "tab-switch-benchmark.jsonl")
    try {
      await mkdir(this.output, { recursive: true })
      await writeFile(file, this.results.flatMap((entry) => entry.records.map((raw) => `${raw}\n`)).join(""), "utf8")
    } catch (error) {
      console.error("Could not save tab-switch benchmark records:", error)
      return { status: "failed" as const }
    }

    console.log(`\nTab-switch benchmark: ${result.status}`)
    Array.from(new Set(this.tests.map((test) => test.title))).forEach((name) => {
      const results = this.results.filter((entry) => entry.test.title === name)
      const unrun = this.tests.filter(
        (test) => test.title === name && !results.some((entry) => entry.test.id === test.id),
      ).length
      const records = results.flatMap((entry) =>
        entry.records.map((raw) => {
          try {
            return { status: entry.status, record: JSON.parse(raw) as BenchmarkRecord | null }
          } catch {
            return { status: entry.status, record: { status: "invalid JSON", metrics: null } }
          }
        }),
      )
      const passed = records.filter((entry) => entry.status === "passed" && entry.record?.status === "passed")
      const valid = passed
        .map((entry) => ({
          firstCorrectObservedMs: entry.record?.metrics?.firstCorrectObservedMs,
          stableObservedMs: entry.record?.metrics?.stableObservedMs,
        }))
        .filter(
          (metrics): metrics is { firstCorrectObservedMs: number; stableObservedMs: number } =>
            typeof metrics.firstCorrectObservedMs === "number" &&
            Number.isFinite(metrics.firstCorrectObservedMs) &&
            typeof metrics.stableObservedMs === "number" &&
            Number.isFinite(metrics.stableObservedMs),
        )

      console.log(`\n${name}`)
      console.log(`  Tests: ${counts(results.map((entry) => entry.status))}; unrun=${unrun}`)
      console.log(
        `  Records: ${counts(records.map((entry) => entry.record?.status ?? "missing status"))}; ` +
          `missing=${results.filter((entry) => entry.records.length === 0).length + unrun}; ` +
          `excluded=${records.length - valid.length}; invalid metrics=${passed.length - valid.length}`,
      )
      ;(["firstCorrectObservedMs", "stableObservedMs"] as const).forEach((metric) => {
        const values = valid.map((entry) => entry[metric]).sort((a, b) => a - b)
        if (values.length === 0) {
          console.log(`  ${metric}: n=0, median=n/a, p95=n/a`)
          return
        }
        const median = (values[Math.floor((values.length - 1) / 2)] + values[Math.floor(values.length / 2)]) / 2
        const p95 = values[Math.ceil(values.length * 0.95) - 1]
        console.log(`  ${metric}: n=${values.length}, median=${median.toFixed(2)} ms, p95=${p95.toFixed(2)} ms`)
      })
    })
    console.log(`\nRaw BENCHMARK records: ${file}`)
  }
}

function counts(statuses: string[]) {
  return (
    Array.from(new Set(statuses))
      .map((status) => `${status}=${statuses.filter((value) => value === status).length}`)
      .join(", ") || "none"
  )
}