diff --git a/packages/app/e2e/performance/README.md b/packages/app/e2e/performance/README.md index ce868d573bb..675057366ba 100644 --- a/packages/app/e2e/performance/README.md +++ b/packages/app/e2e/performance/README.md @@ -56,6 +56,32 @@ Benchmarks do not assert machine-dependent performance budgets. Streaming proces Committed smoke and regression tests continue to own correctness coverage for pagination, tab paint, context resize, collapse state, and composer spacing. +## Desktop profiler + +The desktop profiler launches the existing production build directly, creates isolated desktop state, chooses an available CDP port, and writes reports under the OS temporary directory by default. + +```sh +bun run profile:desktop --help +``` + +Create a private partial snapshot from the default local database and run Home once: + +```sh +bun run profile:desktop --partial-snapshot-out /tmp/opencode-perf.db \ + --window-end 2026-08-04T06:14:26.878Z \ + --scenarios home,calibration --skip-build +``` + +Repeat against the immutable partial snapshot: + +```sh +bun run profile:desktop --mode partial-snapshot --db /tmp/opencode-perf.db \ + --window-end 2026-08-04T06:14:26.878Z \ + --scenarios home,calibration --runs 3 --skip-build +``` + +Partial snapshots contain private application data and must not be committed or shared. The profiler copies each partial snapshot to a per-run working database and remaps selected project paths to temporary workspaces, leaving the source snapshot unchanged. `PROFILE_SUMMARY` is the compact comparison output; `PROFILE_REPORT` points to the complete JSON report with the database hash, invocation parameters, raw runs, and attribution data. + ## Chrome traces Set `OPENCODE_PERFORMANCE_TRACE_DIR` to emit a standard Chrome DevTools trace for every benchmark page automatically: diff --git a/packages/app/e2e/performance/desktop-profile/corpus.ts b/packages/app/e2e/performance/desktop-profile/corpus.ts new file mode 100644 index 00000000000..fe7cf155622 --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/corpus.ts @@ -0,0 +1,137 @@ +import { Database } from "bun:sqlite" +import { mkdir, rm } from "node:fs/promises" +import path from "node:path" +import { progress } from "./progress" +import type { Options, Target } from "./types" + +export async function createPartialSnapshot(source: string, destination: string, options: Options, targets: Target[]) { + await mkdir(path.dirname(destination), { recursive: true }) + await rm(destination, { force: true }) + const input = new Database(source, { readonly: true }) + const schema = input + .query( + `SELECT type, name, sql FROM sqlite_schema + WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' + ORDER BY CASE type WHEN 'table' THEN 0 WHEN 'index' THEN 1 ELSE 2 END, name`, + ) + .all() as { type: string; name: string; sql: string }[] + input.close() + + const output = new Database(destination, { create: true }) + output.run("PRAGMA foreign_keys = OFF") + schema.filter((item) => item.type === "table").forEach((item) => output.run(item.sql)) + output.run("ATTACH DATABASE ? AS source", source) + const selected = [...new Set(targets.map((target) => target.id))] + const placeholders = selected.map(() => "?").join(",") + + for (const table of schema.filter((item) => item.type === "table").map((item) => item.name)) { + progress("copying partial snapshot table", { table }) + if (table === "event") continue + if (table === "message") { + output.run( + `INSERT INTO main.message SELECT * FROM source.message + WHERE (time_created >= ? AND time_created < ? AND session_id IN ( + SELECT id FROM source.session WHERE parent_id IS NULL + )) OR session_id IN (${placeholders})`, + options.windowStart, + options.windowEnd, + ...selected, + ) + continue + } + if (table === "part") { + output.run("INSERT INTO main.part SELECT * FROM source.part WHERE message_id IN (SELECT id FROM main.message)") + continue + } + if (["session_context_epoch", "session_input", "session_message", "session_share", "todo"].includes(table)) { + output.run( + `INSERT INTO main."${table}" SELECT * FROM source."${table}" WHERE session_id IN (${placeholders})`, + ...selected, + ) + continue + } + output.run(`INSERT INTO main."${table}" SELECT * FROM source."${table}"`) + } + output.run("DETACH DATABASE source") + schema.filter((item) => item.type !== "table").forEach((item) => output.run(item.sql)) + output.close() +} + +export async function fingerprint(file: string) { + const input = Bun.file(file) + const hasher = new Bun.CryptoHasher("sha256") + for await (const chunk of input.stream()) hasher.update(chunk) + return { bytes: input.size, sha256: hasher.digest("hex") } +} + +export function loadCorpus(options: Options) { + const database = new Database(options.database, { readonly: true }) + database.run("PRAGMA query_only = ON") + const sessions = database + .query( + `SELECT id, project_id AS projectID, directory, title + FROM session AS candidate + WHERE parent_id IS NULL + AND EXISTS ( + SELECT 1 FROM message + WHERE session_id = candidate.id AND time_created >= ? AND time_created < ? + )`, + ) + .all(options.windowStart, options.windowEnd) as { id: string; projectID: string; directory: string; title: string }[] + const messageRows = database.query( + `SELECT id, data FROM message + WHERE session_id = ? AND time_created >= ? AND time_created < ? + ORDER BY time_created, id`, + ) + const partRows = database.query(`SELECT data FROM part WHERE message_id = ? ORDER BY id`) + const ranked = sessions + .map((session) => { + const messages = messageRows.all(session.id, options.windowStart, options.windowEnd) as { + id: string + data: string + }[] + const parts = messages.flatMap((message) => partRows.all(message.id) as { data: string }[]) + return { + ...session, + bytes: + messages.reduce((sum, message) => sum + Buffer.byteLength(message.data), 0) + + parts.reduce((sum, part) => sum + Buffer.byteLength(part.data), 0), + messages: messages.length, + parts: parts.length, + userTurns: messages.filter((message) => JSON.parse(message.data).role === "user").length, + } + }) + .filter((session) => session.messages > 0) + .sort((a, b) => a.bytes - b.bytes || a.id.localeCompare(b.id)) + if (ranked.length === 0) throw new Error("No sessions found in the profile window") + const select = (label: Target["label"], percentile: number) => ({ + label, + ...ranked[Math.max(0, Math.ceil(ranked.length * percentile) - 1)]!, + }) + const targets = [select("p50", 0.5), select("p95", 0.95), select("max", 1)] satisfies Target[] + const typingText = loadTypingText(database, partRows, messageRows, targets[2]!, options) + const projectIDs = [...new Set(ranked.map((session) => session.projectID))] + database.close() + return { targets, typingText, projectIDs } +} + +function loadTypingText( + database: Database, + partRows: ReturnType, + messageRows: ReturnType, + target: Target, + options: Options, +) { + const messages = messageRows.all(target.id, options.windowStart, options.windowEnd) as { id: string; data: string }[] + const text = messages + .filter((message) => JSON.parse(message.data).role === "user") + .flatMap((message) => + (partRows.all(message.id) as { data: string }[]).flatMap((part) => { + const data = JSON.parse(part.data) + return data.type === "text" && typeof data.text === "string" ? [data.text] : [] + }), + ) + .sort((a, b) => b.length - a.length)[0] + if (!text) throw new Error("No real user prompt found for composer profiling") + return text +} diff --git a/packages/app/e2e/performance/desktop-profile/desktop-state.ts b/packages/app/e2e/performance/desktop-profile/desktop-state.ts new file mode 100644 index 00000000000..a9df97b1860 --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/desktop-state.ts @@ -0,0 +1,116 @@ +import { Database } from "bun:sqlite" +import { mkdir } from "node:fs/promises" +import path from "node:path" +import type { Options } from "./types" + +export async function prepareDesktopState( + options: Options, + databasePath: string, + userData: string, + run: number, + projectIDs: string[], +) { + const database = new Database(databasePath) + const projects = database.query("SELECT id, worktree, sandboxes FROM project ORDER BY id").all() as { + id: string + worktree: string + sandboxes: string + }[] + const selected = new Set(projectIDs) + const profileProjects = projects.filter((project) => selected.has(project.id)) + const worktrees = + options.mode === "partial-snapshot" + ? await remapDirectories(database, profileProjects, path.join(options.output, "workspaces", String(run))) + : profileProjects.map((project) => project.worktree) + database.close() + + await Bun.write( + path.join(userData, "opencode.settings"), + JSON.stringify({ firstLaunchOnboardingComplete: true, oldLayoutEligible: true, tauriMigrated: true }), + ) + await Bun.write( + path.join(userData, "opencode.global.dat"), + JSON.stringify({ + server: JSON.stringify({ + list: [], + projects: { local: worktrees.map((worktree) => ({ worktree, expanded: true })) }, + lastProject: worktrees[0] ? { local: worktrees[0] } : {}, + recentlyClosed: {}, + }), + }), + ) +} + +async function remapDirectories( + database: Database, + projects: { id: string; worktree: string; sandboxes: string }[], + root: string, +) { + await mkdir(root, { recursive: true }) + const mappings = new Map() + const worktrees = await Promise.all( + projects.map(async (project, index) => { + const worktree = path.join(root, `project-${String(index + 1).padStart(3, "0")}`) + await mkdir(worktree, { recursive: true }) + mappings.set(project.worktree, worktree) + const sandboxes = JSON.parse(project.sandboxes) as string[] + const nextSandboxes = await Promise.all( + sandboxes.map(async (sandbox, sandboxIndex) => { + const next = path.join(worktree, `sandbox-${sandboxIndex + 1}`) + await mkdir(next, { recursive: true }) + mappings.set(sandbox, next) + return next + }), + ) + database.run("UPDATE project SET worktree = ?, sandboxes = ? WHERE id = ?", worktree, JSON.stringify(nextSandboxes), project.id) + return worktree + }), + ) + const byProject = new Map(projects.map((project, index) => [project.id, worktrees[index]!])) + const sessions = database.query("SELECT id, project_id, directory FROM session").all() as { + id: string + project_id: string + directory: string + }[] + const directories = database.query("SELECT * FROM project_directory").all() as { + project_id: string + directory: string + type: string | null + strategy: string | null + time_created: number + }[] + const selected = new Set(projects.map((project) => project.id)) + const nextDirectories = await Promise.all( + directories.filter((item) => selected.has(item.project_id)).map(async (item, index) => { + const directory = + mappings.get(item.directory) ?? path.join(byProject.get(item.project_id) ?? root, `directory-${index + 1}`) + await mkdir(directory, { recursive: true }) + return { ...item, directory } + }), + ) + database.transaction(() => { + sessions.filter((session) => selected.has(session.project_id)).forEach((session) => + database.run( + "UPDATE session SET directory = ? WHERE id = ?", + mappings.get(session.directory) ?? byProject.get(session.project_id) ?? worktrees[0]!, + session.id, + ), + ) + database.run( + `DELETE FROM project_directory WHERE project_id IN (${projects.map(() => "?").join(",")})`, + ...projects.map((project) => project.id), + ) + nextDirectories.forEach((item) => + database.run( + `INSERT INTO project_directory (project_id, directory, type, strategy, time_created) + VALUES (?, ?, ?, ?, ?)`, + item.project_id, + item.directory, + item.type, + item.strategy, + item.time_created, + ), + ) + })() + return worktrees +} diff --git a/packages/app/e2e/performance/desktop-profile/options.test.ts b/packages/app/e2e/performance/desktop-profile/options.test.ts new file mode 100644 index 00000000000..dda79da1636 --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/options.test.ts @@ -0,0 +1,54 @@ +import { Database } from "bun:sqlite" +import { afterAll, expect, test } from "bun:test" +import { mkdir, rm } from "node:fs/promises" +import path from "node:path" +import { createPartialSnapshot, fingerprint } from "./corpus" +import { parseOptions } from "./options" + +const directory = path.join(import.meta.dir, `.tmp-${process.pid}`) +const source = path.join(directory, "source.db") +const partialSnapshot = path.join(directory, "partial-snapshot.db") +await mkdir(directory, { recursive: true }) +const database = new Database(source, { create: true }) +database.run("CREATE TABLE sample (value TEXT NOT NULL)") +database.run("INSERT INTO sample VALUES ('repeatable')") +database.close() + +afterAll(() => rm(directory, { recursive: true, force: true })) + +test("parses a portable fixed-window partial snapshot invocation", () => { + const options = parseOptions([ + "--mode", + "partial-snapshot", + "--db", + source, + "--window-end", + "2026-08-04T06:14:26.878Z", + "--window-hours", + "24", + "--scenarios", + "home,calibration", + "--runs", + "3", + "--skip-build", + ])! + + expect(options.database).toBe(source) + expect(options.windowEnd).toBe(1_785_824_066_878) + expect(options.windowStart).toBe(1_785_737_666_878) + expect(options.scenarios).toEqual(["home", "calibration"]) + expect(options.runs).toBe(3) + expect(options.build).toBe(false) +}) + +test("creates a consistent private partial database snapshot", async () => { + const options = parseOptions(["--db", source, "--window-end", "2026-08-04T06:14:26.878Z"])! + await createPartialSnapshot(source, partialSnapshot, options, []) + const copy = new Database(partialSnapshot, { readonly: true }) + expect(copy.query("SELECT value FROM sample").get()).toEqual({ value: "repeatable" }) + copy.close() + expect(await fingerprint(partialSnapshot)).toEqual({ + bytes: expect.any(Number), + sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }) +}) diff --git a/packages/app/e2e/performance/desktop-profile/options.ts b/packages/app/e2e/performance/desktop-profile/options.ts new file mode 100644 index 00000000000..fd5d6496759 --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/options.ts @@ -0,0 +1,88 @@ +import { Global } from "@opencode-ai/core/global" +import { existsSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import { scenarios, type Options, type Scenario } from "./types" + +const help = `Desktop renderer profiler + +Usage: + bun run profile:desktop [options] + +Options: + --mode local|partial-snapshot + Local corpus or fixed partial snapshot (default: local) + --db SQLite database (default: opencode data directory) + --partial-snapshot-out + Copy the benchmark corpus to a private partial snapshot + --output Report directory (default: OS temp directory) + --window-end End of corpus window (default: now; required for partial snapshot) + --window-hours Corpus window size (default: 24) + --scenarios Comma list: ${scenarios.join(",")} (default: all) + --runs Restart Electron and repeat (default: 1) + --skip-build Use the existing desktop production build + --diagnostics Capture Chrome traces + --cpu Capture sampled CPU summaries + --response-urls Attribute Response.text durations by URL + --help Show this message + +Partial snapshots contain private application data. Do not commit or share them. +` + +export function parseOptions(args: string[], now = Date.now()): Options | undefined { + if (args.includes("--help")) { + console.log(help) + return + } + + const value = (name: string) => { + const index = args.indexOf(name) + if (index === -1) return + const result = args[index + 1] + if (!result || result.startsWith("--")) throw new Error(`${name} requires a value`) + return result + } + const mode = value("--mode") ?? "local" + if (mode !== "local" && mode !== "partial-snapshot") throw new Error(`Unsupported mode: ${mode}`) + const endValue = value("--window-end") + if (mode === "partial-snapshot" && !endValue) + throw new Error("--window-end is required in partial-snapshot mode") + const windowEnd = endValue ? parseTime(endValue) : now + const windowHours = number(value("--window-hours") ?? "24", "--window-hours") + const selected = (value("--scenarios")?.split(",") ?? [...scenarios]).map((item) => item.trim()) + if (selected.some((item) => !scenarios.includes(item as Scenario))) + throw new Error(`--scenarios must contain only: ${scenarios.join(", ")}`) + const database = path.resolve(value("--db") ?? path.join(Global.Path.data, "opencode.db")) + if (!existsSync(database)) throw new Error(`Database does not exist: ${database}`) + + return { + mode, + database, + output: path.resolve( + value("--output") ?? path.join(tmpdir(), "opencode-performance", new Date(windowEnd).toISOString().replace(/[:.]/g, "-")), + ), + windowStart: windowEnd - windowHours * 60 * 60 * 1_000, + windowEnd, + scenarios: selected as Scenario[], + runs: number(value("--runs") ?? "1", "--runs"), + build: !args.includes("--skip-build"), + diagnostics: args.includes("--diagnostics"), + cpu: args.includes("--cpu"), + responseURLs: args.includes("--response-urls"), + partialSnapshotOut: value("--partial-snapshot-out") + ? path.resolve(value("--partial-snapshot-out")!) + : undefined, + } +} + +function parseTime(value: string) { + const result = /^\d+$/.test(value) ? Number(value) : Date.parse(value) + if (!Number.isFinite(result)) throw new Error(`Invalid --window-end: ${value}`) + return result +} + +function number(value: string, option: string) { + const result = Number(value) + if (!Number.isFinite(result) || result <= 0) throw new Error(`${option} must be greater than zero`) + return result +} diff --git a/packages/app/e2e/performance/desktop-profile/probe.ts b/packages/app/e2e/performance/desktop-profile/probe.ts new file mode 100644 index 00000000000..70b907c6227 --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/probe.ts @@ -0,0 +1,192 @@ +import type { Page } from "@playwright/test" +import type { Options, ProbeResult } from "./types" + +export async function installProbe(page: Page, options: Options) { + await page.addInitScript((attributeResponses) => { + const state = { + longTasks: [] as number[], + animationFrames: [] as ProbeResult["animationFrames"], + frameGaps: [] as number[], + responseText: [] as ProbeResult["responseText"], + } + ;(window as Window & { __opencodeRendererProfile?: typeof state }).__opencodeRendererProfile = state + if (PerformanceObserver.supportedEntryTypes.includes("longtask")) { + new PerformanceObserver((list) => + state.longTasks.push(...list.getEntries().map((entry) => entry.duration)), + ).observe({ type: "longtask" }) + } + if (PerformanceObserver.supportedEntryTypes.includes("long-animation-frame")) { + new PerformanceObserver((list) => + state.animationFrames.push( + ...list.getEntries().map((entry) => { + const frame = entry as PerformanceEntry & { + blockingDuration: number + scripts?: { + duration: number + forcedStyleAndLayoutDuration?: number + sourceFunctionName?: string + sourceURL?: string + sourceCharPosition?: number + invoker?: string + invokerType?: string + }[] + } + return { + duration: frame.duration, + blockingDuration: frame.blockingDuration, + forcedStyleAndLayoutDuration: + frame.scripts?.reduce((sum, script) => sum + (script.forcedStyleAndLayoutDuration ?? 0), 0) ?? 0, + scripts: + frame.scripts?.map((script) => ({ + function: script.sourceFunctionName || "(anonymous)", + source: script.sourceURL?.split("/").at(-1) || "(document)", + position: script.sourceCharPosition ?? -1, + invoker: script.invoker ?? "(unknown)", + invokerType: script.invokerType ?? "(unknown)", + duration: script.duration, + forcedStyleAndLayoutDuration: script.forcedStyleAndLayoutDuration ?? 0, + })) ?? [], + } + }), + ), + ).observe({ type: "long-animation-frame" }) + } + let previous = performance.now() + const frame = (now: number) => { + const gap = now - previous + if (gap > 20) state.frameGaps.push(gap) + previous = now + requestAnimationFrame(frame) + } + requestAnimationFrame(frame) + if (!attributeResponses) return + const responseText = Response.prototype.text + Response.prototype.text = function () { + const started = performance.now() + const url = this.url + return responseText.call(this).then((text) => { + state.responseText.push({ url, duration: performance.now() - started }) + return text + }) + } + }, options.responseURLs) +} + +export async function resetProbe(page: Page) { + await page.evaluate(() => { + const state = (window as Window & { __opencodeRendererProfile?: ProbeResult }).__opencodeRendererProfile + if (!state) return + state.longTasks.length = 0 + state.animationFrames.length = 0 + state.frameGaps.length = 0 + state.responseText.length = 0 + }) +} + +export async function collectProbe(page: Page) { + return page.evaluate( + () => (window as Window & { __opencodeRendererProfile?: ProbeResult }).__opencodeRendererProfile!, + ) +} + +export function summarizeProbe(probe: ProbeResult) { + const scripts = new Map< + string, + { + function: string + source: string + position: number + invoker: string + invokerType: string + durationMs: number + forcedStyleAndLayoutMs: number + } + >() + probe.animationFrames + .flatMap((frame) => frame.scripts) + .forEach((script) => { + const key = `${script.source}:${script.position}:${script.invoker}` + const current = scripts.get(key) ?? { + function: script.function, + source: script.source, + position: script.position, + invoker: script.invoker, + invokerType: script.invokerType, + durationMs: 0, + forcedStyleAndLayoutMs: 0, + } + current.durationMs += script.duration + current.forcedStyleAndLayoutMs += script.forcedStyleAndLayoutDuration + scripts.set(key, current) + }) + return { + longTasks: { + count: probe.longTasks.length, + totalMs: sum(probe.longTasks), + maxMs: Math.max(0, ...probe.longTasks), + }, + longAnimationFrames: { + count: probe.animationFrames.length, + totalBlockingMs: sum(probe.animationFrames.map((frame) => frame.blockingDuration)), + maxDurationMs: Math.max(0, ...probe.animationFrames.map((frame) => frame.duration)), + forcedStyleAndLayoutMs: sum(probe.animationFrames.map((frame) => frame.forcedStyleAndLayoutDuration)), + scripts: [...scripts.values()].sort((a, b) => b.durationMs - a.durationMs).slice(0, 15), + }, + frameGaps: { + count: probe.frameGaps.length, + maxMs: Math.max(0, ...probe.frameGaps), + }, + responseText: probe.responseText + .map((item) => ({ path: responsePath(item.url), durationMs: item.duration })) + .sort((a, b) => b.durationMs - a.durationMs), + } +} + +export async function startCPUProfile(page: Page, enabled: boolean) { + if (!enabled) return { stop: async () => [] } + const session = await page.context().newCDPSession(page) + await session.send("Profiler.enable") + await session.send("Profiler.setSamplingInterval", { interval: 1_000 }) + await session.send("Profiler.start") + return { + async stop() { + const result = await session.send("Profiler.stop") + await session.detach() + const self = new Map() + result.profile.samples?.forEach((id, index) => { + self.set(id, (self.get(id) ?? 0) + (result.profile.timeDeltas?.[index] ?? 0) / 1_000) + }) + return result.profile.nodes + .map((node) => ({ + function: node.callFrame.functionName || "(anonymous)", + source: sourceName(node.callFrame.url), + line: node.callFrame.lineNumber + 1, + selfMs: self.get(node.id) ?? 0, + })) + .filter((node) => node.selfMs >= 1) + .sort((a, b) => b.selfMs - a.selfMs) + .slice(0, 40) + }, + } +} + +function responsePath(value: string) { + try { + return new URL(value).pathname + } catch { + return value + } +} + +function sourceName(value: string) { + if (!value) return "(native)" + try { + return new URL(value).pathname.split("/").at(-1) || "(document)" + } catch { + return value.split(/[\\/]/).at(-1) || value + } +} + +function sum(values: number[]) { + return values.reduce((total, value) => total + value, 0) +} diff --git a/packages/app/e2e/performance/desktop-profile/progress.ts b/packages/app/e2e/performance/desktop-profile/progress.ts new file mode 100644 index 00000000000..4799e5b14b5 --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/progress.ts @@ -0,0 +1,7 @@ +const started = performance.now() + +export function progress(message: string, details?: Record) { + const elapsed = ((performance.now() - started) / 1_000).toFixed(1) + const suffix = details ? ` ${JSON.stringify(details)}` : "" + console.error(`[desktop-profile +${elapsed}s] ${message}${suffix}`) +} diff --git a/packages/app/e2e/performance/desktop-profile/runtime.ts b/packages/app/e2e/performance/desktop-profile/runtime.ts new file mode 100644 index 00000000000..d6379696eb3 --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/runtime.ts @@ -0,0 +1,156 @@ +import { chromium, type Page } from "@playwright/test" +import { copyFile, mkdir, rm } from "node:fs/promises" +import path from "node:path" +import { prepareDesktopState } from "./desktop-state" +import { progress } from "./progress" +import type { Options } from "./types" + +export async function withDesktop( + options: Options, + desktop: string, + run: number, + projectIDs: string[], + use: (page: Page) => Promise, +) { + const port = availablePort() + const endpoint = `http://127.0.0.1:${port}` + const userData = path.join(options.output, `user-data-${run}`) + const database = + options.mode === "partial-snapshot" ? path.join(options.output, `working-database-${run}.db`) : options.database + await rm(userData, { recursive: true, force: true }) + await mkdir(userData, { recursive: true }) + if (database !== options.database) await copyFile(options.database, database) + await prepareDesktopState(options, database, userData, run, projectIDs) + const electron = path.join( + desktop, + "node_modules", + "electron", + "dist", + (await Bun.file(path.join(desktop, "node_modules", "electron", "path.txt")).text()).trim(), + ) + progress("launching Electron", { run, port }) + const child = Bun.spawn([electron, "."], { + cwd: desktop, + env: { + ...process.env, + OPENCODE_DB: database, + OPENCODE_CHANNEL: "dev", + OPENCODE_PROFILE_LOAF: "1", + OPENCODE_PROFILE_CDP_PORT: String(port), + OPENCODE_PROFILE_USER_DATA: userData, + OPENCODE_PERFORMANCE_TRACE_DIR: options.diagnostics ? path.join(options.output, "traces", String(run)) : "", + OPENCODE_PERFORMANCE_RUN_ID: `desktop-${run}`, + }, + stdout: "pipe", + stderr: "pipe", + }) + const stdout = drain(child.stdout, "stdout") + const stderr = drain(child.stderr, "stderr") + let browser: Awaited> | undefined + + try { + progress("waiting for CDP", { run }) + await waitForCDP(endpoint, child, stdout, stderr) + progress("connecting Playwright", { run }) + browser = await chromium.connectOverCDP(endpoint) + progress("waiting for renderer", { run }) + const page = await waitForRenderer(browser) + progress("waiting for desktop API", { run }) + await page.waitForFunction(() => typeof window.api === "object", undefined, { timeout: 60_000 }) + progress("desktop ready", { run }) + return await use(page) + } finally { + progress("stopping Electron", { run }) + await browser?.close().catch(() => {}) + await killTree(child.pid) + await Promise.allSettled([stdout, stderr]) + if (database !== options.database) { + await Bun.sleep(500) + await rm(database, { force: true }).catch(() => undefined) + } + } +} + +export async function run(command: string[], cwd: string, database: string) { + const child = Bun.spawn(command, { + cwd, + env: { ...process.env, OPENCODE_DB: database, OPENCODE_CHANNEL: "dev" }, + stdout: "inherit", + stderr: "inherit", + }) + const code = await child.exited + if (code !== 0) throw new Error(`${command.join(" ")} exited with ${code}`) +} + +function availablePort() { + const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response() }) + const port = server.port + server.stop(true) + return port +} + +async function waitForCDP( + endpoint: string, + child: ReturnType, + stdout: Promise, + stderr: Promise, +) { + const timeout = Date.now() + 5 * 60_000 + let heartbeat = Date.now() + 10_000 + while (Date.now() < timeout) { + const ready = await fetch(`${endpoint}/json/version`) + .then((response) => response.ok) + .catch(() => false) + if (ready) return + if (child.exitCode !== null) + throw new Error(`Desktop exited before CDP was ready (${child.exitCode})\n${await stdout}\n${await stderr}`) + if (Date.now() >= heartbeat) { + progress("still waiting for CDP") + heartbeat = Date.now() + 10_000 + } + await Bun.sleep(250) + } + throw new Error("Timed out waiting for desktop CDP") +} + +async function waitForRenderer(browser: Awaited>) { + const timeout = Date.now() + 60_000 + let heartbeat = Date.now() + 10_000 + while (Date.now() < timeout) { + const page = browser + .contexts() + .flatMap((context) => context.pages()) + .find((candidate) => candidate.url().startsWith("oc://renderer")) + if (page) return page + if (Date.now() >= heartbeat) { + progress("still waiting for renderer") + heartbeat = Date.now() + 10_000 + } + await Bun.sleep(100) + } + throw new Error("Desktop renderer target was not found") +} + +async function drain(stream: ReadableStream, label: string) { + const decoder = new TextDecoder() + let output = "" + let pending = "" + for await (const chunk of stream) { + const text = decoder.decode(chunk, { stream: true }) + output = (output + text).slice(-50_000) + const lines = (pending + text).split(/\r?\n/) + pending = lines.pop() ?? "" + lines.filter(Boolean).forEach((line) => progress(`Electron ${label}`, { line: line.slice(0, 500) })) + } + if (pending) progress(`Electron ${label}`, { line: pending.slice(0, 500) }) + return output + decoder.decode() +} + +async function killTree(pid: number) { + if (process.platform !== "win32") { + process.kill(pid, "SIGTERM") + return + } + const child = Bun.spawn(["taskkill", "/pid", String(pid), "/T", "/F"], { stdout: "ignore", stderr: "ignore" }) + await child.exited +} diff --git a/packages/app/e2e/performance/desktop-profile/scenario-utils.ts b/packages/app/e2e/performance/desktop-profile/scenario-utils.ts new file mode 100644 index 00000000000..9285b8ee8ed --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/scenario-utils.ts @@ -0,0 +1,75 @@ +import type { Page } from "@playwright/test" +import { progress } from "./progress" + +export async function setDesktopRoute(page: Page, route: string) { + await page.evaluate(async (value) => { + const api = window.api as typeof window.api & { getWindowID?: () => Promise } + const id = (await api.getWindowID?.()) ?? "browser" + localStorage.setItem(`opencode.desktop.window.${id}.last-active-url`, value) + }, route) +} + +export async function waitForQuietDOM(page: Page) { + progress("waiting for DOM to settle") + await page.evaluate( + () => + new Promise((resolve) => { + let settled = false + let timer = setTimeout(done, 750) + const deadline = setTimeout(done, 30_000) + const observer = new MutationObserver(() => { + clearTimeout(timer) + timer = setTimeout(done, 750) + }) + observer.observe(document.body, { childList: true, subtree: true, characterData: true }) + function done() { + if (settled) return + settled = true + clearTimeout(deadline) + observer.disconnect() + requestAnimationFrame(() => requestAnimationFrame(() => resolve())) + } + }), + ) + progress("DOM settled") +} + +export async function waitForSelector(page: Page, selector: string, label: string) { + progress("waiting for UI", { label }) + try { + await page.waitForSelector(selector, { timeout: 30_000 }) + } catch (error) { + progress("UI wait failed", { + label, + url: page.url(), + body: (await page.locator("body").innerText().catch(() => "")).replace(/\s+/g, " ").slice(0, 500), + }) + throw error + } + progress("UI ready", { label }) +} + +export async function domCounts(page: Page, review = false) { + return page.evaluate((review) => ({ + elements: document.getElementsByTagName("*").length, + ...(review + ? { + diffViewers: document.querySelectorAll('[data-component="file"][data-mode="diff"]').length, + diffLines: document.querySelectorAll("[data-line]").length, + } + : { + timelineRows: document.querySelectorAll("[data-timeline-row]").length, + messageRows: document.querySelectorAll("[data-message-id]").length, + markdownRoots: document.querySelectorAll('[data-component="markdown"]').length, + diffViewers: document.querySelectorAll('[data-component="file"][data-mode="diff"]').length, + }), + }), review) +} + +export function sum(values: number[]) { + return values.reduce((total, value) => total + value, 0) +} + +export function percentile(values: number[], quantile: number) { + return values.toSorted((a, b) => a - b)[Math.max(0, Math.ceil(values.length * quantile) - 1)] ?? 0 +} diff --git a/packages/app/e2e/performance/desktop-profile/scenarios.ts b/packages/app/e2e/performance/desktop-profile/scenarios.ts new file mode 100644 index 00000000000..3f05fe0d06b --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/scenarios.ts @@ -0,0 +1,178 @@ +import type { Page } from "@playwright/test" +import { base64Encode } from "@opencode-ai/core/util/encode" +import { startChromeTrace } from "../chrome-trace" +import { collectProbe, resetProbe, startCPUProfile, summarizeProbe } from "./probe" +import { progress } from "./progress" +import { domCounts, percentile, setDesktopRoute, sum, waitForQuietDOM, waitForSelector } from "./scenario-utils" +import type { Options, Target } from "./types" + +export async function runScenarios(page: Page, options: Options, targets: Target[], typingText: string) { + const results: unknown[] = [] + if (options.scenarios.includes("home")) results.push(await profileHome(page, options)) + if (options.scenarios.includes("calibration")) results.push(await profileCalibration(page)) + if (options.scenarios.includes("session")) { + for (const target of targets) results.push(await profileSession(page, options, target)) + } + if (options.scenarios.some((scenario) => ["composer", "history", "review"].includes(scenario))) { + await openSession(page, targets[2]!) + } + if (options.scenarios.includes("composer")) results.push(await profileComposer(page, options, typingText)) + if (options.scenarios.includes("history")) results.push(await profileHistory(page, options, targets[2]!)) + if (options.scenarios.includes("review")) { + const review = await profileReview(page, options) + if (review) results.push(review) + } + return results +} + +async function profileHome(page: Page, options: Options) { + const measured = await measure(page, options, "home", async () => { + await setDesktopRoute(page, "/") + await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 }) + await waitForSelector(page, '[data-component="home-session-row"]', "Home session rows") + await waitForQuietDOM(page) + }) + return { ...measured, dom: await domCounts(page) } +} + +async function profileCalibration(page: Page) { + await resetProbe(page) + await page.evaluate( + () => + new Promise((resolve) => { + setTimeout(function opencodeProfileCalibration() { + const end = performance.now() + 80 + while (performance.now() < end) { + // Deliberate benchmark-only main-thread block. + } + requestAnimationFrame(() => setTimeout(resolve, 100)) + }) + }), + ) + return { name: "attribution-calibration", ...summarizeProbe(await collectProbe(page)) } +} + +async function profileSession(page: Page, options: Options, target: Target) { + await prepareHome(page) + const measured = await measure(page, options, `session-${target.label}`, async () => { + await navigateSession(page, target) + await waitForSelector(page, '[data-component="prompt-input"]', "session composer") + await waitForQuietDOM(page) + }) + return { ...measured, context: targetContext(target), dom: await domCounts(page) } +} + +async function profileComposer(page: Page, options: Options, typingText: string) { + const editor = page.locator('[data-component="prompt-input"][contenteditable="true"]').first() + await editor.click() + await page.keyboard.press("Control+A") + await page.keyboard.press("Backspace") + const printable = [...typingText].filter((character) => !["\r", "\n", "\t"].includes(character)) + const measuredText = printable.slice(-120).join("") + const prefix = printable.slice(0, -measuredText.length).join("") + if (prefix) await page.keyboard.insertText(prefix) + await waitForQuietDOM(page) + const durations: number[] = [] + const measured = await measure(page, options, "composer-typing", async () => { + for (const character of measuredText) { + const started = performance.now() + await page.keyboard.type(character) + durations.push(performance.now() - started) + } + await waitForQuietDOM(page) + }) + await page.keyboard.press("Control+A") + await page.keyboard.press("Backspace") + return { + ...measured, + context: { promptCharacters: printable.length, measuredCharacters: measuredText.length }, + typing: { + totalMs: sum(durations), + meanMs: sum(durations) / durations.length, + p50Ms: percentile(durations, 0.5), + p95Ms: percentile(durations, 0.95), + maxMs: Math.max(...durations), + }, + } +} + +async function profileHistory(page: Page, options: Options, target: Target) { + await waitForSelector(page, '[data-component="prompt-input"]', "history session composer") + await waitForQuietDOM(page) + let requests = 0 + const onResponse = (response: { url(): string }) => { + if (/\/session\/[^/]+\/message(?:\?|$)/.test(response.url())) requests++ + } + page.on("response", onResponse) + const measured = await measure(page, options, "session-max-history-boundary", async () => { + const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") }).first() + await scroller.evaluate((element) => { + element.scrollTop = 0 + element.dispatchEvent(new WheelEvent("wheel", { deltaY: -10_000, bubbles: true })) + element.dispatchEvent(new Event("scroll", { bubbles: true })) + }) + const timeout = Date.now() + 60_000 + while (requests === 0 && Date.now() < timeout) await page.waitForTimeout(50) + if (requests === 0) throw new Error("History boundary did not request a page") + await waitForQuietDOM(page) + }) + page.off("response", onResponse) + return { ...measured, context: targetContext(target), messageRequests: requests } +} + +async function profileReview(page: Page, options: Options) { + const button = page.getByRole("button", { name: "Toggle review" }) + if (!(await button.isVisible().catch(() => false))) return + const panel = page.locator("#review-panel") + if (await panel.isVisible().catch(() => false)) { + await button.click() + await panel.waitFor({ state: "hidden", timeout: 60_000 }) + await waitForQuietDOM(page) + } + const measured = await measure(page, options, "review-open", async () => { + await button.click() + await panel.waitFor({ state: "visible", timeout: 60_000 }) + await waitForQuietDOM(page) + }) + return { ...measured, dom: await domCounts(page, true) } +} + +async function measure(page: Page, options: Options, name: string, action: () => Promise) { + progress("scenario started", { name }) + await resetProbe(page) + const stopTrace = options.diagnostics ? await startChromeTrace(page, name) : undefined + const cpu = await startCPUProfile(page, options.cpu) + const started = performance.now() + await action() + const result = { + name, + elapsedMs: performance.now() - started, + ...summarizeProbe(await collectProbe(page)), + cpu: await cpu.stop(), + trace: await stopTrace?.(), + } + progress("scenario completed", { name, elapsedMs: Math.round(result.elapsedMs), longTasks: result.longTasks.count }) + return result +} + +async function openSession(page: Page, target: Target) { + await navigateSession(page, target) + await waitForSelector(page, '[data-component="prompt-input"]', "session composer") + await waitForQuietDOM(page) +} + +async function prepareHome(page: Page) { + await setDesktopRoute(page, "/") + await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 }) + await waitForSelector(page, '[data-component="home-session-row"]', "Home session rows") + await waitForQuietDOM(page) +} + +async function navigateSession(page: Page, target: Target) { + await setDesktopRoute(page, `/server/${base64Encode("sidecar")}/session/${target.id}`) + await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 }) +} + +function targetContext(target: Target) { + return { serializedBytes: target.bytes, messages: target.messages, parts: target.parts, userTurns: target.userTurns } +} diff --git a/packages/app/e2e/performance/desktop-profile/types.ts b/packages/app/e2e/performance/desktop-profile/types.ts new file mode 100644 index 00000000000..b3d728b2887 --- /dev/null +++ b/packages/app/e2e/performance/desktop-profile/types.ts @@ -0,0 +1,50 @@ +export const scenarios = ["home", "calibration", "session", "composer", "history", "review"] as const + +export type Scenario = (typeof scenarios)[number] + +export type Options = { + mode: "local" | "partial-snapshot" + database: string + output: string + windowStart: number + windowEnd: number + scenarios: Scenario[] + runs: number + build: boolean + diagnostics: boolean + cpu: boolean + responseURLs: boolean + partialSnapshotOut?: string +} + +export type Target = { + label: "p50" | "p95" | "max" + id: string + projectID: string + directory: string + title: string + bytes: number + messages: number + parts: number + userTurns: number +} + +export type ProbeResult = { + longTasks: number[] + animationFrames: { + duration: number + blockingDuration: number + forcedStyleAndLayoutDuration: number + scripts: { + function: string + source: string + position: number + invoker: string + invokerType: string + duration: number + forcedStyleAndLayoutDuration: number + }[] + }[] + frameGaps: number[] + responseText: { url: string; duration: number }[] +} diff --git a/packages/app/e2e/performance/profile-desktop.ts b/packages/app/e2e/performance/profile-desktop.ts new file mode 100644 index 00000000000..896c6c76f9a --- /dev/null +++ b/packages/app/e2e/performance/profile-desktop.ts @@ -0,0 +1,107 @@ +import { mkdir } from "node:fs/promises" +import path from "node:path" +import { createPartialSnapshot, fingerprint, loadCorpus } from "./desktop-profile/corpus" +import { parseOptions } from "./desktop-profile/options" +import { installProbe } from "./desktop-profile/probe" +import { progress } from "./desktop-profile/progress" +import { withDesktop, run } from "./desktop-profile/runtime" +import { runScenarios } from "./desktop-profile/scenarios" + +const root = path.resolve(import.meta.dir, "../../../..") +const desktop = path.join(root, "packages/desktop") +const options = parseOptions(process.argv.slice(2)) +if (!options) process.exit(0) + +await mkdir(options.output, { recursive: true }) +progress("loading corpus", { mode: options.mode }) +let corpus = loadCorpus(options) +if (options.partialSnapshotOut) { + progress("creating partial snapshot") + await createPartialSnapshot(options.database, options.partialSnapshotOut, options, corpus.targets) + options.database = options.partialSnapshotOut + options.mode = "partial-snapshot" + corpus = loadCorpus(options) +} +if (options.build) { + progress("building desktop production bundle") + await run(["bun", "run", "build"], desktop, options.database) +} + +progress("corpus ready", { targets: corpus.targets.map((target) => target.label), runs: options.runs }) +const runs = [] +for (let index = 1; index <= options.runs; index++) { + runs.push( + await withDesktop(options, desktop, index, corpus.projectIDs, async (page) => { + await installProbe(page, options) + await page.evaluate(() => { + const settings = JSON.parse(localStorage.getItem("settings.v3") ?? "{}") + localStorage.setItem( + "settings.v3", + JSON.stringify({ ...settings, general: { ...settings.general, newLayoutDesigns: true } }), + ) + }) + return runScenarios(page, options, corpus.targets, corpus.typingText) + }), + ) +} + +const report = { + schemaVersion: 2, + source: options.mode === "partial-snapshot" ? "partial-database-snapshot" : "local-opencode-db", + command: process.argv.slice(2), + diagnostics: options.diagnostics, + profileCPU: options.cpu, + database: await fingerprint(options.database), + window: { + start: new Date(options.windowStart).toISOString(), + end: new Date(options.windowEnd).toISOString(), + }, + revision: (await Bun.$`git rev-parse HEAD`.cwd(root).text()).trim(), + targets: corpus.targets.map(({ id: _, projectID: __, directory: ___, title: ____, ...target }) => target), + summary: summarize(runs), + runs: runs.map((results, index) => ({ index: index + 1, results })), +} +const file = path.join(options.output, "renderer-profile.json") +await Bun.write(file, JSON.stringify(report, null, 2)) +console.log(`PROFILE_REPORT ${file}`) +console.log(`PROFILE_SUMMARY ${JSON.stringify(report.summary)}`) +console.log(JSON.stringify(report, null, 2)) + +function summarize(runs: unknown[][]) { + type Result = { + name: string + elapsedMs?: number + longTasks: { count: number; totalMs: number; maxMs: number } + longAnimationFrames: { totalBlockingMs: number } + typing?: { p50Ms: number; p95Ms: number; maxMs: number } + } + return Object.fromEntries( + [...Map.groupBy(runs.flat() as Result[], (result) => result.name)].map(([name, samples]) => [ + name, + { + samples: samples.length, + elapsedMedianMs: median(samples.flatMap((sample) => sample.elapsedMs ?? [])), + longTasks: { + maxCount: Math.max(...samples.map((sample) => sample.longTasks.count)), + maxTotalMs: Math.max(...samples.map((sample) => sample.longTasks.totalMs)), + maxTaskMs: Math.max(...samples.map((sample) => sample.longTasks.maxMs)), + }, + maxBlockingMs: Math.max(...samples.map((sample) => sample.longAnimationFrames.totalBlockingMs)), + ...(samples[0]?.typing + ? { + typingMedianMs: { + p50: median(samples.flatMap((sample) => sample.typing?.p50Ms ?? [])), + p95: median(samples.flatMap((sample) => sample.typing?.p95Ms ?? [])), + max: median(samples.flatMap((sample) => sample.typing?.maxMs ?? [])), + }, + } + : {}), + }, + ]), + ) +} + +function median(values: number[]) { + if (values.length === 0) return + return values.toSorted((a, b) => a - b)[Math.floor(values.length / 2)] +} diff --git a/packages/app/e2e/performance/real-desktop-profile.ts b/packages/app/e2e/performance/real-desktop-profile.ts index 2e3dbaaab7f..cd7a85fd2b2 100644 --- a/packages/app/e2e/performance/real-desktop-profile.ts +++ b/packages/app/e2e/performance/real-desktop-profile.ts @@ -1,675 +1 @@ -import { Database } from "bun:sqlite" -import { chromium, type CDPSession, type Page } from "@playwright/test" -import path from "node:path" -import { startChromeTrace } from "./chrome-trace" - -const root = path.resolve(import.meta.dir, "../../../..") -const desktop = path.join(root, "packages/desktop") -const databasePath = process.env.OPENCODE_PROFILE_DB ?? "C:/Users/Lukem/.local/share/opencode/opencode.db" -const output = process.env.OPENCODE_PROFILE_OUTPUT ?? "C:/tmp/opencode/markdown-profile-results" -const cdpPort = process.env.OPENCODE_PROFILE_CDP_PORT ?? String(19_000 + (process.pid % 1_000)) -const endpoint = process.env.OPENCODE_PROFILE_CDP ?? `http://127.0.0.1:${cdpPort}` -const diagnostics = process.env.OPENCODE_PROFILE_DIAGNOSTICS !== "0" -const profileCPU = process.env.OPENCODE_PROFILE_CPU === "1" -const windowEnd = Number(process.env.OPENCODE_PROFILE_WINDOW_END ?? Date.now()) -const windowStart = windowEnd - 24 * 60 * 60 * 1_000 - -type Target = { - label: "p50" | "p95" | "max" - id: string - directory: string - title: string - bytes: number - messages: number - parts: number - userTurns: number -} - -type ProbeResult = { - longTasks: number[] - animationFrames: { - duration: number - blockingDuration: number - forcedStyleAndLayoutDuration: number - scripts: { - function: string - source: string - position: number - invoker: string - invokerType: string - duration: number - forcedStyleAndLayoutDuration: number - }[] - }[] - frameGaps: number[] - responseText: { url: string; duration: number }[] -} - -const targets = loadTargets() -const typingText = loadTypingText(targets.find((target) => target.label === "max")!) -await Bun.$`mkdir -p ${output}` -process.env.OPENCODE_PERFORMANCE_TRACE_DIR = path.join(output, "traces") -process.env.OPENCODE_PERFORMANCE_RUN_ID = new Date(windowEnd).toISOString().replace(/[:.]/g, "-") - -if (process.env.OPENCODE_PROFILE_SKIP_BUILD !== "1") await run(["bun", "run", "build"], desktop) - -const child = Bun.spawn(["bun", "run", "preview"], { - cwd: desktop, - env: { - ...process.env, - OPENCODE_DB: databasePath, - OPENCODE_CHANNEL: "dev", - OPENCODE_PROFILE_LOAF: "1", - OPENCODE_PROFILE_CDP_PORT: cdpPort, - OPENCODE_PROFILE_USER_DATA: - process.env.OPENCODE_PROFILE_USER_DATA ?? "C:/tmp/opencode/markdown-profile-user-data", - }, - stdout: "pipe", - stderr: "pipe", -}) -const stdout = drain(child.stdout) -const stderr = drain(child.stderr) -let browser: Awaited> | undefined - -try { - await waitForCDP() - browser = await chromium.connectOverCDP(endpoint) - const page = await waitForRenderer(browser) - await page.waitForFunction(() => typeof window.api === "object", undefined, { timeout: 60_000 }) - await installProbe(page) - await page.evaluate(() => { - const settings = JSON.parse(localStorage.getItem("settings.v3") ?? "{}") - localStorage.setItem( - "settings.v3", - JSON.stringify({ ...settings, general: { ...settings.general, newLayoutDesigns: true } }), - ) - }) - - const results = [] - results.push(await profileHome(page)) - results.push(await profileCalibration(page)) - for (const target of targets) { - results.push(await profileSession(page, target)) - } - results.push(await profileComposer(page)) - results.push(await profileHistoryBoundary(page, targets.find((target) => target.label === "max")!)) - const review = await profileReview(page) - if (review) results.push(review) - - const report = { - schemaVersion: 1, - source: "real-opencode-db", - diagnostics, - profileCPU, - window: { - start: new Date(windowStart).toISOString(), - end: new Date(windowEnd).toISOString(), - }, - revision: (await Bun.$`git rev-parse HEAD`.cwd(root).text()).trim(), - targets: targets.map(({ id: _, directory: __, title: ___, ...target }) => target), - results, - } - const file = path.join(output, "renderer-profile.json") - await Bun.write(file, JSON.stringify(report, null, 2)) - console.log(`PROFILE_REPORT ${file}`) - console.log(JSON.stringify(report, null, 2)) -} finally { - await browser?.close().catch(() => {}) - await killTree(child.pid) - await Promise.allSettled([stdout, stderr]) -} - -async function profileCalibration(page: Page) { - await resetProbe(page) - await page.evaluate( - () => - new Promise((resolve) => { - setTimeout(function opencodeProfileCalibration() { - const end = performance.now() + 80 - while (performance.now() < end) { - // Deliberate benchmark-only main-thread block. - } - requestAnimationFrame(() => setTimeout(resolve, 100)) - }) - }), - ) - const metrics = await collectProbe(page) - return { name: "attribution-calibration", ...summarizeProbe(metrics) } -} - -function loadTargets() { - const database = new Database(databasePath, { readonly: true }) - database.run("PRAGMA query_only = ON") - const sessions = database - .query( - `SELECT id, directory, title - FROM session AS candidate - WHERE parent_id IS NULL - AND EXISTS ( - SELECT 1 - FROM message - WHERE session_id = candidate.id AND time_created >= ? AND time_created < ? - )`, - ) - .all(windowStart, windowEnd) as { id: string; directory: string; title: string }[] - const messageRows = database.query( - `SELECT id, data - FROM message - WHERE session_id = ? AND time_created >= ? AND time_created < ? - ORDER BY time_created, id`, - ) - const partRows = database.query(`SELECT data FROM part WHERE message_id = ? ORDER BY id`) - const ranked = sessions - .map((session) => { - const messages = messageRows.all(session.id, windowStart, windowEnd) as { id: string; data: string }[] - const parts = messages.flatMap((message) => partRows.all(message.id) as { data: string }[]) - return { - ...session, - bytes: - messages.reduce((sum, message) => sum + Buffer.byteLength(message.data), 0) + - parts.reduce((sum, part) => sum + Buffer.byteLength(part.data), 0), - messages: messages.length, - parts: parts.length, - userTurns: messages.filter((message) => JSON.parse(message.data).role === "user").length, - } - }) - .filter((session) => session.messages > 0) - .sort((a, b) => a.bytes - b.bytes || a.id.localeCompare(b.id)) - database.close() - if (ranked.length === 0) throw new Error("No sessions found in the profile window") - const select = (label: Target["label"], percentile: number) => ({ - label, - ...ranked[Math.max(0, Math.ceil(ranked.length * percentile) - 1)]!, - }) - return [select("p50", 0.5), select("p95", 0.95), select("max", 1)] satisfies Target[] -} - -function loadTypingText(target: Target) { - const database = new Database(databasePath, { readonly: true }) - database.run("PRAGMA query_only = ON") - const messages = database - .query( - `SELECT id, data - FROM message - WHERE session_id = ? AND time_created >= ? AND time_created < ? - ORDER BY time_created, id`, - ) - .all(target.id, windowStart, windowEnd) as { id: string; data: string }[] - const parts = database.query(`SELECT data FROM part WHERE message_id = ? ORDER BY id`) - const text = messages - .filter((message) => JSON.parse(message.data).role === "user") - .flatMap((message) => - (parts.all(message.id) as { data: string }[]).flatMap((part) => { - const data = JSON.parse(part.data) - return data.type === "text" && typeof data.text === "string" ? [data.text] : [] - }), - ) - .sort((a, b) => b.length - a.length)[0] - database.close() - if (!text) throw new Error("No real user prompt found for composer profiling") - return text -} - -async function profileHome(page: Page) { - const stopTrace = diagnostics ? await startChromeTrace(page, "home") : undefined - const cpu = await startCPUProfile(page) - const started = performance.now() - await setDesktopRoute(page, "/") - await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 }) - await page.waitForSelector('[data-component="home-session-row"]', { timeout: 60_000 }) - await waitForQuietDOM(page) - const elapsedMs = performance.now() - started - const metrics = await collectProbe(page) - const dom = await page.evaluate(() => ({ - elements: document.getElementsByTagName("*").length, - timelineRows: document.querySelectorAll("[data-timeline-row]").length, - messageRows: document.querySelectorAll("[data-message-id]").length, - markdownRoots: document.querySelectorAll('[data-component="markdown"]').length, - diffViewers: document.querySelectorAll('[data-component="file"][data-mode="diff"]').length, - })) - return { - name: "home", - elapsedMs, - ...summarizeProbe(metrics), - dom, - cpu: await cpu.stop(), - trace: await stopTrace?.(), - } -} - -async function profileSession(page: Page, target: Target) { - await setDesktopRoute(page, "/") - await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 }) - await page.waitForSelector('[data-component="home-session-row"]', { timeout: 60_000 }) - await waitForQuietDOM(page) - await resetProbe(page) - const stopTrace = diagnostics ? await startChromeTrace(page, `session-${target.label}`) : undefined - const cpu = await startCPUProfile(page) - const started = performance.now() - await page.evaluate((title) => { - const button = [...document.querySelectorAll('[data-component="home-session-row"]')].find( - (element) => element.textContent?.includes(title), - ) - if (!button) throw new Error("Ranked root session was not found on Home") - button.click() - }, target.title) - await page.waitForSelector('[data-component="prompt-input"]', { timeout: 60_000 }) - await waitForQuietDOM(page) - const elapsedMs = performance.now() - started - const metrics = await collectProbe(page) - const dom = await page.evaluate(() => ({ - elements: document.getElementsByTagName("*").length, - timelineRows: document.querySelectorAll("[data-timeline-row]").length, - messageRows: document.querySelectorAll("[data-message-id]").length, - markdownRoots: document.querySelectorAll('[data-component="markdown"]').length, - diffViewers: document.querySelectorAll('[data-component="file"][data-mode="diff"]').length, - })) - return { - name: `session-${target.label}`, - context: { - serializedBytes: target.bytes, - messages: target.messages, - parts: target.parts, - userTurns: target.userTurns, - }, - elapsedMs, - ...summarizeProbe(metrics), - dom, - cpu: await cpu.stop(), - trace: await stopTrace?.(), - } -} - -async function profileComposer(page: Page) { - const editor = page.locator('[data-component="prompt-input"][contenteditable="true"]').first() - await editor.click() - await page.keyboard.press("Control+A") - await page.keyboard.press("Backspace") - const printable = [...typingText].filter( - (character) => character !== "\r" && character !== "\n" && character !== "\t", - ) - const measured = printable.slice(-120).join("") - const prefix = printable.slice(0, -measured.length).join("") - if (prefix) await page.keyboard.insertText(prefix) - await waitForQuietDOM(page) - await resetProbe(page) - const stopTrace = diagnostics ? await startChromeTrace(page, "composer-typing") : undefined - const cpu = await startCPUProfile(page) - const durations: number[] = [] - for (const character of measured) { - const started = performance.now() - await page.keyboard.type(character) - durations.push(performance.now() - started) - } - await waitForQuietDOM(page) - const metrics = await collectProbe(page) - await page.keyboard.press("Control+A") - await page.keyboard.press("Backspace") - return { - name: "composer-typing", - context: { promptCharacters: printable.length, measuredCharacters: measured.length }, - typing: { - totalMs: sum(durations), - meanMs: sum(durations) / durations.length, - p50Ms: percentile(durations, 0.5), - p95Ms: percentile(durations, 0.95), - maxMs: Math.max(...durations), - }, - ...summarizeProbe(metrics), - cpu: await cpu.stop(), - trace: await stopTrace?.(), - } -} - -async function profileHistoryBoundary(page: Page, target: Target) { - await page.waitForSelector('[data-component="prompt-input"]', { timeout: 60_000 }) - await waitForQuietDOM(page) - await resetProbe(page) - const stopTrace = diagnostics ? await startChromeTrace(page, "session-max-history-boundary") : undefined - const cpu = await startCPUProfile(page) - const scroller = page.locator(".scroll-view__viewport", { has: page.locator("[data-timeline-row]") }).first() - const started = performance.now() - let requests = 0 - const onResponse = (response: { url(): string }) => { - if (/\/session\/[^/]+\/message(?:\?|$)/.test(response.url())) requests++ - } - page.on("response", onResponse) - await scroller.evaluate((element) => { - element.scrollTop = 0 - element.dispatchEvent(new WheelEvent("wheel", { deltaY: -10_000, bubbles: true })) - element.dispatchEvent(new Event("scroll", { bubbles: true })) - }) - const timeout = Date.now() + 60_000 - while (requests === 0 && Date.now() < timeout) await page.waitForTimeout(50) - if (requests === 0) throw new Error("History boundary did not request a page") - await waitForQuietDOM(page) - page.off("response", onResponse) - const elapsedMs = performance.now() - started - const metrics = await collectProbe(page) - return { - name: "session-max-history-boundary", - context: { - serializedBytes: target.bytes, - messages: target.messages, - parts: target.parts, - userTurns: target.userTurns, - }, - elapsedMs, - messageRequests: requests, - ...summarizeProbe(metrics), - cpu: await cpu.stop(), - trace: await stopTrace?.(), - } -} - -async function profileReview(page: Page) { - const button = page.getByRole("button", { name: "Toggle review" }) - if (!(await button.isVisible().catch(() => false))) return - const panel = page.locator("#review-panel") - if (await panel.isVisible().catch(() => false)) { - await button.click() - await panel.waitFor({ state: "hidden", timeout: 60_000 }) - await waitForQuietDOM(page) - } - await resetProbe(page) - const stopTrace = diagnostics ? await startChromeTrace(page, "review-open") : undefined - const cpu = await startCPUProfile(page) - const started = performance.now() - await button.click() - await panel.waitFor({ state: "visible", timeout: 60_000 }) - await waitForQuietDOM(page) - const elapsedMs = performance.now() - started - const metrics = await collectProbe(page) - const dom = await page.evaluate(() => ({ - elements: document.getElementsByTagName("*").length, - diffViewers: document.querySelectorAll('[data-component="file"][data-mode="diff"]').length, - diffLines: document.querySelectorAll("[data-line]").length, - })) - return { - name: "review-open", - elapsedMs, - ...summarizeProbe(metrics), - dom, - cpu: await cpu.stop(), - trace: await stopTrace?.(), - } -} - -async function installProbe(page: Page) { - await page.addInitScript((attributeResponses) => { - const state = { - longTasks: [] as number[], - animationFrames: [] as ProbeResult["animationFrames"], - frameGaps: [] as number[], - responseText: [] as ProbeResult["responseText"], - } - ;(window as Window & { __opencodeRendererProfile?: typeof state }).__opencodeRendererProfile = state - if (PerformanceObserver.supportedEntryTypes.includes("longtask")) { - new PerformanceObserver((list) => - state.longTasks.push(...list.getEntries().map((entry) => entry.duration)), - ).observe({ - type: "longtask", - }) - } - if (PerformanceObserver.supportedEntryTypes.includes("long-animation-frame")) { - new PerformanceObserver((list) => - state.animationFrames.push( - ...list.getEntries().map((entry) => { - const frame = entry as PerformanceEntry & { - blockingDuration: number - scripts?: { - duration: number - forcedStyleAndLayoutDuration?: number - sourceFunctionName?: string - sourceURL?: string - sourceCharPosition?: number - invoker?: string - invokerType?: string - }[] - } - return { - duration: frame.duration, - blockingDuration: frame.blockingDuration, - forcedStyleAndLayoutDuration: - frame.scripts?.reduce((sum, script) => sum + (script.forcedStyleAndLayoutDuration ?? 0), 0) ?? 0, - scripts: - frame.scripts?.map((script) => ({ - function: script.sourceFunctionName || "(anonymous)", - source: script.sourceURL?.split("/").at(-1) || "(document)", - position: script.sourceCharPosition ?? -1, - invoker: script.invoker ?? "(unknown)", - invokerType: script.invokerType ?? "(unknown)", - duration: script.duration, - forcedStyleAndLayoutDuration: script.forcedStyleAndLayoutDuration ?? 0, - })) ?? [], - } - }), - ), - ).observe({ type: "long-animation-frame" }) - } - let previous = performance.now() - const frame = (now: number) => { - const gap = now - previous - if (gap > 20) state.frameGaps.push(gap) - previous = now - requestAnimationFrame(frame) - } - requestAnimationFrame(frame) - if (attributeResponses) { - const responseText = Response.prototype.text - Response.prototype.text = function () { - const started = performance.now() - const url = this.url - return responseText.call(this).then((text) => { - state.responseText.push({ url, duration: performance.now() - started }) - return text - }) - } - } - }, process.env.OPENCODE_PROFILE_RESPONSE_URLS === "1") -} - -async function resetProbe(page: Page) { - await page.evaluate(() => { - const state = (window as Window & { __opencodeRendererProfile?: ProbeResult }).__opencodeRendererProfile - if (!state) return - state.longTasks.length = 0 - state.animationFrames.length = 0 - state.frameGaps.length = 0 - state.responseText.length = 0 - }) -} - -async function collectProbe(page: Page) { - return page.evaluate( - () => (window as Window & { __opencodeRendererProfile?: ProbeResult }).__opencodeRendererProfile!, - ) -} - -function summarizeProbe(probe: ProbeResult) { - const scripts = new Map< - string, - { - function: string - source: string - position: number - invoker: string - invokerType: string - durationMs: number - forcedStyleAndLayoutMs: number - } - >() - probe.animationFrames - .flatMap((frame) => frame.scripts) - .forEach((script) => { - const key = `${script.source}:${script.position}:${script.invoker}` - const current = scripts.get(key) ?? { - function: script.function, - source: script.source, - position: script.position, - invoker: script.invoker, - invokerType: script.invokerType, - durationMs: 0, - forcedStyleAndLayoutMs: 0, - } - current.durationMs += script.duration - current.forcedStyleAndLayoutMs += script.forcedStyleAndLayoutDuration - scripts.set(key, current) - }) - return { - longTasks: { - count: probe.longTasks.length, - totalMs: sum(probe.longTasks), - maxMs: Math.max(0, ...probe.longTasks), - }, - longAnimationFrames: { - count: probe.animationFrames.length, - totalBlockingMs: sum(probe.animationFrames.map((frame) => frame.blockingDuration)), - maxDurationMs: Math.max(0, ...probe.animationFrames.map((frame) => frame.duration)), - forcedStyleAndLayoutMs: sum(probe.animationFrames.map((frame) => frame.forcedStyleAndLayoutDuration)), - scripts: [...scripts.values()].sort((a, b) => b.durationMs - a.durationMs).slice(0, 15), - }, - frameGaps: { - count: probe.frameGaps.length, - maxMs: Math.max(0, ...probe.frameGaps), - }, - responseText: probe.responseText - .map((item) => ({ - path: (() => { - try { - return new URL(item.url).pathname - } catch { - return item.url - } - })(), - durationMs: item.duration, - })) - .sort((a, b) => b.durationMs - a.durationMs), - } -} - -async function startCPUProfile(page: Page) { - if (!profileCPU) return { stop: async () => [] } - const session = await page.context().newCDPSession(page) - await session.send("Profiler.enable") - await session.send("Profiler.setSamplingInterval", { interval: 1_000 }) - await session.send("Profiler.start") - return { - async stop() { - const result = await session.send("Profiler.stop") - await session.detach() - const self = new Map() - result.profile.samples?.forEach((id, index) => { - self.set(id, (self.get(id) ?? 0) + (result.profile.timeDeltas?.[index] ?? 0) / 1_000) - }) - return result.profile.nodes - .map((node) => ({ - function: node.callFrame.functionName || "(anonymous)", - source: sourceName(node.callFrame.url), - line: node.callFrame.lineNumber + 1, - selfMs: self.get(node.id) ?? 0, - })) - .filter((node) => node.selfMs >= 1) - .sort((a, b) => b.selfMs - a.selfMs) - .slice(0, 40) - }, - } -} - -async function setDesktopRoute(page: Page, route: string) { - await page.evaluate(async (value) => { - const api = window.api as typeof window.api & { getWindowID?: () => Promise } - const id = (await api.getWindowID?.()) ?? "browser" - localStorage.setItem(`opencode.desktop.window.${id}.last-active-url`, value) - }, route) -} - -async function waitForQuietDOM(page: Page) { - await page.evaluate( - () => - new Promise((resolve) => { - let timer = setTimeout(done, 750) - const observer = new MutationObserver(() => { - clearTimeout(timer) - timer = setTimeout(done, 750) - }) - observer.observe(document.body, { childList: true, subtree: true, characterData: true }) - function done() { - observer.disconnect() - requestAnimationFrame(() => requestAnimationFrame(() => resolve())) - } - }), - ) -} - -async function waitForCDP() { - const timeout = Date.now() + 5 * 60_000 - while (Date.now() < timeout) { - const ready = await fetch(`${endpoint}/json/version`) - .then((response) => response.ok) - .catch(() => false) - if (ready) return - if (child.exitCode !== null) - throw new Error( - `Desktop exited before CDP was ready (${child.exitCode})\n${await stdout}\n${await stderr}`, - ) - await Bun.sleep(250) - } - throw new Error("Timed out waiting for desktop CDP") -} - -async function waitForRenderer(browser: Awaited>) { - const timeout = Date.now() + 60_000 - while (Date.now() < timeout) { - const page = browser - .contexts() - .flatMap((context) => context.pages()) - .find((candidate) => candidate.url().startsWith("oc://renderer")) - if (page) return page - await Bun.sleep(100) - } - throw new Error("Desktop renderer target was not found") -} - -async function run(command: string[], cwd: string) { - const child = Bun.spawn(command, { cwd, env: processEnv(), stdout: "inherit", stderr: "inherit" }) - const code = await child.exited - if (code !== 0) throw new Error(`${command.join(" ")} exited with ${code}`) -} - -function processEnv() { - return { ...process.env, OPENCODE_DB: databasePath, OPENCODE_CHANNEL: "dev" } -} - -async function drain(stream: ReadableStream) { - const decoder = new TextDecoder() - let output = "" - for await (const chunk of stream) output = (output + decoder.decode(chunk, { stream: true })).slice(-50_000) - return output + decoder.decode() -} - -async function killTree(pid: number) { - if (process.platform !== "win32") { - process.kill(pid, "SIGTERM") - return - } - const child = Bun.spawn(["taskkill", "/pid", String(pid), "/T", "/F"], { stdout: "ignore", stderr: "ignore" }) - await child.exited -} - -function sourceName(value: string) { - if (!value) return "(native)" - try { - return new URL(value).pathname.split("/").at(-1) || "(document)" - } catch { - return path.basename(value) - } -} - -function sum(values: number[]) { - return values.reduce((total, value) => total + value, 0) -} - -function percentile(values: number[], quantile: number) { - return values.toSorted((a, b) => a - b)[Math.max(0, Math.ceil(values.length * quantile) - 1)] ?? 0 -} +import "./profile-desktop" diff --git a/packages/app/package.json b/packages/app/package.json index 7eb870b5854..a51b178c4b0 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -28,7 +28,8 @@ "test:e2e:ui": "playwright test --ui", "test:e2e:report": "playwright show-report e2e/playwright-report", "test:stability": "bun test ./e2e/performance/unit/visual-stability.test.ts && playwright test --config e2e/performance/timeline-stability/playwright.config.ts", - "test:bench": "bun test ./e2e/performance/unit && playwright test --config e2e/performance/playwright.config.ts" + "test:bench": "bun test ./e2e/performance/unit && playwright test --config e2e/performance/playwright.config.ts", + "profile:desktop": "bun run e2e/performance/profile-desktop.ts" }, "license": "MIT", "devDependencies": {