From ff2916471a5d151b74773358ab9220a3784c24da Mon Sep 17 00:00:00 2001 From: Dax Raad Date: Thu, 21 May 2026 01:26:31 -0400 Subject: [PATCH] feat(core): add sqlite schema sync --- packages/core/package.json | 3 + packages/core/src/database/database.ts | 45 +++ packages/core/src/database/migration.ts | 248 +++++++++++++ .../{session-event.ts => session/event.ts} | 16 +- .../core/src/{session.ts => session/index.ts} | 6 +- .../message-updater.ts} | 6 +- .../message.ts} | 14 +- .../{session-prompt.ts => session/prompt.ts} | 0 packages/core/src/session/sql.ts | 49 +++ packages/core/test/database-migration.test.ts | 328 ++++++++++++++++++ packages/opencode/src/event-v2-bridge.ts | 2 +- .../instance/httpapi/groups/v2/message.ts | 2 +- .../instance/httpapi/groups/v2/session.ts | 4 +- .../instance/httpapi/handlers/v2/message.ts | 2 +- packages/opencode/src/session/compaction.ts | 2 +- packages/opencode/src/session/processor.ts | 2 +- .../opencode/src/session/projectors-next.ts | 6 +- packages/opencode/src/session/prompt.ts | 4 +- packages/opencode/src/session/schema.ts | 2 +- packages/opencode/src/session/session.sql.ts | 2 +- packages/opencode/src/v2/session.ts | 6 +- .../test/server/httpapi-session.test.ts | 2 +- .../test/v2/session-message-updater.test.ts | 4 +- 23 files changed, 714 insertions(+), 41 deletions(-) create mode 100644 packages/core/src/database/database.ts create mode 100644 packages/core/src/database/migration.ts rename packages/core/src/{session-event.ts => session/event.ts} (96%) rename packages/core/src/{session.ts => session/index.ts} (70%) rename packages/core/src/{session-message-updater.ts => session/message-updater.ts} (98%) rename packages/core/src/{session-message.ts => session/message.ts} (95%) rename packages/core/src/{session-prompt.ts => session/prompt.ts} (100%) create mode 100644 packages/core/src/session/sql.ts create mode 100644 packages/core/test/database-migration.test.ts diff --git a/packages/core/package.json b/packages/core/package.json index e090536419..bfc0e7bc1b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -49,8 +49,10 @@ "@aws-sdk/credential-providers": "3.993.0", "@effect/opentelemetry": "catalog:", "@effect/platform-node": "catalog:", + "@effect/sql-sqlite-bun": "catalog:", "@npmcli/arborist": "9.4.0", "@npmcli/config": "10.8.1", + "@opencode-ai/effect-drizzle-sqlite": "workspace:*", "@opentelemetry/api": "1.9.0", "@opentelemetry/context-async-hooks": "2.6.1", "@opentelemetry/exporter-trace-otlp-http": "0.214.0", @@ -58,6 +60,7 @@ "@openrouter/ai-sdk-provider": "2.8.1", "ai-gateway-provider": "3.1.2", "cross-spawn": "catalog:", + "drizzle-orm": "catalog:", "effect": "catalog:", "gitlab-ai-provider": "6.6.0", "glob": "13.0.5", diff --git a/packages/core/src/database/database.ts b/packages/core/src/database/database.ts new file mode 100644 index 0000000000..a7c56eb220 --- /dev/null +++ b/packages/core/src/database/database.ts @@ -0,0 +1,45 @@ +export * as Database from "./database" + +import { SqliteClient } from "@effect/sql-sqlite-bun" +import { EffectDrizzleSqlite } from "@opencode-ai/effect-drizzle-sqlite" +import { Context, Effect, Layer } from "effect" +import { Global } from "../global" +import { Flag } from "../flag/flag" +import path from "path" + +const makeDatabase = EffectDrizzleSqlite.makeWithDefaults() +type DatabaseShape = Effect.Success + +export class Service extends Context.Service()("@opencode/v2/storage/Database") {} + +const layer = Layer.effect( + Service, + Effect.gen(function* () { + const db = yield* makeDatabase + + yield* db.run("PRAGMA journal_mode = WAL") + yield* db.run("PRAGMA synchronous = NORMAL") + yield* db.run("PRAGMA busy_timeout = 5000") + yield* db.run("PRAGMA cache_size = -64000") + yield* db.run("PRAGMA foreign_keys = ON") + yield* db.run("PRAGMA wal_checkpoint(PASSIVE)") + + return db + }), +) + +export function layerFromPath(filename: string) { + return layer.pipe(Layer.provide(SqliteClient.layer({ filename }))) +} + +export const defaultLayer = Layer.unwrap( + Effect.gen(function* () { + return layerFromPath( + !Flag.OPENCODE_DB + ? path.join(Global.Path.data, "opencode.db") + : Flag.OPENCODE_DB === ":memory:" || path.isAbsolute(Flag.OPENCODE_DB) + ? Flag.OPENCODE_DB + : path.join(Global.Path.data, Flag.OPENCODE_DB), + ) + }), +).pipe(Layer.provide(Global.defaultLayer)) diff --git a/packages/core/src/database/migration.ts b/packages/core/src/database/migration.ts new file mode 100644 index 0000000000..e9575c865a --- /dev/null +++ b/packages/core/src/database/migration.ts @@ -0,0 +1,248 @@ +export * as DatabaseMigration from "./migration" + +import type { EffectDrizzleSqlite } from "@opencode-ai/effect-drizzle-sqlite" +import { Effect } from "effect" +import { getTableName, sql, type SQL, type Table } from "drizzle-orm" +import { getTableConfig, type AnySQLiteTable, type Index, type SQLiteColumn } from "drizzle-orm/sqlite-core" + +export type SchemaAst = { + tables: Record +} + +export type TableAst = { + name: string + columns: Record + indexes: Record +} + +export type ColumnAst = { + name: string + type: string + notNull: boolean + primaryKey: boolean + default?: string +} + +export type IndexAst = { + name: string + table: string + columns: IndexColumnAst[] + unique: boolean + where?: string +} + +export type IndexColumnAst = { type: "column"; name: string } | { type: "expression"; sql: string } + +export type Operation = + | { type: "create_table"; table: TableAst } + | { type: "add_column"; table: string; column: ColumnAst } + | { type: "create_index"; index: IndexAst } + +export function diff(db: EffectDrizzleSqlite.EffectSQLiteDatabase, tables: Table[]) { + return read(db).pipe(Effect.map((actual) => diffSchema(actual, fromTables(tables)))) +} + +export function apply(db: EffectDrizzleSqlite.EffectSQLiteDatabase, operations: Operation[]) { + return Effect.forEach(operations, (operation) => db.run(toSql(operation))).pipe(Effect.asVoid) +} + +function fromTables(tables: Table[]): SchemaAst { + return { + tables: Object.fromEntries(tables.map((table) => { + const config = getTableConfig(table as AnySQLiteTable) + const name = getTableName(table) + return [name, tableFromConfig(name, config.columns, config.indexes)] + })), + } +} + +function diffSchema(actual: SchemaAst, desired: SchemaAst): Operation[] { + return Object.values(desired.tables).flatMap((table) => { + const current = actual.tables[table.name] + if (!current) { + return [createTableOperation(table), ...Object.values(table.indexes).map(createIndexOperation)] + } + return [ + ...Object.values(table.columns) + .filter((column) => current.columns[column.name] === undefined) + .map((column) => addColumnOperation(table.name, column)), + ...Object.values(table.indexes) + .filter((index) => current.indexes[index.name] === undefined) + .map(createIndexOperation), + ] + }) +} + +function createTableOperation(table: TableAst): Operation { + return { type: "create_table", table } +} + +function addColumnOperation(table: string, column: ColumnAst): Operation { + return { type: "add_column", table, column } +} + +function createIndexOperation(index: IndexAst): Operation { + return { type: "create_index", index } +} + +function toSql(operation: Operation) { + if (operation.type === "create_table") { + return `CREATE TABLE ${quoteIdentifier(operation.table.name)} (${Object.values(operation.table.columns) + .map((column) => columnSql(column, true)) + .join(", ")})` + } + if (operation.type === "add_column") { + return `ALTER TABLE ${quoteIdentifier(operation.table)} ADD COLUMN ${columnSql(operation.column, false)}` + } + return [ + "CREATE", + operation.index.unique ? "UNIQUE" : undefined, + "INDEX", + quoteIdentifier(operation.index.name), + "ON", + quoteIdentifier(operation.index.table), + `(${operation.index.columns.map(indexColumnSql).join(", ")})`, + operation.index.where === undefined ? undefined : `WHERE ${operation.index.where}`, + ] + .filter((part) => part !== undefined) + .join(" ") +} + +function read(db: EffectDrizzleSqlite.EffectSQLiteDatabase) { + return Effect.gen(function* () { + const rows = yield* db.all<{ name: string }>(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`) + const tables = yield* Effect.forEach(rows, (row) => readTable(db, row.name)) + return { tables: Object.fromEntries(tables.map((table) => [table.name, table])) } + }) +} + +function readTable(db: EffectDrizzleSqlite.EffectSQLiteDatabase, name: string) { + return Effect.gen(function* () { + const columns = yield* db.all<{ + name: string + type: string + notnull: number + pk: number + dflt_value: string | null + }>(`PRAGMA table_info(${quoteIdentifier(name)})`) + const indexes = yield* db.all<{ name: string; unique: number }>(`PRAGMA index_list(${quoteIdentifier(name)})`) + const indexEntries = yield* Effect.forEach(indexes, (index) => + Effect.gen(function* () { + const statement = yield* db.get<{ sql: string | null }>(sql`SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ${index.name}`) + if (statement?.sql === null || statement?.sql === undefined) return undefined + const columns = yield* db.all<{ seqno: number; name: string | null }>(`PRAGMA index_info(${quoteIdentifier(index.name)})`) + return [ + index.name, + { + name: index.name, + table: name, + columns: columns.map((column) => + column.name === null + ? ({ type: "expression", sql: "" } as const) + : ({ type: "column", name: column.name } as const), + ), + unique: index.unique === 1, + }, + ] as const + }), + ) + return { + name, + columns: Object.fromEntries(columns.map((column) => [ + column.name, + { + name: column.name, + type: column.type, + notNull: column.notnull === 1, + primaryKey: column.pk > 0, + ...(column.dflt_value === null ? {} : { default: column.dflt_value }), + }, + ])), + indexes: Object.fromEntries(indexEntries.filter((entry) => entry !== undefined)), + } + }) +} + +function tableFromConfig(name: string, columns: SQLiteColumn[], indexes: Index[]): TableAst { + return { + name, + columns: Object.fromEntries(columns.map((column) => [column.name, columnFromConfig(column)])), + indexes: Object.fromEntries(indexes.map((index) => [index.config.name, indexFromConfig(index)])), + } +} + +function columnFromConfig(column: SQLiteColumn): ColumnAst { + return { + name: column.name, + type: column.getSQLType(), + notNull: column.notNull, + primaryKey: column.primary, + ...defaultFromColumn(column), + } +} + +function defaultFromColumn(column: SQLiteColumn) { + if (column.default !== undefined) return { default: literal(column.default) } + if (column.defaultFn !== undefined) return { default: literal(column.defaultFn()) } + return {} +} + +function indexFromConfig(index: Index): IndexAst { + return { + name: index.config.name, + table: getTableName(index.config.table), + columns: index.config.columns.map(indexColumnName), + unique: index.config.unique, + ...(index.config.where === undefined ? {} : { where: compileSql(index.config.where) }), + } +} + +function indexColumnName(column: SQLiteColumn | SQL) { + if ("name" in column) return { type: "column", name: column.name } as const + return { type: "expression", sql: compileSql(column) } as const +} + +function compileSql(value: SQL) { + return value.getSQL().toQuery(new SQLiteCompiler()).sql.replace(/"(?:""|[^"])*"\./g, "") +} + +function indexColumnSql(column: IndexColumnAst) { + if (column.type === "column") return quoteIdentifier(column.name) + return column.sql +} + +function columnSql(column: ColumnAst, includePrimaryKey: boolean) { + return [ + quoteIdentifier(column.name), + column.type, + includePrimaryKey && column.primaryKey ? "PRIMARY KEY" : undefined, + column.notNull ? "NOT NULL" : undefined, + column.default === undefined ? undefined : `DEFAULT ${column.default}`, + ] + .filter((part) => part !== undefined) + .join(" ") +} + +class SQLiteCompiler { + inlineParams = true + escapeName = (name: string) => { + return quoteIdentifier(name) + } + escapeParam = () => { + return "?" + } + escapeString = (value: string) => { + return `'${value.replaceAll("'", "''")}'` + } +} + +function literal(value: unknown) { + if (typeof value === "number") return String(value) + if (typeof value === "boolean") return value ? "1" : "0" + if (value === null) return "NULL" + return `'${String(value).replaceAll("'", "''")}'` +} + +function quoteIdentifier(value: string) { + return `"${value.replaceAll('"', '""')}"` +} diff --git a/packages/core/src/session-event.ts b/packages/core/src/session/event.ts similarity index 96% rename from packages/core/src/session-event.ts rename to packages/core/src/session/event.ts index a98d9cc051..2d5f431053 100644 --- a/packages/core/src/session-event.ts +++ b/packages/core/src/session/event.ts @@ -1,11 +1,11 @@ import { Schema } from "effect" -import { EventV2 } from "./event" -import { ModelV2 } from "./model" -import { NonNegativeInt } from "./schema" -import { Session } from "./session" -import { FileAttachment, Prompt } from "./session-prompt" -import { ToolOutput } from "./tool-output" -import { V2Schema } from "./v2-schema" +import { EventV2 } from "../event" +import { ModelV2 } from "../model" +import { NonNegativeInt } from "../schema" +import { ToolOutput } from "../tool-output" +import { V2Schema } from "../v2-schema" +import { Session } from "./index" +import { FileAttachment, Prompt } from "./prompt" export { FileAttachment } @@ -399,4 +399,4 @@ export const All = Schema.Union( export type Event = typeof All.Type export type Type = Event["type"] -export * as SessionEvent from "./session-event" +export * as SessionEvent from "./event" diff --git a/packages/core/src/session.ts b/packages/core/src/session/index.ts similarity index 70% rename from packages/core/src/session.ts rename to packages/core/src/session/index.ts index 756531e328..fa6cf07995 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session/index.ts @@ -1,8 +1,8 @@ -export * as Session from "./session" +export * as Session from "." import { Schema } from "effect" -import { withStatics } from "./schema" -import { Identifier } from "./util/identifier" +import { withStatics } from "../schema" +import { Identifier } from "../util/identifier" export const ID = Schema.String.check(Schema.isStartsWith("ses")).pipe( Schema.brand("SessionID"), diff --git a/packages/core/src/session-message-updater.ts b/packages/core/src/session/message-updater.ts similarity index 98% rename from packages/core/src/session-message-updater.ts rename to packages/core/src/session/message-updater.ts index bbdf59c555..fa5fcc3a4a 100644 --- a/packages/core/src/session-message-updater.ts +++ b/packages/core/src/session/message-updater.ts @@ -1,6 +1,6 @@ import { produce, type WritableDraft } from "immer" -import { SessionEvent } from "./session-event" -import { SessionMessage } from "./session-message" +import { SessionEvent } from "./event" +import { SessionMessage } from "./message" export type MemoryState = { messages: SessionMessage.Message[] @@ -414,4 +414,4 @@ export function update(adapter: Adapter, event: SessionEvent.Eve return adapter.finish() } -export * as SessionMessageUpdater from "./session-message-updater" +export * as SessionMessageUpdater from "./message-updater" diff --git a/packages/core/src/session-message.ts b/packages/core/src/session/message.ts similarity index 95% rename from packages/core/src/session-message.ts rename to packages/core/src/session/message.ts index 73b6dd7da2..305202b0a4 100644 --- a/packages/core/src/session-message.ts +++ b/packages/core/src/session/message.ts @@ -1,10 +1,10 @@ import { Schema } from "effect" -import { Prompt } from "./session-prompt" -import { SessionEvent } from "./session-event" -import { EventV2 } from "./event" -import { ToolOutput } from "./tool-output" -import { V2Schema } from "./v2-schema" -import { ModelV2 } from "./model" +import { EventV2 } from "../event" +import { ModelV2 } from "../model" +import { ToolOutput } from "../tool-output" +import { V2Schema } from "../v2-schema" +import { SessionEvent } from "./event" +import { Prompt } from "./prompt" export const ID = EventV2.ID export type ID = Schema.Schema.Type @@ -170,4 +170,4 @@ export type Message = Schema.Schema.Type export type Type = Message["type"] -export * as SessionMessage from "./session-message" +export * as SessionMessage from "./message" diff --git a/packages/core/src/session-prompt.ts b/packages/core/src/session/prompt.ts similarity index 100% rename from packages/core/src/session-prompt.ts rename to packages/core/src/session/prompt.ts diff --git a/packages/core/src/session/sql.ts b/packages/core/src/session/sql.ts new file mode 100644 index 0000000000..310ffde0f5 --- /dev/null +++ b/packages/core/src/session/sql.ts @@ -0,0 +1,49 @@ +import { index, integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core" +import { Session } from "." + +export const SessionTable = sqliteTable( + "session", + { + id: text().$type().primaryKey(), + project_id: text().notNull(), + workspace_id: text(), + parent_id: text().$type(), + slug: text().notNull(), + directory: text().notNull(), + path: text(), + title: text().notNull(), + version: text().notNull(), + share_url: text(), + summary_additions: integer(), + summary_deletions: integer(), + summary_files: integer(), + summary_diffs: text({ mode: "json" }), + cost: real().notNull().default(0), + tokens_input: integer().notNull().default(0), + tokens_output: integer().notNull().default(0), + tokens_reasoning: integer().notNull().default(0), + tokens_cache_read: integer().notNull().default(0), + tokens_cache_write: integer().notNull().default(0), + revert: text({ mode: "json" }), + permission: text({ mode: "json" }), + agent: text(), + model: text({ mode: "json" }).$type<{ + id: string + providerID: string + variant?: string + }>(), + time_created: integer() + .notNull() + .$default(() => Date.now()), + time_updated: integer() + .notNull() + .$onUpdate(() => Date.now()), + time_compacting: integer(), + time_archived: integer(), + }, + (table) => [ + index("session_project_idx").on(table.project_id), + index("session_workspace_idx").on(table.workspace_id), + index("session_parent_idx").on(table.parent_id), + ], +) diff --git a/packages/core/test/database-migration.test.ts b/packages/core/test/database-migration.test.ts new file mode 100644 index 0000000000..7128c1b95c --- /dev/null +++ b/packages/core/test/database-migration.test.ts @@ -0,0 +1,328 @@ +import { describe, expect, test } from "bun:test" +import { SqliteClient } from "@effect/sql-sqlite-bun" +import { EffectDrizzleSqlite } from "@opencode-ai/effect-drizzle-sqlite" +import { DatabaseMigration } from "@opencode-ai/core/database/migration" +import { Effect } from "effect" +import { index, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { sql, type ColumnBuilderBase } from "drizzle-orm" +import path from "path" +import { tmpdir } from "./fixture/tmpdir" + +const rand = (seed: number) => () => { + seed = (seed * 1664525 + 1013904223) >>> 0 + return seed / 0x100000000 +} + +describe("DatabaseMigration", () => { + test("diff creates missing tables before indexes and apply is idempotent", () => + withDb((db) => + Effect.gen(function* () { + const table = sqliteTable( + "session", + { + id: text().primaryKey(), + slug: text().notNull(), + title: text().notNull().default("untitled"), + }, + (table) => [index("session_slug_idx").on(table.slug), uniqueIndex("session_title_uidx").on(table.title)], + ) + + const operations = yield* DatabaseMigration.diff(db, [table]) + expect(operations.map((operation) => operation.type)).toEqual(["create_table", "create_index", "create_index"]) + + yield* DatabaseMigration.apply(db, operations) + expect(yield* DatabaseMigration.diff(db, [table])).toEqual([]) + expect((yield* columnNames(db, "session")).sort()).toEqual(["id", "slug", "title"]) + expect((yield* indexNames(db, "session")).sort()).toEqual(["session_slug_idx", "session_title_uidx"]) + }), + ), + ) + + test("diff adds missing columns and indexes without recreating existing tables", () => + withDb((db) => + Effect.gen(function* () { + yield* db.run(`CREATE TABLE "session" ("id" text PRIMARY KEY NOT NULL)`) + const table = sqliteTable( + "session", + { + id: text().primaryKey(), + title: text().notNull().default("untitled"), + path: text(), + }, + (table) => [index("session_title_idx").on(table.title)], + ) + + const operations = yield* DatabaseMigration.diff(db, [table]) + expect(operations.map((operation) => operation.type)).toEqual(["add_column", "add_column", "create_index"]) + + yield* DatabaseMigration.apply(db, operations) + expect(yield* DatabaseMigration.diff(db, [table])).toEqual([]) + expect((yield* columnNames(db, "session")).sort()).toEqual(["id", "path", "title"]) + expect(yield* indexNames(db, "session")).toEqual(["session_title_idx"]) + }), + ), + ) + + test("diff is additive only and ignores extra actual schema and definition drift", () => + withDb((db) => + Effect.gen(function* () { + yield* db.run(`CREATE TABLE "session" ("id" text PRIMARY KEY NOT NULL, "title" integer NOT NULL, "extra" text)`) + yield* db.run(`CREATE INDEX "session_extra_idx" ON "session" ("extra")`) + yield* db.run(`CREATE TABLE "extra_table" ("id" text PRIMARY KEY)`) + const table = sqliteTable("session", { + id: text().primaryKey(), + title: text().notNull().default("untitled"), + }) + + expect(yield* DatabaseMigration.diff(db, [table])).toEqual([]) + }), + ), + ) + + test("diff ignores changed indexes to stay downgrade safe", () => + withDb((db) => + Effect.gen(function* () { + yield* db.run(`CREATE TABLE "session" ("id" text PRIMARY KEY NOT NULL, "title" text, "slug" text)`) + yield* db.run(`CREATE INDEX "session_lookup_idx" ON "session" ("slug")`) + yield* db.run(`CREATE INDEX "session_expression_idx" ON "session" (lower("slug"))`) + const table = sqliteTable( + "session", + { + id: text().primaryKey(), + title: text(), + slug: text(), + }, + (table) => [ + uniqueIndex("session_lookup_idx").on(table.title, table.slug), + index("session_expression_idx").on(sql`lower(${table.title})`), + ], + ) + + expect(yield* DatabaseMigration.diff(db, [table])).toEqual([]) + expect((yield* indexColumns(db, "session_lookup_idx")).map((column) => column.name)).toEqual(["slug"]) + }), + ), + ) + + test("diff ignores changed not-null constraints to stay downgrade safe", () => + withDb((db) => + Effect.gen(function* () { + yield* db.run(`CREATE TABLE "session" ("id" text PRIMARY KEY NOT NULL, "required" text, "optional" text NOT NULL)`) + const table = sqliteTable("session", { + id: text().primaryKey(), + required: text().notNull(), + optional: text(), + }) + + expect(yield* DatabaseMigration.diff(db, [table])).toEqual([]) + expect(yield* columnFlags(db, "session")).toMatchObject({ required: { notnull: 0 }, optional: { notnull: 1 } }) + }), + ), + ) + + test("apply handles quoted identifiers, composite indexes, unique indexes, and expression indexes", () => + withDb((db) => + Effect.gen(function* () { + const table = sqliteTable( + "table \" with spaces", + { + id: text('id " col').primaryKey(), + value: text('value " one').notNull().default("a'b"), + other: text("other space"), + }, + (table) => [ + uniqueIndex('idx " composite').on(table.value, table.other), + index('idx " expression').on(sql`lower(${table.value})`.inlineParams()), + ], + ) + + yield* DatabaseMigration.apply(db, yield* DatabaseMigration.diff(db, [table])) + expect(yield* DatabaseMigration.diff(db, [table])).toEqual([]) + expect((yield* columnNames(db, 'table " with spaces')).sort()).toEqual(["id \" col", "other space", "value \" one"]) + expect((yield* indexNames(db, 'table " with spaces')).sort()).toEqual(['idx " composite', 'idx " expression']) + }), + ), + ) + + test("random schema reconciliation reaches a fixed point with additive operations", () => + withDb((db) => + Effect.gen(function* () { + for (let seed = 1; seed <= 75; seed++) { + const random = rand(seed) + const specs = Array.from({ length: 1 + Math.floor(random() * 4) }, (_, index) => randomSpec(random, seed, index)) + for (const spec of specs) yield* seedActualSchema(db, spec, random) + + const tables = specs.map(tableFromSpec) + const operations = yield* DatabaseMigration.diff(db, tables) + expect( + operations.every((operation) => ["create_table", "add_column", "create_index"].includes(operation.type)), + ).toBe(true) + + yield* DatabaseMigration.apply(db, operations) + expect(yield* DatabaseMigration.diff(db, tables)).toEqual([]) + } + }), + ), + 20000, + ) +}) + +type TableSpec = { + name: string + columns: ColumnSpec[] + indexes: IndexSpec[] +} + +type ColumnSpec = { + key: string + name: string + primaryKey: boolean + notNull: boolean + default?: string +} + +type IndexSpec = { + name: string + columns: string[] + unique: boolean +} + +function tableFromSpec(spec: TableSpec) { + const columns = Object.fromEntries(spec.columns.map((column) => [column.key, columnBuilder(column)])) as Record + return sqliteTable(spec.name, columns, (table) => + spec.indexes.map((item) => { + const columns = item.columns.map((key) => table[key]).filter((column) => column !== undefined) + const first = columns[0] + if (!first) throw new Error(`index ${item.name} has no columns`) + return (item.unique ? uniqueIndex(item.name) : index(item.name)).on(first, ...columns.slice(1)) + }), + ) +} + +function columnBuilder(spec: ColumnSpec): ColumnBuilderBase { + if (spec.primaryKey) return text(spec.name).primaryKey() + if (spec.default !== undefined) return text(spec.name).notNull().default(spec.default) + if (spec.notNull) return text(spec.name).notNull() + return text(spec.name) +} + +function randomSpec(random: () => number, seed: number, index: number): TableSpec { + const name = identifier(random, `table_${seed}_${index}`) + const columns = Array.from({ length: 1 + Math.floor(random() * 8) }, (_, i): ColumnSpec => { + const primaryKey = i === 0 + const notNull = primaryKey || random() > 0.5 + return { + key: `column_${i}`, + name: identifier(random, `column_${i}`), + primaryKey, + notNull, + ...(notNull && !primaryKey ? { default: `default_${Math.floor(random() * 1000)}` } : {}), + } + }) + const indexes = columns + .filter((column) => !column.primaryKey && random() > 0.45) + .map((column, i): IndexSpec => ({ + name: identifier(random, `${name}_${column.name}_${i}_idx`), + columns: random() > 0.65 ? columns.filter((item) => !item.primaryKey).slice(0, 2).map((item) => item.key) : [column.key], + unique: random() > 0.8, + })) + .filter((item) => item.columns.length > 0) + return { name, columns, indexes } +} + +function seedActualSchema(db: EffectDrizzleSqlite.EffectSQLiteDatabase, spec: TableSpec, random: () => number) { + return Effect.gen(function* () { + if (random() < 0.25) return + const columns = spec.columns.filter((column) => column.primaryKey || random() > 0.35) + yield* db.run(`CREATE TABLE ${quoteIdentifier(spec.name)} (${columns.map(columnSql).join(", ")})`) + for (const column of columns.filter((column) => !column.primaryKey && !column.notNull && random() > 0.6)) { + yield* db.run(`ALTER TABLE ${quoteIdentifier(spec.name)} ALTER COLUMN ${quoteIdentifier(column.name)} SET NOT NULL`) + } + for (const item of spec.indexes.filter(() => random() > 0.5)) { + if (!item.columns.every((key) => columns.some((column) => column.key === key))) continue + const changed = random() > 0.5 + yield* db.run( + indexSql(spec.name, { + ...item, + unique: changed ? !item.unique : item.unique, + columns: changed ? [...item.columns].reverse() : item.columns, + }), + ) + } + }) +} + +function columnSql(spec: ColumnSpec) { + return [ + quoteIdentifier(spec.name), + "text", + spec.primaryKey ? "PRIMARY KEY" : undefined, + spec.notNull ? "NOT NULL" : undefined, + spec.default === undefined ? undefined : `DEFAULT ${literal(spec.default)}`, + ] + .filter((item) => item !== undefined) + .join(" ") +} + +function indexSql(table: string, spec: IndexSpec) { + return [ + "CREATE", + spec.unique ? "UNIQUE" : undefined, + "INDEX", + quoteIdentifier(spec.name), + "ON", + quoteIdentifier(table), + `(${spec.columns.map((column) => quoteIdentifier(columnName(column))).join(", ")})`, + ] + .filter((item) => item !== undefined) + .join(" ") +} + +function indexColumns(db: EffectDrizzleSqlite.EffectSQLiteDatabase, index: string) { + return db.all<{ name: string | null }>(`PRAGMA index_info(${quoteIdentifier(index)})`) +} + +function columnName(key: string) { + return key.replace(/^column_/, "column_") +} + +function identifier(random: () => number, fallback: string) { + const suffixes = ["", " space", ' " quote', " select", "-dash", "_underscore"] + return `${fallback}${suffixes[Math.floor(random() * suffixes.length)]}` +} + +function columnNames(db: EffectDrizzleSqlite.EffectSQLiteDatabase, table: string) { + return db.all<{ name: string }>(`PRAGMA table_info(${quoteIdentifier(table)})`).pipe(Effect.map((rows) => rows.map((row) => row.name))) +} + +function indexNames(db: EffectDrizzleSqlite.EffectSQLiteDatabase, table: string) { + return db + .all<{ name: string }>(`PRAGMA index_list(${quoteIdentifier(table)})`) + .pipe(Effect.map((rows) => rows.map((row) => row.name).filter((name) => !name.startsWith("sqlite_autoindex_")))) +} + +function columnFlags(db: EffectDrizzleSqlite.EffectSQLiteDatabase, table: string) { + return db + .all<{ name: string; notnull: number }>(`PRAGMA table_info(${quoteIdentifier(table)})`) + .pipe(Effect.map((rows) => Object.fromEntries(rows.map((row) => [row.name, { notnull: row.notnull }])))) +} + +async function withDb(fn: (db: EffectDrizzleSqlite.EffectSQLiteDatabase) => Effect.Effect) { + const dir = await tmpdir() + try { + return await Effect.gen(function* () { + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + return yield* fn(db) + }).pipe(Effect.provide(SqliteClient.layer({ filename: path.join(dir.path, "test.db") })), Effect.runPromise) + } finally { + await dir[Symbol.asyncDispose]() + } +} + +function quoteIdentifier(value: string) { + return `"${value.replaceAll('"', '""')}"` +} + +function literal(value: string) { + return `'${value.replaceAll("'", "''")}'` +} diff --git a/packages/opencode/src/event-v2-bridge.ts b/packages/opencode/src/event-v2-bridge.ts index 4c6c79a707..ff3ede4750 100644 --- a/packages/opencode/src/event-v2-bridge.ts +++ b/packages/opencode/src/event-v2-bridge.ts @@ -9,7 +9,7 @@ import { SyncEvent } from "@/sync" import { EventV2 } from "@opencode-ai/core/event" import "@opencode-ai/core/account" import "@opencode-ai/core/catalog" -import "@opencode-ai/core/session-event" +import "@opencode-ai/core/session/event" import { Context, Effect, Layer, Option } from "effect" export function toSyncDefinition(definition: D) { diff --git a/packages/opencode/src/server/routes/instance/httpapi/groups/v2/message.ts b/packages/opencode/src/server/routes/instance/httpapi/groups/v2/message.ts index 47bb01cd8a..7a8ff9c249 100644 --- a/packages/opencode/src/server/routes/instance/httpapi/groups/v2/message.ts +++ b/packages/opencode/src/server/routes/instance/httpapi/groups/v2/message.ts @@ -1,5 +1,5 @@ import { SessionID } from "@/session/schema" -import { SessionMessage } from "@opencode-ai/core/session-message" +import { SessionMessage } from "@opencode-ai/core/session/message" import { Schema } from "effect" import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" import { InvalidCursorError } from "../../errors" diff --git a/packages/opencode/src/server/routes/instance/httpapi/groups/v2/session.ts b/packages/opencode/src/server/routes/instance/httpapi/groups/v2/session.ts index f74a9f50bf..8b9b5e8f20 100644 --- a/packages/opencode/src/server/routes/instance/httpapi/groups/v2/session.ts +++ b/packages/opencode/src/server/routes/instance/httpapi/groups/v2/session.ts @@ -1,6 +1,6 @@ import { SessionID } from "@/session/schema" -import { SessionMessage } from "@opencode-ai/core/session-message" -import { Prompt } from "@opencode-ai/core/session-prompt" +import { SessionMessage } from "@opencode-ai/core/session/message" +import { Prompt } from "@opencode-ai/core/session/prompt" import { SessionV2 } from "@/v2/session" import { Schema } from "effect" import { HttpApiEndpoint, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi" diff --git a/packages/opencode/src/server/routes/instance/httpapi/handlers/v2/message.ts b/packages/opencode/src/server/routes/instance/httpapi/handlers/v2/message.ts index c809f6485d..c00f90dac2 100644 --- a/packages/opencode/src/server/routes/instance/httpapi/handlers/v2/message.ts +++ b/packages/opencode/src/server/routes/instance/httpapi/handlers/v2/message.ts @@ -1,4 +1,4 @@ -import { SessionMessage } from "@opencode-ai/core/session-message" +import { SessionMessage } from "@opencode-ai/core/session/message" import { SessionV2 } from "@/v2/session" import { Effect, Schema } from "effect" import * as DateTime from "effect/DateTime" diff --git a/packages/opencode/src/session/compaction.ts b/packages/opencode/src/session/compaction.ts index ef007fe74d..b0a762a5a8 100644 --- a/packages/opencode/src/session/compaction.ts +++ b/packages/opencode/src/session/compaction.ts @@ -19,7 +19,7 @@ import { isOverflow as overflow, usable } from "./overflow" import { serviceUse } from "@/effect/service-use" import { RuntimeFlags } from "@/effect/runtime-flags" import { EventV2Bridge } from "@/event-v2-bridge" -import { SessionEvent } from "@opencode-ai/core/session-event" +import { SessionEvent } from "@opencode-ai/core/session/event" const log = Log.create({ service: "session.compaction" }) diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index 3b6fbcc7bf..9ba8c52cd8 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -23,7 +23,7 @@ import * as Log from "@opencode-ai/core/util/log" import { isRecord } from "@/util/record" import { EventV2 } from "@opencode-ai/core/event" import { EventV2Bridge } from "@/event-v2-bridge" -import { SessionEvent } from "@opencode-ai/core/session-event" +import { SessionEvent } from "@opencode-ai/core/session/event" import { ModelV2 } from "@opencode-ai/core/model" import { ProviderV2 } from "@opencode-ai/core/provider" import * as DateTime from "effect/DateTime" diff --git a/packages/opencode/src/session/projectors-next.ts b/packages/opencode/src/session/projectors-next.ts index ae5b9c5d2f..d7b2d14824 100644 --- a/packages/opencode/src/session/projectors-next.ts +++ b/packages/opencode/src/session/projectors-next.ts @@ -1,8 +1,8 @@ import { and, desc, eq } from "@/storage/db" import type { Database } from "@/storage/db" -import { SessionMessage } from "@opencode-ai/core/session-message" -import { SessionMessageUpdater } from "@opencode-ai/core/session-message-updater" -import { SessionEvent } from "@opencode-ai/core/session-event" +import { SessionMessage } from "@opencode-ai/core/session/message" +import { SessionMessageUpdater } from "@opencode-ai/core/session/message-updater" +import { SessionEvent } from "@opencode-ai/core/session/event" import * as DateTime from "effect/DateTime" import { SyncEvent } from "@/sync" import { EventV2Bridge } from "@/event-v2-bridge" diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index e39d0016ab..a78ef34bda 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -49,10 +49,10 @@ import { SessionRunState } from "./run-state" import { RuntimeFlags } from "@/effect/runtime-flags" import { EventV2 } from "@opencode-ai/core/event" import { EventV2Bridge } from "@/event-v2-bridge" -import { SessionEvent } from "@opencode-ai/core/session-event" +import { SessionEvent } from "@opencode-ai/core/session/event" import { ModelV2 } from "@opencode-ai/core/model" import { ProviderV2 } from "@opencode-ai/core/provider" -import { AgentAttachment, FileAttachment, ReferenceAttachment, Source } from "@opencode-ai/core/session-prompt" +import { AgentAttachment, FileAttachment, ReferenceAttachment, Source } from "@opencode-ai/core/session/prompt" import { Reference } from "@/reference/reference" import * as DateTime from "effect/DateTime" import { eq } from "@/storage/db" diff --git a/packages/opencode/src/session/schema.ts b/packages/opencode/src/session/schema.ts index f1622b6958..aa3a1e28d9 100644 --- a/packages/opencode/src/session/schema.ts +++ b/packages/opencode/src/session/schema.ts @@ -1,7 +1,7 @@ import { Schema } from "effect" import { Identifier } from "@/id/id" -import { Session as CoreSession } from "@opencode-ai/core/session" +import { Session as CoreSession } from "@opencode-ai/core/session/index" import { withStatics } from "@opencode-ai/core/schema" export const SessionID = CoreSession.ID diff --git a/packages/opencode/src/session/session.sql.ts b/packages/opencode/src/session/session.sql.ts index 610ca72c46..b1f40dcf1d 100644 --- a/packages/opencode/src/session/session.sql.ts +++ b/packages/opencode/src/session/session.sql.ts @@ -1,7 +1,7 @@ import { sqliteTable, text, integer, index, primaryKey, real } from "drizzle-orm/sqlite-core" import { ProjectTable } from "../project/project.sql" import type { MessageV2 } from "./message-v2" -import type { SessionMessage } from "@opencode-ai/core/session-message" +import type { SessionMessage } from "@opencode-ai/core/session/message" import type { Snapshot } from "../snapshot" import type { Permission } from "../permission" import type { ProjectID } from "../project/schema" diff --git a/packages/opencode/src/v2/session.ts b/packages/opencode/src/v2/session.ts index ec67d1820a..3cda673010 100644 --- a/packages/opencode/src/v2/session.ts +++ b/packages/opencode/src/v2/session.ts @@ -4,10 +4,10 @@ import { WorkspaceID } from "@/control-plane/schema" import { and, asc, desc, eq, gt, gte, isNull, like, lt, or, type SQL } from "@/storage/db" import * as Database from "@/storage/db" import { Context, DateTime, Effect, Layer, Option, Schema } from "effect" -import { SessionMessage } from "@opencode-ai/core/session-message" -import type { Prompt } from "@opencode-ai/core/session-prompt" +import { SessionMessage } from "@opencode-ai/core/session/message" +import type { Prompt } from "@opencode-ai/core/session/prompt" import { ProjectID } from "@/project/schema" -import { SessionEvent } from "@opencode-ai/core/session-event" +import { SessionEvent } from "@opencode-ai/core/session/event" import { V2Schema } from "@opencode-ai/core/v2-schema" import { optionalOmitUndefined } from "@opencode-ai/core/schema" import { EventV2 } from "@opencode-ai/core/event" diff --git a/packages/opencode/test/server/httpapi-session.test.ts b/packages/opencode/test/server/httpapi-session.test.ts index 313245f732..25d0762643 100644 --- a/packages/opencode/test/server/httpapi-session.test.ts +++ b/packages/opencode/test/server/httpapi-session.test.ts @@ -19,7 +19,7 @@ import { MessageID, PartID, SessionID, type SessionID as SessionIDType } from ". import { MessageV2 } from "../../src/session/message-v2" import { Database } from "@/storage/db" import { SessionMessageTable, SessionTable } from "@/session/session.sql" -import { SessionMessage } from "@opencode-ai/core/session-message" +import { SessionMessage } from "@opencode-ai/core/session/message" import { ModelV2 } from "@opencode-ai/core/model" import { ProviderV2 } from "@opencode-ai/core/provider" import * as DateTime from "effect/DateTime" diff --git a/packages/opencode/test/v2/session-message-updater.test.ts b/packages/opencode/test/v2/session-message-updater.test.ts index 588521281c..394a4a8708 100644 --- a/packages/opencode/test/v2/session-message-updater.test.ts +++ b/packages/opencode/test/v2/session-message-updater.test.ts @@ -4,8 +4,8 @@ import { SessionID } from "../../src/session/schema" import { EventV2 } from "@opencode-ai/core/event" import { ModelV2 } from "@opencode-ai/core/model" import { ProviderV2 } from "@opencode-ai/core/provider" -import { SessionEvent } from "@opencode-ai/core/session-event" -import { SessionMessageUpdater } from "@opencode-ai/core/session-message-updater" +import { SessionEvent } from "@opencode-ai/core/session/event" +import { SessionMessageUpdater } from "@opencode-ai/core/session/message-updater" test("step snapshots carry over to assistant messages", () => { const state: SessionMessageUpdater.MemoryState = { messages: [] }