From ffdb4a70d760f672b4948a336e0609ac8ae4e9fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pascal=20Andr=C3=A9?= Date: Sat, 7 Nov 2026 15:34:22 +0100 Subject: [PATCH] fix(storage): support native split assistant parts across history and pruning (#1359) ## Summary Support OpenCode's optional `experimental.part_rows` storage throughout direct history/search, navigation/outline, image gallery and technical cleanup. Reconstruct ordinal split rows followed by any inline tail, including existing split data after the flag is disabled. Missions source projections share the same ordering. Validate the actual storage schema, part ownership/type/call identity and byte/row budgets before reconstruction. Keep metadata reads free of JSON decoding and preserve native rowid seeks. Cleanup deletes selected split rows and inline entries atomically with the envelope CAS, media-reference release and retry receipt; retained split rows keep their exact bytes and ordinals. ## Validation - Storage-specific regressions: 10/10 passed, covering mixed messages, stale revisions, rollback, media ownership, gallery pixels/provenance, metadata queries and malformed/oversized storage. - Final-source pruning and affected native Missions tests: 318/318 passed. - Full server suite: 4,098 passed, 23 skipped, one timeout in the 70-passage recurring-Missions scenario; the same scenario passed in isolation in 35 seconds. - Server and workspace typechecks passed; help tests passed 31/31 over all ten locales. - Rebuilt final committed bundles: official private CLI 2.2.3 passed the complete default-storage journey; 2.2.6 passed split storage with real UI cleanup and persisted reload, media release, claim concurrency, next model payload and mixed content after flag-off/restart. - Official private CLI 2.2.6 passed the split Design gallery journey: exact pixels, proven lineage and late read-back with unverifiable provenance. - Add a latest-runtime split-storage CI journey. Update technical documentation and history help in all ten locales. Minimum runtime and dependency pins stay at 2.2.3; the recommendation remains 2.2.5. No user database or shared service is modified by the isolated fixtures. This PR does not include the separate scheduled-wake fix in #1350. --- .github/workflows/pr-build.yml | 4 + dev-docs/OPENCODE_COMPATIBILITY.md | 8 + dev-docs/SESSION_HISTORY_QUERIES.md | 5 + dev-docs/SESSION_PRUNING_RPC.md | 7 +- dev-docs/SESSION_PRUNING_SAFETY.md | 10 +- docs/help/de/history.md | 2 + docs/help/en/history.md | 2 + docs/help/es/history.md | 2 + docs/help/fr/history.md | 2 + docs/help/he/history.md | 2 + docs/help/ja/history.md | 2 + docs/help/ne/history.md | 2 + docs/help/ru/history.md | 2 + docs/help/tr/history.md | 2 + docs/help/zh-Hans/history.md | 2 + .../opencode/missions/native-message-data.ts | 9 +- .../missions/native-source-fence.test.ts | 4 +- .../session-pruning/assets-gallery-proof.ts | 10 +- .../session-pruning/assets-gallery.ts | 13 +- .../opencode/session-pruning/assets-store.ts | 12 +- .../session-pruning/history-ranked-store.ts | 12 +- .../opencode/session-pruning/history-store.ts | 6 +- .../session-pruning/message-storage.test.ts | 212 ++++++++++++++++++ .../session-pruning/message-storage.ts | 72 ++++++ .../native-assistant-content.ts | 5 + .../session-pruning/navigation-store.ts | 6 +- .../opencode/session-pruning/outline-index.ts | 12 +- .../session-pruning/outline-preview.ts | 9 +- .../src/opencode/session-pruning/package.json | 2 +- .../opencode/session-pruning/preview-store.ts | 6 +- .../opencode/session-pruning/transaction.ts | 13 +- scripts/test-design-companion-native.mjs | 2 +- scripts/test-session-media-blob-native.mjs | 5 +- scripts/test-session-pruning-native.mjs | 40 +++- 34 files changed, 453 insertions(+), 51 deletions(-) create mode 100644 packages/server/src/opencode/session-pruning/message-storage.test.ts create mode 100644 packages/server/src/opencode/session-pruning/message-storage.ts create mode 100644 packages/server/src/opencode/session-pruning/native-assistant-content.ts diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index ae6372d36..3cede8944 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -197,6 +197,10 @@ jobs: if ($LASTEXITCODE) { exit $LASTEXITCODE } node scripts/test-session-pruning-native.mjs $cli if ($LASTEXITCODE) { exit $LASTEXITCODE } + if ('${{ matrix.runtime }}' -eq 'latest') { + node scripts/test-session-pruning-native.mjs $cli --part-rows + if ($LASTEXITCODE) { exit $LASTEXITCODE } + } # Shipped Missions bundle: loading, model-visible tools through the # ordinary prompt loop, journal RPC and recurrence capability. node scripts/test-missions-native-contract.mjs $cli diff --git a/dev-docs/OPENCODE_COMPATIBILITY.md b/dev-docs/OPENCODE_COMPATIBILITY.md index a27979bc5..f159c38b8 100644 --- a/dev-docs/OPENCODE_COMPATIBILITY.md +++ b/dev-docs/OPENCODE_COMPATIBILITY.md @@ -145,6 +145,14 @@ connections and Missions authority checks keep their blocking contracts. See ### Replay contract +SQL history/search, navigation/outline, gallery and pruning consume assistant +`session_part` rows in ordinal order followed by inline content. The shared bounded +`message-storage.ts` relation mirrors native history, including mixed messages and +stored split rows while `experimental.part_rows` is disabled. Pruning deletes only +selected split rows and inline entries under its existing claim and transaction fences. +Missions assistant-message projections use the same ordered-content SQL expression, +retaining their own read budgets and source fences for mixed messages. + Text/reasoning/tool/step provider data is one `native` field (on a key shared by a tool call and its result, the result wins). Summary compactions reference `metadata.keepID`, with kept text in `metadata.recent`; provider compactions diff --git a/dev-docs/SESSION_HISTORY_QUERIES.md b/dev-docs/SESSION_HISTORY_QUERIES.md index 70635084e..9eb92f30f 100644 --- a/dev-docs/SESSION_HISTORY_QUERIES.md +++ b/dev-docs/SESSION_HISTORY_QUERIES.md @@ -50,6 +50,11 @@ finishes first. JSON is materialized one message at a time, limited to 16 MiB. Oversized or unsupported content is counted as skipped and reported to the UI. Never add an index, trigger, FTS table or persistent transcript copy. +The shared `session-pruning/message-storage.ts` relation reconstructs assistant +content from ordered `session_part` rows followed by the inline tail. Stored-byte +bounds cover both forms before reconstruction; all resulting part indexes use that +combined order. Existing split rows retain this read behavior after `part_rows` is disabled. + Cursors bind scope and query; the broker also binds the authorized directory inventory. A rowid horizon excludes new rows appended during each location scan. The search window never starts a separate statistics traversal, and an empty diff --git a/dev-docs/SESSION_PRUNING_RPC.md b/dev-docs/SESSION_PRUNING_RPC.md index e2f1e63ad..830c07987 100644 --- a/dev-docs/SESSION_PRUNING_RPC.md +++ b/dev-docs/SESSION_PRUNING_RPC.md @@ -24,6 +24,7 @@ methods, the integration OpenCode's maintainers recommend for plugins - `media-blob.ts`: the native read form of rows stored with OpenCode's optional media storage (`experimental.media_blob`: `media:` refs into `blob`, owned through `blob_ref`). Preview, pruning, prune candidates, navigation windows and assets inline refs, or give the native unavailable placeholder, exactly as native reads do, within the same byte budgets measured on that form. Rows naming no ref pass through untouched; ref-backed rows whose blob tables are missing or unrecognized fail closed as `unsupported_storage`. - `database-path.ts`: daemon-side OpenCode data-directory/environment conventions, with an optional path override. - `transaction.ts`: synchronous content-only CAS and atomic idempotency receipt. +- `message-storage.ts`: bounded native message relation for inline content and ordinal `session_part` rows followed by any inline tail; schema inspection selects the stored representation. Headers account for both stores before JSON materialization. Invalid part ownership, type or call identity is refused. Metadata-only rowid/sequence seeks retain their native tables. - `claim-fence.ts`: native execution-claim and database-identity checks. - `service.ts`: database connection and a fresh `ctx.storage` challenge. - `isolated-store.ts`: in-memory-only test adapter; never a live write bypass. @@ -42,8 +43,10 @@ The UI offers individual, per-message, group and session cleanup entry points. I fetches the native message, resolves selected tools by ID and reasoning by a unique content/time match, then sends indexes plus a canonical SHA-256 content revision. Ambiguous reasoning or stale content fails closed. It never submits replacement text. -The plugin compares that revision with the native read form (`media-blob.ts`), writes -surviving parts in their stored form, and in the same transaction releases the +The plugin compares that revision with the reconstructed native read form (`message-storage.ts`, `media-blob.ts`), +deletes selected split rows and removes selected entries from the inline tail under the same write lock. +Retained split rows keep their original ordinals and JSON bytes; native readers order ordinals without requiring contiguous values. +The envelope's exact-data CAS, part deletion and receipt commit together. In the same transaction it releases the `blob_ref` rows of refs the removed parts alone named, deleting blobs no ref keeps, as native content updates do. Selections allow up to 100,001 unique indices (0-100,000), within a 1 MiB broker diff --git a/dev-docs/SESSION_PRUNING_SAFETY.md b/dev-docs/SESSION_PRUNING_SAFETY.md index 02b771119..3fef4fe49 100644 --- a/dev-docs/SESSION_PRUNING_SAFETY.md +++ b/dev-docs/SESSION_PRUNING_SAFETY.md @@ -20,8 +20,8 @@ writers are unsupported. a subsequent primary request and a restart both see pruned content (native smoke test). - `session/projector.ts` / `bus.ts`: native projections run transactionally. The - plugin changes only an already-completed assistant row's content, with exact-data - CAS; it never manufactures sequence values or emits replacement events. + plugin changes only an already-completed assistant's stored content, with exact-data + CAS on its envelope and split-part deletions under the same write lock; it never manufactures sequence values or emits replacement events. Rather than relying on move/revert controls, the plugin rechecks directory/project/workspace identity and staged-revert/compaction state *inside* @@ -51,7 +51,9 @@ keep their text. Never rewrite/replay events to repair them or mutate V1 tables. 4. Bound/read the row, check assistant completion, completed/error tool state, known content kinds, selected technical indices and canonical SHA-256 revision. - Update only `$.content`, with CAS against the exact stored row data. + Reconstruct ordinal split parts followed by the inline tail. Delete only selected + split rows and selected inline entries; retained split ordinals and bytes stay intact. + Update only envelope `$.content`, with CAS against its exact stored row data. 5. Insert a content-free receipt in the same transaction; commit or roll back both. Identical requests cannot accidentally delete the next block after indices shift. Receipts are retained indefinitely. @@ -71,7 +73,7 @@ DB are unsupported. history is in the context window. Copied summaries, native checkpoints and forks keep their content. Provider-specific state continuation and budget estimates need their own validation. -- **Persistence**: only selected V2 content arrays change. Usage/cost record past +- **Persistence**: only selected V2 technical content changes, in split rows or inline arrays. Usage/cost record past execution and stay unchanged. - **Physical storage**: freed SQLite pages become reusable. The DB file shrinks only through separately consented maintenance such as VACUUM; this plugin never diff --git a/docs/help/de/history.md b/docs/help/de/history.md index ce9801c22..1dd233dbd 100644 --- a/docs/help/de/history.md +++ b/docs/help/de/history.md @@ -1,5 +1,7 @@ # Verlauf, Suche und Bereinigung +Suche, Zeitleiste, Bildergalerie und technische Bereinigung unterstützen auch OpenCodes experimentellen `part_rows`-Speicher und Nachrichten mit getrennten und eingebetteten Teilen. Bereits getrennte Teile bleiben nach dem Deaktivieren der Option lesbar. + Gespeicherter Verlauf, aktuell angezeigter Gesprächsverlauf und komprimierter Modellkontext sind getrennte Dinge. Wählen Sie Navigation oder Filter, bevor Sie etwas löschen. ## Gesamten Verlauf durchsuchen {#full-history-search} diff --git a/docs/help/en/history.md b/docs/help/en/history.md index 852ecd95c..b2e81ace9 100644 --- a/docs/help/en/history.md +++ b/docs/help/en/history.md @@ -1,5 +1,7 @@ # History, search and cleanup +Search, timeline navigation, the image gallery and technical cleanup also support OpenCode's experimental `part_rows` storage, including messages containing both split and inline parts. Existing split parts remain supported after the option is disabled. + Stored history, what the transcript currently displays and the model's compacted context are separate. Choose navigation or filtering before deleting anything. ## Search full history {#full-history-search} diff --git a/docs/help/es/history.md b/docs/help/es/history.md index 29c55628e..a8b9b8079 100644 --- a/docs/help/es/history.md +++ b/docs/help/es/history.md @@ -1,5 +1,7 @@ # Historial, búsqueda y limpieza +La búsqueda, la cronología, la galería de imágenes y la limpieza técnica admiten también el almacenamiento experimental `part_rows` de OpenCode y los mensajes con partes separadas e integradas. Las partes ya separadas siguen siendo compatibles tras desactivar la opción. + El historial almacenado, lo que muestra actualmente la transcripción y el contexto compactado del modelo son cosas distintas. Elige navegar o filtrar antes de eliminar nada. ## Buscar en todo el historial {#full-history-search} diff --git a/docs/help/fr/history.md b/docs/help/fr/history.md index c82de3dc3..2346a6fe0 100644 --- a/docs/help/fr/history.md +++ b/docs/help/fr/history.md @@ -1,5 +1,7 @@ # Historique, recherche et nettoyage +La recherche, la chronologie, la galerie d’images et le nettoyage technique prennent aussi en charge le stockage expérimental `part_rows` d’OpenCode, y compris les messages mixtes. Les éléments déjà séparés restent lisibles après désactivation de l’option. + L’historique enregistré, ce que la transcription affiche actuellement et le contexte compacté du modèle sont distincts. Préférez la navigation ou le filtrage avant toute suppression. ## Rechercher dans tout l’historique {#full-history-search} diff --git a/docs/help/he/history.md b/docs/help/he/history.md index 6683cd466..c80d32268 100644 --- a/docs/help/he/history.md +++ b/docs/help/he/history.md @@ -1,5 +1,7 @@ # היסטוריה, חיפוש וניקוי +החיפוש, ציר הזמן, גלריית התמונות והניקוי הטכני תומכים גם באחסון הניסיוני `part_rows` של OpenCode, כולל הודעות עם חלקים נפרדים וחלקים מוטמעים יחד. חלקים שכבר נשמרו בנפרד נשארים נתמכים גם לאחר כיבוי האפשרות. + ההיסטוריה השמורה, מה שהתמליל מציג כרגע וההקשר המסוכם של המודל הם דברים נפרדים. בחרו בניווט או בסינון לפני מחיקת דבר כלשהו. ## חיפוש בהיסטוריה המלאה {#full-history-search} diff --git a/docs/help/ja/history.md b/docs/help/ja/history.md index 0bdfb936f..50b1708cb 100644 --- a/docs/help/ja/history.md +++ b/docs/help/ja/history.md @@ -1,5 +1,7 @@ # 履歴、検索、クリーンアップ +検索、タイムライン、画像ギャラリー、技術的な内容の削除は、OpenCode の実験的な `part_rows` ストレージにも対応しています。分離されたパーツとインラインのパーツが混在するメッセージや、オプションを無効にする前に分離されたパーツも扱えます。 + 保存された履歴、会話欄に現在表示されている内容、モデルのコンパクト化されたコンテキストは別物です。何かを削除する前に、移動やフィルターを選んでください。 ## 全履歴を検索する {#full-history-search} diff --git a/docs/help/ne/history.md b/docs/help/ne/history.md index 9ced1f5e5..84e51f92b 100644 --- a/docs/help/ne/history.md +++ b/docs/help/ne/history.md @@ -1,5 +1,7 @@ # इतिहास, खोज र सफाइ +खोज, टाइमलाइन, छवि ग्यालरी र प्राविधिक सामग्रीको सफाइले OpenCode को प्रयोगात्मक `part_rows` भण्डारण र छुट्टाछुट्टै तथा inline अंश मिसिएका सन्देशहरू पनि समर्थन गर्छन्। विकल्प बन्द गरेपछि पनि पहिले छुट्टै भण्डारण भएका अंशहरू पढ्न सकिन्छ। + सुरक्षित इतिहास, transcript ले अहिले देखाउने कुरा र मोडेलको संक्षिप्त सन्दर्भ फरक हुन्। केही मेट्नुअघि नेभिगेसन वा filtering रोज्नुहोस्। ## पूर्ण इतिहास खोज्नुहोस् {#full-history-search} diff --git a/docs/help/ru/history.md b/docs/help/ru/history.md index e0664861a..c31d45cf2 100644 --- a/docs/help/ru/history.md +++ b/docs/help/ru/history.md @@ -1,5 +1,7 @@ # История, поиск и очистка +Поиск, временная шкала, галерея изображений и очистка технического содержимого также поддерживают экспериментальное хранилище OpenCode `part_rows`, включая сообщения с отдельными и встроенными частями. Уже сохранённые отдельно части остаются доступными после отключения этой настройки. + Сохранённая история, текущее отображение переписки и компактированный контекст модели — разные вещи. Прежде чем что-либо удалять, выберите навигацию или фильтрацию. ## Поиск по всей истории {#full-history-search} diff --git a/docs/help/tr/history.md b/docs/help/tr/history.md index cacf1e7e1..c5d565d80 100644 --- a/docs/help/tr/history.md +++ b/docs/help/tr/history.md @@ -1,5 +1,7 @@ # Geçmiş, arama ve temizleme +Arama, zaman çizelgesi, görsel galerisi ve teknik temizleme, OpenCode'un deneysel `part_rows` depolamasını ve ayrı parçalarla satır içi parçaları birlikte içeren iletileri de destekler. Seçenek kapatıldıktan sonra da önceden ayrılmış parçalar okunabilir. + Saklanan geçmiş, konuşma dökümünün şu anda görüntüledikleri ve modelin sıkıştırılmış bağlamı ayrıdır. Herhangi bir şeyi silmeden önce gezinmeyi veya filtrelemeyi seçin. ## Tüm geçmişi ara {#full-history-search} diff --git a/docs/help/zh-Hans/history.md b/docs/help/zh-Hans/history.md index 1014eb390..3fac0a2c6 100644 --- a/docs/help/zh-Hans/history.md +++ b/docs/help/zh-Hans/history.md @@ -1,5 +1,7 @@ # 历史记录、搜索与清理 +搜索、时间线、图片库和技术内容清理也支持 OpenCode 的实验性 `part_rows` 存储,包括同时包含分离部分和内联部分的消息。关闭该选项后,已单独存储的部分仍可读取。 + 已存储历史、对话当前显示的内容和模型压缩后的上下文彼此独立。删除任何内容前,先考虑导航或筛选。 ## 搜索完整历史 {#full-history-search} diff --git a/packages/server/src/opencode/missions/native-message-data.ts b/packages/server/src/opencode/missions/native-message-data.ts index 821c234a9..002921544 100644 --- a/packages/server/src/opencode/missions/native-message-data.ts +++ b/packages/server/src/opencode/missions/native-message-data.ts @@ -1,8 +1,9 @@ -/** Native assistant reads prefer ordinal session_part rows, retaining inline - * content when none exist. Callers alias session_message as `m` and keep their +import { nativeAssistantContentSQL } from "../session-pruning/native-assistant-content" + +/** Native assistant reads combine ordinal session_part rows and the inline tail. + * Callers alias session_message as `m` and keep their * existing message/byte bounds and transaction graph. */ export const nativeMessageDataSQL = `CASE WHEN m.type='assistant' AND EXISTS( SELECT 1 FROM session_part WHERE message_id=m.id) - THEN json_set(m.data,'$.content',json((SELECT json_group_array(json(data)) FROM - (SELECT data FROM session_part WHERE message_id=m.id ORDER BY ordinal)))) + THEN json_set(m.data,'$.content',json(${nativeAssistantContentSQL("m.id", "m.data")})) ELSE m.data END` diff --git a/packages/server/src/opencode/missions/native-source-fence.test.ts b/packages/server/src/opencode/missions/native-source-fence.test.ts index 752c3df59..87b2eebbd 100644 --- a/packages/server/src/opencode/missions/native-source-fence.test.ts +++ b/packages/server/src/opencode/missions/native-source-fence.test.ts @@ -35,9 +35,9 @@ test("source projections use the document transaction graph and reject changed m const prefix = factory(); prefix.message("source", message.id, nativePassageMessageDigest(Schema.decodeUnknownSync(SessionMessage.Info)(message))) const family = factory(); family.family("source") const assistant = { type: "assistant", id: "msg_assistant", time: { created: 1001, completed: 1002 }, agent: "worker", - model: { providerID: "fixture", id: "fixture" }, content: [{ type: "text", text: "split source" }], + model: { providerID: "fixture", id: "fixture" }, content: [{ type: "text", text: "split source" }, { type: "text", text: "inline tail" }], cost: 0, tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, finish: "stop" } - db.prepare("INSERT INTO session_message VALUES(?,?,?,?,?)").run(assistant.id, 0, "assistant", JSON.stringify({ ...assistant, content: [] }), "source") + db.prepare("INSERT INTO session_message VALUES(?,?,?,?,?)").run(assistant.id, 0, "assistant", JSON.stringify({ ...assistant, content: assistant.content.slice(1) }), "source") db.prepare("INSERT INTO session_part VALUES(?,?,?,?,?)").run(assistant.id, "source", 0, "text", JSON.stringify(assistant.content[0])) const split = factory(); split.message("source", assistant.id, nativePassageMessageDigest(Schema.decodeUnknownSync(SessionMessage.Info)(assistant))) assert.equal(split.current(), true) diff --git a/packages/server/src/opencode/session-pruning/assets-gallery-proof.ts b/packages/server/src/opencode/session-pruning/assets-gallery-proof.ts index b9e113c17..1d292071e 100644 --- a/packages/server/src/opencode/session-pruning/assets-gallery-proof.ts +++ b/packages/server/src/opencode/session-pruning/assets-gallery-proof.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto" import path from "node:path" import type { DatabaseSync, SQLInputValue } from "node:sqlite" +import { messageStorage } from "./message-storage" import { MESSAGE_LIMIT } from "./assets-store" // Generation annotations of the image gallery, proven from the transcript alone @@ -88,9 +89,10 @@ export function promptFields(part: Part) { */ export function coverage(db: DatabaseSync, sessionID: string, boundary: string | undefined, top: number, through: number, preloaded: Map, windowRows: number): Coverage { - const rows = db.prepare(`SELECT id,seq,length(CAST(data AS BLOB)) AS bytes FROM session_message WHERE session_id=? AND type='assistant' + const storage = messageStorage(db) + const rows = db.prepare(`SELECT id,seq,${storage.bytes()} AS bytes FROM ${storage.table} WHERE session_id=? AND type='assistant' AND seq<=?${visible(boundary)} ORDER BY seq DESC LIMIT ?`).all(sessionID, top, ...after(boundary), windowRows + LOOKBACK_MESSAGES) - const read = db.prepare("SELECT data FROM session_message WHERE session_id=? AND id=?") + const read = db.prepare(`SELECT data FROM ${storage.table} WHERE session_id=? AND id=?`) const calls: Call[] = [], scanned: Array<{ seq: number; bytes: number }> = [] let budget = 0, stopped = false for (const row of rows) { @@ -99,7 +101,9 @@ export function coverage(db: DatabaseSync, sessionID: string, boundary: string | if (data === undefined) { if (bytes > MESSAGE_LIMIT || budget + bytes > LOOKBACK_BYTES) { stopped = true; break } budget += bytes - data = JSON.parse(String(read.get(sessionID, row.id)?.data)) + const stored = read.get(sessionID, row.id)?.data + if (typeof stored !== "string") { stopped = true; break } + data = JSON.parse(stored) } scanned.push({ seq, bytes }) for (const part of Array.isArray(data?.content) ? data.content : []) { diff --git a/packages/server/src/opencode/session-pruning/assets-gallery.ts b/packages/server/src/opencode/session-pruning/assets-gallery.ts index 75cf25903..d0590c03b 100644 --- a/packages/server/src/opencode/session-pruning/assets-gallery.ts +++ b/packages/server/src/opencode/session-pruning/assets-gallery.ts @@ -7,6 +7,7 @@ import { assetBytes, cursorSchema, hash, listed, MESSAGE_LIMIT, storedAsset } fr import { GALLERY_RESPONSE_LIMIT, type GalleryEntry, type GenerationTarget, type ImageReferenceResult, type ImagesResult } from "./assets-gallery-contract" import { after, annotate, coverage, generationState, IMAGE_TOOL, interval, isGalleryFile, key, labelOf, promptFields, relativeInside, resolveName, sameFile, visible, writes, writesAnyFile, type Coverage, type Part } from "./assets-gallery-proof" +import { messageStorage } from "./message-storage" // The image gallery (dev-docs/SESSION_PRUNING_RPC.md#image-gallery): image `read` parts and // `experimental_image_generate` calls of one session, recomputed from the transcript on every request. @@ -36,9 +37,10 @@ export async function listGalleryImages(db: DatabaseSync, scope: HistoryScope, c if (previous && previous.binding !== binding) return { status: "blocked", reason: "conflict" } const through = previous?.through ?? throughOf(db, sessionID, boundary) const where = "session_id=? AND type='assistant'" + visible(boundary), params = [sessionID, ...after(boundary)] - const rows = db.prepare(`SELECT id,seq,length(CAST(data AS BLOB)) AS bytes FROM session_message WHERE ${where} + const storage = messageStorage(db) + const rows = db.prepare(`SELECT id,seq,${storage.bytes()} AS bytes FROM ${storage.table} WHERE ${where} AND seq<=? AND seq<=? ORDER BY seq DESC LIMIT ?`).all(...params, through, previous?.sequence ?? through, Math.min(windowMessages, WINDOW_MESSAGES)) - const read = db.prepare("SELECT data FROM session_message WHERE session_id=? AND id=?") + const read = db.prepare(`SELECT data FROM ${storage.table} WHERE session_id=? AND id=?`) const preloaded = new Map(), window: Array<{ id: string; seq: number; data: any; items: Array<{ part: number; index?: number }> }> = [] let examined = 0, next: { sequence: number; offset: number } | undefined for (const row of rows) { @@ -48,7 +50,9 @@ export async function listGalleryImages(db: DatabaseSync, scope: HistoryScope, c // A row the byte budget refuses is the continuation; every admitted row is emitted first. if (examined && examined + bytes > WINDOW_BYTES) { next = { sequence: seq, offset: 0 }; break } examined += bytes - const data = JSON.parse(String(read.get(sessionID, row.id)?.data)) + const stored = read.get(sessionID, row.id)?.data + if (typeof stored !== "string") return { status: "blocked", reason: "unsupported_storage" } + const data = JSON.parse(stored) const files = listed(db, data) if (!files) return { status: "blocked", reason: "unsupported_storage" } preloaded.set(String(row.id), data) @@ -131,7 +135,8 @@ export async function galleryImageReference(db: DatabaseSync, scope: HistoryScop if (annotate(cov, scope.directory, sessionID, seq, found.tool, file).origin !== "proven" || !span) return refused("unverifiable") start = span.start } else { - const row = db.prepare(`SELECT seq,length(CAST(data AS BLOB)) AS bytes,CASE WHEN length(CAST(data AS BLOB))<=? THEN data END AS data FROM session_message + const storage = messageStorage(db) + const row = db.prepare(`SELECT seq,${storage.bytes()} AS bytes,CASE WHEN ${storage.bytes()}<=? THEN data END AS data FROM ${storage.table} WHERE session_id=? AND type='assistant' AND id=?${visible(boundary)}`).get(MESSAGE_LIMIT, sessionID, target.messageID, ...after(boundary)) if (!row) return refused("missing") if (typeof row.data !== "string") return refused("unverifiable") diff --git a/packages/server/src/opencode/session-pruning/assets-store.ts b/packages/server/src/opencode/session-pruning/assets-store.ts index 1737ad656..693c7ceac 100644 --- a/packages/server/src/opencode/session-pruning/assets-store.ts +++ b/packages/server/src/opencode/session-pruning/assets-store.ts @@ -6,6 +6,7 @@ import { ownedSession } from "./navigation-scope" import type { HistoryScope } from "./history-store" import { ASSET_URI_LIMIT, isEmbeddedAsset, type AssetTarget, type AssetsResult, type AssetReadResult } from "./assets-contract" import { mediaBlobBase64, mediaBlobSizes, mediaDataUri, mediaDataUriLength, toolMediaHash } from "./media-blob" +import { messageStorage } from "./message-storage" export const MESSAGE_LIMIT = 16 * 1024 * 1024 export const cursorSchema = z.object({ binding: z.string().length(64), through: z.number().int().min(0), @@ -53,9 +54,10 @@ export async function listSessionAssets(db: DatabaseSync, scope: HistoryScope, c const params: SQLInputValue[] = [scope.sessionID!, ...(boundary ? [boundary] : [])] const maximum = Number(db.prepare(`SELECT coalesce(max(seq),0) AS value FROM session_message WHERE ${where}`).get(...params)?.value) const through = previous?.through ?? maximum - const rows = db.prepare(`SELECT id,seq,length(CAST(data AS BLOB)) AS bytes FROM session_message WHERE ${where} + const storage = messageStorage(db) + const rows = db.prepare(`SELECT id,seq,${storage.bytes()} AS bytes FROM ${storage.table} WHERE ${where} AND seq<=? AND seq<=? ORDER BY seq DESC LIMIT 32`).all(...params, through, previous?.sequence ?? through) - const read = db.prepare("SELECT data FROM session_message WHERE session_id=? AND id=?") + const read = db.prepare(`SELECT data FROM ${storage.table} WHERE session_id=? AND id=?`) const entries: Extract["entries"] = [] let examined = 0 for (const row of rows) { @@ -66,7 +68,8 @@ export async function listSessionAssets(db: DatabaseSync, scope: HistoryScope, c if (examined && examined + bytes > 32 * 1024 * 1024) return page(sequence, 0) examined += bytes const data = read.get(scope.sessionID!, row.id)?.data - const assets = typeof data === "string" ? listed(db, JSON.parse(data)) : [] + if (typeof data !== "string") return { status: "blocked", reason: "unsupported_storage" } + const assets = listed(db, JSON.parse(data)) if (!assets) return { status: "blocked", reason: "unsupported_storage" } const offset = sequence === previous?.sequence ? previous.offset : 0 for (let index = offset; index < assets.length; index++) { @@ -96,7 +99,8 @@ export async function readSessionAsset(db: DatabaseSync, scope: HistoryScope, ta } /** The digest-bound tool file a target names, inside the caller's read transaction; the size is checked before its JSON loads. */ export function storedAsset(db: DatabaseSync, sessionID: string, boundary: string | undefined, target: AssetTarget) { - const row = db.prepare(`SELECT seq,length(CAST(data AS BLOB)) AS bytes,CASE WHEN length(CAST(data AS BLOB))<=? THEN data END AS data FROM session_message + const storage = messageStorage(db) + const row = db.prepare(`SELECT seq,${storage.bytes()} AS bytes,CASE WHEN ${storage.bytes()}<=? THEN data END AS data FROM ${storage.table} WHERE session_id=? AND type='assistant' AND id=? ${boundary ? "AND id < ?" : ""}`).get(MESSAGE_LIMIT, sessionID, target.messageID, ...(boundary ? [boundary] : [])) if (!row) return { status: "missing" } as const if (typeof row.data !== "string") return { status: Number(row.bytes) > MESSAGE_LIMIT ? "limit" : "missing" } as const diff --git a/packages/server/src/opencode/session-pruning/history-ranked-store.ts b/packages/server/src/opencode/session-pruning/history-ranked-store.ts index 4bcad3540..5c6dc34c6 100644 --- a/packages/server/src/opencode/session-pruning/history-ranked-store.ts +++ b/packages/server/src/opencode/session-pruning/history-ranked-store.ts @@ -9,6 +9,7 @@ import { compareRank, dateBounds, diverseHits, idfWeights, rankedExcerpt, termMa import { mayContainHistoryTerms } from "./history-ranked-prefilter" import type { HistoryRankedInput, HistoryRankedPage } from "./history-ranked-contract" import { rankedDirectoryOwnership, type CheckRankedDirectory } from "./history-ranked-ownership" +import { messageStorage } from "./message-storage" export const RANKED_LIMITS = { chunk: 128, messages: 16_384, candidates: 30_000, sessions: 600, judged: 600, prefilterBytes: 64 * 1024, scanMs: 1500, totalMs: 2000 } as const interface Candidate { position: number; messageID: string; sessionID: string; createdAt: number; flags?: boolean[]; score: number } @@ -85,11 +86,12 @@ export async function queryRankedHistory(db: DatabaseSync, scope: HistoryScope, return present } catch { return -2 } }) + const storage = messageStorage(db) const rows = db.prepare(`SELECT m.rowid AS position,m.id,m.session_id,m.time_created,s.directory,s.workspace_id,s.title, - CASE WHEN length(CAST(m.data AS BLOB)) <= ${MAX_MESSAGE_BYTES} - THEN codenomad_ranked_mask(CAST(${input.includeTechnical ? "m.data" : `CASE WHEN length(CAST(m.data AS BLOB)) > ${RANKED_LIMITS.prefilterBytes} + CASE WHEN ${storage.bytes("m")} <= ${MAX_MESSAGE_BYTES} + THEN codenomad_ranked_mask(CAST(${input.includeTechnical ? "m.data" : `CASE WHEN ${storage.bytes("m")} > ${RANKED_LIMITS.prefilterBytes} THEN ${textProjection} ELSE m.data END`} AS BLOB),m.type) ELSE -2 END AS mask - FROM session_message m CROSS JOIN session_v2 s ON s.id=m.session_id + FROM ${storage.table} m CROSS JOIN session_v2 s ON s.id=m.session_id WHERE ${where}${dates} AND m.rowid > ? AND m.rowid <= ? ORDER BY m.rowid DESC LIMIT ${RANKED_LIMITS.chunk}`) let candidates: Candidate[] = [] const owners = new Map() @@ -188,8 +190,8 @@ export async function queryRankedHistory(db: DatabaseSync, scope: HistoryScope, candidates.sort((a, b) => Number(!a.flags) - Number(!b.flags) || b.score - a.score || b.createdAt - a.createdAt || b.position - a.position) const judge = textJudge(terms, input.query, weights) const body = db.prepare(`SELECT m.type,m.time_created,s.directory,s.workspace_id,s.title, - CASE WHEN length(CAST(m.data AS BLOB)) <= ${MAX_MESSAGE_BYTES} THEN ${input.includeTechnical ? "m.data" : textProjection} END AS data - FROM session_message m JOIN session_v2 s ON s.id=m.session_id + CASE WHEN ${storage.bytes("m")} <= ${MAX_MESSAGE_BYTES} THEN ${input.includeTechnical ? "m.data" : textProjection} END AS data + FROM ${storage.table} m JOIN session_v2 s ON s.id=m.session_id WHERE ${where}${dates} AND m.rowid=? AND m.id=? AND m.session_id=?`) const judged: Judged[] = [] if (candidates.length > RANKED_LIMITS.judged) cap("judge_limit") diff --git a/packages/server/src/opencode/session-pruning/history-store.ts b/packages/server/src/opencode/session-pruning/history-store.ts index c0b297717..b35cb190c 100644 --- a/packages/server/src/opencode/session-pruning/history-store.ts +++ b/packages/server/src/opencode/session-pruning/history-store.ts @@ -5,6 +5,7 @@ import { z } from "zod" import { historyScopeSQL, historyContent, historyTexts, MAX_MESSAGE_BYTES, HISTORY_SLICE_MS, textProjection } from "./history-content" import { previewContent } from "./planner" import { projectStoredMedia } from "./media-blob" +import { messageStorage } from "./message-storage" import type { HistoryQuery, HistoryNativePage, HistoryHit } from "./history-contract" export interface HistoryScope { directory: string; workspaceID?: string; sessionID?: string; projectID?: string } @@ -44,10 +45,11 @@ export async function queryHistoryPage(db: DatabaseSync, scope: HistoryScope, in const countMessagesOnly = input.purpose === "stats" && !input.includeTechnical const textOnly = input.purpose === "search" && !input.includeTechnical const rowLimit = countMessagesOnly || input.purpose === "search" ? 1024 : 32 - const dataColumn = countMessagesOnly ? "NULL" : `CASE WHEN length(CAST(m.data AS BLOB)) <= ? THEN ${textOnly ? textProjection : "m.data"} END` + const storage = messageStorage(db) + const dataColumn = countMessagesOnly ? "NULL" : `CASE WHEN ${storage.bytes("m")} <= ? THEN ${textOnly ? textProjection : "m.data"} END` const rows = db.prepare(`SELECT m.rowid AS position,m.id,m.session_id,m.type,s.directory,s.workspace_id, ${dataColumn} AS data - FROM session_message m JOIN session_v2 s ON s.id=m.session_id + FROM ${countMessagesOnly ? "session_message" : storage.table} m JOIN session_v2 s ON s.id=m.session_id WHERE ${where} AND m.rowid > ? AND m.rowid <= ? ORDER BY m.rowid LIMIT ${rowLimit}`) let after = cursor.after const started = performance.now() diff --git a/packages/server/src/opencode/session-pruning/message-storage.test.ts b/packages/server/src/opencode/session-pruning/message-storage.test.ts new file mode 100644 index 000000000..d96052c1b --- /dev/null +++ b/packages/server/src/opencode/session-pruning/message-storage.test.ts @@ -0,0 +1,212 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { DatabaseSync } from "node:sqlite" +import path from "node:path" +import { createHash } from "node:crypto" +import { messageStorage } from "./message-storage" +import { queryHistoryPage } from "./history-store" +import { queryRankedHistory } from "./history-ranked-store" +import { readNavigationWindow } from "./navigation-store" +import { readSessionOutline } from "./outline-index" +import { readOutlinePreviews } from "./outline-preview" +import { listSessionAssets } from "./assets-store" +import { pruneTransaction } from "./transaction" +import { previewContent } from "./planner" +import { projectStoredMedia } from "./media-blob" +import { openFixture } from "./assets-gallery-fixture" + +const signal = () => new AbortController().signal +const partTable = `CREATE TABLE session_part(message_id TEXT NOT NULL,session_id TEXT NOT NULL,ordinal INTEGER NOT NULL, + type TEXT NOT NULL,call_id TEXT,time_created INTEGER NOT NULL,time_updated INTEGER NOT NULL,data TEXT NOT NULL, + PRIMARY KEY(message_id,ordinal));` +const content = [ + { type: "text", text: "Split needle answer", time: { created: 1, completed: 2 } }, + { type: "tool", id: "call_read", name: "read", state: { status: "completed", input: {}, content: [{ type: "text", text: "technical needle" }], metadata: {} }, time: { created: 3, completed: 4 } }, + { type: "reasoning", text: "private reasoning", time: { created: 5, completed: 6 } }, + { type: "text", text: "Inline tail", time: { created: 7, completed: 8 } }, +] +function split(db: DatabaseSync, count: number, messageID = "m") { + const row = db.prepare("SELECT session_id,data FROM session_message WHERE id=?").get(messageID)! + const data = JSON.parse(String(row.data)) + const insert = db.prepare("INSERT INTO session_part VALUES(?,?,?,?,?,?,?,?)") + data.content.slice(0, count).forEach((part: any, index: number) => insert.run(messageID, row.session_id, index, part.type, part.id ?? null, 1, 1, JSON.stringify(part))) + db.prepare("UPDATE session_message SET data=? WHERE id=?").run(JSON.stringify({ ...data, content: data.content.slice(count) }), messageID) +} +function fixture(t: test.TestContext) { + const db = new DatabaseSync(":memory:") + t.after(() => db.close()) + db.exec(`CREATE TABLE session_v2(id TEXT PRIMARY KEY,directory TEXT,project_id TEXT,workspace_id TEXT,parent_id TEXT,title TEXT,metadata TEXT,revert TEXT); + CREATE TABLE session_message(id TEXT PRIMARY KEY,session_id TEXT,type TEXT,seq INTEGER,time_created INTEGER,time_updated INTEGER,data TEXT); + CREATE TABLE kv(key TEXT PRIMARY KEY,value TEXT,time_created INTEGER,time_updated INTEGER); ${partTable}`) + const directory = path.resolve("split-fixture").replaceAll("\\", "/") + db.prepare("INSERT INTO session_v2 VALUES('s',?,'project',NULL,NULL,'Title','{}',NULL)").run(directory) + db.prepare("INSERT INTO session_message VALUES('m','s','assistant',1,1,1,?)").run(JSON.stringify({ time: { created: 1, completed: 9 }, metadata: { keep: true }, content })) + const scope = { sessionID: "s", directory, projectID: "project" } + const read = () => JSON.parse(String(db.prepare(`SELECT data FROM ${messageStorage(db).table} WHERE id='m'`).get()!.data)) + return { db, scope, read } +} + +test("inline, split and mixed messages share history, ranked search, navigation and outline positions", async t => { + for (const count of [0, 2, 4]) { + const f = fixture(t) + split(f.db, count) + assert.deepEqual(f.read().content, content) + const literal = await queryHistoryPage(f.db, f.scope, { purpose: "search", query: "split needle", includeTechnical: false }, signal()) + assert.equal(literal.hits.length, 1) + assert.equal(literal.hits[0].partIndex, 0) + const ranked = await queryRankedHistory(f.db, f.scope, { version: 1, query: "split needle", includeTechnical: false, limit: 10 }, signal(), async () => "owned") + assert.equal(ranked.hits.length, 1) + const window = await readNavigationWindow(f.db, f.scope, { kind: "oldest" }, signal()) + assert.equal(window.status, "window") + if (window.status === "window") assert.deepEqual(window.messages[0].content, content) + const outline = await readSessionOutline(f.db, f.scope, undefined, signal(), -1, [], true) + assert.equal(outline.status, "outline") + if (outline.status === "outline") assert.deepEqual([outline.entries[0].tools, outline.entries[0].reasoning, outline.entries[0].text], [1, 1, 2]) + const previews = await readOutlinePreviews(f.db, f.scope, ["m"], signal()) + assert.equal(previews.status, "previews") + if (previews.status === "previews") assert.match(previews.entries[0].text, /Split needle answer[\s\S]*Inline tail/) + } +}) + +test("split pruning preserves retained bytes, native ordinals and inline tail, with atomic replay receipts", t => { + const f = fixture(t) + split(f.db, 3) + const before = f.db.prepare("SELECT ordinal,data FROM session_part ORDER BY ordinal").all() + const preview = previewContent(f.read()) + assert.equal(preview.status, "preview") + if (preview.status !== "preview") return + const request = { sessionID: "s", messageID: "m", revision: preview.revision, indexes: [1, 2] } + const result = pruneTransaction(f.db, request, () => undefined, "receipt/") + assert.equal(result.status, "pruned") + assert.deepEqual(f.read().content, [content[0], content[3]]) + assert.deepEqual(f.db.prepare("SELECT ordinal,data FROM session_part ORDER BY ordinal").all(), [before[0]]) + assert.deepEqual(JSON.parse(String(f.db.prepare("SELECT data FROM session_message").get()!.data)).content, [content[3]]) + assert.deepEqual(pruneTransaction(f.db, request, () => undefined, "receipt/"), result) + assert.deepEqual(f.read().content, [content[0], content[3]]) + assert.equal(f.read().metadata.keep, true) +}) + +test("a failed inline CAS rolls back split deletion and receipt together", t => { + const f = fixture(t) + split(f.db, 3) + const before = f.read(), rows = f.db.prepare("SELECT * FROM session_part ORDER BY ordinal").all() + const preview = previewContent(before) + assert.equal(preview.status, "preview") + if (preview.status !== "preview") return + const proxy = new Proxy(f.db, { get(db, name) { + if (name === "prepare") return (sql: string) => sql.startsWith("UPDATE session_message SET data=json_set") + ? { run: () => ({ changes: 0 }) } : db.prepare(sql) + const value = Reflect.get(db, name, db) + return typeof value === "function" ? value.bind(db) : value + } }) + assert.deepEqual(pruneTransaction(proxy, { sessionID: "s", messageID: "m", revision: preview.revision, indexes: [1] }, () => undefined, "receipt/"), { status: "blocked", reason: "conflict" }) + assert.deepEqual(f.db.prepare("SELECT * FROM session_part ORDER BY ordinal").all(), rows) + assert.deepEqual(f.read(), before) + assert.equal(f.db.prepare("SELECT count(*) AS n FROM kv").get()!.n, 0) +}) + +test("part-only edits invalidate a pruning revision even when the inline envelope is unchanged", t => { + const f = fixture(t) + split(f.db, 3) + const preview = previewContent(f.read()) + assert.equal(preview.status, "preview") + if (preview.status !== "preview") return + const inline = f.db.prepare("SELECT data FROM session_message").get()!.data + f.db.prepare("UPDATE session_part SET data=json_set(data,'$.text','changed reasoning') WHERE ordinal=2").run() + const rows = f.db.prepare("SELECT * FROM session_part ORDER BY ordinal").all() + assert.deepEqual(pruneTransaction(f.db, { sessionID: "s", messageID: "m", revision: preview.revision, indexes: [2] }, () => undefined), { status: "blocked", reason: "conflict" }) + assert.deepEqual(f.db.prepare("SELECT * FROM session_part ORDER BY ordinal").all(), rows) + assert.equal(f.db.prepare("SELECT data FROM session_message").get()!.data, inline) +}) + +test("foreign ownership and mismatched tool identity fail closed; unknown table shapes are refused", async t => { + for (const mutation of ["session_id='foreign'", "call_id='wrong'"]) { + const f = fixture(t) + split(f.db, 3) + f.db.exec(`UPDATE session_part SET ${mutation} WHERE ordinal=1`) + assert.equal((await readNavigationWindow(f.db, f.scope, { kind: "oldest" }, signal())).status, "blocked") + } + const f = fixture(t) + f.db.exec("DROP TABLE session_part; CREATE TABLE session_part(data TEXT)") + assert.throws(() => messageStorage(f.db), /Unsupported native part storage/) +}) + +test("split pruning releases only dropped message media refs and preserves blobs shared by another owner", t => { + for (const shared of [false, true]) { + const f = fixture(t), bytes = Buffer.from("synthetic image"), hash = createHash("sha256").update(bytes).digest("hex") + f.db.exec("CREATE TABLE blob(hash TEXT PRIMARY KEY,data BLOB); CREATE TABLE blob_ref(session_id TEXT,message_id TEXT,hash TEXT)") + f.db.prepare("INSERT INTO blob VALUES(?,?)").run(hash, bytes) + f.db.prepare("INSERT INTO blob_ref VALUES('s','m',?)").run(hash) + if (shared) f.db.prepare("INSERT INTO blob_ref VALUES('other','other',?)").run(hash) + const data = f.read() + data.content[1].state.content = [{ type: "file", mime: "image/png", name: "image.png", uri: `media:${hash}` }] + f.db.prepare("UPDATE session_message SET data=?").run(JSON.stringify(data)) + split(f.db, 3) + const stored = f.read(), projection = projectStoredMedia(f.db, stored, Buffer.byteLength(JSON.stringify(stored)), 16 * 1024 * 1024) + assert.equal(projection.status, "ok") + if (projection.status !== "ok") continue + const preview = previewContent(projection.data) + assert.equal(preview.status, "preview") + if (preview.status !== "preview") continue + const result = pruneTransaction(f.db, { sessionID: "s", messageID: "m", revision: preview.revision, indexes: [1] }, () => undefined) + assert.equal(result.status, "pruned") + assert.equal(f.db.prepare("SELECT count(*) AS n FROM blob_ref WHERE message_id='m'").get()!.n, 0) + assert.equal(f.db.prepare("SELECT count(*) AS n FROM blob").get()!.n, shared ? 1 : 0) + assert.deepEqual(f.read().content, [content[0], content[2], content[3]]) + } +}) + +test("plain and blob-backed image galleries retain targets, pixels and provenance with mixed storage", async t => { + for (const kind of ["plain", "blob"] as const) { + const f = openFixture(kind) + t.after(f.close) + const before = await f.walk(true) + const image = before.all.find(entry => entry.kind === "image")! + assert(image.kind === "image") + const pixels = await f.read(image.target), reference = await f.reference(image.target) + f.raw.exec(partTable) + for (const row of f.raw.prepare("SELECT id,data FROM session_message WHERE type='assistant'").all()) { + const n = JSON.parse(String(row.data)).content.length + split(f.raw, Math.max(1, n - 1), String(row.id)) + } + assert.deepEqual(await f.walk(true), before) + assert.deepEqual(await f.read(image.target), pixels) + assert.deepEqual(await f.reference(image.target), reference) + assert.equal((await listSessionAssets(f.db, f.scope, undefined, signal())).status, "page") + } +}) + +test("split headers retain rowid seeks and do not execute JSON decoding", t => { + const f = fixture(t) + split(f.db, 3) + const storage = messageStorage(f.db) + const bytecode = f.db.prepare(`EXPLAIN SELECT ${storage.bytes()},time_updated FROM ${storage.table} WHERE id='m'`).all() + assert(!bytecode.some(op => String(op.p4).includes("json_")), "metadata headers do not reconstruct or decode messages") + const plan = f.db.prepare(`EXPLAIN QUERY PLAN SELECT m.rowid FROM ${storage.table} m WHERE m.rowid>? AND m.rowid<=? ORDER BY m.rowid DESC LIMIT 128`).all(0, 128) + assert(plan.some(row => /SEARCH raw USING INTEGER PRIMARY KEY/.test(String(row.detail)))) +}) + +test("too many small split rows are refused before aggregate JSON is built", t => { + const f = fixture(t) + f.db.prepare("UPDATE session_message SET data=json_set(data,'$.content',json('[]'))").run() + const insert = f.db.prepare("INSERT INTO session_part VALUES('m','s',?,'text',NULL,1,1,?)") + f.db.exec("BEGIN") + for (let index = 0; index < 100_002; index++) insert.run(index, '{"type":"text","text":""}') + f.db.exec("COMMIT") + const storage = messageStorage(f.db) + const row = f.db.prepare(`SELECT ${storage.bytes()} AS bytes,data FROM ${storage.table} WHERE id='m'`).get()! + assert(Number(row.bytes) < 16 * 1024 * 1024) + assert.equal(row.data, null) +}) + +test("oversized or malformed split rows cannot become empty successful reads or writes", async t => { + const f = fixture(t) + split(f.db, 3) + const inline = f.db.prepare("SELECT data FROM session_message").get()!.data + f.db.prepare("UPDATE session_part SET data=? WHERE ordinal=0").run("x".repeat(16 * 1024 * 1024 + 1)) + assert.equal((await readNavigationWindow(f.db, f.scope, { kind: "oldest" }, signal())).status, "blocked") + assert.equal(pruneTransaction(f.db, { sessionID: "s", messageID: "m", revision: "a".repeat(64), indexes: [1] }, () => undefined).status, "blocked") + assert.equal(f.db.prepare("SELECT data FROM session_message").get()!.data, inline) + f.db.prepare("UPDATE session_part SET data='{}' WHERE ordinal=0").run() + assert.equal((await readNavigationWindow(f.db, f.scope, { kind: "oldest" }, signal())).status, "blocked") +}) diff --git a/packages/server/src/opencode/session-pruning/message-storage.ts b/packages/server/src/opencode/session-pruning/message-storage.ts new file mode 100644 index 000000000..3dcc7304a --- /dev/null +++ b/packages/server/src/opencode/session-pruning/message-storage.ts @@ -0,0 +1,72 @@ +import type { DatabaseSync } from "node:sqlite" +import { nativeAssistantContentSQL } from "./native-assistant-content" + +const LIMIT = 16 * 1024 * 1024 +const PARTS = 100_001 +const quote = (name: string) => `"${name.replaceAll('"', '""')}"` +const cache = new WeakMap() + +/** A bounded native read relation. Split parts precede the inline tail, as in + * OpenCode's history.withParts. No table, index, trigger or copy is created. */ +export function messageStorage(db: DatabaseSync) { + const version = Number(db.prepare("PRAGMA schema_version").get()!.schema_version) + let known = cache.get(db) + if (!known || known.version !== version) { + const found = db.prepare("SELECT type FROM sqlite_schema WHERE name='session_part'").get() + if (!found) known = { version, table: "session_message", split: false } + else { + const columns = db.prepare("PRAGMA table_info(session_part)").all() + const required = ["message_id", "session_id", "ordinal", "type", "call_id", "time_created", "time_updated", "data"] + if (found.type !== "table" || required.some(name => !columns.some(column => column.name === name)) + || columns.some(column => required.includes(String(column.name)) && (String(column.type).toUpperCase() + !== (["ordinal", "time_created", "time_updated"].includes(String(column.name)) ? "INTEGER" : "TEXT") + || (column.name !== "call_id" && column.notnull !== 1))) + || !columns.some(column => column.name === "message_id" && column.pk === 1) + || !columns.some(column => column.name === "ordinal" && column.pk === 2)) throw new Error("Unsupported native part storage") + const owner = "p.message_id=raw.id" + // The LIMIT bounds metadata examination even for malformed foreign writers. + const parts = (fields: string) => `(SELECT ${fields} FROM session_part p WHERE ${owner} ORDER BY ordinal LIMIT ${PARTS + 1})` + const count = `(SELECT count(*) FROM ${parts("1")})` + // octet_length reads TEXT byte counts from SQLite metadata, before loading + // potentially oversized part payloads for JSON validation/reconstruction. + const bytes = `(octet_length(raw.data) + (SELECT coalesce(sum(bytes),0)+count(*) FROM ${parts("octet_length(data) AS bytes")}))` + const has = `EXISTS(SELECT 1 FROM session_part p WHERE ${owner})` + const invalid = `EXISTS(SELECT 1 FROM ${parts("session_id,ordinal,type,call_id,data")} WHERE session_id IS NOT raw.session_id + OR typeof(ordinal)<>'integer' OR ordinal<0 OR ordinal>9007199254740991 OR typeof(data)<>'text' OR NOT json_valid(data) + OR CASE WHEN json_valid(data) THEN json_type(data)<>'object' OR json_extract(data,'$.type') IS NOT type + OR CASE WHEN type='tool' THEN json_type(data,'$.id') IS NOT 'text' OR call_id IS NOT json_extract(data,'$.id') + ELSE call_id IS NOT NULL END ELSE 1 END)` + const merged = `json_set(raw.data,'$.content',json(${nativeAssistantContentSQL("raw.id", "raw.data")}))` + const data = `CASE WHEN ${has} THEN CASE WHEN raw.type='assistant' AND ${count}<=${PARTS} + AND ${bytes}<=${LIMIT} THEN CASE WHEN json_valid(raw.data) THEN CASE + WHEN json_type(raw.data,'$.content')='array' AND NOT ${invalid} THEN ${merged} END END END ELSE raw.data END` + const names = db.prepare("PRAGMA table_info(session_message)").all().map(column => String(column.name)) + if (names.includes("codenomad_stored_bytes")) throw new Error("Unsupported native message columns") + const fields = names.map(name => name === "data" ? `${data} AS data` + : name === "time_updated" ? `max(raw.time_updated,coalesce((SELECT max(time_updated) FROM ${parts("time_updated")}),raw.time_updated)) AS time_updated` + : `raw.${quote(name)}`) + known = { version, split: true, table: `(SELECT raw.rowid AS rowid,${fields.join(",")}, + CASE WHEN ${has} THEN ${bytes} ELSE octet_length(raw.data) END AS codenomad_stored_bytes + FROM session_message raw)` } + } + cache.set(db, known) + } + return { table: known.table, bytes: (alias = "") => known!.split + ? `${alias ? `${alias}.` : ""}codenomad_stored_bytes` : `length(CAST(${alias ? `${alias}.` : ""}data AS BLOB))` } +} + +/** Retained part ordinals stay unchanged. Native reads use ordinal order, not + * contiguous numbering. Caller owns BEGIN IMMEDIATE and the completed-row fence. */ +export function pruneSplitParts(db: DatabaseSync, sessionID: string, messageID: string, indexes: ReadonlySet): number { + if (!db.isTransaction) throw new Error("Part pruning requires a write transaction") + messageStorage(db) + if (!cache.get(db)!.split) return 0 + const rows = db.prepare("SELECT ordinal FROM session_part WHERE message_id=? AND session_id=? ORDER BY ordinal LIMIT ?") + .all(messageID, sessionID, PARTS + 1) + if (rows.length > PARTS) throw new Error("Native part count exceeds limit") + const remove = db.prepare("DELETE FROM session_part WHERE message_id=? AND session_id=? AND ordinal=?") + rows.forEach((row, index) => { + if (indexes.has(index) && remove.run(messageID, sessionID, row.ordinal).changes !== 1) throw new Error("Native part changed") + }) + return rows.length +} diff --git a/packages/server/src/opencode/session-pruning/native-assistant-content.ts b/packages/server/src/opencode/session-pruning/native-assistant-content.ts new file mode 100644 index 000000000..d98fc1584 --- /dev/null +++ b/packages/server/src/opencode/session-pruning/native-assistant-content.ts @@ -0,0 +1,5 @@ +/** Shared internal SQL expression: native history reads ordinal parts first, then any + * inline tail. Callers own storage validation, message/byte bounds and fences. */ +export const nativeAssistantContentSQL = (messageID: string, inlineData: string) => `(SELECT json_group_array(json(part)) FROM ( + SELECT 0 AS source,ordinal AS position,data AS part FROM session_part WHERE message_id=${messageID} + UNION ALL SELECT 1,CAST(key AS INTEGER),value FROM json_each(${inlineData},'$.content') ORDER BY source,position))` diff --git a/packages/server/src/opencode/session-pruning/navigation-store.ts b/packages/server/src/opencode/session-pruning/navigation-store.ts index bcfe7bd03..83288340c 100644 --- a/packages/server/src/opencode/session-pruning/navigation-store.ts +++ b/packages/server/src/opencode/session-pruning/navigation-store.ts @@ -2,6 +2,7 @@ import type { DatabaseSync, SQLInputValue } from "node:sqlite" import { setImmediate as yieldTurn } from "node:timers/promises" import { ownedSession } from "./navigation-scope" import { projectStoredMedia } from "./media-blob" +import { messageStorage } from "./message-storage" import type { HistoryScope } from "./history-store" import { navigationWindowResultSchema, type NavigationTarget, type NavigationWindowResult } from "./navigation-contract" export { readSessionOutline } from "./outline-index" @@ -24,8 +25,9 @@ export async function readNavigationWindow(db: DatabaseSync, scope: HistoryScope if (!anchor) return { status: "blocked", reason: "anchor_missing" } sequence = Number(anchor.seq) } + const storage = messageStorage(db) const select = (clause: string, args: SQLInputValue[], ascending: boolean, count: number) => db.prepare( - `SELECT id,type,seq,length(CAST(data AS BLOB)) AS bytes FROM session_message WHERE ${where} ${clause} ORDER BY seq ${ascending ? "ASC" : "DESC"} LIMIT ?`, + `SELECT id,type,seq,${storage.bytes()} AS bytes FROM ${storage.table} WHERE ${where} ${clause} ORDER BY seq ${ascending ? "ASC" : "DESC"} LIMIT ?`, ).all(...params, ...args, count) const rows = target.kind === "around" ? [...select("AND seq < ?", [sequence!], false, 80).reverse(), ...select("AND seq >= ?", [sequence!], true, 120)] @@ -38,7 +40,7 @@ export async function readNavigationWindow(db: DatabaseSync, scope: HistoryScope return { status: "blocked", reason: "unsupported_storage" } } const messages = [] - const read = db.prepare("SELECT data FROM session_message WHERE session_id=? AND id=?") + const read = db.prepare(`SELECT data FROM ${storage.table} WHERE session_id=? AND id=?`) let windowBytes = 0 for (const row of rows) { signal.throwIfAborted() diff --git a/packages/server/src/opencode/session-pruning/outline-index.ts b/packages/server/src/opencode/session-pruning/outline-index.ts index 1c503140d..ddeb5dcc9 100644 --- a/packages/server/src/opencode/session-pruning/outline-index.ts +++ b/packages/server/src/opencode/session-pruning/outline-index.ts @@ -5,6 +5,7 @@ import type { HistoryScope } from "./history-store" import type { OutlineCheckpoint, OutlineEntry, OutlineResult } from "./navigation-contract" import { displayLayout } from "./outline-layout" import { ownedSession } from "./navigation-scope" +import { messageStorage } from "./message-storage" function boundedToolName(value: unknown): string { if (typeof value !== "string") return "" @@ -37,9 +38,10 @@ export async function readSessionOutline(db: DatabaseSync, scope: HistoryScope, const through = cursor?.through ?? Math.max(maximum, known.at(-1)?.through ?? 0) const total = Number(db.prepare(`SELECT count(*) AS value FROM session_message WHERE ${where} AND seq<=?`).get(...params, through)?.value) let after = cursor?.after ?? startAfter - const headers = db.prepare(`SELECT id,type,seq,time_updated,length(CAST(data AS BLOB)) AS bytes - FROM session_message WHERE ${where} AND seq>? AND seq<=? ORDER BY seq LIMIT 512`) - const project = db.prepare(`SELECT id,type,seq, + const storage = messageStorage(db) + const headers = db.prepare(`SELECT id,type,seq,time_updated,${storage.bytes()} AS bytes + FROM ${storage.table} WHERE ${where} AND seq>? AND seq<=? ORDER BY seq LIMIT 512`) + const project = db.prepare(`SELECT id,type,seq,data IS NULL AS unsupported, CASE WHEN type='assistant' THEN (SELECT count(*) FROM json_each(data,'$.content') WHERE json_extract(value,'$.type')='tool') ELSE 0 END AS tools, CASE WHEN type='assistant' THEN coalesce((SELECT substr(CASE WHEN json_type(value,'$.name')='text' THEN json_extract(value,'$.name') @@ -59,7 +61,7 @@ export async function readSessionOutline(db: DatabaseSync, scope: HistoryScope, CASE WHEN type='assistant' THEN coalesce(json_array_length(data,'$.content'),0) + (coalesce(json_type(data,'$.error'),'null')<>'null' AND NOT EXISTS (SELECT 1 FROM json_each(data,'$.content') WHERE json_extract(value,'$.type')='text')) END AS display_parts` : ""} - FROM session_message WHERE ${where} AND seq>? AND seq<=? ORDER BY seq LIMIT 512`) + FROM ${storage.table} WHERE ${where} AND seq>? AND seq<=? ORDER BY seq LIMIT 512`) const entries: OutlineEntry[] = [], checkpoints: Array = [] let examined = 0, layoutCharacters = 0 // Row counts do not bound time: one assistant can contain megabytes of tool @@ -75,6 +77,7 @@ export async function readSessionOutline(db: DatabaseSync, scope: HistoryScope, const rows = [] for (const row of headers.iterate(...params, after, end)) { signal.throwIfAborted() + if (row.bytes === null || Number(row.bytes) > 16 * 1024 * 1024) return { status: "blocked", reason: "unsupported_storage" } rows.push(row) if (rows.length % 128 === 0 || performance.now() - turnStarted >= 8) { await yieldTurn(undefined, { signal }) @@ -92,6 +95,7 @@ export async function readSessionOutline(db: DatabaseSync, scope: HistoryScope, if (changed) { for (const row of project.iterate(...params, after, chunkEnd)) { signal.throwIfAborted() + if (row.unsupported) return { status: "blocked", reason: "unsupported_storage" } const tools = Number(row.tools) const layout = layouts && row.type === "assistant" ? displayLayout(row.layout, Number(row.display_parts)) : {} layoutCharacters += layout.layout?.length ?? 0 diff --git a/packages/server/src/opencode/session-pruning/outline-preview.ts b/packages/server/src/opencode/session-pruning/outline-preview.ts index c16490f07..b7d2a1ed6 100644 --- a/packages/server/src/opencode/session-pruning/outline-preview.ts +++ b/packages/server/src/opencode/session-pruning/outline-preview.ts @@ -3,20 +3,23 @@ import { setImmediate as yieldTurn } from "node:timers/promises" import type { HistoryScope } from "./history-store" import type { OutlinePreviewResult } from "./navigation-contract" import { ownedSession } from "./navigation-scope" +import { messageStorage } from "./message-storage" /** Demand-only excerpts, independent of index geometry and transcript windows. */ export async function readOutlinePreviews(db: DatabaseSync, scope: HistoryScope, ids: string[], signal: AbortSignal): Promise { signal.throwIfAborted() const boundary = ownedSession(db, scope) const params: SQLInputValue[] = [scope.sessionID!, ...(boundary ? [boundary] : [])] - const read = db.prepare(`SELECT id, CASE WHEN length(CAST(data AS BLOB))<=16777216 THEN data END AS data - FROM session_message WHERE session_id=? ${boundary ? "AND id < ?" : ""} AND id=?`) + const storage = messageStorage(db) + const read = db.prepare(`SELECT id, CASE WHEN ${storage.bytes()}<=16777216 THEN data END AS data + FROM ${storage.table} WHERE session_id=? ${boundary ? "AND id < ?" : ""} AND id=?`) const entries: Array<{ id: string; text: string; tools: string }> = [] for (const id of [...new Set(ids)].slice(0, 12)) { signal.throwIfAborted() const row = read.get(...params, id) if (!row) continue - const data = typeof row.data === "string" ? JSON.parse(row.data) : {} + if (typeof row.data !== "string") return { status: "blocked", reason: "unsupported_storage" } + const data = JSON.parse(row.data) let text = "", tools = "" const append = (current: string, value: unknown) => typeof value === "string" && current.length < 4096 ? (current + (current ? "\n\n" : "") + value.slice(0, 4096 - current.length)).slice(0, 4096) : current diff --git a/packages/server/src/opencode/session-pruning/package.json b/packages/server/src/opencode/session-pruning/package.json index 7cdf7afd7..b72d85c16 100644 --- a/packages/server/src/opencode/session-pruning/package.json +++ b/packages/server/src/opencode/session-pruning/package.json @@ -18,7 +18,7 @@ "assets-gallery-contract.ts", "assets-gallery-proof.ts", "assets-gallery.ts", "assets-conversation.ts", "history-database.ts", "navigation-contract.ts", "navigation-store.ts", "navigation-scope.ts", "outline-index.ts", "outline-layout.ts", "outline-preview.ts", - "transaction.ts" + "transaction.ts", "message-storage.ts", "native-assistant-content.ts" ], "dependencies": { "@opencode/plugin": "2.2.3", diff --git a/packages/server/src/opencode/session-pruning/preview-store.ts b/packages/server/src/opencode/session-pruning/preview-store.ts index 3f40f3c2a..15f93632e 100644 --- a/packages/server/src/opencode/session-pruning/preview-store.ts +++ b/packages/server/src/opencode/session-pruning/preview-store.ts @@ -4,6 +4,7 @@ import type { messageTargetSchema } from "./contract" import { storageDirectory } from "./storage-path" import type { LocationRef } from "@opencode/client" import { projectStoredMedia, type MediaProjection } from "./media-blob" +import { messageStorage } from "./message-storage" export async function readPruningPreview( filename: unknown, target: z.infer, location: LocationRef, projectID: string, @@ -14,11 +15,12 @@ export async function readPruningPreview( const db = new DatabaseSync(filename, { readOnly: true, timeout: 1000 }) try { db.exec("PRAGMA query_only=ON") - const row = db.prepare(`SELECT m.data, length(CAST(m.data AS BLOB)) AS bytes FROM session_message m + const storage = messageStorage(db) + const row = db.prepare(`SELECT m.data, ${storage.bytes("m")} AS bytes FROM ${storage.table} m JOIN session_v2 s ON s.id = m.session_id WHERE m.id = ? AND m.session_id = ? AND m.type = 'assistant' AND s.directory = ? AND s.project_id = ? AND s.workspace_id IS ? - AND length(CAST(m.data AS BLOB)) <= 16777216`).get(target.messageID, target.sessionID, + AND ${storage.bytes("m")} <= 16777216`).get(target.messageID, target.sessionID, storageDirectory(location.directory), projectID, location.workspaceID ?? null) // The native form, as the pruning transaction compares it. return typeof row?.data === "string" ? projectStoredMedia(db, JSON.parse(row.data), Number(row.bytes), 16777216) : undefined diff --git a/packages/server/src/opencode/session-pruning/transaction.ts b/packages/server/src/opencode/session-pruning/transaction.ts index 10cad84ed..23d207085 100644 --- a/packages/server/src/opencode/session-pruning/transaction.ts +++ b/packages/server/src/opencode/session-pruning/transaction.ts @@ -4,6 +4,7 @@ import { pruneAllRequestSchema, pruneRequestSchema, pruneResultSchema, type Prun import { planPrune, previewContent, revision } from "./planner" import { canonicalContent } from "./revision" import { canReleaseMedia, projectStoredMedia, releaseDroppedMedia } from "./media-blob" +import { messageStorage, pruneSplitParts } from "./message-storage" const MESSAGE_LIMIT = 16 * 1024 * 1024 type BlockReason = Extract["reason"] @@ -33,8 +34,9 @@ export function pruneTransaction( return result } } - const row = db.prepare(`SELECT type, data, length(CAST(data AS BLOB)) AS bytes FROM session_message WHERE session_id=? AND id=? - AND length(CAST(data AS BLOB)) <= ?`).get(request.sessionID, request.messageID, MESSAGE_LIMIT) + const storage = messageStorage(db) + const row = db.prepare(`SELECT type, data, ${storage.bytes()} AS bytes FROM ${storage.table} WHERE session_id=? AND id=? + AND ${storage.bytes()} <= ?`).get(request.sessionID, request.messageID, MESSAGE_LIMIT) if (!row || row.type !== "assistant" || typeof row.data !== "string") return { status: "blocked", reason: "not_deletable" } const data = JSON.parse(row.data) // Compare what the caller hashed from native reads; the same budget bounds the native form. @@ -47,8 +49,13 @@ export function pruneTransaction( if (!indexes.length) return { status: "blocked", reason: "not_deletable" } const plan = planPrune(data, { ...request, indexes }, projection.data) if (plan.status !== "planned") return plan + const inline = db.prepare("SELECT data FROM session_message WHERE session_id=? AND id=?").get(request.sessionID, request.messageID)?.data + if (typeof inline !== "string") return { status: "blocked", reason: "conflict" } + const removed = new Set(indexes) + const splitCount = pruneSplitParts(db, request.sessionID, request.messageID, removed) + const inlineContent = splitCount ? JSON.parse(inline).content.filter((_: unknown, index: number) => !removed.has(splitCount + index)) : plan.content const update = db.prepare("UPDATE session_message SET data=json_set(data,'$.content',json(?)) WHERE session_id=? AND id=? AND data=?") - .run(JSON.stringify(plan.content), request.sessionID, request.messageID, row.data) + .run(JSON.stringify(inlineContent), request.sessionID, request.messageID, inline) if (update.changes !== 1) return { status: "blocked", reason: "conflict" } releaseDroppedMedia(db, request.sessionID, request.messageID, data, { ...data, content: plan.content }) const result: PruneResult = { status: "pruned", messageID: request.messageID, revision: revision(plan.projected), removedCount: indexes.length } diff --git a/scripts/test-design-companion-native.mjs b/scripts/test-design-companion-native.mjs index 52fdb7aa9..b329b3369 100644 --- a/scripts/test-design-companion-native.mjs +++ b/scripts/test-design-companion-native.mjs @@ -150,7 +150,7 @@ const password = randomUUID() const env = await fixtureEnvironment(root, { OPENCODE_CONFIG_DIR: configDirectory, OPENCODE_DB: path.join(root, "fixture.db"), OPENCODE_SERVER_PASSWORD: password, OPENCODE_DISABLE_MODELS_FETCH: "1", OPENCODE_DISABLE_FFF: "1", - OPENCODE_CONFIG_CONTENT: JSON.stringify({ model: "fixture/vision", small_model: "fixture/vision", enabled_providers: ["fixture", "fiximg"], experimental: { image_generate: { model: "fiximg/image" } }, providers: { + OPENCODE_CONFIG_CONTENT: JSON.stringify({ model: "fixture/vision", small_model: "fixture/vision", enabled_providers: ["fixture", "fiximg"], experimental: { image_generate: { model: "fiximg/image" }, ...(process.argv.includes("--part-rows") ? { part_rows: true } : {}) }, providers: { fixture: { package: "@opencode/ai/providers/openai-compatible", settings, models: { vision: { name: "Fixture vision", capabilities: { tools: true, input: ["text", "image/png"], output: ["text"] } } } }, fiximg: { package: "@opencode/ai/providers/openai-compatible/images", settings, models: { image: { name: "Fixture image", type: "image", capabilities: { tools: false, input: ["text", "image/png"], output: ["image"] } } } }, } }), diff --git a/scripts/test-session-media-blob-native.mjs b/scripts/test-session-media-blob-native.mjs index d900c6bee..4fd3579fd 100644 --- a/scripts/test-session-media-blob-native.mjs +++ b/scripts/test-session-media-blob-native.mjs @@ -21,7 +21,10 @@ export async function testSessionMediaBlobNative({ client, location, locationOpt const bytes = Buffer.from(MEDIA_PROBE_HTML), hash = createHash("sha256").update(bytes).digest("hex") const uri = `data:text/html;base64,${bytes.toString("base64")}` assert.equal(target.content[index].state.content[0].uri, uri, "native reads return the inline file") - const stored = JSON.parse(db.prepare("SELECT data FROM session_message WHERE id=?").get(target.id).data).content[index].state.content[0].uri + const inline = JSON.parse(db.prepare("SELECT data FROM session_message WHERE id=?").get(target.id).data).content + const split = db.prepare("SELECT 1 FROM sqlite_schema WHERE type='table' AND name='session_part'").get() + ? db.prepare("SELECT data FROM session_part WHERE message_id=? ORDER BY ordinal").all(target.id).map(row => JSON.parse(row.data)) : [] + const stored = [...split, ...inline][index].state.content[0].uri const blobs = Boolean(db.prepare("SELECT 1 FROM sqlite_schema WHERE type='table' AND name='blob'").get()) const refs = () => db.prepare("SELECT count(*) AS n FROM blob_ref WHERE session_id=? AND hash=?").get(session.id, hash).n const blob = () => db.prepare("SELECT count(*) AS n FROM blob WHERE hash=?").get(hash).n diff --git a/scripts/test-session-pruning-native.mjs b/scripts/test-session-pruning-native.mjs index 25dda59a0..c8e47bea4 100644 --- a/scripts/test-session-pruning-native.mjs +++ b/scripts/test-session-pruning-native.mjs @@ -12,11 +12,13 @@ import { fileURLToPath, pathToFileURL } from "node:url" import { OpenCode } from "@opencode/client" import { tsImport } from "tsx/esm/api" import { createNativeFixtureReporter } from "./native-fixture-diagnostics.mjs" +import { fixtureEnvironment } from "./native-product-fixture.mjs" const cli = process.argv[2] if (!cli || !path.isAbsolute(cli)) throw new Error("Pass an absolute path to the CLI executable to test in isolation") const runtimeVersion = execFileSync(cli, ["--version"], { encoding: "utf8" }).trim().replace(/^opencode2? v/, "") const ui = process.argv.includes("--ui") +const splitParts = process.argv.includes("--part-rows") const pluginArgument = process.argv[3]?.startsWith("--") ? undefined : process.argv[3] const pluginDirectory = pluginArgument ?? fileURLToPath(new URL("../packages/server/src/opencode/session-pruning/", import.meta.url)) if (!path.isAbsolute(pluginDirectory)) throw new Error("Plugin directory must be absolute") @@ -33,9 +35,7 @@ if (pruningUI) { await pruningUI.loadPruningUIDependencies(diagnostics) diagnostics.stage("ui-dependencies-ready-before-native-preamble") } -const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("OPENCODE_") && !key.startsWith("XDG_"))) -for (const key of ["XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME"]) env[key] = path.join(root, key) -Object.assign(env, { +const env = await fixtureEnvironment(root, { USERPROFILE: root, HOME: root, OPENCODE_TEST_HOME: root, OPENCODE_CONFIG_DIR: path.join(root, "config"), OPENCODE_DB: path.join(root, "test.db"), OPENCODE_SERVER_PASSWORD: "isolated-pruning-fixture", OPENCODE_CONFIG_PROJECT_DISABLE: "1", @@ -126,7 +126,7 @@ env.OPENCODE_CONFIG_CONTENT = JSON.stringify({ }, models: { fixture: {} } } }, plugins: [path.join(root, "plugin")], // Runtimes with optional blob storage externalize tool files; others ignore the key. - experimental: { media_blob: true }, + experimental: { media_blob: true, ...(splitParts ? { part_rows: true } : {}) }, }) function start() { const child = spawn(cli, ["serve", "--hostname", "127.0.0.1", "--port", "0", "--log-level", "debug", "--print-logs"], { cwd: root, env, windowsHide: true }) @@ -353,6 +353,7 @@ try { await delay(150) assert.equal(requests.length, before) db.prepare("UPDATE session_message SET data=json_set(data,'$.content',json('[]')) WHERE session_id=? AND id=?").run(session.id, target.id) + if (splitParts) db.prepare("DELETE FROM session_part WHERE session_id=? AND message_id=?").run(session.id, target.id) db.exec("COMMIT") await competing; await wait() const next = requests.slice(before).filter(item => item.kind === "primary") @@ -386,6 +387,19 @@ try { const { testSessionImagesNative } = await import("./test-session-images-native.mjs") const imagesConversation = await testSessionImagesNative({ client, second: makeClient(), location, locationOptions, generate: generateMedia, other: { location: historyLocation, locationOptions: locationRequestOptions(historyLocation) } }) + let mixed + if (splitParts) { + const carry = await client.session.create({ location }, locationOptions) + primaryCount = toolSteps + 1 + await client.session.prompt({ sessionID: carry.id, text: "Retain an existing split answer across disabling part_rows." }) + await client.session.wait({ sessionID: carry.id }, { signal: AbortSignal.timeout(20_000) }) + const answer = (await client.message.list({ sessionID: carry.id, order: "asc", limit: 100 })).data + .find(message => message.type === "assistant" && message.content.some(part => part.type === "text")) + assert(answer) + assert(db.prepare("SELECT count(*) AS n FROM session_part WHERE message_id=?").get(answer.id).n > 0) + const text = answer.content.find(part => part.type === "text") + mixed = { sessionID: carry.id, messageID: answer.id, content: [...answer.content, { ...text, text: "MIXED_INLINE_TAIL_CANARY" }] } + } assert.equal(db.prepare("PRAGMA integrity_check").get().integrity_check, "ok") const enqueued = first.filter(event => event.type === "session.inbox.enqueued") assert(enqueued.length > 0) @@ -394,12 +408,30 @@ try { db.close(); db = undefined child.kill(); await stopped await writeFile(path.join(root, "before-restart.log"), output) + if (mixed) { + // An older writer can append an inline tail beside split parts. Only this + // stopped, fixture-owned DB is edited to reproduce that native storage shape. + const stoppedDB = new DatabaseSync(env.OPENCODE_DB) + try { stoppedDB.prepare("UPDATE session_message SET data=json_set(data,'$.content',json(?)) WHERE session_id=? AND id=?") + .run(JSON.stringify([mixed.content.at(-1)]), mixed.sessionID, mixed.messageID) } finally { stoppedDB.close() } + const config = JSON.parse(env.OPENCODE_CONFIG_CONTENT) + config.experimental.part_rows = false + env.OPENCODE_CONFIG_CONTENT = JSON.stringify(config) + } output = "" ;({ child, stopped } = start()) await until(() => /http:\/\/127\.0\.0\.1:\d+/.test(output)) const restartedEndpoint = { ...connection.endpoint, url: output.match(/http:\/\/127\.0\.0\.1:\d+/)[0] } rememberRuntime(restartedEndpoint, { version: runtimeVersion, pid: child.pid, discovery: "info" }) const restarted = OpenCode.make({ baseUrl: restartedEndpoint.url, fetch: createRuntimeFetch(restartedEndpoint) }) + if (mixed) { + assert.deepEqual((await restarted.session.message.get({ sessionID: mixed.sessionID, messageID: mixed.messageID })).content, mixed.content) + const found = (await restarted.rpc.call({ rpcID: "codenomad.session-pruning", method: "history", location, + input: { sessionID: mixed.sessionID, purpose: "search", query: "MIXED_INLINE_TAIL_CANARY", includeTechnical: false } }, locationOptions)).output + assert.equal(found.status, "page") + assert(found.hits.some(hit => hit.messageID === mixed.messageID && hit.partIndex === mixed.content.length - 1)) + console.log("PASS: split parts plus inline tail stay readable and searchable after part_rows is disabled and the daemon restarts") + } assert.deepEqual((await restarted.session.message.get({ sessionID: session.id, messageID: target.id })).content, []) assert.deepEqual((await restarted.session.message.get({ sessionID: fork.id, messageID: forkTarget.id })).content, []) assert.deepEqual(await restarted.session.context({ sessionID: fork.id }), contextBefore)