diff --git a/.changeset/fix-tui-startup-freeze.md b/.changeset/fix-tui-startup-freeze.md new file mode 100644 index 000000000..71991eb22 --- /dev/null +++ b/.changeset/fix-tui-startup-freeze.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Fix multi-second typing and rendering freezes at startup or while idle when a large search index loads, replays, or rebuilds. diff --git a/.oxlintrc.json b/.oxlintrc.json index 786b0f5cf..6eeb43089 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -91,12 +91,13 @@ } }, { - // The stage-6 worker closure: these modules (and everything - // packages/minidb/src/worker/ pulls in) are loaded by a bare + // The worker closures: these modules (and everything + // packages/minidb/src/worker/ and + // packages/kap-server/src/search/worker/ pull in) are loaded by a bare // node:worker_threads Worker under Node's native type stripping with // `execArgv: ['--experimental-transform-types']`, which requires // explicit `.ts` import specifiers (the strip loader does not remap - // `.js` -> `.ts`). Keep the exception scoped to exactly that closure. + // `.js` -> `.ts`). Keep the exception scoped to exactly those closures. "files": [ "packages/minidb/src/worker/**/*.ts", "packages/minidb/src/codec.ts", @@ -104,7 +105,10 @@ "packages/minidb/src/trigram.ts", "packages/minidb/src/text-postings.ts", "packages/minidb/src/text-index/tokenize.ts", - "packages/minidb/src/gen-codec.ts" + "packages/minidb/src/gen-codec.ts", + "packages/kap-server/src/search/worker/**/*.ts", + "packages/kap-server/src/search/indexCore.ts", + "packages/kap-server/src/search/match.ts" ], "rules": { "import/extensions": "off" diff --git a/AGENTS.md b/AGENTS.md index 1ba77bf76..b18f4db8b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,9 @@ This is a TypeScript monorepo built for agent-assisted development. Keep the roo ## Experimental Features -- Gate a not-yet-public feature behind an experimental flag. Add the flag to the registry at `packages/agent-core/src/flags/registry.ts`, then check it with `flags.enabled('my-feature')`. Flags are env-driven and default off: `KIMI_CODE_EXPERIMENTAL_` toggles one, `KIMI_CODE_EXPERIMENTAL_FLAG` enables all. Release by flipping the entry's `default` to `true`. +- Gate a not-yet-public feature behind an experimental flag. Flags are env-driven and default off: `KIMI_CODE_EXPERIMENTAL_` toggles one, `KIMI_CODE_EXPERIMENTAL_FLAG` enables all. Release by flipping the entry's `default` to `true`. + - `packages/agent-core` (v1): add the flag to the central registry at `packages/agent-core/src/flags/registry.ts`, then check it with `flags.enabled('my-feature')`. + - `packages/agent-core-v2` and kap-server modules: there is no central catalog — declare the flag in the owning domain via `registerFlagDefinition` at import time (see `packages/agent-core-v2/docs/flag.md`), then check it with `IFlagService.enabled(id)`. Current search-index-separation flags: `persistence_minidb_readmodel` (session read model, default on) and `search_worker` (global search worker host, default on). ## Where to Update Instructions diff --git a/apps/kimi-code/package.json b/apps/kimi-code/package.json index 7aaa004a9..2329f4409 100644 --- a/apps/kimi-code/package.json +++ b/apps/kimi-code/package.json @@ -50,7 +50,7 @@ "provenance": true }, "scripts": { - "build": "tsdown && node scripts/copy-native-assets.mjs && node scripts/check-web-assets.mjs", + "build": "tsdown && tsdown --config tsdown.dist-worker.config.ts && node scripts/copy-native-assets.mjs && node scripts/check-web-assets.mjs", "prebuild": "node scripts/build-vis-asset.mjs", "catalog:update": "node scripts/update-catalog.mjs --out dist/built-in-catalog.json", "smoke": "node scripts/smoke.mjs", diff --git a/apps/kimi-code/scripts/native/01-bundle.mjs b/apps/kimi-code/scripts/native/01-bundle.mjs index 9f917e019..7b19157f3 100644 --- a/apps/kimi-code/scripts/native/01-bundle.mjs +++ b/apps/kimi-code/scripts/native/01-bundle.mjs @@ -15,11 +15,12 @@ export async function runBundleStep() { // miss it (npm builds get it via the `prebuild` script). await run(process.execPath, [buildVisAssetPath]); await run(process.execPath, [tsdownCliPath, '--config', 'tsdown.native.config.ts']); - // Bundle the minidb text-build worker into one self-contained ESM file so - // it can ride the SEA blob as an asset (02-sea-blob.mjs) and be spawned - // from disk at runtime — bundled binaries otherwise lack the worker entry - // and heavy text-index builds degrade to the inline main-thread core. - // Runs after the main bundle with clean:false so both verified files remain. + // Bundle the off-main-thread workers (the minidb text-build worker and + // the kap-server global-search worker) into self-contained ESM files so + // they can ride the SEA blob as assets (02-sea-blob.mjs) and be spawned + // from disk at runtime — bundled binaries otherwise lack the worker + // entries and heavy index work degrades to inline main-thread cores. + // Runs after the main bundle with clean:false so all verified files remain. await run(process.execPath, [tsdownCliPath, '--config', 'tsdown.worker.config.ts']); await run(process.execPath, [checkBundlePath]); } diff --git a/apps/kimi-code/scripts/native/assets.mjs b/apps/kimi-code/scripts/native/assets.mjs index 3c9f3b204..41fb600e3 100644 --- a/apps/kimi-code/scripts/native/assets.mjs +++ b/apps/kimi-code/scripts/native/assets.mjs @@ -6,6 +6,7 @@ import { dirname, extname, isAbsolute, join, relative, resolve } from 'node:path import { pathToFileURL } from 'node:url'; import { + KAP_SEARCH_WORKER_ASSET, MINIDB_TEXT_BUILD_WORKER_ASSET, NATIVE_ASSET_MANIFEST_VERSION, buildManifestKey, @@ -272,19 +273,23 @@ export async function collectNativeAssets({ appRoot, target }) { Object.assign(assets, result.assets); } - const workerSource = resolve(appRoot, 'dist-native', 'intermediates', 'text-build-worker.mjs'); - const workerBytes = await readFile(workerSource); - const workerAssetKey = buildRuntimeAssetKey(target, MINIDB_TEXT_BUILD_WORKER_ASSET.key); - const runtimeFiles = [ - { - key: MINIDB_TEXT_BUILD_WORKER_ASSET.key, + const runtimeFiles = []; + for (const [fileName, asset] of [ + ['text-build-worker.mjs', MINIDB_TEXT_BUILD_WORKER_ASSET], + ['search-worker.mjs', KAP_SEARCH_WORKER_ASSET], + ]) { + const workerSource = resolve(appRoot, 'dist-native', 'intermediates', fileName); + const workerBytes = await readFile(workerSource); + const workerAssetKey = buildRuntimeAssetKey(target, asset.key); + runtimeFiles.push({ + key: asset.key, assetKey: workerAssetKey, - relativePath: MINIDB_TEXT_BUILD_WORKER_ASSET.relativePath, + relativePath: asset.relativePath, sha256: sha256(workerBytes), - mode: MINIDB_TEXT_BUILD_WORKER_ASSET.mode, - }, - ]; - assets[workerAssetKey] = workerSource; + mode: asset.mode, + }); + assets[workerAssetKey] = workerSource; + } const manifest = { version: NATIVE_ASSET_MANIFEST_VERSION, diff --git a/apps/kimi-code/scripts/native/check-bundle.mjs b/apps/kimi-code/scripts/native/check-bundle.mjs index bf6306406..8b3519db7 100644 --- a/apps/kimi-code/scripts/native/check-bundle.mjs +++ b/apps/kimi-code/scripts/native/check-bundle.mjs @@ -71,6 +71,7 @@ function checkBundle(bundlePath, { worker = false } = {}) { const bundles = [ { path: nativeJsBundlePath(), worker: false }, { path: resolve(nativeIntermediatesDir(), 'text-build-worker.mjs'), worker: true }, + { path: resolve(nativeIntermediatesDir(), 'search-worker.mjs'), worker: true }, ]; let failed = false; for (const bundle of bundles) { diff --git a/apps/kimi-code/scripts/native/manifest.mjs b/apps/kimi-code/scripts/native/manifest.mjs index 1344a24f4..99f3b3a3d 100644 --- a/apps/kimi-code/scripts/native/manifest.mjs +++ b/apps/kimi-code/scripts/native/manifest.mjs @@ -7,6 +7,12 @@ export const MINIDB_TEXT_BUILD_WORKER_ASSET = Object.freeze({ mode: 0o644, }); +export const KAP_SEARCH_WORKER_ASSET = Object.freeze({ + key: 'kap-search-worker', + relativePath: 'runtime/kap-server/search-worker.mjs', + mode: 0o644, +}); + export function buildManifestKey(target) { return `native/${target}/manifest.json`; } diff --git a/apps/kimi-code/scripts/native/smoke.mjs b/apps/kimi-code/scripts/native/smoke.mjs index 0d0f2604b..dbff8c7e9 100644 --- a/apps/kimi-code/scripts/native/smoke.mjs +++ b/apps/kimi-code/scripts/native/smoke.mjs @@ -84,6 +84,7 @@ try { }); assertIncludes(nativeAssetOutput, `Native asset smoke passed: ${target}`, 'native asset smoke'); assertIncludes(nativeAssetOutput, 'MiniDb worker build passed', 'MiniDb worker smoke'); + assertIncludes(nativeAssetOutput, 'search worker ready', 'search worker smoke'); } finally { await rm(smokeHome, { recursive: true, force: true }); } diff --git a/apps/kimi-code/src/main.ts b/apps/kimi-code/src/main.ts index 7c4e1040b..cfcfb0928 100644 --- a/apps/kimi-code/src/main.ts +++ b/apps/kimi-code/src/main.ts @@ -38,6 +38,7 @@ import { createKimiCodeHostIdentity, getVersion } from './cli/version'; import { CLI_SHUTDOWN_TIMEOUT_MS, CLI_UI_MODE, PROCESS_NAME } from './constant/app'; import { cleanupStaleNativeCacheForCurrent } from './native/native-assets'; import { installMinidbTextBuildWorker } from './native/minidb-worker'; +import { installKapSearchWorker } from './native/search-worker'; import { installNativeModuleHook } from './native/module-hook'; import { runNativeAssetSmokeIfRequested } from './native/smoke'; @@ -158,6 +159,17 @@ export function main(): void { ? `minidb-worker:failed code=${workerInstall.errorCode} sha256=${workerInstall.assetSha256 ?? 'unknown'}` : `minidb-worker:${workerInstall.status}`, ); + // Same pattern for the global-search worker: extracted from the SEA blob so + // the search index runs off the main thread; a failure leaves the search + // surface degraded (the `search_worker` flag restores the inline host). + const searchWorkerInstall = installKapSearchWorker(); + startupTrace( + searchWorkerInstall.status === 'installed' + ? `search-worker:installed basename=${searchWorkerInstall.basename} sha256=${searchWorkerInstall.assetSha256}` + : searchWorkerInstall.status === 'failed' + ? `search-worker:failed code=${searchWorkerInstall.errorCode} sha256=${searchWorkerInstall.assetSha256 ?? 'unknown'}` + : `search-worker:${searchWorkerInstall.status}`, + ); if (runNativeAssetSmokeIfRequested()) return; // Start the background cleanup of stale native cache. Fire-and-forget; must not block startup or throw. diff --git a/apps/kimi-code/src/native/native-assets.ts b/apps/kimi-code/src/native/native-assets.ts index a69224661..51db1af9f 100644 --- a/apps/kimi-code/src/native/native-assets.ts +++ b/apps/kimi-code/src/native/native-assets.ts @@ -16,6 +16,7 @@ import { join as joinPosix } from 'pathe'; import { KIMI_BUILD_INFO } from '#/cli/build-info'; import { + KAP_SEARCH_WORKER_ASSET, MINIDB_TEXT_BUILD_WORKER_ASSET, NATIVE_ASSET_MANIFEST_VERSION as MANIFEST_VERSION, buildManifestKey, @@ -416,6 +417,10 @@ export function getMinidbTextBuildWorkerFile( return getNativeRuntimeFile(MINIDB_TEXT_BUILD_WORKER_ASSET.key, options); } +export function getKapSearchWorkerFile(options: NativeAssetOptions = {}): string | null { + return getNativeRuntimeFile(KAP_SEARCH_WORKER_ASSET.key, options); +} + export function getNativePackageRoot( packageName: string, options: NativeAssetOptions = {}, diff --git a/apps/kimi-code/src/native/search-worker.ts b/apps/kimi-code/src/native/search-worker.ts new file mode 100644 index 000000000..158825ce9 --- /dev/null +++ b/apps/kimi-code/src/native/search-worker.ts @@ -0,0 +1,72 @@ +import { basename } from 'node:path'; + +import { + configureSearchWorkerRuntime, + getSearchWorkerRuntimeState, +} from '@moonshot-ai/kap-server/search-worker-runtime'; + +import { KAP_SEARCH_WORKER_ASSET } from '../../scripts/native/manifest.mjs'; +import { + getEmbeddedNativeAssetManifest, + getKapSearchWorkerFile, + getSeaAssetSource, + type NativeAssetOptions, +} from './native-assets'; + +export type KapSearchWorkerInstallStatus = + | { readonly status: 'not-sea' } + | { readonly status: 'asset-missing' } + | { + readonly status: 'installed'; + readonly assetSha256: string; + readonly basename: string; + } + | { + readonly status: 'failed'; + readonly errorCode: string; + readonly assetSha256?: string; + }; + +function errorCode(error: unknown): string { + const code = (error as NodeJS.ErrnoException | undefined)?.code; + if (typeof code === 'string' && code.length > 0) return code; + return error instanceof Error ? error.name : 'UNKNOWN'; +} + +/** + * Install the SEA-bundled global-search worker without making optional + * extraction fatal. Without it the search service resolves no worker entry + * inside the single-file binary and reports the index as degraded; the + * `search_worker` experimental flag restores the in-process host. + */ +export function installKapSearchWorker( + options: NativeAssetOptions = {}, +): KapSearchWorkerInstallStatus { + const source = options.source ?? getSeaAssetSource(); + if (source === null) return { status: 'not-sea' }; + + let assetSha256: string | undefined; + try { + const manifest = options.manifest ?? getEmbeddedNativeAssetManifest(source); + const file = manifest?.runtimeFiles.find((entry) => entry.key === KAP_SEARCH_WORKER_ASSET.key); + if (manifest === null || file === undefined) return { status: 'asset-missing' }; + assetSha256 = file.sha256; + + const workerPath = getKapSearchWorkerFile({ ...options, source, manifest }); + if (workerPath === null) return { status: 'asset-missing' }; + configureSearchWorkerRuntime(workerPath); + const runtime = getSearchWorkerRuntimeState(); + if (!runtime.configured) throw new Error('search worker runtime was not configured'); + return { + status: 'installed', + assetSha256, + basename: basename(workerPath), + }; + } catch (error) { + return { + status: 'failed', + errorCode: errorCode(error), + assetSha256, + }; + } +} diff --git a/apps/kimi-code/src/native/smoke.ts b/apps/kimi-code/src/native/smoke.ts index 1d330bc87..a7f80957c 100644 --- a/apps/kimi-code/src/native/smoke.ts +++ b/apps/kimi-code/src/native/smoke.ts @@ -1,8 +1,11 @@ import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'; import { createRequire } from 'node:module'; +import { once } from 'node:events'; import { dirname, join } from 'node:path'; +import { Worker } from 'node:worker_threads'; import { MiniDb } from '@moonshot-ai/minidb'; +import { getSearchWorkerRuntimeState } from '@moonshot-ai/kap-server/search-worker-runtime'; import { getEmbeddedNativeAssetManifest, @@ -74,6 +77,34 @@ async function smokeMinidbWorker(): Promise { } } +async function smokeSearchWorker(): Promise { + // The SEA-extracted global-search worker entry must boot from disk and + // complete the versioned ready handshake. + const runtime = getSearchWorkerRuntimeState(); + if (!runtime.configured) { + throw new Error('search worker runtime was not configured'); + } + const cacheBase = getNativeCacheBase(); + mkdirSync(cacheBase, { recursive: true }); + const dir = mkdtempSync(join(cacheBase, 'sea-search-worker-')); + const worker = new Worker(runtime.path, { + workerData: { dir, bootSalt: 'sea-smoke' }, + }); + try { + const ready = once(worker, 'message', { + signal: AbortSignal.timeout(15_000), + }) as Promise; + const [event] = await ready; + const v = (event as { type?: string; v?: number }).v; + if ((event as { type?: string }).type !== 'ready' || typeof v !== 'number') { + throw new Error(`search worker handshake is unexpected: ${JSON.stringify(event)}`); + } + } finally { + await worker.terminate().catch(() => {}); + rmSync(dir, { recursive: true, force: true }); + } +} + async function runSmoke(): Promise { const manifest = getEmbeddedNativeAssetManifest(); if (manifest === null) throw new Error('Native asset manifest is not available.'); @@ -84,7 +115,10 @@ async function runSmoke(): Promise { } smokePiTuiNativeLoad(); await smokeMinidbWorker(); - process.stdout.write(`Native asset smoke passed: ${manifest.target}; MiniDb worker build passed\n`); + await smokeSearchWorker(); + process.stdout.write( + `Native asset smoke passed: ${manifest.target}; MiniDb worker build passed; search worker ready\n`, + ); } export function runNativeAssetSmokeIfRequested(): boolean { diff --git a/apps/kimi-code/tsdown.dist-worker.config.ts b/apps/kimi-code/tsdown.dist-worker.config.ts new file mode 100644 index 000000000..43c26570e --- /dev/null +++ b/apps/kimi-code/tsdown.dist-worker.config.ts @@ -0,0 +1,41 @@ +// Bundles the kap-server global-search worker +// (packages/kap-server/src/search/worker/entry.ts) into ONE self-contained +// `dist/search-worker.mjs` sibling of the main bundle. The search worker +// host resolves it at runtime next to `dist/main.mjs` (dev/tests use the TS +// source; the SEA binary uses the extracted asset from +// tsdown.worker.config.ts). Separate config because rolldown forbids +// `codeSplitting: false` with multiple inputs. + +import { resolve } from 'node:path'; + +import { defineConfig } from 'tsdown'; + +const appRoot = import.meta.dirname; + +export default defineConfig({ + entry: { + 'search-worker': resolve( + appRoot, + '../../packages/kap-server/src/search/worker/entry.ts', + ), + }, + format: ['esm'], + // Shares the main bundle's dist (never wipe it) and lands as + // `dist/search-worker.mjs`. + outDir: 'dist', + clean: false, + dts: false, + hash: false, + platform: 'node', + target: 'node24', + sourcemap: false, + minify: false, + silent: true, + deps: { + onlyBundle: false, + }, + outputOptions: { + codeSplitting: false, + entryFileNames: '[name].mjs', + }, +}); diff --git a/apps/kimi-code/tsdown.worker.config.ts b/apps/kimi-code/tsdown.worker.config.ts index fc3335314..34a010954 100644 --- a/apps/kimi-code/tsdown.worker.config.ts +++ b/apps/kimi-code/tsdown.worker.config.ts @@ -1,11 +1,18 @@ -// Dedicated tsdown config that bundles the minidb text-build worker -// (packages/minidb/src/worker/text-build-worker.ts) and its whole import -// closure into ONE self-contained plain-JS ESM file. The SEA single-file -// binary embeds it as an asset (scripts/native/02-sea-blob.mjs) and spawns a -// real worker thread from it at runtime (src/native/minidb-worker.ts); -// without it the bundled binary has no worker entry file on disk and heavy -// text-index builds degrade to the inline main-thread core, stalling the -// event loop on large corpora. +// Dedicated tsdown config that bundles the off-main-thread workers into +// self-contained ESM files so they can ride the SEA blob as assets +// (02-sea-blob.mjs) and be spawned from disk at runtime: +// - text-build-worker.mjs: the minidb text-build worker +// (packages/minidb/src/worker/text-build-worker.ts); +// - search-worker.mjs: the kap-server global-search worker +// (packages/kap-server/src/search/worker/entry.ts). +// Without them the bundled binary lacks the worker entry files on disk and +// heavy index work degrades to the inline main-thread cores, stalling the +// event loop on large corpora. Runs after the main bundle with clean:false +// so all verified files remain. +// +// One config per entry: rolldown forbids `codeSplitting: false` with +// multiple inputs, and each worker must be a single self-contained file +// (check-bundle.mjs enforces zero remaining externals/relative imports). import { resolve } from 'node:path'; @@ -13,18 +20,31 @@ import { defineConfig } from 'tsdown'; const here = import.meta.dirname; -export default defineConfig({ - entry: [resolve(here, '../../packages/minidb/src/worker/text-build-worker.ts')], - format: ['esm'], - outDir: resolve(here, 'dist-native/intermediates'), - entryFileNames: 'text-build-worker.mjs', - codeSplitting: false, - platform: 'node', - target: 'node24', - dts: false, - sourcemap: false, - minify: false, - silent: true, - // The intermediates dir also holds main.cjs & co. — never wipe it. - clean: false, -}); +function workerConfig(name: string, entry: string) { + return defineConfig({ + entry: { [name]: resolve(here, entry) }, + format: ['esm'], + outDir: resolve(here, 'dist-native/intermediates'), + entryFileNames: '[name].mjs', + codeSplitting: false, + platform: 'node', + target: 'node24', + dts: false, + sourcemap: false, + minify: false, + silent: true, + deps: { + // Force-bundle the workspace packages the entries import + // (`@moonshot-ai/minidb` and its subpaths) so the output is + // self-contained. + alwaysBundle: [/^@moonshot-ai\//], + }, + // The intermediates dir also holds main.cjs & co. — never wipe it. + clean: false, + }); +} + +export default [ + workerConfig('text-build-worker', '../../packages/minidb/src/worker/text-build-worker.ts'), + workerConfig('search-worker', '../../packages/kap-server/src/search/worker/entry.ts'), +]; diff --git a/packages/acp-server/src/start.ts b/packages/acp-server/src/start.ts index a90635401..47835ecee 100644 --- a/packages/acp-server/src/start.ts +++ b/packages/acp-server/src/start.ts @@ -16,9 +16,13 @@ import { Readable, Writable } from 'node:stream'; import { ndJsonStream, type AgentConnection, type Stream } from '@agentclientprotocol/sdk'; import { bootstrap, + drainQueryStoreDisposals, + drainSessionIndexMirror, + drainSessionMetadataWrites, getLiveSessionById, IAppendLogStore, ISessionContext, + ISessionIndexMirror, logSeed, resolveConfigPath, resolveKimiHome, @@ -166,7 +170,19 @@ export async function runAcpServerWithStream( } catch { // ignore — disposal proceeds regardless } + // Same shutdown order as kap-server: settle queued session-metadata + // writes, then drain the session-index mirror while the query store is + // still open, so a queued summary lands in the read model. + await drainSessionMetadataWrites(); + await core.accessor.get(ISessionIndexMirror).drain(); core.dispose(); + // `core.dispose()` runs the mirror's and the query store's synchronous + // `dispose()`, whose drains/closes are asynchronous — await them so an + // embedding host that removes homeDir right after close() never races + // an in-flight shard close (ENOTEMPTY on teardown). + await drainSessionIndexMirror(); + await drainQueryStoreDisposals(); + await drainSessionMetadataWrites(); })(); return closePromise; }; diff --git a/packages/agent-core-v2/AGENTS.md b/packages/agent-core-v2/AGENTS.md index 392b6cad4..edb4c14c5 100644 --- a/packages/agent-core-v2/AGENTS.md +++ b/packages/agent-core-v2/AGENTS.md @@ -78,6 +78,10 @@ Business domains **do not implement persistence themselves** — they depend on Business code must not `import 'node:fs'`, write SQL, hand-roll append-logs / atomic writes, or hold file handles. Generic Stores are named by **access pattern** (`IAppendLogStore`, `IAtomicDocumentStore`); only domain-unique Stores are named after the domain (`ISessionIndex`). See `.agents/skills/agent-core-dev/persistence.md` for the full layering rules and decision tree. +## Session index + +`ISessionIndex` (`src/app/sessionIndex/`, App scope) serves session list/resume reads over two paths: the authoritative directory scan (`sessionIndexSource`, always correct, linear) and the minidb-backed derived read model (`IQueryStore` at `/cache/query-store`, keyset-paged, `O(log N + limit)`), gated by the `persistence_minidb_readmodel` flag (default ON; roll back via `KIMI_CODE_EXPERIMENTAL_PERSISTENCE_MINIDB_READMODEL=false` or the `[experimental]` config section). The read model has an explicit lifecycle — `uninitialized → preparing → ready/degraded` via `prepare()`/`status()`; reads while preparing answer from the authoritative store immediately and fold the `ISessionIndexMirror` queue in for read-your-writes; the first list shares one single-flight authoritative scan with the initial projection. The query-store is structural-only — text-index definitions are rejected at definition level, so session operations never touch the global full-text index (`/search-index`, owned by kap-server's search surface). + ## Conversation undo `context.undo` is the only persisted undo fact. `contextMemory/conversationTime.ts` owns the conversation clock (`isUndoAnchor` — the single tick predicate used by `computeUndoCut`, the checkpoint reducers, and the transcript reducer) and the checkpoint protocol. A wire Model whose state must follow conversation undo (todo, plan, task-notification delivery, …) **MUST** be defined with `defineCheckpointedModel` — never hand-roll the push/clear/restore reducers — which also registers it into `CHECKPOINTED_MODELS` for the undo pipeline's pre-cut depth check. World-time state (turn counters, task registries, revision counters) must stay outside checkpointed Models. diff --git a/packages/agent-core-v2/src/app/sessionIndex/sessionIndex.ts b/packages/agent-core-v2/src/app/sessionIndex/sessionIndex.ts index 5a42ed42f..10ddf32b6 100644 --- a/packages/agent-core-v2/src/app/sessionIndex/sessionIndex.ts +++ b/packages/agent-core-v2/src/app/sessionIndex/sessionIndex.ts @@ -128,6 +128,12 @@ export interface ISessionIndexMirror { record(summary: SessionSummary): void; /** Summaries accepted but not yet flushed (read-your-writes window). */ pending(): readonly SessionSummary[]; + /** + * Forget a session on the delete path: drop any queued summary and wait + * out an in-flight flush that may still carry it, so the caller's + * follow-up query-store delete is not resurrected by the mirror. + */ + evict(id: string): Promise; /** Flush everything currently queued; resolves with the queue empty. */ drain(): Promise; } diff --git a/packages/agent-core-v2/src/app/sessionIndex/sessionIndexMirrorService.ts b/packages/agent-core-v2/src/app/sessionIndex/sessionIndexMirrorService.ts index 36658493e..0070d6d05 100644 --- a/packages/agent-core-v2/src/app/sessionIndex/sessionIndexMirrorService.ts +++ b/packages/agent-core-v2/src/app/sessionIndex/sessionIndexMirrorService.ts @@ -8,7 +8,10 @@ * per session (only the newest summary is kept) and flush in chunks — on a * short timer or as soon as a batch fills — writing summaries (with the * recency column declared) and per-workspace counter deltas into the - * currently published generation. + * currently published generation. `evict()` forgets a deleted session (the + * `ISessionIndex.remove` path): reads fold the queue in for + * read-your-writes, so a queued creation must be dropped — and an in-flight + * flush waited out — before the store delete, or the entry is resurrected. * * Everything here is best-effort: a flush failure keeps the entries queued, * backs off, and after repeated failures gives up until the next `record` — @@ -110,6 +113,14 @@ export class SessionIndexMirror extends Disposable implements ISessionIndexMirro return [...this.pendingMap.values()]; } + async evict(id: string): Promise { + this.pendingMap.delete(id); + // A flush that already snapshotted this id may still be writing it; + // wait it out so the caller's store delete lands after that batch. + await this.flushing; + this.pendingMap.delete(id); + } + async drain(): Promise { this.timer.cancel(); while (this.pendingMap.size > 0) { diff --git a/packages/agent-core-v2/src/app/sessionIndex/sessionIndexProjector.ts b/packages/agent-core-v2/src/app/sessionIndex/sessionIndexProjector.ts index cd313cbde..385b962e8 100644 --- a/packages/agent-core-v2/src/app/sessionIndex/sessionIndexProjector.ts +++ b/packages/agent-core-v2/src/app/sessionIndex/sessionIndexProjector.ts @@ -11,10 +11,24 @@ * generation; the next run clears its own stragglers before writing. * Publishing also schedules the previous generation's drop. * + * The initial projection coincides with the first list request (read paths + * kick `prepare()` single-flight and fall back to the authoritative scan + * while unprepared). `sharedScan()` makes that ONE scan serve both: the + * projection joins a running scan — or reuses one that just settled within + * a short window — instead of enumerating every session directory a second + * time. Fallback reads use `sharedScanForRead()`: they join only a scan + * that is still in flight and otherwise drive a fresh one, so a read never + * serves a settled snapshot that could predate a just-created session. (A + * joined scan may still have started before the read; the index folds the + * mirror's pending queue into the result to cover that window.) A + * projection run consumes the slot on settle, so a later re-projection + * always scans fresh. + * * Reconciliation runs against the *published* generation: it re-scans the - * authoritative set, upserts summaries that drifted (mirror loss, external - * edits), deletes entries whose document disappeared, and rewrites every - * counter from the authoritative scan — bounding counter drift to one + * authoritative set (always fresh — a stale snapshot could regress counters + * the mirror just updated), upserts summaries that drifted (mirror loss, + * external edits), deletes entries whose document disappeared, and rewrites + * every counter from the authoritative scan — bounding counter drift to one * reconcile interval. * * This is an internal collaborator of `FileSessionIndex`, not a DI service: @@ -46,6 +60,15 @@ import { const WRITE_CHUNK = 500; const SCAN_CONCURRENCY = 16; +/** + * How long a settled shared scan stays reusable BY A PROJECTION. The window + * only needs to cover the gap between a fallback read finishing its scan and + * the kicked projection reaching its own scan call (query-store open + + * collection housekeeping in between); a projection never reuses a slot + * older than this. Fallback reads never reuse a settled scan at all (see + * `sharedScanForRead`). + */ +const SHARED_SCAN_REUSE_MS = 30_000; export interface SessionIndexProjectorDeps { readonly storage: IFileSystemStorageService; @@ -66,11 +89,87 @@ export interface ReconcileResult { readonly removed: number; } +/** One consistent pass over the authoritative session metadata set. */ +export interface AuthoritativeScan { + readonly summaries: SessionSummary[]; + readonly counts: Map; +} + +interface ScanSlot { + readonly promise: Promise; + readonly reusableUntil: number; + settled: boolean; +} + export class SessionIndexProjector { + private scanSlot: ScanSlot | undefined; + constructor(private readonly deps: SessionIndexProjectorDeps) {} + /** + * The projection's scan: joins a running shared scan, reuses one that + * settled within the reuse window, or starts a fresh one. The projection + * publishes a point-in-time derived model by design, so a just-finished + * snapshot is safe for it (the mirror queue and reconciliation heal the + * gap) — and this is what keeps a fast first read + kicked projection + * from scanning the directory tree twice. + */ + sharedScan(): Promise { + const slot = this.scanSlot; + if (slot !== undefined && (!slot.settled || Date.now() < slot.reusableUntil)) { + return slot.promise; + } + return this.startScan(); + } + + /** + * A fallback read's scan: joins a scan that is still in flight or starts a + * fresh one. A settled snapshot is NEVER served to a read — it could + * predate a session this process just created, breaking read-your-writes. + * Joining an in-flight scan is NOT the same freshness as enumerating here + * and now: the scan may have started (and passed a directory) before this + * call, so the caller folds the mirror's pending queue into the result — + * every pending entry is known to be durable on disk. + */ + sharedScanForRead(): Promise { + const slot = this.scanSlot; + if (slot !== undefined && !slot.settled) return slot.promise; + return this.startScan(); + } + + private startScan(): Promise { + const slot: ScanSlot = { + promise: this.scanAuthoritative(), + reusableUntil: Date.now() + SHARED_SCAN_REUSE_MS, + settled: false, + }; + const markSettled = (): void => { + slot.settled = true; + }; + void slot.promise.then(markSettled, markSettled); + this.scanSlot = slot; + return slot.promise; + } + /** Scan the authoritative set into a fresh generation and publish it. */ async project(generation: number): Promise { + // Captured BEFORE the housekeeping below: the scan overlaps it, and the + // finally invalidates exactly the slot this run consumed — never a newer + // scan a concurrent fallback read started meanwhile. + const scan = this.sharedScan(); + try { + return await this.doProject(generation, scan); + } finally { + // The consumed slot must not serve a LATER projection: a re-projection + // always scans the authoritative set fresh. + if (this.scanSlot?.promise === scan) this.scanSlot = undefined; + } + } + + private async doProject( + generation: number, + scan: Promise, + ): Promise { const { queryStore, log } = this.deps; const collection = sessionCollection(generation); const counters = sessionCountersCollection(generation); @@ -83,7 +182,7 @@ export class SessionIndexProjector { field: `custom.${PARENT_SESSION_ID_KEY}`, }); - const { summaries, counts } = await this.scanAuthoritative(); + const { summaries, counts } = await scan; await this.batchChunks( summaries.map((summary) => ({ kind: 'put' as const, @@ -156,10 +255,7 @@ export class SessionIndexProjector { return result; } - private async scanAuthoritative(): Promise<{ - summaries: SessionSummary[]; - counts: Map; - }> { + private async scanAuthoritative(): Promise { const { storage, docs, sessionsScope } = this.deps; const summaries: SessionSummary[] = []; const counts = new Map(); diff --git a/packages/agent-core-v2/src/app/sessionIndex/sessionIndexService.ts b/packages/agent-core-v2/src/app/sessionIndex/sessionIndexService.ts index 3f428f499..012e997db 100644 --- a/packages/agent-core-v2/src/app/sessionIndex/sessionIndexService.ts +++ b/packages/agent-core-v2/src/app/sessionIndex/sessionIndexService.ts @@ -25,6 +25,17 @@ * model is never healed by per-request backfill. Degraded reads retry * `prepare()` after a short backoff. * + * The first list request coincides with the initial projection: the read is + * served by the authoritative fallback AND kicks `prepare()`. To keep that + * request from paying two full directory scans, the uninitialized/preparing + * fallback joins the projector's in-flight shared scan — or drives the one + * the projection will reuse (`SessionIndexProjector.sharedScanForRead`) — so + * one authoritative pass serves both, and the mirror's pending queue is + * folded into the result so a session created or mutated after the scan + * passed its directory still shows up. Flag-off hosts and the `degraded` + * fallback keep the targeted per-workspace enumeration (with the same + * pending fold). + * * Keyset pagination is canonical (`updatedAt` desc, `id` desc): a cursor is a * session id resolved by point lookup, and the window's boundary tie group is * re-fetched and merged so same-millisecond ties never lose or duplicate an @@ -331,11 +342,14 @@ export class FileSessionIndex extends Disposable implements ISessionIndex { * Evict a deleted session's derived state so `get` / `listRecent` stop * answering for the id immediately: the authoritative directory is deleted * by the caller (`sessionLifecycle.delete`), and the next projection would - * drop the entry anyway — this closes the stale-read window in between. A - * summary still queued in the mirror heals at the next projection. With the - * read model off there is no derived state to evict. + * drop the entry anyway — this closes the stale-read window in between. The + * mirror queue is evicted first (waiting out an in-flight flush): reads + * fold the queue in for read-your-writes, and a late flush would otherwise + * resurrect the entry after the store delete. With the read model off + * there is no derived state to evict beyond the queue. */ async remove(id: string): Promise { + await this.mirror.evict(id); await this.withReadModel( async (generation) => { await this.queryStore.delete(sessionCollection(generation), id); @@ -635,17 +649,11 @@ export class FileSessionIndex extends Disposable implements ISessionIndex { return { items: query.limit !== undefined ? items.slice(0, query.limit) : items }; } - const workspaceIds = query.workspaceIds ?? (await listWorkspaceIds(this.storage, this.sessionsScope)); - const collected: SessionSummary[] = []; - for (const workspaceId of workspaceIds) { - for (const sessionId of await listSessionIds(this.storage, this.sessionsScope, workspaceId)) { - const summary = await readSessionSummary(this.docs, this.sessionsScope, workspaceId, sessionId); - if (summary === undefined) continue; - if (summary.archived && query.includeArchived !== true) continue; - if (!summaryMatchesChildOf(summary, query.childOf)) continue; - collected.push(summary); - } - } + const collected = (await this.collectAuthoritative(query.workspaceIds)).filter( + (summary) => + (query.includeArchived === true || !summary.archived) && + summaryMatchesChildOf(summary, query.childOf), + ); const items = collected.toSorted(canonicalOrder); let start = 0; @@ -678,18 +686,59 @@ export class FileSessionIndex extends Disposable implements ISessionIndex { private async countLegacy(query: SessionCountQuery): Promise { let count = 0; - const workspaceIds = - query.workspaceIds ?? (await listWorkspaceIds(this.storage, this.sessionsScope)); - for (const workspaceId of workspaceIds) { - for (const sessionId of await listSessionIds(this.storage, this.sessionsScope, workspaceId)) { - const summary = await readSessionSummary(this.docs, this.sessionsScope, workspaceId, sessionId); - if (summary === undefined) continue; - if (query.includeArchived === true || !summary.archived) count += 1; - } + for (const summary of await this.collectAuthoritative(query.workspaceIds)) { + if (query.includeArchived === true || !summary.archived) count += 1; } return count; } + /** + * Collect the authoritative summaries behind a legacy read. While the read + * model is enabled but not yet ready, the kicked initial projection is + * scanning the same authoritative set, so the read joins that in-flight + * scan (or drives the one the projection will reuse) instead of running a + * second full directory scan. Flag-off hosts and the degraded fallback + * keep the targeted per-workspace enumeration. + * + * Either way the mirror's pending queue is folded in by id (pending + * entries win): every queued summary was recorded only after its + * `state.json` is durable, and a scan/enumeration that started before the + * write may legitimately have passed the directory already — the fold is + * what keeps read-your-writes on this path too. + */ + private async collectAuthoritative( + workspaceIds: readonly string[] | undefined, + ): Promise { + let collected: SessionSummary[]; + if ( + this.readModelEnabled() && + (this.state === 'uninitialized' || this.state === 'preparing') + ) { + const { summaries } = await this.projector.sharedScanForRead(); + collected = + workspaceIds === undefined + ? summaries + : summaries.filter((summary) => workspaceIds.includes(summary.workspaceId)); + } else { + const ids = workspaceIds ?? (await listWorkspaceIds(this.storage, this.sessionsScope)); + collected = []; + for (const workspaceId of ids) { + for (const sessionId of await listSessionIds(this.storage, this.sessionsScope, workspaceId)) { + const summary = await readSessionSummary(this.docs, this.sessionsScope, workspaceId, sessionId); + if (summary !== undefined) collected.push(summary); + } + } + } + const pending = this.mirror.pending(); + if (pending.length === 0) return collected; + const byId = new Map(collected.map((summary) => [summary.id, summary])); + for (const summary of pending) { + if (workspaceIds !== undefined && !workspaceIds.includes(summary.workspaceId)) continue; + byId.set(summary.id, summary); + } + return [...byId.values()]; + } + private readModelEnabled(): boolean { return this.flags.enabled(READ_MODEL_FLAG); } diff --git a/packages/agent-core-v2/src/persistence/backends/minidb/flag.ts b/packages/agent-core-v2/src/persistence/backends/minidb/flag.ts index 23be408e3..1a1f2433f 100644 --- a/packages/agent-core-v2/src/persistence/backends/minidb/flag.ts +++ b/packages/agent-core-v2/src/persistence/backends/minidb/flag.ts @@ -2,9 +2,9 @@ * `minidb` persistence backend — flag contribution. * * Gates the minidb-backed derived read-model (`IQueryStore`) and the consumers - * that read through it. Off by default; enable via - * `KIMI_CODE_EXPERIMENTAL_PERSISTENCE_MINIDB_READMODEL` or the `[experimental]` - * config section. + * that read through it. On by default; roll back to the legacy authoritative + * read path via `KIMI_CODE_EXPERIMENTAL_PERSISTENCE_MINIDB_READMODEL=false` or + * the `[experimental]` config section. */ import { type FlagDefinitionInput, registerFlagDefinition } from '#/app/flag/flagRegistry'; @@ -15,7 +15,7 @@ export const persistenceMiniDbReadModelFlag: FlagDefinitionInput = { description: 'Use the minidb-backed IQueryStore as a derived read model for session indexing and wire replay.', env: 'KIMI_CODE_EXPERIMENTAL_PERSISTENCE_MINIDB_READMODEL', - default: false, + default: true, surface: 'core', }; diff --git a/packages/agent-core-v2/src/persistence/backends/minidb/miniDbQueryStore.ts b/packages/agent-core-v2/src/persistence/backends/minidb/miniDbQueryStore.ts index 666c75fd3..1545fbcbc 100644 --- a/packages/agent-core-v2/src/persistence/backends/minidb/miniDbQueryStore.ts +++ b/packages/agent-core-v2/src/persistence/backends/minidb/miniDbQueryStore.ts @@ -38,6 +38,14 @@ * cluster-wide registry, and value indexes are created `sparse` so documents * from other collections (which lack the indexed field) are skipped. * + * This store is a STRUCTURAL read model: stores, datetime/order columns, and + * secondary/compound indexes only. `ensureIndex` rejects text-index + * definitions — a text index would pull tokenizers, postings files, and + * full-text generation builds into the session-list critical path, which is + * exactly what the search-index separation forbids here; full-text search + * lives in the kap-server search-index database (`IGlobalSearchService`), + * never in this cluster. + * * Ordered columns map to the engine's `dt` channels: `put`/`batch` forward * `columns` as `SetOptions.dt`, and `pageByColumn` issues a dt-bounded, * dt-sorted, limited query — which the engine serves by walking its ordered @@ -146,6 +154,9 @@ export class MiniDbQueryStore extends Disposable implements IQueryStore { } private openFresh(): Promise { + // Answers "which database does a session-list read touch" from logs alone: + // the read model lives in this cluster, one MiniDb per shard directory. + this.log.info('minidb query-store opening', { dir: this.dir, shardCount: SHARD_COUNT }); return ClusterDb.open({ dir: this.dir, shardCount: SHARD_COUNT, @@ -272,6 +283,11 @@ export class MiniDbQueryStore extends Disposable implements IQueryStore { } async ensureIndex(collection: string, def: IndexDef): Promise { + if (def.kind === 'text') { + throw new Error( + `minidb query-store is a structural read model: text index "${def.name}" on collection "${collection}" is rejected; full-text search lives in the kap-server search-index database`, + ); + } const guard = `${collection}:${def.kind}:${def.name}`; if (this.ensuredIndexes.has(guard)) return; const name = indexName(collection, def.name); @@ -279,10 +295,8 @@ export class MiniDbQueryStore extends Disposable implements IQueryStore { try { if (def.kind === 'value') { await db.createIndex(name, { field: def.field, sparse: true, unique: def.unique }); - } else if (def.kind === 'compound') { - await db.createCompoundIndex(name, { groupBy: def.groupBy, orderBy: def.orderBy }); } else { - await db.createTextIndex(name, { fields: def.fields }); + await db.createCompoundIndex(name, { groupBy: def.groupBy, orderBy: def.orderBy }); } } catch (error) { if (!(error instanceof Error) || !error.message.includes('already exists')) throw error; diff --git a/packages/agent-core-v2/src/session/sessionMetadata/sessionMetadataService.ts b/packages/agent-core-v2/src/session/sessionMetadata/sessionMetadataService.ts index 87fbd5f7d..70505926e 100644 --- a/packages/agent-core-v2/src/session/sessionMetadata/sessionMetadataService.ts +++ b/packages/agent-core-v2/src/session/sessionMetadata/sessionMetadataService.ts @@ -20,7 +20,9 @@ * `ISessionIndexMirror` — a bounded, coalescing queue that flushes to the * `IQueryStore` read model off the user completion path. The mutation * completes with the authoritative `state.json` write; it never waits on the - * derived store (no mirror flush, no query-store lock). First-time creation in + * derived store (no mirror flush, no query-store lock), and a mirror failure + * is logged and swallowed — the read model heals by reconciliation, the + * session lifecycle never sees it. First-time creation in * `load()` records too — a new session must appear in listings immediately * (the mirror's pending queue feeds the index's read-your-writes merge); * loading an *existing* document (session resume) stays silent. Queued writes @@ -160,20 +162,30 @@ export class SessionMetadata extends Service implements ISessionMetadata { } private mirrorToReadModel(): void { - this.mirror.record( - buildSessionSummary({ - id: this.data.id, - workspaceId: this.ctx.workspaceId, - cwd: this.ctx.cwd, - title: this.data.title, - lastPrompt: this.data.lastPrompt, - createdAt: this.data.createdAt, - updatedAt: this.data.updatedAt, - archived: this.data.archived === true, - custom: this.data.custom, - lastTurnReason: this.data.lastTurnReason, - }), - ); + try { + this.mirror.record( + buildSessionSummary({ + id: this.data.id, + workspaceId: this.ctx.workspaceId, + cwd: this.ctx.cwd, + title: this.data.title, + lastPrompt: this.data.lastPrompt, + createdAt: this.data.createdAt, + updatedAt: this.data.updatedAt, + archived: this.data.archived === true, + custom: this.data.custom, + lastTurnReason: this.data.lastTurnReason, + }), + ); + } catch (error) { + // The authoritative document is already durable at this point; a mirror + // failure only degrades the read model (reconciliation heals it) and + // must never fail the session mutation itself. + this.log.warn('session index mirror record failed; the read model heals by reconciliation', { + sessionId: this.ctx.sessionId, + error: error instanceof Error ? error.message : String(error), + }); + } } private async load(): Promise { diff --git a/packages/agent-core-v2/test/app/sessionIndex/sessionIndex.test.ts b/packages/agent-core-v2/test/app/sessionIndex/sessionIndex.test.ts index f72192188..85410d3da 100644 --- a/packages/agent-core-v2/test/app/sessionIndex/sessionIndex.test.ts +++ b/packages/agent-core-v2/test/app/sessionIndex/sessionIndex.test.ts @@ -32,6 +32,7 @@ import { FileStorageService } from '#/persistence/backends/node-fs/fileStorageSe import { IAtomicDocumentStore } from '#/persistence/interface/atomicDocumentStore'; import { IQueryStore, + type Checkpoint, type ColumnPageQuery, type IQuery, type Page, @@ -707,6 +708,89 @@ describe('FileSessionIndex (read model)', () => { expect(unknown.items).toEqual([]); }); + it('remove evicts a queued mirror entry so a deleted session stays unlisted', async () => { + await seedSession('a', { createdAt: 1, updatedAt: 2 }); + const store = build(); + await store.prepare(); + + // Created after the projection: present only in the mirror queue. + mirror.record(summary('fresh', { createdAt: 3, updatedAt: 10 })); + const before = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(before.items.map((s) => s.id)).toEqual(['fresh', 'a']); + + // sessionLifecycle.delete removes the authoritative doc, then evicts + // through remove(): the queued summary must not fold the session back in. + await store.remove('fresh'); + expect(mirror.pending()).toEqual([]); + const after = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(after.items.map((s) => s.id)).toEqual(['a']); + + // A later flush must not resurrect the evicted summary either. + await mirror.drain(); + const settled = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(settled.items.map((s) => s.id)).toEqual(['a']); + }); + + it('remove waits out an in-flight mirror flush before deleting from the store', async () => { + await seedSession('a', { createdAt: 1, updatedAt: 2 }); + + let batchGate: Promise | undefined; + let releaseBatch: () => void = () => {}; + let notifyBatchEntered: (() => void) | undefined; + class GatedQueryStore extends MiniDbQueryStore { + override async batch(ops: readonly WriteOp[]): Promise { + notifyBatchEntered?.(); + const gate = batchGate; + batchGate = undefined; + if (gate !== undefined) await gate; + return super.batch(ops); + } + } + registerScopedService( + LifecycleScope.App, + IQueryStore, + GatedQueryStore, + ScopeActivation.OnDemand, + 'storage', + ); + const fileStorage = new FileStorageService(homeDir); + const host = createScopedTestHost([ + stubPair(IFileSystemStorageService, fileStorage), + stubPair(IAtomicDocumentStore, new JsonAtomicDocumentStore(fileStorage)), + stubPair(IBootstrapService, stubBootstrap(homeDir)), + stubPair(ILogService, stubLog()), + stubPair(IFlagService, stubFlag(true)), + ]); + disposeHost = () => { + host.dispose(); + }; + queryStore = host.app.accessor.get(IQueryStore); + mirror = host.app.accessor.get(ISessionIndexMirror); + const store = host.app.accessor.get(ISessionIndex) as FileSessionIndex; + await store.prepare(); + + // Arm the gate, queue a summary, and start a flush that parks inside + // batch while still carrying the id. + const entered = new Promise((resolve) => { + notifyBatchEntered = resolve; + }); + batchGate = new Promise((resolve) => { + releaseBatch = resolve; + }); + mirror.record(summary('fresh', { createdAt: 3, updatedAt: 10 })); + const draining = mirror.drain(); + await entered; + + // remove() must block behind the in-flight flush; the flush lands first, + // then the store delete erases the resurrected entry. + const removing = store.remove('fresh'); + releaseBatch(); + await Promise.all([removing, draining]); + + const page = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(page.items.map((s) => s.id)).toEqual(['a']); + }); + it('count folds the mirror queue in before the flush lands', async () => { await seedSession('a', { createdAt: 1, updatedAt: 2 }); await seedSession('b', { createdAt: 2, updatedAt: 3 }); @@ -765,9 +849,17 @@ describe('FileSessionIndex (read model)', () => { const status = await store.prepare(); expect(status.state).toBe('degraded'); expect(status.degradedCount).toBe(1); + // The degraded state stays diagnosable: the reason names the failing stage. + expect(status.reason).toBe('projection failed'); // Reads still work — from the authoritative documents. const fallback = await store.listRecent({ workspaceIds: [workspaceId] }); expect(fallback.items.map((s) => s.id)).toEqual(['b', 'a']); + // The fallback keeps serving while degraded (no silent read-model answers). + expect(store.status()).toEqual({ + state: 'degraded', + reason: 'projection failed', + degradedCount: 1, + }); const recovered = await store.prepare(); expect(recovered).toEqual({ state: 'ready', generation: 1, degradedCount: 1 }); @@ -880,6 +972,471 @@ describe('FileSessionIndex (read model)', () => { expect(await store.count({ workspaceIds: [workspaceId] })).toBe(2); }); + it('the first read kicks one initial projection and shares its authoritative scan', async () => { + await seedSession('a', { title: 'a', createdAt: 1, updatedAt: 3 }); + await seedSession('b', { title: 'b', createdAt: 2, updatedAt: 2 }); + await seedSession('c', { title: 'c', createdAt: 3, updatedAt: 1 }); + + // Counts authoritative document reads: one full scan is exactly two gets + // per seeded session (the `/state.json` miss plus the + // `/session-meta/state.json` fallback hit). The first list + // kicks the initial projection AND falls back to the authoritative path; + // both must be served by ONE shared scan, not one scan each. + class CountingDocs extends JsonAtomicDocumentStore { + gets = 0; + override async get(scope: string, key: string): Promise { + this.gets += 1; + return super.get(scope, key); + } + } + const fileStorage = new FileStorageService(homeDir); + const docs = new CountingDocs(fileStorage); + const host = createScopedTestHost([ + stubPair(IFileSystemStorageService, fileStorage), + stubPair(IAtomicDocumentStore, docs), + stubPair(IBootstrapService, stubBootstrap(homeDir)), + stubPair(ILogService, stubLog()), + stubPair(IFlagService, stubFlag(true)), + ]); + disposeHost = () => { + host.dispose(); + }; + queryStore = host.app.accessor.get(IQueryStore); + mirror = host.app.accessor.get(ISessionIndexMirror); + const store = host.app.accessor.get(ISessionIndex) as FileSessionIndex; + + const [page, status, sameFlight] = await Promise.all([ + store.listRecent({ workspaceIds: [workspaceId] }), + store.prepare(), + store.prepare(), + ]); + expect(page.items.map((s) => s.id)).toEqual(['a', 'b', 'c']); + // Single-flight: two concurrent prepares plus the read's kick produced + // exactly one projection (generation 1, never 2). + expect(status).toEqual({ state: 'ready', generation: 1, degradedCount: 0 }); + expect(sameFlight).toEqual(status); + expect(docs.gets).toBe(6); + + // Warm reads come from the read model — no further document reads. + const warm = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(warm.items.map((s) => s.id)).toEqual(['a', 'b', 'c']); + expect(await store.count({ workspaceIds: [workspaceId] })).toBe(3); + expect(docs.gets).toBe(6); + }); + + it('serves authoritative reads immediately while preparing', async () => { + await seedSession('a', { title: 'a', createdAt: 1, updatedAt: 2 }); + await seedSession('b', { title: 'b', createdAt: 2, updatedAt: 3 }); + + // Blocks the read-model open at the checkpoint read: prepare() stays in + // `preparing` until the gate is released. + class GatedQueryStore extends MiniDbQueryStore { + private gate: Promise | undefined; + private openGate: (() => void) | undefined; + hold(): void { + this.gate = new Promise((resolve) => { + this.openGate = resolve; + }); + } + release(): void { + this.openGate?.(); + this.gate = undefined; + } + override async getCheckpoint(source: string): Promise { + await this.gate; + return super.getCheckpoint(source); + } + } + registerScopedService( + LifecycleScope.App, + IQueryStore, + GatedQueryStore, + ScopeActivation.OnDemand, + 'storage', + ); + const fileStorage = new FileStorageService(homeDir); + const host = createScopedTestHost([ + stubPair(IFileSystemStorageService, fileStorage), + stubPair(IAtomicDocumentStore, new JsonAtomicDocumentStore(fileStorage)), + stubPair(IBootstrapService, stubBootstrap(homeDir)), + stubPair(ILogService, stubLog()), + stubPair(IFlagService, stubFlag(true)), + ]); + disposeHost = () => { + host.dispose(); + }; + queryStore = host.app.accessor.get(IQueryStore); + mirror = host.app.accessor.get(ISessionIndexMirror); + const store = host.app.accessor.get(ISessionIndex) as FileSessionIndex; + + (queryStore as GatedQueryStore).hold(); + const preparing = store.prepare(); + expect(store.status().state).toBe('preparing'); + + // Neither the list, the count, nor the point lookup waits for the blocked + // projection: all answer from the authoritative metadata immediately. + const page = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(page.items.map((s) => s.id)).toEqual(['b', 'a']); + expect(await store.count({ workspaceIds: [workspaceId] })).toBe(2); + expect((await store.get('b'))?.title).toBe('b'); + expect(store.status().state).toBe('preparing'); + + (queryStore as GatedQueryStore).release(); + const status = await preparing; + expect(status).toEqual({ state: 'ready', generation: 1, degradedCount: 0 }); + const warm = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(warm.items.map((s) => s.id)).toEqual(['b', 'a']); + }); + + it('folds the mirror queue into reads that join an in-flight scan (read-your-writes)', async () => { + await seedSession('a', { title: 'a', createdAt: 1, updatedAt: 2 }); + await seedSession('b', { title: 'b', createdAt: 2, updatedAt: 3 }); + + // Parks the shared scan mid-flight at the first document read, so a + // mutation landing while the scan is running is guaranteed to postdate + // the scan's pass over its directory. + class GatedDocs extends JsonAtomicDocumentStore { + private gate: Promise | undefined; + private openGate: (() => void) | undefined; + private markFirstGet: (() => void) | undefined; + readonly firstGet = new Promise((resolve) => { + this.markFirstGet = resolve; + }); + hold(): void { + this.gate = new Promise((resolve) => { + this.openGate = resolve; + }); + } + release(): void { + this.openGate?.(); + } + override async get(scope: string, key: string): Promise { + this.markFirstGet?.(); + await this.gate; + return super.get(scope, key); + } + } + const fileStorage = new FileStorageService(homeDir); + const docs = new GatedDocs(fileStorage); + const host = createScopedTestHost([ + stubPair(IFileSystemStorageService, fileStorage), + stubPair(IAtomicDocumentStore, docs), + stubPair(IBootstrapService, stubBootstrap(homeDir)), + stubPair(ILogService, stubLog()), + stubPair(IFlagService, stubFlag(true)), + ]); + disposeHost = () => { + host.dispose(); + }; + queryStore = host.app.accessor.get(IQueryStore); + mirror = host.app.accessor.get(ISessionIndexMirror); + const store = host.app.accessor.get(ISessionIndex) as FileSessionIndex; + + docs.hold(); + const first = store.listRecent({ workspaceIds: [workspaceId] }); + await docs.firstGet; // the shared scan is now parked mid-flight + + // A creation and an archive flip land while the scan is running: both + // state.json documents are durable and their summaries sit in the mirror + // queue (exactly what SessionMetadata does after persisting). + await seedSession('c', { title: 'c', createdAt: 3, updatedAt: 4 }); + mirror.record(summary('c', { title: 'c', createdAt: 3, updatedAt: 4 })); + mirror.record(summary('a', { archived: true, updatedAt: 5 })); + + // The second read JOINS the in-flight scan; the scan's snapshot contains + // neither 'c' nor the archive flip. The pending fold must restore both. + const second = store.listRecent({ workspaceIds: [workspaceId] }); + docs.release(); + const [firstPage, secondPage] = await Promise.all([first, second]); + for (const page of [firstPage, secondPage]) { + expect(page.items.map((s) => s.id)).toEqual(['c', 'b']); + } + + // The projection publishes the scan's snapshot (without 'c'); the queued + // mirror entries flush into it and warm reads converge on the same view. + const status = await store.prepare(); + expect(status.state).toBe('ready'); + await mirror.drain(); + const warm = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(warm.items.map((s) => s.id)).toEqual(['c', 'b']); + const all = await store.listRecent({ workspaceIds: [workspaceId], includeArchived: true }); + expect(all.items.map((s) => s.id)).toEqual(['a', 'c', 'b']); + }); + + it('never serves a settled shared scan to a fallback read', async () => { + await seedSession('a', { title: 'a', createdAt: 1, updatedAt: 2 }); + await seedSession('b', { title: 'b', createdAt: 2, updatedAt: 3 }); + + // Holds prepare() at the checkpoint read so the state stays `preparing` + // across several reads; counts document reads to prove scan freshness. + class GatedQueryStore extends MiniDbQueryStore { + private gate: Promise | undefined; + private openGate: (() => void) | undefined; + hold(): void { + this.gate = new Promise((resolve) => { + this.openGate = resolve; + }); + } + release(): void { + this.openGate?.(); + this.gate = undefined; + } + override async getCheckpoint(source: string): Promise { + await this.gate; + return super.getCheckpoint(source); + } + } + class CountingDocs extends JsonAtomicDocumentStore { + gets = 0; + override async get(scope: string, key: string): Promise { + this.gets += 1; + return super.get(scope, key); + } + } + registerScopedService( + LifecycleScope.App, + IQueryStore, + GatedQueryStore, + ScopeActivation.OnDemand, + 'storage', + ); + const fileStorage = new FileStorageService(homeDir); + const docs = new CountingDocs(fileStorage); + const host = createScopedTestHost([ + stubPair(IFileSystemStorageService, fileStorage), + stubPair(IAtomicDocumentStore, docs), + stubPair(IBootstrapService, stubBootstrap(homeDir)), + stubPair(ILogService, stubLog()), + stubPair(IFlagService, stubFlag(true)), + ]); + disposeHost = () => { + host.dispose(); + }; + queryStore = host.app.accessor.get(IQueryStore); + mirror = host.app.accessor.get(ISessionIndexMirror); + const store = host.app.accessor.get(ISessionIndex) as FileSessionIndex; + + (queryStore as GatedQueryStore).hold(); + const preparing = store.prepare(); + // The first fallback read drives a fresh scan (2 sessions × 2 gets). + const first = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(first.items.map((s) => s.id)).toEqual(['b', 'a']); + expect(docs.gets).toBe(4); + + // Written behind the mirror's back (an external-process creation): the + // first scan is settled by now, so the second read must scan fresh and + // see it — a reused snapshot would answer [b, a] forever. + await seedSession('c', { title: 'c', createdAt: 3, updatedAt: 4 }); + const second = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(second.items.map((s) => s.id)).toEqual(['c', 'b', 'a']); + expect(docs.gets).toBe(10); // 3 sessions × 2 gets on the second scan + + // The kicked projection still avoids a redundant pass: it reuses the + // just-settled second scan within the reuse window. + (queryStore as GatedQueryStore).release(); + const status = await preparing; + expect(status).toEqual({ state: 'ready', generation: 1, degradedCount: 0 }); + expect(docs.gets).toBe(10); + }); + + it('status() walks the read-model lifecycle and stays diagnosable through degradation', async () => { + await seedSession('a', { title: 'a', createdAt: 1, updatedAt: 2 }); + + // Gate the checkpoint read so `preparing` is observable; fail one batch + // later so the `degraded` transition is observable in the same walk. + class GatedFlakyQueryStore extends MiniDbQueryStore { + private gate: Promise | undefined; + private openGate: (() => void) | undefined; + failNextBatch = false; + hold(): void { + this.gate = new Promise((resolve) => { + this.openGate = resolve; + }); + } + release(): void { + this.openGate?.(); + this.gate = undefined; + } + override async getCheckpoint(source: string): Promise { + await this.gate; + return super.getCheckpoint(source); + } + override async batch(ops: readonly WriteOp[]): Promise { + if (this.failNextBatch) { + this.failNextBatch = false; + throw new Error('injected projection crash'); + } + return super.batch(ops); + } + } + registerScopedService( + LifecycleScope.App, + IQueryStore, + GatedFlakyQueryStore, + ScopeActivation.OnDemand, + 'storage', + ); + const fileStorage = new FileStorageService(homeDir); + const host = createScopedTestHost([ + stubPair(IFileSystemStorageService, fileStorage), + stubPair(IAtomicDocumentStore, new JsonAtomicDocumentStore(fileStorage)), + stubPair(IBootstrapService, stubBootstrap(homeDir)), + stubPair(ILogService, stubLog()), + stubPair(IFlagService, stubFlag(true)), + ]); + disposeHost = () => { + host.dispose(); + }; + queryStore = host.app.accessor.get(IQueryStore); + mirror = host.app.accessor.get(ISessionIndexMirror); + const store = host.app.accessor.get(ISessionIndex) as FileSessionIndex; + const gated = queryStore as GatedFlakyQueryStore; + + // uninitialized → preparing → ready, each observable through status(). + expect(store.status()).toEqual({ state: 'uninitialized', degradedCount: 0 }); + gated.hold(); + const preparing = store.prepare(); + expect(store.status()).toEqual({ state: 'preparing', degradedCount: 0 }); + gated.release(); + expect(await preparing).toEqual({ state: 'ready', generation: 1, degradedCount: 0 }); + + // A failing re-projection with a published generation keeps serving it: + // generation failures stay local to the read model's next attempt. + gated.failNextBatch = true; + await store.reprojectNow(); + expect(store.status()).toEqual({ state: 'ready', generation: 1, degradedCount: 0 }); + + // A store that lost its published base (corruption rebuild) degrades on + // the next prepare — diagnosably — while reads keep answering from the + // authoritative documents. + disposeHost?.(); + disposeHost = undefined; + await drainSessionIndexMirror(); + await drainQueryStoreDisposals(); + await fsp.rm(join(homeDir, 'cache', 'query-store'), { recursive: true, force: true }); + + const second = build(); + (queryStore as GatedFlakyQueryStore).failNextBatch = true; + const degraded = await second.prepare(); + expect(degraded.state).toBe('degraded'); + expect(degraded.reason).toBe('projection failed'); + expect(degraded.degradedCount).toBe(1); + const fallback = await second.listRecent({ workspaceIds: [workspaceId] }); + expect(fallback.items.map((s) => s.id)).toEqual(['a']); + expect(await second.prepare()).toEqual({ state: 'ready', generation: 1, degradedCount: 1 }); + }); + + it('a restart loads the published generation instead of re-scanning', async () => { + await seedSession('a', { title: 'a', createdAt: 1, updatedAt: 3 }); + await seedSession('b', { title: 'b', createdAt: 2, updatedAt: 2 }); + await seedSession('c', { title: 'c', createdAt: 3, updatedAt: 1 }); + + const first = build(); + await first.prepare(); + expect(first.status()).toEqual({ state: 'ready', generation: 1, degradedCount: 0 }); + disposeHost?.(); + disposeHost = undefined; + await drainSessionIndexMirror(); + await drainQueryStoreDisposals(); + + class CountingDocs extends JsonAtomicDocumentStore { + gets = 0; + override async get(scope: string, key: string): Promise { + this.gets += 1; + return super.get(scope, key); + } + } + const fileStorage = new FileStorageService(homeDir); + const docs = new CountingDocs(fileStorage); + const host = createScopedTestHost([ + stubPair(IFileSystemStorageService, fileStorage), + stubPair(IAtomicDocumentStore, docs), + stubPair(IBootstrapService, stubBootstrap(homeDir)), + stubPair(ILogService, stubLog()), + stubPair(IFlagService, stubFlag(true)), + ]); + disposeHost = () => { + host.dispose(); + }; + queryStore = host.app.accessor.get(IQueryStore); + mirror = host.app.accessor.get(ISessionIndexMirror); + const second = host.app.accessor.get(ISessionIndex) as FileSessionIndex; + + // The healthy published generation attaches directly: same generation, + // no projection, and not a single authoritative document read. + const status = await second.prepare(); + expect(status).toEqual({ state: 'ready', generation: 1, degradedCount: 0 }); + expect(docs.gets).toBe(0); + const warm = await second.listRecent({ workspaceIds: [workspaceId] }); + expect(warm.items.map((s) => s.id)).toEqual(['a', 'b', 'c']); + expect(docs.gets).toBe(0); + }); + + it('the resume-startup sequence pays one scan: point lookup, projection, then warm lists', async () => { + await seedSession('a', { title: 'a', createdAt: 1, updatedAt: 3 }); + await seedSession('b', { title: 'b', createdAt: 2, updatedAt: 2 }); + await seedSession('c', { title: 'c', createdAt: 3, updatedAt: 1 }); + + class CountingDocs extends JsonAtomicDocumentStore { + gets = 0; + override async get(scope: string, key: string): Promise { + this.gets += 1; + return super.get(scope, key); + } + } + const fileStorage = new FileStorageService(homeDir); + const docs = new CountingDocs(fileStorage); + const host = createScopedTestHost([ + stubPair(IFileSystemStorageService, fileStorage), + stubPair(IAtomicDocumentStore, docs), + stubPair(IBootstrapService, stubBootstrap(homeDir)), + stubPair(ILogService, stubLog()), + stubPair(IFlagService, stubFlag(true)), + ]); + disposeHost = () => { + host.dispose(); + }; + queryStore = host.app.accessor.get(IQueryStore); + mirror = host.app.accessor.get(ISessionIndexMirror); + const store = host.app.accessor.get(ISessionIndex) as FileSessionIndex; + + // `kimi --resume `: a targeted point lookup (2 document reads), which + // also kicks the initial projection in the background (one shared scan = + // 3 sessions × 2 gets). The TUI's later fetchSessions must then be warm. + expect((await store.get('b'))?.title).toBe('b'); + expect(await store.prepare()).toEqual({ state: 'ready', generation: 1, degradedCount: 0 }); + expect(docs.gets).toBe(8); + + // fetchSessions after the resume: served by the read model — zero + // additional document reads, zero directory scans. + const page = await store.listRecent({ workspaceIds: [workspaceId] }); + expect(page.items.map((s) => s.id)).toEqual(['a', 'b', 'c']); + expect(docs.gets).toBe(8); + }); + + it('the session query-store carries no full-text index artifacts', async () => { + await seedSession('a', { title: 'alpha', createdAt: 1, updatedAt: 2 }); + await seedSession('b', { title: 'beta', createdAt: 2, updatedAt: 3 }); + + const store = build(); + await store.prepare(); + mirror.record(summary('c', { title: 'gamma', createdAt: 3, updatedAt: 4 })); + await mirror.drain(); + expect(await store.count({ workspaceIds: [workspaceId] })).toBe(3); + + // The read model is a structural store: value documents, ordered columns + // and secondary indexes only. No shard may carry a text-index definition + // sidecar, a legacy root postings file, or generation text artifacts + // (dictionary/docs/postings). + const storeDir = join(homeDir, 'cache', 'query-store'); + const entries = await fsp.readdir(storeDir, { recursive: true, withFileTypes: true }); + const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name); + expect(files.length).toBeGreaterThan(0); + expect(files.filter((name) => name === 'db.textindexes.json')).toEqual([]); + expect(files.filter((name) => /^db\.text-.*\.postings$/.test(name))).toEqual([]); + expect(files.filter((name) => /^text-.*\.(dictionary|postings|docs)$/.test(name))).toEqual([]); + }); + // -- stage-3 performance baselines ------------------------------------------ // Medians are logged as JSON for phase-to-phase comparison, but no // wall-clock budget gates CI: shared-runner load can inflate any fixed diff --git a/packages/agent-core-v2/test/app/sessionIndex/stubs.ts b/packages/agent-core-v2/test/app/sessionIndex/stubs.ts index a4caf45a5..78defb83d 100644 --- a/packages/agent-core-v2/test/app/sessionIndex/stubs.ts +++ b/packages/agent-core-v2/test/app/sessionIndex/stubs.ts @@ -21,6 +21,7 @@ export function stubSessionIndexMirror(): ISessionIndexMirror & { recorded.push(summary); }, pending: () => recorded, + evict: async () => {}, drain: async () => {}, }; } diff --git a/packages/agent-core-v2/test/app/sessionLegacy/sessionLegacy.test.ts b/packages/agent-core-v2/test/app/sessionLegacy/sessionLegacy.test.ts index 2e0599673..507607914 100644 --- a/packages/agent-core-v2/test/app/sessionLegacy/sessionLegacy.test.ts +++ b/packages/agent-core-v2/test/app/sessionLegacy/sessionLegacy.test.ts @@ -81,6 +81,7 @@ function stubSessionChain(ix: TestInstantiationService, session: ISessionScopeHa _serviceBrand: undefined, record: () => {}, pending: () => [], + evict: () => Promise.resolve(), drain: () => Promise.resolve(), }); ix.stub(IWorkspaceLifecycleService, { diff --git a/packages/agent-core-v2/test/app/workspaceLifecycle/workspaceLifecycle.test.ts b/packages/agent-core-v2/test/app/workspaceLifecycle/workspaceLifecycle.test.ts index 8dfa2e820..d2bf1ce45 100644 --- a/packages/agent-core-v2/test/app/workspaceLifecycle/workspaceLifecycle.test.ts +++ b/packages/agent-core-v2/test/app/workspaceLifecycle/workspaceLifecycle.test.ts @@ -106,6 +106,7 @@ function sessionIndexMirrorStub(): ISessionIndexMirror { _serviceBrand: undefined, record: () => {}, pending: () => [], + evict: () => Promise.resolve(), drain: () => Promise.resolve(), }; } diff --git a/packages/agent-core-v2/test/persistence/backends/minidb/miniDbQueryStore.test.ts b/packages/agent-core-v2/test/persistence/backends/minidb/miniDbQueryStore.test.ts index 6b906dbd1..f71be91ef 100644 --- a/packages/agent-core-v2/test/persistence/backends/minidb/miniDbQueryStore.test.ts +++ b/packages/agent-core-v2/test/persistence/backends/minidb/miniDbQueryStore.test.ts @@ -9,7 +9,7 @@ import { createScopedTestHost, stubPair } from '#/_base/di/test'; import { ILogService } from '#/_base/log/log'; import { IBootstrapService } from '#/app/bootstrap/bootstrap'; import { ClusterDb } from '@moonshot-ai/minidb/cluster'; -import { MiniDbQueryStore } from '#/persistence/backends/minidb/miniDbQueryStore'; +import { drainQueryStoreDisposals, MiniDbQueryStore } from '#/persistence/backends/minidb/miniDbQueryStore'; import { IQueryStore } from '#/persistence/interface/queryStore'; import { stubBootstrap } from '../../../app/bootstrap/stubs'; import { stubLog } from '../../../_base/log/stubs'; @@ -36,6 +36,9 @@ describe('MiniDbQueryStore', () => { afterEach(async () => { disposeHost?.(); disposeHost = undefined; + // The host's synchronous dispose() fires the store's async close; await it + // so the rm below never races a late shard close (ENOTEMPTY flake). + await drainQueryStoreDisposals(); await fsp.rm(homeDir, { recursive: true, force: true }); }); @@ -113,18 +116,32 @@ describe('MiniDbQueryStore', () => { expect(page2.nextCursor).toBeUndefined(); }); - it('ensureIndex is idempotent across value, compound and text kinds', async () => { + it('ensureIndex is idempotent across value and compound kinds', async () => { const store = build(); await store.put(COLLECTION, 'a', { id: 'a', ws: 'x', n: 1, body: 'hello world' }); await store.ensureIndex(COLLECTION, { kind: 'value', name: 'byWs', field: 'ws' }); await store.ensureIndex(COLLECTION, { kind: 'value', name: 'byWs', field: 'ws' }); await store.ensureIndex(COLLECTION, { kind: 'compound', name: 'byWsN', groupBy: 'ws', orderBy: 'n' }); - await store.ensureIndex(COLLECTION, { kind: 'text', name: 'body', fields: ['body'] }); - await store.ensureIndex(COLLECTION, { kind: 'text', name: 'body', fields: ['body'] }); + await store.ensureIndex(COLLECTION, { kind: 'compound', name: 'byWsN', groupBy: 'ws', orderBy: 'n' }); const page = await store.query(COLLECTION).where({ ws: 'x' }).execute(); expect(page.items).toHaveLength(1); }); + it('rejects text indexes: the query-store is a structural read model', async () => { + const store = build(); + await store.put(COLLECTION, 'a', { id: 'a', body: 'hello world' }); + await expect( + store.ensureIndex(COLLECTION, { kind: 'text', name: 'body', fields: ['body'] }), + ).rejects.toThrow(/structural read model/); + // The rejected definition must not lodge a partial registration. + await expect( + store.ensureIndex(COLLECTION, { kind: 'text', name: 'body', fields: ['body'] }), + ).rejects.toThrow(/structural read model/); + const storeDir = join(homeDir, 'cache', 'query-store'); + const shardEntries = await fsp.readdir(join(storeDir, 'shard-00')); + expect(shardEntries.filter((name) => name.includes('text'))).toEqual([]); + }); + it('stores checkpoints', async () => { const store = build(); expect(await store.getCheckpoint('wire:abc')).toBeUndefined(); diff --git a/packages/agent-core-v2/test/session/sessionMetadata/sessionMetadata.test.ts b/packages/agent-core-v2/test/session/sessionMetadata/sessionMetadata.test.ts index f92e3b5d6..574eec152 100644 --- a/packages/agent-core-v2/test/session/sessionMetadata/sessionMetadata.test.ts +++ b/packages/agent-core-v2/test/session/sessionMetadata/sessionMetadata.test.ts @@ -119,6 +119,44 @@ describe('SessionMetadata', () => { expect(mirror.recorded[0]).toMatchObject({ id: 's1', archived: false }); }); + it('persists the authoritative document before recording to the mirror', async () => { + const store = ix.get(IAtomicDocumentStore); + // Read the persisted document back from inside record(): at that point + // the mutation must already be durable. + const persistedAtRecord: Promise | undefined>[] = []; + const baseRecord = mirror.record; + mirror.record = (summary) => { + persistedAtRecord.push(store.get>(META_SCOPE, 'state.json')); + baseRecord(summary); + }; + + const meta = ix.get(ISessionMetadata); + await meta.ready; // first-time creation records too + await meta.update({ title: 'durable-first' }); + + expect(persistedAtRecord).toHaveLength(2); + const [atCreate, atUpdate] = await Promise.all(persistedAtRecord); + expect(atCreate).toMatchObject({ id: 's1', archived: false }); + expect(atUpdate).toMatchObject({ title: 'durable-first' }); + }); + + it('a mirror failure degrades the read model but never fails the metadata mutation', async () => { + mirror.record = () => { + throw new Error('mirror down'); + }; + + const meta = ix.get(ISessionMetadata); + // The creation-time record throws inside load(); the load must survive. + await meta.ready; + await meta.update({ title: 'still fine' }); + expect(await meta.read()).toMatchObject({ title: 'still fine' }); + + // The mutation reached the authoritative document: a fresh instance reads + // it back even though every mirror record failed. + const fresh = createFreshMetadata(ix); + expect(await fresh.read()).toMatchObject({ title: 'still fine' }); + }); + it('persists across instances', async () => { const meta = ix.get(ISessionMetadata); await meta.update({ title: 'persisted' }); diff --git a/packages/agent-core-v2/test/setup.ts b/packages/agent-core-v2/test/setup.ts index a20398448..3fc219eeb 100644 --- a/packages/agent-core-v2/test/setup.ts +++ b/packages/agent-core-v2/test/setup.ts @@ -11,3 +11,11 @@ for (const key of Object.keys(process.env)) { delete process.env[key]; } } + +// `persistence_minidb_readmodel` defaults ON in production, but the shared +// harness boots full containers against a fixed homeDir — with the read +// model engaged, every container opens the same query-store minidb (teardown +// races, mirror flush timers under fake timers). Pin the legacy path here; +// read-model suites opt in with explicit flag service overrides +// (test/app/sessionIndex). +process.env['KIMI_CODE_EXPERIMENTAL_PERSISTENCE_MINIDB_READMODEL'] = 'false'; diff --git a/packages/agent-core-v2/test/workspace/sessionLifecycle/sessionLifecycle.test.ts b/packages/agent-core-v2/test/workspace/sessionLifecycle/sessionLifecycle.test.ts index 95780c60b..eb024c0eb 100644 --- a/packages/agent-core-v2/test/workspace/sessionLifecycle/sessionLifecycle.test.ts +++ b/packages/agent-core-v2/test/workspace/sessionLifecycle/sessionLifecycle.test.ts @@ -290,6 +290,7 @@ function sessionIndexMirrorStub(): ISessionIndexMirror { _serviceBrand: undefined, record: () => {}, pending: () => [], + evict: () => Promise.resolve(), drain: () => Promise.resolve(), }; } diff --git a/packages/kap-server/AGENTS.md b/packages/kap-server/AGENTS.md index 906b10c8c..131c15f2c 100644 --- a/packages/kap-server/AGENTS.md +++ b/packages/kap-server/AGENTS.md @@ -30,8 +30,11 @@ Implements the op-batch sequencing contract: The global search surface is `POST /api/v1/search` (`src/search/` + `src/routes/search.ts`): a cross-session full-text search over user messages, assistant text, and session titles, backed by a single minidb database at `/search-index` (`IGlobalSearchService`, App scope — the write-lock holder is the indexer, other processes open read-only and catch up via WAL). +- Everything that touches the search-index MiniDb lives in the host-agnostic core (`src/search/indexCore.ts`): open/reopen (writer election, corruption rebuild), read-only freshness (fingerprint + WAL catch-up), the incremental sync pass, the bounded index-route query execution, reindex, and close. It runs in one of two hosts behind the `SearchBackend` seam: a dedicated **worker thread** (`src/search/worker/` — `host.ts` RPC client + `entry.ts` worker entry + `protocol.ts` contract; the default) or the **inline** in-process host (the rollback). The `search_worker` experimental flag (`KIMI_CODE_EXPERIMENTAL_SEARCH_WORKER`, default ON) selects the host at service construction. The main thread keeps only query normalization, page-token encode/decode, the sync coordinator, the live route, and hit projection; worker failure/crash surfaces as recognizable `building`/`degraded` pages — never an implicit inline fallback. Host lifecycle guards: the worker reports the write-lock token via a `lockToken` event the moment the lock is acquired (`OpenOptions.onLockAcquired`), so a mid-open crash leaves a reapable lock; an orphan detector reaps a same-pid lock line whose token belongs to no live worker of the process and restarts (never a silent permanent read-only); every RPC carries a watchdog timeout (a wedged worker is terminated and restarted with backoff); `beginClose` reaches the worker-side core via a control message so dispose never wedges behind a running sync; and the token-pinning generation is salted per worker boot, so page tokens never validate across a worker restart. - It serves two modes: `terms` (the default — minidb's inverted text index over ASCII words + CJK uni/bigrams, no positions, term-level AND) and `literal` (substring-exact search: a hashed 2/3-gram index supplies candidates, every candidate's text is then confirmed with `includes`, so hits carry zero false positives; literal ignores `sort` and returns newest-first). - The index route is fully bounded (stage 4): a search request serves the currently published generation and never awaits a sync/reopen/reindex — it kicks the single-flight + debounced background coordinator instead, and reports `index_state.stale` / `index_state.degraded` when serving a behind view or after a failed refresh, and `index_state.state: 'building'` while the served handle's text base is still being (re)built by the deferred fallback build (searches get the empty building page, never a partial result). -- Every query runs under explicit budgets (max terms, postings visits via `MiniDb.searchBoundedAsync`, candidate caps, confirmation text volume, a match deadline) with over-budget pages flagged `incomplete: 'candidate_cap' | 'postings_budget' | 'deadline'`. +- The aggregate lifecycle (stage 5) — `stopped → opening → ready → building/degraded → closing` — is diagnosable two ways: `IGlobalSearchService.status()` keeps its historical semantics (may kick/await the open + read-only refresh) but never throws — a failed open/worker answers `lifecycle.state: 'degraded'` with the error as detail — and `lifecycleReport()` is the synchronous local read that never kicks an open or spawns the worker, so it still answers during a minutes-long first open or a worker backoff (both surface on `/api/v1/debug/globalSearch/*`). A corruption-triggered rebuild is its own logged outcome (`search-index corruption detected; rebuilding from scratch`), distinct from building/degraded lines. +- Every query runs under explicit budgets (max terms, postings visits via `MiniDb.searchBoundedAsync`, candidate caps, confirmation text volume, a match deadline) with over-budget pages flagged `incomplete: 'candidate_cap' | 'postings_budget' | 'deadline'`. The budget VALUES are centralized and commented: per-query work budgets at the top of `src/search/searchService.ts` (mirrored as service test knobs), worker-host budgets at the top of `src/search/worker/host.ts` (ready/close/request/sync watchdogs, heap cap, crash backoff), minidb's process-wide worker slots and slice budgets in `packages/minidb/src/maintenance.ts` / `recovery.ts`. Query concurrency is structural, not a semaphore: one worker thread per service serializes all index-route CPU work, each query is individually budgeted and watchdogged, and the worker heap cap turns a pathological burst into a degraded-restart rather than a main-process stall. - Pagination is keyset over `(time, key)` / `(score, time, key)` with versioned v2 tokens pinning the index generation (a rebuild/reopen/rescan invalidates old tokens with `invalid_page_token`; legacy v1 offset tokens are still accepted and upgraded), and per-session sync scans only that session's file-meta keys (`\0meta\file\\`, migrated from the pre-v2 hash-only keys by a one-time background pass). - When `container.session_id` is provided and that session is live in this process (`TranscriptService.forSessionLive` returns a store, wired via `setLiveTranscriptSource` in `start.ts`), BOTH modes instead scan the in-memory transcript store (turn prompts + assistant text frames, history established via `whenReady`/`ensureAgentHistory`) — no index involved; terms-mode live hits are scored Σ log(1+tf) (comparable only within a route, per the `GlobalSearchSource` contract), live-route errors never fall back to the index, and the response's `source: 'live' | 'index'` field (also mixed into the page-token fingerprint, so a mid-pagination route flip invalidates the old token) tells the caller which route served the page. +- Dev/test worker runtime: the worker entry (`src/search/worker/entry.ts`) loads from TS source under Node's native type stripping plus a repo-local `.js`→`.ts` resolve hook (`src/search/worker/register-dev-hooks.mjs`); the npm bundle emits `dist/search-worker.mjs` (`apps/kimi-code/tsdown.dist-worker.config.ts`); the SEA binary embeds it as the `kap-search-worker` runtime asset (`apps/kimi-code/scripts/native/manifest.mjs`), installed at startup by `apps/kimi-code/src/native/search-worker.ts`. diff --git a/packages/kap-server/package.json b/packages/kap-server/package.json index e2b869539..bf6ff7fab 100644 --- a/packages/kap-server/package.json +++ b/packages/kap-server/package.json @@ -12,6 +12,10 @@ "./contract": { "types": "./src/contract.ts", "default": "./src/contract.ts" + }, + "./search-worker-runtime": { + "types": "./src/search/worker/runtime.ts", + "default": "./src/search/worker/runtime.ts" } }, "scripts": { diff --git a/packages/kap-server/src/search/contract.ts b/packages/kap-server/src/search/contract.ts index 93388b82e..cc563b5a8 100644 --- a/packages/kap-server/src/search/contract.ts +++ b/packages/kap-server/src/search/contract.ts @@ -72,6 +72,29 @@ export interface GlobalSearchQuery { readonly pageToken?: string; } +// ---- errors ---------------------------------------------------------------- + +export type GlobalSearchErrorReason = + | 'invalid_query' + | 'invalid_page_token' + | 'readonly_index' + | 'index_unavailable'; + +/** + * Service-level error with a machine-readable reason. Lives in the contract + * (not the service module) so the search-index core — which also runs inside + * the search worker thread — can raise it without importing the service. + */ +export class GlobalSearchError extends Error { + constructor( + readonly reason: GlobalSearchErrorReason, + message: string, + ) { + super(message); + this.name = 'GlobalSearchError'; + } +} + // ---- response -------------------------------------------------------------- export interface GlobalSearchHit { diff --git a/packages/kap-server/src/search/docs.ts b/packages/kap-server/src/search/docs.ts new file mode 100644 index 000000000..28d90fc07 --- /dev/null +++ b/packages/kap-server/src/search/docs.ts @@ -0,0 +1,122 @@ +/** + * `search` module — stored document shapes of the `/search-index` + * minidb database (pure types, no runtime imports). + * + * Extracted from `searchService.ts` so both the main-process service (hit + * assembly, live route) and the host-agnostic index core (`indexCore.ts` — + * which also runs inside the search worker thread) share one definition. + * The worker entry's import closure is loaded under Node's native type + * stripping, so every RELATIVE import in that closure uses an explicit `.ts` + * specifier. + */ + +/** Cap one indexed document's text so huge pastes do not bloat the index. */ +export const MAX_DOC_TEXT_CHARS = 20_000; + +export interface MessageDoc { + readonly kind: 'message'; + readonly sessionId: string; + readonly workspaceId: string; + readonly sessionTitle: string; + readonly agentId: string; + readonly role: 'user' | 'assistant'; + readonly text: string; + readonly time: number; + /** + * 0-based turn ordinal in the transcript view (groupTurns numbering). Absent + * for docs indexed before turn tracking existed. + */ + readonly turn?: number; + /** + * Transcript step id (`t.`, engine live numbering from the wire + * record's `step` field) of the step that produced this assistant text. + * Absent for user docs and docs indexed before step tracking existed. + */ + readonly stepId?: string; +} + +export interface TitleDoc { + readonly kind: 'title'; + readonly sessionId: string; + readonly workspaceId: string; + readonly sessionTitle: string; + /** Titles belong to the session, not an agent — always ''. */ + readonly agentId: ''; + readonly role: 'title'; + readonly text: string; + readonly time: number; +} + +// -- turn/step counter states, persisted on file metas for resume ------------ + +export interface TurnOpener { + readonly turn: number; + readonly anchor: boolean; +} + +export interface TurnCounterState { + /** Ordinal the next opened turn will get (0-based). */ + readonly next: number; + /** Whether a turn is currently open (groupTurns' `ensureTurn` gate). */ + readonly hasTurn: boolean; + /** Turn openers, in order — the replay stack for `context.undo`. */ + readonly openers: readonly TurnOpener[]; +} + +export interface StepTrackerState { + /** Current turn's step uuid → ordinal (the wire `step` field, else the fallback counter). */ + readonly byUuid: Record; + /** `step.begin` count within the current turn — the fallback ordinal source. */ + readonly begins: number; +} + +export interface FileMetaDoc { + readonly kind: 'fileMeta'; + /** Owning session, used to drop metas when a session disappears. */ + readonly sessionId: string; + /** Doc-key coordinates of this file's documents (see `docKeyPrefix`). */ + readonly agentId: string; + readonly source: 'root' | 'agents'; + /** Absolute wire path (debugging aid; the key is its session + hash). */ + readonly path: string; + /** Byte offset up to which the wire file has been indexed. */ + readonly offset: number; + readonly size: number; + /** + * File mtime/inode at the last sync pass. A changed inode (atomic + * replacement) or a bumped mtime at an unchanged size (in-place rewrite) + * forces a rescan even when `size === offset`. Absent in metas written + * before change tracking — such metas are simply refreshed with the + * current stat on the next pass, without a rescan. + */ + readonly mtimeMs?: number; + readonly ino?: number; + /** + * Turn counter state at `offset` — persisted with the watermark so an + * incremental pass resumes counting instead of restarting at turn 0. + * Absent in metas written before turn tracking; treated as the initial + * state, which makes a legacy meta resume mid-file with a zeroed counter — + * an accepted one-time drift, self-healing on the next shrink/rescan. + */ + readonly turnState?: TurnCounterState; + /** + * Step tracker state at `offset` — persisted with the watermark for the + * same resume reason as `turnState`. Absent in metas written before step + * tracking: such a file is RESCANNED from scratch (docs dropped, offset + * reset) so stepIds are all-or-nothing per file instead of drifting. + */ + readonly stepState?: StepTrackerState; +} + +export interface SessionMetaDoc { + readonly kind: 'sessionMeta'; +} + +export interface StatsDoc { + readonly kind: 'stats'; + readonly sessions: number; + readonly documents: number; + readonly lastIndexedAt: number; +} + +export type SearchDoc = MessageDoc | TitleDoc | FileMetaDoc | SessionMetaDoc | StatsDoc; diff --git a/packages/kap-server/src/search/indexCore.ts b/packages/kap-server/src/search/indexCore.ts new file mode 100644 index 000000000..6b4b2e83c --- /dev/null +++ b/packages/kap-server/src/search/indexCore.ts @@ -0,0 +1,1425 @@ +/** + * `search` module — the host-agnostic search-index core (stage 4 worker + * isolation). + * + * Everything that touches the `/search-index` MiniDb lives here: + * open/reopen (writer election via the write lock, corruption rebuild), the + * read-only freshness machinery (fingerprint + WAL catch-up), the + * incremental sync pass (wire-file scanning, doc projection, stats), the + * bounded index-route query execution (candidate search + match/confirm + + * keyset pagination), reindex, and close. + * + * The core runs in TWO hosts with identical semantics: + * - INLINE host (rollback switch): the main process instantiates it + * directly (`InlineSearchBackend` in searchService.ts) — the pre-stage-4 + * behavior; + * - WORKER host (default): `worker/entry.ts` instantiates it inside a + * dedicated worker thread and serves it over the RPC protocol + * (`worker/protocol.ts`), so generation loads, WAL replays, syncs and + * query execution never share the TUI/HTTP main thread's event loop. + * + * Because the worker entry is loaded under Node's native type stripping, + * every RELATIVE import in this file (and its whole closure) uses an + * explicit `.ts` specifier, and no decorators / non-erasable TS syntax are + * allowed in the closure. + * + * The concurrency comments carried over from searchService.ts ("the lock is + * the election", "requests serve a published generation, never wait") all + * apply per-host: in the worker host the interleaving between a background + * sync/refresh and a query is the same single-threaded interleaving the + * pre-worker service had on the main thread. + */ + +import { createHash } from 'node:crypto'; +import { open, readFile, readdir, rm, stat } from 'node:fs/promises'; +import { join, relative } from 'node:path'; + +import { + LockError, + MiniDb, + OpTracker, + TextIndexBuildingError, + type BatchInputOp, +} from '@moonshot-ai/minidb'; + +import { GlobalSearchError, type GlobalSearchIncomplete } from './contract.ts'; +import { + MAX_DOC_TEXT_CHARS, + type FileMetaDoc, + type MessageDoc, + type SearchDoc, + type SessionMetaDoc, + type StatsDoc, + type StepTrackerState, + type TitleDoc, + type TurnCounterState, +} from './docs.ts'; +import { + decodePageToken, + matchDocs, + paginateRows, + type MatchBudget, + type MatchedRow, + type NormalizedQuery, + type SearchBudgets, +} from './match.ts'; +import { analyzeWireLine, type StepEffect, type TurnEffect } from './wireExtract.ts'; + +// --------------------------------------------------------------------------- +// Constants & key namespaces (moved verbatim from the pre-worker service) +// --------------------------------------------------------------------------- + +const TEXT_INDEX_NAME = 'body'; +/** n-gram substring index backing literal mode, alongside 'body'. */ +const TRI_INDEX_NAME = 'tri'; +const WIRE_FILENAME = 'wire.jsonl'; + +/** Key namespaces inside the single db. */ +const FILE_META_PREFIX = '\0meta\\file\\'; +const SESSION_META_PREFIX = '\0meta\\session\\'; +const STATS_KEY = '\0meta\\stats'; + +function hashPath(filePath: string): string { + return createHash('sha256').update(filePath).digest('hex').slice(0, 32); +} + +/** + * minidb keys are limited to 128 bytes, far shorter than an absolute wire + * path — the file meta key carries the owning session id plus a hash of the + * path (the path itself lives in the value). The session segment makes a + * per-session prefix scan (`fileMetaPrefixFor`) touch only that session's + * metas instead of the global meta namespace. + */ +function fileMetaKey(sessionId: string, filePath: string): string { + return `${FILE_META_PREFIX}${sessionId}\\${hashPath(filePath)}`; +} + +/** All file-meta keys of one session (prefix-scan argument). */ +function fileMetaPrefixFor(sessionId: string): string { + return `${FILE_META_PREFIX}${sessionId}\\`; +} + +/** + * Pre-v2 file-meta key: hash-only, the owning session identifiable only via + * the value — a per-session lookup required scanning every file meta. + * Read side of the migration: `syncWireFile` still resolves it by point + * lookup; `migrateFileMetaKeys` rewrites the rest in one background pass. + */ +function legacyFileMetaKey(filePath: string): string { + return FILE_META_PREFIX + hashPath(filePath); +} + +/** One wire-delta read slice: growth is consumed in bounded chunks instead + * of one `size - offset` allocation. */ +const WIRE_READ_CHUNK_BYTES = 1 << 20; +/** Flush doc ops to the db in batches of this size while scanning a delta. */ +const WIRE_BATCH_OPS = 1_000; +const EMPTY_BUFFER = Buffer.alloc(0); + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +// --------------------------------------------------------------------------- +// Turn counter / step tracker (replay over the wire file; states in docs.ts) +// --------------------------------------------------------------------------- + +const INITIAL_TURN_STATE: TurnCounterState = { next: 0, hasTurn: false, openers: [] }; + +function initialTurnState(): TurnCounterState { + return INITIAL_TURN_STATE; +} + +/** + * Replay `context.undo {count}`: drop the last `count` anchor-opened turns. + * The counter rewinds to the ordinal of the earliest dropped anchor, and the + * opener stack is truncated there. An undo with fewer anchors than `count` + * never reaches the wire (the engine's precheck rejects it) — left untouched. + */ +function applyUndoToTurnState(state: TurnCounterState, count: number): TurnCounterState { + let found = 0; + for (let i = state.openers.length - 1; i >= 0; i--) { + if (state.openers[i]!.anchor) { + found++; + if (found === count) { + return { + next: state.openers[i]!.turn, + hasTurn: i > 0, + openers: state.openers.slice(0, i), + }; + } + } + } + return state; +} + +/** + * Advance the counter with one record's turn effect. Returns the ordinal that + * documents extracted from the SAME record belong to: a user opener carries + * the turn it opens; assistant content carries the current turn (after the + * `ensure` gate). Undefined when the record owns no turn. + * + * The counter is monotonic except for `undo` rewinds: `apply_compaction` and + * `clear` do NOT renumber (the transcript's cold replay keeps full history + * and groupTurns numbers it continuously; the live TurnModel is monotonic + * too), so they are `none` effects by construction. Docs indexed BEFORE an + * `undo` keep their pre-undo ordinals — those messages no longer exist in the + * transcript view, so their ordinals point nowhere (known, accepted + * deviation, same class as "folded-away messages stay searchable"). + */ +function advanceTurnCounter( + state: TurnCounterState, + effect: TurnEffect, +): { docTurn: number | undefined; state: TurnCounterState } { + switch (effect.kind) { + case 'open': + return { + docTurn: state.next, + state: { + next: state.next + 1, + hasTurn: true, + openers: [...state.openers, { turn: state.next, anchor: effect.anchor }], + }, + }; + case 'ensure': { + const next = state.hasTurn ? state : { ...state, next: state.next + 1, hasTurn: true }; + return { docTurn: next.next - 1, state: next }; + } + case 'undo': + return { docTurn: undefined, state: applyUndoToTurnState(state, effect.count) }; + case 'none': + return { docTurn: undefined, state }; + } +} + +const INITIAL_STEP_STATE: StepTrackerState = { byUuid: {}, begins: 0 }; + +function initialStepState(): StepTrackerState { + return INITIAL_STEP_STATE; +} + +/** + * Advance the tracker with one record's step effect. `begin` maps the step's + * uuid to its ordinal: the wire record's own `step` field when present (the + * engine's live 1-based numbering — the same numbering transcript step ids + * use), otherwise the count of begins seen in this turn (v1 loops had no + * loop-level retries, so counting matches the surviving-step numbering). + * The mapping is never narrowed per step — it is reset wholesale at turn + * boundaries (`open`, a fallback-opening `ensure`, `undo`) by the caller. + */ +function advanceStepTracker(state: StepTrackerState, effect: StepEffect): StepTrackerState { + if (effect.kind !== 'begin') return state; + const begins = state.begins + 1; + const ordinal = effect.ordinal ?? begins; + if (state.byUuid[effect.uuid] === ordinal) return state; + return { byUuid: { ...state.byUuid, [effect.uuid]: ordinal }, begins }; +} + +// --------------------------------------------------------------------------- +// Core options & wire types +// --------------------------------------------------------------------------- + +/** Minimal logger surface the core needs (the worker forwards these over RPC). */ +export interface SearchCoreLog { + info(message: string, meta?: Record): void; + warn(message: string, meta?: Record): void; +} + +export interface SearchCoreOptions { + /** Absolute path of the search-index database directory. */ + readonly indexDir: string; + readonly log: SearchCoreLog; + /** + * Unique-per-host-boot salt mixed into the token-pinning generation: a + * worker/process restart resets the local generation counter, and the + * salt makes tokens issued before the restart fail validation instead of + * colliding with the fresh counter (see `tokenGeneration`). + */ + readonly bootSalt: string; + /** + * Fired synchronously when an open acquires the write lock — BEFORE the + * heavy recovery work runs. The worker entry forwards it to the host so a + * mid-open crash leaves a reapable lock. + */ + readonly onLockToken?: (token: string) => void; +} + +/** + * One session to index, with its persistence directory PRE-RESOLVED by the + * caller (the main process owns `sessionDirOf`/`workspacePersistenceScope`; + * the worker closure deliberately does not import agent-core-v2). + */ +export interface SyncSessionInput { + readonly id: string; + readonly workspaceId: string; + readonly title?: string; + readonly updatedAt: number; + /** Absolute session directory (the parent of wire.jsonl / agents/). */ + readonly dir: string; +} + +/** The core's view of the served index, embedded in every search response. */ +export interface CoreIndexView { + readonly state: 'building' | 'ready' | 'readonly'; + readonly indexedSessions: number; + readonly documents: number; + readonly readOnly: boolean; + /** + * Read-only-branch staleness: the on-disk fingerprint changed (a refresh + * is pending) or a refresh is in flight. Writer-side staleness is the + * caller's coordinator state and is OR'ed in by the service. + */ + readonly freshnessStale: boolean; + /** Last background refresh failure, when serving a stale view. */ + readonly degraded?: string; + /** + * Token of the db.lock line this core published (writer only) — the main + * process uses it to reap the lock file after a worker crash without ever + * deleting another owner's lock. + */ + readonly lockToken?: string; +} + +export type CoreSearchResult = + | { + readonly kind: 'page'; + readonly rows: MatchedRow[]; + readonly hasMore: boolean; + readonly incomplete?: GlobalSearchIncomplete; + /** Token-pinning generation (`bootSalt:counter`) — see tokenGeneration. */ + readonly generation: string; + readonly index: CoreIndexView; + } + | { readonly kind: 'building'; readonly index: CoreIndexView }; + +export interface CoreSearchParams { + readonly q: NormalizedQuery; + /** Raw opaque token from the request; decoded (and generation-checked) here. */ + readonly pageToken?: string; + readonly budgets: SearchBudgets; +} + +export interface CoreSyncOutcome { + /** True when the pass did not run (no db yet, or a read-only instance). */ + readonly noop: boolean; + readonly sessions: number; + readonly documents: number; + readonly lockToken?: string; + /** Post-pass lifecycle snapshot (stage 5) — keeps the host's cached + * aggregate state exact on the sync-first path, where no search/status + * response ever carries it. */ + readonly lifecycle: CoreLifecycleReport; +} + +/** The pass body's return — the wrapper adds the lock token and lifecycle. */ +type CoreSyncPassOutcome = Omit; + +/** + * The aggregate lifecycle of the served index (stage 5): the diagnostic state + * machine behind the service's status surface — + * `stopped → opening → ready → building/degraded → closing`. Distinct from + * the per-page `CoreIndexView.state` (which answers "can this page serve hits + * now"): the lifecycle also covers the no-db phases and carries the failure + * detail, so logs/diagnostics can tell building, stale-serving, degraded, + * corrupt-rebuild and worker-unavailable apart. + */ +export type CoreLifecycleState = + /** Never opened, or fully closed. */ + | 'stopped' + /** An open/reopen is in flight (generation load / WAL replay / rebuild). */ + | 'opening' + /** A db is published but its text base is still (re)building. */ + | 'building' + /** A published generation is serving (writer or read-only). A base that + * keeps serving after a failed background refresh stays 'ready' — the + * failure rides the `degraded` message field of the page/status surfaces. */ + | 'ready' + /** No base is serving: the last open failed, or (worker host) the worker + * is down/backing off. The detail carries the error. */ + | 'degraded' + /** Close has started; in-flight ops are draining. */ + | 'closing'; + +export interface CoreLifecycleReport { + readonly state: CoreLifecycleState; + readonly detail?: string; +} + +export interface CoreStatus { + readonly sessions: number; + readonly documents: number; + readonly lastIndexedAt: number | null; + readonly generation: number; + readonly readOnly: boolean; + readonly lockToken?: string; + readonly degraded?: string; + /** Post-open/post-refresh lifecycle snapshot (stage 5). */ + readonly lifecycle: CoreLifecycleReport; +} + +// --------------------------------------------------------------------------- +// The core +// --------------------------------------------------------------------------- + +export class SearchIndexCore { + /** WAL watermark (bytes applied) for read-only catch-up. */ + private walOffset = 0; + private fingerprint = ''; + private disposed = false; + /** + * Close-gate + drain for lifecycle-managed background ops (sync passes and + * read-only refreshes): close() closes the gate (new ops skip) and drains + * the in-flight ones BEFORE closing the db, so no background task ever + * touches a closed handle. + */ + private readonly ops = new OpTracker(); + /** + * Identity of the published index base: bumped on every open/reopen + * (initial open, read-only swap, reindex) and on a sync pass that REPLACED + * already-indexed documents (shrink rescan, title overwrite). Page tokens + * pin it; additive/deletion-only passes deliberately keep it stable so + * keyset pagination over a live index is not constantly restarted (see + * contract.ts for the weak-consistency semantics). + */ + private generation = 0; + /** Set by a sync pass when it replaced indexed documents → generation bump. */ + private syncReplaced = false; + /** Last background refresh failure — surfaced as degraded. */ + private lastRefreshError: { at: number; message: string } | null = null; + /** Last open failure — a search with no published generation fails fast. */ + private openError: string | null = null; + /** One-time per-process migration flag for pre-v2 file-meta keys. */ + private fileMetaMigrated = false; + /** Token of the published db.lock line (writer only) — see CoreIndexView. */ + private lockToken: string | undefined; + + // Visible to the inline host and to tests (the worker host never touches + // these directly — it talks RPC). + db: MiniDb | null = null; + openPromise: Promise | null = null; + refreshPromise: Promise | null = null; + fullSyncDone = false; + + constructor(private readonly options: SearchCoreOptions) {} + + /** The published db.lock token (writer only) — see CoreIndexView.lockToken. */ + get lockTokenView(): string | undefined { + return this.lockToken; + } + + private get indexDir(): string { + return this.options.indexDir; + } + + private get log(): SearchCoreLog { + return this.options.log; + } + + // -- lifecycle --------------------------------------------------------------- + + ensureOpen(): Promise { + this.openPromise ??= this.openDb().then( + () => { + this.openError = null; + }, + (error: unknown) => { + this.openPromise = null; + this.openError = errorMessage(error); + throw error; + }, + ); + return this.openPromise; + } + + private async openDb(): Promise { + const db = await this.openSearchDb(); + // The host may have been closed while the (slow) open was in flight — + // close the handle immediately instead of leaking it and writing the + // text-index definition below into a directory the caller may already be + // deleting. + if (this.disposed) { + await db.close().catch(() => {}); + throw new GlobalSearchError('index_unavailable', 'search service is disposed'); + } + await this.publishDb(db, null); + } + + /** + * The generation page tokens pin: `:`. The local + * counter alone restarts from 0 when the worker (or, pre-workerization, + * the process) respawns — a token issued before the restart (say g=1) + * would otherwise validate against the fresh host's g=1 and silently + * paginate a different base. The boot salt makes every pre-restart token + * fail with `invalid_page_token` so the client restarts the search. + */ + private tokenGeneration(): string { + return `${this.options.bootSalt}:${this.generation}`; + } + + /** + * Swap a freshly opened db in as the new published generation: writer-side + * text-index definitions and the (handle-independent) fingerprint are + * computed BEFORE the swap, so a failure closes `next` and leaves `prev` + * (or the no-db state) untouched; the swap itself is one synchronous + * segment with no failure point between publishing `next` and closing + * `prev`. + */ + private async publishDb(next: MiniDb, prev: MiniDb | null): Promise { + let fingerprint: string; + try { + if (!next.readOnly) { + // Both indexes are created here (not at first write) so a + // pre-existing db gets the tri index built over its current documents + // on first open after the upgrade, and a read-only peer only ever + // reopens on the definitions-file fingerprint change. + for (const [name, options] of [ + [TEXT_INDEX_NAME, { fields: ['text'] }], + [TRI_INDEX_NAME, { fields: ['text'], tokenizer: 'ngram' }], + ] as const) { + try { + await next.createTextIndex(name, options); + } catch (error) { + if (!(error instanceof Error && error.message.includes('already exists'))) throw error; + } + } + } + fingerprint = await this.computeFingerprint(); + } catch (error) { + await next.close().catch(() => {}); + throw error; + } + this.db = next; + this.walOffset = next.recoveryInfo?.walScanEnd ?? 0; + this.generation++; + this.fingerprint = fingerprint; + this.lockToken = next.readOnly ? undefined : await this.readLockToken(); + if (prev !== null) await prev.close().catch(() => {}); + // The lifecycle read model answers from logs alone which database this + // surface serves and which path its open took (a published-generation + // attach vs a full recovery), and how expensive the open was. + const lifecycle = next.lifecycleStatus(); + this.log.info('global search: index opened', { + dir: this.indexDir, + readOnly: next.readOnly, + state: lifecycle.state, + generation: next.getIndexGeneration()?.id ?? null, + openMs: Math.round(lifecycle.phases.openMs), + fullRecoveryMs: Math.round(lifecycle.phases.fullRecoveryMs), + }); + } + + /** + * The db.lock token this core published, so the main process can reap the + * lock after a worker crash. Read from the lock file (the LockFile + * instance's token is instance-private by design); anything unreadable or + * foreign-owned yields undefined and the reaper stays conservative. + */ + private async readLockToken(): Promise { + try { + const raw = await readFile(join(this.indexDir, 'db.lock'), 'utf8'); + const parsed = JSON.parse(raw) as { pid?: unknown; token?: unknown }; + if (parsed.pid !== process.pid || typeof parsed.token !== 'string') return undefined; + return parsed.token; + } catch { + return undefined; + } + } + + /** + * Open the index db, rebuilding from scratch on unrecoverable corruption + * (the index is derived data — never repaired, only rebuilt). + * + * Rebuild is WRITER-ONLY: a process that fails to grab the write lock must + * never delete the directory out from under the live indexer. Lock state is + * not observable once `open` throws, so corruption is disambiguated with a + * probe open WITHOUT `onLockFail`: it throws `LockError` before recovery + * when another process holds the lock, and re-throws the corruption + * (releasing the lock) when the lock is free — in which case this process + * is the would-be writer and may rebuild. + */ + private async openSearchDb(): Promise> { + const opts = { + dir: this.indexDir, + valueCodec: 'json', + fsyncPolicy: 'everysec', + onLockFail: 'readonly', + onLockAcquired: (info: { readonly token: string }) => { + // The lock is held from this instant; publish the token immediately + // so a mid-open crash leaves a reapable lock (the token read back at + // publishDb remains the final authority). + this.lockToken = info.token; + this.options.onLockToken?.(info.token); + }, + } as const; + try { + return await MiniDb.open(opts); + } catch (error) { + if (!isRebuildableCorruption(error)) throw error; + let probeError: unknown; + try { + const probe = await MiniDb.open({ dir: opts.dir, valueCodec: opts.valueCodec }); + await probe.close().catch(() => {}); + probeError = undefined; // lock free AND data fine — cannot happen, but treat as rebuildable + } catch (error) { + probeError = error; + } + if (probeError instanceof LockError) { + // Another process holds the write lock: leave its files alone. The + // caller's open fails; the next search retries from scratch. + throw error; + } + // The index is derived data — never repaired, only rebuilt. The rebuild + // destroys the previous index files, so the corruption that justified it + // must be visible in the logs (stage 5: 'corrupt' is a distinguishable + // diagnostic outcome, separate from building/degraded). + this.log.warn('global search: search-index corruption detected; rebuilding from scratch', { + dir: this.indexDir, + error: errorMessage(error), + }); + await rm(this.indexDir, { recursive: true, force: true }); + return MiniDb.open(opts); + } + } + + /** + * Synchronously mark the core closed: in-flight sync/refresh passes skip + * their remaining work at the next `disposed` check, and no new background + * work starts. The async close() then drains and releases the handle. The + * split exists so a host's synchronous dispose() can gate mid-pass writes + * BEFORE awaiting anything (the pre-worker service did this with its + * synchronous `disposed = true`). + */ + beginClose(): void { + this.disposed = true; + } + + /** + * Close the core: close the op gate (new syncs / refreshes skip at + * enter()), wait for every in-flight op and the in-flight open to settle, + * then release and close the handle — no background task can touch a + * closed db. + */ + async close(): Promise { + this.disposed = true; + await this.ops.close(); + await this.openPromise?.catch(() => {}); + const db = this.db; + this.db = null; + if (db) await db.close().catch(() => {}); + } + + /** + * Run one lifecycle-managed background op under the close drain gate: + * skipped once close has started, and close waits for every op that + * already entered before it closes the db. + */ + private async tracked(op: () => Promise): Promise { + if (!this.ops.enter()) return; + try { + await op(); + } finally { + this.ops.leave(); + } + } + + // -- read-only freshness (fingerprint + WAL catch-up) ------------------------- + + private async computeFingerprint(): Promise { + const parts: string[] = []; + for (const name of ['db.wal', 'db.snapshot', 'db.textindexes.json']) { + try { + const s = await stat(join(this.indexDir, name)); + parts.push(`${name}:${s.dev}:${s.ino}:${s.mtimeMs}:${s.size}`); + } catch { + parts.push(`${name}:-`); + } + } + return parts.join('|'); + } + + /** + * Bring a read-only instance up to date with the indexer's committed + * writes. Unchanged fingerprint → zero IO; WAL pure-append → incremental + * `catchUpFromWal`; anything else → open the replacement db and swap (which + * may also promote this process to indexer when the old writer's lock is + * gone). Single-flight; a failure is recorded in `lastRefreshError` and + * the stale generation keeps serving (surfaced as `indexState.degraded`). + */ + refresh(): Promise { + this.refreshPromise ??= this.tracked(() => this.doRefreshReadonly()) + .then( + () => { + this.lastRefreshError = null; + }, + (error: unknown) => { + // A failed refresh must not fail the search — serve the stale view, + // but no longer swallow the error silently. + this.lastRefreshError = { at: Date.now(), message: errorMessage(error) }; + this.log.warn('global search: read-only refresh failed; serving the stale view', { + error: errorMessage(error), + }); + }, + ) + .finally(() => { + this.refreshPromise = null; + }); + return this.refreshPromise; + } + + private async doRefreshReadonly(): Promise { + const db = this.db; + if (!db || !db.readOnly || this.disposed) return; + const fp = await this.computeFingerprint(); + if (fp === this.fingerprint) return; + const [, snapPrev, defsPrev] = this.fingerprint.split('|'); + const [, snapNow, defsNow] = fp.split('|'); + if (snapPrev === snapNow && defsPrev === defsNow) { + const res = await db.catchUpFromWal(this.walOffset); + if (res !== null) { + this.walOffset = res.offset; + this.fingerprint = fp; + return; + } + } + // WAL rotated/truncated, snapshot or index definitions changed, or the + // watermark no longer aligns: reopen from scratch. The replacement is + // opened and published BEFORE the stale handle closes, so a failed + // reopen leaves the previous generation servable instead of dropping + // the index out from under in-flight searches. + const next = await this.openSearchDb(); + if (this.disposed) { + await next.close().catch(() => {}); + return; + } + if (this.db !== db) { + // A concurrent refresh already swapped: just close the duplicate. + await next.close().catch(() => {}); + return; + } + await this.publishDb(next, db); + } + + // -- sync pass (indexer only) ------------------------------------------------- + // + // The caller (the service's coordinator) owns debounce/single-flight and + // enumerates the sessions; the pass itself — wire-file scanning, doc + // projection, stats — runs here. A sync that REPLACED indexed documents + // bumps the generation, invalidating older page tokens. + + async sync(sessions: readonly SyncSessionInput[]): Promise { + let outcome: CoreSyncPassOutcome = { noop: true, sessions: 0, documents: 0 }; + await this.tracked(async () => { + outcome = await this.runSync(sessions); + }); + return { ...outcome, lockToken: this.lockToken, lifecycle: this.lifecycleState() }; + } + + private async runSync(sessions: readonly SyncSessionInput[]): Promise { + if (this.disposed) return { noop: true, sessions: 0, documents: 0 }; + this.syncReplaced = false; + await this.ensureOpen(); + const db = this.db; + if (!db || db.readOnly || this.disposed) return { noop: true, sessions: 0, documents: 0 }; + + // One-time rewrite of pre-v2 hash-only file-meta keys, inside the + // background pass — never in the query path. After it, every per-session + // lookup below scans only that session's meta prefix. + await this.migrateFileMetaKeys(db); + + const currentIds = new Set(sessions.map((s) => s.id)); + + // Drop sessions whose directory disappeared since the last sync. The + // disposed gate covers this loop and the trailing stats write: once + // close starts, the pass skips them instead of writing into a db whose + // close is already draining. + for (const row of db.query({ key: { prefix: SESSION_META_PREFIX }, project: [] })) { + if (this.disposed) return { noop: true, sessions: 0, documents: 0 }; + const sessionId = row.key.slice(SESSION_META_PREFIX.length); + if (!currentIds.has(sessionId)) await this.deleteSessionDocs(db, sessionId); + } + + let indexed = 0; + for (const summary of sessions) { + if (this.disposed) return { noop: true, sessions: 0, documents: 0 }; + try { + await this.syncSession(db, summary); + indexed++; + } catch (error) { + // One unreadable session must not abort the whole pass. + this.log.warn('global search: failed to index session', { + sessionId: summary.id, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + if (this.disposed) return { noop: true, sessions: 0, documents: 0 }; + const metaCount = db.query({ key: { prefix: '\0meta\\' }, project: [] }).length; + const stats: StatsDoc = { + kind: 'stats', + sessions: indexed, + documents: db.size - metaCount, + lastIndexedAt: Date.now(), + }; + await db.set(STATS_KEY, stats); + this.fullSyncDone = true; + if (this.syncReplaced) { + // The pass REPLACED indexed documents (shrink rescan / title + // overwrite), so their sort keys may have moved: page tokens from the + // previous generation must restart instead of drifting. + this.generation++; + } + return { noop: false, sessions: indexed, documents: stats.documents }; + } + + /** + * One-time per-process migration of pre-v2 hash-only file-meta keys to the + * session-scoped format (`fileMetaKey`). A single full prefix scan of the + * meta namespace; per-session work afterwards only scans that session's + * keys. Idempotent — a crash mid-migration just rescans on the next pass. + */ + private async migrateFileMetaKeys(db: MiniDb): Promise { + if (this.fileMetaMigrated) return; + const ops: BatchInputOp[] = []; + for (const row of db.query({ key: { prefix: FILE_META_PREFIX }, project: [] })) { + const rest = row.key.slice(FILE_META_PREFIX.length); + if (rest.includes('\\')) continue; // already session-scoped + const meta = row.value; + if (meta.kind !== 'fileMeta') continue; + ops.push({ op: 'set', key: fileMetaKey(meta.sessionId, meta.path), value: meta }); + ops.push({ op: 'del', key: row.key }); + } + // Batch the rewrite instead of one op per key; empty on every later pass. + if (ops.length > 0) await db.batch(ops); + this.fileMetaMigrated = true; + } + + private async deleteSessionDocs(db: MiniDb, sessionId: string): Promise { + for (const row of db.query({ key: { prefix: `${sessionId}/` }, project: [] })) { + await db.del(row.key); + } + for (const row of db.query({ key: { prefix: fileMetaPrefixFor(sessionId) }, project: [] })) { + await db.del(row.key); + } + await db.del(SESSION_META_PREFIX + sessionId); + } + + private async syncSession(db: MiniDb, summary: SyncSessionInput): Promise { + const wireFiles = await collectWireFiles(summary.dir); + const seenPaths = new Set(wireFiles.map((file) => file.path)); + + // A wire file that vanished on its own (e.g. one agent's log deleted + // while the session lives on): drop its docs and meta. Session-level + // disappearance is handled separately in runSync. The scan is scoped to + // THIS session's meta prefix — O(files of this session), independent of + // the global session count. + for (const row of db.query({ key: { prefix: fileMetaPrefixFor(summary.id) } })) { + const meta = row.value; + if (meta.kind !== 'fileMeta') continue; + if (seenPaths.has(meta.path)) continue; + await this.deleteFileDocs(db, meta); + await db.del(row.key); + } + + for (const file of wireFiles) { + await this.syncWireFile(db, summary, file); + } + + const title = summary.title ?? ''; + const titleKey = `${summary.id}/$title`; + const existing = db.get(titleKey); + if (title.length > 0) { + if (existing?.kind !== 'title' || existing.text !== title) { + const doc: TitleDoc = { + kind: 'title', + sessionId: summary.id, + workspaceId: summary.workspaceId, + sessionTitle: title, + agentId: '', + role: 'title', + text: title, + time: summary.updatedAt, + }; + await db.set(titleKey, doc); + // Overwriting an existing title doc moves its sort key mid-pagination + // — a replacing change, unlike the additive first-time create. + if (existing !== undefined) this.syncReplaced = true; + } + } else if (existing !== undefined) { + await db.del(titleKey); + } + // Session marker: presence is the information — write only when missing. + if (db.get(SESSION_META_PREFIX + summary.id) === undefined) { + const sessionMeta: SessionMetaDoc = { kind: 'sessionMeta' }; + await db.set(SESSION_META_PREFIX + summary.id, sessionMeta); + } + } + + private async deleteFileDocs(db: MiniDb, meta: FileMetaDoc): Promise { + const prefix = `${meta.sessionId}/${meta.agentId}/${meta.source}:`; + for (const row of db.query({ key: { prefix }, project: [] })) { + await db.del(row.key); + } + } + + private async syncWireFile( + db: MiniDb, + summary: SyncSessionInput, + file: WireFileRef, + ): Promise { + let st: { size: number; mtimeMs: number; ino: number }; + try { + st = await stat(file.path); + } catch { + return; // transiently unreadable — retry next pass + } + const size = st.size; + const metaKey = fileMetaKey(summary.id, file.path); + // New session-scoped key first, then the pre-v2 hash-only key (a cheap + // point lookup, not a scan): metas written before the key migration are + // honored and opportunistically rewritten under the new key. + let meta = db.get(metaKey); + let legacyKey: string | null = null; + if (meta?.kind !== 'fileMeta') { + const oldKey = legacyFileMetaKey(file.path); + const legacy = db.get(oldKey); + if (legacy?.kind === 'fileMeta') { + meta = legacy; + legacyKey = oldKey; + } + } + const known = meta?.kind === 'fileMeta' ? meta : undefined; + let offset = known?.offset ?? 0; + let turnState: TurnCounterState = known?.turnState ?? initialTurnState(); + let stepState: StepTrackerState = known?.stepState ?? initialStepState(); + const fileMeta = ( + nextOffset: number, + turns: TurnCounterState, + steps: StepTrackerState, + ): FileMetaDoc => ({ + kind: 'fileMeta', + sessionId: summary.id, + agentId: file.agentId, + source: file.source, + path: file.path, + offset: nextOffset, + size, + mtimeMs: st.mtimeMs, + ino: st.ino, + turnState: turns, + stepState: steps, + }); + // Metas written before step tracking carry no `stepState`: rescan the + // file from scratch so stepIds are all-or-nothing per file rather than + // drifting mid-file (the shrink path does exactly this). + const legacyMeta = known !== undefined && known.stepState === undefined; + // An inode change means the file was replaced (atomic rewrite); a bumped + // mtime at an unchanged size means an in-place rewrite. Both invalidate + // the byte-offset watermark even though the size alone would not. + const replacedFile = known?.ino !== undefined && known.ino !== st.ino; + const rewrittenInPlace = + known?.mtimeMs !== undefined && size === known.offset && st.mtimeMs > known.mtimeMs; + if (size < offset || legacyMeta || replacedFile || rewrittenInPlace) { + // File was rebuilt/truncated: drop its docs and rescan from scratch — + // the turn counter and step tracker restart with it. A replacing + // change: the docs' sort keys may move → bump the generation. + this.syncReplaced = true; + await this.deleteFileDocs(db, fileMeta(0, initialTurnState(), initialStepState())); + offset = 0; + turnState = initialTurnState(); + stepState = initialStepState(); + } + if (size === offset) { + // No growth: only rewrite the meta when something actually changed + // (first sight, stat refresh after an upgrade, legacy key cleanup) — + // an unchanged file must not cost a WAL record per pass. + if ( + legacyKey !== null || + known === undefined || + known.size !== size || + known.mtimeMs !== st.mtimeMs || + known.ino !== st.ino || + known.offset !== offset + ) { + const ops: BatchInputOp[] = [ + { op: 'set', key: metaKey, value: fileMeta(offset, turnState, stepState) }, + ]; + if (legacyKey !== null) ops.push({ op: 'del', key: legacyKey }); + await db.batch(ops); + } + return; + } + + // Read only the new byte range, in bounded chunks, consuming complete + // lines; a trailing partial line (or a short read from a mid-read + // truncation) is left for the next pass — the watermark below never + // advances past bytes that were actually consumed. The line loop keeps + // only line-sized strings alive instead of one `size - offset` buffer + // plus a full split array. + const handle = await open(file.path, 'r'); + const ops: BatchInputOp[] = []; + let byteCursor = offset; + try { + let position = offset; + let pending: Buffer = EMPTY_BUFFER; // partial-line bytes starting at byteCursor + const chunk = Buffer.allocUnsafe(WIRE_READ_CHUNK_BYTES); + while (position < size) { + if (this.disposed) return; // meta not advanced: the next pass redoes the file + const { bytesRead } = await handle.read( + chunk, + 0, + Math.min(chunk.length, size - position), + position, + ); + if (bytesRead === 0) break; + const slice = chunk.subarray(0, bytesRead); + position += bytesRead; + let start = 0; + for (;;) { + const nl = slice.indexOf(0x0a, start); + if (nl === -1) break; + const lineBuf = + pending.length > 0 + ? Buffer.concat([pending, slice.subarray(start, nl)]) + : slice.subarray(start, nl); + pending = EMPTY_BUFFER; + const lineOffset = byteCursor; + byteCursor += lineBuf.length + 1; + ({ turnState, stepState } = this.collectWireLine( + ops, + summary, + file, + lineBuf.toString('utf8'), + lineOffset, + { turnState, stepState }, + )); + start = nl + 1; + } + // The chunk buffer is reused, so the unconsumed tail must be copied. + pending = + pending.length > 0 + ? Buffer.concat([pending, slice.subarray(start)]) + : Buffer.from(slice.subarray(start)); + if (ops.length >= WIRE_BATCH_OPS) { + await db.batch(ops); + ops.length = 0; + } + } + } finally { + await handle.close(); + } + + if (byteCursor === offset && legacyKey === null) return; // no complete line yet + ops.push({ op: 'set', key: metaKey, value: fileMeta(byteCursor, turnState, stepState) }); + if (legacyKey !== null) ops.push({ op: 'del', key: legacyKey }); + await db.batch(ops); + } + + /** + * Turn/step counting and doc extraction for one complete wire line. + * Returns the counter states advanced by the line (they are immutable and + * replaced per line, so the caller threads them through the chunk loop). + */ + private collectWireLine( + ops: BatchInputOp[], + summary: SyncSessionInput, + file: WireFileRef, + line: string, + lineOffset: number, + counters: { turnState: TurnCounterState; stepState: StepTrackerState }, + ): { turnState: TurnCounterState; stepState: StepTrackerState } { + let { turnState, stepState } = counters; + const analysis = analyzeWireLine(line); + // Turn counting runs independently of indexing: every line moves the + // counter (a text-less user message still opens a turn). + const advanced = advanceTurnCounter(turnState, analysis.turn); + // A turn boundary invalidates the step mapping: a new turn opens + // (`open`, or `ensure` opening a fallback turn from no-turn), or an + // `undo` rewinds the counter mid-turn. + if ( + analysis.turn.kind === 'open' || + analysis.turn.kind === 'undo' || + (analysis.turn.kind === 'ensure' && !turnState.hasTurn) + ) { + stepState = initialStepState(); + } + turnState = advanced.state; + stepState = advanceStepTracker(stepState, analysis.step); + const extracted = analysis.messages; + for (let i = 0; i < extracted.length; i++) { + const e = extracted[i]!; + const stepOrdinal = e.stepUuid !== undefined ? stepState.byUuid[e.stepUuid] : undefined; + const doc: MessageDoc = { + kind: 'message', + sessionId: summary.id, + workspaceId: summary.workspaceId, + sessionTitle: summary.title ?? '', + agentId: file.agentId, + role: e.role, + text: e.text.length > MAX_DOC_TEXT_CHARS ? e.text.slice(0, MAX_DOC_TEXT_CHARS) : e.text, + time: e.time ?? summary.updatedAt, + turn: advanced.docTurn, + // A doc whose step cannot be resolved (no `step.begin` seen, or a + // turn boundary invalidated the mapping) just omits the id. + stepId: + advanced.docTurn !== undefined && stepOrdinal !== undefined + ? `t${advanced.docTurn}.${stepOrdinal}` + : undefined, + }; + // A line can yield several docs — the per-line index keeps keys unique. + ops.push({ + op: 'set', + key: `${docKeyPrefix(summary.id, file)}${lineOffset}:${i}`, + value: doc, + }); + } + return { turnState, stepState }; + } + + // -- index-route query --------------------------------------------------------- + + /** + * Serve one page from the currently published generation. Never waits for + * an open, sync, reopen or reindex: with no published base it answers with + * `building` semantics (or fails fast when the last open failed), and the + * page-token generation check runs against the base pinned at request + * time. + */ + async search(params: CoreSearchParams): Promise { + const { q, budgets } = params; + const db = this.db; + if (db === null) { + if (this.disposed) { + throw new GlobalSearchError('index_unavailable', 'search service is disposed'); + } + if (this.openError !== null) { + // The last open failed (e.g. a read-only open racing a writer's + // compaction): surface the failure; the caller kicks a background + // retry, so search traffic self-heals the index once the transient + // cause goes away — a successful retry clears openError. + throw new GlobalSearchError( + 'index_unavailable', + `search index failed to open: ${this.openError}`, + ); + } + if (params.pageToken !== undefined) { + // No generation to validate the token against — the client restarts + // the search once a base is published. + throw new GlobalSearchError( + 'invalid_page_token', + 'the search index is not ready yet; restart the search', + ); + } + return { kind: 'building', index: this.buildingView() }; + } + + let freshnessStale = false; + let serveDb = db; + if (serveDb.readOnly) { + // Cheap freshness probe (3 stats). A changed fingerprint refreshes in + // the BACKGROUND — this request deliberately serves the stale + // generation instead of waiting for a catch-up or a full reopen. + let fp: string | null = null; + try { + fp = await this.computeFingerprint(); + } catch (error) { + this.lastRefreshError = { at: Date.now(), message: errorMessage(error) }; + } + // A background refresh may have swapped (and closed) the captured + // handle during the await. Re-pin to the currently published handle. + // (The async query path awaits again below, so a later swap can still + // close it mid-query — the bounded pass retries once on the fresh + // handle for exactly that race.) + if (this.db === null) { + throw new GlobalSearchError('index_unavailable', 'search service is disposed'); + } + serveDb = this.db; + if (serveDb.readOnly) { + freshnessStale = fp === null || fp !== this.fingerprint || this.refreshPromise !== null; + if (fp !== null && fp !== this.fingerprint) void this.refresh(); + } + // else: the reopen promoted this instance to writer (the old writer's + // lock was gone) — the caller's coordinator picks up sync work from + // here; freshnessStale stays false. + } + const generation = this.tokenGeneration(); + const page = decodePageToken(q, 'index', params.pageToken, generation); + + // The served handle's text base is still (re)building — the deferred + // open-time build on the no-generation fallback path has not committed (or + // finally failed). Answer with the building page instead of running a + // pass that would raise TextIndexBuildingError; the background build + // commits and a later search serves real hits. Tokens from an older + // generation already failed validation above, so reaching here with a + // building handle is always a first-page situation. + if (serveDb.textIndexBuilding(q.mode === 'literal' ? TRI_INDEX_NAME : TEXT_INDEX_NAME)) { + return { kind: 'building', index: this.buildingView(serveDb) }; + } + + // One bounded text-index pass: db.searchBoundedAsync returns at most the + // budgeted candidates with their scores (the async variant — postings + // reads and disk-mode value reads run off the event loop); + // container/role/time filters and the requested sort are applied in + // memory. (A separate db.query({text}) for pagination would scan the + // same postings a second time.) + let candidates: { key: string; value: SearchDoc | undefined; score: number }[]; + let incomplete: GlobalSearchIncomplete | undefined; + const runBounded = ( + db2: MiniDb, + ): Promise<{ + hits: { key: string; value: SearchDoc; score: number }[]; + visits: number; + truncated: boolean; + }> => { + if (q.mode === 'literal') { + // Ask for one past the cap so an over-cap candidate set is + // detectable; the postings budget bounds the index-side work before + // confirmation even starts. + return db2.searchBoundedAsync(TRI_INDEX_NAME, q.query, { + op: 'AND', + limit: budgets.literalCandidateCap + 1, + maxVisits: budgets.postingsVisitBudget, + }); + } + return db2.searchBoundedAsync(TEXT_INDEX_NAME, q.query, { + op: q.op, + limit: budgets.maxTextHits + 1, + maxVisits: budgets.postingsVisitBudget, + }); + }; + try { + let res: { + hits: { key: string; value: SearchDoc; score: number }[]; + visits: number; + truncated: boolean; + }; + try { + res = await runBounded(serveDb); + } catch (error) { + // The async query path awaits: a background read-only refresh may + // have swapped (and closed) the pinned handle mid-query. Re-pin the + // currently published handle and retry ONCE — anything else is a + // real failure. + const msg = error instanceof Error ? error.message : String(error); + const closedRace = + msg.includes('postings file is closed') || + msg.includes('MiniDb is closed') || + msg.includes('ValueReader is not open'); + if (!closedRace || this.db === null || this.db === serveDb) throw error; + serveDb = this.db; + res = await runBounded(serveDb); + } + if (q.mode === 'literal') { + candidates = res.hits; + if (res.truncated) incomplete = 'postings_budget'; + if (candidates.length > budgets.literalCandidateCap) { + candidates.length = budgets.literalCandidateCap; + incomplete ??= 'candidate_cap'; + } + } else { + candidates = res.hits; + if (res.truncated) incomplete = 'postings_budget'; + if (candidates.length > budgets.maxTextHits) { + candidates.length = budgets.maxTextHits; + incomplete ??= 'candidate_cap'; + } + } + } catch (error) { + // The base build's state flipped between the early check and the pass + // (or a read-only refresh swapped in a still-building handle mid-page): + // serve the same building page the early check produces. + if (error instanceof TextIndexBuildingError) { + return { kind: 'building', index: this.buildingView(serveDb) }; + } + // A read-only instance can open before the writer has created the text + // index — serve an empty page instead of failing the search. + if (error instanceof Error && error.message.includes('no such text index')) { + return { + kind: 'page', + rows: [], + hasMore: false, + incomplete: undefined, + generation, + index: this.readIndexView(serveDb, freshnessStale), + }; + } + throw error; + } + + const budget: MatchBudget = { + deadlineAt: Date.now() + budgets.queryDeadlineMs, + textCharsLeft: budgets.queryTextBudgetChars, + }; + const boundary = page.kind === 'keyset' ? page.boundary : undefined; + const matched = matchDocs(q, candidates, boundary, budget); + incomplete ??= matched.incomplete; + const { pageRows, hasMore } = paginateRows(q, page, matched.rows); + return { + kind: 'page', + rows: pageRows, + hasMore, + incomplete, + generation, + index: this.readIndexView(serveDb, freshnessStale), + }; + } + + // -- reindex & status ------------------------------------------------------------ + + /** + * Full rebuild: close the handle, wipe the directory and reopen. The + * caller (the service) blocks new sync passes beforehand and runs the + * authoritative sync afterwards. + */ + async reindex(): Promise { + await this.ensureOpen(); + if (this.db?.readOnly === true) { + throw new GlobalSearchError( + 'readonly_index', + 'another process holds the search-index write lock; reindex from that process', + ); + } + // Let an in-flight refresh settle before closing the db it swaps. + await this.refreshPromise?.catch(() => {}); + const db = this.db; + if (db) { + await db.close().catch(() => {}); + this.db = null; + } + this.openPromise = null; + this.fullSyncDone = false; + this.lockToken = undefined; + await rm(this.indexDir, { recursive: true, force: true }); + await this.ensureOpen(); + } + + /** + * Synchronous lifecycle snapshot (stage 5) — never kicks an open, never + * awaits: the diagnostic view of `stopped → opening → ready → + * building/degraded → closing`. 'degraded' means NO base is serving (the + * last open failed); a published base that keeps serving after a failed + * background refresh stays 'ready' — the failure rides the `degraded` + * message field of the page/status surfaces, same as the per-page + * `indexState` discipline. + */ + lifecycleState(): CoreLifecycleReport { + if (this.disposed) return { state: this.db === null ? 'stopped' : 'closing' }; + const db = this.db; + if (db === null) { + if (this.openPromise !== null) return { state: 'opening' }; + if (this.openError !== null) return { state: 'degraded', detail: this.openError }; + return { state: 'stopped' }; + } + if (db.textIndexBuilding(TEXT_INDEX_NAME) || db.textIndexBuilding(TRI_INDEX_NAME)) { + return { state: 'building' }; + } + return { state: 'ready' }; + } + + async status(): Promise { + await this.ensureOpen(); + if (this.db?.readOnly === true) { + // An explicit status call may wait for the refresh; searches may not. + await this.refresh(); + } + const stats = this.db?.get(STATS_KEY); + return { + sessions: stats?.kind === 'stats' ? stats.sessions : 0, + documents: stats?.kind === 'stats' ? stats.documents : 0, + lastIndexedAt: stats?.kind === 'stats' ? stats.lastIndexedAt : null, + generation: this.generation, + readOnly: this.db?.readOnly === true, + lockToken: this.lockToken, + degraded: this.lastRefreshError?.message, + lifecycle: this.lifecycleState(), + }; + } + + // -- index-state views ------------------------------------------------------------ + + /** The view served while no queryable base is available. */ + private buildingView(db?: MiniDb): CoreIndexView { + const handle = db ?? this.db; + const stats = handle?.get(STATS_KEY); + const indexed = stats?.kind === 'stats' ? stats.sessions : 0; + return { + state: 'building', + indexedSessions: indexed, + documents: stats?.kind === 'stats' ? stats.documents : 0, + readOnly: handle?.readOnly === true, + freshnessStale: true, + degraded: this.lastRefreshError?.message, + lockToken: this.lockToken, + }; + } + + private readIndexView(db: MiniDb, freshnessStale: boolean): CoreIndexView { + const stats = db.get(STATS_KEY); + const indexed = stats?.kind === 'stats' ? stats.sessions : 0; + const documents = stats?.kind === 'stats' ? stats.documents : 0; + // A deferred open-time base build (no-generation fallback path) puts the + // served handle's text indexes into the building state — surface it as + // the same 'building' the first-sync window uses, whatever the process role. + const building = db.textIndexBuilding(TEXT_INDEX_NAME) || db.textIndexBuilding(TRI_INDEX_NAME); + return { + state: building ? 'building' : db.readOnly ? 'readonly' : this.fullSyncDone ? 'ready' : 'building', + indexedSessions: indexed, + documents, + readOnly: db.readOnly, + freshnessStale, + degraded: this.lastRefreshError?.message, + lockToken: this.lockToken, + }; + } +} + +// --------------------------------------------------------------------------- +// wire file enumeration & doc keys +// --------------------------------------------------------------------------- + +interface WireFileRef { + readonly path: string; + /** 'main' or a subagent id, for both legacy and v2 layouts. */ + readonly agentId: string; + /** + * Key discriminator: a session can carry BOTH a legacy root wire.jsonl and + * v2 per-agent logs; without this their `/` keys collide. + */ + readonly source: 'root' | 'agents'; +} + +async function collectWireFiles(sessionDir: string): Promise { + const files: WireFileRef[] = []; + const root = join(sessionDir, WIRE_FILENAME); + try { + if ((await stat(root)).isFile()) files.push({ path: root, agentId: 'main', source: 'root' }); + } catch { + // no legacy root log + } + const agentsDir = join(sessionDir, 'agents'); + try { + const entries = await readdir(agentsDir, { recursive: true, withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile() || entry.name !== WIRE_FILENAME) continue; + const path = join(entry.parentPath, entry.name); + files.push({ path, agentId: relative(agentsDir, entry.parentPath), source: 'agents' }); + } + } catch { + // no agents dir + } + return files; +} + +function docKeyPrefix(sessionId: string, file: WireFileRef): string { + return `${sessionId}/${file.agentId}/${file.source}:`; +} + +/** Same rebuildability test as `MiniDb.openOrRebuild`. */ +function isRebuildableCorruption(error: unknown): boolean { + return ( + error instanceof SyntaxError || + (error !== null && + typeof error === 'object' && + (error as { name?: string }).name === 'CorruptFrameError') + ); +} diff --git a/packages/kap-server/src/search/match.ts b/packages/kap-server/src/search/match.ts new file mode 100644 index 000000000..6ae69e5b8 --- /dev/null +++ b/packages/kap-server/src/search/match.ts @@ -0,0 +1,385 @@ +/** + * `search` module — query shape, sort order, keyset boundaries, bounded + * collection and the shared match/confirm pass (pure logic). + * + * Extracted from `searchService.ts` so the SAME implementation backs both + * consumers: + * - the main-process service (the live transcript route), and + * - the host-agnostic index core (`indexCore.ts`), which runs the index + * route inside the search worker thread. + * + * The worker entry's import closure is loaded under Node's native type + * stripping, so every relative import here uses an explicit `.ts` specifier. + */ + +import { createHash } from 'node:crypto'; + +import { normalizeLiteral } from '@moonshot-ai/minidb'; + +import { + GlobalSearchError, + type GlobalSearchIncomplete, + type GlobalSearchSource, +} from './contract.ts'; +import type { MessageDoc, SearchDoc, TitleDoc } from './docs.ts'; + +// --------------------------------------------------------------------------- +// Normalized query (produced by the service, consumed by both routes) +// --------------------------------------------------------------------------- + +export interface NormalizedQuery { + readonly query: string; + readonly mode: 'terms' | 'literal'; + /** + * Literal mode only: `normalizeLiteral(query)`, computed once and reused by + * candidate confirmation and the snippet anchor. The n-gram index's query + * tokenizer applies the same normalization to the query terms, so index and + * comparison agree by construction. + */ + readonly literalQuery?: string; + /** + * Terms mode only: the query's deduplicated terms under minidb's default + * `tokenize` (the same tokenizer the 'body' text index applies to both + * sides). Computed once here so the live route's in-memory AND match agrees + * with the index route by construction. Empty when the query tokenizes to + * nothing (e.g. punctuation only) — both routes then match zero docs, + * mirroring `TextIndex.search`. + */ + readonly termsQuery?: readonly string[]; + readonly op: 'AND' | 'OR'; + readonly container?: { readonly sessionId?: string; readonly agentId?: string }; + readonly role?: 'user' | 'assistant' | 'title'; + readonly startTime?: number; + readonly endTime?: number; + readonly sort: 'score' | 'time_desc' | 'time_asc'; + readonly pageSize: number; +} + +/** Per-query work budget knobs (service fields, forwarded to the core). */ +export interface SearchBudgets { + readonly literalCandidateCap: number; + readonly maxTextHits: number; + readonly postingsVisitBudget: number; + readonly queryDeadlineMs: number; + readonly queryTextBudgetChars: number; +} + +// --------------------------------------------------------------------------- +// Page-token decoding products (tokens themselves are the service's business) +// --------------------------------------------------------------------------- + +/** + * Sort boundary of the last returned hit — the keyset cursor: + * - literal mode / `time_desc` / `time_asc`: `[time, key]`; + * - `score` (terms mode): `[score, time, key]`. + * The key is the doc's stable identity: the minidb key on the index route, a + * synthetic per-frame key on the live route. + */ +export type SortBoundary = readonly (number | string)[]; + +export type DecodedPage = + | { readonly kind: 'first' } + | { readonly kind: 'keyset'; readonly boundary: SortBoundary } + /** Legacy v1 offset token, accepted during the transition window. */ + | { readonly kind: 'legacy'; readonly skip: number }; + +/** Boundary tuple width for the query's effective sort order. */ +export function boundaryWidth(q: NormalizedQuery): 2 | 3 { + return q.mode !== 'literal' && q.sort === 'score' ? 3 : 2; +} + +// --------------------------------------------------------------------------- +// Sort order, boundary filtering and bounded collection (both routes) +// --------------------------------------------------------------------------- + +/** One matched document with its stable key and match context. */ +export interface MatchedRow { + readonly key: string; + readonly value: MessageDoc | TitleDoc; + readonly score: number; + /** Literal mode: offset of the confirmed match, reused as snippet anchor. */ + readonly anchor?: number; +} + +/** Per-query work budget for the match/confirm phase (both routes). */ +export interface MatchBudget { + /** Date.now() timestamp after which matching stops with 'deadline'. */ + readonly deadlineAt: number; + /** Remaining document text (UTF-16 code units) literal confirmation may + * process before stopping with 'deadline'. */ + textCharsLeft: number; +} + +function cmpKey(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** + * The query's total order (negative = `a` ranks before `b`): + * - literal mode (sort is a terms-mode concept) and `time_desc`: + * (time desc, key asc); + * - `time_asc`: (time asc, key asc); + * - `score`: (score desc, time desc, key asc). + */ +export function compareRows(q: NormalizedQuery, a: MatchedRow, b: MatchedRow): number { + if (q.mode !== 'literal' && q.sort === 'score') { + return b.score - a.score || b.value.time - a.value.time || cmpKey(a.key, b.key); + } + if (q.mode !== 'literal' && q.sort === 'time_asc') { + return a.value.time - b.value.time || cmpKey(a.key, b.key); + } + return b.value.time - a.value.time || cmpKey(a.key, b.key); +} + +/** The boundary tuple of a row — the keyset cursor payload. */ +export function boundaryOf(q: NormalizedQuery, row: MatchedRow): SortBoundary { + return boundaryWidth(q) === 3 ? [row.score, row.value.time, row.key] : [row.value.time, row.key]; +} + +/** Whether the row ranks strictly AFTER the boundary in the sort order. */ +function rowAfterBoundary(q: NormalizedQuery, row: MatchedRow, boundary: SortBoundary): boolean { + let cmp: number; + if (boundary.length === 3) { + const [bs, bt, bk] = boundary as readonly [number, number, string]; + cmp = bs - row.score || bt - row.value.time || cmpKey(row.key, bk); + } else { + const [bt, bk] = boundary as readonly [number, string]; + cmp = + q.mode !== 'literal' && q.sort === 'time_asc' + ? row.value.time - bt || cmpKey(row.key, bk) + : bt - row.value.time || cmpKey(row.key, bk); + } + return cmp > 0; +} + +/** + * Bounded collector for the K best rows in the query's sort order — same + * worst-at-root heap shape as minidb's TopK: O(log K) per row and K rows in + * memory instead of an O(E log E) sort over every eligible row. Deep pages + * stay proportional to pageSize. + */ +export class RowTopK { + private readonly a: MatchedRow[] = []; + + constructor( + private readonly q: NormalizedQuery, + private readonly k: number, + ) {} + + private worse(x: MatchedRow, y: MatchedRow): boolean { + return compareRows(this.q, x, y) > 0; // x ranks after y + } + + offer(row: MatchedRow): void { + const a = this.a; + if (a.length < this.k) { + a.push(row); + let i = a.length - 1; + while (i > 0) { + const p = (i - 1) >> 1; + if (!this.worse(a[i]!, a[p]!)) break; + [a[p], a[i]] = [a[i]!, a[p]!]; + i = p; + } + return; + } + if (this.k === 0 || !this.worse(a[0]!, row)) return; // must beat the worst kept + a[0] = row; + let i = 0; + for (;;) { + let w = i; + const l = 2 * i + 1; + const r = 2 * i + 2; + if (l < a.length && this.worse(a[l]!, a[w]!)) w = l; + if (r < a.length && this.worse(a[r]!, a[w]!)) w = r; + if (w === i) break; + [a[w], a[i]] = [a[i]!, a[w]!]; + i = w; + } + } + + /** The kept rows in final rank order. */ + sorted(): MatchedRow[] { + return this.a.sort((x, y) => compareRows(this.q, x, y)); + } +} + +/** How often the match loop re-checks the deadline (candidate iterations). */ +const DEADLINE_CHECK_STRIDE = 64; + +/** + * Container/role/time filtering, keyset-boundary filtering and literal + * confirmation — one implementation shared by the index route (confirming + * n-gram candidates) and the live route (scanning every in-memory + * document). The query work budgets apply at this match stage: the + * wall-clock deadline is re-checked every DEADLINE_CHECK_STRIDE candidates + * and literal confirmation additionally charges each processed document's + * text against `budget.textCharsLeft`. A budget stop is reported as + * `incomplete: 'deadline'`, never a silent truncation. + */ +export function matchDocs( + q: NormalizedQuery, + docs: Iterable<{ key: string; value: SearchDoc | undefined; score: number }>, + boundary: SortBoundary | undefined, + budget: MatchBudget, +): { rows: MatchedRow[]; incomplete?: GlobalSearchIncomplete } { + const literalQuery = q.literalQuery; + const rows: MatchedRow[] = []; + let i = 0; + for (const { key, value: doc, score } of docs) { + if ((i++ & (DEADLINE_CHECK_STRIDE - 1)) === 0 && Date.now() > budget.deadlineAt) { + return { rows, incomplete: 'deadline' }; + } + if (doc === undefined || (doc.kind !== 'message' && doc.kind !== 'title')) continue; + if (q.container?.sessionId !== undefined && doc.sessionId !== q.container.sessionId) continue; + if (q.container?.agentId !== undefined && doc.agentId !== q.container.agentId) continue; + if (q.role !== undefined && doc.role !== q.role) continue; + if (q.startTime !== undefined && doc.time < q.startTime) continue; + if (q.endTime !== undefined && doc.time > q.endTime) continue; + // The boundary check only needs the sort key (score/time/key), so it + // runs BEFORE the expensive literal confirmation. + if (boundary !== undefined && !rowAfterBoundary(q, { key, value: doc, score }, boundary)) { + continue; + } + if (literalQuery !== undefined) { + budget.textCharsLeft -= doc.text.length; + if (budget.textCharsLeft < 0) return { rows, incomplete: 'deadline' }; + // Two-phase execution (same model as Elasticsearch's wildcard field): + // candidates (from the n-gram index, or every in-memory doc on the + // live route) are confirmed against the document text — hash + // collisions and non-contiguous n-gram coverage can produce false + // positives. Zero false positives is the hard guarantee of literal + // mode. The match offset doubles as the snippet anchor. Deliberate + // deviation from ES: the comparison is case-insensitive (NFKC + + // lowercase), aligned with the terms tokenizer. + const at = normalizeLiteral(doc.text).indexOf(literalQuery); + if (at === -1) continue; + rows.push({ key, value: doc, score: 0, anchor: at }); + } else { + rows.push({ key, value: doc, score }); + } + } + return { rows }; +} + +/** + * Sort, paginate and project the matched docs into a page (both routes). + * Keyset pages collect the best `pageSize + 1` rows past the boundary in a + * bounded heap; legacy v1 offset tokens get one last offset slice and are + * answered with a v2 keyset token. + */ +export function paginateRows( + q: NormalizedQuery, + page: DecodedPage, + rows: MatchedRow[], +): { pageRows: MatchedRow[]; hasMore: boolean } { + let pageRows: MatchedRow[]; + let hasMore: boolean; + if (page.kind === 'legacy') { + rows.sort((a, b) => compareRows(q, a, b)); + const slice = rows.slice(page.skip, page.skip + q.pageSize + 1); + hasMore = slice.length > q.pageSize; + pageRows = slice.slice(0, q.pageSize); + } else { + const top = new RowTopK(q, q.pageSize + 1); + for (const row of rows) top.offer(row); + const slice = top.sorted(); + hasMore = slice.length > q.pageSize; + pageRows = slice.slice(0, q.pageSize); + } + return { pageRows, hasMore }; +} + +// --------------------------------------------------------------------------- +// Page tokens v2 — keyset cursor + generation, legacy v1 offset compat +// --------------------------------------------------------------------------- + +/** + * The page token encodes a fingerprint of the query conditions — changing + * conditions mid-pagination invalidates the token (same rule as Lark's + * search API). The serving route (`source`) is part of the fingerprint: a + * route flip mid-pagination (e.g. the container session closed and the live + * route fell away) invalidates the token too, so the client restarts the + * search instead of silently switching result sets. + */ +export function tokenFingerprint(q: NormalizedQuery, source: GlobalSearchSource): string { + const basis = JSON.stringify([ + q.query, + q.mode, + q.op, + q.container?.sessionId, + q.container?.agentId, + q.role, + q.startTime, + q.endTime, + q.sort, + source, + ]); + return createHash('sha256').update(basis).digest('base64url').slice(0, 16); +} + +const PAGE_TOKEN_VERSION = 2; + +export function encodePageToken( + q: NormalizedQuery, + source: GlobalSearchSource, + boundary: SortBoundary, + generation: string | undefined, +): string { + return Buffer.from( + JSON.stringify({ v: PAGE_TOKEN_VERSION, f: tokenFingerprint(q, source), g: generation, b: boundary }), + ).toString('base64url'); +} + +export function decodePageToken( + q: NormalizedQuery, + source: GlobalSearchSource, + token: string | undefined, + generation: string | undefined, +): DecodedPage { + if (token === undefined) return { kind: 'first' }; + let parsed: unknown; + try { + parsed = JSON.parse(Buffer.from(token, 'base64url').toString('utf8')); + } catch { + throw new GlobalSearchError('invalid_page_token', 'pageToken is malformed'); + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new GlobalSearchError('invalid_page_token', 'pageToken is malformed'); + } + const p = parsed as { v?: unknown; f?: unknown; s?: unknown; g?: unknown; b?: unknown }; + if (p.f !== tokenFingerprint(q, source)) { + throw new GlobalSearchError( + 'invalid_page_token', + 'pageToken does not match the query conditions; query conditions must not change mid-pagination', + ); + } + if (p.v === undefined) { + // Legacy v1 offset token (`{f, s}`) — transition window: answer it with + // offset semantics; the response issues a v2 keyset token back. + if (typeof p.s !== 'number' || !Number.isInteger(p.s) || p.s < 0) { + throw new GlobalSearchError('invalid_page_token', 'pageToken is malformed'); + } + return { kind: 'legacy', skip: p.s }; + } + if (p.v !== PAGE_TOKEN_VERSION) { + throw new GlobalSearchError('invalid_page_token', 'pageToken has an unsupported version'); + } + if (generation !== undefined && p.g !== generation) { + throw new GlobalSearchError( + 'invalid_page_token', + 'pageToken was issued by an older index generation (the index was rebuilt, reopened or rescanned); restart the search', + ); + } + const width = boundaryWidth(q); + if ( + !Array.isArray(p.b) || + p.b.length !== width || + typeof p.b[0] !== 'number' || + typeof p.b[width - 1] !== 'string' || + (width === 3 && typeof p.b[1] !== 'number') + ) { + throw new GlobalSearchError('invalid_page_token', 'pageToken is malformed'); + } + return { kind: 'keyset', boundary: p.b as SortBoundary }; +} diff --git a/packages/kap-server/src/search/searchService.ts b/packages/kap-server/src/search/searchService.ts index 4ad065ceb..02e559ef3 100644 --- a/packages/kap-server/src/search/searchService.ts +++ b/packages/kap-server/src/search/searchService.ts @@ -19,7 +19,25 @@ * - Refresh/sync failures are recorded in `lastRefreshError` and surface * as `indexState.degraded` while the previous generation keeps serving. * - * Concurrency model — "the lock is the election": + * Execution hosts (stage 4 worker isolation): + * - Everything that touches the search-index MiniDb lives in the + * host-agnostic core (`indexCore.ts`). The service never opens the + * global search MiniDb itself. + * - WORKER host (default): the core runs inside a dedicated worker thread + * (`worker/entry.ts`), driven over RPC by `worker/host.ts` + * (`SearchWorkerHost`) — generation loads, WAL replays, sync passes and + * query execution never share this process's event loop. A worker that + * fails to start or dies mid-flight yields recognizable degraded + * responses (never a silent fallback onto the main thread); the + * `search_worker` experimental flag (`KIMI_CODE_EXPERIMENTAL_SEARCH_WORKER`, + * default ON) is the explicit rollback switch back to the inline host. + * - INLINE host (rollback / tests): the same core runs in-process. + * - The main thread keeps only: query normalization, page-token + * encode/decode, the sync coordinator (debounce/single-flight over the + * backend's sync), the live-transcript route, and hit projection. + * + * Concurrency model — "the lock is the election" (unchanged; the write lock + * is now held by the worker thread on behalf of this process): * - `MiniDb.open({ onLockFail: 'readonly' })`: the process that grabs the * exclusive write lock becomes the indexer (build + incremental sync); * every other process opens read-only and never rescans wire files. @@ -30,19 +48,6 @@ * replacement db first, then swap — a failed reopen keeps the stale * generation servable). When the indexer dies, the next opener takes the * lock and becomes the new indexer. - * - In-process, syncs are serialized behind a single-flight promise. - * - * Incremental indexing anchors on wire.jsonl byte offsets (the files are - * append-only JSONL): a `\0meta\file\\` key per wire - * file records how far it has been indexed plus the file's size/mtime/inode; - * growth re-reads only the new byte range (in bounded chunks), shrinkage or - * an inode/mtime change drops the file's docs and rescans. Session title - * docs (`/$title`) are overwritten each sync; disappeared sessions are - * dropped by key prefix. Pre-v2 hash-only file-meta keys - * (`\0meta\file\`) are migrated to the session-scoped format by a - * one-time background pass (`migrateFileMetaKeys`) and opportunistically on - * per-file lookup; readers never enumerate the global meta namespace per - * session, so one session's sync touches only that session's metas. * * Registration: this module is side-effect-imported by `start.ts` BEFORE * `bootstrap()` runs, so the module-level `registerScopedService` below lands @@ -51,91 +56,66 @@ * reflection surface as `globalSearch` with zero extra code. */ -import { createHash } from 'node:crypto'; -import { open, readdir, rm, stat } from 'node:fs/promises'; -import { join, relative } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { join } from 'node:path'; +import { stat } from 'node:fs/promises'; import { createDecorator, IBootstrapService, + IFlagService, ILogService, ISessionIndex, LifecycleScope, ScopeActivation, + registerFlagDefinition, registerScopedService, sessionDirOf, workspacePersistenceScope, type SessionSummary, } from '@moonshot-ai/agent-core-v2'; -import { LockError, MiniDb, OpTracker, TextIndexBuildingError, normalizeLiteral, tokenize, type BatchInputOp } from '@moonshot-ai/minidb'; +import { normalizeLiteral, tokenize } from '@moonshot-ai/minidb'; import type { TranscriptStore } from '@moonshot-ai/transcript'; -import type { - GlobalSearchHit, - GlobalSearchIndexState, - GlobalSearchIncomplete, - GlobalSearchPage, - GlobalSearchQuery, - GlobalSearchSource, +import { + GlobalSearchError, + type GlobalSearchHit, + type GlobalSearchIndexState, + type GlobalSearchPage, + type GlobalSearchQuery, } from './contract'; +import { MAX_DOC_TEXT_CHARS, type MessageDoc, type TitleDoc } from './docs'; +import { + SearchIndexCore, + type CoreIndexView, + type CoreLifecycleReport, + type CoreSearchParams, + type CoreSearchResult, + type CoreStatus, + type CoreSyncOutcome, + type SyncSessionInput, +} from './indexCore'; +import { + boundaryOf, + decodePageToken, + encodePageToken, + matchDocs, + paginateRows, + type MatchedRow, + type NormalizedQuery, + type SearchBudgets, +} from './match'; import { makeSnippet } from './snippet'; -import { analyzeWireLine, type StepEffect, type TurnEffect } from './wireExtract'; +import { SearchWorkerError, SearchWorkerHost, dropLiveLockToken, noteLiveLockToken } from './worker/host'; + +export { GlobalSearchError } from './contract'; +export type { GlobalSearchErrorReason } from './contract'; // --------------------------------------------------------------------------- -// Constants & stored document shapes +// Constants // --------------------------------------------------------------------------- const INDEX_DIR_NAME = 'search-index'; -const TEXT_INDEX_NAME = 'body'; -/** n-gram substring index backing literal mode, alongside 'body'. */ -const TRI_INDEX_NAME = 'tri'; -const WIRE_FILENAME = 'wire.jsonl'; - -/** Key namespaces inside the single db. */ -const FILE_META_PREFIX = '\0meta\\file\\'; -const SESSION_META_PREFIX = '\0meta\\session\\'; -const STATS_KEY = '\0meta\\stats'; - -function hashPath(filePath: string): string { - return createHash('sha256').update(filePath).digest('hex').slice(0, 32); -} - -/** - * minidb keys are limited to 128 bytes, far shorter than an absolute wire - * path — the file meta key carries the owning session id plus a hash of the - * path (the path itself lives in the value). The session segment makes a - * per-session prefix scan (`fileMetaPrefixFor`) touch only that session's - * metas instead of the global meta namespace. - */ -function fileMetaKey(sessionId: string, filePath: string): string { - return `${FILE_META_PREFIX}${sessionId}\\${hashPath(filePath)}`; -} - -/** All file-meta keys of one session (prefix-scan argument). */ -function fileMetaPrefixFor(sessionId: string): string { - return `${FILE_META_PREFIX}${sessionId}\\`; -} - -/** - * Pre-v2 file-meta key: hash-only, the owning session identifiable only via - * the value — a per-session lookup required scanning every file meta. - * Read side of the migration: `syncWireFile` still resolves it by point - * lookup; `migrateFileMetaKeys` rewrites the rest in one background pass. - */ -function legacyFileMetaKey(filePath: string): string { - return FILE_META_PREFIX + hashPath(filePath); -} - -/** Cap one indexed document's text so huge pastes do not bloat the index. */ -const MAX_DOC_TEXT_CHARS = 20_000; -/** Upper bound for text-index candidates handed to the scoring map / query. */ -const MAX_TEXT_HITS = 100_000; -/** - * Upper bound for literal-mode n-gram candidates handed to the confirmation - * pass (a store `get` plus a substring scan each — pure CPU). Beyond the cap - * the page is truncated and flagged `incomplete: 'candidate_cap'`. - */ -const LITERAL_CANDIDATE_CAP = 10_000; /** Sessions are listed in pages of this size. */ const SESSION_PAGE_SIZE = 500; @@ -157,230 +137,59 @@ const QUERY_DEADLINE_MS = 500; /** Max document text processed by literal confirmation per query (UTF-16 * code units) — the backstop for pathological huge-document corpora. */ const QUERY_TEXT_BUDGET_CHARS = 16_000_000; -/** How often the match loop re-checks the deadline (candidate iterations). */ -const DEADLINE_CHECK_STRIDE = 64; - -/** One wire-delta read slice: growth is consumed in bounded chunks instead - * of one `size - offset` allocation. */ -const WIRE_READ_CHUNK_BYTES = 1 << 20; -/** Flush doc ops to the db in batches of this size while scanning a delta. */ -const WIRE_BATCH_OPS = 1_000; -const EMPTY_BUFFER = Buffer.alloc(0); +/** Upper bound for text-index candidates handed to the scoring map / query. */ +const MAX_TEXT_HITS = 100_000; +/** + * Upper bound for literal-mode n-gram candidates handed to the confirmation + * pass (a store `get` plus a substring scan each — pure CPU). Beyond the cap + * the page is truncated and flagged `incomplete: 'candidate_cap'`. + */ +const LITERAL_CANDIDATE_CAP = 10_000; function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } -interface MessageDoc { - readonly kind: 'message'; - readonly sessionId: string; - readonly workspaceId: string; - readonly sessionTitle: string; - readonly agentId: string; - readonly role: 'user' | 'assistant'; - readonly text: string; - readonly time: number; - /** - * 0-based turn ordinal in the transcript view (groupTurns numbering). Absent - * for docs indexed before turn tracking existed. - */ - readonly turn?: number; - /** - * Transcript step id (`t.`, engine live numbering from the wire - * record's `step` field) of the step that produced this assistant text. - * Absent for user docs and docs indexed before step tracking existed. - */ - readonly stepId?: string; -} - -interface TitleDoc { - readonly kind: 'title'; - readonly sessionId: string; - readonly workspaceId: string; - readonly sessionTitle: string; - /** Titles belong to the session, not an agent — always ''. */ - readonly agentId: ''; - readonly role: 'title'; - readonly text: string; - readonly time: number; -} - -interface FileMetaDoc { - readonly kind: 'fileMeta'; - /** Owning session, used to drop metas when a session disappears. */ - readonly sessionId: string; - /** Doc-key coordinates of this file's documents (see `docKeyPrefix`). */ - readonly agentId: string; - readonly source: 'root' | 'agents'; - /** Absolute wire path (debugging aid; the key is its session + hash). */ - readonly path: string; - /** Byte offset up to which the wire file has been indexed. */ - readonly offset: number; - readonly size: number; - /** - * File mtime/inode at the last sync pass. A changed inode (atomic - * replacement) or a bumped mtime at an unchanged size (in-place rewrite) - * forces a rescan even when `size === offset`. Absent in metas written - * before change tracking — such metas are simply refreshed with the - * current stat on the next pass, without a rescan. - */ - readonly mtimeMs?: number; - readonly ino?: number; - /** - * Turn counter state at `offset` — persisted with the watermark so an - * incremental pass resumes counting instead of restarting at turn 0. - * Absent in metas written before turn tracking; treated as the initial - * state, which makes a legacy meta resume mid-file with a zeroed counter — - * an accepted one-time drift, self-healing on the next shrink/rescan. - */ - readonly turnState?: TurnCounterState; - /** - * Step tracker state at `offset` — persisted with the watermark for the - * same resume reason as `turnState`. Absent in metas written before step - * tracking: such a file is RESCANNED from scratch (docs dropped, offset - * reset) so stepIds are all-or-nothing per file instead of drifting. - */ - readonly stepState?: StepTrackerState; -} - -interface SessionMetaDoc { - readonly kind: 'sessionMeta'; -} - -// --------------------------------------------------------------------------- -// Turn counter (transcript groupTurns numbering, replayed over the wire file) -// --------------------------------------------------------------------------- - -interface TurnOpener { - readonly turn: number; - readonly anchor: boolean; -} - -interface TurnCounterState { - /** Ordinal the next opened turn will get (0-based). */ - readonly next: number; - /** Whether a turn is currently open (groupTurns' `ensureTurn` gate). */ - readonly hasTurn: boolean; - /** Turn openers, in order — the replay stack for `context.undo`. */ - readonly openers: readonly TurnOpener[]; -} - -const INITIAL_TURN_STATE: TurnCounterState = { next: 0, hasTurn: false, openers: [] }; - -function initialTurnState(): TurnCounterState { - return INITIAL_TURN_STATE; -} - -/** - * Replay `context.undo {count}`: drop the last `count` anchor-opened turns. - * The counter rewinds to the ordinal of the earliest dropped anchor, and the - * opener stack is truncated there. An undo with fewer anchors than `count` - * never reaches the wire (the engine's precheck rejects it) — left untouched. - */ -function applyUndoToTurnState(state: TurnCounterState, count: number): TurnCounterState { - let found = 0; - for (let i = state.openers.length - 1; i >= 0; i--) { - if (state.openers[i]!.anchor) { - found++; - if (found === count) { - return { - next: state.openers[i]!.turn, - hasTurn: i > 0, - openers: state.openers.slice(0, i), - }; - } - } - } - return state; -} - -/** - * Advance the counter with one record's turn effect. Returns the ordinal that - * documents extracted from the SAME record belong to: a user opener carries - * the turn it opens; assistant content carries the current turn (after the - * `ensure` gate). Undefined when the record owns no turn. - * - * The counter is monotonic except for `undo` rewinds: `apply_compaction` and - * `clear` do NOT renumber (the transcript's cold replay keeps full history - * and groupTurns numbers it continuously; the live TurnModel is monotonic - * too), so they are `none` effects by construction. Docs indexed BEFORE an - * `undo` keep their pre-undo ordinals — those messages no longer exist in the - * transcript view, so their ordinals point nowhere (known, accepted - * deviation, same class as "folded-away messages stay searchable"). - */ -function advanceTurnCounter( - state: TurnCounterState, - effect: TurnEffect, -): { docTurn: number | undefined; state: TurnCounterState } { - switch (effect.kind) { - case 'open': - return { - docTurn: state.next, - state: { - next: state.next + 1, - hasTurn: true, - openers: [...state.openers, { turn: state.next, anchor: effect.anchor }], - }, - }; - case 'ensure': { - const next = state.hasTurn ? state : { ...state, next: state.next + 1, hasTurn: true }; - return { docTurn: next.next - 1, state: next }; - } - case 'undo': - return { docTurn: undefined, state: applyUndoToTurnState(state, effect.count) }; - case 'none': - return { docTurn: undefined, state }; +async function pathExists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; } } // --------------------------------------------------------------------------- -// Step tracker (transcript step ids `t.`, per-turn uuid → ordinal) +// Experimental flag (the rollback switch for the worker host) // --------------------------------------------------------------------------- -interface StepTrackerState { - /** Current turn's step uuid → ordinal (the wire `step` field, else the fallback counter). */ - readonly byUuid: Record; - /** `step.begin` count within the current turn — the fallback ordinal source. */ - readonly begins: number; -} - -const INITIAL_STEP_STATE: StepTrackerState = { byUuid: {}, begins: 0 }; - -function initialStepState(): StepTrackerState { - return INITIAL_STEP_STATE; -} - /** - * Advance the tracker with one record's step effect. `begin` maps the step's - * uuid to its ordinal: the wire record's own `step` field when present (the - * engine's live 1-based numbering — the same numbering transcript step ids - * use), otherwise the count of begins seen in this turn (v1 loops had no - * loop-level retries, so counting matches the surviving-step numbering). - * The mapping is never narrowed per step — it is reset wholesale at turn - * boundaries (`open`, a fallback-opening `ensure`, `undo`) by the caller. + * `search_worker` — run the global search index in a dedicated worker + * thread (default ON). Disable via `KIMI_CODE_EXPERIMENTAL_SEARCH_WORKER=false` + * or the `[experimental]` config section to fall back to the in-process + * (inline) host. Read once at service construction. */ -function advanceStepTracker(state: StepTrackerState, effect: StepEffect): StepTrackerState { - if (effect.kind !== 'begin') return state; - const begins = state.begins + 1; - const ordinal = effect.ordinal ?? begins; - if (state.byUuid[effect.uuid] === ordinal) return state; - return { byUuid: { ...state.byUuid, [effect.uuid]: ordinal }, begins }; -} +export const SEARCH_WORKER_FLAG_ID = 'search_worker'; -interface StatsDoc { - readonly kind: 'stats'; - readonly sessions: number; - readonly documents: number; - readonly lastIndexedAt: number; -} +registerFlagDefinition({ + id: SEARCH_WORKER_FLAG_ID, + title: 'search worker isolation', + description: + 'Run the global search-index MiniDB (open, WAL replay, sync, queries) in a dedicated worker thread instead of the server main thread.', + env: 'KIMI_CODE_EXPERIMENTAL_SEARCH_WORKER', + default: true, + surface: 'core', +}); -type SearchDoc = MessageDoc | TitleDoc | FileMetaDoc | SessionMetaDoc | StatsDoc; +// --------------------------------------------------------------------------- +// Disposal drain (unchanged contract with start.ts) +// --------------------------------------------------------------------------- /** * Fire-and-forget close promises produced by `dispose()` (DI disposal is * synchronous). The server shutdown path awaits these via * `drainGlobalSearchDisposals()` before the homeDir is released, so a - * teardown `rm()` never races an in-flight minidb open/close. + * teardown `rm()` never races an in-flight backend close. */ const pendingDisposals = new Set>(); @@ -399,27 +208,20 @@ export async function drainGlobalSearchDisposals(): Promise { // Service interface // --------------------------------------------------------------------------- -export type GlobalSearchErrorReason = - | 'invalid_query' - | 'invalid_page_token' - | 'readonly_index' - | 'index_unavailable'; - -export class GlobalSearchError extends Error { - constructor( - readonly reason: GlobalSearchErrorReason, - message: string, - ) { - super(message); - this.name = 'GlobalSearchError'; - } -} - export interface IGlobalSearchService { readonly _serviceBrand: undefined; search(query: GlobalSearchQuery): Promise; /** Full rebuild: wipe the index and rescan every wire file. */ reindex(): Promise<{ sessions: number; documents: number }>; + /** + * Diagnostic status (the `/api/v1/debug` surface reflects it). Never + * throws: a backend that cannot answer (failed open, worker down) reports + * a degraded lifecycle instead of rejecting. `lifecycle` is the aggregate + * state machine (stage 5): stopped → opening → ready → building/degraded → + * closing. NOTE the historical contract: the call may kick/await the + * backend's open and read-only refresh — use `lifecycleReport()` for a + * non-intrusive local read. + */ status(): Promise<{ sessions: number; documents: number; @@ -428,7 +230,14 @@ export interface IGlobalSearchService { generation: number; /** Last background refresh/sync/reindex failure, if serving stale. */ degraded?: string; + lifecycle: CoreLifecycleReport; }>; + /** + * Synchronous local lifecycle report (stage 5): never kicks an open, never + * spawns the worker, never awaits. Answers the transitional states + * (stopped/opening/degraded-backoff/closing) that status() would block on. + */ + lifecycleReport(): CoreLifecycleReport; /** * Wire the live-transcript source for the in-memory search route. Called * once from the composition root (start.ts) after `TranscriptService` is @@ -455,37 +264,9 @@ export interface LiveTranscriptSource { } // --------------------------------------------------------------------------- -// Query normalization & page tokens +// Query normalization // --------------------------------------------------------------------------- -interface NormalizedQuery { - readonly query: string; - readonly mode: 'terms' | 'literal'; - /** - * Literal mode only: `normalizeLiteral(query)`, computed once here and - * reused by candidate confirmation and the snippet anchor. The n-gram - * index's query tokenizer applies the same normalization to the query - * terms, so index and comparison agree by construction. - */ - readonly literalQuery?: string; - /** - * Terms mode only: the query's deduplicated terms under minidb's default - * `tokenize` (the same tokenizer the 'body' text index applies to both - * sides). Computed once here so the live route's in-memory AND match agrees - * with the index route by construction. Empty when the query tokenizes to - * nothing (e.g. punctuation only) — both routes then match zero docs, - * mirroring `TextIndex.search`. - */ - readonly termsQuery?: readonly string[]; - readonly op: 'AND' | 'OR'; - readonly container?: { readonly sessionId?: string; readonly agentId?: string }; - readonly role?: 'user' | 'assistant' | 'title'; - readonly startTime?: number; - readonly endTime?: number; - readonly sort: 'score' | 'time_desc' | 'time_asc'; - readonly pageSize: number; -} - function normalizeQuery(input: GlobalSearchQuery, maxQueryTerms: number): NormalizedQuery { const mode = input.mode ?? 'terms'; // Literal matching is byte-exact (mod NFKC/case) — whitespace is part of @@ -525,233 +306,86 @@ function normalizeQuery(input: GlobalSearchQuery, maxQueryTerms: number): Normal }; } -/** - * The page token encodes a fingerprint of the query conditions — changing - * conditions mid-pagination invalidates the token (same rule as Lark's - * search API). The serving route (`source`) is part of the fingerprint: a - * route flip mid-pagination (e.g. the container session closed and the live - * route fell away) invalidates the token too, so the client restarts the - * search instead of silently switching result sets. - */ -function tokenFingerprint(q: NormalizedQuery, source: GlobalSearchSource): string { - const basis = JSON.stringify([ - q.query, - q.mode, - q.op, - q.container?.sessionId, - q.container?.agentId, - q.role, - q.startTime, - q.endTime, - q.sort, - source, - ]); - return createHash('sha256').update(basis).digest('base64url').slice(0, 16); -} - // --------------------------------------------------------------------------- -// Page tokens v2 — keyset cursor + generation, legacy v1 offset compat +// Execution backends (worker RPC vs inline core) // --------------------------------------------------------------------------- -const PAGE_TOKEN_VERSION = 2; - -/** - * Sort boundary of the last returned hit — the keyset cursor: - * - literal mode / `time_desc` / `time_asc`: `[time, key]`; - * - `score` (terms mode): `[score, time, key]`. - * The key is the doc's stable identity: the minidb key on the index route, a - * synthetic per-frame key on the live route. - */ -type SortBoundary = readonly (number | string)[]; - -type DecodedPage = - | { readonly kind: 'first' } - | { readonly kind: 'keyset'; readonly boundary: SortBoundary } - /** Legacy v1 offset token, accepted during the transition window. */ - | { readonly kind: 'legacy'; readonly skip: number }; - -/** Boundary tuple width for the query's effective sort order. */ -function boundaryWidth(q: NormalizedQuery): 2 | 3 { - return q.mode !== 'literal' && q.sort === 'score' ? 3 : 2; +/** The service's view of an execution host for the search-index core. */ +export interface SearchBackend { + /** + * Synchronously stop accepting new background work (called from the + * service's synchronous dispose() before any awaiting): in-flight passes + * skip their remaining writes at the next gate check. + */ + beginClose(): void; + /** + * Local, round-trip-free aggregate lifecycle (stage 5): the states a wedged + * or not-yet-started backend must be able to report without being asked + * (stopped/opening/degraded/closing). The full status() round trip refines + * the up states (ready/building) with exact stats. + */ + lifecycleSnapshot(): CoreLifecycleReport; + ensureOpen(): Promise; + search(params: CoreSearchParams): Promise; + sync(sessions: readonly SyncSessionInput[]): Promise; + refresh(): Promise; + reindex(): Promise; + status(): Promise; + dispose(): Promise; } -function encodePageToken( - q: NormalizedQuery, - source: GlobalSearchSource, - boundary: SortBoundary, - generation: number | undefined, -): string { - return Buffer.from( - JSON.stringify({ v: PAGE_TOKEN_VERSION, f: tokenFingerprint(q, source), g: generation, b: boundary }), - ).toString('base64url'); -} +/** Rollback host: the search-index core running on the main thread. */ +export class InlineSearchBackend implements SearchBackend { + readonly core: SearchIndexCore; -function decodePageToken( - q: NormalizedQuery, - source: GlobalSearchSource, - token: string | undefined, - generation: number | undefined, -): DecodedPage { - if (token === undefined) return { kind: 'first' }; - let parsed: unknown; - try { - parsed = JSON.parse(Buffer.from(token, 'base64url').toString('utf8')); - } catch { - throw new GlobalSearchError('invalid_page_token', 'pageToken is malformed'); - } - if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new GlobalSearchError('invalid_page_token', 'pageToken is malformed'); - } - const p = parsed as { v?: unknown; f?: unknown; s?: unknown; g?: unknown; b?: unknown }; - if (p.f !== tokenFingerprint(q, source)) { - throw new GlobalSearchError( - 'invalid_page_token', - 'pageToken does not match the query conditions; query conditions must not change mid-pagination', - ); - } - if (p.v === undefined) { - // Legacy v1 offset token (`{f, s}`) — transition window: answer it with - // offset semantics; the response issues a v2 keyset token back. - if (typeof p.s !== 'number' || !Number.isInteger(p.s) || p.s < 0) { - throw new GlobalSearchError('invalid_page_token', 'pageToken is malformed'); - } - return { kind: 'legacy', skip: p.s }; - } - if (p.v !== PAGE_TOKEN_VERSION) { - throw new GlobalSearchError('invalid_page_token', 'pageToken has an unsupported version'); - } - if (generation !== undefined && p.g !== generation) { - throw new GlobalSearchError( - 'invalid_page_token', - 'pageToken was issued by an older index generation (the index was rebuilt, reopened or rescanned); restart the search', - ); - } - const width = boundaryWidth(q); - if ( - !Array.isArray(p.b) || - p.b.length !== width || - typeof p.b[0] !== 'number' || - typeof p.b[width - 1] !== 'string' || - (width === 3 && typeof p.b[1] !== 'number') - ) { - throw new GlobalSearchError('invalid_page_token', 'pageToken is malformed'); - } - return { kind: 'keyset', boundary: p.b as SortBoundary }; -} - -// --------------------------------------------------------------------------- -// Sort order, boundary filtering and bounded collection (both routes) -// --------------------------------------------------------------------------- - -/** One matched document with its stable key and match context. */ -interface MatchedRow { - readonly key: string; - readonly value: MessageDoc | TitleDoc; - readonly score: number; - /** Literal mode: offset of the confirmed match, reused as snippet anchor. */ - readonly anchor?: number; -} - -/** Per-query work budget for the match/confirm phase (both routes). */ -interface MatchBudget { - /** Date.now() timestamp after which matching stops with 'deadline'. */ - readonly deadlineAt: number; - /** Remaining document text (UTF-16 code units) literal confirmation may - * process before stopping with 'deadline'. */ - textCharsLeft: number; -} - -function cmpKey(a: string, b: string): number { - return a < b ? -1 : a > b ? 1 : 0; -} - -/** - * The query's total order (negative = `a` ranks before `b`): - * - literal mode (sort is a terms-mode concept) and `time_desc`: - * (time desc, key asc); - * - `time_asc`: (time asc, key asc); - * - `score`: (score desc, time desc, key asc). - */ -function compareRows(q: NormalizedQuery, a: MatchedRow, b: MatchedRow): number { - if (q.mode !== 'literal' && q.sort === 'score') { - return b.score - a.score || b.value.time - a.value.time || cmpKey(a.key, b.key); - } - if (q.mode !== 'literal' && q.sort === 'time_asc') { - return a.value.time - b.value.time || cmpKey(a.key, b.key); - } - return b.value.time - a.value.time || cmpKey(a.key, b.key); -} - -/** The boundary tuple of a row — the keyset cursor payload. */ -function boundaryOf(q: NormalizedQuery, row: MatchedRow): SortBoundary { - return boundaryWidth(q) === 3 ? [row.score, row.value.time, row.key] : [row.value.time, row.key]; -} - -/** Whether the row ranks strictly AFTER the boundary in the sort order. */ -function rowAfterBoundary(q: NormalizedQuery, row: MatchedRow, boundary: SortBoundary): boolean { - let cmp: number; - if (boundary.length === 3) { - const [bs, bt, bk] = boundary as readonly [number, number, string]; - cmp = bs - row.score || bt - row.value.time || cmpKey(row.key, bk); - } else { - const [bt, bk] = boundary as readonly [number, string]; - cmp = - q.mode !== 'literal' && q.sort === 'time_asc' - ? row.value.time - bt || cmpKey(row.key, bk) - : bt - row.value.time || cmpKey(row.key, bk); - } - return cmp > 0; -} - -/** - * Bounded collector for the K best rows in the query's sort order — same - * worst-at-root heap shape as minidb's TopK: O(log K) per row and K rows in - * memory instead of an O(E log E) sort over every eligible row. Deep pages - * stay proportional to pageSize. - */ -class RowTopK { - private readonly a: MatchedRow[] = []; - - constructor( - private readonly q: NormalizedQuery, - private readonly k: number, - ) {} - - private worse(x: MatchedRow, y: MatchedRow): boolean { - return compareRows(this.q, x, y) > 0; // x ranks after y + constructor(options: { indexDir: string; log: ILogService }) { + // A fresh boot salt per service instance: page tokens never validate + // across a service/process restart (the core's local generation counter + // restarts from 0), mirroring the worker host's per-spawn salt. The held + // lock token is registered process-wide so a worker-host peer's orphan + // detector never mistakes this inline writer's lock for a dead worker's. + this.core = new SearchIndexCore({ + ...options, + bootSalt: randomUUID(), + onLockToken: noteLiveLockToken, + }); } - offer(row: MatchedRow): void { - const a = this.a; - if (a.length < this.k) { - a.push(row); - let i = a.length - 1; - while (i > 0) { - const p = (i - 1) >> 1; - if (!this.worse(a[i]!, a[p]!)) break; - [a[p], a[i]] = [a[i]!, a[p]!]; - i = p; - } - return; - } - if (this.k === 0 || !this.worse(a[0]!, row)) return; // must beat the worst kept - a[0] = row; - let i = 0; - for (;;) { - let w = i; - const l = 2 * i + 1; - const r = 2 * i + 2; - if (l < a.length && this.worse(a[l]!, a[w]!)) w = l; - if (r < a.length && this.worse(a[r]!, a[w]!)) w = r; - if (w === i) break; - [a[w], a[i]] = [a[i]!, a[w]!]; - i = w; - } + beginClose(): void { + this.core.beginClose(); } - /** The kept rows in final rank order. */ - sorted(): MatchedRow[] { - return this.a.sort((x, y) => compareRows(this.q, x, y)); + lifecycleSnapshot(): CoreLifecycleReport { + return this.core.lifecycleState(); + } + + ensureOpen(): Promise { + return this.core.ensureOpen(); + } + + search(params: CoreSearchParams): Promise { + return this.core.search(params); + } + + sync(sessions: readonly SyncSessionInput[]): Promise { + return this.core.sync(sessions); + } + + refresh(): Promise { + return this.core.refresh(); + } + + reindex(): Promise { + return this.core.reindex(); + } + + status(): Promise { + return this.core.status(); + } + + dispose(): Promise { + dropLiveLockToken(this.core.lockTokenView); + return this.core.close(); } } @@ -783,56 +417,35 @@ export class GlobalSearchService implements IGlobalSearchService { /** Max distinct query terms in terms mode (test knob). */ maxQueryTerms = MAX_QUERY_TERMS; - private db: MiniDb | null = null; - private openPromise: Promise | null = null; + private readonly backend: SearchBackend; private syncPromise: Promise | null = null; private refreshPromise: Promise | null = null; private lastSyncStartedAt = 0; - private fullSyncDone = false; - /** WAL watermark (bytes applied) for read-only catch-up. */ - private walOffset = 0; - private fingerprint = ''; private summaries = new Map(); private disposed = false; - /** - * Dispose drain gate (plan 12's OpTracker primitive, consumed per plan - * 13): every lifecycle-managed background op (sync pass, read-only - * refresh) enters it; dispose() closes the gate (new ops skip) and drains - * the in-flight ones BEFORE closing the db, so no background task ever - * touches a closed handle (review #20). - */ - private readonly ops = new OpTracker(); /** Set while `reindex()` swaps the db — syncs started meanwhile are no-ops. */ private reindexing = false; /** Live-transcript source for the in-memory route; null until start.ts wires it. */ private liveSource: LiveTranscriptSource | null = null; - /** - * Identity of the published index base: bumped on every open/reopen - * (initial open, read-only swap, reindex) and on a sync pass that REPLACED - * already-indexed documents (shrink rescan, title overwrite). Page tokens - * pin it; additive/deletion-only passes deliberately keep it stable so - * keyset pagination over a live index is not constantly restarted (see - * contract.ts for the weak-consistency semantics). - */ - private generation = 0; - /** Set by a sync pass when it replaced indexed documents → generation bump. */ - private syncReplaced = false; /** One queued follow-up pass behind the in-flight one (backpressure). */ private syncQueued = false; /** Trailing-pass timer behind the debounce window. */ private syncTimer: ReturnType | null = null; - /** Last background refresh/sync/reindex failure — surfaced as degraded. */ + /** Last background sync/reindex/worker failure — surfaced as degraded. */ private lastRefreshError: { at: number; message: string } | null = null; - /** Last open failure — a search with no published generation fails fast. */ - private openError: string | null = null; - /** One-time per-process migration flag for pre-v2 file-meta keys. */ - private fileMetaMigrated = false; + /** Set when dispose()'s async drain finished — lifecycle 'stopped'. */ + private drainSettled = false; constructor( @ISessionIndex private readonly sessionIndex: ISessionIndex, @IBootstrapService private readonly bootstrap: IBootstrapService, @ILogService private readonly log: ILogService, + @IFlagService private readonly flags: IFlagService, ) { + const indexDir = join(this.bootstrap.homeDir, INDEX_DIR_NAME); + this.backend = this.flags.enabled(SEARCH_WORKER_FLAG_ID) + ? new SearchWorkerHost({ dir: indexDir, log: this.log }) + : new InlineSearchBackend({ indexDir, log: this.log }); // App-scope OnScopeCreated activation: kick the first full sync off in the // background so server bootstrap never blocks on indexing. this.requestSync(); @@ -848,111 +461,18 @@ export class GlobalSearchService implements IGlobalSearchService { return join(this.bootstrap.homeDir, INDEX_DIR_NAME); } - private ensureOpen(): Promise { - this.openPromise ??= this.openDb().then( - () => { - this.openError = null; - }, - (error: unknown) => { - this.openPromise = null; - this.openError = errorMessage(error); - throw error; - }, - ); - return this.openPromise; - } - - private async openDb(): Promise { - const db = await this.openSearchDb(); - // The scope may have been disposed while the (slow) open was in flight — - // close the handle immediately instead of leaking it and writing the - // text-index definition below into a directory the caller may already be - // deleting. - if (this.disposed) { - await db.close().catch(() => {}); - throw new GlobalSearchError('index_unavailable', 'search service is disposed'); - } - await this.publishDb(db, null); - } - - /** - * Swap a freshly opened db in as the new published generation: writer-side - * text-index definitions and the (handle-independent) fingerprint are - * computed BEFORE the swap, so a failure closes `next` and leaves `prev` - * (or the no-db state) untouched; the swap itself is one synchronous - * segment with no failure point between publishing `next` and closing - * `prev`. - */ - private async publishDb(next: MiniDb, prev: MiniDb | null): Promise { - let fingerprint: string; - try { - if (!next.readOnly) { - // Both indexes are created here (not at first write) so a - // pre-existing db gets the tri index built over its current documents - // on first open after the upgrade, and a read-only peer only ever - // reopens on the definitions-file fingerprint change. - for (const [name, options] of [ - [TEXT_INDEX_NAME, { fields: ['text'] }], - [TRI_INDEX_NAME, { fields: ['text'], tokenizer: 'ngram' }], - ] as const) { - try { - await next.createTextIndex(name, options); - } catch (error) { - if (!(error instanceof Error && error.message.includes('already exists'))) throw error; - } - } - } - fingerprint = await this.computeFingerprint(); - } catch (error) { - await next.close().catch(() => {}); - throw error; - } - this.db = next; - this.walOffset = next.recoveryInfo?.walScanEnd ?? 0; - this.generation++; - this.fingerprint = fingerprint; - if (prev !== null) await prev.close().catch(() => {}); - } - - /** - * Open the index db, rebuilding from scratch on unrecoverable corruption - * (the index is derived data — never repaired, only rebuilt). - * - * Rebuild is WRITER-ONLY: a process that fails to grab the write lock must - * never delete the directory out from under the live indexer. Lock state is - * not observable once `open` throws, so corruption is disambiguated with a - * probe open WITHOUT `onLockFail`: it throws `LockError` before recovery - * when another process holds the lock, and re-throws the corruption - * (releasing the lock) when the lock is free — in which case this process - * is the would-be writer and may rebuild. - */ - private async openSearchDb(): Promise> { - const opts = { - dir: this.indexDir, - valueCodec: 'json', - fsyncPolicy: 'everysec', - onLockFail: 'readonly', - } as const; - try { - return await MiniDb.open(opts); - } catch (error) { - if (!isRebuildableCorruption(error)) throw error; - let probeError: unknown; - try { - const probe = await MiniDb.open({ dir: opts.dir, valueCodec: opts.valueCodec }); - await probe.close().catch(() => {}); - probeError = undefined; // lock free AND data fine — cannot happen, but treat as rebuildable - } catch (error) { - probeError = error; - } - if (probeError instanceof LockError) { - // Another process holds the write lock: leave its files alone. The - // caller's open fails; the next search retries from scratch. - throw error; - } - await rm(this.indexDir, { recursive: true, force: true }); - return MiniDb.open(opts); - } + private toSyncInput(summary: SessionSummary): SyncSessionInput { + return { + id: summary.id, + workspaceId: summary.workspaceId, + title: summary.title, + updatedAt: summary.updatedAt, + dir: sessionDirOf( + this.bootstrap.homeDir, + workspacePersistenceScope(this.bootstrap.scope('sessions'), summary.workspaceId), + summary.id, + ), + }; } dispose(): void { @@ -961,116 +481,32 @@ export class GlobalSearchService implements IGlobalSearchService { clearTimeout(this.syncTimer); this.syncTimer = null; } - // DI disposal is synchronous, but draining background work and closing a - // MiniDb are not: close the op gate SYNCHRONOUSLY (new syncs/refreshes - // skip at enter()), wait for every in-flight op and the in-flight open - // to settle, then release and close the handle — no background task can - // touch a closed db (review #20). The promise is registered module-level - // so the server shutdown path (`drainGlobalSearchDisposals` in start.ts) - // can await it before the homeDir is torn down — otherwise teardown rm() - // races the close and fails with ENOTEMPTY. + // Synchronously gate the backend BEFORE any awaiting: an in-flight sync + // pass skips its remaining writes at the next disposed check (the + // review-#20 drain contract). Both hosts honor it: the inline core flips + // its own flag; the worker host forwards a beginClose control message so + // the worker-side core gates mid-pass too. + this.backend.beginClose(); + // DI disposal is synchronous, but draining background work and closing + // the backend are not: wait for the in-flight coordinator facades to + // settle (bounded now — a gated pass returns early), then dispose the + // backend (the inline core drains its tracked ops before closing the db; + // the worker drains and closes on the `close` request, with terminate() + // as the host's bounded backstop). The promise is + // registered module-level so the server shutdown path + // (`drainGlobalSearchDisposals` in start.ts) can await it before the + // homeDir is torn down — otherwise teardown rm() races the close and + // fails with ENOTEMPTY. const pending = (async () => { - await this.ops.close(); - await this.openPromise?.catch(() => {}); - const db = this.db; - this.db = null; - if (db) await db.close().catch(() => {}); + await this.syncPromise?.catch(() => {}); + await this.refreshPromise?.catch(() => {}); + await this.backend.dispose(); + this.drainSettled = true; })(); pendingDisposals.add(pending); void pending.finally(() => pendingDisposals.delete(pending)); } - /** - * Run one lifecycle-managed background op under the dispose drain gate: - * skipped once dispose has started, and dispose waits for every op that - * already entered before it closes the db (review #20). - */ - private async tracked(op: () => Promise): Promise { - if (!this.ops.enter()) return; - try { - await op(); - } finally { - this.ops.leave(); - } - } - - // -- read-only freshness (fingerprint + WAL catch-up) ------------------------- - - private async computeFingerprint(): Promise { - const parts: string[] = []; - for (const name of ['db.wal', 'db.snapshot', 'db.textindexes.json']) { - try { - const s = await stat(join(this.indexDir, name)); - parts.push(`${name}:${s.dev}:${s.ino}:${s.mtimeMs}:${s.size}`); - } catch { - parts.push(`${name}:-`); - } - } - return parts.join('|'); - } - - /** - * Bring a read-only instance up to date with the indexer's committed - * writes. Unchanged fingerprint → zero IO; WAL pure-append → incremental - * `catchUpFromWal`; anything else → open the replacement db and swap (which - * may also promote this process to indexer when the old writer's lock is - * gone). Single-flight; a failure is recorded in `lastRefreshError` and - * the stale generation keeps serving (surfaced as `indexState.degraded`). - */ - private refreshReadonly(): Promise { - this.refreshPromise ??= this.tracked(() => this.doRefreshReadonly()) - .then( - () => { - this.lastRefreshError = null; - }, - (error: unknown) => { - // A failed refresh must not fail the search — serve the stale view, - // but no longer swallow the error silently. - this.lastRefreshError = { at: Date.now(), message: errorMessage(error) }; - this.log.warn('global search: read-only refresh failed; serving the stale view', { - error: errorMessage(error), - }); - }, - ) - .finally(() => { - this.refreshPromise = null; - }); - return this.refreshPromise; - } - - private async doRefreshReadonly(): Promise { - const db = this.db; - if (!db || !db.readOnly || this.disposed) return; - const fp = await this.computeFingerprint(); - if (fp === this.fingerprint) return; - const [, snapPrev, defsPrev] = this.fingerprint.split('|'); - const [, snapNow, defsNow] = fp.split('|'); - if (snapPrev === snapNow && defsPrev === defsNow) { - const res = await db.catchUpFromWal(this.walOffset); - if (res !== null) { - this.walOffset = res.offset; - this.fingerprint = fp; - return; - } - } - // WAL rotated/truncated, snapshot or index definitions changed, or the - // watermark no longer aligns: reopen from scratch. The replacement is - // opened and published BEFORE the stale handle closes, so a failed - // reopen leaves the previous generation servable instead of dropping - // the index out from under in-flight searches. - const next = await this.openSearchDb(); - if (this.disposed) { - await next.close().catch(() => {}); - return; - } - if (this.db !== db) { - // A concurrent refresh already swapped: just close the duplicate. - await next.close().catch(() => {}); - return; - } - await this.publishDb(next, db); - } - // -- sync coordinator (indexer only) ------------------------------------------- // // Requests never await a sync; they ask the coordinator to schedule one. @@ -1119,7 +555,7 @@ export class GlobalSearchService implements IGlobalSearchService { /** Single-flight: concurrent callers share the in-flight sync. */ private ensureSyncStarted(): Promise { if (this.syncPromise === null) { - const p = this.tracked(() => this.runSync()).finally(() => { + const p = this.runSync().finally(() => { if (this.syncPromise === p) this.syncPromise = null; }); this.syncPromise = p; @@ -1131,94 +567,19 @@ export class GlobalSearchService implements IGlobalSearchService { // `reindexing`: a rebuild is swapping the db out — this pass is a no-op; // the rebuild itself runs the authoritative sync when done. if (this.disposed || this.reindexing) return; - this.syncReplaced = false; const sessions = await this.listAllSessions(); + if (this.disposed) return; // dispose landed while sessions were enumerated // Nothing to index and no index on disk yet: don't even create the // `/search-index` directory — it would show up in the fs folder // picker and cost every server boot a pointless db open. if (sessions.length === 0 && !(await pathExists(this.indexDir))) { this.summaries = new Map(); this.lastSyncStartedAt = Date.now(); - this.fullSyncDone = true; return; } - - await this.ensureOpen(); - const db = this.db; - if (!db || db.readOnly || this.disposed) return; - this.lastSyncStartedAt = Date.now(); - - // One-time rewrite of pre-v2 hash-only file-meta keys, inside the - // background pass — never in the query path. After it, every per-session - // lookup below scans only that session's meta prefix. - await this.migrateFileMetaKeys(db); - this.summaries = new Map(sessions.map((s) => [s.id, s])); - const currentIds = new Set(sessions.map((s) => s.id)); - - // Drop sessions whose directory disappeared since the last sync. The - // disposed gate covers this loop and the trailing stats write (review - // #20): once dispose starts, the pass skips them instead of writing into - // a db whose close is already draining. - for (const row of db.query({ key: { prefix: SESSION_META_PREFIX }, project: [] })) { - if (this.disposed) return; - const sessionId = row.key.slice(SESSION_META_PREFIX.length); - if (!currentIds.has(sessionId)) await this.deleteSessionDocs(db, sessionId); - } - - let indexed = 0; - for (const summary of sessions) { - if (this.disposed) return; - try { - await this.syncSession(db, summary); - indexed++; - } catch (error) { - // One unreadable session must not abort the whole pass. - this.log.warn('global search: failed to index session', { - sessionId: summary.id, - error: error instanceof Error ? error.message : String(error), - }); - } - } - - if (this.disposed) return; // dispose started mid-pass: skip the trailing write (review #20) - const metaCount = db.query({ key: { prefix: '\0meta\\' }, project: [] }).length; - const stats: StatsDoc = { - kind: 'stats', - sessions: indexed, - documents: db.size - metaCount, - lastIndexedAt: Date.now(), - }; - await db.set(STATS_KEY, stats); - this.fullSyncDone = true; - if (this.syncReplaced) { - // The pass REPLACED indexed documents (shrink rescan / title - // overwrite), so their sort keys may have moved: page tokens from the - // previous generation must restart instead of drifting. - this.generation++; - } - } - - /** - * One-time per-process migration of pre-v2 hash-only file-meta keys to the - * session-scoped format (`fileMetaKey`). A single full prefix scan of the - * meta namespace; per-session work afterwards only scans that session's - * keys. Idempotent — a crash mid-migration just rescans on the next pass. - */ - private async migrateFileMetaKeys(db: MiniDb): Promise { - if (this.fileMetaMigrated) return; - const ops: BatchInputOp[] = []; - for (const row of db.query({ key: { prefix: FILE_META_PREFIX }, project: [] })) { - const rest = row.key.slice(FILE_META_PREFIX.length); - if (rest.includes('\\')) continue; // already session-scoped - const meta = row.value; - if (meta.kind !== 'fileMeta') continue; - ops.push({ op: 'set', key: fileMetaKey(meta.sessionId, meta.path), value: meta }); - ops.push({ op: 'del', key: row.key }); - } - // Batch the rewrite instead of one op per key; empty on every later pass. - if (ops.length > 0) await db.batch(ops); - this.fileMetaMigrated = true; + this.lastSyncStartedAt = Date.now(); + await this.backend.sync(sessions.map((s) => this.toSyncInput(s))); } private async listAllSessions(): Promise { @@ -1232,291 +593,33 @@ export class GlobalSearchService implements IGlobalSearchService { return out; } - private async deleteSessionDocs(db: MiniDb, sessionId: string): Promise { - for (const row of db.query({ key: { prefix: `${sessionId}/` }, project: [] })) { - await db.del(row.key); - } - for (const row of db.query({ key: { prefix: fileMetaPrefixFor(sessionId) }, project: [] })) { - await db.del(row.key); - } - await db.del(SESSION_META_PREFIX + sessionId); - } - - private async syncSession(db: MiniDb, summary: SessionSummary): Promise { - const sessionDir = sessionDirOf( - this.bootstrap.homeDir, - workspacePersistenceScope(this.bootstrap.scope('sessions'), summary.workspaceId), - summary.id, - ); - const wireFiles = await collectWireFiles(sessionDir); - const seenPaths = new Set(wireFiles.map((file) => file.path)); - - // A wire file that vanished on its own (e.g. one agent's log deleted - // while the session lives on): drop its docs and meta. Session-level - // disappearance is handled separately in runSync. The scan is scoped to - // THIS session's meta prefix — O(files of this session), independent of - // the global session count. - for (const row of db.query({ key: { prefix: fileMetaPrefixFor(summary.id) } })) { - const meta = row.value; - if (meta.kind !== 'fileMeta') continue; - if (seenPaths.has(meta.path)) continue; - await this.deleteFileDocs(db, meta); - await db.del(row.key); - } - - for (const file of wireFiles) { - await this.syncWireFile(db, summary, file); - } - - const title = summary.title ?? ''; - const titleKey = `${summary.id}/$title`; - const existing = db.get(titleKey); - if (title.length > 0) { - if (existing?.kind !== 'title' || existing.text !== title) { - const doc: TitleDoc = { - kind: 'title', - sessionId: summary.id, - workspaceId: summary.workspaceId, - sessionTitle: title, - agentId: '', - role: 'title', - text: title, - time: summary.updatedAt, - }; - await db.set(titleKey, doc); - // Overwriting an existing title doc moves its sort key mid-pagination - // — a replacing change, unlike the additive first-time create. - if (existing !== undefined) this.syncReplaced = true; - } - } else if (existing !== undefined) { - await db.del(titleKey); - } - // Session marker: presence is the information — write only when missing. - if (db.get(SESSION_META_PREFIX + summary.id) === undefined) { - const sessionMeta: SessionMetaDoc = { kind: 'sessionMeta' }; - await db.set(SESSION_META_PREFIX + summary.id, sessionMeta); - } - } - - private async deleteFileDocs(db: MiniDb, meta: FileMetaDoc): Promise { - const prefix = `${meta.sessionId}/${meta.agentId}/${meta.source}:`; - for (const row of db.query({ key: { prefix }, project: [] })) { - await db.del(row.key); - } - } - - private async syncWireFile( - db: MiniDb, - summary: SessionSummary, - file: WireFileRef, - ): Promise { - let st: { size: number; mtimeMs: number; ino: number }; - try { - st = await stat(file.path); - } catch { - return; // transiently unreadable — retry next pass - } - const size = st.size; - const metaKey = fileMetaKey(summary.id, file.path); - // New session-scoped key first, then the pre-v2 hash-only key (a cheap - // point lookup, not a scan): metas written before the key migration are - // honored and opportunistically rewritten under the new key. - let meta = db.get(metaKey); - let legacyKey: string | null = null; - if (meta?.kind !== 'fileMeta') { - const oldKey = legacyFileMetaKey(file.path); - const legacy = db.get(oldKey); - if (legacy?.kind === 'fileMeta') { - meta = legacy; - legacyKey = oldKey; - } - } - const known = meta?.kind === 'fileMeta' ? meta : undefined; - let offset = known?.offset ?? 0; - let turnState: TurnCounterState = known?.turnState ?? initialTurnState(); - let stepState: StepTrackerState = known?.stepState ?? initialStepState(); - const fileMeta = ( - nextOffset: number, - turns: TurnCounterState, - steps: StepTrackerState, - ): FileMetaDoc => ({ - kind: 'fileMeta', - sessionId: summary.id, - agentId: file.agentId, - source: file.source, - path: file.path, - offset: nextOffset, - size, - mtimeMs: st.mtimeMs, - ino: st.ino, - turnState: turns, - stepState: steps, - }); - // Metas written before step tracking carry no `stepState`: rescan the - // file from scratch so stepIds are all-or-nothing per file rather than - // drifting mid-file (the shrink path does exactly this). - const legacyMeta = known !== undefined && known.stepState === undefined; - // An inode change means the file was replaced (atomic rewrite); a bumped - // mtime at an unchanged size means an in-place rewrite. Both invalidate - // the byte-offset watermark even though the size alone would not. - const replacedFile = known?.ino !== undefined && known.ino !== st.ino; - const rewrittenInPlace = - known?.mtimeMs !== undefined && size === known.offset && st.mtimeMs > known.mtimeMs; - if (size < offset || legacyMeta || replacedFile || rewrittenInPlace) { - // File was rebuilt/truncated: drop its docs and rescan from scratch — - // the turn counter and step tracker restart with it. A replacing - // change: the docs' sort keys may move → bump the generation. - this.syncReplaced = true; - await this.deleteFileDocs(db, fileMeta(0, initialTurnState(), initialStepState())); - offset = 0; - turnState = initialTurnState(); - stepState = initialStepState(); - } - if (size === offset) { - // No growth: only rewrite the meta when something actually changed - // (first sight, stat refresh after an upgrade, legacy key cleanup) — - // an unchanged file must not cost a WAL record per pass. - if ( - legacyKey !== null || - known === undefined || - known.size !== size || - known.mtimeMs !== st.mtimeMs || - known.ino !== st.ino || - known.offset !== offset - ) { - const ops: BatchInputOp[] = [ - { op: 'set', key: metaKey, value: fileMeta(offset, turnState, stepState) }, - ]; - if (legacyKey !== null) ops.push({ op: 'del', key: legacyKey }); - await db.batch(ops); - } - return; - } - - // Read only the new byte range, in bounded chunks, consuming complete - // lines; a trailing partial line (or a short read from a mid-read - // truncation) is left for the next pass — the watermark below never - // advances past bytes that were actually consumed. The line loop keeps - // only line-sized strings alive instead of one `size - offset` buffer - // plus a full split array. - const handle = await open(file.path, 'r'); - const ops: BatchInputOp[] = []; - let byteCursor = offset; - try { - let position = offset; - let pending: Buffer = EMPTY_BUFFER; // partial-line bytes starting at byteCursor - const chunk = Buffer.allocUnsafe(WIRE_READ_CHUNK_BYTES); - while (position < size) { - if (this.disposed) return; // meta not advanced: the next pass redoes the file - const { bytesRead } = await handle.read( - chunk, - 0, - Math.min(chunk.length, size - position), - position, - ); - if (bytesRead === 0) break; - const slice = chunk.subarray(0, bytesRead); - position += bytesRead; - let start = 0; - for (;;) { - const nl = slice.indexOf(0x0a, start); - if (nl === -1) break; - const lineBuf = - pending.length > 0 - ? Buffer.concat([pending, slice.subarray(start, nl)]) - : slice.subarray(start, nl); - pending = EMPTY_BUFFER; - const lineOffset = byteCursor; - byteCursor += lineBuf.length + 1; - ({ turnState, stepState } = this.collectWireLine( - ops, - summary, - file, - lineBuf.toString('utf8'), - lineOffset, - { turnState, stepState }, - )); - start = nl + 1; - } - // The chunk buffer is reused, so the unconsumed tail must be copied. - pending = - pending.length > 0 - ? Buffer.concat([pending, slice.subarray(start)]) - : Buffer.from(slice.subarray(start)); - if (ops.length >= WIRE_BATCH_OPS) { - await db.batch(ops); - ops.length = 0; - } - } - } finally { - await handle.close(); - } - - if (byteCursor === offset && legacyKey === null) return; // no complete line yet - ops.push({ op: 'set', key: metaKey, value: fileMeta(byteCursor, turnState, stepState) }); - if (legacyKey !== null) ops.push({ op: 'del', key: legacyKey }); - await db.batch(ops); - } - /** - * Turn/step counting and doc extraction for one complete wire line. - * Returns the counter states advanced by the line (they are immutable and - * replaced per line, so the caller threads them through the chunk loop). + * Drive a read-only refresh of the backend (single-flight facade). + * Production note: request-path searches no longer call this — the core + * kicks read-only refreshes internally and reports freshness via + * `CoreIndexView.freshnessStale`. The facade remains as the deterministic + * refresh drive for tests (`refreshNow`) and its promise feeds the + * read-only-side `stale` bit while an explicit refresh is in flight. The + * core records refresh failures itself (surfaced as `degraded`); only + * worker-availability failures land in the rejection branch here — a + * failed refresh must never fail the search that kicked it. */ - private collectWireLine( - ops: BatchInputOp[], - summary: SessionSummary, - file: WireFileRef, - line: string, - lineOffset: number, - counters: { turnState: TurnCounterState; stepState: StepTrackerState }, - ): { turnState: TurnCounterState; stepState: StepTrackerState } { - let { turnState, stepState } = counters; - const analysis = analyzeWireLine(line); - // Turn counting runs independently of indexing: every line moves the - // counter (a text-less user message still opens a turn). - const advanced = advanceTurnCounter(turnState, analysis.turn); - // A turn boundary invalidates the step mapping: a new turn opens - // (`open`, or `ensure` opening a fallback turn from no-turn), or an - // `undo` rewinds the counter mid-turn. - if ( - analysis.turn.kind === 'open' || - analysis.turn.kind === 'undo' || - (analysis.turn.kind === 'ensure' && !turnState.hasTurn) - ) { - stepState = initialStepState(); - } - turnState = advanced.state; - stepState = advanceStepTracker(stepState, analysis.step); - const extracted = analysis.messages; - for (let i = 0; i < extracted.length; i++) { - const e = extracted[i]!; - const stepOrdinal = e.stepUuid !== undefined ? stepState.byUuid[e.stepUuid] : undefined; - const doc: MessageDoc = { - kind: 'message', - sessionId: summary.id, - workspaceId: summary.workspaceId, - sessionTitle: summary.title ?? '', - agentId: file.agentId, - role: e.role, - text: e.text.length > MAX_DOC_TEXT_CHARS ? e.text.slice(0, MAX_DOC_TEXT_CHARS) : e.text, - time: e.time ?? summary.updatedAt, - turn: advanced.docTurn, - // A doc whose step cannot be resolved (no `step.begin` seen, or a - // turn boundary invalidated the mapping) just omits the id. - stepId: - advanced.docTurn !== undefined && stepOrdinal !== undefined - ? `t${advanced.docTurn}.${stepOrdinal}` - : undefined, - }; - // A line can yield several docs — the per-line index keeps keys unique. - ops.push({ - op: 'set', - key: `${docKeyPrefix(summary.id, file)}${lineOffset}:${i}`, - value: doc, + private refreshReadonly(): Promise { + if (this.refreshPromise === null) { + this.refreshPromise = this.backend.refresh().then( + () => {}, + (error: unknown) => { + this.lastRefreshError = { at: Date.now(), message: errorMessage(error) }; + this.log.warn('global search: read-only refresh failed; serving the stale view', { + error: errorMessage(error), + }); + }, + ); + void this.refreshPromise.finally(() => { + this.refreshPromise = null; }); } - return { turnState, stepState }; + return this.refreshPromise; } // -- public API --------------------------------------------------------------- @@ -1568,7 +671,7 @@ export class GlobalSearchService implements IGlobalSearchService { await source.ensureAgentHistory(sessionId, agentId); } const docs = await this.collectLiveDocs(sessionId, store, agentIds); - const budget: MatchBudget = { + const budget = { deadlineAt: Date.now() + this.queryDeadlineMs, textCharsLeft: this.queryTextBudgetChars, }; @@ -1578,19 +681,29 @@ export class GlobalSearchService implements IGlobalSearchService { // in-memory AND match first, scoring each hit. const matched = q.mode === 'literal' - ? this.matchDocs( + ? matchDocs( q, docs.map(({ key, value }) => ({ key, value, score: 0 })), boundary, budget, ) - : this.matchDocs(q, matchLiveTerms(q.termsQuery ?? [], docs), boundary, budget); - return this.toPage(q, 'live', page, matched.rows, matched.incomplete, { - state: 'ready', - indexedSessions: 1, - totalSessions: 1, - documents: docs.length, - }); + : matchDocs(q, matchLiveTerms(q.termsQuery ?? [], docs), boundary, budget); + const { pageRows, hasMore } = paginateRows(q, page, matched.rows); + return { + items: pageRows.map((row) => this.projectHit(q, row)), + hasMore, + pageToken: hasMore + ? encodePageToken(q, 'live', boundaryOf(q, pageRows[pageRows.length - 1]!), undefined) + : undefined, + incomplete: matched.incomplete, + indexState: { + state: 'ready', + indexedSessions: 1, + totalSessions: 1, + documents: docs.length, + }, + source: 'live', + }; } /** @@ -1686,7 +799,17 @@ export class GlobalSearchService implements IGlobalSearchService { return docs; } - // -- index route (minidb) ------------------------------------------------------- + // -- index route (backend: search worker, or the inline core) -------------------- + + private budgets(): SearchBudgets { + return { + literalCandidateCap: this.literalCandidateCap, + maxTextHits: this.maxTextHits, + postingsVisitBudget: this.postingsVisitBudget, + queryDeadlineMs: this.queryDeadlineMs, + queryTextBudgetChars: this.queryTextBudgetChars, + }; + } private async searchIndex( q: NormalizedQuery, @@ -1715,301 +838,142 @@ export class GlobalSearchService implements IGlobalSearchService { } // The request path serves the currently published generation and never - // waits for an open, sync, reopen or reindex: with no published base yet - // it answers with `building` semantics and lets the background - // coordinator catch up. - const db = this.db; - if (db === null) { - if (this.disposed) { - throw new GlobalSearchError('index_unavailable', 'search service is disposed'); - } - if (this.openError !== null) { - // The last open failed (e.g. a read-only open racing a writer's - // compaction): surface the failure, but ALSO kick a background retry - // (runSync → ensureOpen), so search traffic self-heals the index once - // the transient cause goes away — a successful retry clears openError. - this.requestSync(); - throw new GlobalSearchError( - 'index_unavailable', - `search index failed to open: ${this.openError}`, - ); - } - if (pageToken !== undefined) { - // No generation to validate the token against — the client restarts - // the search once a base is published. - throw new GlobalSearchError( - 'invalid_page_token', - 'the search index is not ready yet; restart the search', - ); - } - this.requestSync(); // kicks the open + first sync if nothing is running - return this.buildingPage(null); - } - - let stale: boolean; - let serveDb = db; - if (serveDb.readOnly) { - // Cheap freshness probe (3 stats). A changed fingerprint refreshes in - // the BACKGROUND — this request deliberately serves the stale - // generation instead of waiting for a catch-up or a full reopen. - let fp: string | null = null; - try { - fp = await this.computeFingerprint(); - } catch (error) { - this.lastRefreshError = { at: Date.now(), message: errorMessage(error) }; - } - // A background refresh may have swapped (and closed) the captured - // handle during the await. Re-pin to the currently published handle. - // (The stage-6 async query path awaits again below, so a later swap - // can still close it mid-query — the bounded pass retries once on the - // fresh handle for exactly that race.) - if (this.db === null) { - throw new GlobalSearchError('index_unavailable', 'search service is disposed'); - } - serveDb = this.db; - if (serveDb.readOnly) { - stale = fp === null || fp !== this.fingerprint || this.refreshPromise !== null; - if (fp !== null && fp !== this.fingerprint) void this.refreshReadonly(); - } else { - // The reopen promoted this process to writer (the old writer's lock - // was gone): serve from it and kick the coordinator like a writer. - this.requestSync(); - stale = this.syncPromise !== null || this.syncQueued || this.syncTimer !== null; - } - } else { - // Writer: kick the coordinator (never awaited); the served generation - // is the one published by the last completed pass. - this.requestSync(); - stale = this.syncPromise !== null || this.syncQueued || this.syncTimer !== null; - } - const generation = this.generation; - const page = decodePageToken(q, 'index', pageToken, generation); - - // The served handle's text base is still (re)building — the deferred - // open-time build on the no-generation fallback path has not committed (or - // finally failed). Answer with the building page instead of running a - // pass that would raise TextIndexBuildingError; the background build - // commits and a later search serves real hits. Tokens from an older - // generation already failed validation above, so reaching here with a - // building handle is always a first-page situation. - if (serveDb.textIndexBuilding(q.mode === 'literal' ? TRI_INDEX_NAME : TEXT_INDEX_NAME)) { - return this.buildingPage(serveDb); - } - - // One bounded text-index pass: db.searchBoundedAsync returns at most the - // budgeted candidates with their scores (stage 6: the async variant — - // postings reads and disk-mode value reads run off the event loop); - // container/role/time filters and the requested sort are applied in - // memory. (A separate db.query({text}) for pagination would scan the - // same postings a second time.) - let candidates: { key: string; value: SearchDoc | undefined; score: number }[]; - let incomplete: GlobalSearchIncomplete | undefined; - const runBounded = (db2: MiniDb): Promise<{ hits: { key: string; value: SearchDoc; score: number }[]; visits: number; truncated: boolean }> => { - if (q.mode === 'literal') { - // Ask for one past the cap so an over-cap candidate set is - // detectable; the postings budget bounds the index-side work before - // confirmation even starts. - return db2.searchBoundedAsync(TRI_INDEX_NAME, q.query, { - op: 'AND', - limit: this.literalCandidateCap + 1, - maxVisits: this.postingsVisitBudget, - }); - } - return db2.searchBoundedAsync(TEXT_INDEX_NAME, q.query, { - op: q.op, - limit: this.maxTextHits + 1, - maxVisits: this.postingsVisitBudget, - }); - }; + // waits for an open, sync, reopen or reindex: the backend answers from + // its published base (or with building semantics) while the coordinator + // below catches up in the background. + let result: CoreSearchResult; try { - let res: { hits: { key: string; value: SearchDoc; score: number }[]; visits: number; truncated: boolean }; - try { - res = await runBounded(serveDb); - } catch (error) { - // The async query path awaits: a background read-only refresh may - // have swapped (and closed) the pinned handle mid-query. Re-pin the - // currently published handle and retry ONCE — anything else is a - // real failure. - const msg = error instanceof Error ? error.message : String(error); - const closedRace = msg.includes('postings file is closed') || msg.includes('MiniDb is closed') || msg.includes('ValueReader is not open'); - if (!closedRace || this.db === null || this.db === serveDb) throw error; - serveDb = this.db; - res = await runBounded(serveDb); - } - if (q.mode === 'literal') { - candidates = res.hits; - if (res.truncated) incomplete = 'postings_budget'; - if (candidates.length > this.literalCandidateCap) { - candidates.length = this.literalCandidateCap; - incomplete ??= 'candidate_cap'; - } - } else { - candidates = res.hits; - if (res.truncated) incomplete = 'postings_budget'; - if (candidates.length > this.maxTextHits) { - candidates.length = this.maxTextHits; - incomplete ??= 'candidate_cap'; - } - } + result = await this.backend.search({ q, pageToken, budgets: this.budgets() }); } catch (error) { - // The base build's state flipped between the early check and the pass - // (or a read-only refresh swapped in a still-building handle mid-page): - // serve the same building page the early check produces. - if (error instanceof TextIndexBuildingError) { - return this.buildingPage(serveDb); + if (error instanceof GlobalSearchError) { + // A failed open self-heals through search traffic: kick a background + // retry (runSync → backend.sync → ensureOpen), exactly like the + // pre-worker service did. + if (error.reason === 'index_unavailable') this.requestSync(); + throw error; } - // A read-only instance can open before the writer has created the text - // index — serve an empty page instead of failing the search. - if (error instanceof Error && error.message.includes('no such text index')) { - return { - items: [], - hasMore: false, - pageToken: undefined, - incomplete: undefined, - indexState: this.readIndexState(serveDb, stale), - source: 'index', - }; + if (error instanceof SearchWorkerError) { + // The worker is down (spawn failure, crash, backoff): never restore + // the heavy work inline onto this thread — report a recognizable + // degraded state. Recovery is LAZY: the respawn happens on the next + // sync/search that arrives after the backoff window (kicked here), + // never on a timer. + this.lastRefreshError = { at: Date.now(), message: error.message }; + this.log.warn('global search: search worker unavailable; serving a degraded page', { + error: error.message, + code: error.code, + }); + if (error.code === 'disposed') { + // Same contract as the inline host after dispose: fail fast. + throw new GlobalSearchError('index_unavailable', 'search service is disposed'); + } + this.requestSync(); + if (pageToken !== undefined) { + // No generation to validate the token against — the client + // restarts the search once a base is published. + throw new GlobalSearchError( + 'invalid_page_token', + 'the search index is not ready yet; restart the search', + ); + } + return this.buildingPage(null); } throw error; } - const budget: MatchBudget = { - deadlineAt: Date.now() + this.queryDeadlineMs, - textCharsLeft: this.queryTextBudgetChars, - }; - const boundary = page.kind === 'keyset' ? page.boundary : undefined; - const matched = this.matchDocs(q, candidates, boundary, budget); - incomplete ??= matched.incomplete; - return this.toPage( - q, - 'index', - page, - matched.rows, - incomplete, - this.readIndexState(serveDb, stale), - generation, - ); - } + // Writer-side kick (the read-only refresh kick happens inside the core): + // the served generation is the one published by the last completed pass. + if (!result.index.readOnly) this.requestSync(); - // -- shared match & page assembly (both routes) -------------------------------- - - /** - * Container/role/time filtering, keyset-boundary filtering and literal - * confirmation — one implementation shared by the index route (confirming - * n-gram candidates) and the live route (scanning every in-memory - * document). The query work budgets apply at this match stage: the - * wall-clock deadline is re-checked every DEADLINE_CHECK_STRIDE candidates - * and literal confirmation additionally charges each processed document's - * text against `budget.textCharsLeft`. A budget stop is reported as - * `incomplete: 'deadline'`, never a silent truncation. - */ - private matchDocs( - q: NormalizedQuery, - docs: Iterable<{ key: string; value: SearchDoc | undefined; score: number }>, - boundary: SortBoundary | undefined, - budget: MatchBudget, - ): { rows: MatchedRow[]; incomplete?: GlobalSearchIncomplete } { - const literalQuery = q.literalQuery; - const rows: MatchedRow[] = []; - let i = 0; - for (const { key, value: doc, score } of docs) { - if ((i++ & (DEADLINE_CHECK_STRIDE - 1)) === 0 && Date.now() > budget.deadlineAt) { - return { rows, incomplete: 'deadline' }; - } - if (doc === undefined || (doc.kind !== 'message' && doc.kind !== 'title')) continue; - if (q.container?.sessionId !== undefined && doc.sessionId !== q.container.sessionId) continue; - if (q.container?.agentId !== undefined && doc.agentId !== q.container.agentId) continue; - if (q.role !== undefined && doc.role !== q.role) continue; - if (q.startTime !== undefined && doc.time < q.startTime) continue; - if (q.endTime !== undefined && doc.time > q.endTime) continue; - // The boundary check only needs the sort key (score/time/key), so it - // runs BEFORE the expensive literal confirmation. - if (boundary !== undefined && !rowAfterBoundary(q, { key, value: doc, score }, boundary)) { - continue; - } - if (literalQuery !== undefined) { - budget.textCharsLeft -= doc.text.length; - if (budget.textCharsLeft < 0) return { rows, incomplete: 'deadline' }; - // Two-phase execution (same model as Elasticsearch's wildcard field): - // candidates (from the n-gram index, or every in-memory doc on the - // live route) are confirmed against the document text — hash - // collisions and non-contiguous n-gram coverage can produce false - // positives. Zero false positives is the hard guarantee of literal - // mode. The match offset doubles as the snippet anchor. Deliberate - // deviation from ES: the comparison is case-insensitive (NFKC + - // lowercase), aligned with the terms tokenizer. - const at = normalizeLiteral(doc.text).indexOf(literalQuery); - if (at === -1) continue; - rows.push({ key, value: doc, score: 0, anchor: at }); - } else { - rows.push({ key, value: doc, score }); - } - } - return { rows }; - } - - /** - * Sort, paginate and project the matched docs into a page (both routes). - * Keyset pages collect the best `pageSize + 1` rows past the boundary in a - * bounded heap; legacy v1 offset tokens get one last offset slice and are - * answered with a v2 keyset token. - */ - private toPage( - q: NormalizedQuery, - source: GlobalSearchSource, - page: DecodedPage, - rows: MatchedRow[], - incomplete: GlobalSearchIncomplete | undefined, - indexState: GlobalSearchIndexState, - generation?: number, - ): GlobalSearchPage { - // Literal mode: the normalized query (computed in normalizeQuery), reused - // by confirmation and the snippet anchor. - const literalQuery = q.literalQuery; - let pageRows: MatchedRow[]; - let hasMore: boolean; - if (page.kind === 'legacy') { - rows.sort((a, b) => compareRows(q, a, b)); - const slice = rows.slice(page.skip, page.skip + q.pageSize + 1); - hasMore = slice.length > q.pageSize; - pageRows = slice.slice(0, q.pageSize); - } else { - const top = new RowTopK(q, q.pageSize + 1); - for (const row of rows) top.offer(row); - const slice = top.sorted(); - hasMore = slice.length > q.pageSize; - pageRows = slice.slice(0, q.pageSize); - } - const items: GlobalSearchHit[] = pageRows.map((row) => { - const doc = row.value; - return { - sessionId: doc.sessionId, - workspaceId: doc.workspaceId, - sessionTitle: this.summaries.get(doc.sessionId)?.title ?? doc.sessionTitle, - agentId: doc.agentId, - role: doc.role, - snippet: - doc.kind === 'title' - ? doc.text - : row.anchor !== undefined && literalQuery !== undefined - ? makeSnippet(doc.text, q.query, 80, { at: row.anchor, len: literalQuery.length }) - : makeSnippet(doc.text, q.query), - time: doc.time, - turn: doc.kind === 'message' ? doc.turn : undefined, - stepId: doc.kind === 'message' ? doc.stepId : undefined, - score: row.score, - }; - }); + if (result.kind === 'building') return this.buildingPage(result.index); return { - items, - hasMore, - pageToken: hasMore - ? encodePageToken(q, source, boundaryOf(q, pageRows[pageRows.length - 1]!), generation) + items: result.rows.map((row) => this.projectHit(q, row)), + hasMore: result.hasMore, + pageToken: result.hasMore + ? encodePageToken( + q, + 'index', + boundaryOf(q, result.rows[result.rows.length - 1]!), + result.generation, + ) : undefined, - incomplete, - indexState, - source, + incomplete: result.incomplete, + indexState: this.composeIndexState(result.index), + source: 'index', + }; + } + + // -- shared hit projection & index-state assembly ------------------------------- + + private projectHit(q: NormalizedQuery, row: MatchedRow): GlobalSearchHit { + const doc = row.value; + return { + sessionId: doc.sessionId, + workspaceId: doc.workspaceId, + sessionTitle: this.summaries.get(doc.sessionId)?.title ?? doc.sessionTitle, + agentId: doc.agentId, + role: doc.role, + snippet: + doc.kind === 'title' + ? doc.text + : row.anchor !== undefined && q.literalQuery !== undefined + ? makeSnippet(doc.text, q.query, 80, { at: row.anchor, len: q.literalQuery.length }) + : makeSnippet(doc.text, q.query), + time: doc.time, + turn: doc.kind === 'message' ? doc.turn : undefined, + stepId: doc.kind === 'message' ? doc.stepId : undefined, + score: row.score, + }; + } + + /** + * Merge the backend's index view with the coordinator's writer-side state: + * a page is stale when the backend knows its view is behind (read-only + * freshness) OR a sync pass is in flight/queued/pending behind the + * debounce window. + */ + private composeIndexState(view: CoreIndexView): GlobalSearchIndexState { + const coordinatorStale = view.readOnly + ? this.refreshPromise !== null + : this.syncPromise !== null || this.syncQueued || this.syncTimer !== null; + return { + state: view.state, + indexedSessions: view.indexedSessions, + totalSessions: view.readOnly + ? view.indexedSessions + : Math.max(view.indexedSessions, this.summaries.size), + documents: view.documents, + stale: view.freshnessStale || coordinatorStale || undefined, + degraded: this.lastRefreshError?.message ?? view.degraded, + }; + } + + /** + * The page served while the index base is unavailable: the first full sync + * has not finished yet (no db yet), a deferred open-time base build is + * still running / finally failed on the served handle, or the search + * worker is down. Same "never wait" rule as every other request path — + * the background coordinator/build catches up and a later search serves + * real hits. + */ + private buildingPage(view: CoreIndexView | null): GlobalSearchPage { + const indexed = view?.indexedSessions ?? 0; + const readOnly = view?.readOnly === true; + return { + items: [], + hasMore: false, + pageToken: undefined, + incomplete: undefined, + indexState: { + state: 'building', + indexedSessions: indexed, + totalSessions: readOnly ? indexed : Math.max(indexed, this.summaries.size), + documents: view?.documents ?? 0, + stale: true, + degraded: this.lastRefreshError?.message ?? view?.degraded, + }, + source: 'index', }; } @@ -2018,26 +982,13 @@ export class GlobalSearchService implements IGlobalSearchService { // Block new background passes BEFORE the first await, so no sync can // start writing into the db this rebuild is about to swap out. this.reindexing = true; - await this.ensureOpen(); - if (this.db?.readOnly === true) { - throw new GlobalSearchError( - 'readonly_index', - 'another process holds the search-index write lock; reindex from that process', - ); - } - // Let the in-flight sync settle before closing the db it writes into. - // Syncs triggered while we wait see `reindexing` and return as no-ops, - // so one await is sufficient — no new writer of the old db can appear. + await this.backend.ensureOpen(); + // Let the in-flight sync settle before the backend closes the db it + // writes into. Syncs triggered while we wait see `reindexing` and + // return as no-ops, so one await is sufficient — no new writer of the + // old db can appear. await this.syncPromise?.catch(() => {}); - const db = this.db; - if (db) { - await db.close().catch(() => {}); - this.db = null; - } - this.openPromise = null; - this.fullSyncDone = false; - await rm(this.indexDir, { recursive: true, force: true }); - await this.ensureOpen(); + await this.backend.reindex(); // The rebuild runs the authoritative sync itself — an explicit // maintenance operation, never ordinary in-request work. this.reindexing = false; @@ -2048,11 +999,8 @@ export class GlobalSearchService implements IGlobalSearchService { this.lastRefreshError = { at: Date.now(), message: errorMessage(error) }; throw error; } - const stats = this.db?.get(STATS_KEY); - return { - sessions: stats?.kind === 'stats' ? stats.sessions : 0, - documents: stats?.kind === 'stats' ? stats.documents : 0, - }; + const stats = await this.backend.status(); + return { sessions: stats.sessions, documents: stats.documents }; } async status(): Promise<{ @@ -2061,67 +1009,47 @@ export class GlobalSearchService implements IGlobalSearchService { lastIndexedAt: number | null; generation: number; degraded?: string; + lifecycle: CoreLifecycleReport; }> { - await this.ensureOpen(); - if (this.db?.readOnly === true) { - // An explicit status call may wait for the refresh; searches may not. - await this.refreshReadonly(); - } else { - this.requestSync(); + const empty = { sessions: 0, documents: 0, lastIndexedAt: null, generation: 0 }; + if (this.disposed) { + return { + ...empty, + lifecycle: { state: this.drainSettled ? 'stopped' : 'closing' }, + }; + } + try { + const status = await this.backend.status(); + if (!status.readOnly) this.requestSync(); + return { + sessions: status.sessions, + documents: status.documents, + lastIndexedAt: status.lastIndexedAt, + generation: status.generation, + degraded: this.lastRefreshError?.message ?? status.degraded, + lifecycle: status.lifecycle, + }; + } catch (error) { + // A backend that cannot answer (failed open, worker down, wedged call + // reaped by the watchdog) reports degraded — this diagnostic surface + // never throws, exactly so the failure states stay observable. + const message = errorMessage(error); + return { ...empty, degraded: message, lifecycle: { state: 'degraded', detail: message } }; } - const stats = this.db?.get(STATS_KEY); - return { - sessions: stats?.kind === 'stats' ? stats.sessions : 0, - documents: stats?.kind === 'stats' ? stats.documents : 0, - lastIndexedAt: stats?.kind === 'stats' ? stats.lastIndexedAt : null, - generation: this.generation, - degraded: this.lastRefreshError?.message, - }; } /** - * The page served while the index base is unavailable: the first full sync - * has not finished (no db yet), or a deferred open-time base build is - * still running / finally failed on the served handle. Same "never wait" - * rule as every other request path — the background coordinator/build - * catches up and a later search serves real hits. + * Synchronous LOCAL lifecycle report (stage 5): never kicks an open, never + * spawns the worker, never awaits — the view that still answers DURING a + * minutes-long first open (or while the worker backs off), where status() + * would block. The up states (ready/building) come from the backend's + * cached last response and may lag one RPC; status() is the exact, + * round-trip variant. Reflected on the `/api/v1/debug` surface like every + * Service method. */ - private buildingPage(db: MiniDb | null): GlobalSearchPage { - const stats = db?.get(STATS_KEY); - const indexed = stats?.kind === 'stats' ? stats.sessions : 0; - return { - items: [], - hasMore: false, - pageToken: undefined, - incomplete: undefined, - indexState: { - state: 'building', - indexedSessions: indexed, - totalSessions: db === null ? this.summaries.size : db.readOnly ? indexed : Math.max(indexed, this.summaries.size), - documents: stats?.kind === 'stats' ? stats.documents : 0, - stale: true, - degraded: this.lastRefreshError?.message, - }, - source: 'index', - }; - } - - private readIndexState(db: MiniDb, stale: boolean): GlobalSearchIndexState { - const stats = db.get(STATS_KEY); - const indexed = stats?.kind === 'stats' ? stats.sessions : 0; - const documents = stats?.kind === 'stats' ? stats.documents : 0; - // A deferred open-time base build (no-generation fallback path) puts the - // served handle's text indexes into the building state — surface it as - // the same 'building' the first-sync window uses, whatever the process role. - const building = db.textIndexBuilding(TEXT_INDEX_NAME) || db.textIndexBuilding(TRI_INDEX_NAME); - return { - state: building ? 'building' : db.readOnly ? 'readonly' : this.fullSyncDone ? 'ready' : 'building', - indexedSessions: indexed, - totalSessions: db.readOnly ? indexed : Math.max(indexed, this.summaries.size), - documents, - stale: stale || undefined, - degraded: this.lastRefreshError?.message, - }; + lifecycleReport(): CoreLifecycleReport { + if (this.disposed) return { state: this.drainSettled ? 'stopped' : 'closing' }; + return this.backend.lifecycleSnapshot(); } } @@ -2136,8 +1064,9 @@ export class GlobalSearchService implements IGlobalSearchService { * uses — so a document matches when EVERY query term appears in its term set * (AND). The score is Σ log(1 + tf) per query term: it is only comparable * within the live route, since there is no corpus-wide IDF in memory (the - * `GlobalSearchSource` contract comment says the same). The shared `toPage` - * applies the final (score, time, key) order over the returned rows. + * `GlobalSearchSource` contract comment says the same). The shared + * pagination applies the final (score, time, key) order over the returned + * rows. */ function matchLiveTerms( terms: readonly string[], @@ -2164,66 +1093,6 @@ function matchLiveTerms( return matched; } -// --------------------------------------------------------------------------- -// wire file enumeration & doc keys -// --------------------------------------------------------------------------- - -interface WireFileRef { - readonly path: string; - /** 'main' or a subagent id, for both legacy and v2 layouts. */ - readonly agentId: string; - /** - * Key discriminator: a session can carry BOTH a legacy root wire.jsonl and - * v2 per-agent logs; without this their `/` keys collide. - */ - readonly source: 'root' | 'agents'; -} - -async function pathExists(path: string): Promise { - try { - await stat(path); - return true; - } catch { - return false; - } -} - -async function collectWireFiles(sessionDir: string): Promise { - const files: WireFileRef[] = []; - const root = join(sessionDir, WIRE_FILENAME); - try { - if ((await stat(root)).isFile()) files.push({ path: root, agentId: 'main', source: 'root' }); - } catch { - // no legacy root log - } - const agentsDir = join(sessionDir, 'agents'); - try { - const entries = await readdir(agentsDir, { recursive: true, withFileTypes: true }); - for (const entry of entries) { - if (!entry.isFile() || entry.name !== WIRE_FILENAME) continue; - const path = join(entry.parentPath, entry.name); - files.push({ path, agentId: relative(agentsDir, entry.parentPath), source: 'agents' }); - } - } catch { - // no agents dir - } - return files; -} - -function docKeyPrefix(sessionId: string, file: WireFileRef): string { - return `${sessionId}/${file.agentId}/${file.source}:`; -} - -/** Same rebuildability test as `MiniDb.openOrRebuild`. */ -function isRebuildableCorruption(error: unknown): boolean { - return ( - error instanceof SyntaxError || - (error !== null && - typeof error === 'object' && - (error as { name?: string }).name === 'CorruptFrameError') - ); -} - registerScopedService( LifecycleScope.App, IGlobalSearchService, diff --git a/packages/kap-server/src/search/worker/dev-hooks.mjs b/packages/kap-server/src/search/worker/dev-hooks.mjs new file mode 100644 index 000000000..6bc9a4a31 --- /dev/null +++ b/packages/kap-server/src/search/worker/dev-hooks.mjs @@ -0,0 +1,23 @@ +// Resolve hook for the DEV search worker (`--experimental-transform-types`). +// +// The worker entry's import closure runs from TypeScript source in dev and +// tests. minidb's internals import sibling modules with `.js` specifiers +// while the files on disk are `.ts` (the repo builds with bundler module +// resolution), and Node's native type stripping does no specifier remapping +// — so retry a missing relative `.js` specifier as `.ts` here. Registered by +// register-dev-hooks.mjs via `--import`; never bundled (the bundled worker +// is plain JS and needs no hook). +export async function resolve(specifier, context, nextResolve) { + try { + return await nextResolve(specifier, context); + } catch (error) { + if ( + error?.code === 'ERR_MODULE_NOT_FOUND' && + (specifier.startsWith('./') || specifier.startsWith('../')) && + /\.m?js$/.test(specifier) + ) { + return nextResolve(specifier.replace(/\.m?js$/, '.ts'), context); + } + throw error; + } +} diff --git a/packages/kap-server/src/search/worker/entry.ts b/packages/kap-server/src/search/worker/entry.ts new file mode 100644 index 000000000..7495ae42e --- /dev/null +++ b/packages/kap-server/src/search/worker/entry.ts @@ -0,0 +1,207 @@ +/** + * `search/worker` — worker-thread entry: hosts the search-index core inside + * a dedicated worker thread. + * + * Loaded by the main-process host (`host.ts`) in one of three shapes: + * - dev / tests: this `.ts` file itself, under Node's native type + * stripping plus the repo-local `.js` → `.ts` resolve hook + * (`register-dev-hooks.mjs`); + * - bundled CLI (`dist/main.mjs`): the bundled `search-worker.mjs` + * sibling emitted by tsdown; + * - SEA single-file binary: the extracted runtime asset configured via + * `configureSearchWorkerRuntime` (`runtime.ts`). + * + * The whole import closure of this file must stay loadable under + * `--experimental-transform-types`: relative imports carry explicit `.ts` + * specifiers, no decorators, erasable TypeScript only. In particular it must + * NOT import the service (`searchService.ts`, decorators) or the + * agent-core-v2 barrel (`.md?raw` imports need a bundler). + * + * Concurrency: requests are handled as their messages arrive (async + * interleaving, same semantics the in-process service had). A `close` + * request first lets every in-flight request settle, then closes the core — + * which drains tracked background ops (sync/refresh) before closing the db — + * responds, and lets the thread exit. The host's terminate() remains the + * hard backstop. + */ + +import { parentPort, workerData } from 'node:worker_threads'; + +import { configureTextBuildWorkerRuntime } from '@moonshot-ai/minidb/worker-runtime'; + +import { GlobalSearchError, type GlobalSearchErrorReason } from '../contract.ts'; +import { + SearchIndexCore, + type CoreSearchParams, + type SyncSessionInput, +} from '../indexCore.ts'; +import { + SEARCH_WORKER_PROTOCOL_VERSION, + type SearchWorkerData, + type SearchWorkerErrorPayload, + type SearchWorkerEvent, + type SearchWorkerOpenResult, + type SearchWorkerCall, + type SearchWorkerRequest, +} from './protocol.ts'; + +const data = workerData as SearchWorkerData; + +// The worker owns the search-index MiniDb, so its heavy text-generation +// builds are triggered from THIS thread — give minidb the packaged +// text-build worker entry when the main process extracted one (SEA). In dev +// minidb resolves its own sibling source entry; a missing entry degrades to +// the bounded inline core inside this worker, never the main thread. +if (typeof data.textBuildWorkerPath === 'string') { + try { + configureTextBuildWorkerRuntime(data.textBuildWorkerPath); + } catch { + // Best-effort: minidb falls back to the bounded inline build in this worker. + } +} + +const port = parentPort; +if (port === null) { + throw new Error('search worker entry must run inside a worker thread'); +} + +const post = (event: SearchWorkerEvent): void => { + port.postMessage(event); +}; + +const core = new SearchIndexCore({ + indexDir: data.dir, + bootSalt: data.bootSalt, + log: { + info: (message, meta) => { + post({ type: 'log', level: 'info', message, meta }); + }, + warn: (message, meta) => { + post({ type: 'log', level: 'warn', message, meta }); + }, + }, + // The lock token must reach the host BEFORE the heavy open work completes + // — a worker that dies mid-open would otherwise leave an unreapable lock + // (the lock line carries this process's still-alive pid). + onLockToken: (token) => { + post({ type: 'lockToken', token }); + }, +}); + +function toErrorPayload(error: unknown): SearchWorkerErrorPayload { + if (error instanceof GlobalSearchError) { + return { message: error.message, reason: error.reason as GlobalSearchErrorReason }; + } + return { message: error instanceof Error ? error.message : String(error) }; +} + +async function dispatch(request: SearchWorkerCall): Promise { + switch (request.type) { + case 'open': { + await core.ensureOpen(); + const result: SearchWorkerOpenResult = { + readOnly: core.db?.readOnly === true, + lockToken: core.lockTokenView, + lifecycle: core.lifecycleState(), + }; + return result; + } + case 'search': + return core.search(request.params as CoreSearchParams); + case 'sync': + return core.sync((request.params as { sessions: readonly SyncSessionInput[] }).sessions); + case 'refresh': { + await core.refresh(); + const result: SearchWorkerOpenResult = { + readOnly: core.db?.readOnly === true, + lockToken: core.lockTokenView, + lifecycle: core.lifecycleState(), + }; + return result; + } + case 'reindex': { + await core.reindex(); + const result: SearchWorkerOpenResult = { + readOnly: core.db?.readOnly === true, + lockToken: core.lockTokenView, + lifecycle: core.lifecycleState(), + }; + return result; + } + case 'status': + return core.status(); + case 'close': + return null; + } +} + +const inFlight = new Set>(); +let closing = false; + +async function handle(request: SearchWorkerCall): Promise { + if (request.v !== SEARCH_WORKER_PROTOCOL_VERSION) { + post({ + id: request.id, + type: 'error', + error: { + message: `search worker protocol mismatch: host v${request.v}, worker v${SEARCH_WORKER_PROTOCOL_VERSION}`, + }, + }); + return; + } + if (request.type === 'close') { + // Gate the core FIRST (an in-flight sync abandons its remaining work at + // the next checkpoint), then let every in-flight request settle and + // close — the close itself drains tracked background ops. The host's + // terminate() is the backstop when a request wedges. + closing = true; + core.beginClose(); + await Promise.all(inFlight); + let error: SearchWorkerErrorPayload | null = null; + try { + await core.close(); + } catch (closeError) { + error = toErrorPayload(closeError); + } + if (error !== null) post({ id: request.id, type: 'error', error }); + else post({ id: request.id, type: 'result', result: null }); + // Releasing the port lets the thread's event loop drain and exit. + port!.close(); + return; + } + if (closing) { + post({ + id: request.id, + type: 'error', + error: { message: 'search service is disposed', reason: 'index_unavailable' }, + }); + return; + } + try { + const result = await dispatch(request); + post({ id: request.id, type: 'result', result }); + } catch (error) { + post({ id: request.id, type: 'error', error: toErrorPayload(error) }); + } +} + +port.on('message', (value: unknown) => { + const request = value as SearchWorkerRequest; + if (request === null || typeof request !== 'object') return; + if (request.type === 'beginClose') { + // Control message (no id, no response): flip the core into closing + // state NOW so an in-flight sync abandons its remaining work at the + // next checkpoint instead of wedging the host's dispose behind it. + closing = true; + core.beginClose(); + return; + } + if (typeof request.id !== 'number' || typeof request.type !== 'string') return; + const tracked = handle(request); + inFlight.add(tracked); + void tracked.finally(() => { + inFlight.delete(tracked); + }); +}); + +post({ type: 'ready', v: SEARCH_WORKER_PROTOCOL_VERSION }); diff --git a/packages/kap-server/src/search/worker/host.ts b/packages/kap-server/src/search/worker/host.ts new file mode 100644 index 000000000..e030baebf --- /dev/null +++ b/packages/kap-server/src/search/worker/host.ts @@ -0,0 +1,813 @@ +/** + * `search/worker` — the main-process host for the search worker thread. + * + * The host owns the worker lifecycle and speaks the request/response + * protocol (`protocol.ts`); to the search service it presents the same + * surface as the inline core (`SearchIndexCore`), so the service's + * coordinator / live route / page assembly never care which host runs the + * index. + * + * Lifecycle semantics (stage 4 work items 3, 6 and 7): + * - lazy spawn with single-flight + backoff: the first RPC spawns the + * worker; spawn failures and dirty exits back off exponentially + * (500ms → 10s cap) instead of hot-looping, and RPCs during the backoff + * window fail fast with a recognizable SearchWorkerError — nothing + * ever falls back to running the index on the main thread implicitly + * (the explicit rollback switch is the `search_worker` flag); + * - a dirty exit rejects every in-flight request (no dangling promises), + * records the failure for the service's degraded state, and reaps the + * search-index `db.lock` — but ONLY when the lock line still carries + * the dead worker's own token, so a live owner's lock is never deleted + * (the lock file's pid is unusable for this judgment because worker + * threads share the main process pid); + * - dispose() sends `close` (the worker drains in-flight requests and + * tracked background ops, closes the db and exits) with terminate() as + * the bounded backstop. + * + * Entry resolution: the configured packaged asset (SEA) → the sibling + * `entry.ts` source (dev/tests, run under Node's native type stripping plus + * the repo-local `.js` → `.ts` resolve hook) → the bundled + * `search-worker.mjs` sibling (packaged npm layout). + */ + +import fsSync from 'node:fs'; +import { randomUUID } from 'node:crypto'; +import { readFile, rm } from 'node:fs/promises'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { Worker } from 'node:worker_threads'; + +import { getTextBuildWorkerRuntimeState } from '@moonshot-ai/minidb/worker-runtime'; + +import { GlobalSearchError } from '../contract'; +import type { + CoreIndexView, + CoreLifecycleReport, + CoreSearchParams, + CoreSearchResult, + CoreStatus, + CoreSyncOutcome, + SearchCoreLog, + SyncSessionInput, +} from '../indexCore'; +import { + SEARCH_WORKER_PROTOCOL_VERSION, + type SearchWorkerData, + type SearchWorkerEvent, + type SearchWorkerOpenResult, + type SearchWorkerCallType, + type SearchWorkerResultMap, +} from './protocol'; +import { getSearchWorkerRuntimeState } from './runtime'; + +export type SearchWorkerErrorCode = + /** No worker entry file could be resolved in this installation. */ + | 'runtime-unavailable' + /** Constructor/bootstrap/handshake failure (incl. protocol mismatch). */ + | 'spawn-failed' + /** The worker died after a successful handshake (crash / OOM / kill). */ + | 'crashed' + /** A restart is backing off; the RPC failed fast without spawning. */ + | 'backoff' + /** The host is disposed. */ + | 'disposed'; + +/** + * Recognizable worker-availability failure. The service maps it to a + * building/degraded response (searches) or the degraded state (background + * ops) — it is never swallowed silently. + */ +export class SearchWorkerError extends Error { + constructor( + readonly code: SearchWorkerErrorCode, + message: string, + ) { + super(message); + this.name = 'SearchWorkerError'; + } +} + +export interface SearchWorkerHostOptions { + /** Absolute path of the search-index database directory. */ + readonly dir: string; + readonly log: SearchCoreLog; + /** Ready-handshake budget (ms). Default 15_000. */ + readonly readyTimeoutMs?: number; + /** Grace period for the drain-on-close before terminate() (ms). Default 30_000. */ + readonly closeTimeoutMs?: number; + /** Watchdog budget per query/lifecycle request (ms). Default 60_000. */ + readonly requestTimeoutMs?: number; + /** Watchdog budget per sync/reindex request (ms). Default 30 min. */ + readonly syncTimeoutMs?: number; + /** Worker heap cap, mirroring the text-build worker. Default 1024. */ + readonly maxOldSpaceMb?: number; + /** Test hook: worker factory override. */ + readonly workerFactory?: (entry: { url: URL; data: SearchWorkerData; execArgv: string[] }) => Worker; +} + +interface WorkerEntryResolution { + readonly url: URL; + /** File-system path for packaged entries (stat-able, absolute). */ + readonly execArgv: string[]; +} + +type PendingRequest = { + readonly method: SearchWorkerCallType; + readonly resolve: (value: unknown) => void; + readonly reject: (error: unknown) => void; + /** Absent for `close` — doDispose races it against closeTimeoutMs itself. */ + readonly watchdog?: ReturnType; +}; + +const DEFAULT_READY_TIMEOUT_MS = 15_000; +const DEFAULT_CLOSE_TIMEOUT_MS = 30_000; +const DEFAULT_MAX_OLD_SPACE_MB = 1024; +/** A worker session that survived this long resets the crash backoff. */ +const STABLE_SESSION_MS = 60_000; +const BACKOFF_BASE_MS = 500; +const BACKOFF_CAP_MS = 10_000; +/** + * Watchdog budgets per request. Queries and lifecycle calls are expected to + * be bounded (the query path carries its own work budgets); sync/reindex of + * a large corpus legitimately takes minutes, so their leash is long — the + * watchdog exists to catch a WEDGED worker, not to police slow work. + */ +const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; +const DEFAULT_SYNC_TIMEOUT_MS = 30 * 60_000; +/** + * Grace window before treating a lock line carrying THIS process's pid but + * an unknown token as an orphan: a live sibling worker's token event rides + * the same-process port and lands within milliseconds, so a token still + * unknown after the window belongs to a dead worker. + */ +const ORPHAN_LOCK_GRACE_MS = 250; + +/** + * Lock tokens held by LIVE search-index holders of this process (worker + * threads share the process pid, so the pid in a lock line cannot + * distinguish a live holder from a dead one). Fed by the workers' + * `lockToken` events (and by the inline backend's core) and pruned on + * worker exit / backend dispose; the orphan-lock detector reaps a same-pid + * lock line only when its token is NOT in this set. + */ +const liveLockTokens = new Set(); + +/** Register a held search-index lock token (inline backend's core). */ +export function noteLiveLockToken(token: string): void { + liveLockTokens.add(token); +} + +/** Drop a previously registered token (inline backend close/dispose). */ +export function dropLiveLockToken(token: string | undefined): void { + if (token !== undefined) liveLockTokens.delete(token); +} + +export class SearchWorkerHost { + private worker: Worker | null = null; + private spawnPromise: Promise | null = null; + private reapPromise: Promise | null = null; + private readonly requests = new Map(); + private nextId = 1; + /** Token of the db.lock line the current/last worker published. */ + private lockToken: string | undefined; + private failures = 0; + private nextRetryAfter = 0; + private lastFailure: string | null = null; + private readyAt = 0; + private exiting = false; + private exitResolve: (() => void) | null = null; + private orphanCheckScheduled = false; + /** + * The worker-side core's lifecycle as of the last RPC response that carried + * it (every response shape does: status/open/refresh/reindex directly, + * search via its index view). Feeds `lifecycleSnapshot` — the local, + * round-trip-free answer for the service's status surface. + */ + private lastCoreLifecycle: CoreLifecycleReport | null = null; + + constructor(private readonly options: SearchWorkerHostOptions) {} + + private get log(): SearchCoreLog { + return this.options.log; + } + + private get dir(): string { + return this.options.dir; + } + + /** Test/diagnostic: the lock token the current/last worker reported. */ + get reportedLockToken(): string | undefined { + return this.lockToken; + } + + /** + * The aggregate search lifecycle from the host's LOCAL knowledge (stage 5): + * never spawns the worker, never waits on an RPC — the states a wedged or + * not-yet-running worker cannot answer for itself are exactly the ones this + * must report (stopped/opening/degraded-backoff/closing). A live worker's + * state comes from the cached last response (`lastCoreLifecycle`). + */ + lifecycleSnapshot(): CoreLifecycleReport { + if (this.exiting) { + return this.worker !== null || this.spawnPromise !== null + ? { state: 'closing' } + : { state: 'stopped' }; + } + if (this.spawnPromise !== null) return { state: 'opening' }; + if (this.worker === null) { + if (this.lastFailure !== null) { + // Crash/spawn failure with a lazy restart pending on the next RPC + // (possibly still inside the backoff window): the index is unserved. + const wait = this.nextRetryAfter - Date.now(); + return { + state: 'degraded', + detail: + wait > 0 + ? `search worker restart backing off for ${wait}ms (last failure: ${this.lastFailure})` + : `search worker restart pending (last failure: ${this.lastFailure})`, + }; + } + // Never spawned: the lazy spawn fires on the first RPC. + return { state: 'stopped' }; + } + // The worker is up but no response has landed yet (its first open is + // still in flight): the core is opening. + return this.lastCoreLifecycle ?? { state: 'opening' }; + } + + // -- backend surface ------------------------------------------------------------ + + async ensureOpen(): Promise { + return this.call('open'); + } + + async search(params: CoreSearchParams): Promise { + return this.call('search', params); + } + + async sync(sessions: readonly SyncSessionInput[]): Promise { + return this.call('sync', { sessions }); + } + + async refresh(): Promise { + return this.call('refresh'); + } + + async reindex(): Promise { + return this.call('reindex'); + } + + async status(): Promise { + return this.call('status'); + } + + /** Test hook: hard-kill the worker (crash/restart semantics). */ + async killWorkerForTest(): Promise { + const worker = this.worker; + if (worker === null) return; + await worker.terminate(); + await this.waitForExit(); + } + + // -- RPC -------------------------------------------------------------------------- + + private async call( + type: T, + params?: unknown, + ): Promise { + await this.ensureWorker(); + const worker = this.worker; + if (worker === null) { + throw new SearchWorkerError('crashed', 'search worker is not running'); + } + return this.sendRequest(worker, type, params) as Promise; + } + + private sendRequest( + worker: Worker, + type: SearchWorkerCallType, + params?: unknown, + ): Promise { + const id = this.nextId++; + return new Promise((resolve, reject) => { + // Watchdog budget: `open` shares the long leash with sync/reindex — a + // first open over a huge corpus replays the full WAL / runs full + // recovery with no wall-clock budget of its own, and killing it at + // the short leash would discard all progress into a backoff loop. + // `close` gets no watchdog: doDispose races it against closeTimeoutMs, + // and a shorter request leash would misjudge a legitimate drain as a + // wedged worker. + let watchdog: ReturnType | undefined; + if (type !== 'close') { + const timeoutMs = + type === 'sync' || type === 'reindex' || type === 'open' + ? this.options.syncTimeoutMs ?? DEFAULT_SYNC_TIMEOUT_MS + : this.options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + watchdog = setTimeout(() => this.onRequestTimeout(id), timeoutMs); + watchdog.unref?.(); + } + this.requests.set(id, { method: type, resolve, reject, watchdog }); + try { + worker.postMessage({ id, v: SEARCH_WORKER_PROTOCOL_VERSION, type, params }); + } catch (error) { + this.requests.delete(id); + if (watchdog !== undefined) clearTimeout(watchdog); + reject( + new SearchWorkerError( + 'crashed', + `search worker channel broke: ${error instanceof Error ? error.message : String(error)}`, + ), + ); + } + }); + } + + /** + * Watchdog: a request outlived its budget — the worker is presumed wedged. + * Reject the requester and take the crash path (terminate → reject the + * remaining in-flight requests → reap → backed-off restart). + */ + private onRequestTimeout(id: number): void { + const pending = this.requests.get(id); + if (pending === undefined) return; + this.requests.delete(id); + const message = `search worker request '${pending.method}' timed out`; + this.log.warn(`global search: ${message}; terminating the wedged worker`, { dir: this.dir }); + pending.reject(new SearchWorkerError('crashed', message)); + const worker = this.worker; + if (worker !== null) { + void worker.terminate().catch(() => {}); + } + } + + private settleRequest(id: number, error: Error | null, result: unknown): void { + const pending = this.requests.get(id); + if (pending === undefined) return; + this.requests.delete(id); + if (pending.watchdog !== undefined) clearTimeout(pending.watchdog); + if (error !== null) { + pending.reject(error); + return; + } + this.noteResult(result); + pending.resolve(result); + } + + /** + * Track the worker-published db.lock token and the read-only role from any + * response carrying them; a read-only answer triggers the orphan-lock + * detector (see recoverOrphanedLock). Also refreshes the cached core + * lifecycle (`lastCoreLifecycle`) — every response shape carries it: + * status/open/refresh/reindex directly, search via its index view. + */ + private noteResult(result: unknown): void { + if (result === null || typeof result !== 'object') return; + const direct = (result as { lockToken?: unknown }).lockToken; + const nested = (result as { index?: { lockToken?: unknown } }).index?.lockToken; + const token = typeof direct === 'string' ? direct : typeof nested === 'string' ? nested : undefined; + if (token !== undefined) this.lockToken = token; + const readOnly = + (result as { readOnly?: unknown }).readOnly === true || + (result as { index?: { readOnly?: unknown } }).index?.readOnly === true; + if (readOnly) this.scheduleOrphanCheck(); + const lifecycle = (result as { lifecycle?: CoreLifecycleReport }).lifecycle; + if (lifecycle !== undefined) { + this.lastCoreLifecycle = lifecycle; + return; + } + const index = (result as { index?: CoreIndexView }).index; + if (index !== undefined) { + // A search response arrived at all, so the worker is serving: the page + // view's building/ready maps directly; a degraded MESSAGE on a serving + // page keeps the lifecycle 'ready' (the message rides status()'s + // degraded field, same as the per-page surface). + this.lastCoreLifecycle = { state: index.state === 'building' ? 'building' : 'ready' }; + } + } + + // -- worker lifecycle -------------------------------------------------------------- + + private ensureWorker(): Promise { + if (this.worker !== null) return Promise.resolve(); + if (this.exiting) { + return Promise.reject(new SearchWorkerError('disposed', 'search worker is disposed')); + } + if (this.spawnPromise !== null) return this.spawnPromise; + const wait = this.nextRetryAfter - Date.now(); + if (wait > 0) { + return Promise.reject( + new SearchWorkerError( + 'backoff', + `search worker restart backing off for ${wait}ms (last failure: ${this.lastFailure ?? 'unknown'})`, + ), + ); + } + this.spawnPromise = this.doSpawn().finally(() => { + this.spawnPromise = null; + }); + return this.spawnPromise; + } + + private resolveEntry(): WorkerEntryResolution { + const configured = getSearchWorkerRuntimeState(); + if (configured.configured) { + return { url: pathToFileURL(configured.path), execArgv: [] }; + } + const source = new URL('./entry.ts', import.meta.url); + try { + if (fsSync.statSync(source).isFile()) { + // Dev/tests: run the TypeScript entry under Node's native type + // stripping; the repo-local resolve hook maps minidb's internal + // `.js` specifiers onto the on-disk `.ts` sources. + return { + url: source, + execArgv: [ + '--experimental-transform-types', + '--disable-warning=ExperimentalWarning', + '--import', + new URL('./register-dev-hooks.mjs', import.meta.url).href, + ], + }; + } + } catch { + // not a source layout — try the bundled sibling + } + const bundled = new URL('./search-worker.mjs', import.meta.url); + try { + if (fsSync.statSync(bundled).isFile()) { + return { url: bundled, execArgv: [] }; + } + } catch { + // fall through to the explicit failure + } + throw new SearchWorkerError( + 'runtime-unavailable', + 'search worker entry not found (no configured packaged asset, no sibling source, no bundled sibling)', + ); + } + + private async doSpawn(): Promise { + // A previous worker's lock reaping must land before the new worker's + // MiniDb open evaluates the lock. + await this.reapPromise; + const entry = this.resolveEntry(); + const textBuild = getTextBuildWorkerRuntimeState(); + const data: SearchWorkerData = { + dir: this.dir, + bootSalt: randomUUID(), + textBuildWorkerPath: + textBuild.configured && textBuild.entry.kind === 'packaged' ? textBuild.entry.path : undefined, + }; + let worker: Worker; + try { + worker = + this.options.workerFactory?.({ + url: entry.url, + data, + execArgv: entry.execArgv, + }) ?? + new Worker(entry.url, { + workerData: data, + execArgv: entry.execArgv, + resourceLimits: { maxOldGenerationSizeMb: this.options.maxOldSpaceMb ?? DEFAULT_MAX_OLD_SPACE_MB }, + name: 'kimi-search-worker', + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.noteFailure(`spawn failed: ${message}`); + throw new SearchWorkerError('spawn-failed', `search worker failed to start: ${message}`); + } + worker.unref(); + this.attach(worker); + try { + await this.awaitReady(worker); + } catch (error) { + // Never attached as current: detach without exit handling — the + // worker's own (late) exit event is stale-guarded in onExit. + this.detach(); + await worker.terminate().catch(() => {}); + const message = error instanceof Error ? error.message : String(error); + this.noteFailure(`handshake failed: ${message}`); + throw new SearchWorkerError('spawn-failed', `search worker handshake failed: ${message}`); + } + this.readyAt = Date.now(); + this.worker = worker; + // The failure that armed the backoff is consumed: a fresh worker starts + // with a clean slate so a LATER dirty exit records its own cause (the + // backoff counters themselves are kept for the exponential math). + this.lastFailure = null; + // The cached core lifecycle belonged to the PREVIOUS worker generation: + // the new core has not even opened its db yet, and its first open over a + // large corpus can take minutes — serving the dead worker's 'ready' from + // the snapshot in that window would misreport the index as servable. + this.lastCoreLifecycle = null; + if (this.exiting) { + // beginClose()/dispose() landed while the handshake was in flight; the + // control message posted then found no worker and was dropped — forward + // it now so the worker-side core gates any pass about to start. + try { + worker.postMessage({ v: SEARCH_WORKER_PROTOCOL_VERSION, type: 'beginClose' }); + } catch { + // A broken channel here is settled by the exit path. + } + } + this.log.info('global search: worker started', { dir: this.dir }); + } + + private attach(worker: Worker): void { + worker.on('message', (event: SearchWorkerEvent) => this.onMessage(worker, event)); + worker.on('error', (error: Error) => this.onError(worker, error)); + worker.on('exit', (code: number) => this.onExit(worker, code)); + } + + private detach(): void { + this.worker = null; + this.exitResolve?.(); + this.exitResolve = null; + } + + private onMessage(worker: Worker, event: SearchWorkerEvent): void { + // Ignore a stale worker's late messages (it was replaced after a + // handshake abort; only the current worker's traffic is meaningful). + if (this.worker !== worker) return; + if (event === null || typeof event !== 'object') return; + if (event.type === 'log') { + this.log[event.level](event.message, event.meta); + return; + } + if (event.type === 'lockToken') { + // The worker just acquired the write lock (posted before the heavy + // open work runs). Register it live so a mid-open crash leaves a + // reapable lock and the orphan detector never touches it. + this.lockToken = event.token; + liveLockTokens.add(event.token); + return; + } + if (event.type === 'ready') return; // consumed by awaitReady's one-shot listener + if (event.type === 'result') { + this.settleRequest(event.id, null, event.result); + return; + } + if (event.type === 'error') { + const payload = event.error; + const error = + payload.reason !== undefined + ? new GlobalSearchError(payload.reason, payload.message) + : new Error(payload.message); + this.settleRequest(event.id, error, undefined); + } + } + + private onError(worker: Worker, error: Error): void { + if (this.worker !== worker) return; + // The 'exit' event follows and performs the teardown; remember the cause. + this.lastFailure = `worker error: ${error.message}`; + } + + private onExit(worker: Worker, code: number): void { + // A stale worker's exit must not tear down its replacement: the + // handshake-abort path detaches a worker BEFORE its exit event lands. + if (this.worker !== worker) return; + const wasExiting = this.exiting; + const sessionMs = Date.now() - this.readyAt; + const deadToken = this.lockToken; + this.detach(); + this.lockToken = undefined; + if (deadToken !== undefined) liveLockTokens.delete(deadToken); + if (this.requests.size > 0) { + // During a clean close the pending requests race the worker's exit — + // they are disposed, not crashed (the close itself already settled). + const error = wasExiting + ? new SearchWorkerError('disposed', 'search worker is disposed') + : new SearchWorkerError( + 'crashed', + `search worker exited with in-flight requests (code ${code}${this.lastFailure !== null ? `; ${this.lastFailure}` : ''})`, + ); + for (const pending of this.requests.values()) { + if (pending.watchdog !== undefined) clearTimeout(pending.watchdog); + pending.reject(error); + } + this.requests.clear(); + } + if (wasExiting) return; // clean close / handshake abort — no backoff, no reaping needed + // Dirty exit. Reap the lock the dead worker published (guarded by its + // token) so the replacement worker can acquire it — the pid in the lock + // line is useless here because worker threads share the main process + // pid and it stays alive. + // A bare kill leaves no 'error'-event message: record the exit itself so + // the backoff rejection and the lifecycle report can name the failure. + this.lastFailure ??= `worker exited with code ${code}`; + this.reapPromise = this.reapLockFile(deadToken).finally(() => { + this.reapPromise = null; + }); + this.failures = sessionMs > STABLE_SESSION_MS ? 1 : this.failures + 1; + const backoff = Math.min(BACKOFF_BASE_MS * 2 ** (this.failures - 1), BACKOFF_CAP_MS); + this.nextRetryAfter = Date.now() + backoff; + this.log.warn('global search: worker exited unexpectedly; restart backed off', { + code, + backoffMs: backoff, + lastFailure: this.lastFailure ?? undefined, + }); + } + + /** Record a spawn/handshake failure and arm the restart backoff. */ + private noteFailure(message: string): void { + this.failures += 1; + const backoff = Math.min(BACKOFF_BASE_MS * 2 ** (this.failures - 1), BACKOFF_CAP_MS); + this.nextRetryAfter = Date.now() + backoff; + this.lastFailure = message; + } + + /** + * Remove the search-index db.lock ONLY when it still carries the dead + * worker's own token. A lock line written by anyone else (a live worker of + * another service instance in this process, another process entirely) is + * never touched. + */ + private async reapLockFile(token: string | undefined): Promise { + if (token === undefined) return; + const lockPath = join(this.dir, 'db.lock'); + try { + const raw = await readFile(lockPath, 'utf8'); + const parsed = JSON.parse(raw) as { pid?: unknown; token?: unknown }; + if (parsed.pid === process.pid && parsed.token === token) { + await rm(lockPath, { force: true }); + this.log.warn('global search: reaped the lock left by the dead worker', { dir: this.dir }); + } + } catch { + // Unreadable or already gone — the next open re-evaluates from disk. + } + } + + private awaitReady(worker: Worker): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error(`ready handshake timed out after ${this.options.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS}ms`)); + }, this.options.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS); + timer.unref?.(); + const onMessage = (event: SearchWorkerEvent): void => { + if (event?.type !== 'ready') return; + cleanup(); + if (event.v !== SEARCH_WORKER_PROTOCOL_VERSION) { + reject( + new Error( + `protocol mismatch: host v${SEARCH_WORKER_PROTOCOL_VERSION}, worker v${event.v ?? 'unknown'}`, + ), + ); + return; + } + resolve(); + }; + const onError = (error: Error): void => { + cleanup(); + reject(error); + }; + const onExit = (code: number): void => { + cleanup(); + reject(new Error(`worker exited before ready (code ${code})`)); + }; + const cleanup = (): void => { + clearTimeout(timer); + worker.off('message', onMessage); + worker.off('error', onError); + worker.off('exit', onExit); + }; + worker.on('message', onMessage); + worker.on('error', onError); + worker.on('exit', onExit); + }); + } + + private waitForExit(): Promise { + if (this.worker === null) return Promise.resolve(); + return new Promise((resolve) => { + const previous = this.exitResolve; + this.exitResolve = () => { + previous?.(); + resolve(); + }; + }); + } + + // -- dispose ------------------------------------------------------------------------- + + /** + * Synchronously stop accepting new work (the service's dispose() calls + * this before any awaiting): every later RPC fails fast with 'disposed', + * and the worker's core is told NOW (control message) so an in-flight + * sync abandons its remaining work at the next checkpoint instead of + * wedging the dispose behind a running pass. + */ + beginClose(): void { + this.exiting = true; + try { + this.worker?.postMessage({ v: SEARCH_WORKER_PROTOCOL_VERSION, type: 'beginClose' }); + } catch { + // A broken channel means the worker is already gone — nothing to notify. + } + } + + /** + * A read-only answer while the lock could be an orphaned same-pid line + * (left by a dead worker whose token event never landed) schedules a + * re-check after a grace window — see recoverOrphanedLock. + */ + private scheduleOrphanCheck(): void { + if (this.orphanCheckScheduled || this.exiting) return; + this.orphanCheckScheduled = true; + const timer = setTimeout(() => { + this.orphanCheckScheduled = false; + void this.recoverOrphanedLock(); + }, ORPHAN_LOCK_GRACE_MS); + timer.unref?.(); + } + + /** + * Safety net behind the `lockToken` event: this instance is being served + * read-only while the lock line carries THIS process's pid. The line is + * legitimate only when its token belongs to a live worker of this process + * (`liveLockTokens`); anything else is an orphan left by a dead worker + * (its token event was lost with it) — reap it and restart the worker so + * the next request reopens as the writer instead of silently serving a + * frozen read-only view until process exit. + */ + private async recoverOrphanedLock(): Promise { + if (this.exiting || this.worker === null) return; + const lockPath = join(this.dir, 'db.lock'); + let token: string; + try { + const parsed = JSON.parse(await readFile(lockPath, 'utf8')) as { + pid?: unknown; + token?: unknown; + }; + if (parsed.pid !== process.pid || typeof parsed.token !== 'string') return; + token = parsed.token; + } catch { + return; // gone or unreadable — nothing to reap + } + if (liveLockTokens.has(token)) return; // a live worker of this process owns it + // Compare-then-delete: re-read the line right before removing it — a new + // owner may have replaced it since the judgment above, and deleting a + // live owner's lock would admit a second writer. + try { + const again = JSON.parse(await readFile(lockPath, 'utf8')) as { + pid?: unknown; + token?: unknown; + }; + if (again.pid !== process.pid || again.token !== token) return; + if (liveLockTokens.has(token)) return; + } catch { + return; // vanished or replaced mid-check — nothing safe to reap + } + this.log.warn('global search: reaping an orphaned same-pid lock and restarting the worker', { + dir: this.dir, + }); + await rm(lockPath, { force: true }).catch(() => {}); + const worker = this.worker; + if (worker === null) return; + // The current worker opened read-only because of the orphan; terminate + // it — the dirty-exit path (backoff, then a lazy respawn on the next + // RPC) brings up a writer. + await worker.terminate().catch(() => {}); + } + + dispose(): Promise { + this.exiting = true; + this.disposePromise ??= this.doDispose(); + return this.disposePromise; + } + + private disposePromise: Promise | null = null; + + private async doDispose(): Promise { + // A spawn in flight must settle (successfully or not) before the close + // decision; a spawn that wins the race still gets closed below. + await this.spawnPromise?.catch(() => {}); + const worker = this.worker; + if (worker === null) return; + const closeTimeoutMs = this.options.closeTimeoutMs ?? DEFAULT_CLOSE_TIMEOUT_MS; + let timeout: ReturnType | null = null; + try { + // The close request bypasses ensureWorker (the host is already marked + // exiting) but still rides the request multiplexer. + await Promise.race([ + this.sendRequest(worker, 'close'), + new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + reject(new Error(`search worker close timed out after ${closeTimeoutMs}ms`)); + }, closeTimeoutMs); + timeout.unref?.(); + }), + ]); + } catch { + // Drain failures and timeouts both fall through to the terminate backstop. + } finally { + if (timeout !== null) clearTimeout(timeout); + } + if (this.worker !== null) { + await worker.terminate().catch(() => {}); + await this.waitForExit(); + } + } +} diff --git a/packages/kap-server/src/search/worker/protocol.ts b/packages/kap-server/src/search/worker/protocol.ts new file mode 100644 index 000000000..1fab7bd64 --- /dev/null +++ b/packages/kap-server/src/search/worker/protocol.ts @@ -0,0 +1,131 @@ +/** + * `search/worker` — the RPC protocol between the main-process search worker + * host (`host.ts`) and the worker thread entry (`entry.ts`). + * + * One long-lived worker exclusively owns the `/search-index` MiniDb + * handle (open/replay, sync, refresh, queries, reindex, close). The protocol + * is a small request/response envelope over `postMessage` with a versioned + * ready handshake — deliberately NOT an extension of minidb's bounded + * text-build worker (a one-shot build runner), which has a different + * lifecycle. + * + * Pure types + constants; safe to import from both sides. The worker entry's + * import closure runs under Node's native type stripping, so relative + * imports in this file's closure use explicit `.ts` specifiers. + */ + +import type { GlobalSearchErrorReason } from '../contract.ts'; +import type { + CoreLifecycleReport, + CoreSearchParams, + CoreSearchResult, + CoreStatus, + CoreSyncOutcome, + SyncSessionInput, +} from '../indexCore.ts'; + +export const SEARCH_WORKER_PROTOCOL_VERSION = 1; + +/** `workerData` payload for the search worker. */ +export interface SearchWorkerData { + /** Absolute path of the search-index database directory. */ + readonly dir: string; + /** + * Unique-per-spawn boot identifier, mixed into the index generation the + * worker reports: page tokens issued before a worker restart NEVER + * validate against the respawned worker (its local generation counter + * restarts from 0), so mid-pagination clients get `invalid_page_token` + * and restart the search instead of drifting across two generations that + * happen to share a number. + */ + readonly bootSalt: string; + /** + * Packaged (SEA-extracted) minidb text-build worker file. The main process + * already extracted it for its own minidb; the search worker configures + * its own copy of the runtime so full-text generation builds triggered by + * the search-index MiniDb keep running in a nested worker instead of + * inline in the search worker. Absent in dev (minidb resolves its own + * sibling .ts entry) and in plain bundles (inline-in-worker fallback). + */ + readonly textBuildWorkerPath?: string; +} + +// -- requests (main → worker) ------------------------------------------------- + +/** RPC calls: carry a request id and always produce a response. */ +export type SearchWorkerCall = + | { readonly id: number; readonly v: number; readonly type: 'open' } + | { readonly id: number; readonly v: number; readonly type: 'search'; readonly params: CoreSearchParams } + | { readonly id: number; readonly v: number; readonly type: 'sync'; readonly params: { readonly sessions: readonly SyncSessionInput[] } } + | { readonly id: number; readonly v: number; readonly type: 'refresh' } + | { readonly id: number; readonly v: number; readonly type: 'reindex' } + | { readonly id: number; readonly v: number; readonly type: 'status' } + | { readonly id: number; readonly v: number; readonly type: 'close' }; + +export type SearchWorkerCallType = SearchWorkerCall['type']; + +/** + * Control message (no id, no response): flip the worker-side core into + * closing state NOW so an in-flight sync abandons its remaining work at the + * next checkpoint — the host's beginClose()/dispose() must never be wedged + * behind a running pass. + */ +export interface SearchWorkerControlMessage { + readonly v: number; + readonly type: 'beginClose'; +} + +/** Everything the host may post to the worker. */ +export type SearchWorkerRequest = SearchWorkerCall | SearchWorkerControlMessage; + +// -- responses & events (worker → main) ---------------------------------------- + +/** Response payload of the `open` / `refresh` / `reindex` requests. */ +export interface SearchWorkerOpenResult { + readonly readOnly: boolean; + readonly lockToken?: string; + /** + * The core's lifecycle right after the call (stage 5): lets the host keep + * its cached aggregate state exact across worker (re)opens — an open that + * published a still-building text base reports 'building', not 'ready'. + */ + readonly lifecycle: CoreLifecycleReport; +} + +export interface SearchWorkerResultMap { + readonly open: SearchWorkerOpenResult; + readonly search: CoreSearchResult; + readonly sync: CoreSyncOutcome; + readonly refresh: SearchWorkerOpenResult; + readonly reindex: SearchWorkerOpenResult; + readonly status: CoreStatus; + readonly close: null; +} + +/** Serializable failure: typed search errors keep their reason. */ +export interface SearchWorkerErrorPayload { + readonly message: string; + readonly reason?: GlobalSearchErrorReason; +} + +export type SearchWorkerEvent = + | { readonly type: 'ready'; readonly v: number } + | { + readonly type: 'log'; + readonly level: 'info' | 'warn'; + readonly message: string; + readonly meta?: Record; + } + | { + /** + * Fired the moment the worker's MiniDb acquires the write lock — + * BEFORE the heavy open work (WAL replay / generation load) runs. The + * host needs the token up front: a worker that dies mid-open + * otherwise leaves a lock line carrying this process's (still alive) + * pid, which minidb's pid-liveness rule can never reclaim. + */ + readonly type: 'lockToken'; + readonly token: string; + } + | { readonly id: number; readonly type: 'result'; readonly result: unknown } + | { readonly id: number; readonly type: 'error'; readonly error: SearchWorkerErrorPayload }; diff --git a/packages/kap-server/src/search/worker/register-dev-hooks.mjs b/packages/kap-server/src/search/worker/register-dev-hooks.mjs new file mode 100644 index 000000000..b426debeb --- /dev/null +++ b/packages/kap-server/src/search/worker/register-dev-hooks.mjs @@ -0,0 +1,7 @@ +// Registers the dev search worker's module hooks (see dev-hooks.mjs). +// Passed to the worker via execArgv `--import `; the specifier is +// resolved relative to this file, so the hook module rides along regardless +// of the process cwd. Never bundled. +import { register } from 'node:module'; + +register('./dev-hooks.mjs', import.meta.url); diff --git a/packages/kap-server/src/search/worker/runtime.ts b/packages/kap-server/src/search/worker/runtime.ts new file mode 100644 index 000000000..6ade233f6 --- /dev/null +++ b/packages/kap-server/src/search/worker/runtime.ts @@ -0,0 +1,58 @@ +/** + * `search/worker` — packaged worker-entry registration (mirrors minidb's + * `worker-runtime.ts` for the text-build worker). + * + * In the SEA single-file binary the search worker entry exists only as an + * embedded asset; the CLI extracts it to the native-asset cache at startup + * and configures the absolute path here ONCE (see + * `apps/kimi-code/src/native/search-worker.ts`). Everywhere else the host + * resolves the entry itself (sibling `.ts` source in dev/tests, bundled + * `.mjs` sibling in the packaged npm layout), and this stays unconfigured. + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +export type SearchWorkerRuntimeState = + | { readonly configured: false } + | { readonly configured: true; readonly path: string }; + +let configuredPath: string | null = null; + +/** Configure the packaged search worker entry once during process startup. */ +export function configureSearchWorkerRuntime(entry: string): SearchWorkerRuntimeState { + if (!path.isAbsolute(entry)) { + throw new TypeError('search worker entry must be an absolute path'); + } + let stat: fs.Stats; + try { + stat = fs.statSync(entry); + } catch (error) { + throw new TypeError( + `search worker entry is not readable: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + if (!stat.isFile()) { + throw new TypeError('search worker entry must be a regular file'); + } + if (configuredPath !== null) { + if (configuredPath !== entry) { + throw new Error('search worker runtime is already configured'); + } + return { configured: true, path: configuredPath }; + } + configuredPath = entry; + return { configured: true, path: configuredPath }; +} + +/** Reset process-wide configuration. Intended for tests and controlled hosts. */ +export function resetSearchWorkerRuntime(): void { + configuredPath = null; +} + +export function getSearchWorkerRuntimeState(): SearchWorkerRuntimeState { + return configuredPath === null + ? { configured: false } + : { configured: true, path: configuredPath }; +} diff --git a/packages/kap-server/test/search/searchRoute.test.ts b/packages/kap-server/test/search/searchRoute.test.ts index 8176e8f73..e0f009f62 100644 --- a/packages/kap-server/test/search/searchRoute.test.ts +++ b/packages/kap-server/test/search/searchRoute.test.ts @@ -1,7 +1,12 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +// Keep these server boots on the WORKER search host (the suite-level setup +// disables the flag to spare non-search suites the background worker load): +// this file is the end-to-end coverage of the production worker path. +process.env['KIMI_CODE_EXPERIMENTAL_SEARCH_WORKER'] = '1'; + import { ISessionIndex, type SessionSummary } from '@moonshot-ai/agent-core-v2'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -212,3 +217,114 @@ describe('server-v2 /api/v1/search', () => { expect(body!.data.incomplete).toBeUndefined(); }); }); + +/** + * Index-separation contract (phase 2): with the global search database + * unavailable, session list / get / create / cold resume keep working — only + * the real full-text search request reports the outage. The sabotage below + * plants a plain FILE at `/search-index`, so every search-MiniDb open + * fails for the whole server lifetime. + */ +describe('server-v2 session routes with the global search DB unavailable', () => { + let server: RunningServer | undefined; + let home: string | undefined; + let base: string; + + beforeEach(async () => { + home = await mkdtemp(join(tmpdir(), 'kimi-server-v2-search-down-')); + await writeFile(join(home, 'search-index'), 'not a minidb directory', 'utf8'); + }); + + afterEach(async () => { + if (server !== undefined) { + await server.close(); + server = undefined; + } + if (home !== undefined) { + await rm(home, { recursive: true, force: true }); + home = undefined; + } + }); + + async function boot(): Promise { + server = await startServer({ + hostIdentity: TEST_HOST_IDENTITY, + host: '127.0.0.1', + port: 0, + homeDir: home as string, + logLevel: 'silent', + }); + base = `http://127.0.0.1:${server.port}`; + } + + async function getJson(path: string): Promise> { + const res = await authedFetch(server as RunningServer, base, path); + return (await res.json()) as Envelope; + } + + async function postJson(path: string, body?: unknown): Promise> { + const res = await authedFetch(server as RunningServer, base, path, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body ?? {}), + }); + return (await res.json()) as Envelope; + } + + it('session list / create / get / cold resume pass with the search index down', { timeout: 30_000 }, async () => { + await boot(); + // create (the write path) + const created = await postJson<{ id: string }>('/api/v1/sessions', { + metadata: { cwd: home }, + }); + expect(created.code).toBe(0); + const id = created.data.id; + // list (the listSessions path) + const list = await getJson<{ items: { id: string }[] }>('/api/v1/sessions'); + expect(list.code).toBe(0); + expect(list.data.items.map((item) => item.id)).toContain(id); + await server!.close(); + server = undefined; + + // A fresh server over the same home: every session is cold, so the + // messages route resumes it from authoritative metadata — the + // `--resume` / `--continue` equivalent. + await boot(); + const coldList = await getJson<{ items: { id: string }[] }>('/api/v1/sessions'); + expect(coldList.code).toBe(0); + expect(coldList.data.items.map((item) => item.id)).toContain(id); + const got = await getJson<{ id: string }>(`/api/v1/sessions/${id}`); + expect(got.code).toBe(0); + const messages = await getJson<{ items: unknown[] }>(`/api/v1/sessions/${id}/messages`); + expect(messages.code).toBe(0); + + // The session flows never opened the search database: the sabotage file + // is still exactly what the test planted. + const probe = await stat(join(home as string, 'search-index')); + expect(probe.isFile()).toBe(true); + }); + + it('only the full-text search request reports the index outage', { timeout: 30_000 }, async () => { + await boot(); + const created = await postJson<{ id: string }>('/api/v1/sessions', { + metadata: { cwd: home }, + }); + expect(created.code).toBe(0); + + // The search request surfaces the failure (50001) once the boot-time + // background open has failed; until then it may answer `building`. + await expect + .poll( + async () => (await postJson('/api/v1/search', { query: 'anything' })).code, + { timeout: 10_000, interval: 100 }, + ) + .toBe(50001); + const search = await postJson('/api/v1/search', { query: 'anything' }); + expect(search.code).toBe(50001); + expect(search.msg).toContain('search index failed to open'); + + // Session routes stay green throughout. + const list = await getJson<{ items: unknown[] }>('/api/v1/sessions'); + expect(list.code).toBe(0); + }); +}); diff --git a/packages/kap-server/test/search/searchService.test.ts b/packages/kap-server/test/search/searchService.test.ts index 4f2619d5a..8b1f90bc9 100644 --- a/packages/kap-server/test/search/searchService.test.ts +++ b/packages/kap-server/test/search/searchService.test.ts @@ -1,11 +1,13 @@ import { createHash } from 'node:crypto'; -import { appendFile, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { appendFile, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { monitorEventLoopDelay, type IntervalHistogram } from 'node:perf_hooks'; +import { monitorEventLoopDelay, performance, type IntervalHistogram } from 'node:perf_hooks'; +import { Worker } from 'node:worker_threads'; import type { IBootstrapService, + IFlagService, ILogService, ISessionIndex, SessionSummary, @@ -14,12 +16,21 @@ import { MiniDb } from '@moonshot-ai/minidb'; import { TranscriptStore, type TranscriptOperation } from '@moonshot-ai/transcript'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { SyncSessionInput } from '../../src/search/indexCore'; import { GlobalSearchError, GlobalSearchService, + InlineSearchBackend, + SEARCH_WORKER_FLAG_ID, drainGlobalSearchDisposals, type LiveTranscriptSource, + type SearchBackend, } from '../../src/search/searchService'; +import { + SearchWorkerError, + SearchWorkerHost, +} from '../../src/search/worker/host'; +import type { SearchWorkerRequest } from '../../src/search/worker/protocol'; // --------------------------------------------------------------------------- // fixtures & stubs @@ -118,12 +129,76 @@ const noopLog = { debug: () => {}, } as unknown as ILogService; +function makeFlags(workerEnabled: boolean): IFlagService { + return { + enabled: (id: string) => id === SEARCH_WORKER_FLAG_ID && workerEnabled, + } as unknown as IFlagService; +} + +/** + * Worker-host service (the production default): the index core runs in a + * dedicated worker thread. Most behavior tests run against this host. + */ function makeService(home: string, index: ISessionIndex): GlobalSearchService { - const service = new GlobalSearchService(index, makeBootstrap(home), noopLog); + const service = new GlobalSearchService(index, makeBootstrap(home), noopLog, makeFlags(true)); service.syncDebounceMs = 0; return service; } +/** + * Inline-host service (the rollback host): the index core runs in-process so + * tests can drive its internals (direct db writes, patched probes) — the + * worker thread's core is not reachable from the test thread. + */ +function makeInlineService(home: string, index: ISessionIndex): GlobalSearchService { + const service = new GlobalSearchService(index, makeBootstrap(home), noopLog, makeFlags(false)); + service.syncDebounceMs = 0; + return service; +} + +/** Loose db handle shape for direct fixture writes (same as the pre-split suite). */ +interface TestDb { + get(key: string): Record | undefined; + set(key: string, value: unknown): Promise; + del(key: string): Promise; + batch(ops: { op: 'set' | 'del'; key: string; value?: unknown }[]): Promise; + query(criteria: { key: { prefix: string }; project?: string[] }): { + key: string; + value: Record; + }[]; + compact(): Promise; + close(): Promise; +} + +/** The inline backend's index core (test drive point; worker mode has none). */ +type TestableCore = { + db: TestDb | null; + doRefreshReadonly(): Promise; + computeFingerprint(): Promise; + openSearchDb(): Promise; + deleteSessionDocs(db: TestDb, sessionId: string): Promise; + syncSession(db: TestDb, summary: SyncSessionInput): Promise; +}; + +function coreOf(service: GlobalSearchService): TestableCore { + const backend = (service as unknown as { backend: SearchBackend }).backend; + if (!(backend instanceof InlineSearchBackend)) { + throw new Error('this test drives core internals and must use makeInlineService'); + } + return backend.core as unknown as TestableCore; +} + +/** SessionSummary → the core's sync input (dir resolved like the service does). */ +function syncInput(homeDir: string, s: SessionSummary): SyncSessionInput { + return { + id: s.id, + workspaceId: s.workspaceId, + title: s.title, + updatedAt: s.updatedAt, + dir: join(homeDir, 'sessions', s.workspaceId, s.id), + }; +} + // --------------------------------------------------------------------------- // test-only drives for the background coordinator / private internals // --------------------------------------------------------------------------- @@ -150,25 +225,7 @@ function refreshNow(service: GlobalSearchService): Promise { } interface ServiceInternals { - db: { - get(key: string): Record | undefined; - set(key: string, value: unknown): Promise; - del(key: string): Promise; - batch(ops: { op: 'set' | 'del'; key: string; value?: unknown }[]): Promise; - query(criteria: { key: { prefix: string }; project?: string[] }): { - key: string; - value: Record; - }[]; - compact(): Promise; - close(): Promise; - } | null; - generation: number; syncPromise: Promise | null; - fileMetaMigrated: boolean; - doRefreshReadonly(): Promise; - syncSession(db: NonNullable, summary: SessionSummary): Promise; - openSearchDb(): Promise; - computeFingerprint(): Promise; } function internals(service: GlobalSearchService): ServiceInternals { @@ -187,12 +244,13 @@ async function flush(rounds = 20): Promise { } /** - * Patch a private async method to signal its first entry and hold it until - * released — the deterministic "dispose lands mid-op" interleaving for the - * lifecycle tests. Later calls pass through to the original. + * Patch a private async method of the inline core to signal its first entry + * and hold it until released — the deterministic "dispose lands mid-op" + * interleaving for the lifecycle tests. Later calls pass through to the + * original. */ function blockFirstCall( - service: GlobalSearchService, + core: TestableCore, method: 'deleteSessionDocs' | 'computeFingerprint', ): { entered: Promise; release: () => void } { let enteredResolve!: () => void; @@ -203,7 +261,7 @@ function blockFirstCall( const gate = new Promise((resolve) => { releaseResolve = resolve; }); - const host = service as unknown as Record; + const host = core as unknown as Record; const orig = host[method] as (this: unknown, ...args: unknown[]) => Promise; let first = true; host[method] = async function (this: unknown, ...args: unknown[]) { @@ -552,12 +610,12 @@ describe('GlobalSearchService', () => { const file = await writeWire(home!, 's1', 'main', [userLine('苹果 base', T1)]); const index = staticIndex([s1]); - const writer = track(makeService(home!, index)); + const writer = track(makeInlineService(home!, index)); await writer.reindex(); // Same process, same homeDir: the lock is held by `writer`, so this // instance must downgrade to read-only instead of rebuilding. - const reader = track(makeService(home!, index)); + const reader = track(makeInlineService(home!, index)); const status = await reader.status(); expect(status.documents).toBe(2); // replayed at open: message + title @@ -581,14 +639,14 @@ describe('GlobalSearchService', () => { // WAL rotation on the writer forces the reader's reopen path (open the // replacement, then swap); results stay correct afterwards. - const writerDb = internals(writer).db; + const writerDb = coreOf(writer).db; await writerDb?.compact(); await refreshNow(reader); const afterRotation = await reader.search({ query: '苹果' }); expect(afterRotation.items.length).toBe(2); }); - it('rejects reindex on a read-only instance', async () => { + it('rejects reindex on a read-only instance', { timeout: 30_000 }, async () => { const s1 = summary('s1', 'lock', T1); await writeWire(home!, 's1', 'main', [userLine('苹果 lock', T1)]); const index = staticIndex([s1]); @@ -604,13 +662,13 @@ describe('GlobalSearchService', () => { it('serves the building page while the index base is rebuilding, real hits after commit', async () => { const s1 = summary('s1', 'shared', T1); await writeWire(home!, 's1', 'main', [userLine('苹果 base', T1)]); - const service = track(makeService(home!, staticIndex([s1]))); + const service = track(makeInlineService(home!, staticIndex([s1]))); await service.reindex(); // Force the base-building state on the served handle (the deferred // open-time build sets exactly this): searches must return the building // page — never throw, never silently serve partial results. - const db = internals(service).db as unknown as { textIndexBuilding(name: string): boolean }; + const db = coreOf(service).db as unknown as { textIndexBuilding(name: string): boolean }; const original = db.textIndexBuilding.bind(db); db.textIndexBuilding = () => true; try { @@ -637,12 +695,12 @@ describe('GlobalSearchService', () => { const s1 = summary('s1', 'shared', T1); await writeWire(home!, 's1', 'main', [userLine('苹果 base', T1)]); const index = staticIndex([s1]); - const writer = track(makeService(home!, index)); + const writer = track(makeInlineService(home!, index)); await writer.reindex(); - const reader = track(makeService(home!, index)); + const reader = track(makeInlineService(home!, index)); await reader.status(); // forces the read-only open - const db = internals(reader).db as unknown as { textIndexBuilding(name: string): boolean }; + const db = coreOf(reader).db as unknown as { textIndexBuilding(name: string): boolean }; const original = db.textIndexBuilding.bind(db); db.textIndexBuilding = () => true; try { @@ -938,12 +996,12 @@ describe('GlobalSearchService', () => { stepBeginLine('u1', 1, T1 + 100), assistantStepLine('苹果 reply one', 'u1', T1 + 200), ]); - const service = track(makeService(home!, staticIndex([s1]))); + const service = track(makeInlineService(home!, staticIndex([s1]))); await service.reindex(); // Simulate a file meta written before step tracking existed by stripping // stepState from the persisted meta. - const db = internals(service).db; + const db = coreOf(service).db; expect(db).not.toBeNull(); const metaRows = db!.query({ key: { prefix: '\0meta\\file\\' } }); expect(metaRows.length).toBe(1); @@ -1157,12 +1215,12 @@ describe('GlobalSearchService', () => { it('scopes one session sync to its own file-meta keys among 10k sessions', async () => { const s1 = summary('s1', 'scoped', T1); await writeWire(home!, 's1', 'main', [userLine('苹果 scoped', T1)]); - const service = track(makeService(home!, staticIndex([s1]))); + const service = track(makeInlineService(home!, staticIndex([s1]))); await service.reindex(); // Seed 10k foreign sessions × 3 file-meta rows directly (no wire files // on disk), simulating a large pre-existing index. - const db = internals(service).db!; + const db = coreOf(service).db!; for (let from = 0; from < 10_000; from += 2_500) { const ops: { op: 'set'; key: string; value: unknown }[] = []; for (let i = from; i < from + 2_500; i++) { @@ -1195,7 +1253,7 @@ describe('GlobalSearchService', () => { if (criteria.key.prefix.startsWith('\0meta\\file\\')) metaRowsScanned += rows.length; return rows; }; - await internals(service).syncSession(db, s1); + await coreOf(service).syncSession(db, syncInput(home!, s1)); expect(metaRowsScanned).toBeLessThanOrEqual(5); // s1 has exactly 1 meta }); @@ -1203,13 +1261,13 @@ describe('GlobalSearchService', () => { const s1 = summary('s1', 'migration', T1); const main = await writeWire(home!, 's1', 'main', [userLine('苹果 main', T1)]); await writeWire(home!, 's1', 'agent-1', [userLine('苹果 sub', T2)]); - const first = track(makeService(home!, staticIndex([s1]))); + const first = track(makeInlineService(home!, staticIndex([s1]))); await first.reindex(); expect((await first.search({ query: '苹果' })).items.length).toBe(2); // Rewrite every file meta under its pre-v2 hash-only key (same value), // simulating an index written before the key migration. - const db = internals(first).db!; + const db = coreOf(first).db!; const metas = db.query({ key: { prefix: '\0meta\\file\\' } }); expect(metas.length).toBe(2); const legacyOffsets = new Map(); @@ -1225,9 +1283,9 @@ describe('GlobalSearchService', () => { await drainGlobalSearchDisposals(); // A fresh instance migrates on its first background pass. - const second = track(makeService(home!, staticIndex([s1]))); + const second = track(makeInlineService(home!, staticIndex([s1]))); await settleSync(second); - const db2 = internals(second).db!; + const db2 = coreOf(second).db!; const after = db2.query({ key: { prefix: '\0meta\\file\\' } }); expect(after.length).toBe(2); for (const row of after) { @@ -1265,7 +1323,10 @@ describe('GlobalSearchService', () => { Buffer.from(page1.pageToken!, 'base64url').toString('utf8'), ) as Record; expect(decoded['v']).toBe(2); - expect(typeof decoded['g']).toBe('number'); + // The pinned generation is `:` — the salt makes + // tokens fail validation across a worker/process restart. + expect(typeof decoded['g']).toBe('string'); + expect(decoded['g']).toMatch(/:\d+$/); expect(Array.isArray(decoded['b'])).toBe(true); // Additive writes land mid-pagination: they do NOT change the @@ -1351,14 +1412,14 @@ describe('GlobalSearchService', () => { const s1 = summary('s1', 'degraded', T1); const file = await writeWire(home!, 's1', 'main', [userLine('苹果 base', T1)]); const index = staticIndex([s1]); - const writer = track(makeService(home!, index)); + const writer = track(makeInlineService(home!, index)); await writer.reindex(); - const reader = track(makeService(home!, index)); + const reader = track(makeInlineService(home!, index)); await reader.status(); expect((await reader.search({ query: '苹果' })).items.length).toBe(1); - const original = internals(reader).doRefreshReadonly; - internals(reader).doRefreshReadonly = async () => { + const original = coreOf(reader).doRefreshReadonly; + coreOf(reader).doRefreshReadonly = async () => { throw new Error('refresh boom'); }; await appendFile(file, `${userLine('苹果 delta', T2)}\n`, 'utf8'); @@ -1375,7 +1436,7 @@ describe('GlobalSearchService', () => { expect(degraded.items.length).toBe(1); // still the stale generation // Restoring the refresh path self-heals the flag. - internals(reader).doRefreshReadonly = original; + coreOf(reader).doRefreshReadonly = original; await refreshNow(reader); const healed = await reader.search({ query: '苹果' }); expect(healed.indexState.degraded).toBeUndefined(); @@ -1469,16 +1530,16 @@ describe('GlobalSearchService', () => { it('self-heals a failed open through search traffic', async () => { const s1 = summary('s1', 'heal', T1); await writeWire(home!, 's1', 'main', [userLine('苹果 heal', T1)]); - const service = track(makeService(home!, staticIndex([s1]))); + const service = track(makeInlineService(home!, staticIndex([s1]))); // The db open fails transiently (e.g. a read-only open racing a // writer's compaction). - const si = internals(service); - const origOpen = si.openSearchDb; + const core = coreOf(service); + const origOpen = core.openSearchDb; let failOpen = true; - si.openSearchDb = async () => { + core.openSearchDb = async () => { if (failOpen) throw new Error('open boom'); - return origOpen.call(service); + return origOpen.call(core); }; // First search: building semantics while the (doomed) pass runs. @@ -1511,20 +1572,20 @@ describe('GlobalSearchService', () => { const s1 = summary('s1', 'swap', T1); await writeWire(home!, 's1', 'main', [userLine('苹果 base', T1)]); const index = staticIndex([s1]); - const writer = track(makeService(home!, index)); + const writer = track(makeInlineService(home!, index)); await writer.reindex(); - const reader = track(makeService(home!, index)); + const reader = track(makeInlineService(home!, index)); await reader.status(); expect((await reader.search({ query: '苹果' })).items.length).toBe(1); // Rotate the writer's WAL so the reader's refresh must take the reopen // path (which swaps the handle and closes the previous one). - await internals(writer).db!.compact(); + await coreOf(writer).db!.compact(); // Park the search's fingerprint probe at a gate; while it is parked, a // background refresh completes the reopen and closes the handle the // search captured. - const ri = internals(reader); + const ri = coreOf(reader); const origFp = ri.computeFingerprint.bind(ri); let fpCalls = 0; let releaseProbe!: () => void; @@ -2200,7 +2261,7 @@ describe('GlobalSearchService', () => { it('closes the handle when post-open index setup fails, and the next open becomes the writer again (review #19)', async () => { const s1 = summary('s1', 'open failure', T1); await writeWire(home!, 's1', 'main', [userLine('苹果 recovery', T1)]); - const service = track(makeService(home!, staticIndex([s1]))); + const service = track(makeInlineService(home!, staticIndex([s1]))); // Inject a failure into the writer-side text-index creation that runs // after MiniDb.open has handed out the handle. @@ -2214,11 +2275,11 @@ describe('GlobalSearchService', () => { // must be closed — otherwise the leaked writer keeps the lock for the // rest of the process lifetime and every later open degrades to // read-only (the review #19 self-lock). - expect(internals(service).db).toBeNull(); + expect(coreOf(service).db).toBeNull(); // The very next open takes the write lock again (same process). await service.reindex(); - const db = internals(service).db; + const db = coreOf(service).db; expect(db).not.toBeNull(); expect((db as unknown as { readOnly: boolean }).readOnly).toBe(false); expect((await service.search({ query: '苹果' })).items.length).toBe(1); @@ -2233,6 +2294,7 @@ describe('GlobalSearchService', () => { makeSessionIndex(async () => ({ items: sessions, nextCursor: undefined })), makeBootstrap(home!), log, + makeFlags(false), ); service.syncDebounceMs = 0; track(service); @@ -2243,7 +2305,7 @@ describe('GlobalSearchService', () => { await settleSync(service); // Record trailing writes; the gated pass must skip its STATS_KEY write. - const db = internals(service).db!; + const db = coreOf(service).db!; const setKeys: string[] = []; const origSet = db.set.bind(db); db.set = async (key: string, value: unknown) => { @@ -2254,7 +2316,7 @@ describe('GlobalSearchService', () => { // The next pass deletes the disappeared session; block it inside the // deleteSessionDocs window, then dispose mid-pass. sessions.length = 0; - const blocked = blockFirstCall(service, 'deleteSessionDocs'); + const blocked = blockFirstCall(coreOf(service), 'deleteSessionDocs'); const sync = syncNow(service); await blocked.entered; service.dispose(); @@ -2270,7 +2332,7 @@ describe('GlobalSearchService', () => { await sync; await drain; expect(drained).toBe(true); - expect(internals(service).db).toBeNull(); + expect(coreOf(service).db).toBeNull(); // The gate closed before the trailing stats write: it was skipped, and // no background write ever hit the closed handle. expect(setKeys).not.toContain('\0meta\\stats'); @@ -2280,20 +2342,20 @@ describe('GlobalSearchService', () => { it('dispose drains an in-flight read-only refresh before closing the db (review #20)', async () => { const s1 = summary('s1', 'drain refresh', T1); await writeWire(home!, 's1', 'main', [userLine('苹果 refresh', T1)]); - const writer = track(makeService(home!, staticIndex([s1]))); + const writer = track(makeInlineService(home!, staticIndex([s1]))); await writer.reindex(); // A second instance on the same home opens read-only (the writer holds // the lock) and catches up by refreshing from the writer's commits. const { log, warnings } = recordingLog(); - const reader = new GlobalSearchService(staticIndex([s1]), makeBootstrap(home!), log); + const reader = new GlobalSearchService(staticIndex([s1]), makeBootstrap(home!), log, makeFlags(false)); reader.syncDebounceMs = 0; track(reader); await syncNow(reader); // join the constructor-kicked pass: opens the read-only handle - expect((internals(reader).db as unknown as { readOnly: boolean } | null)?.readOnly).toBe(true); + expect((coreOf(reader).db as unknown as { readOnly: boolean } | null)?.readOnly).toBe(true); // Block the next refresh inside its fingerprint probe, then dispose. - const blocked = blockFirstCall(reader, 'computeFingerprint'); + const blocked = blockFirstCall(coreOf(reader), 'computeFingerprint'); const refresh = refreshNow(reader); await blocked.entered; reader.dispose(); @@ -2309,7 +2371,7 @@ describe('GlobalSearchService', () => { await refresh; await drain; expect(drained).toBe(true); - expect(internals(reader).db).toBeNull(); + expect(coreOf(reader).db).toBeNull(); expect(warnings.filter((w) => w.includes('closed'))).toEqual([]); }); @@ -2324,12 +2386,13 @@ describe('GlobalSearchService', () => { makeSessionIndex(async () => ({ items: sessions, nextCursor: undefined })), makeBootstrap(root), noopLog, + makeFlags(false), ); service.syncDebounceMs = 0; track(service); await service.reindex(); sessions.length = 0; - const blocked = blockFirstCall(service, 'deleteSessionDocs'); + const blocked = blockFirstCall(coreOf(service), 'deleteSessionDocs'); const sync = syncNow(service); await blocked.entered; return { service, blocked, sync }; @@ -2353,7 +2416,7 @@ describe('GlobalSearchService', () => { // Let the first service finish: a snapshot-only drain (Promise.all // over the set at call time) would return here, leaving b behind. - const aDb = internals(a.service).db!; + const aDb = coreOf(a.service).db!; a.blocked.release(); await a.sync; // Join A's in-pending close (close is idempotent and shared) so its @@ -2366,8 +2429,8 @@ describe('GlobalSearchService', () => { await b.sync; await drain; expect(drained).toBe(true); - expect(internals(a.service).db).toBeNull(); - expect(internals(b.service).db).toBeNull(); + expect(coreOf(a.service).db).toBeNull(); + expect(coreOf(b.service).db).toBeNull(); } finally { await rm(homeB, { recursive: true, force: true }); } @@ -2376,8 +2439,871 @@ describe('GlobalSearchService', () => { }); // --------------------------------------------------------------------------- -// stage-1 performance baseline over a synthetic corpus +// stage-4: search worker host lifecycle // --------------------------------------------------------------------------- +// The worker-mode service runs the index core in a dedicated worker thread. +// These tests cover the host's lifecycle contracts: crash restart with the +// token-guarded lock reap, corruption rebuild inside the worker, cross-worker +// lock contention, read-only reopen after WAL/snapshot replacement, in-flight +// request convergence, dispose backstop, and the main-thread latency probe. + +describe('search worker host (stage 4)', () => { + // These tests spawn real worker threads (≈200ms each, several per test); + // the explicit 30s timeouts keep them load-proof under a saturated + // full-suite CI run (vitest's 5s default is meant for in-process work). + let home: string | undefined; + const services: GlobalSearchService[] = []; + const hosts: SearchWorkerHost[] = []; + + beforeEach(async () => { + home = await mkdtemp(join(tmpdir(), 'kimi-kap-search-worker-')); + }); + + afterEach(async () => { + for (const service of services.splice(0)) service.dispose(); + for (const host of hosts.splice(0)) await host.dispose().catch(() => {}); + await drainGlobalSearchDisposals(); + if (home !== undefined) { + await rm(home, { recursive: true, force: true }); + home = undefined; + } + }); + + function track(service: GlobalSearchService): GlobalSearchService { + services.push(service); + return service; + } + + function hostOf(service: GlobalSearchService): SearchWorkerHost { + const backend = (service as unknown as { backend: SearchBackend }).backend; + if (!(backend instanceof SearchWorkerHost)) { + throw new Error('expected the worker backend'); + } + return backend; + } + + const lockPath = (): string => join(home!, 'search-index', 'db.lock'); + + async function waitForGone(path: string): Promise { + await vi.waitFor(async () => { + await expect(stat(path)).rejects.toThrow(); + }); + } + + it('restarts a killed worker, reaps its lock, and keeps serving', { timeout: 30_000 }, async () => { + const s1 = summary('s1', 'crash', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 crash', T1)]); + const service = track(makeService(home!, staticIndex([s1]))); + await service.reindex(); + expect((await service.search({ query: '苹果' })).items.length).toBe(1); + + // The worker holds the write lock with THIS process's pid (worker threads + // share it) — the stale-by-pid rule alone could never reclaim it here. + const lockRaw = JSON.parse(await readFile(lockPath(), 'utf8')) as { pid: number }; + expect(lockRaw.pid).toBe(process.pid); + + await hostOf(service).killWorkerForTest(); + // The dead worker's lock line is reaped (guarded by its token). + await waitForGone(lockPath()); + + // Inside the backoff window the search reports a recognizable degraded + // page instead of hanging or silently serving nothing. + const degraded = await service.search({ query: '苹果' }); + expect(degraded.items).toEqual([]); + expect(degraded.indexState.state).toBe('building'); + expect(degraded.indexState.degraded).toContain('worker'); + + // After the backoff the coordinator's retry respawns the worker, reopens + // the index and serves real hits again. + await new Promise((resolve) => setTimeout(resolve, 700)); + await settleSync(service); + const page = await service.search({ query: '苹果' }); + expect(page.items.length).toBe(1); + expect(page.indexState.state).toBe('ready'); + }); + + it('reports the lock token at acquire time; a mid-open kill leaves a reapable lock', { timeout: 30_000 }, async () => { + // Seed a corpus with an inline writer so the worker's first open has a + // real WAL/generation to chew — the kill lands while the open RPC is + // still in flight, so the token could only have arrived via the early + // `lockToken` event (no response had a chance to carry it). + const summaries: SessionSummary[] = []; + for (let i = 0; i < 300; i++) { + const s = summary(`midopen-${i}`, `midopen 会话 ${i}`, T1 + i); + summaries.push(s); + const lines: string[] = []; + for (let j = 0; j < 30; j++) lines.push(userLine(`中途退出 ${i}-${j} 检索`, T1 + i * 100 + j)); + await writeWire(home!, s.id, 'main', lines); + } + const inline = track(makeInlineService(home!, staticIndex(summaries))); + await inline.reindex(); + inline.dispose(); + await drainGlobalSearchDisposals(); + + const dir = join(home!, 'search-index'); + const host = new SearchWorkerHost({ dir, log: noopLog }); + hosts.push(host); + let openSettled = false; + const opening = host.ensureOpen().then( + () => { + openSettled = true; + }, + () => { + openSettled = true; + }, + ); + await vi.waitFor( + () => { + expect(host.reportedLockToken).toBeDefined(); + }, + { interval: 5, timeout: 10_000 }, + ); + // The token event fired at lock-acquire time, long before the replay + // finishes — this is the property the mid-open reap depends on. + expect(openSettled).toBe(false); + + await host.killWorkerForTest(); + await opening; // settled (rejected as crashed) — captured above + await waitForGone(lockPath()); + + // After the backoff the replacement worker reopens as the WRITER — never + // a silent permanent read-only. + await new Promise((resolve) => setTimeout(resolve, 700)); + const reopened = await host.ensureOpen(); + expect(reopened.readOnly).toBe(false); + }); + + it('recovers a read-only open caused by an orphaned same-pid lock', { timeout: 30_000 }, async () => { + const dir = join(home!, 'search-index'); + await mkdir(dir, { recursive: true }); + // An orphan lock line as a crashed worker would leave it: this process's + // (alive) pid, but a token no live worker can claim. + await writeFile( + join(dir, 'db.lock'), + JSON.stringify({ pid: process.pid, ts: Date.now(), token: 'orphan-token' }), + 'utf8', + ); + const host = new SearchWorkerHost({ dir, log: noopLog }); + hosts.push(host); + const first = await host.ensureOpen(); + expect(first.readOnly).toBe(true); // the planted lock looks alive (same pid) + + // The orphan detector reaps it (the token belongs to no live worker) and + // restarts the worker, which reopens as the writer. + await vi.waitFor( + async () => { + const status = await host.status(); + expect(status.readOnly).toBe(false); + }, + { timeout: 10_000, interval: 200 }, + ); + }); + + it('beginClose abandons an in-flight worker sync and dispose stays bounded', { timeout: 30_000 }, async () => { + // 200 sessions × 30 lines: the first full pass takes hundreds of ms in + // the worker — long enough that beginClose lands before the pass can + // complete, with orders of magnitude of margin. + const summaries: SessionSummary[] = []; + for (let i = 0; i < 200; i++) { + const s = summary(`drain-${i}`, `drain 会话 ${i}`, T1 + i); + summaries.push(s); + const lines: string[] = []; + for (let j = 0; j < 30; j++) lines.push(userLine(`排空 ${i}-${j} 检索`, T1 + i * 100 + j)); + await writeWire(home!, s.id, 'main', lines); + } + const inputs = summaries.map((s) => syncInput(home!, s)); + const host = new SearchWorkerHost({ dir: join(home!, 'search-index'), log: noopLog }); + hosts.push(host); + + await host.ensureOpen(); // worker up first, so the sync posts before beginClose + const sync = host.sync(inputs); // first full pass (open + index) + await new Promise((resolve) => setTimeout(resolve, 20)); + host.beginClose(); + // The pass saw the closing gate and returned early as a no-op instead of + // running to completion. + const outcome = await sync; + expect(outcome.noop).toBe(true); + + const startedAt = performance.now(); + await host.dispose(); + expect(performance.now() - startedAt).toBeLessThan(5_000); + expect((host as unknown as { worker: unknown }).worker).toBeNull(); + }); + + it('times out a wedged request and terminates the worker (watchdog)', { timeout: 30_000 }, async () => { + const dir = join(home!, 'search-index'); + // Swallow `status` while the gate is up: the request is never delivered, + // so only the watchdog can settle it. The respawned worker is ungated. + let gate = true; + const host = new SearchWorkerHost({ + dir, + log: noopLog, + requestTimeoutMs: 300, + workerFactory: ({ url, data, execArgv }) => { + const worker = new Worker(url, { workerData: data, execArgv }); + const original = worker.postMessage.bind(worker); + worker.postMessage = ((message: unknown, ...rest: unknown[]) => { + if (gate && (message as { type?: string } | null)?.type === 'status') return true; + return original(message as Parameters[0], ...(rest as never[])); + }) as Worker['postMessage']; + return worker; + }, + }); + hosts.push(host); + await host.ensureOpen(); + + const wedged = host.status(); + wedged.catch(() => {}); + await expect(wedged).rejects.toMatchObject({ code: 'crashed' }); + await expect(wedged).rejects.toThrow(/timed out/); + + // The wedged worker was terminated; after the backoff the next call + // respawns a healthy (ungated) one. + gate = false; + await new Promise((resolve) => setTimeout(resolve, 700)); + const status = await host.status(); + expect(status.readOnly).toBe(false); + }); + + it('rejects in-flight requests as disposed during a clean close', { timeout: 30_000 }, async () => { + const dir = join(home!, 'search-index'); + const host = new SearchWorkerHost({ + dir, + log: noopLog, + workerFactory: ({ url, data, execArgv }) => { + const worker = new Worker(url, { workerData: data, execArgv }); + const original = worker.postMessage.bind(worker); + // Park `sync` in the channel; `close` flows through normally, so the + // worker drains and exits while the sync is still host-side pending. + worker.postMessage = ((message: unknown, ...rest: unknown[]) => { + if ((message as { type?: string } | null)?.type === 'sync') return true; + return original(message as Parameters[0], ...(rest as never[])); + }) as Worker['postMessage']; + return worker; + }, + }); + hosts.push(host); + await host.ensureOpen(); + + const sync = host.sync([]); + sync.catch(() => {}); + await host.dispose(); + // A clean close rejects the racing request as 'disposed', not 'crashed'. + await expect(sync).rejects.toMatchObject({ code: 'disposed' }); + }); + + it('reports index_unavailable for searches after dispose', { timeout: 30_000 }, async () => { + const s1 = summary('s1', 'disposed', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 disposed', T1)]); + const service = track(makeService(home!, staticIndex([s1]))); + await service.reindex(); + + service.dispose(); + // Same fail-fast contract as the inline host: a typed index_unavailable, + // never a bare channel error. + await expect(service.search({ query: '苹果' })).rejects.toMatchObject({ + reason: 'index_unavailable', + message: 'search service is disposed', + }); + await drainGlobalSearchDisposals(); + }); + + it('invalidates page tokens across a worker restart (boot salt)', { timeout: 30_000 }, async () => { + const s1 = summary('s1', 'boot', T1); + const lines: string[] = []; + for (let i = 0; i < 30; i++) lines.push(userLine(`苹果 boot ${i}`, T1 + i)); + await writeWire(home!, 's1', 'main', lines); + const service = track(makeService(home!, staticIndex([s1]))); + await service.reindex(); + const page1 = await service.search({ query: '苹果', sort: 'time_asc', pageSize: 10 }); + expect(page1.pageToken).toBeDefined(); + + await hostOf(service).killWorkerForTest(); + await new Promise((resolve) => setTimeout(resolve, 700)); + await settleSync(service); // respawn + reopen + resync + + // The respawned worker's local generation counter restarted from 0; the + // boot salt makes the dead worker's token fail validation instead of + // colliding with the fresh counter. + await expect( + service.search({ query: '苹果', sort: 'time_asc', pageSize: 10, pageToken: page1.pageToken }), + ).rejects.toMatchObject({ reason: 'invalid_page_token' }); + const restarted = await service.search({ query: '苹果', sort: 'time_asc', pageSize: 10 }); + expect(restarted.items.length).toBe(10); + }); + + it('rebuilds a corrupt database inside the worker', { timeout: 30_000 }, async () => { + const s1 = summary('s1', 'corrupt', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 corrupt', T1)]); + const first = track(makeService(home!, staticIndex([s1]))); + await first.reindex(); + expect((await first.search({ query: '苹果' })).items.length).toBe(1); + first.dispose(); + await drainGlobalSearchDisposals(); + + // Corrupt the snapshot: the worker's open must detect it and rebuild + // (the index is derived data — never repaired, only rebuilt). + await writeFile(join(home!, 'search-index', 'db.snapshot'), 'not a snapshot {{{', 'utf8'); + + const second = track(makeService(home!, staticIndex([s1]))); + await settleSync(second); + const page = await second.search({ query: '苹果' }); + expect(page.items.length).toBe(1); + expect(page.indexState.state).toBe('ready'); + }); + + it('runs a second worker read-only and a fresh worker becomes the writer after the writer dies', { timeout: 30_000 }, async () => { + const s1 = summary('s1', 'election', T1); + const file = await writeWire(home!, 's1', 'main', [userLine('苹果 election', T1)]); + const index = staticIndex([s1]); + + const writer = track(makeService(home!, index)); + await writer.reindex(); + const reader = track(makeService(home!, index)); + await reader.status(); + const ro = await reader.search({ query: '苹果' }); + expect(ro.indexState.state).toBe('readonly'); + expect(ro.items.length).toBe(1); + + // WAL catch-up across two workers of one process. + await appendFile(file, `${userLine('苹果 delta', T2)}\n`, 'utf8'); + await settleSync(writer); + await refreshNow(reader); + expect((await reader.search({ query: '苹果' })).items.length).toBe(2); + + // The writer's worker dies; its lock is reaped. A fresh instance's worker + // then acquires the write lock (the election) and can reindex. + await hostOf(writer).killWorkerForTest(); + await waitForGone(lockPath()); + writer.dispose(); + await drainGlobalSearchDisposals(); + + const third = track(makeService(home!, index)); + await settleSync(third); + const ready = await third.search({ query: '苹果' }); + expect(ready.indexState.state).toBe('ready'); + expect(ready.items.length).toBe(2); + await third.reindex(); // would throw readonly_index without the write lock + expect((await third.search({ query: '苹果' })).items.length).toBe(2); + }); + + it('reader worker reopens when the writer replaces the WAL/snapshot (reindex)', { timeout: 30_000 }, async () => { + const s1 = summary('s1', 'rotate', T1); + const file = await writeWire(home!, 's1', 'main', [userLine('苹果 rotate', T1)]); + const index = staticIndex([s1]); + + const writer = track(makeService(home!, index)); + await writer.reindex(); + const reader = track(makeService(home!, index)); + await reader.status(); + expect((await reader.search({ query: '苹果' })).items.length).toBe(1); + + // Reindex wipes and rebuilds the whole directory: the reader's watermark + // no longer aligns, so its refresh takes the reopen-and-swap path. + await appendFile(file, `${userLine('苹果 rotated', T2)}\n`, 'utf8'); + await writer.reindex(); + await refreshNow(reader); + const page = await reader.search({ query: '苹果' }); + expect(page.indexState.state).toBe('readonly'); + expect(page.items.length).toBe(2); + expect(page.items.some((h) => h.snippet.includes('rotated'))).toBe(true); + }); + + it('rejects in-flight requests when the worker dies', { timeout: 30_000 }, async () => { + const dir = join(home!, 'search-index'); + let gateSync = false; + const held: SearchWorkerRequest[] = []; + const host = new SearchWorkerHost({ + dir, + log: noopLog, + workerFactory: ({ url, data, execArgv }) => { + const worker = new Worker(url, { workerData: data, execArgv }); + const original = worker.postMessage.bind(worker); + // Park `sync` requests in the channel: never delivered to the worker, + // so the request is guaranteed in-flight when the worker dies. + worker.postMessage = ((message: unknown, ...rest: unknown[]) => { + if (gateSync && (message as { type?: string } | null)?.type === 'sync') { + held.push(message as SearchWorkerRequest); + return true; + } + return original(message as Parameters[0], ...(rest as never[])); + }) as Worker['postMessage']; + return worker; + }, + }); + hosts.push(host); + await host.ensureOpen(); + + gateSync = true; + const sync = host.sync([]); + sync.catch(() => {}); // settle handling below + await vi.waitFor(() => { + expect(held.length).toBe(1); + }); + + await host.killWorkerForTest(); + await expect(sync).rejects.toBeInstanceOf(SearchWorkerError); + await expect(sync).rejects.toMatchObject({ code: 'crashed' }); + await waitForGone(join(dir, 'db.lock')); + }); + + it('dispose terminates a wedged worker after the close timeout', { timeout: 30_000 }, async () => { + const dir = join(home!, 'search-index'); + const host = new SearchWorkerHost({ + dir, + log: noopLog, + closeTimeoutMs: 300, + workerFactory: ({ url, data, execArgv }) => { + const worker = new Worker(url, { workerData: data, execArgv }); + const original = worker.postMessage.bind(worker); + // Swallow `close`: the drain can never complete, so the host's + // terminate() backstop must end the worker within the timeout. + worker.postMessage = ((message: unknown, ...rest: unknown[]) => { + if ((message as { type?: string } | null)?.type === 'close') return true; + return original(message as Parameters[0], ...(rest as never[])); + }) as Worker['postMessage']; + return worker; + }, + }); + hosts.push(host); + await host.ensureOpen(); + + const startedAt = performance.now(); + await host.dispose(); + const elapsed = performance.now() - startedAt; + expect(elapsed).toBeGreaterThan(250); + expect(elapsed).toBeLessThan(10_000); + expect((host as unknown as { worker: unknown }).worker).toBeNull(); + }); + + it('keeps the main thread responsive while the worker opens and syncs a corpus', { timeout: 30_000 }, async () => { + // 120 sessions × 30 lines — enough for the worker's open + first full + // sync to take measurable time, small enough for CI. + const summaries: SessionSummary[] = []; + for (let i = 0; i < 120; i++) { + const s = summary(`probe-${i}`, `probe 会话 ${i}`, T1 + i); + summaries.push(s); + const lines: string[] = []; + for (let j = 0; j < 30; j++) { + lines.push(userLine(`检索词 probe ${i}-${j} 持久化`, T1 + i * 100 + j)); + } + await writeWire(home!, s.id, 'main', lines); + } + + const eld = monitorEventLoopDelay({ resolution: 5 }); + let probing = true; + let ticks = 0; + const probe = (async () => { + while (probing) { + await new Promise((resolve) => setImmediate(resolve)); + ticks++; + } + })(); + eld.enable(); + + const service = track(makeService(home!, staticIndex(summaries))); + // A search issued while the worker boots/opens/syncs answers with + // building semantics instead of waiting for the heavy work. + const early = await service.search({ query: '检索词' }); + expect(early.indexState.state).toBe('building'); + await settleSync(service); + + probing = false; + await probe; + eld.disable(); + const p99Ms = eld.percentile(99) / 1e6; + const maxMs = eld.max / 1e6; + // Logged for phase-to-phase comparison; asserted against CI-safe + // ceilings that still catch a main-thread stall regression — the point + // is "no main-thread stall", not speed. + console.log('[stage-4 worker probe]', JSON.stringify({ p99Ms, maxMs, ticks })); + expect(ticks).toBeGreaterThan(0); + expect(p99Ms).toBeLessThan(20); + expect(maxMs).toBeLessThan(100); + + const page = await service.search({ query: '检索词' }); + expect(page.items.length).toBeGreaterThan(0); + expect(page.indexState.state).toBe('ready'); + }); + + it('keeps the main thread responsive while the worker rebuilds and swaps the generation (reindex)', { timeout: 30_000 }, async () => { + // Same corpus scale as the open/sync probe. The measured reindex reruns + // the full build inside the worker and atomically swaps the published + // generation (the reader-reopen path); the main thread only passes RPC + // messages, so its event loop must stay live throughout. + const summaries: SessionSummary[] = []; + for (let i = 0; i < 120; i++) { + const s = summary(`reindex-${i}`, `reindex 会话 ${i}`, T1 + i); + summaries.push(s); + const lines: string[] = []; + for (let j = 0; j < 30; j++) { + lines.push(userLine(`检索词 reindex ${i}-${j} 持久化`, T1 + i * 100 + j)); + } + await writeWire(home!, s.id, 'main', lines); + } + + const service = track(makeService(home!, staticIndex(summaries))); + await service.reindex(); + expect((await service.search({ query: '检索词' })).indexState.state).toBe('ready'); + + const eld = monitorEventLoopDelay({ resolution: 5 }); + const stop = { done: false }; + let ticks = 0; + const probe = (async () => { + while (!stop.done) { + await new Promise((resolve) => setImmediate(resolve)); + ticks++; + } + })(); + eld.enable(); + + await service.reindex(); + + stop.done = true; + await probe; + eld.disable(); + const p99Ms = eld.percentile(99) / 1e6; + const maxMs = eld.max / 1e6; + // Same CI-safe ceilings as the open/sync probe: the point is "no + // main-thread stall", not speed. + console.log('[stage-4 reindex probe]', JSON.stringify({ p99Ms, maxMs, ticks })); + expect(ticks).toBeGreaterThan(0); + expect(p99Ms).toBeLessThan(20); + expect(maxMs).toBeLessThan(100); + + const page = await service.search({ query: '检索词' }); + expect(page.items.length).toBeGreaterThan(0); + expect(page.indexState.state).toBe('ready'); + }); +}); + +// --------------------------------------------------------------------------- +// stage-5: aggregate lifecycle diagnostics +// --------------------------------------------------------------------------- +// The search side's explicit state machine (stopped → opening → ready → +// building/degraded → closing) and the rules around it: status() keeps its +// historical kick/await semantics and never throws, lifecycleReport() is the +// non-intrusive local read, a corrupt rebuild is logged as its own outcome, +// concurrent first calls spawn/open exactly once, a clean dispose releases +// the lock, and a spawn failure never falls back to the inline host. + +describe('search lifecycle diagnostics (stage 5)', () => { + let home: string | undefined; + const services: GlobalSearchService[] = []; + const hosts: SearchWorkerHost[] = []; + + beforeEach(async () => { + home = await mkdtemp(join(tmpdir(), 'kimi-kap-search-lifecycle-')); + }); + + afterEach(async () => { + for (const service of services.splice(0)) service.dispose(); + for (const host of hosts.splice(0)) await host.dispose().catch(() => {}); + await drainGlobalSearchDisposals(); + if (home !== undefined) { + await rm(home, { recursive: true, force: true }); + home = undefined; + } + }); + + function track(service: GlobalSearchService): GlobalSearchService { + services.push(service); + return service; + } + + function hostOf(service: GlobalSearchService): SearchWorkerHost { + const backend = (service as unknown as { backend: SearchBackend }).backend; + if (!(backend instanceof SearchWorkerHost)) { + throw new Error('expected the worker backend'); + } + return backend; + } + + const lockPath = (): string => join(home!, 'search-index', 'db.lock'); + + async function waitForGone(path: string): Promise { + await vi.waitFor(async () => { + await expect(stat(path)).rejects.toThrow(); + }); + } + + it('walks stopped → ready → closing → stopped and never throws (inline)', async () => { + // No sessions and no index dir: the constructor's sync no-ops before + // touching the backend, so nothing has opened — and nothing gets created. + const service = track(makeInlineService(home!, staticIndex([]))); + expect(service.lifecycleReport()).toEqual({ state: 'stopped' }); + await expect(stat(join(home!, 'search-index'))).rejects.toThrow(); + + // status() keeps its historical semantics: it may kick the open, then + // reports the exact post-open lifecycle with real (zero) stats. + const status = await service.status(); + expect(status).toMatchObject({ sessions: 0, documents: 0, lifecycle: { state: 'ready' } }); + expect(service.lifecycleReport()).toEqual({ state: 'ready' }); + + service.dispose(); + // DI disposal is synchronous; the async drain is still settling. + expect(service.lifecycleReport()).toEqual({ state: 'closing' }); + await expect(service.status()).resolves.toMatchObject({ lifecycle: { state: 'closing' } }); + await drainGlobalSearchDisposals(); + expect(service.lifecycleReport()).toEqual({ state: 'stopped' }); + await expect(service.status()).resolves.toMatchObject({ lifecycle: { state: 'stopped' } }); + }); + + it('reports degraded instead of throwing when the open fails (inline)', async () => { + const s1 = summary('s1', 'open 失败', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 open-failure', T1)]); + const service = track(makeInlineService(home!, staticIndex([s1]))); + // Patch before the constructor-kicked sync reaches the open (the patch + // lands synchronously, the sync only reaches openSearchDb after awaits). + const core = coreOf(service) as unknown as { openSearchDb(): Promise }; + core.openSearchDb = async () => { + throw new Error('disk gone'); + }; + + // The constructor-kicked pass and the explicit retries all fail the open. + await settleSync(service).catch(() => {}); + await expect(service.search({ query: '苹果' })).rejects.toMatchObject({ + reason: 'index_unavailable', + }); + expect(service.lifecycleReport().state).toBe('degraded'); + expect(service.lifecycleReport().detail).toContain('disk gone'); + const status = await service.status(); + expect(status.lifecycle.state).toBe('degraded'); + expect(status.degraded).toContain('disk gone'); + }); + + it('logs the corruption rebuild as its own diagnostic outcome (inline)', { timeout: 30_000 }, async () => { + const s1 = summary('s1', 'corrupt 日志', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 corrupt-log', T1)]); + const first = track(makeInlineService(home!, staticIndex([s1]))); + await first.reindex(); + first.dispose(); + await drainGlobalSearchDisposals(); + + // Corrupt the text-index definitions sidecar: its JSON.parse failure is a + // rebuildable corruption (SyntaxError), so the open throws, the probe + // confirms the lock is free, and the derived index is rebuilt from + // scratch — with the warn line this test asserts. (A corrupt db.snapshot + // would NOT exercise this path: resync-mode recovery skips bad frames and + // the intact WAL heals the open.) + await writeFile(join(home!, 'search-index', 'db.textindexes.json'), 'not json {{{', 'utf8'); + + const { log, warnings } = recordingLog(); + const second = new GlobalSearchService(staticIndex([s1]), makeBootstrap(home!), log, makeFlags(false)); + track(second); + second.syncDebounceMs = 0; + await settleSync(second); + expect(warnings.some((line) => line.includes('corruption detected'))).toBe(true); + const page = await second.search({ query: '苹果' }); + expect(page.items.length).toBe(1); + expect(page.indexState.state).toBe('ready'); + }); + + it('a failing session index degrades search only — construction, search and status keep answering', async () => { + // The dependency direction pin (stage 5 work item 2): global search READS + // the session index, so a session-index failure surfaces inside search as + // a degraded note — it never propagates into construction or the answers. + const failing = makeSessionIndex(async () => { + throw new Error('metadata store down'); + }); + const service = track(makeInlineService(home!, failing)); + // Every sync pass fails on listRecent — the failure is recorded, never + // propagated into construction or the request paths. + await settleSync(service).catch(() => {}); + const page = await service.search({ query: 'anything' }); + expect(page.items).toEqual([]); + expect(page.indexState.state).toBe('building'); + const status = await service.status(); + expect(status.degraded).toContain('metadata store down'); + }); + + it('a restart attaches the published generation instead of rebuilding (inline)', async () => { + const s1 = summary('s1', '代际复用', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 generation-reuse', T1)]); + const first = track(makeInlineService(home!, staticIndex([s1]))); + await first.reindex(); + // Publish a persistent generation deterministically: the close-time + // best-effort publish is gated on a 4 MiB WAL-staleness rule that a tiny + // test corpus never crosses. + const firstCore = coreOf(first) as unknown as { + db: { buildGeneration(trigger: 'manual'): Promise } | null; + }; + await firstCore.db!.buildGeneration('manual'); + first.dispose(); + await drainGlobalSearchDisposals(); + + const second = track(makeInlineService(home!, staticIndex([s1]))); + await settleSync(second); + const page = await second.search({ query: '苹果' }); + expect(page.items.length).toBe(1); + // The open attached the published generation (and replayed at most the + // WAL delta past its checkpoint) — no full recovery ran. + const core = coreOf(second) as unknown as { + db: { lifecycleStatus(): { path: string[] } } | null; + }; + const path = core.db!.lifecycleStatus().path; + expect(path).toContain('generation-load'); + expect(path).not.toContain('full-rebuild'); + }); + + it('concurrent cold calls open the index exactly once (inline)', async () => { + const s1 = summary('s1', '单次打开', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 single-open', T1)]); + const service = track(makeInlineService(home!, staticIndex([s1]))); + const core = coreOf(service) as unknown as { openSearchDb(): Promise }; + const originalOpen = core.openSearchDb.bind(core); + let openCalls = 0; + core.openSearchDb = async () => { + openCalls++; + return originalOpen(); + }; + + // Cold searches answer building pages and kick the coordinator; the + // single-flight open underneath must run exactly once. + const pages = await Promise.all([ + service.search({ query: '苹果' }), + service.search({ query: '苹果' }), + service.search({ query: '苹果' }), + ]); + for (const page of pages) expect(page.indexState.state).toBe('building'); + await settleSync(service); + expect(openCalls).toBe(1); + expect((await service.search({ query: '苹果' })).items.length).toBe(1); + }); + + it('concurrent first RPCs spawn exactly one worker', { timeout: 30_000 }, async () => { + let spawns = 0; + const host = new SearchWorkerHost({ + dir: join(home!, 'search-index'), + log: noopLog, + workerFactory: ({ url, data, execArgv }) => { + spawns++; + return new Worker(url, { workerData: data, execArgv }); + }, + }); + hosts.push(host); + await Promise.all([host.ensureOpen(), host.ensureOpen(), host.ensureOpen(), host.ensureOpen()]); + expect(spawns).toBe(1); + expect(host.lifecycleSnapshot().state).toBe('ready'); + }); + + it('reports opening while the worker boots, ready after the sync, degraded after a crash', { timeout: 30_000 }, async () => { + const s1 = summary('s1', '生命周期', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 lifecycle', T1)]); + const service = track(makeService(home!, staticIndex([s1]))); + // Let the constructor-kicked sync reach the worker spawn (a worker boot + // takes ~200ms, far longer than these flush rounds), then the local + // report must show the transitional state WITHOUT any RPC. + await flush(); + expect(service.lifecycleReport().state).toBe('opening'); + await settleSync(service); + expect(service.lifecycleReport().state).toBe('ready'); + const status = await service.status(); + expect(status.lifecycle.state).toBe('ready'); + expect(status.sessions).toBe(1); + + await hostOf(service).killWorkerForTest(); + // Inside the backoff window the local report names the crash — no RPC + // needed, so the state is observable while the worker is down. + const down = service.lifecycleReport(); + expect(down.state).toBe('degraded'); + expect(down.detail).toContain('worker'); + + await new Promise((resolve) => setTimeout(resolve, 700)); + await settleSync(service); // respawn after the backoff + expect(service.lifecycleReport().state).toBe('ready'); + expect((await service.search({ query: '苹果' })).items.length).toBe(1); + }); + + it('does not serve a dead worker generation’s cached lifecycle after a respawn', { timeout: 30_000 }, async () => { + const s1 = summary('s1', '缓存失效', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 stale-cache', T1)]); + const service = track(makeService(home!, staticIndex([s1]))); + await settleSync(service); + expect(service.lifecycleReport().state).toBe('ready'); + + await hostOf(service).killWorkerForTest(); + await new Promise((resolve) => setTimeout(resolve, 700)); // out of backoff + + // Drive a respawn, then pin the window between the handshake completing + // and the first RPC response landing: the new worker's core has not even + // opened the db yet, so the snapshot must report 'opening' — never the + // dead worker's cached 'ready'. The assertion runs synchronously right + // after the poll, so the response message cannot have been processed yet. + const host = hostOf(service); + const respawn = syncNow(service); + respawn.catch(() => {}); + await vi.waitFor( + () => { + expect((host as unknown as { worker: unknown }).worker).not.toBeNull(); + }, + { interval: 5, timeout: 10_000 }, + ); + expect(service.lifecycleReport().state).toBe('opening'); + + await respawn; + await settleSync(service); + expect(service.lifecycleReport().state).toBe('ready'); + expect((await service.search({ query: '苹果' })).items.length).toBe(1); + }); + + it('a clean dispose releases the search-index lock and the lifecycle settles at stopped', { timeout: 30_000 }, async () => { + const s1 = summary('s1', '退出顺序', T1); + await writeWire(home!, 's1', 'main', [userLine('苹果 shutdown', T1)]); + const service = track(makeService(home!, staticIndex([s1]))); + await service.reindex(); + await expect(stat(lockPath())).resolves.toBeDefined(); + + const host = hostOf(service); + host.beginClose(); + expect(host.lifecycleSnapshot().state).toBe('closing'); + service.dispose(); + await drainGlobalSearchDisposals(); + // The worker drained, closed the db (releasing the lock) and exited + // before the disposal promise resolved. + await waitForGone(lockPath()); + expect(service.lifecycleReport()).toEqual({ state: 'stopped' }); + + // A fresh instance can take the write lock immediately — no stale-lock + // window after a clean shutdown. + const next = track(makeService(home!, staticIndex([s1]))); + await next.reindex(); + expect((await next.search({ query: '苹果' })).items.length).toBe(1); + }); + + it('a spawn failure never falls back to the inline host and reports degraded', { timeout: 30_000 }, async () => { + const service = track(makeService(home!, staticIndex([]))); + // Swap in a host whose worker cannot spawn (no worker runtime in this + // environment): the search must degrade recognizably, never restore the + // index work onto the main thread. + const failingHost = new SearchWorkerHost({ + dir: join(home!, 'search-index'), + log: noopLog, + workerFactory: () => { + throw new Error('threads unavailable'); + }, + }); + hosts.push(failingHost); + (service as unknown as { backend: SearchBackend }).backend = failingHost; + + const page = await service.search({ query: 'anything' }); + expect(page.items).toEqual([]); + expect(page.source).toBe('index'); + expect(page.indexState.state).toBe('building'); + expect(page.indexState.degraded).toContain('threads unavailable'); + // No inline db materialized: the index directory was never created. + await expect(stat(join(home!, 'search-index'))).rejects.toThrow(); + + expect(service.lifecycleReport().state).toBe('degraded'); + // A second call inside the backoff window fails fast; status() still + // answers (never throws) with the degraded lifecycle. + const status = await service.status(); + expect(status.lifecycle.state).toBe('degraded'); + expect(status.degraded).toContain('worker'); + }); +}); + + // Numbers are logged as JSON for phase-to-phase comparison; only a loose // complexity budget is asserted (no tight absolute millisecond thresholds in // shared CI), so an accidental quadratic regression trips the test anywhere. diff --git a/packages/kap-server/test/sessions.test.ts b/packages/kap-server/test/sessions.test.ts index dd767b1b0..3a3d6ede9 100644 --- a/packages/kap-server/test/sessions.test.ts +++ b/packages/kap-server/test/sessions.test.ts @@ -1354,6 +1354,10 @@ describe('server-v2 /api/v1/sessions (minidb read model)', () => { let home: string | undefined; let base: string; + // The suite-level setup pins the read-model flag OFF (env outranks the + // `[experimental]` config section), so this describe re-enables it per test. + const READ_MODEL_ENV = 'KIMI_CODE_EXPERIMENTAL_PERSISTENCE_MINIDB_READMODEL'; + const READ_MODEL_CONFIG = [ 'default_model = "stub"', '', @@ -1367,12 +1371,10 @@ describe('server-v2 /api/v1/sessions (minidb read model)', () => { 'model = "stub"', 'max_context_size = 1000', '', - '[experimental]', - 'persistence_minidb_readmodel = true', - '', ].join('\n'); beforeEach(async () => { + process.env[READ_MODEL_ENV] = '1'; home = await mkdtemp(join(tmpdir(), 'kimi-server-v2-sessions-rm-')); await writeFile(join(home, 'config.toml'), READ_MODEL_CONFIG, 'utf8'); server = await startServer({ @@ -1387,6 +1389,7 @@ describe('server-v2 /api/v1/sessions (minidb read model)', () => { }); afterEach(async () => { + process.env[READ_MODEL_ENV] = 'false'; if (server !== undefined) { await server.close(); server = undefined; @@ -1465,4 +1468,45 @@ describe('server-v2 /api/v1/sessions (minidb read model)', () => { const relisted = await getJson('/api/v1/sessions?include_archive=true'); expect(relisted.body.data.items.map((s) => s.id)).toEqual([id]); }); + + it('serves session routes from the authoritative store when the read model cannot open', async () => { + // Break the read model at its root: a plain FILE where the query-store + // directory must be. Boot-time prepare fails; every later access retries + // and fails the same way for the whole server lifetime. + await (server as RunningServer).close(); + server = undefined; + await rm(join(home as string, 'cache', 'query-store'), { recursive: true, force: true }); + await writeFile(join(home as string, 'cache', 'query-store'), 'sabotage', 'utf8'); + + // The boot itself must survive the read-model failure (prepare's failure + // is logged, never propagated). + server = await startServer({ + hostIdentity: TEST_HOST_IDENTITY, + host: '127.0.0.1', + port: 0, + homeDir: home, + logLevel: 'silent', + debugEndpoints: true, + }); + base = `http://127.0.0.1:${server.port}`; + + // The degradation is diagnosable through the debug surface. + const status = await getJson<{ state: string; reason?: string; degradedCount: number }>( + '/api/v1/debug/sessionIndex/status', + ); + expect(status.body.data.state).toBe('degraded'); + expect(status.body.data.degradedCount).toBeGreaterThan(0); + + // Session lifecycle is untouched: create, list, point-lookup all answer + // from the authoritative metadata. + const created = await postJson('/api/v1/sessions', { + metadata: { cwd: home as string }, + }); + expect(created.body.code).toBe(0); + const id = created.body.data.id; + const listed = await getJson('/api/v1/sessions'); + expect(listed.body.data.items.some((s) => s.id === id)).toBe(true); + const fetched = await getJson<{ id: string }>(`/api/v1/sessions/${id}`); + expect(fetched.body.data.id).toBe(id); + }); }); diff --git a/packages/kap-server/test/setup.ts b/packages/kap-server/test/setup.ts index b22ac8598..14ce4deab 100644 --- a/packages/kap-server/test/setup.ts +++ b/packages/kap-server/test/setup.ts @@ -1,13 +1,13 @@ /** * Vitest setup — hermetic experimental flags. * - * The kap-server suites pin the default (flag-off) behavior of the engine: - * several scenarios assert wire semantics that an experimental flag - * deliberately changes (e.g. the minidb session read model makes externally - * written sessions eventually consistent). A developer shell exporting + * The kap-server suites pin flag-off engine behavior: several scenarios + * assert wire semantics that an experimental flag deliberately changes + * (e.g. the minidb session read model makes externally written sessions + * eventually consistent). A developer shell exporting * `KIMI_CODE_EXPERIMENTAL_FLAG` (or a single-flag variant) must not flip the - * whole suite — scrub the env here; a test that wants a flag enables it - * explicitly through the boot config. + * whole suite — scrub the env here, then pin the defaults-ON flags OFF + * below; a test that wants a flag re-enables its env var explicitly. */ delete process.env['KIMI_CODE_EXPERIMENTAL_FLAG']; @@ -16,3 +16,22 @@ for (const key of Object.keys(process.env)) { delete process.env[key]; } } + +// Stage-4 note: default the `search_worker` flag OFF for server-booting +// suites. Every startServer would otherwise lazily spawn a real search +// worker thread, and in dev/test the entry loads its TypeScript closure via +// Node's type stripping — CPU-heavy enough that the accumulated background +// load pushed unrelated heavy tests (e.g. prompts image compression) past +// the 5s default timeout under full-suite parallelism. Suites that exercise +// the search surface pin their host explicitly: searchService.test.ts +// injects flag stubs per service, and searchRoute.test.ts re-enables the +// env var for its end-to-end worker coverage. +process.env['KIMI_CODE_EXPERIMENTAL_SEARCH_WORKER'] = 'false'; + +// Stage-6 note: default the `persistence_minidb_readmodel` flag OFF for +// server-booting suites. The flag defaults ON in production, but several +// suites assert wire semantics the read model deliberately changes +// (externally written sessions become eventually consistent). The dedicated +// read-model describe in sessions.test.ts re-enables the env var per test — +// the env source outranks the `[experimental]` config section. +process.env['KIMI_CODE_EXPERIMENTAL_PERSISTENCE_MINIDB_READMODEL'] = 'false'; diff --git a/packages/minidb/AGENTS.md b/packages/minidb/AGENTS.md index 4ea13a732..3a3a19f05 100644 --- a/packages/minidb/AGENTS.md +++ b/packages/minidb/AGENTS.md @@ -1,6 +1,6 @@ # minidb Agent Guide -The embedded JSON document store (`MiniDb`) behind kap-server's search index — snapshot + WAL persistence with an exclusive write lock (losers open read-only and catch up from the WAL), plus a larger-than-RAM full-text layer. +The embedded JSON document store (`MiniDb`) behind kap-server's search index — snapshot + WAL persistence with an exclusive write lock (losers open read-only and catch up from the WAL; `OpenOptions.onLockAcquired` reports the held lock token right after acquisition, before recovery work — supervisors hosting MiniDb in a worker thread need it because worker threads share the host process pid, so the pid in the lock line alone cannot drive stale reclamation), plus a larger-than-RAM full-text layer. ## Text index @@ -12,14 +12,16 @@ Derived state (store image, dt/secondary/compound indexes, text dictionary + pos On the fallback path the corpus-scale text rebuild is no longer awaited inside `open()`: it runs as a `'text-build'` maintenance task on the same bounded engine pinned at the recovery checkpoint (rollback: `OpenOptions.deferOpenTextBuilds: false`), searches on a not-yet-committed index raise `TextIndexBuildingError` (state surfaced via `MiniDb.textIndexBuilding` — the guard is a dedicated `basePending` flag, so staged builds over a live old base keep serving), and a read-only opener builds into a private scratch dir next to the db dir (`.ro-scratch/-*`, dropped on close) and adopts the disk base there instead of aggregating a full in-RAM base. Async base reads are commit-safe via a base-swap epoch (`TextIndex.baseEpoch`): a read straddling a base commit re-reads from the fresh base and never caches a stale list. -A healthy writer also keeps a valid generation around at runtime — the per-write WAL-growth trigger (`MiniDb.maybeAutoGenerationBuild`, 4 MiB staleness rule, throttled with failure backoff) covers the started-from-empty window the open-time kick cannot, and `close()` publishes a missing/stale generation best-effort (`buildGeneration('close')`). Load-time integrity is per-file crc32 + definition hashes — a corrupt or definition-mismatched image rebuilds only the affected index from the loaded store. +A healthy writer also keeps a valid generation around at runtime — the per-write WAL-growth trigger (`MiniDb.maybeAutoGenerationBuild`, 4 MiB staleness rule, throttled with failure backoff) covers the started-from-empty window the open-time kick cannot, and `close()` publishes a missing/stale generation best-effort (`buildGeneration('close')`). Load-time integrity is per-file crc32 + definition hashes — a corrupt or definition-mismatched image rebuilds only the affected index from the loaded store. The whole open path is cooperative (phase 3, main-thread relief): payload verification is chunked with yields (`readGenerationFileCheckedAsync`, `verifyFileIntegrityAsync` — same error semantics as the sync originals), the store/image map construction is sliced (`Store.bulkLoadRefsAsync` — with the active-expire timer paused for the load's duration, so a mid-load reap can never diverge the map from the not-yet-rebuilt ordered index — the async image parsers, the secondary/compound `loadImageAsync`, `TextIndex.attachImageAsync`), and the WAL-delta apply yields on primitive-op + wall-clock budgets (`walApplySlicer` — a batch frame unrolls into thousands of ops, so frame-granular yielding could never bound a slice). + +Open-time lifecycle telemetry — the `no-generation` / `generation-load` / `wal-catch-up` / `full-rebuild` / `ready` / `degraded` state machine plus per-phase wall-clock timings (image loads, postings crc, WAL scan/apply, full recovery, text rebuilds incl. their hosting mode) — lives in the `LifecycleTracker` (`src/lifecycle-status.ts`), fed by `lifecycle.ts` / `generation-loader.ts` / `recovery.ts` and read through `MiniDb.lifecycleStatus()`; `bench/open-lifecycle.ts` (`pnpm bench:open`) is the repeatable event-loop-delay / input-latency baseline over the four open paths (small corpus, large WAL delta, large full-text generation, corrupt generation). ## Maintenance and worker builds Heavy maintenance runs through the unified **maintenance scheduler** (`src/maintenance.ts` — one heavy task per database at a time, queue backpressure, disk free-space preflight, deadline/cancellation, shutdown drain-or-cancel, `MiniDb.maintenanceStatus()` read model; nested submissions from inside a task run inline via AsyncLocalStorage to avoid self-deadlock). -Full-text generation artifacts (tokenization → bounded-memory aggregation → segmented external merge → postings/dictionary/base-docs) are produced **off the main thread** by a worker build (`src/worker/text-build-core.ts`, hosted by `src/worker/text-build.ts` + `src/worker/text-build-worker.ts` — Node-native type-stripping with `execArgv: ['--experimental-transform-types']`, explicit `.ts` import specifiers in the whole worker closure; worker writes only inside the tmp generation dir, the main thread verifies (sanity + streaming crc) and swaps the live base via `TextIndex.commitRebase` after `beginRebase`; `OpenOptions.textBuildWorker: false` is the rollback switch; a missing worker file or slot pressure hosts the SAME bounded core inline on the main thread instead — the in-thread staged aggregation is kept only for small corpora (< 4096 docs), custom function tokenizers, and the explicit rollback). The same bounded engine (worker-or-inline + rebase, `MiniDb.boundedTextBuild`) also backs the two full-corpus initial-build entries — `createTextIndex` and the open-time loader rebuild of a corrupt/definition-mismatched image — so first-time indexing of a large existing store no longer aggregates the whole term->postings map in RAM. +Full-text generation artifacts (tokenization → bounded-memory aggregation → segmented external merge → postings/dictionary/base-docs) are produced **off the main thread** by a worker build (`src/worker/text-build-core.ts`, hosted by `src/worker/text-build.ts` + `src/worker/text-build-worker.ts` — Node-native type-stripping with `execArgv: ['--experimental-transform-types']`, explicit `.ts` import specifiers in the whole worker closure; worker writes only inside the tmp generation dir, the main thread verifies (sanity + streaming crc) and swaps the live base via `TextIndex.commitRebase` after `beginRebase`; `OpenOptions.textBuildWorker: false` is the rollback switch; a missing worker file hosts the SAME bounded core inline on the main thread, while worker-slot pressure first QUEUES for a process-wide slot (`WorkerSlots.acquireBounded`, bounded by `MiniDb.textBuildSlotWaitMs`, default 30 s, abort-aware) and only a persisted drought hosts the inline core as the explicit last resort — the in-thread staged aggregation is kept only for small corpora (< 4096 docs), custom function tokenizers, and the explicit rollback). The same bounded engine (worker-or-inline + rebase, `MiniDb.boundedTextBuild`) also backs the two full-corpus initial-build entries — `createTextIndex` and the open-time loader rebuild of a corrupt/definition-mismatched image — so first-time indexing of a large existing store no longer aggregates the whole term->postings map in RAM. ## Async read surface -The async read surface is additive: `getAsync` / `searchAsync` / `searchBoundedAsync` / `queryAsync` (`ValueReader.readAsync`, `PostingsFile.readAsync`, byte-bounded decoded-postings cache via `TextIndexOptions.cacheBytes`); recovery scans use the async sequential scanner (`scanFrameRefsFdAsync` — windowed reads, sliced CRC, periodic yields, AbortSignal, bounded corruption-resync candidate budget shared with the sync scanner); compaction's disk-mode snapshot groups live refs by (file, offset) and reads them with bounded-concurrency async positioned reads (`src/snapshot.ts`). +The async read surface is additive: `getAsync` / `searchAsync` / `searchBoundedAsync` / `queryAsync` (`ValueReader.readAsync`, `PostingsFile.readAsync`, byte-bounded decoded-postings cache via `TextIndexOptions.cacheBytes`); recovery scans use the async sequential scanner (`scanFrameRefsFdAsync` — windowed reads, sliced CRC, periodic yields, AbortSignal, bounded corruption-resync candidate budget shared with the sync scanner); compaction's disk-mode snapshot groups live refs by (file, offset) and reads them with bounded-concurrency async positioned reads (`src/snapshot.ts`). Read-replica catch-up (`MiniDb.catchUpFromWal`) is cooperative too: the delta scan rides the same async scanner and the apply yields on primitive-op + wall-clock budgets (`walApplySlicer`), with calls serialized per instance so overlapping catch-ups never interleave mid-apply — a replica that fell far behind no longer stalls the host loop in one synchronous scan+apply. diff --git a/packages/minidb/bench/open-lifecycle.ts b/packages/minidb/bench/open-lifecycle.ts new file mode 100644 index 000000000..828d7452f --- /dev/null +++ b/packages/minidb/bench/open-lifecycle.ts @@ -0,0 +1,367 @@ +// bench/open-lifecycle.ts +// +// Phase-1 baseline (plan/01-baseline-and-boundaries.md): what MiniDb's open() +// costs the host process, per lifecycle path. Four scenario families — +// 1. small corpus with a healthy published generation; +// 2. a large WAL delta past the generation checkpoint (wal-catch-up); +// 3. a large full-text generation (store image + text images + postings crc); +// 4. a corrupt generation (the full-recovery fallback, then the deferred +// background text rebuild: the 'degraded' → 'ready' window) +// each measuring the open's wall time, event-loop delay, simulated input +// latency (a setImmediate probe racing the operation — how fast the loop +// services a fresh macrotask, i.e. a keypress handler), and memory peaks, and +// recording the lifecycleStatus() phase breakdown so a generation attach can +// be told apart from a full rebuild by numbers, not by inference. +// +// Run: node --import tsx bench/open-lifecycle.ts (full sizes) +// node --import tsx bench/open-lifecycle.ts --quick (smoke sizes) +// node --import tsx bench/open-lifecycle.ts --json .tmp/open-lifecycle.json +// +// Knobs (env): BENCH_SEED. + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { monitorEventLoopDelay } from 'node:perf_hooks'; +import { MiniDb } from '../src/index.js'; +import { GEN_BUILD_WAL_DELTA_BYTES } from '../src/generation-builder.js'; + +const fmt = (n) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); +const MIB = 1024 * 1024; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-open-bench-')); +} + +// ---- fixed-seed synthetic data (same generator family as bench.ts) ---------- + +function mulberry32(seed) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const LATIN_VOCAB = + 'wal sync snapshot compaction recovery index query cache buffer frame codec store delta merge rotate flush token parse schema server client socket thread worker queue stream ledger journal cursor segment batch commit'.split( + ' ', + ); +const CJK_VOCAB = ['持久化', '快照', '索引', '恢复', '压缩', '查询', '缓存', '日志', '事务', '复制']; + +function makeMessages(count, seed) { + const rng = mulberry32(seed); + const pick = (arr) => arr[(rng() * arr.length) | 0]; + const docs = []; + for (let i = 0; i < count; i++) { + const words = []; + const n = 20 + ((rng() * 15) | 0); + for (let w = 0; w < n; w++) words.push(rng() < 0.15 ? pick(CJK_VOCAB) : pick(LATIN_VOCAB)); + if (i % 97 === 0) words.push('walrus'); + if (i % 131 === 0) words.push('持久化'); + docs.push({ key: `m${i}`, body: words.join(' '), ts: 1_700_000_000_000 + i * 1000 }); + } + return docs; +} + +// ---- measurement machinery (same conventions as bench.ts) ------------------- + +function percentileOf(sorted, p) { + if (sorted.length === 0) return 0; + const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1); + return sorted[Math.max(0, idx)]; +} + +function latencySummary(samples) { + if (!samples || samples.length === 0) return undefined; + const sorted = [...samples].sort((a, b) => a - b); + return { + count: sorted.length, + p50: percentileOf(sorted, 50), + p95: percentileOf(sorted, 95), + p99: percentileOf(sorted, 99), + max: sorted[sorted.length - 1], + }; +} + +class MemSampler { + constructor() { + this.peakRssBytes = 0; + this.peakHeapUsedBytes = 0; + this.timer = null; + } + start() { + const sample = () => { + const mu = process.memoryUsage(); + if (mu.rss > this.peakRssBytes) this.peakRssBytes = mu.rss; + if (mu.heapUsed > this.peakHeapUsedBytes) this.peakHeapUsedBytes = mu.heapUsed; + }; + sample(); + this.timer = setInterval(sample, 25); + this.timer.unref?.(); + } + stop() { + if (this.timer) clearInterval(this.timer); + this.timer = null; + return { peakRssBytes: this.peakRssBytes, peakHeapUsedBytes: this.peakHeapUsedBytes }; + } +} + +const finite = (n) => (Number.isFinite(n) ? Math.round(n * 1000) / 1000 : 0); + +/** Simulated input latency: how long a fresh macrotask (a keypress handler's + * setImmediate) waits while the measured operation holds the event loop. */ +async function probeInputLatency(stop) { + const samples = []; + while (!stop.done) { + const t = performance.now(); + await new Promise((r) => setImmediate(r)); + samples.push(performance.now() - t); + } + return samples; +} + +const results = []; + +/** Measure one operation (an open, or a degraded→ready wait): wall time, + * event-loop delay, input-latency probes, memory peaks. */ +async function measure(name, op, extra) { + if (global.gc) global.gc(); + const eld = monitorEventLoopDelay(); + const mem = new MemSampler(); + const stop = { done: false }; + eld.enable(); + mem.start(); + const probe = probeInputLatency(stop); + const t0 = performance.now(); + const out = (await op()) || {}; + const durationMs = performance.now() - t0; + stop.done = true; + const inputSamples = await probe; + const memPeaks = mem.stop(); + eld.disable(); + const row = { + name, + durationMs: Math.round(durationMs * 1000) / 1000, + eventLoopDelayMs: { + mean: finite(eld.mean / 1e6), + p50: finite(eld.percentile(50) / 1e6), + p95: finite(eld.percentile(95) / 1e6), + p99: finite(eld.percentile(99) / 1e6), + max: finite(eld.max / 1e6), + }, + inputLatencyMs: latencySummary(inputSamples.map(finite)), + peakRssBytes: memPeaks.peakRssBytes, + peakHeapUsedBytes: memPeaks.peakHeapUsedBytes, + extra: { ...extra, ...out.extra }, + }; + results.push(row); + const input = row.inputLatencyMs; + console.log( + ` ${name.padEnd(52)} ${durationMs.toFixed(1).padStart(9)} ms` + + ` [eld p99 ${row.eventLoopDelayMs.p99.toFixed(1)} / max ${row.eventLoopDelayMs.max.toFixed(1)} ms` + + (input ? `, input p99 ${input.p99.toFixed(1)} / max ${input.max.toFixed(1)} ms` : '') + + `, rss ${(row.peakRssBytes / MIB).toFixed(0)} MiB]`, + ); + return row; +} + +/** The lifecycle fields every open row reports (rounded for a stable JSON). */ +function lifecycleExtra(db) { + const status = db.lifecycleStatus(); + const phases = Object.fromEntries(Object.entries(status.phases).map(([k, v]) => [k, finite(v)])); + return { + state: status.state, + path: status.path, + textIndexes: status.textIndexes, + pendingTextIndexes: status.pendingTextIndexes, + phases, + generationLoads: db.stats.generationLoads, + generationLoadFallbacks: db.stats.generationLoadFallbacks, + walDeltaAppliedOps: db.recoveryInfo?.walDeltaAppliedOps ?? 0, + recoveryBytes: db.stats.recoveryBytes, + recoveryFrames: db.stats.recoveryFrames, + indexRebuildDecoded: db.stats.indexRebuildDecoded, + }; +} + +// ---- scenario helpers --------------------------------------------------------- + +/** Populate a corpus with word + ngram text indexes and publish a generation. + * The runtime wal-growth trigger is suppressed so exactly ONE generation + * (the explicit rebuild below) is ever published — the corrupt scenario's + * candidate set stays deterministic at any corpus size. */ +async function populateWithGeneration(dir, docs) { + const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + db.genBuildKickMinIntervalMs = Number.POSITIVE_INFINITY; + const CHUNK = 1000; + for (let base = 0; base < docs.length; base += CHUNK) { + await db.batch(docs.slice(base, base + CHUNK).map((d) => ({ op: 'set', key: d.key, value: d }))); + } + await db.createTextIndex('word', { fields: ['body'] }); + await db.createTextIndex('ngram', { fields: ['body'], tokenizer: 'ngram' }); + await db.rebuildGeneration(); + await db.close(); +} + +const OPEN_OPTS = { valueCodec: 'json', fsyncPolicy: 'everysec', autoCompact: false }; + +/** Scenario 1: a healthy generation attach over a small corpus. */ +async function smallCorpusScenario({ docs }) { + const dir = await tmpDir(); + await populateWithGeneration(dir, docs); + let db; + await measure(`open small corpus, healthy generation, N=${fmt(docs.length)}`, async () => { + db = await MiniDb.open({ dir, ...OPEN_OPTS }); + return { extra: { docs: docs.length, ...lifecycleExtra(db) } }; + }); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +} + +/** Scenario 2: a published generation plus a WAL delta just below the + * close-time republish threshold — the open is generation-load + wal-catch-up. */ +async function largeWalScenario({ docs, deltaBytes }) { + const dir = await tmpDir(); + await populateWithGeneration(dir, docs); + // Append a WAL delta past the checkpoint WITHOUT crossing + // GEN_BUILD_WAL_DELTA_BYTES, so neither the wal-growth trigger nor the + // close-time publish refresh the generation: the next open must replay it. + const target = Math.min(deltaBytes, Math.floor(GEN_BUILD_WAL_DELTA_BYTES * 0.8)); + let deltaOps = 0; + let closePublished = false; + { + const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + const generation = db.getIndexGeneration()?.id ?? null; + const writtenBase = db.stats.walBytesWritten; + let i = 0; + while (db.stats.walBytesWritten - writtenBase < target) { + const ops = []; + for (let k = 0; k < 100; k++) { + ops.push({ op: 'set', key: `d${i * 100 + k}`, value: { body: `delta ${i} ${k} ${'y'.repeat(200)}`, ts: 1_700_100_000_000 + i * 100 + k } }); + } + await db.batch(ops); + i++; + } + deltaOps = i * 100; + await db.close(); + // A close-time republish would invalidate the scenario (nothing left to + // replay): verify the WAL delta survived below the threshold. + closePublished = db.getIndexGeneration()?.id !== generation; + } + let db; + await measure(`open with large WAL delta (${fmt(target)} B), N=${fmt(docs.length)}`, async () => { + db = await MiniDb.open({ dir, ...OPEN_OPTS }); + return { extra: { docs: docs.length, deltaBytes: target, deltaOps, closePublishedGeneration: closePublished, ...lifecycleExtra(db) } }; + }); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +} + +/** Scenario 3: a large full-text generation — store image + text images + + * the whole-file postings crc dominate the open. */ +async function largeGenerationScenario({ docs }) { + const dir = await tmpDir(); + await populateWithGeneration(dir, docs); + let db; + await measure(`open large full-text generation, N=${fmt(docs.length)}`, async () => { + db = await MiniDb.open({ dir, ...OPEN_OPTS }); + return { extra: { docs: docs.length, ...lifecycleExtra(db) } }; + }); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +} + +/** Scenario 4: a corrupt generation — the open falls back to the legacy full + * recovery, returns 'degraded' (the corpus-scale text rebuild is deferred), + * and reaches 'ready' when the background bounded build commits. */ +async function corruptGenerationScenario({ docs, readyTimeoutMs }) { + const dir = await tmpDir(); + await populateWithGeneration(dir, docs); + // Flip one byte inside EVERY generation's store image: the manifest crc + // rejects every candidate and the next open must take the legacy full + // recovery (retention keeps a previous generation around as a fallback + // candidate — corrupting only the newest would measure THAT path instead). + { + const gens = path.join(dir, 'generations'); + for (const entry of await fs.readdir(gens)) { + if (entry.includes('.tmp-')) continue; + const storeImage = path.join(gens, entry, 'store'); + const raw = await fs.readFile(storeImage); + raw[Math.floor(raw.length / 2)] ^= 0xff; + await fs.writeFile(storeImage, raw); + } + } + let db; + await measure(`open with corrupt generation (full-recovery fallback), N=${fmt(docs.length)}`, async () => { + db = await MiniDb.open({ dir, ...OPEN_OPTS }); + return { extra: { docs: docs.length, ...lifecycleExtra(db) } }; + }); + await measure(`deferred text rebuild (degraded -> ready), N=${fmt(docs.length)}`, async () => { + const t0 = Date.now(); + while (db.lifecycleStatus().state !== 'ready') { + if (Date.now() - t0 > readyTimeoutMs) throw new Error(`timed out waiting for ready (state=${db.lifecycleStatus().state})`); + await new Promise((r) => setTimeout(r, 5)); + } + return { + extra: { + docs: docs.length, + ...lifecycleExtra(db), + textDeferredBuilds: db.stats.textDeferredBuilds, + textDeferredBuildErrors: db.stats.textDeferredBuildErrors, + textWorkerBuilds: db.stats.textWorkerBuilds, + textWorkerFallbacks: db.stats.textWorkerFallbacks, + }, + }; + }); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +} + +async function main() { + const argv = process.argv.slice(2); + const jsonIdx = argv.indexOf('--json'); + const jsonPath = jsonIdx !== -1 ? argv[jsonIdx + 1] : process.env.BENCH_JSON; + const quick = argv.includes('--quick') || process.env.BENCH_QUICK === '1'; + + const SEED = Number(process.env.BENCH_SEED || 42); + const sizes = quick + ? { small: 2_000, walCorpus: 5_000, walDeltaBytes: 1.5 * MIB, largeGen: 8_000, corrupt: 8_000, readyTimeoutMs: 120_000 } + : { small: 20_000, walCorpus: 50_000, walDeltaBytes: 3 * MIB, largeGen: 200_000, corrupt: 100_000, readyTimeoutMs: 600_000 }; + + console.log(`\nminidb open-lifecycle baseline (seed=${SEED}${quick ? ', QUICK' : ''}, node ${process.version})\n`); + + await smallCorpusScenario({ docs: makeMessages(sizes.small, SEED) }); + await largeWalScenario({ docs: makeMessages(sizes.walCorpus, SEED), deltaBytes: sizes.walDeltaBytes }); + await largeGenerationScenario({ docs: makeMessages(sizes.largeGen, SEED) }); + await corruptGenerationScenario({ docs: makeMessages(sizes.corrupt, SEED), readyTimeoutMs: sizes.readyTimeoutMs }); + + const report = { + schemaVersion: 1, + tool: 'minidb/bench/open-lifecycle', + quick, + startedAt: new Date().toISOString(), + node: process.version, + platform: process.platform, + arch: process.arch, + seed: SEED, + scenarios: results, + }; + const json = JSON.stringify(report, null, 2); + if (jsonPath) { + await fs.mkdir(path.dirname(path.resolve(jsonPath)), { recursive: true }); + await fs.writeFile(jsonPath, json + '\n', 'utf8'); + console.log(`\nJSON report written to ${jsonPath}`); + } else { + console.log('\n--- bench JSON ---'); + console.log(json); + } + console.log('\ndone.\n'); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/package.json b/packages/minidb/package.json index f29416468..99c63a9eb 100644 --- a/packages/minidb/package.json +++ b/packages/minidb/package.json @@ -49,6 +49,7 @@ "test": "vitest run", "bench": "node --import tsx bench/bench.ts", "bench:cluster": "node --import tsx bench/cluster.ts", + "bench:open": "node --import tsx bench/open-lifecycle.ts", "clean": "rm -rf dist" } } diff --git a/packages/minidb/src/compound-index.ts b/packages/minidb/src/compound-index.ts index 622946a47..58fa08ffb 100644 --- a/packages/minidb/src/compound-index.ts +++ b/packages/minidb/src/compound-index.ts @@ -292,4 +292,33 @@ export class CompoundIndexManager { entry.groups = groups; entry.byPk = byPk; } + + /** Sliced variant of loadImage (the open-time main-thread path): identical + * resulting state, but the group/byPk map construction yields to the + * event loop every `sliceEvery` entries. The state swap itself stays one + * synchronous segment (the containers are detached until then), and the + * store is not published until open() returns, so a mid-load yield is + * never observable. */ + async loadImageAsync(image: CompoundImageIndex, opts: { sliceEvery?: number } = {}): Promise { + const sliceEvery = opts.sliceEvery ?? 32768; + const entry = this.indexes.get(image.name); + if (!entry) throw new Error(`no such compound index: ${image.name}`); + const groups = new Map>(); + const byPk = new Map(); + let n = 0; + for (const g of image.groups) { + const list = await SkipList.bulkLoadAsync( + g.entries.map((e) => ({ key: e.order, val: e.pk })), + { compareKey: entry.cmp, compareVal: cmpString }, + { sliceEvery }, + ); + groups.set(g.group, list); + for (const e of g.entries) { + byPk.set(e.pk, { group: g.group, order: e.order }); + if (++n % sliceEvery === 0) await new Promise((r) => setImmediate(r)); + } + } + entry.groups = groups; + entry.byPk = byPk; + } } diff --git a/packages/minidb/src/gen-codec.ts b/packages/minidb/src/gen-codec.ts index 9db785be4..eac67d3ca 100644 --- a/packages/minidb/src/gen-codec.ts +++ b/packages/minidb/src/gen-codec.ts @@ -392,6 +392,8 @@ export async function readGenerationFileCheckedAsync( return new ByteReader(buf.subarray(8, buf.length - 4)); } +const yieldToLoop = (): Promise => new Promise((r) => setImmediate(r)); + /** Verify a raw file against the manifest's integrity record by streaming * its bytes (bounded memory — used for the postings files, which have no * envelope of their own and are otherwise only verified lazily, per-record, @@ -417,6 +419,36 @@ export function verifyFileIntegritySync(path: string, expected: { bytes: number; } } +/** The read/verify chunk size of verifyFileIntegrityAsync: one crc32 slice per + * chunk, then a yield — a large postings file verifies in bounded ~ms + * slices instead of one synchronous pass. */ +const VERIFY_CHUNK_BYTES = 1 << 20; + +/** Async sliced variant of verifyFileIntegritySync (the open-time main-thread + * path): bounded 1 MiB positioned reads, a crc32 slice and an event-loop + * yield per chunk, so verifying a large postings file never blocks the loop + * for the whole pass. Identical error semantics. */ +export async function verifyFileIntegrityAsync(path: string, expected: { bytes: number; crc32: number }): Promise { + const fh = await fs.open(path, 'r'); + try { + const st = await fh.stat(); + if (st.size !== expected.bytes) throw new GenerationCorruptError('file size does not match manifest record'); + let crc = 0; + const buf = Buffer.allocUnsafe(Math.min(VERIFY_CHUNK_BYTES, Math.max(st.size, 1))); + let pos = 0; + while (pos < st.size) { + const { bytesRead } = await fh.read(buf, 0, Math.min(buf.length, st.size - pos), pos); + if (bytesRead === 0) throw new GenerationCorruptError('file shrank during integrity check'); + crc = crc32(buf.subarray(0, bytesRead), crc); + pos += bytesRead; + await yieldToLoop(); + } + if ((crc >>> 0) !== expected.crc32) throw new GenerationCorruptError('file crc does not match manifest record'); + } finally { + await fh.close().catch(() => {}); + } +} + // ---- store image ------------------------------------------------------------ const STORE_MAGIC = 'MDGS'; @@ -670,6 +702,52 @@ export function readSecondaryIndexImage(r: ByteReader): SecondaryImageIndex[] { return out; } +/** Sliced variant of readSecondaryIndexImage (stage 6): identical result, + * with event-loop yields every `yieldEvery` parsed entries so a large image + * never parses in one synchronous run. */ +export async function readSecondaryIndexImageAsync(r: ByteReader, yieldEvery = 32768): Promise { + const count = r.u32(); + const out: SecondaryImageIndex[] = []; + let n = 0; + const tick = async (): Promise => { + if (++n % yieldEvery === 0) await yieldToLoop(); + }; + for (let i = 0; i < count; i++) { + const name = r.text(); + const field = r.text(); + const typeTag = r.u8(); + const flags = r.u8(); + const type = typeTag === 2 ? 'range' : typeTag === 1 ? 'equality' : null; + if (type === null) throw new GenerationCorruptError(`secondary image: unknown index type ${typeTag}`); + let equality: SecondaryImageIndex['equality'] = null; + let range: SecondaryImageIndex['range'] = null; + if (type === 'equality') { + equality = []; + const valueCount = r.u64(); + for (let v = 0; v < valueCount; v++) { + const scalarKey = r.text(); + const pkCount = r.u64(); + const pks: string[] = []; + for (let p = 0; p < pkCount; p++) { + pks.push(r.key()); + await tick(); + } + equality.push({ scalarKey, pks }); + } + } else { + range = []; + const m = r.u64(); + for (let j = 0; j < m; j++) { + range.push({ value: r.f64(), pk: r.key() }); + await tick(); + } + } + out.push({ name, field, type, unique: (flags & 1) !== 0, sparse: (flags & 2) !== 0, equality, range }); + } + if (!r.done) throw new GenerationCorruptError('secondary index image: trailing bytes'); + return out; +} + // ---- compound index image --------------------------------------------------- const COMPOUND_MAGIC = 'MDCI'; @@ -781,6 +859,41 @@ export function readCompoundIndexImage(r: ByteReader): CompoundImageIndex[] { return out; } +/** Sliced variant of readCompoundIndexImage (stage 6): identical result, with + * event-loop yields every `yieldEvery` parsed entries. */ +export async function readCompoundIndexImageAsync(r: ByteReader, yieldEvery = 32768): Promise { + const count = r.u32(); + const out: CompoundImageIndex[] = []; + let n = 0; + const tick = async (): Promise => { + if (++n % yieldEvery === 0) await yieldToLoop(); + }; + for (let i = 0; i < count; i++) { + const name = r.text(); + const groupBy = r.text(); + const orderBy = r.text(); + const ot = r.u8(); + const orderType = ot === 2 ? 'string' : ot === 1 ? 'number' : null; + if (orderType === null) throw new GenerationCorruptError(`compound image: unknown order type ${ot}`); + const groupCount = r.u64(); + const groups: CompoundImageIndex['groups'] = []; + for (let g = 0; g < groupCount; g++) { + const group = readGroupValue(r); + const m = r.u64(); + const entries: { order: number | string; pk: string }[] = []; + for (let j = 0; j < m; j++) { + const order = orderType === 'string' ? r.text() : r.f64(); + entries.push({ order, pk: r.key() }); + await tick(); + } + groups.push({ group, entries }); + } + out.push({ name, groupBy, orderBy, orderType, groups }); + } + if (!r.done) throw new GenerationCorruptError('compound index image: trailing bytes'); + return out; +} + // ---- text index images ------------------------------------------------------ const TEXT_DICT_MAGIC = 'MDTD'; @@ -923,3 +1036,51 @@ export function readTextDocsImage(r: ByteReader): TextDocsImage { if (!r.done) throw new GenerationCorruptError('text docs image: trailing bytes'); return { keys, docLens, liveCount, removed, delta }; } + +/** Sliced variant of readTextDocsImage (stage 6): identical result, with + * event-loop yields every `yieldEvery` parsed entries so a large doc table + * or delta never parses in one synchronous run. */ +export async function readTextDocsImageAsync(r: ByteReader, yieldEvery = 32768): Promise { + const docCount = r.u64(); + const liveCount = r.u64(); + const removedCount = r.u64(); + const deltaCount = r.u64(); + let n = 0; + const tick = async (): Promise => { + if (++n % yieldEvery === 0) await yieldToLoop(); + }; + const keys: (string | undefined)[] = []; + const docLens: (number | undefined)[] = []; + for (let i = 0; i < docCount; i++) { + const present = r.u8(); + if (present === 1) { + keys.push(r.key()); + docLens.push(r.u32()); + } else if (present === 0) { + keys.push(undefined); + const len = r.u32(); + docLens.push(len === 0 ? undefined : len); + } else { + throw new GenerationCorruptError(`text docs image: unknown presence tag ${present}`); + } + await tick(); + } + const removed: number[] = []; + for (let i = 0; i < removedCount; i++) { + removed.push(r.u32()); + await tick(); + } + const delta: TextDocsImage['delta'] = []; + for (let i = 0; i < deltaCount; i++) { + const term = r.term(); + const m = r.u64(); + const docs: { docID: number; freq: number }[] = []; + for (let j = 0; j < m; j++) { + docs.push({ docID: r.u32(), freq: r.u32() }); + await tick(); + } + delta.push({ term, docs }); + } + if (!r.done) throw new GenerationCorruptError('text docs image: trailing bytes'); + return { keys, docLens, liveCount, removed, delta }; +} diff --git a/packages/minidb/src/generation-builder.ts b/packages/minidb/src/generation-builder.ts index 4511b3f0f..81ab3ba73 100644 --- a/packages/minidb/src/generation-builder.ts +++ b/packages/minidb/src/generation-builder.ts @@ -76,11 +76,12 @@ import { TextRegistry } from './text-registry.js'; import type { TextIndexDef } from './text-registry.js'; import { ValueReader } from './value-reader.js'; import type { RecoveryMode, RecoveryInfo, ValueMode, RecoveredOp } from './recovery.js'; +import type { LifecycleTracker } from './lifecycle-status.js'; import { fsyncDir } from './compaction.js'; -import { defaultWorkerSlots } from './maintenance.js'; +import { defaultWorkerSlots, MaintenanceCancelledError } from './maintenance.js'; import type { MaintenanceContext, MaintenanceScheduler } from './maintenance.js'; import { startWorkerTextBuild, textBuildWorkerAvailable, verifyFileCrcAsync, WorkerTextBuildError } from './worker/text-build.js'; -import type { WorkerTextBuildHandle, TextBuildCheckpoint } from './worker/text-build.js'; +import type { WorkerTextBuildFallbackReason, WorkerTextBuildHandle, TextBuildCheckpoint } from './worker/text-build.js'; import { readBaseDocsImageAsync, BASE_DOCS_MAGIC, BASE_DOCS_VERSION } from './worker/text-build-core.js'; import type { TextBuildCoreResult } from './worker/text-build-core.js'; import { TYPE_SET } from './codec.js'; @@ -186,11 +187,18 @@ export interface GenerationBuilderDeps { * disables the worker path for the rest of the instance. */ disableTextWorker: () => void; textBuildMemoryBytes: () => number; + /** TUI-safe worker-slot policy: how long the worker build queues for a + * process-wide slot before the bounded inline core is allowed as the last + * resort (see MiniDb.textBuildSlotWaitMs). */ + textBuildSlotWaitMs: () => number; dt: DtIndex; indexes: IndexManager; compound: CompoundIndexManager; textRegistry: TextRegistry; stats: GenerationStats; + /** Per-open lifecycle telemetry, shared with the loader facet (the open + * flow's state machine + phase timings; the build path does not feed it). */ + lifecycle: LifecycleTracker; decode: (b: Buffer | undefined) => V | undefined; indexable: (v: unknown) => v is Record; /** Live records (decoded values), for per-index rebuilds. */ @@ -295,9 +303,10 @@ export class GenerationBuilder { * thread verifies the worker's output (sanity + streaming crc) and swaps * the live base in via commitRebase, and the stage-5 atomic publish stays * the safety boundary — the worker only ever writes inside the tmp - * generation directory. Custom-tokenizer indexes, small corpora, worker - * slot pressure, and deployments without the worker file stay on the - * in-thread staged build. */ + * generation directory. Custom-tokenizer indexes and small corpora stay + * on the in-thread staged build; a deployment without the worker file — + * or a worker-slot drought that outlasts the bounded queue wait — hosts + * the SAME bounded core inline instead. */ private async runGenerationBuild(ctx: MaintenanceContext): Promise { const t0 = performance.now(); const gens = generationsDir(this.deps.dir()); @@ -483,11 +492,26 @@ export class GenerationBuilder { } // Host the bounded build in a worker thread when its entry file // exists; otherwise run the SAME bounded core inline on the main - // thread (worker-slot pressure, or a bundled single-file deployment - // without the worker file). Inline is still memory-bounded — the - // unbounded staged aggregation is NOT the fallback here. + // thread (a bundled single-file deployment without the worker file, + // or a persisted slot drought). Inline is still memory-bounded — the + // unbounded staged aggregation is NOT the fallback here. TUI-safe + // slot policy: queue for a process-wide slot first (bounded by + // textBuildSlotWaitMs and the build's abort signal) instead of + // dropping the build onto the main thread the moment every slot is + // busy. const workerAvailable = textBuildWorkerAvailable(); - workerSlotRelease = workerAvailable ? defaultWorkerSlots.tryAcquire() : null; + let inlineReason: WorkerTextBuildFallbackReason | undefined; + if (workerAvailable) { + try { + workerSlotRelease = await defaultWorkerSlots.acquireBounded(this.deps.textBuildSlotWaitMs(), aborter.signal); + } catch (e) { + if (e instanceof MaintenanceCancelledError) throw new GenerationBuildAborted('worker slot wait cancelled'); + throw e; + } + if (workerSlotRelease === null) inlineReason = 'slot-pressure'; + } else { + inlineReason = 'runtime-unavailable'; + } const inline = workerSlotRelease === null; workerHandle = startWorkerTextBuild( { @@ -512,7 +536,7 @@ export class GenerationBuilder { signal: aborter.signal, shouldAbort: () => gb.aborted || this.deps.wal() !== gb.wal || this.deps.state() !== 'open', inline, - inlineReason: workerAvailable ? 'slot-pressure' : 'runtime-unavailable', + inlineReason, onFallback: (reason) => { this.deps.stats.textWorkerFallbacks++; this.deps.stats.lastTextWorkerFallback = reason; diff --git a/packages/minidb/src/generation-loader.ts b/packages/minidb/src/generation-loader.ts index eb8345a8d..12a049c70 100644 --- a/packages/minidb/src/generation-loader.ts +++ b/packages/minidb/src/generation-loader.ts @@ -28,26 +28,28 @@ import { generationDir, listGenerations, readCurrent, readManifest } from './gen import { GenerationCorruptError, STORE_VERSION, - readGenerationFileChecked, + readGenerationFileCheckedAsync, readStoreImage, readDtIndexImage, - readSecondaryIndexImage, - readCompoundIndexImage, - readTextDictionaryImage, - readTextDocsImage, - verifyFileIntegritySync, + readSecondaryIndexImageAsync, + readCompoundIndexImageAsync, + readTextDictionaryImageAsync, + readTextDocsImageAsync, + verifyFileIntegrityAsync, } from './gen-codec.js'; import type { StoreImageRecord } from './gen-codec.js'; import { IndexManager } from './index-manager.js'; import type { IndexInfo } from './index-manager.js'; import { CompoundIndexManager } from './compound-index.js'; import type { CompoundIndexInfo } from './compound-index.js'; +import type { LifecycleTracker } from './lifecycle-status.js'; import { TextRegistry } from './text-registry.js'; import type { TextIndexDef } from './text-registry.js'; import type { TextIndex } from './text-index/index.js'; +import { yieldToLoop } from './text-index/tokenize.js'; import type { TextBuildCheckpoint } from './worker/text-build.js'; import { ValueReader } from './value-reader.js'; -import { frameToOps } from './recovery.js'; +import { frameToOps, walApplySlicer } from './recovery.js'; import type { RecoveryMode, RecoveryInfo, ValueMode, RecoveredOp } from './recovery.js'; import { scanFrameRefsFdAsync } from './codec.js'; import { toKStr } from './value-codec.js'; @@ -57,6 +59,10 @@ import type { DtIndex } from './dt-index.js'; import type { ValueCodecName } from './types.js'; import type { GenerationStats } from './generation-builder.js'; +/** Store-image parse/filter records per event-loop slice on the open path + * (the bulk load's own slicing lives in Store.bulkLoadRefsAsync). */ +const STORE_IMAGE_RECORDS_PER_SLICE = 8192; + /** The owner-injected surface the load path needs (a structural subset of * GenerationBuilderDeps — the builder passes its own deps through). */ export interface GenerationLoaderDeps { @@ -75,6 +81,8 @@ export interface GenerationLoaderDeps { compound: CompoundIndexManager; textRegistry: TextRegistry; stats: GenerationStats; + /** Per-open lifecycle telemetry (state machine + phase timings). */ + lifecycle: LifecycleTracker; indexable: (v: unknown) => v is Record; /** Live records (decoded values), for per-index rebuilds. */ liveRecords: () => Generator<{ key: Buffer; value: V | undefined; dt: Record | null }>; @@ -163,13 +171,18 @@ export class GenerationLoader { // Store image. Records expire-past at load time are dropped here AND // noted, so their loaded index entries can be reconciled below (the // image legitimately contains records whose TTL elapsed after the build). + // The payload verify, the parse/filter walk, and the bulk load are all + // sliced with event-loop yields (safe mid-load: nothing is published + // until open() returns). + const tStore = performance.now(); const storeInfo = manifest.files[STORE_IMAGE_FILE]; if (!storeInfo) throw new GenerationCorruptError('store image missing from manifest'); - const storePayload = await readGenerationFileChecked(path.join(genDir, STORE_IMAGE_FILE), 'MDGS', STORE_VERSION, storeInfo); + const storePayload = await readGenerationFileCheckedAsync(path.join(genDir, STORE_IMAGE_FILE), 'MDGS', STORE_VERSION, storeInfo); const now = Date.now(); const droppedExpired: string[] = []; const records: StoreImageRecord[] = []; let imageCount = 0; + let parsed = 0; for (const rec of readStoreImage(storePayload)) { imageCount++; if (rec.expireAt && rec.expireAt <= now) { @@ -180,17 +193,21 @@ export class GenerationLoader { throw new GenerationCorruptError('store image carries disk refs for a memory-mode open'); } records.push(rec); + if (++parsed % STORE_IMAGE_RECORDS_PER_SLICE === 0) await yieldToLoop(); } - this.deps.store().bulkLoadRefs(records); + await this.deps.store().bulkLoadRefsAsync(records); if (manifest.counts && typeof manifest.counts.records === 'number' && manifest.counts.records !== imageCount) { throw new GenerationCorruptError(`store image record count mismatch (${imageCount} != ${manifest.counts.records})`); } + this.deps.lifecycle.time('storeImageLoadMs', performance.now() - tStore); // Derived-index images. Every failure here is LOCAL: a corrupt or missing // image rebuilds exactly the affected index(es) from the loaded store. + const tNonText = performance.now(); await this.loadDtImage(genDir, manifest); await this.loadSecondaryImages(genDir, manifest); await this.loadCompoundImages(genDir, manifest); + this.deps.lifecycle.time('nonTextImageLoadMs', performance.now() - tNonText); await this.loadTextImages(genDir, manifest); // Reconcile the expired-at-load drops out of the loaded index states. @@ -204,6 +221,7 @@ export class GenerationLoader { // Replay the WAL delta past the checkpoint with the exact same per-frame // interpretation the legacy recovery uses (frameToOps), maintaining every // derived index incrementally (applyRecoveredOp). + this.deps.lifecycle.transition('wal-catch-up'); const replay = await this.replayWalDelta(cp.walOffset, mode); // A rotation racing the load invalidates the coordinate system the // recoveryInfo below is anchored to (and, in disk mode, the value reader @@ -258,32 +276,38 @@ export class GenerationLoader { * candidate loaded (the caller runs the legacy full recovery). */ async tryLoadGeneration(mode: RecoveryMode): Promise { const t0 = performance.now(); - const candidates: string[] = []; + const lifecycle = this.deps.lifecycle; try { - const current = await readCurrent(this.deps.dir()); - if (current) candidates.push(current); - for (const g of await listGenerations(this.deps.dir())) { - if (!g.tmp && g.id !== current && candidates.length < 3) candidates.push(g.id); - } - } catch (e) { - this.deps.stats.generationLoadFallbacks++; - this.deps.stats.lastGenerationFallback = `list: ${(e as Error).message}`; - return false; - } - for (const id of candidates) { + const candidates: string[] = []; try { - await this.loadOneGeneration(id, mode); - this.deps.stats.generationLoads++; - this.deps.stats.generationLoadDurationMs += performance.now() - t0; - return true; + const current = await readCurrent(this.deps.dir()); + if (current) candidates.push(current); + for (const g of await listGenerations(this.deps.dir())) { + if (!g.tmp && g.id !== current && candidates.length < 3) candidates.push(g.id); + } } catch (e) { - if (!(e instanceof GenerationCorruptError) && (e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; this.deps.stats.generationLoadFallbacks++; - this.deps.stats.lastGenerationFallback = `${id}: ${(e as Error).message}`; - this.resetAfterFailedGenerationLoad(); + this.deps.stats.lastGenerationFallback = `list: ${(e as Error).message}`; + return false; } + for (const id of candidates) { + try { + lifecycle.transition('generation-load'); + await this.loadOneGeneration(id, mode); + this.deps.stats.generationLoads++; + this.deps.stats.generationLoadDurationMs += performance.now() - t0; + return true; + } catch (e) { + if (!(e instanceof GenerationCorruptError) && (e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + this.deps.stats.generationLoadFallbacks++; + this.deps.stats.lastGenerationFallback = `${id}: ${(e as Error).message}`; + this.resetAfterFailedGenerationLoad(); + } + } + return false; + } finally { + lifecycle.time('generationCandidateLoadMs', performance.now() - t0); } - return false; } /** Load the dt image; rebuild the (cheap, metadata-only) dt index from the @@ -292,7 +316,7 @@ export class GenerationLoader { const info = manifest.files[DT_INDEX_FILE]; if (info) { try { - const payload = await readGenerationFileChecked(path.join(genDir, DT_INDEX_FILE), 'MDGD', 1, info); + const payload = await readGenerationFileCheckedAsync(path.join(genDir, DT_INDEX_FILE), 'MDGD', 1, info); this.deps.dt.loadImage(readDtIndexImage(payload)); return; } catch (e) { @@ -310,16 +334,17 @@ export class GenerationLoader { /** Load secondary-index images for definitions whose hash still matches; * rebuild exactly the affected indexes otherwise (plan: only the affected - * index is rebuilt, never the whole registry). */ + * index is rebuilt, never the whole registry). The payload verify, the + * parse, and each image's map construction are event-loop sliced. */ private async loadSecondaryImages(genDir: string, manifest: GenerationManifest): Promise { const live = this.deps.indexes.list(); if (live.length === 0) return; - let images: Map[number]> | null = null; + let images: Map>[number]> | null = null; const info = manifest.files[SECONDARY_INDEX_FILE]; if (info) { try { - const payload = await readGenerationFileChecked(path.join(genDir, SECONDARY_INDEX_FILE), 'MDSI', 1, info); - images = new Map(readSecondaryIndexImage(payload).map((i) => [i.name, i])); + const payload = await readGenerationFileCheckedAsync(path.join(genDir, SECONDARY_INDEX_FILE), 'MDSI', 1, info); + images = new Map((await readSecondaryIndexImageAsync(payload)).map((i) => [i.name, i])); } catch (e) { if (!(e instanceof GenerationCorruptError)) throw e; } @@ -328,7 +353,7 @@ export class GenerationLoader { const image = images?.get(def.name); if (image && manifest.indexDefs.secondary[def.name] === indexDefHash(def)) { try { - this.deps.indexes.loadImage(image); + await this.deps.indexes.loadImageAsync(image); continue; } catch { /* shape mismatch: rebuild below */ @@ -348,16 +373,17 @@ export class GenerationLoader { this.deps.indexes.indexes.set(def.name, fresh.indexes.get(def.name)!); } - /** Load compound-index images (same per-index discipline as secondary). */ + /** Load compound-index images (same per-index discipline — and the same + * event-loop slicing — as secondary). */ private async loadCompoundImages(genDir: string, manifest: GenerationManifest): Promise { const live = this.deps.compound.list(); if (live.length === 0) return; - let images: Map[number]> | null = null; + let images: Map>[number]> | null = null; const info = manifest.files[COMPOUND_INDEX_FILE]; if (info) { try { - const payload = await readGenerationFileChecked(path.join(genDir, COMPOUND_INDEX_FILE), 'MDCI', 1, info); - images = new Map(readCompoundIndexImage(payload).map((i) => [i.name, i])); + const payload = await readGenerationFileCheckedAsync(path.join(genDir, COMPOUND_INDEX_FILE), 'MDCI', 1, info); + images = new Map((await readCompoundIndexImageAsync(payload)).map((i) => [i.name, i])); } catch (e) { if (!(e instanceof GenerationCorruptError)) throw e; } @@ -366,7 +392,7 @@ export class GenerationLoader { const image = images?.get(def.name); if (image && manifest.indexDefs.compound[def.name] === indexDefHash(def)) { try { - this.deps.compound.loadImage(image); + await this.deps.compound.loadImageAsync(image); continue; } catch { /* shape mismatch: rebuild below */ @@ -401,33 +427,29 @@ export class GenerationLoader { let attached = false; if (dictInfo && docsInfo && postingsInfo && manifest.indexDefs.text[def.name] === indexDefHash(TextRegistry.canonicalTextDef(def))) { try { - const dictPayload = await readGenerationFileChecked(path.join(genDir, textDictionaryFile(def.name)), 'MDTD', 1, dictInfo); - const docsPayload = await readGenerationFileChecked(path.join(genDir, textDocsFile(def.name)), 'MDTC', 1, docsInfo); + const tImage = performance.now(); + const dictPayload = await readGenerationFileCheckedAsync(path.join(genDir, textDictionaryFile(def.name)), 'MDTD', 1, dictInfo); + const docsPayload = await readGenerationFileCheckedAsync(path.join(genDir, textDocsFile(def.name)), 'MDTC', 1, docsInfo); // The postings file carries the base every search reads: verify it // wholesale against the manifest NOW (one streaming crc pass), so a // corrupt base is rebuilt at open instead of failing a query later - // (its per-record CRCs would only trip on the first read). + // (its per-record CRCs would only trip on the first read). The + // verify is chunked with event-loop yields — a large postings file + // no longer pins the main thread for the whole pass. const postingsPath = path.join(genDir, textPostingsFile(def.name)); - verifyFileIntegritySync(postingsPath, postingsInfo); - const dict = new Map(readTextDictionaryImage(dictPayload).map((e) => [e.term, { off: e.off, len: e.len, df: e.df }])); - const docs = readTextDocsImage(docsPayload); - const docLens = new Map(); - for (let i = 0; i < docs.docLens.length; i++) { - const len = docs.docLens[i]; - if (len !== undefined) docLens.set(i, len); - } - ti.attachImage({ - postingsPath, - dict, - keys: docs.keys, - docLens, - liveCount: docs.liveCount, - removed: new Set(docs.removed), - delta: new Map(docs.delta.map((d) => [d.term, new Map(d.docs.map((x) => [x.docID, x.freq] as [number, number]))])), - }); + const tCrc = performance.now(); + await verifyFileIntegrityAsync(postingsPath, postingsInfo); + this.deps.lifecycle.time('postingsIntegrityCheckMs', performance.now() - tCrc); + // The map constructions (dictionary, doc table, delta) are built + // inside the sliced attach — no intermediate wholesale Maps here. + const dictEntries = await readTextDictionaryImageAsync(dictPayload); + const docs = await readTextDocsImageAsync(docsPayload); + await ti.attachImageAsync({ postingsPath, dictEntries, docs }); // Carry the integrity record forward: a later CLEAN fast-path build // re-publishes this unchanged file without re-reading it. ti.postingsFileInfo = { bytes: postingsInfo.bytes, crc32: postingsInfo.crc32 }; + this.deps.lifecycle.time('textImageLoadMs', performance.now() - tImage); + this.deps.lifecycle.noteTextIndexSource(def.name, 'image'); attached = true; } catch (e) { if (!(e instanceof GenerationCorruptError) && (e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; @@ -441,6 +463,7 @@ export class GenerationLoader { // The bounded build is discardable: on ANY failure the staged // rebuild below is the fallback (it reads the already-loaded store, // so opens always complete). + const tRebuild = performance.now(); let hosted: 'worker' | 'inline' | null = null; try { hosted = await this.deps.boundedTextBuild(def.name, ti, def, manifest.checkpoint); @@ -448,6 +471,8 @@ export class GenerationLoader { /* fall through to the staged rebuild */ } if (hosted === null) await ti.build(this.deps.textRecords()); + this.deps.lifecycle.time('textRebuildMs', performance.now() - tRebuild); + this.deps.lifecycle.noteTextIndexSource(def.name, hosted ?? 'staged'); } } } @@ -469,15 +494,25 @@ export class GenerationLoader { const fd = fsSync.openSync(this.deps.walPath(), 'r'); try { const st = fsSync.fstatSync(fd); + const tScan = performance.now(); const r = await scanFrameRefsFdAsync(fd, { onCorrupt: mode, startOffset }); + this.deps.lifecycle.time('walScanMs', performance.now() - tScan); + const tApply = performance.now(); let corruptBatches = 0; let appliedOps = 0; + // Cooperative slicing by PRIMITIVE op count and elapsed time (a batch + // frame can unroll into thousands of ops, so frame-granular yielding + // could not bound an apply slice — the open-time freeze this removes). + // Safe mid-apply: the store is not published until open() returns. + const slice = walApplySlicer(); for (const f of r.frames) { for (const op of frameToOps(f, 'wal', fd, this.deps.valueMode(), () => corruptBatches++)) { this.deps.applyRecoveredOp(op); appliedOps++; + if (slice()) await yieldToLoop(); } } + this.deps.lifecycle.time('walApplyMs', performance.now() - tApply); let truncatedWal = false; const last = r.corruptRanges[r.corruptRanges.length - 1]; if (last && last[1] === st.size && !this.deps.readOnly()) { diff --git a/packages/minidb/src/index-manager.ts b/packages/minidb/src/index-manager.ts index 3f179b9fb..0ab0722f6 100644 --- a/packages/minidb/src/index-manager.ts +++ b/packages/minidb/src/index-manager.ts @@ -542,4 +542,54 @@ export class IndexManager { throw new Error(`index "${image.name}" image payload missing`); } } + + /** Sliced variant of loadImage (the open-time main-thread path): identical + * resulting state, but the forward/reverse map construction yields to the + * event loop every `sliceEvery` entries. The state swap itself (assigning + * the freshly built containers) stays one synchronous segment, so a + * mid-load yield can never expose a half-built index. Safe to yield while + * building: the containers are detached until the swap, and the store is + * not published until open() returns. */ + async loadImageAsync(image: SecondaryImageIndex, opts: { sliceEvery?: number } = {}): Promise { + const sliceEvery = opts.sliceEvery ?? 32768; + const idx = this.indexes.get(image.name); + if (!idx) throw new Error(`no such index: ${image.name}`); + if (idx.type !== image.type) throw new Error(`index "${image.name}" image type mismatch`); + let n = 0; + const tick = async (): Promise => { + if (++n % sliceEvery === 0) await new Promise((r) => setImmediate(r)); + }; + if (idx.type === 'range' && image.range) { + const list = await SkipList.bulkLoadAsync( + image.range.map((e) => ({ key: e.value, val: e.pk })), + { compareKey: cmpNumber, compareVal: cmpString }, + { sliceEvery }, + ); + const byPk = new Map(); + for (const e of image.range) { + const arr = byPk.get(e.pk); + if (arr) arr.push(e.value); + else byPk.set(e.pk, [e.value]); + await tick(); + } + idx.list = list; + idx.byPk = byPk; + } else if (idx.type === 'equality' && image.equality) { + const map = new Map>(); + const byPk = new Map(); + for (const v of image.equality) { + map.set(v.scalarKey, new Set(v.pks)); + for (const pk of v.pks) { + const arr = byPk.get(pk); + if (arr) arr.push(v.scalarKey); + else byPk.set(pk, [v.scalarKey]); + await tick(); + } + } + idx.map = map; + idx.byPk = byPk; + } else { + throw new Error(`index "${image.name}" image payload missing`); + } + } } diff --git a/packages/minidb/src/index.ts b/packages/minidb/src/index.ts index ace5cbfae..2415de9ce 100644 --- a/packages/minidb/src/index.ts +++ b/packages/minidb/src/index.ts @@ -20,7 +20,13 @@ export { OpTracker } from './op-tracker.js'; export { TextIndexBuildingError } from './text-index/index.js'; export { normalizeLiteral, createNgramTokenizer } from './trigram.js'; export { tokenize } from './text-index/index.js'; -export type { RecoveryInfo } from './recovery.js'; +export type { RecoveryInfo, RecoveryPhaseTimings } from './recovery.js'; +export type { + MiniDbLifecycleState, + MiniDbLifecycleStatus, + OpenPhaseTimings, + OpenTextIndexSource, +} from './lifecycle-status.js'; export type { IndexDef, IndexInfo, IndexType } from './index-manager.js'; export type { CompoundIndexDef, CompoundIndexInfo } from './compound-index.js'; export type { TextIndexTokenizerName } from './trigram.js'; diff --git a/packages/minidb/src/lifecycle-status.ts b/packages/minidb/src/lifecycle-status.ts new file mode 100644 index 000000000..51c95877b --- /dev/null +++ b/packages/minidb/src/lifecycle-status.ts @@ -0,0 +1,179 @@ +// src/lifecycle-status.ts +// +// Structured per-open lifecycle telemetry: the state machine one open() walks +// through and the wall-clock cost of every lifecycle phase, so operators and +// benches can tell a published-generation attach apart from a legacy full +// recovery WITHOUT inferring it from the cumulative stats counters. One +// tracker instance lives exactly one open (a MiniDb instance opens once); +// lifecycle.ts, generation-loader.ts, recovery.ts (through a sink) and the +// deferred text-build task feed it, and the public read model is +// MiniDb.lifecycleStatus(). + +/** The lifecycle states one open() walks through. */ +export type MiniDbLifecycleState = + /** Nothing has been loaded: the initial state, and the state of an open + * that found no published generation to even attempt (a fresh/legacy + * database, or generations disabled). */ + | 'no-generation' + /** Loading a published generation's images (store + derived indexes). */ + | 'generation-load' + /** Replaying the WAL delta past the loaded generation's checkpoint. */ + | 'wal-catch-up' + /** The legacy full recovery + open-time index rebuild is running (no usable + * generation, or every candidate failed validation). */ + | 'full-rebuild' + /** open() completed and every index is servable. */ + | 'ready' + /** open() completed but at least one text index's base is still being built + * by the deferred background task — searches on it raise + * TextIndexBuildingError until the build commits. */ + | 'degraded'; + +/** Wall-clock milliseconds per lifecycle phase of the LAST open (0 for a + * phase the open never entered). Phases are NOT strictly nested: + * generationCandidateLoadMs contains the image-load / postings-check / + * wal-catch-up phases of the generation path, fullRecoveryMs contains the + * full-recovery path's walScan/walApply (and its staged text commit also + * lands in textRebuildMs), and a deferred text build's textRebuildMs is + * recorded after open() returned, while the instance sits in 'degraded'. */ +export interface OpenPhaseTimings { + /** The whole open() call. */ + openMs: number; + /** tryLoadGeneration across every candidate (manifest validation, image + * loads, WAL-delta replay, failed-candidate resets). */ + generationCandidateLoadMs: number; + /** Store image read + decode + bulk load (generation path). */ + storeImageLoadMs: number; + /** dt / secondary / compound image loads, including the per-index rebuild + * of a corrupt or definition-mismatched image (generation path). */ + nonTextImageLoadMs: number; + /** Text dictionary/docs image reads + base attach (generation path). */ + textImageLoadMs: number; + /** Whole-file postings crc32 verification (generation path). */ + postingsIntegrityCheckMs: number; + /** Frame scanning: the generation path's WAL-delta scan, or the full + * recovery's snapshot + WAL scans (the async scanner's time only). */ + walScanMs: number; + /** Applying scanned frames to the store and the derived indexes. */ + walApplyMs: number; + /** The legacy full recovery + open-time index rebuild block (recover() + + * rebuildAllIndexes()); 0 when a generation served the open. */ + fullRecoveryMs: number; + /** Corpus tokenization work: per-index rebuilds during a generation load + * (bounded worker/inline or staged), the full-rebuild walk's staged text + * commit, and — after open() returned, under 'degraded' — the deferred + * background base builds. 0 on a healthy generation attach. */ + textRebuildMs: number; +} + +/** How one text index's base was served by the last open. */ +export type OpenTextIndexSource = + /** Attached from the generation's images — no tokenization at all. */ + | 'image' + /** Rebuilt by the bounded engine hosted in a worker thread. */ + | 'worker' + /** Rebuilt by the bounded engine hosted inline on this thread. */ + | 'inline' + /** Rebuilt by the staged in-thread build (the unbounded small-corpus path). */ + | 'staged' + /** Still building in the deferred background task when open() returned. */ + | 'deferred'; + +/** The public read model (MiniDb.lifecycleStatus()). */ +export interface MiniDbLifecycleStatus { + state: MiniDbLifecycleState; + /** The states the last open walked through, in order (deduped transitions; + * always starts at 'no-generation' — nothing is loaded before the first + * attempt). */ + path: MiniDbLifecycleState[]; + /** Epoch ms when open() completed; null while an open is still in flight + * (or after a failed open — a failed open never reaches ready/degraded). */ + openedAt: number | null; + phases: OpenPhaseTimings; + /** Per text index: how its base was served (absent = the index did not + * exist at open). A 'deferred' entry flips to its hosting mode when the + * background build commits. */ + textIndexes: Record; + /** Text index names whose base build was still pending when open() + * returned — the reason for 'degraded'. */ + pendingTextIndexes: string[]; +} + +/** The mutable tracker behind MiniDb.lifecycleStatus(); driven by the + * lifecycle facets through the LifecycleHost / GenerationLoaderDeps views + * (same shared-by-reference discipline as the stats object). */ +export class LifecycleTracker { + private current: MiniDbLifecycleState = 'no-generation'; + private readonly transitions: MiniDbLifecycleState[] = ['no-generation']; + private completedAt: number | null = null; + private readonly pending = new Set(); + private readonly textSources = new Map(); + readonly phases: OpenPhaseTimings = { + openMs: 0, + generationCandidateLoadMs: 0, + storeImageLoadMs: 0, + nonTextImageLoadMs: 0, + textImageLoadMs: 0, + postingsIntegrityCheckMs: 0, + walScanMs: 0, + walApplyMs: 0, + fullRecoveryMs: 0, + textRebuildMs: 0, + }; + + get state(): MiniDbLifecycleState { + return this.current; + } + + transition(next: MiniDbLifecycleState): void { + if (next === this.current) return; + this.current = next; + this.transitions.push(next); + } + + /** Add `ms` to one phase (phases accumulate across candidates/passes). */ + time(phase: keyof OpenPhaseTimings, ms: number): void { + this.phases[phase] += ms; + } + + noteTextIndexSource(name: string, source: OpenTextIndexSource): void { + this.textSources.set(name, source); + } + + /** A text index's base build was deferred to the background task: pending + * from here until the build commits (finishOpen maps this to 'degraded'). */ + markTextIndexPending(name: string): void { + this.pending.add(name); + this.textSources.set(name, 'deferred'); + } + + /** The deferred build for one index committed (source = its hosting mode); + * the last pending clear flips a completed open back to 'ready'. A finally + * failed build stays pending — the index keeps raising + * TextIndexBuildingError, which IS the degraded state. */ + clearTextIndexPending(name: string, source: Exclude): void { + this.pending.delete(name); + this.textSources.set(name, source); + if (this.pending.size === 0 && this.completedAt !== null && this.current === 'degraded') { + this.transition('ready'); + } + } + + /** open() is about to return: servable ('ready') unless a deferred text + * build is still pending ('degraded'). */ + finishOpen(): void { + this.completedAt = Date.now(); + this.transition(this.pending.size > 0 ? 'degraded' : 'ready'); + } + + snapshot(): MiniDbLifecycleStatus { + return { + state: this.current, + path: [...this.transitions], + openedAt: this.completedAt, + phases: { ...this.phases }, + textIndexes: Object.fromEntries(this.textSources), + pendingTextIndexes: [...this.pending], + }; + } +} diff --git a/packages/minidb/src/lifecycle.ts b/packages/minidb/src/lifecycle.ts index 78cd1e127..f1019d153 100644 --- a/packages/minidb/src/lifecycle.ts +++ b/packages/minidb/src/lifecycle.ts @@ -25,6 +25,7 @@ import { import { readManifest, sweepGenerationTemps } from './generation-files.js'; import { MaintenanceScheduler } from './maintenance.js'; import { LockFile, LockError } from './lockfile.js'; +import type { LifecycleTracker } from './lifecycle-status.js'; import { ValueReader } from './value-reader.js'; import { Store } from './store.js'; import type { StoreRecord } from './store.js'; @@ -92,6 +93,9 @@ export interface LifecycleHost { store: Store; wal: WAL; valueReader?: ValueReader; + /** Per-open lifecycle telemetry (state machine + phase timings), driven by + * this flow and the generation loader; read via MiniDb.lifecycleStatus(). */ + lifecycle: LifecycleTracker; recoveryInfo: RecoveryInfo | null; stats: WalStats & CompactionTarget['stats'] & { @@ -112,6 +116,7 @@ export interface LifecycleHost { * maintenance. On success the host instance is fully open; on failure every * acquired resource is released before the error rethrows. */ export async function openMiniDb(db: LifecycleHost, opts: OpenOptions, hooks: LifecycleHooks): Promise { + const openT0 = performance.now(); if (!opts || !opts.dir) throw new TypeError('MiniDb.open: opts.dir is required'); db.dir = opts.dir; db.walPath = path.join(db.dir, WAL_FILE); @@ -190,6 +195,20 @@ export async function openMiniDb(db: LifecycleHost, opts: OpenOptions, hoo } else { throw new LockError(`database is locked by another process: ${db.dir}`); } + } else { + // Report the held token BEFORE the heavy recovery work below: a + // supervisor (another thread orchestrating this open) learns the lock + // identity immediately and can reap the lock if this opener dies + // mid-recovery. The callback is purely observational — a throwing + // supervisor must not fail this open with the lock already held. + const heldToken = db.lock.heldToken; + if (heldToken !== undefined) { + try { + opts.onLockAcquired?.({ token: heldToken }); + } catch { + // Intentionally swallowed — see above. + } + } } } @@ -254,13 +273,19 @@ export async function openMiniDb(db: LifecycleHost, opts: OpenOptions, hoo if (db.indexGenerationsEnabled) generationLoaded = await hooks.tryLoadGeneration(opts.recovery ?? 'resync'); if (!generationLoaded) { + // The loader already recorded a 'generation-load' attempt when it tried + // candidates; none at all leaves the state at 'no-generation'. Either + // way the legacy full recovery now runs. + db.lifecycle.transition('full-rebuild'); const recT0 = performance.now(); + const scanApply = { walScanMs: 0, walApplyMs: 0 }; db.recoveryInfo = await recover({ dir: db.dir, store: db.store, mode: opts.recovery ?? 'resync', truncate: !db.readOnly, valueMode: db.valueMode, + timings: scanApply, // Disk-backed values need the positioned reader attached to the SAME // inodes recovery scanned; recovery's generation pairing re-verifies // the attach and retries the whole pass when a rotation landed in @@ -296,6 +321,8 @@ export async function openMiniDb(db: LifecycleHost, opts: OpenOptions, hoo : undefined, }); db.stats.recoveryDurationMs += performance.now() - recT0; + db.lifecycle.time('walScanMs', scanApply.walScanMs); + db.lifecycle.time('walApplyMs', scanApply.walApplyMs); db.stats.recoveryBytes += db.recoveryInfo.snapshotBytes + db.recoveryInfo.walBytes; db.stats.recoveryFrames += db.recoveryInfo.snapshotFrames + db.recoveryInfo.walFrames; // Recovery may have truncated a torn WAL tail behind the WAL's back; @@ -304,6 +331,7 @@ export async function openMiniDb(db: LifecycleHost, opts: OpenOptions, hoo if (db.recoveryInfo.truncatedWal) await db.wal.refreshSize(); hooks.seedAccessFromStore(); await hooks.rebuildAllIndexes(); + db.lifecycle.time('fullRecoveryMs', performance.now() - recT0); } // A read-only instance never compacts: rotation would rename the live @@ -331,6 +359,10 @@ export async function openMiniDb(db: LifecycleHost, opts: OpenOptions, hoo void hooks.buildGeneration('open').catch(() => {}); } } + // open() is about to return: 'ready', or 'degraded' while a deferred + // text-index base build is still pending in the background. + db.lifecycle.time('openMs', performance.now() - openT0); + db.lifecycle.finishOpen(); } catch (err) { // A background open-time compaction may still be in flight: settle it // before tearing down the WAL/store/handles it touches. diff --git a/packages/minidb/src/lockfile.ts b/packages/minidb/src/lockfile.ts index a4f5eb108..c7bb3e20c 100644 --- a/packages/minidb/src/lockfile.ts +++ b/packages/minidb/src/lockfile.ts @@ -74,10 +74,18 @@ function hookExit(): void { export class LockFile { readonly path: string; held = false; - /** Identity of the current acquire() attempt: minted fresh per attempt, + /** Token of the current acquire attempt: minted fresh per attempt, * carried by every file this instance publishes (lock/bid/watch), and the * sole ownership criterion (`mine`). Null before the first acquire(). */ private token: string | null = null; + + /** The held instance's ownership token (undefined unless currently held). + * Lets a host supervising this database from another thread learn WHO + * holds the lock file without parsing it — worker threads share the main + * process pid, so the pid in the lock line cannot distinguish them. */ + get heldToken(): string | undefined { + return this.held && this.token !== null ? this.token : undefined; + } /** Serializes acquire/renew/release (the shared promise-chain pattern of * serialize.ts): each op's whole read-check-write completes before the next * one starts, so a renew already in flight finishes before a release diff --git a/packages/minidb/src/maintenance.ts b/packages/minidb/src/maintenance.ts index 48af87559..ed22bdee2 100644 --- a/packages/minidb/src/maintenance.ts +++ b/packages/minidb/src/maintenance.ts @@ -349,6 +349,40 @@ export class WorkerSlots { }); } + /** TUI-safe slot policy: queue for a slot up to `waitMs` (observing the + * optional abort signal) instead of dropping a large worker-eligible + * build onto the main thread the moment every slot is busy. Resolves null + * on timeout — the caller decides whether its bounded inline core is an + * acceptable last resort. Rejects MaintenanceCancelledError on abort. */ + async acquireBounded(waitMs: number, signal?: AbortSignal): Promise<(() => void) | null> { + const immediate = this.tryAcquire(); + if (immediate) return immediate; + if (signal?.aborted) throw new MaintenanceCancelledError('generation-build'); + return new Promise<(() => void) | null>((resolve, reject) => { + const cleanup = (): void => { + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + const i = this.waiters.indexOf(grant); + if (i >= 0) this.waiters.splice(i, 1); + }; + const timer = setTimeout(() => { + cleanup(); + resolve(null); + }, waitMs); + timer.unref?.(); + const onAbort = (): void => { + cleanup(); + reject(new MaintenanceCancelledError('generation-build')); + }; + const grant = (): void => { + cleanup(); + resolve(() => this.release()); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + this.waiters.push(grant); + }); + } + private release(): void { const next = this.waiters.shift(); if (next) { @@ -363,3 +397,9 @@ export class WorkerSlots { * capped at two — one worker build already saturates a core for seconds, * and the main thread must stay responsive. */ export const defaultWorkerSlots = new WorkerSlots(Math.max(1, Math.min(2, Math.floor(os.cpus().length / 2)))); + +/** Default budget of the TUI-safe slot queue (WorkerSlots.acquireBounded): + * a worker-eligible text build waits this long for a slot before falling + * back to the bounded inline core as the explicit last resort — large + * full-text builds are never dropped onto the main thread UNCONDITIONALLY. */ +export const TEXT_BUILD_SLOT_WAIT_MS = 30_000; diff --git a/packages/minidb/src/mini-db.ts b/packages/minidb/src/mini-db.ts index 49026a30c..e4b84dca3 100644 --- a/packages/minidb/src/mini-db.ts +++ b/packages/minidb/src/mini-db.ts @@ -15,7 +15,7 @@ import { Store } from './store.js'; import type { StoreRecord } from './store.js'; import { WAL } from './wal.js'; import { ValueReader } from './value-reader.js'; -import { catchUpWal } from './recovery.js'; +import { catchUpWalAsync } from './recovery.js'; import { compact, shouldCompact } from './compaction.js'; import { OpTracker } from './op-tracker.js'; import { TEXT_INDEXES_FILE, SNAPSHOT_FILE, isPersistentFile, rootPostingsFile, textDictionaryFile, textDocsFile } from './generation.js'; @@ -26,7 +26,7 @@ import { CompoundIndexManager } from './compound-index.js'; import { toKStr, writeFileAtomic } from './value-codec.js'; import { LockFile } from './lockfile.js'; import { createSerializer } from './serialize.js'; -import { MaintenanceScheduler, defaultWorkerSlots } from './maintenance.js'; +import { MaintenanceScheduler, defaultWorkerSlots, TEXT_BUILD_SLOT_WAIT_MS } from './maintenance.js'; import { MemoryGuard } from './memory-guard.js'; import { backup as runBackup, backupInProgressError as newBackupInProgressError } from './backup.js'; import type { BackupDeps } from './backup.js'; @@ -39,6 +39,8 @@ import { openMiniDb, closeMiniDb, renewMiniDbLock, openOrRebuildMiniDb } from '. import { IndexAdmin } from './index-admin.js'; import { ReadPath } from './read-path.js'; import { createMiniDbStats } from './stats.js'; +import { LifecycleTracker } from './lifecycle-status.js'; +import type { MiniDbLifecycleStatus } from './lifecycle-status.js'; import type { LifecycleHooks } from './lifecycle.js'; import { TextRegistry } from './text-registry.js'; import type { TextIndexDef } from './text-registry.js'; @@ -52,7 +54,7 @@ import type { RangeOptions } from './skiplist.js'; import type { TextIndexTokenizerName } from './trigram.js'; import type { PostingEntry } from './text-postings.js'; import { startWorkerTextBuild, textBuildWorkerAvailable, verifyFileCrcAsync, WorkerTextBuildError } from './worker/text-build.js'; -import type { TextBuildCheckpoint } from './worker/text-build.js'; +import type { TextBuildCheckpoint, WorkerTextBuildFallbackReason } from './worker/text-build.js'; import { readBaseDocsImageAsync, BASE_DOCS_MAGIC, BASE_DOCS_VERSION } from './worker/text-build-core.js'; import type { TextBuildCoreResult } from './worker/text-build-core.js'; import { readGenerationFileCheckedAsync, readTextDictionaryImageAsync } from './gen-codec.js'; @@ -112,6 +114,12 @@ export class MiniDb { * offset as advanced by the last successful catch-up (recoveryInfo's scan * endpoint anchors the first call). */ private walTail: { dev: number; ino: number; size: number } | null = null; + /** Catch-up serializer: the sliced async apply yields to the event loop, so + * overlapping catch-ups would interleave op-by-op (the pre-async apply was + * atomic per call by virtue of being synchronous). Chaining restores that + * per-call atomicity; a caller queued behind another catch-up observes the + * advanced watermark and no-ops or falls back to a full reopen. */ + private catchUpChain: Promise = Promise.resolve(); readOnly = false; /* Non-private (package-internal): lifecycle.ts reads/writes this through its LifecycleHost view. */ lock: LockFile | null = null; @@ -193,6 +201,10 @@ export class MiniDb { private textWorkerDisabled = false; /** Stage 6: worker aggregation memory budget. */ /* Non-private (package-internal): lifecycle.ts reads/writes this through its LifecycleHost view. */ textBuildMemoryBytes = 128 * 1024 * 1024; + /** TUI-safe worker-slot policy: how long a worker-eligible text build + * queues for a process-wide slot before the bounded inline core is + * allowed as the last resort (never the unbounded staged aggregation). */ + /* Non-private (package-internal): read through the GenerationBuilderDeps view. */ textBuildSlotWaitMs = TEXT_BUILD_SLOT_WAIT_MS; /** Stage 6: maintenance I/O concurrency (snapshot grouped reads). */ /* Non-private (package-internal): lifecycle.ts reads/writes this through its LifecycleHost view. */ maintenanceIoConcurrency = 8; /** Defer the open-time full text rebuild (no-generation fallback) to a background @@ -236,6 +248,10 @@ export class MiniDb { * this state the caller cannot assume a rejected write had no effect. */ writeDisabled: unknown = null; readonly stats = createMiniDbStats(); + /** Per-open lifecycle telemetry (the open() state machine + per-phase + * wall-clock timings): driven by lifecycle.ts and the generation loader, + * read through lifecycleStatus(). */ + /* Non-private (package-internal): lifecycle.ts drives it through its LifecycleHost view. */ readonly lifecycle = new LifecycleTracker(); /** The text-index registry facet (text-registry.ts): owns the live TextIndex * map, the persisted definition list, and the staged-drop marks (declared @@ -327,11 +343,13 @@ export class MiniDb { this.textWorkerDisabled = true; }, textBuildMemoryBytes: () => this.textBuildMemoryBytes, + textBuildSlotWaitMs: () => this.textBuildSlotWaitMs, dt: this.dt, indexes: this.indexes, compound: this.compound, textRegistry: this.textRegistry, stats: this.stats, + lifecycle: this.lifecycle, decode: (b) => this.decode(b), indexable: (v): v is Record => this.indexable(v), liveRecords: () => this.liveRecords(), @@ -561,7 +579,14 @@ export class MiniDb { deferred.add(name); } } + const textRebuildMsBefore = this.stats.textRebuildDurationMs; await this.indexAdmin.rebuildAllIndexes({ skipTextIndex: (name) => deferred.has(name) }); + this.lifecycle.time('textRebuildMs', this.stats.textRebuildDurationMs - textRebuildMsBefore); + // Every non-deferred text index was just rebuilt by the staged in-thread + // walk above (the full-recovery path leaves them empty). + for (const name of this.text.keys()) { + if (!deferred.has(name)) this.lifecycle.noteTextIndexSource(name, 'staged'); + } if (deferred.size > 0) this.deferOpenTextBuilds([...deferred]); } @@ -590,6 +615,7 @@ export class MiniDb { ti.basePending = true; ti.beginRebase(); armed.push({ name, ti, def }); + this.lifecycle.markTextIndexPending(name); } if (armed.length === 0) return; // The checkpoint the build covers: the recovery scan endpoint (see the @@ -644,6 +670,8 @@ export class MiniDb { const output = scratchDir !== null ? { dir: scratchDir, postingsPath: path.join(scratchDir, rootPostingsFile(name)) } : null; let checkpoint = pin(); let committed = false; + let hostedMode: 'worker' | 'inline' | null = null; + const tBuild = performance.now(); for (let attempt = 1; attempt <= 3 && !committed; attempt++) { if (attempt > 1) checkpoint = repin(ti); try { @@ -652,13 +680,16 @@ export class MiniDb { // mid-task): no build will come — stop retrying. if (hosted === null) break; committed = true; + hostedMode = hosted; } catch { // Shutdown cancels the task: leave close() to finish teardown. if (ctx.signal.aborted) return; } } - if (committed) { + this.lifecycle.time('textRebuildMs', performance.now() - tBuild); + if (committed && hostedMode !== null) { this.stats.textDeferredBuilds++; + this.lifecycle.clearTextIndexPending(name, hostedMode); } else { // Every attempt failed: disarm and mark the base pending — // searches keep the typed building signal instead of silently @@ -827,11 +858,21 @@ export class MiniDb { const postingsPath = tmpDir !== null ? path.join(tmpDir, rootPostingsFile(name)) : output!.postingsPath; const dictionaryPath = path.join(artifactsDir, textDictionaryFile(name)); const baseDocsPath = path.join(artifactsDir, `${textDocsFile(name)}.base`); - // Worker thread when its entry file exists; the inline bounded core - // otherwise (slot pressure or a single-file deployment) — never the - // unbounded staged aggregation. + // Worker thread when its entry file exists; the inline bounded core is + // the last resort — never the unbounded staged aggregation. TUI-safe + // slot policy: queue for a process-wide worker slot first (bounded by + // textBuildSlotWaitMs and the caller's abort signal) instead of + // dropping a large build onto the main thread the moment every slot is + // busy; only a persisted slot drought (or a deployment without the + // worker file) hosts the SAME bounded core inline. const workerAvailable = textBuildWorkerAvailable(); - slotRelease = workerAvailable ? defaultWorkerSlots.tryAcquire() : null; + let inlineReason: WorkerTextBuildFallbackReason | undefined; + if (workerAvailable) { + slotRelease = await defaultWorkerSlots.acquireBounded(this.textBuildSlotWaitMs, signal); + if (slotRelease === null) inlineReason = 'slot-pressure'; + } else { + inlineReason = 'runtime-unavailable'; + } const inline = slotRelease === null; const handle = startWorkerTextBuild( { @@ -857,7 +898,7 @@ export class MiniDb { { shouldAbort: () => this.state !== 'open' || signal?.aborted === true, inline, - inlineReason: workerAvailable ? 'slot-pressure' : 'runtime-unavailable', + inlineReason, signal, onFallback: (reason) => { this.stats.textWorkerFallbacks++; @@ -1114,8 +1155,8 @@ export class MiniDb { * derived-index maintenance applyOp performs on the write path — minus * unique checks: the writer already validated, and intermediate frame * states must apply literally (LWW). */ - private applyRecoveredFrame(f: FrameRef, fd: number): void { - this.writePath.applyRecoveredFrame(f, fd); + private applyRecoveredFrameAsync(f: FrameRef, fd: number, slice: () => boolean): Promise { + return this.writePath.applyRecoveredFrameAsync(f, fd, slice); } private applyRecoveredOp(op: RecoveredOp): void { @@ -1352,13 +1393,34 @@ export class MiniDb { * rotated file and a shrunken one all return null, meaning: reopen from * scratch. A partial/torn tail left by a writer mid-writev is NOT an * error: the scan stops at the last fully-valid frame; call again later - * and its CRC validates once the writev landed. */ + * and its CRC validates once the writev landed. + * + * Cooperative: the scan runs through the windowed async scanner and the + * apply yields between primitive ops on the walApplySlicer budgets, so a + * replica that fell far behind does not block the host's event loop in + * one synchronous scan+apply. Calls are serialized per instance (see + * catchUpChain). */ async catchUpFromWal(offset: number): Promise<{ offset: number; appliedFrames: number } | null> { + this.ensureOpen(); + const run = this.catchUpChain.then(() => this.doCatchUpFromWal(offset)); + // The chain itself must survive a caller's rejection (treated as "reopen + // from scratch" by every caller): swallow for the chain, not for the caller. + this.catchUpChain = run.then( + () => undefined, + () => undefined, + ); + return run; + } + + private async doCatchUpFromWal(offset: number): Promise<{ offset: number; appliedFrames: number } | null> { + // Re-check on dequeue: the instance may have been closed while this call + // waited on the chain behind another catch-up (both production callers + // gate close externally; this is the library-contract backstop). this.ensureOpen(); const ri = this.recoveryInfo; const anchor = this.walTail ?? (ri && ri.walIno ? { dev: ri.walDev, ino: ri.walIno, size: ri.walScanEnd } : null); if (!anchor || offset !== anchor.size) return null; - const res = catchUpWal(this.walPath, offset, anchor, (f, fd) => this.applyRecoveredFrame(f, fd)); + const res = await catchUpWalAsync(this.walPath, offset, anchor, (f, fd, slice) => this.applyRecoveredFrameAsync(f, fd, slice)); if (res) this.walTail = { dev: anchor.dev, ino: anchor.ino, size: res.offset }; return res; } @@ -1376,6 +1438,16 @@ export class MiniDb { return this.maintenance.status(); } + /** The last open()'s lifecycle read model: which path served the open + * ('generation-load' + 'wal-catch-up' vs 'full-rebuild'), whether the + * instance is 'ready' or still 'degraded' (a deferred text-index base + * build in flight), the per-phase wall-clock timings, and how every text + * index's base was served (generation image vs worker/inline/staged + * rebuild). Diagnostics only — the cumulative counters stay in `stats`. */ + lifecycleStatus(): MiniDbLifecycleStatus { + return this.lifecycle.snapshot(); + } + async close(): Promise { return closeMiniDb(this, this.lifecycleHooks); } diff --git a/packages/minidb/src/recovery.ts b/packages/minidb/src/recovery.ts index 800b41bd9..f9d4271a6 100644 --- a/packages/minidb/src/recovery.ts +++ b/packages/minidb/src/recovery.ts @@ -6,7 +6,7 @@ // // The per-frame interpretation (expiry drop, batch unrolling, value refs, dt // meta) lives in frameToOps so that open-time recovery and read-replica WAL -// catch-up (catchUpWal) can never drift apart. +// catch-up (catchUpWalAsync) can never drift apart. // // GENERATION PAIRING (stat-pairing — the LEGACY fallback path). MiniDb's // disk state is a compound document: a compaction rotation swaps db.snapshot @@ -35,7 +35,7 @@ import fs from 'node:fs/promises'; import fsSync from 'node:fs'; import path from 'node:path'; -import { scanFrameRefsFd, scanFrameRefsFdAsync, scanBatchOpRefs, TYPE_SET, TYPE_DEL, TYPE_BATCH, MAGIC } from './codec.js'; +import { scanFrameRefsFdAsync, scanBatchOpRefs, TYPE_SET, TYPE_DEL, TYPE_BATCH, MAGIC } from './codec.js'; import type { FrameRef } from './codec.js'; import { SNAPSHOT_FILE, WAL_FILE } from './generation.js'; import { yieldToLoop } from './text-index/tokenize.js'; @@ -44,6 +44,16 @@ import type { Store, ValueLoc, ValueRef } from './store.js'; export type RecoveryMode = 'resync' | 'strict'; export type ValueMode = 'memory' | 'disk'; +/** Optional wall-clock sink for the two recovery sub-phases (scan vs apply), + * accumulated across every pass (generation-churn retries included). Fed by + * the open-time lifecycle telemetry; recover() works unchanged without it. */ +export interface RecoveryPhaseTimings { + /** Async frame scanning of the snapshot and the WAL. */ + walScanMs: number; + /** Applying the scanned frames to the store. */ + walApplyMs: number; +} + export interface RecoveryInfo { snapshotFrames: number; walFrames: number; @@ -176,8 +186,30 @@ export function* frameToOps( } } -/** Yield to the event loop every this many applied frames (see applyFrames). */ -const APPLY_YIELD_FRAMES = 4096; +/** Budgets for the cooperative slicing of recovered-op apply loops. A BATCH + * frame unrolls into thousands of primitive ops, so frame-granular yielding + * cannot bound an apply slice — primitive-op count and elapsed time can. */ +export const WAL_APPLY_OPS_PER_SLICE = 512; +export const WAL_APPLY_SLICE_MS = 8; + +/** Cooperative slicing for the recovered-op apply loops: reports true when + * either budget (primitive ops, or wall-clock since the last yield) is + * exhausted and the loop should yieldToLoop(). Yielding mid-apply is safe on + * both consumers: at open the store is not published until open() returns, + * so nothing observes a half-applied pass; during replica catch-up + * (catchUpWalAsync) concurrent readers can observe intermediate apply + * states, which the replica's eventual-consistency contract permits (the + * caller's watermark only advances once the whole delta has applied). */ +export function walApplySlicer(): () => boolean { + let ops = 0; + let sliceStart = performance.now(); + return () => { + if (++ops < WAL_APPLY_OPS_PER_SLICE && performance.now() - sliceStart < WAL_APPLY_SLICE_MS) return false; + ops = 0; + sliceStart = performance.now(); + return true; + }; +} /** Apply recovered frames to the store. This is pure in-memory bookkeeping — * one synchronous pass per file would be a noticeable event-loop stall on a @@ -193,13 +225,13 @@ async function applyFrames( valueMode: ValueMode, onCorruptBatch?: () => void, ): Promise { - let applied = 0; + const slice = walApplySlicer(); for (const f of frames) { for (const op of frameToOps(f, file, fd, valueMode, onCorruptBatch)) { if (op.type === TYPE_SET) store.setRef(op.key, op.ref!, op.expireAt, op.dt); else if (op.type === TYPE_DEL) store.del(op.key); + if (slice()) await yieldToLoop(); } - if (++applied % APPLY_YIELD_FRAMES === 0) await yieldToLoop(); } } @@ -272,6 +304,7 @@ export async function recover({ maxGenerationRetries = 4, attachValueReader, signal, + timings, }: { dir: string; store: Store; @@ -292,6 +325,8 @@ export async function recover({ /** Cancellation for the async frame scans (stage 6): an aborted scan * throws an 'AbortError' and leaves the pass's partial state discarded. */ signal?: AbortSignal; + /** Optional scan/apply wall-clock sink (see RecoveryPhaseTimings). */ + timings?: RecoveryPhaseTimings; }): Promise { const snapPath = path.join(dir, SNAPSHOT_FILE); const walPath = path.join(dir, WAL_FILE); @@ -299,7 +334,7 @@ export async function recover({ for (let attempt = 0; ; attempt++) { let pass: RecoverPassResult; try { - pass = await recoverPass({ snapPath, walPath, store, mode, truncate, valueMode, signal }); + pass = await recoverPass({ snapPath, walPath, store, mode, truncate, valueMode, signal, timings }); } catch (e) { // A cancelled scan may have applied a prefix of the pass's frames: // discard the partial application so the error never carries state @@ -336,6 +371,7 @@ async function recoverPass({ truncate, valueMode, signal, + timings, }: { snapPath: string; walPath: string; @@ -344,6 +380,7 @@ async function recoverPass({ truncate: boolean; valueMode: ValueMode; signal?: AbortSignal; + timings?: RecoveryPhaseTimings; }): Promise { let corruptBatches = 0; const countCorruptBatch = (): void => { @@ -362,8 +399,12 @@ async function recoverPass({ // Stage 6: the scan itself is async (sequential window reads, CRC // slices, periodic yields, abortable); only the in-memory apply below // runs on the event loop. + const snapScanT0 = performance.now(); const r = await scanFrameRefsFdAsync(fd, { onCorrupt: mode, signal }); + if (timings) timings.walScanMs += performance.now() - snapScanT0; + const snapApplyT0 = performance.now(); await applyFrames(r.frames, 'snapshot', fd, store, valueMode, countCorruptBatch); + if (timings) timings.walApplyMs += performance.now() - snapApplyT0; snapshotFrames = r.frames.length; snapshotCorrupt = r.corruptRanges; } finally { @@ -388,8 +429,12 @@ async function recoverPass({ walScanned = { dev: st.dev, ino: st.ino, size: st.size }; walSizeFloor = st.size; walBytes = st.size; + const walScanT0 = performance.now(); const r = await scanFrameRefsFdAsync(fd, { onCorrupt: mode, signal }); + if (timings) timings.walScanMs += performance.now() - walScanT0; + const walApplyT0 = performance.now(); await applyFrames(r.frames, 'wal', fd, store, valueMode, countCorruptBatch); + if (timings) timings.walApplyMs += performance.now() - walApplyT0; walFrames = r.frames.length; walCorrupt = r.corruptRanges; walScanEnd = r.eofOffset; @@ -443,7 +488,7 @@ async function recoverPass({ } /** Continue a replica from a WAL watermark: strictly scan frames in - * [offset, EOF) of `walPath` and hand each to `apply(f, fd)` in order. + * [offset, EOF) of `walPath` and hand each to `apply(f, fd, slice)` in order. * * Returns the new continuation offset (end of the last fully-valid frame) * and how many frames were applied. An invalid/partial frame anywhere stops @@ -453,13 +498,24 @@ async function recoverPass({ * cannot be a clean frame-boundary continuation (negative, beyond EOF, * pointing at bytes that start no frame, or the file is not the `anchor` * inode — rotation swapped it in the microseconds between the caller's stat - * and this open): the caller must fully reopen. */ -export function catchUpWal( + * and this open): the caller must fully reopen. + * + * Cooperative (stage 5 follow-up of the open-path slicing): the frame scan + * runs through the windowed async scanner and the apply loop yields between + * primitive ops on the shared walApplySlicer budgets, so a replica that fell + * far behind no longer blocks the host loop in one synchronous scan+apply. + * `apply` receives the slicer and MUST await-yield when it fires — a BATCH + * frame unrolls into thousands of primitive ops, so frame-granular yielding + * could never bound a slice. Yielding mid-catch-up lets concurrent readers + * observe intermediate apply states; that is the documented replica contract + * (a replica is eventually consistent — the caller's watermark only advances + * once the whole delta has applied). */ +export async function catchUpWalAsync( walPath: string, offset: number, anchor: { dev: number; ino: number }, - apply: (f: FrameRef, fd: number) => void, -): { offset: number; appliedFrames: number } | null { + apply: (f: FrameRef, fd: number, slice: () => boolean) => Promise, +): Promise<{ offset: number; appliedFrames: number } | null> { let fd: number; try { fd = fsSync.openSync(walPath, 'r'); @@ -472,7 +528,7 @@ export function catchUpWal( if (st.dev !== anchor.dev || st.ino !== anchor.ino) return null; const size = st.size; if (offset < 0 || offset > size) return null; - const r = scanFrameRefsFd(fd, { onCorrupt: 'strict', startOffset: offset }); + const r = await scanFrameRefsFdAsync(fd, { onCorrupt: 'strict', startOffset: offset }); if (r.frames.length === 0 && r.eofOffset < size) { // Bytes at `offset` start no valid frame. An append-only file grows // whole frames sequentially, so a torn tail is always a frame PREFIX @@ -483,7 +539,8 @@ export function catchUpWal( if (!MAGIC.subarray(0, n).equals(head)) return null; return { offset, appliedFrames: 0 }; } - for (const f of r.frames) apply(f, fd); + const slice = walApplySlicer(); + for (const f of r.frames) await apply(f, fd, slice); return { offset: r.eofOffset, appliedFrames: r.frames.length }; } finally { fsSync.closeSync(fd); diff --git a/packages/minidb/src/skiplist.ts b/packages/minidb/src/skiplist.ts index 42a11b497..d881aa719 100644 --- a/packages/minidb/src/skiplist.ts +++ b/packages/minidb/src/skiplist.ts @@ -123,6 +123,63 @@ export class SkipList { return list; } + /** Sliced variant of bulkLoad (the open-time main-thread load paths): + * builds the exact same balanced tower, but yields to the event loop every + * `sliceEvery` source entries in the two O(N) passes (node creation, + * level linking), so a large image load never runs as one synchronous + * slice. */ + static async bulkLoadAsync( + entries: readonly RangeEntry[], + opts: SkipListOptions = {}, + slice: { sliceEvery?: number } = {}, + ): Promise> { + const sliceEvery = slice.sliceEvery ?? 65536; + const yieldToLoop = (): Promise => new Promise((r) => setImmediate(r)); + const list = new SkipList(opts); + const n = entries.length; + if (n === 0) return list; + const nodes: SkipNode[] = []; + for (let i = 0; i < n; i++) { + if (i > 0 && i % sliceEvery === 0) await yieldToLoop(); + const e = entries[i]!; + if (nodes.length > 0) { + const prev = nodes[nodes.length - 1]!; + if (list.cmpK(prev.key, e.key) === 0 && list.cmpV(prev.val, e.val) === 0) continue; + } + // Balanced tower: index i (0-based) reaches level 1 + v4(i+1), capped. + let lvl = 1; + for (let m = i + 1; m % 4 === 0 && lvl < MAX_LEVEL; m = m / 4) lvl++; + nodes.push(new SkipNode(e.key, e.val, lvl)); + } + const count = nodes.length; + list.level = 1; + for (const node of nodes) if (node.level.length > list.level) list.level = node.level.length; + // Link every level: forward pointers + spans (level-0 distance to the + // forward node; 0 for a tail's null forward, matching insert()). + const lastAt: { node: SkipNode; index: number }[] = []; + for (let l = 0; l < list.level; l++) lastAt.push({ node: list.header, index: -1 }); + for (let i = 0; i < count; i++) { + if (i > 0 && i % sliceEvery === 0) await yieldToLoop(); + const node = nodes[i]!; + for (let l = 0; l < node.level.length; l++) { + const pred = lastAt[l]!; + pred.node.level[l]!.forward = node; + pred.node.level[l]!.span = i - pred.index; + lastAt[l] = { node, index: i }; + } + node.backward = i === 0 ? null : nodes[i - 1]!; + } + for (let l = 0; l < list.level; l++) { + const pred = lastAt[l]!; + // Header spans at unused levels keep the insert() convention (distance + // from the header's virtual index -1, i.e. count); real tail nodes get 0. + pred.node.level[l]!.span = pred.index === -1 ? count : 0; + } + list.tail = nodes[count - 1]!; + list.length = count; + return list; + } + private nodeLess(a: SkipNode, b: { key: K; val: V }): boolean { const c = this.cmpK(a.key, b.key); return c < 0 || (c === 0 && this.cmpV(a.val, b.val) < 0); diff --git a/packages/minidb/src/store.ts b/packages/minidb/src/store.ts index 7d9a3a3cc..247b2c9db 100644 --- a/packages/minidb/src/store.ts +++ b/packages/minidb/src/store.ts @@ -132,6 +132,16 @@ export class Store { * expiry backlog), making the next ticks aggressive until the storm drains * — like Redis's aggressive expire cycle. */ private expireAggressive = false; + /** In-flight async bulk load (bulkLoadRefsAsync only — the sync bulk load + * yields nothing, so no timer can fire mid-load there). Active expiry is + * paused for the load's duration: a mid-load reap would delete the key + * from the map (and from the about-to-be-replaced OLD ordered index, a + * no-op) while the load still rebuilds `order` from its accumulated + * entries — resurrecting the reaped key in the ordered index, where a + * later re-set would duplicate it in ordered scans. Whatever elapsed + * during the load is reaped by the first tick after it, which is exactly + * the sync bulkLoadRefs behavior. */ + private bulkLoading = false; private timer: ReturnType | null = null; private readonly onExpire?: (key: string, record: StoreRecord) => void; private readonly readValue?: ValueReader; @@ -389,7 +399,48 @@ export class Store { this.order = SkipList.bulkLoad(orderEntries, { compareKey: cmpString }); } + /** Sliced variant of bulkLoadRefs (the open-time main-thread path): + * identical resulting state, but the per-record map insertions yield to + * the event loop every `sliceEvery` records, so a large store image never + * loads in one synchronous run. The final ordered-index bulk build is a + * single O(N) pass over the sorted entries and is likewise sliced via + * SkipList.bulkLoadAsync. Safe to yield mid-load: the store is not + * published until open() returns, and active expiry is paused for the + * load's duration (see `bulkLoading`) so a mid-load reap cannot diverge + * the map from the not-yet-rebuilt ordered index. */ + async bulkLoadRefsAsync( + records: Iterable<{ kstr: string; ref: ValueRef; expireAt: number; dt: Record | null; metaBytes?: number }>, + opts: { sliceEvery?: number } = {}, + ): Promise { + const sliceEvery = opts.sliceEvery ?? 8192; + const orderEntries: RangeEntry[] = []; + this.bulkLoading = true; + try { + let n = 0; + for (const { kstr, ref, expireAt, dt, metaBytes } of records) { + const seq = ++this.seq; + this.map.set(kstr, { ref, expireAt: expireAt || 0, seq, dt }); + this.bytes += Buffer.byteLength(kstr, 'binary') + this.refBytes(ref) + (metaBytes ?? 0); + if (expireAt) { + this.expiring++; + this.heap.push({ t: expireAt, k: kstr, seq }); + } + orderEntries.push({ key: kstr, val: kstr }); + if (++n % sliceEvery === 0) await new Promise((r) => setImmediate(r)); + } + this.order = await SkipList.bulkLoadAsync(orderEntries, { compareKey: cmpString }, { sliceEvery }); + } finally { + this.bulkLoading = false; + } + } + private activeExpire(): void { + // A sliced bulk load holds the map/order consistency contract: reaping + // mid-load would drop the key from the map while the load still rebuilds + // `order` from its accumulated entries. Skip this tick; the next one + // reaps whatever elapsed meanwhile (identical to the sync bulk load, + // during which no timer can fire at all). + if (this.bulkLoading) return; const now = Date.now(); // Normal ticks stay within the small budget; a tick that still finds a // full quota of expired keys flips to aggressive mode (larger budget, like diff --git a/packages/minidb/src/text-index/image.ts b/packages/minidb/src/text-index/image.ts index 8265b3a0d..fc08680a3 100644 --- a/packages/minidb/src/text-index/image.ts +++ b/packages/minidb/src/text-index/image.ts @@ -150,6 +150,92 @@ export function attachImage( s.baseEpoch++; } +/** The raw parsed images attachImageAsync consumes (the generation loader's + * readTextDictionaryImageAsync / readTextDocsImageAsync output, structurally + * — avoiding a gen-codec import here). */ +export interface AttachImageRaw { + postingsPath: string; + dictEntries: Iterable<{ term: string; off: number; len: number; df: number }>; + docs: { + keys: (string | undefined)[]; + docLens: (number | undefined)[]; + liveCount: number; + removed: number[]; + delta: { term: string; docs: { docID: number; freq: number }[] }[]; + }; +} + +/** Sliced variant of attachImage (the open-time main-thread path): identical + * resulting state and the same side-effect order, but every O(terms/docs) + * map construction happens here, in slices with event-loop yields, so + * attaching a large generation's base never stalls the loop for the whole + * pass. Safe to yield mid-attach: the index is not serving until open() + * returns, and the readonly-bound containers (delta, deltaDocs, removed — + * their bindings are stable by class invariant) are cleared-then-refilled + * in place exactly like the sync attach, only with yields interleaved. */ +export async function attachImageAsync( + s: TextIndexImageState, + args: AttachImageRaw, + opts: { sliceEvery?: number } = {}, +): Promise { + const sliceEvery = opts.sliceEvery ?? 32768; + let n = 0; + const tick = async (): Promise => { + if (++n % sliceEvery === 0) await yieldToLoop(); + }; + s.close(); // release any previous postings handle + s.memBase = null; + const postings = new Map(); + for (const e of args.dictEntries) { + postings.set(e.term, { off: e.off, len: e.len, df: e.df }); + await tick(); + } + s.postings = postings; + s.pf = PostingsFile.open(args.postingsPath); + const keys = args.docs.keys; // adopted (fresh from the parser — see commitRebase's containers) + const docLen = new Map(); + for (let i = 0; i < keys.length; i++) { + const len = args.docs.docLens[i]; + if (len !== undefined) docLen.set(i, len); + await tick(); + } + const keyToId = new Map(); + for (let i = 0; i < keys.length; i++) { + const k = keys[i]; + if (k !== undefined) keyToId.set(k, i); + await tick(); + } + s.docLen = docLen; + s.keys = keys; + s.keyToId = keyToId; + s.delta.clear(); + s.deltaDocs.clear(); + s.deltaCount = 0; + for (const d of args.docs.delta) { + const m = new Map(); + s.delta.set(d.term, m); + for (const doc of d.docs) { + m.set(doc.docID, doc.freq); + s.deltaCount++; + let set = s.deltaDocs.get(doc.docID); + if (!set) s.deltaDocs.set(doc.docID, (set = new Set())); + set.add(d.term); + await tick(); + } + } + s.removed.clear(); + for (const id of args.docs.removed) { + s.removed.add(id); + await tick(); + } + s.clearCache(); + s.N = args.docs.liveCount; + // A committed base is now attached — the index no longer owes a build, and + // any async read in flight must re-read from the attached base. + s.basePending = false; + s.baseEpoch++; +} + /** Stage-5 generation build: after the atomic publish rename, repoint the * live base handle from the build's tmp directory to the published * generation directory (same file, final name). On Windows an open handle diff --git a/packages/minidb/src/text-index/index.ts b/packages/minidb/src/text-index/index.ts index c53217d56..9313cc94f 100644 --- a/packages/minidb/src/text-index/index.ts +++ b/packages/minidb/src/text-index/index.ts @@ -51,15 +51,16 @@ import type { import { feedBuild, StagedBuild } from './builder.js'; import { attachImage as attachImageImpl, + attachImageAsync as attachImageAsyncImpl, exportImageState as exportImageStateImpl, exportImageStateAsync as exportImageStateAsyncImpl, repointPostings as repointPostingsImpl, } from './image.js'; -import type { TextIndexImage } from './image.js'; +import type { AttachImageRaw, TextIndexImage } from './image.js'; export * from './tokenize.js'; export * from './types.js'; -export type { TextIndexImage } from './image.js'; +export type { AttachImageRaw, TextIndexImage } from './image.js'; /** * Raised by text searches while the index's base is (re)building. The @@ -931,6 +932,15 @@ export class TextIndex { attachImageImpl(this, args); } + /** Sliced variant of attachImage (the open-time main-thread path): + * identical resulting state, but every O(terms/docs) map construction is + * built from the raw parsed images in slices with event-loop yields, so + * attaching a large generation's base never stalls the loop for the whole + * pass. */ + async attachImageAsync(args: AttachImageRaw, opts: { sliceEvery?: number } = {}): Promise { + await attachImageAsyncImpl(this, args, opts); + } + /** Stage-5 generation build: after the atomic publish rename, repoint the * live base handle from the build's tmp directory to the published * generation directory (same file, final name). On Windows an open handle diff --git a/packages/minidb/src/types.ts b/packages/minidb/src/types.ts index 8ca56349d..f4f67858a 100644 --- a/packages/minidb/src/types.ts +++ b/packages/minidb/src/types.ts @@ -31,6 +31,15 @@ export interface OpenOptions { recovery?: RecoveryMode; readOnly?: boolean; onLockFail?: 'readonly'; + /** + * Writer opens only: invoked synchronously right after the exclusive write + * lock is acquired — BEFORE any recovery/replay work runs. Hosts that + * supervise the open from another thread (e.g. kap-server's search worker, + * whose threads share the main process pid so pid-liveness alone can never + * reclaim the lock) use it to learn the lock token immediately and reap + * the lock after a mid-open crash. + */ + onLockAcquired?: (info: { readonly token: string }) => void; /** Where to keep value bulk. 'memory' keeps values in RAM; 'disk' keeps only * value pointers in RAM and reads values from the snapshot/WAL on demand. */ valueMode?: ValueModeSetting; diff --git a/packages/minidb/src/write-path.ts b/packages/minidb/src/write-path.ts index 74f20858c..b645126bc 100644 --- a/packages/minidb/src/write-path.ts +++ b/packages/minidb/src/write-path.ts @@ -19,6 +19,7 @@ import { encodeFrame, encodeBatchOps, scanBatchOpRefs, HEADER_SIZE, TYPE_SET, TY import type { BatchOp as EncodedBatchOp, FrameRef } from './codec.js'; import { frameToOps } from './recovery.js'; import type { ValueMode, RecoveredOp } from './recovery.js'; +import { yieldToLoop } from './text-index/tokenize.js'; import { toBuf, toKStr, normDt, MAX_KEY_LEN } from './value-codec.js'; import type { Store, StoreRecord, ValueLoc } from './store.js'; import type { WAL } from './wal.js'; @@ -643,9 +644,15 @@ export class WritePath { * open-time recovery derives from it (frameToOps), plus the incremental * derived-index maintenance applyOp performs on the write path — minus * unique checks: the writer already validated, and intermediate frame - * states must apply literally (LWW). */ - applyRecoveredFrame(f: FrameRef, fd: number): void { - for (const op of frameToOps(f, 'wal', fd, this.deps.valueMode())) this.applyRecoveredOp(op); + * states must apply literally (LWW). Cooperative: yields between primitive + * ops when the caller's slicer (walApplySlicer budgets) fires — a BATCH + * frame unrolls into thousands of ops, so per-op yielding is what bounds a + * catch-up slice on the host's event loop. */ + async applyRecoveredFrameAsync(f: FrameRef, fd: number, slice: () => boolean): Promise { + for (const op of frameToOps(f, 'wal', fd, this.deps.valueMode())) { + this.applyRecoveredOp(op); + if (slice()) await yieldToLoop(); + } } applyRecoveredOp(op: RecoveredOp): void { diff --git a/packages/minidb/test/bench-json.test.ts b/packages/minidb/test/bench-json.test.ts index 450422137..8ed11b355 100644 --- a/packages/minidb/test/bench-json.test.ts +++ b/packages/minidb/test/bench-json.test.ts @@ -97,3 +97,84 @@ test('bench --quick emits a JSON report with a stable schema', async () => { await fs.rm(dir, { recursive: true, force: true }); } }, 300_000); + +test('open-lifecycle bench --quick emits the phase-1 baseline report', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-open-bench-json-')); + try { + const reportPath = path.join(dir, 'report.json'); + await promisify(execFile)( + process.execPath, + ['--import', 'tsx', path.join(pkgDir, 'bench', 'open-lifecycle.ts'), '--quick', '--json', reportPath], + { cwd: pkgDir, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 }, + ); + const report = JSON.parse(await fs.readFile(reportPath, 'utf8')); + + // Top-level envelope. + assert.equal(report.schemaVersion, 1); + assert.equal(report.tool, 'minidb/bench/open-lifecycle'); + assert.ok(Array.isArray(report.scenarios)); + + const byName = (re) => report.scenarios.find((s) => re.test(s.name)); + + // Every row carries the event-loop-delay + input-latency baselines. + for (const s of report.scenarios) { + assert.equal(typeof s.durationMs, 'number', `${s.name}.durationMs`); + for (const k of ['mean', 'p50', 'p95', 'p99', 'max']) { + assert.equal(typeof s.eventLoopDelayMs[k], 'number', `${s.name}.eventLoopDelayMs.${k}`); + } + assert.ok(s.inputLatencyMs, `${s.name}.inputLatencyMs`); + for (const k of ['count', 'p50', 'p95', 'p99', 'max']) { + assert.equal(typeof s.inputLatencyMs[k], 'number', `${s.name}.inputLatencyMs.${k}`); + } + } + + // The four scenario families exist with their lifecycle phase breakdown. + const PHASES = [ + 'openMs', + 'generationCandidateLoadMs', + 'storeImageLoadMs', + 'nonTextImageLoadMs', + 'textImageLoadMs', + 'postingsIntegrityCheckMs', + 'walScanMs', + 'walApplyMs', + 'fullRecoveryMs', + 'textRebuildMs', + ]; + const small = byName(/small corpus, healthy generation/); + const wal = byName(/large WAL delta/); + const large = byName(/large full-text generation/); + const corrupt = byName(/corrupt generation \(full-recovery fallback\)/); + const deferred = byName(/deferred text rebuild \(degraded -> ready\)/); + for (const [name, s] of Object.entries({ small, wal, large, corrupt, deferred })) { + assert.ok(s, `${name} scenario exists`); + for (const k of PHASES) assert.equal(typeof s.extra.phases[k], 'number', `${name}.extra.phases.${k}`); + } + + // Healthy generation opens: no full recovery, no corpus tokenization. + for (const [name, s] of Object.entries({ small, wal, large })) { + assert.equal(s.extra.state, 'ready', name); + assert.deepEqual(s.extra.path, ['no-generation', 'generation-load', 'wal-catch-up', 'ready'], name); + assert.equal(s.extra.phases.fullRecoveryMs, 0, name); + assert.equal(s.extra.phases.textRebuildMs, 0, name); + assert.equal(s.extra.generationLoads, 1, name); + assert.equal(s.extra.indexRebuildDecoded, 0, name); + } + // The WAL-delta open really replayed the delta (and close did NOT + // republish it into a fresh generation — the scenario would be void). + assert.ok(wal.extra.walDeltaAppliedOps > 0, 'wal delta replayed'); + assert.equal(wal.extra.closePublishedGeneration, false, 'close left the generation untouched'); + assert.ok(wal.extra.phases.walApplyMs > 0, 'wal apply measured'); + + // The corrupt generation fell back to a full recovery and returned + // degraded; the background deferred build then brought it to ready. + assert.equal(corrupt.extra.state, 'degraded'); + assert.deepEqual(corrupt.extra.path, ['no-generation', 'generation-load', 'full-rebuild', 'degraded']); + assert.ok(corrupt.extra.phases.fullRecoveryMs > 0, 'full recovery measured'); + assert.equal(corrupt.extra.generationLoadFallbacks, 1); + assert.equal(deferred.extra.state, 'ready'); + assert.ok(deferred.extra.textDeferredBuilds >= 1, 'deferred build committed'); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}, 300_000); diff --git a/packages/minidb/test/generation.test.ts b/packages/minidb/test/generation.test.ts index 1d272030a..b89d91514 100644 --- a/packages/minidb/test/generation.test.ts +++ b/packages/minidb/test/generation.test.ts @@ -14,6 +14,7 @@ import fsSync from 'node:fs'; import path from 'node:path'; import { afterEach, describe, expect, test, vi } from 'vitest'; import { MiniDb } from '../src/index.js'; +import { TextIndex } from '../src/text-index/index.js'; import { SkipList, cmpNumber, cmpString } from '../src/skiplist.js'; import { GenerationCorruptError, @@ -23,6 +24,11 @@ import { readSecondaryIndexImage, readCompoundIndexImage, readTextDocsImage, + readSecondaryIndexImageAsync, + readCompoundIndexImageAsync, + readTextDocsImageAsync, + verifyFileIntegritySync, + verifyFileIntegrityAsync, writeStoreImage, writeDtIndexImage, writeSecondaryIndexImage, @@ -30,6 +36,10 @@ import { writeTextDocsImage, } from '../src/gen-codec.js'; import type { StoreImageRecord } from '../src/gen-codec.js'; +import { crc32 } from '../src/crc32.js'; +import { Store } from '../src/store.js'; +import { IndexManager } from '../src/index-manager.js'; +import { CompoundIndexManager } from '../src/compound-index.js'; import { readManifest, listGenerations } from '../src/generation-files.js'; import { tmpDir, rmrf, waitFor, deferred } from './helpers.js'; @@ -156,6 +166,32 @@ describe('skiplist bulkLoad', () => { { key: 1, val: 'b' }, ]); }); + + test('bulkLoadAsync builds the identical tower, sliced (incl. duplicates)', async () => { + const entries: { key: number; val: string }[] = []; + for (let i = 0; i < 5000; i++) { + entries.push({ key: i * 2, val: `v${i}` }); + if (i % 997 === 0) entries.push({ key: i * 2, val: `v${i}` }); // exact duplicates + } + const sync = SkipList.bulkLoad(entries, { compareKey: cmpNumber, compareVal: cmpString }); + const asyncList = await SkipList.bulkLoadAsync(entries, { compareKey: cmpNumber, compareVal: cmpString }, { sliceEvery: 1 }); + expect(asyncList.length).toBe(sync.length); + expect(asyncList.toArray()).toEqual(sync.toArray()); + for (const probe of [0, 1, 42, 2499, 4999]) { + const e = entries[probe]!; + expect(asyncList.getRank(e.key, e.val)).toBe(sync.getRank(e.key, e.val)); + } + expect(asyncList.getByRank(0)).toEqual(sync.getByRank(0)); + expect(asyncList.getByRank(asyncList.length - 1)).toEqual(sync.getByRank(sync.length - 1)); + expect(asyncList.range({ gte: 100, lte: 200, count: 7 })).toEqual(sync.range({ gte: 100, lte: 200, count: 7 })); + // Mutations after the sliced bulk load keep every invariant. + asyncList.insert(3, 'v-new'); + asyncList.delete(100, 'v50'); + sync.insert(3, 'v-new'); + sync.delete(100, 'v50'); + expect(asyncList.toArray()).toEqual(sync.toArray()); + expect(asyncList.getRank(3, 'v-new')).toBe(sync.getRank(3, 'v-new')); + }); }); // ---- unit: gen-codec round-trips -------------------------------------------- @@ -285,6 +321,217 @@ describe('gen-codec', () => { }); }); +// ---- unit: the sliced (event-loop-friendly) open-path variants --------------- + +describe('sliced open-path variants', () => { + test('verifyFileIntegrityAsync streams a chunked verify with the sync error semantics', async () => { + const dir = await openTmp('verify-async'); + const p = path.join(dir, 'raw.bin'); + const payload = Buffer.alloc(2 * 1024 * 1024 + 123); // spans multiple 1 MiB chunks + for (let i = 0; i < payload.length; i++) payload[i] = (i * 31 + 7) & 0xff; + await fs.writeFile(p, payload); + const good = { bytes: payload.length, crc32: crc32(payload) >>> 0 }; + + // Same verdict as the sync verifier, on accept and on both rejections. + await verifyFileIntegrityAsync(p, good); + expect(() => verifyFileIntegritySync(p, good)).not.toThrow(); + + const sizeErr = await verifyFileIntegrityAsync(p, { bytes: good.bytes + 1, crc32: good.crc32 }).catch((e) => e); + expect(sizeErr).toBeInstanceOf(GenerationCorruptError); + expect((sizeErr as Error).message).toBe('file size does not match manifest record'); + + const crcErr = await verifyFileIntegrityAsync(p, { bytes: good.bytes, crc32: (good.crc32 ^ 1) >>> 0 }).catch((e) => e); + expect(crcErr).toBeInstanceOf(GenerationCorruptError); + expect((crcErr as Error).message).toBe('file crc does not match manifest record'); + expect(() => verifyFileIntegritySync(p, { bytes: good.bytes, crc32: (good.crc32 ^ 1) >>> 0 })).toThrow( + 'file crc does not match manifest record', + ); + + // An empty file is a valid edge: zero chunks, crc of nothing. + const empty = path.join(dir, 'empty.bin'); + await fs.writeFile(empty, Buffer.alloc(0)); + await verifyFileIntegrityAsync(empty, { bytes: 0, crc32: 0 }); + }); + + test('async image parsers match the sync parsers entry-for-entry', async () => { + const dir = await openTmp('parse-async'); + const secImage = [ + { + name: 'byKind', + field: 'kind', + type: 'equality' as const, + unique: false, + sparse: true, + equality: [ + { scalarKey: 'string:t1', pks: ['a', 'b'] }, + { scalarKey: 'number:7', pks: ['c'] }, + ], + range: null, + }, + { + name: 'byScore', + field: 'score', + type: 'range' as const, + unique: false, + sparse: true, + equality: null, + range: [ + { value: 1.5, pk: 'a' }, + { value: 2, pk: 'b' }, + { value: 2, pk: 'c' }, + ], + }, + ]; + await writeSecondaryIndexImage(path.join(dir, 'sec'), secImage); + const secPayload = parseGenerationBuffer(await fs.readFile(path.join(dir, 'sec')), 'MDSI', 1).payload; + const secSync = readSecondaryIndexImage(secPayload); + const secAsync = await readSecondaryIndexImageAsync(parseGenerationBuffer(await fs.readFile(path.join(dir, 'sec')), 'MDSI', 1).payload, 1); + expect(secAsync).toEqual(secSync); + + const cmpImage = [ + { + name: 'g', + groupBy: 'kind', + orderBy: 'ts', + orderType: 'number' as const, + groups: [ + { + group: 't1', + entries: [ + { order: 5, pk: 'a' }, + { order: 6, pk: 'b' }, + ], + }, + { group: null, entries: [] }, + { group: true, entries: [{ order: 2, pk: 'y' }] }, + ], + }, + ]; + await writeCompoundIndexImage(path.join(dir, 'cmp'), cmpImage); + const cmpSync = readCompoundIndexImage(parseGenerationBuffer(await fs.readFile(path.join(dir, 'cmp')), 'MDCI', 1).payload); + const cmpAsync = await readCompoundIndexImageAsync(parseGenerationBuffer(await fs.readFile(path.join(dir, 'cmp')), 'MDCI', 1).payload, 1); + expect(cmpAsync).toEqual(cmpSync); + + const docsImage = { + keys: ['a', undefined, 'c', 'd'] as (string | undefined)[], + docLens: [10, undefined, 3, 40] as (number | undefined)[], + liveCount: 3, + removed: [1], + delta: [ + { term: 'hello', docs: [{ docID: 2, freq: 4 }] }, + { + term: 'world', + docs: [ + { docID: 0, freq: 1 }, + { docID: 3, freq: 2 }, + ], + }, + ], + }; + await writeTextDocsImage(path.join(dir, 'docs'), docsImage); + const docsSync = readTextDocsImage(parseGenerationBuffer(await fs.readFile(path.join(dir, 'docs')), 'MDTC', 1).payload); + const docsAsync = await readTextDocsImageAsync(parseGenerationBuffer(await fs.readFile(path.join(dir, 'docs')), 'MDTC', 1).payload, 1); + expect(docsAsync).toEqual(docsSync); + expect(docsAsync).toEqual(docsImage); + }); + + test('Store.bulkLoadRefsAsync builds the same store as bulkLoadRefs', async () => { + const records: StoreImageRecord[] = []; + for (let i = 0; i < 300; i++) { + const kstr = `k${String(i).padStart(4, '0')}`; // already ascending + records.push({ + kstr, + ref: { kind: 'memory', value: Buffer.from(`v${i}`) }, + expireAt: i % 10 === 0 ? 9999999999999 : 0, + dt: i % 3 === 0 ? { ts: i } : null, + metaBytes: i % 3 === 0 ? Buffer.byteLength(JSON.stringify({ dt: { ts: i } }), 'utf8') : 0, + }); + } + const syncStore = new Store({ activeExpireIntervalMs: 0 }); + const asyncStore = new Store({ activeExpireIntervalMs: 0 }); + try { + syncStore.bulkLoadRefs(records); + await asyncStore.bulkLoadRefsAsync(records, { sliceEvery: 1 }); + expect([...asyncStore.rawKeys()]).toEqual([...syncStore.rawKeys()]); + expect(asyncStore.size).toBe(syncStore.size); + expect(asyncStore.bytes).toBe(syncStore.bytes); + expect(asyncStore.get('k0007')).toEqual(syncStore.get('k0007')); + expect(asyncStore.getRecord('k0150')).toMatchObject({ expireAt: 9999999999999, dt: { ts: 150 } }); + } finally { + syncStore.close(); + asyncStore.close(); + } + }); + + test('loadImageAsync (secondary + compound) builds the same queryable state', async () => { + const docs = []; + for (let i = 0; i < 200; i++) docs.push({ pk: `k${i}`, doc: { kind: `t${i % 7}`, score: i % 100, ts: 1700000000000 + i } }); + + const secSrc = new IndexManager(); + secSrc.create('byKind', { field: 'kind' }); + secSrc.create('byScore', { field: 'score', type: 'range' }); + const cmpSrc = new CompoundIndexManager(); + cmpSrc.create('byKindTime', { groupBy: 'kind', orderBy: 'ts' }); + for (const { pk, doc } of docs) { + secSrc.add(pk, doc); + cmpSrc.add(pk, doc, null); + } + + const secDst = new IndexManager(); + secDst.create('byKind', { field: 'kind' }); + secDst.create('byScore', { field: 'score', type: 'range' }); + for (const image of secSrc.exportImage()) await secDst.loadImageAsync(image, { sliceEvery: 1 }); + expect(secDst.findEq('byKind', 't3').sort()).toEqual(secSrc.findEq('byKind', 't3').sort()); + expect(secDst.findRange('byScore', { min: 10, max: 42 })).toEqual(secSrc.findRange('byScore', { min: 10, max: 42 })); + + const cmpDst = new CompoundIndexManager(); + cmpDst.create('byKindTime', { groupBy: 'kind', orderBy: 'ts' }); + for (const image of cmpSrc.exportImage().images) await cmpDst.loadImageAsync(image, { sliceEvery: 1 }); + expect(cmpDst.range('byKindTime', 't2', { limit: 5 })).toEqual(cmpSrc.range('byKindTime', 't2', { limit: 5 })); + expect(cmpDst.range('byKindTime', 't5', { limit: 50 })).toEqual(cmpSrc.range('byKindTime', 't5', { limit: 50 })); + }); + + test('TextIndex.attachImageAsync attaches the same base as attachImage', async () => { + const dir = await openTmp('attach-async'); + const postingsPath = path.join(dir, 'ft.postings'); + const src = new TextIndex({ name: 'ft', fields: ['text'], postingsPath }); + const syncTi = new TextIndex({ name: 'ft', fields: ['text'] }); + const asyncTi = new TextIndex({ name: 'ft', fields: ['text'] }); + try { + const corpus = []; + for (let i = 0; i < 300; i++) corpus.push({ key: `k${i}`, value: { text: `hello world ${i} 持久化 ${i % 7}` } }); + await src.build(corpus); + src.add('late1', { text: 'late write walrus' }); // lands in the delta + src.remove('k5'); // tombstone + const image = src.exportImageState(); + + syncTi.attachImage({ postingsPath, ...image }); + const docsImage = { + keys: image.keys, + docLens: (() => { + const out: (number | undefined)[] = []; + for (let i = 0; i < image.keys.length; i++) out.push(image.docLens.get(i)); + return out; + })(), + liveCount: image.liveCount, + removed: [...image.removed], + delta: [...image.delta].map(([term, m]) => ({ term, docs: [...m].map(([docID, freq]) => ({ docID, freq })) })), + }; + const dictEntries = [...image.dict].map(([term, e]) => ({ term, off: e.off, len: e.len, df: e.df })); + await asyncTi.attachImageAsync({ postingsPath, dictEntries, docs: docsImage }, { sliceEvery: 1 }); + + expect(asyncTi.exportImageState()).toEqual(syncTi.exportImageState()); + expect(asyncTi.N).toBe(syncTi.N); + expect(asyncTi.search('walrus').length).toBe(syncTi.search('walrus').length); + expect(asyncTi.search('持久化').length).toBeGreaterThan(0); + } finally { + src.close(); + syncTi.close(); + asyncTi.close(); + } + }); +}); + // ---- integration: build / publish / load ------------------------------------ describe('generation build + load', () => { @@ -1043,3 +1290,120 @@ describe('generation availability triggers (wal-growth + close publish)', () => await db.close(); }, 60000); }); + +describe('open lifecycle status (phase timings + state machine)', () => { + test('a healthy generation open reports generation-load with zero corpus work', async () => { + const dir = await openTmp('lifecycle-healthy'); + let db = await MiniDb.open>({ dir, valueCodec: 'json' }); + await seedIndexedDb(db, 2000); + await db.rebuildGeneration(); + const gen = db.getIndexGeneration()!; + await db.close(); + + // Hard proof that no full-corpus tokenization runs on the attach path: + // the staged builder (the only in-open corpus tokenizer besides the + // bounded rebuild) must not be entered at all. + const buildSpy = vi.spyOn(TextIndex.prototype, 'build'); + try { + db = await MiniDb.open>({ dir, valueCodec: 'json' }); + expect(buildSpy).not.toHaveBeenCalled(); + } finally { + buildSpy.mockRestore(); + } + expect(db.stats.generationLoads).toBe(1); + expect(db.stats.generationIndexRebuilds).toBe(0); + expect(db.stats.indexRebuildDecoded).toBe(0); // no value re-decode + expect(db.stats.textRebuildDurationMs).toBe(0); // no tokenization + + const status = db.lifecycleStatus(); + expect(status.state).toBe('ready'); + expect(status.path).toEqual(['no-generation', 'generation-load', 'wal-catch-up', 'ready']); + expect(status.openedAt).not.toBeNull(); + expect(status.phases.openMs).toBeGreaterThan(0); + expect(status.phases.generationCandidateLoadMs).toBeGreaterThan(0); + expect(status.phases.storeImageLoadMs).toBeGreaterThan(0); + expect(status.phases.postingsIntegrityCheckMs).toBeGreaterThan(0); + expect(status.phases.textImageLoadMs).toBeGreaterThan(0); + expect(status.phases.walScanMs).toBeGreaterThanOrEqual(0); + expect(status.phases.fullRecoveryMs).toBe(0); + expect(status.phases.textRebuildMs).toBe(0); + expect(status.textIndexes).toEqual({ ft: 'image', tri: 'image' }); + expect(status.pendingTextIndexes).toEqual([]); + expect(db.recoveryInfo?.indexGeneration?.id).toBe(gen.id); + expect(db.recoveryInfo?.walDeltaAppliedOps).toBe(0); + assertSeededDb(db, 2000); + await db.close(); + }); + + test('a corrupt generation falls back to full-rebuild (only then is the corpus re-tokenized)', async () => { + const dir = await openTmp('lifecycle-corrupt'); + let db = await MiniDb.open>({ dir, valueCodec: 'json' }); + await seedIndexedDb(db, 2000); + await db.rebuildGeneration(); + const gen = db.getIndexGeneration()!; + await db.close(); + + // Flip one byte inside the generation's store image: the manifest crc + // rejects the candidate and the open must fall back to the legacy full + // recovery — the ONLY path allowed to re-tokenize the corpus. + const storeImage = path.join(dir, 'generations', gen.id, 'store'); + const raw = await fs.readFile(storeImage); + raw[Math.floor(raw.length / 2)]! ^= 0xff; + await fs.writeFile(storeImage, raw); + + db = await MiniDb.open>({ dir, valueCodec: 'json' }); + expect(db.stats.generationLoads).toBe(0); + expect(db.stats.generationLoadFallbacks).toBe(1); + expect(db.stats.lastGenerationFallback).toContain(gen.id); + expect(db.recoveryInfo?.indexGeneration).toBeUndefined(); + expect(db.stats.indexRebuildDecoded).toBeGreaterThan(0); // corpus re-decoded + expect(db.stats.textRebuildDurationMs).toBeGreaterThan(0); // corpus re-tokenized + + const status = db.lifecycleStatus(); + // 2000 docs < TEXT_BUILD_WORKER_MIN_DOCS: the rebuild stays in-open + // (staged), so the instance is fully ready when open() returns. + expect(status.state).toBe('ready'); + expect(status.path).toEqual(['no-generation', 'generation-load', 'full-rebuild', 'ready']); + expect(status.phases.fullRecoveryMs).toBeGreaterThan(0); + expect(status.phases.walScanMs).toBeGreaterThan(0); + expect(status.phases.walApplyMs).toBeGreaterThan(0); + expect(status.phases.textRebuildMs).toBeGreaterThan(0); + expect(status.textIndexes).toEqual({ ft: 'staged', tri: 'staged' }); + assertSeededDb(db, 2000); + await db.close(); + }); + + test('a missing generation with a deferrable corpus reports degraded until the background text build commits', async () => { + const dir = await openTmp('lifecycle-degraded'); + let db = await MiniDb.open>({ dir, valueCodec: 'json' }); + await seedIndexedDb(db, 5000); // >= TEXT_BUILD_WORKER_MIN_DOCS: deferrable + await db.rebuildGeneration(); + await db.close(); + + // No candidate at all: the open starts from 'no-generation'. + await fs.rm(path.join(dir, 'generations'), { recursive: true, force: true }); + await fs.rm(path.join(dir, 'CURRENT'), { force: true }); + + db = await MiniDb.open>({ dir, valueCodec: 'json' }); + expect(db.stats.generationLoads).toBe(0); + const building = db.lifecycleStatus(); + expect(building.state).toBe('degraded'); + expect(building.path).toEqual(['no-generation', 'full-rebuild', 'degraded']); + expect(building.pendingTextIndexes.sort()).toEqual(['ft', 'tri']); + expect(building.textIndexes).toEqual({ ft: 'deferred', tri: 'deferred' }); + expect(db.textIndexBuilding('ft')).toBe(true); + expect(() => db.search('ft', 'hello')).toThrowError(/still building/); + + await waitFor(() => db.lifecycleStatus().state === 'ready', 'deferred text builds'); + const ready = db.lifecycleStatus(); + expect(ready.path).toEqual(['no-generation', 'full-rebuild', 'degraded', 'ready']); + expect(ready.pendingTextIndexes).toEqual([]); + expect(['worker', 'inline']).toContain(ready.textIndexes['ft']); + expect(['worker', 'inline']).toContain(ready.textIndexes['tri']); + expect(ready.phases.textRebuildMs).toBeGreaterThan(0); + expect(db.stats.textDeferredBuilds).toBe(2); + expect(db.stats.textDeferredBuildErrors).toBe(0); + assertSeededDb(db, 5000); + await db.close(); + }, 60000); +}); diff --git a/packages/minidb/test/lock.test.ts b/packages/minidb/test/lock.test.ts index 79d62f6e2..3963aff0d 100644 --- a/packages/minidb/test/lock.test.ts +++ b/packages/minidb/test/lock.test.ts @@ -312,3 +312,40 @@ test('close() waits out a failing in-flight compaction and still cleans up every await fs.rm(dir, { recursive: true, force: true }); } }); + +test('onLockAcquired fires with the held token; read-only fallback opens skip it', async () => { + const dir = await tmpDir(); + let reported; + const db1 = await MiniDb.open({ + dir, + valueCodec: 'string', + onLockAcquired: (info) => { + reported = info.token; + }, + }); + try { + // Fired before open() resolved, and matches the published lock line — + // a supervisor learns the lock identity before any heavy recovery ran. + assert.ok(reported); + const line = JSON.parse(await fs.readFile(path.join(dir, 'db.lock'), 'utf8')); + assert.equal(line.token, reported); + assert.equal(line.pid, process.pid); + + // A read-only fallback open (the lock is busy) never fires the callback. + let second; + const ro = await MiniDb.open({ + dir, + valueCodec: 'string', + onLockFail: 'readonly', + onLockAcquired: (info) => { + second = info.token; + }, + }); + assert.equal(ro.readOnly, true); + assert.equal(second, undefined); + await ro.close(); + } finally { + await db1.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/recovery.test.ts b/packages/minidb/test/recovery.test.ts index 8eae278b7..f197a512f 100644 --- a/packages/minidb/test/recovery.test.ts +++ b/packages/minidb/test/recovery.test.ts @@ -199,6 +199,138 @@ for (const valueMode of ['memory', 'disk'] as const) { }); } +test('catchUpFromWal: a torn tail pauses the catch-up and a later call completes it', async () => { + const dir = await tmpDir(); + try { + const writer = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + for (let i = 0; i < 10; i++) await writer.set(`k${i}`, { n: i }); + const reader = await MiniDb.open({ dir, valueCodec: 'json', readOnly: true }); + const ri = reader.recoveryInfo!; + + // Simulate a writer mid-writev: only a PREFIX of the next frame is on + // disk. The catch-up must stop at the last fully-valid frame without + // erroring, and must not advance the watermark past it. + const frame = encodeFrame({ + type: TYPE_SET, + key: Buffer.from('torn', 'utf8'), + value: Buffer.from(JSON.stringify({ n: 99 }), 'utf8'), + }); + const half = Math.floor(frame.length / 2); + const walPath = path.join(dir, 'db.wal'); + await fs.appendFile(walPath, frame.subarray(0, half)); + const paused = await reader.catchUpFromWal(ri.walScanEnd); + assert.deepEqual(paused, { offset: ri.walScanEnd, appliedFrames: 0 }); + assert.equal(reader.get('torn'), undefined); + + // The writev lands: the tail's CRC now validates and the frame applies. + await fs.appendFile(walPath, frame.subarray(half)); + const done = await reader.catchUpFromWal(ri.walScanEnd); + assert.deepEqual(done, { offset: ri.walScanEnd + frame.length, appliedFrames: 1 }); + assert.deepEqual(reader.get('torn'), { n: 99 }); + + await reader.close(); + await writer.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('catchUpFromWal: a compaction rotation swaps the WAL inode and the catch-up declines', async () => { + const dir = await tmpDir(); + try { + const writer = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + for (let i = 0; i < 100; i++) await writer.set(`k${i}`, { n: i }); + const reader = await MiniDb.open({ dir, valueCodec: 'json', readOnly: true }); + const ri = reader.recoveryInfo!; + assert.ok(ri.walScanEnd > 0); + + // Rotation: compaction swaps db.snapshot/db.wal in two renames, so the + // WAL the reader anchored on is gone — incremental catch-up must decline + // (null) and leave the reopen decision to the caller. + await writer.compact(); + assert.equal(await reader.catchUpFromWal(ri.walScanEnd), null); + + // A fresh read-only reopen sees every write, including post-rotation ones. + await writer.set('post-rotation', { n: 101 }); + await reader.close(); + const reopened = await MiniDb.open({ dir, valueCodec: 'json', readOnly: true }); + assert.equal(reopened.size, 101); + assert.deepEqual(reopened.get('post-rotation'), { n: 101 }); + await reopened.close(); + await writer.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('catchUpFromWal: concurrent calls are serialized instead of interleaving mid-apply', async () => { + const dir = await tmpDir(); + try { + const writer = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + for (let i = 0; i < 50; i++) await writer.set(`k${i}`, { n: i }); + const reader = await MiniDb.open({ dir, valueCodec: 'json', readOnly: true }); + const ri = reader.recoveryInfo!; + for (let i = 50; i < 150; i++) await writer.set(`k${i}`, { n: i }); + + // The sliced async apply yields to the event loop, so two overlapping + // catch-ups at the same watermark would interleave op-by-op without the + // per-instance chain. The second call runs after the first advanced the + // watermark and declines (the caller then falls back to a reopen). + const [first, second] = await Promise.all([ + reader.catchUpFromWal(ri.walScanEnd), + reader.catchUpFromWal(ri.walScanEnd), + ]); + assert.ok(first && first.appliedFrames === 100); + assert.equal(second, null); + assert.equal(reader.size, 150); + + await reader.close(); + await writer.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('catchUpFromWal: a large tail applies in cooperative slices (event loop stays live)', { timeout: 60_000 }, async () => { + const dir = await tmpDir(); + try { + const writer = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + for (let i = 0; i < 100; i++) await writer.set(`pre:${i}`, { n: i, t: `alpha ${i}` }); + await writer.createTextIndex('t', { fields: ['t'] }); + const reader = await MiniDb.open({ dir, valueCodec: 'json', readOnly: true }); + const ri = reader.recoveryInfo!; + // 8000 single-op frames: WAL_APPLY_OPS_PER_SLICE (512) alone forces ~15 + // yields mid-apply even when every op is cheap. + for (let i = 0; i < 8000; i++) await writer.set(`s:${i}`, { n: i, t: `alpha beta ${i}` }); + + // Count event-loop turns observed while the catch-up runs. The async + // scan contributes about one (a sub-window file fills in one read); an + // UNSLICED apply would add none, the sliced one adds one per 512 ops. + let loopTurns = 0; + let catchUpDone = false; + const probe = (async () => { + for (;;) { + await new Promise((r) => setImmediate(r)); + if (catchUpDone) return; + loopTurns++; + } + })(); + const res = await reader.catchUpFromWal(ri.walScanEnd); + catchUpDone = true; + await probe; + + assert.ok(res && res.appliedFrames === 8000); + assert.ok(loopTurns >= 3, `expected the sliced apply to yield repeatedly, observed ${loopTurns} loop turn(s)`); + assert.equal(reader.size, 8100); + assert.ok(reader.search('t', 'beta').length > 0); + + await reader.close(); + await writer.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + // ---- generation pairing (stat-pairing; review #15) -------------------------- // // Fault-injection tests for recover()'s snapshot/WAL generation pairing. The @@ -643,3 +775,65 @@ test('async scanner: resync candidate budget bounds fake-magic storms', async () await fs.rm(dir, { recursive: true, force: true }); } }); + +// ---- phase 3: cooperative slicing of the recovered-op apply loops ----------- + +import { walApplySlicer, WAL_APPLY_OPS_PER_SLICE, WAL_APPLY_SLICE_MS } from '../src/recovery.js'; + +test('walApplySlicer trips on the op budget, then resets; the time budget trips independently', () => { + const slice = walApplySlicer(); + for (let i = 1; i < WAL_APPLY_OPS_PER_SLICE; i++) assert.equal(slice(), false); + assert.equal(slice(), true); // the op budget trips exactly at the cap + assert.equal(slice(), false); // counters reset for the next slice + // The time budget trips below the op cap once the slice ran long enough. + const timed = walApplySlicer(); + assert.equal(timed(), false); // one op, well under the op cap + const t0 = performance.now(); + while (performance.now() - t0 < WAL_APPLY_SLICE_MS + 5) { + // busy-wait: keep this slice over its time budget with ops to spare + } + assert.equal(timed(), true); +}); + +test('a generation WAL delta of large batch frames replays every op through the sliced apply loop', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + db.genBuildKickMinIntervalMs = Number.POSITIVE_INFINITY; // exactly one explicit generation + for (let base = 0; base < 2000; base += 500) { + await db.batch(Array.from({ length: 500 }, (_, i) => ({ op: 'set' as const, key: `b${base + i}`, value: { n: base + i } }))); + } + await db.rebuildGeneration(); + await db.close(); + + // A WAL delta of batch frames past the checkpoint: thousands of primitive + // ops in a handful of frames — frame-granular yielding could never slice + // this; the op/time budgets can. Small values keep the delta far below + // the close-time republish threshold. + db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + const gen = db.getIndexGeneration()?.id; + for (let base = 0; base < 3000; base += 500) { + await db.batch(Array.from({ length: 500 }, (_, i) => ({ op: 'set' as const, key: `d${base + i}`, value: { n: base + i } }))); + } + await db.close(); + assert.equal(db.getIndexGeneration()?.id, gen); // no close-time republish: the delta must be replayed + + db = await MiniDb.open({ dir, valueCodec: 'json', autoCompact: false }); + try { + assert.ok(db.recoveryInfo?.indexGeneration); + assert.equal(db.recoveryInfo?.walDeltaAppliedOps, 3000); + assert.equal(db.size, 5000); + assert.deepEqual(db.get('d0'), { n: 0 }); + assert.deepEqual(db.get('d2999'), { n: 2999 }); + assert.deepEqual(db.get('b1999'), { n: 1999 }); + const status = db.lifecycleStatus(); + assert.equal(status.state, 'ready'); + assert.deepEqual(status.path, ['no-generation', 'generation-load', 'wal-catch-up', 'ready']); + assert.ok(status.phases.walApplyMs > 0); + } finally { + await db.close(); + } + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/store.test.ts b/packages/minidb/test/store.test.ts index 4c3925040..d26ddef2c 100644 --- a/packages/minidb/test/store.test.ts +++ b/packages/minidb/test/store.test.ts @@ -205,3 +205,40 @@ test('reapExpiredDue falls back to a full scan when the heap diverged from the m assert.equal(s.map.size, 0); s.close(); }); + +test('bulkLoadRefsAsync pauses active expiry: a TTL expiring mid-load cannot diverge the ordered index', async () => { + // A TTL that elapses DURING the sliced load: with the active-expire tick + // firing mid-load, the reaped key would vanish from the map but stay in + // the rebuilt ordered index — and a later re-set would insert a SECOND + // ordered entry, producing duplicate scan results. Pausing expiry for the + // load keeps the resulting state identical to the sync bulk load. + const short = 'a-short-ttl'; // sorts before every kNNNNN + const records = [ + { kstr: short, ref: { kind: 'memory' as const, value: B('v0') }, expireAt: Date.now() + 1, dt: null }, + ...Array.from({ length: 20000 }, (_, i) => ({ + kstr: `k${String(i).padStart(5, '0')}`, + ref: { kind: 'memory' as const, value: B(`v${i}`) }, + expireAt: 0, + dt: null, + })), + ]; + const s = new Store({ activeExpireIntervalMs: 1 }); // a tick lands within the first yields + try { + await s.bulkLoadRefsAsync(records, { sliceEvery: 50 }); // 400 yields: the tick fires mid-load + // The divergence repro: re-set the key — a stale ordered entry left by a + // mid-load reap would now show it twice in ordered scans. + s.set(short, B('v1')); + const scanned = [...s.scan()].map((e) => e.key.toString()); + assert.equal(scanned.length, new Set(scanned).size, 'ordered scan must not contain duplicates'); + assert.equal(scanned.length, 20001); + // The ordered index and the map agree exactly (no stale, no missing). + assert.deepEqual([...s.rawKeys()], [...s.map.keys()].sort()); + // And the next tick after the load reaps normally (expiry only deferred). + s.set(short, B('v2'), Date.now() - 1); // already expired + await sleep(20); + assert.equal(s.map.has(short), false); + assert.deepEqual([...s.rawKeys()], [...s.map.keys()].sort()); + } finally { + s.close(); + } +}); diff --git a/packages/minidb/test/worker-build.test.ts b/packages/minidb/test/worker-build.test.ts index eefaaaeb9..428b37115 100644 --- a/packages/minidb/test/worker-build.test.ts +++ b/packages/minidb/test/worker-build.test.ts @@ -22,6 +22,7 @@ import { import { buildTextArtifacts } from '../src/worker/text-build-core.js'; import type { TextBuildCoreSpec } from '../src/worker/text-build-core.js'; import { startWorkerTextBuild, WorkerTextBuildError, verifyFileCrcAsync } from '../src/worker/text-build.js'; +import { WorkerSlots, MaintenanceCancelledError, defaultWorkerSlots } from '../src/maintenance.js'; import { tmpDir, rmrf, waitFor } from './helpers.js'; const cleanups: (() => Promise | void)[] = []; @@ -897,3 +898,101 @@ describe('single-file deployment (worker entry absent)', () => { } }, 120000); }); + +describe('worker slot queue (TUI-safe slot pressure policy)', () => { + test('acquireBounded grants a freed slot, times out to null, and rejects on abort', async () => { + const slots = new WorkerSlots(1); + // Queued: the waiter is granted the slot the holder releases. + const first = slots.tryAcquire(); + expect(first).not.toBeNull(); + let granted = false; + const waiting = slots.acquireBounded(5000).then((r) => { + granted = r !== null; + return r; + }); + await new Promise((r) => setTimeout(r, 20)); + expect(granted).toBe(false); + first!(); + const release = await waiting; + expect(granted).toBe(true); + release!(); + // Timeout: a drought outlasting the wait resolves null (the caller's + // inline-last-resort decision point). + const held = slots.tryAcquire(); + expect(held).not.toBeNull(); + expect(await slots.acquireBounded(20)).toBeNull(); + // Abort: a queued waiter rejects with MaintenanceCancelledError. + const controller = new AbortController(); + const aborted = slots.acquireBounded(5000, controller.signal); + controller.abort(); + await expect(aborted).rejects.toBeInstanceOf(MaintenanceCancelledError); + held!(); + }); + + test('a worker-eligible build queues for a slot under pressure instead of going inline', async () => { + const dir = await openTmp('slot-queue'); + const db = await MiniDb.open>({ dir, valueCodec: 'json' }); + await seedDataOnly(db, 5000); + // Hold every process-wide slot: the build cannot start a worker NOW, and + // the TUI-safe policy must QUEUE it, not drop it onto the main thread. + const held: (() => void)[] = []; + for (let i = 0; i < defaultWorkerSlots.total; i++) { + const r = defaultWorkerSlots.tryAcquire(); + assert(r !== null); + held.push(r); + } + db.textBuildSlotWaitMs = 60_000; // the grant, not the timeout, must end the wait + let settled = false; + const createP = db + .createTextIndex('ft', { fields: ['text'] }) + .then(() => { + settled = true; + }) + .catch((e) => { + settled = true; + throw e; + }); + // startInline records its fallback synchronously, so after a settle-free + // window with no fallback stat the build provably did NOT go inline. + await new Promise((r) => setTimeout(r, 200)); + expect(settled).toBe(false); + expect(db.stats.textWorkerFallbacks).toBe(0); + expect(db.stats.textWorkerBuilds).toBe(0); + // Free a slot: the queued build takes it and completes through the worker. + held[0]!(); + try { + await createP; + } finally { + for (const r of held.slice(1)) r(); + } + expect(db.stats.textWorkerBuilds).toBe(1); + expect(db.stats.textWorkerFallbacks).toBe(0); + expect(db.search('ft', 'hello').length).toBeGreaterThan(0); + await db.close(); + }, 60000); + + test('a persisted slot drought hosts the bounded inline core (explicit last resort)', async () => { + const dir = await openTmp('slot-drought'); + const db = await MiniDb.open>({ dir, valueCodec: 'json' }); + await seedDataOnly(db, 5000); + db.textBuildSlotWaitMs = 50; // the drought outlasts the queue wait + const held: (() => void)[] = []; + for (let i = 0; i < defaultWorkerSlots.total; i++) { + const r = defaultWorkerSlots.tryAcquire(); + assert(r !== null); + held.push(r); + } + try { + await db.createTextIndex('ft', { fields: ['text'] }); + } finally { + for (const r of held) r(); + } + // The bounded inline core built the index (never the unbounded staged + // aggregation), and the fallback is accounted explicitly. + expect(db.stats.textWorkerBuilds).toBe(0); + expect(db.stats.textWorkerFallbacks).toBe(1); + expect(db.stats.lastTextWorkerFallback).toBe('slot-pressure'); + expect(db.search('ft', 'hello').length).toBeGreaterThan(0); + await db.close(); + }, 60000); +}); diff --git a/packages/node-sdk/src/sdk-rpc-client-v2.ts b/packages/node-sdk/src/sdk-rpc-client-v2.ts index de0da25c7..a0d82e662 100644 --- a/packages/node-sdk/src/sdk-rpc-client-v2.ts +++ b/packages/node-sdk/src/sdk-rpc-client-v2.ts @@ -158,6 +158,8 @@ import { applyPromptMetadataUpdate, bootstrap, DEFAULT_AGENT_PROFILE_NAME, + drainQueryStoreDisposals, + drainSessionIndexMirror, ensureKimiHome, ensureMainAgent, IAgentActivityView, @@ -187,6 +189,7 @@ import { ISessionCronService, ISessionExportService, ISessionIndex, + ISessionIndexMirror, ISessionInitService, ISessionMcpHandle, ISessionMetadata, @@ -487,7 +490,14 @@ export class SDKRpcClientV2 extends SDKRpcClientBase { subscription.dispose(); } await this.klient.close(); + // Same shutdown order as kap-server: drain the session-index mirror while + // the query store is still open, then await the asynchronous closes that + // disposal fires — a host that removes homeDir right after close() must + // not race an in-flight shard close (ENOTEMPTY on teardown). + await this.app.accessor.get(ISessionIndexMirror).drain(); this.app.dispose(); + await drainSessionIndexMirror(); + await drainQueryStoreDisposals(); } /** diff --git a/packages/node-sdk/test/list-sessions.test.ts b/packages/node-sdk/test/list-sessions.test.ts index 02c17192e..8ee0e60bd 100644 --- a/packages/node-sdk/test/list-sessions.test.ts +++ b/packages/node-sdk/test/list-sessions.test.ts @@ -9,9 +9,15 @@ import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from 'node:fs/promise import { tmpdir } from 'node:os'; import { basename, dirname, join } from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createKimiHarness } from '#/index'; +import { + drainQueryStoreDisposals, + drainSessionIndexMirror, + ISessionIndex, +} from '@moonshot-ai/agent-core-v2'; + +import { createKimiHarness, SDKRpcClientV2 } from '#/index'; import type { KimiError } from '#/index'; import { @@ -430,3 +436,74 @@ describe('KimiHarness.listSessions', () => { } }); }); + +describe('SDKRpcClientV2 search-index separation', () => { + // The global full-text search database (`/search-index`) belongs + // to the kap-server search surface. The TUI-side chain (rpc client → + // klient → `ISessionIndex`) must list, resume and continue sessions without + // ever opening it — including while the session read model is still + // preparing. + + it('listSessions / resumeSession never open the global search index (read model off)', async () => { + vi.stubEnv('KIMI_CODE_EXPERIMENTAL_FLAG', '0'); + vi.stubEnv('KIMI_CODE_EXPERIMENTAL_PERSISTENCE_MINIDB_READMODEL', '0'); + const homeDir = await makeTempDir(); + const workDir = await makeTempDir(); + const client = new SDKRpcClientV2({ homeDir, identity: TEST_IDENTITY }); + + try { + const created = await client.createSession({ id: 'ses_search_sep_off', workDir }); + await client.closeSession({ sessionId: created.id }); + + const sessions = await client.listSessions({ workDir }); + expect(sessions.map((item) => item.id)).toEqual([created.id]); + const resumed = await client.resumeSession({ id: created.id }); + expect(resumed.id).toBe(created.id); + + expect(existsSync(join(homeDir, 'search-index'))).toBe(false); + // With the read model off, the session query-store is never opened either. + expect(existsSync(join(homeDir, 'cache', 'query-store'))).toBe(false); + } finally { + await client.close(); + vi.unstubAllEnvs(); + } + }); + + it('listSessions / resumeSession never open the global search index (read model on)', async () => { + vi.stubEnv('KIMI_CODE_EXPERIMENTAL_FLAG', '0'); + vi.stubEnv('KIMI_CODE_EXPERIMENTAL_PERSISTENCE_MINIDB_READMODEL', '1'); + const homeDir = await makeTempDir(); + const workDir = await makeTempDir(); + const client = new SDKRpcClientV2({ homeDir, identity: TEST_IDENTITY }); + + try { + const created = await client.createSession({ id: 'ses_search_sep_on', workDir }); + await client.closeSession({ sessionId: created.id }); + + // The read model is still preparing here: the first list kicks the + // background projection and answers from authoritative metadata, the + // resume reads the authoritative document — neither waits for, nor + // opens, any full-text index. + const sessions = await client.listSessions({ workDir }); + expect(sessions.map((item) => item.id)).toEqual([created.id]); + const resumed = await client.resumeSession({ id: created.id }); + expect(resumed.id).toBe(created.id); + + expect(existsSync(join(homeDir, 'search-index'))).toBe(false); + + // Settle the kicked projection before close so teardown never races it, + // and prove the read model really did engage (the flag took effect). + const status = await client.engineAccessor.get(ISessionIndex).prepare(); + expect(status.state).toBe('ready'); + expect(existsSync(join(homeDir, 'cache', 'query-store'))).toBe(true); + expect(existsSync(join(homeDir, 'search-index'))).toBe(false); + } finally { + await client.close(); + // Dispose fired the mirror/query-store async closes; await them before + // the shared afterEach removes the temp home. + await drainSessionIndexMirror(); + await drainQueryStoreDisposals(); + vi.unstubAllEnvs(); + } + }); +}); diff --git a/packages/node-sdk/test/sdk-rpc-client-v2.test.ts b/packages/node-sdk/test/sdk-rpc-client-v2.test.ts index aa2a3b2e5..160131655 100644 --- a/packages/node-sdk/test/sdk-rpc-client-v2.test.ts +++ b/packages/node-sdk/test/sdk-rpc-client-v2.test.ts @@ -23,7 +23,11 @@ import { type KimiConfig, } from '#/index'; import { foldAgentWireReplay } from '#/v2/resume-replay'; -import { IHostRequestHeaders } from '@moonshot-ai/agent-core-v2'; +import { + drainQueryStoreDisposals, + drainSessionIndexMirror, + IHostRequestHeaders, +} from '@moonshot-ai/agent-core-v2'; import { TEST_IDENTITY } from './test-identity'; import { recordingTelemetry, type TelemetryRecord } from './telemetry'; @@ -31,6 +35,10 @@ import { recordingTelemetry, type TelemetryRecord } from './telemetry'; const tempDirs: string[] = []; afterEach(async () => { + // The read-model mirror/query-store close asynchronously on dispose; await + // the drains so the rm below never races their final flush (ENOTEMPTY). + await drainSessionIndexMirror(); + await drainQueryStoreDisposals(); for (const dir of tempDirs.splice(0)) { await rm(dir, { recursive: true, force: true }); }