diff --git a/flake.nix b/flake.nix index c50d0474e..4a00e0b83 100644 --- a/flake.nix +++ b/flake.nix @@ -71,6 +71,7 @@ ./packages/kimi-migration-legacy ./packages/kosong ./packages/migration-legacy + ./packages/minidb ./packages/node-sdk ./packages/oauth ./packages/protocol @@ -91,6 +92,7 @@ "@moonshot-ai/server-e2e" "@moonshot-ai/kaos" "@moonshot-ai/kosong" + "@moonshot-ai/minidb" "@moonshot-ai/migration-legacy" "@moonshot-ai/kimi-code-sdk" "@moonshot-ai/kimi-code-oauth" diff --git a/packages/minidb/.gitignore b/packages/minidb/.gitignore new file mode 100644 index 000000000..e1927171a --- /dev/null +++ b/packages/minidb/.gitignore @@ -0,0 +1,9 @@ +# External reference implementations (not part of the package) +references/ + +# Runtime database files produced by minidb during tests / local runs +*.wal +*.snapshot +*.snapshot.tmp +*.wal.tmp +data/ diff --git a/packages/minidb/DESIGN_NOTES.md b/packages/minidb/DESIGN_NOTES.md new file mode 100644 index 000000000..984b2b479 --- /dev/null +++ b/packages/minidb/DESIGN_NOTES.md @@ -0,0 +1,285 @@ +# minidb — Design Notes + +A pure-Node.js (no native addons) embedded key-value database that mixes +**Redis** (in-memory KV speed, data structures, AOF/RDB persistence) with +**SQLite** (durable single-file persistence, WAL, indexed queries). + +This document distills what we learned by reading real database source code, and +records the concrete decisions for our implementation. + +## Sources studied + +| Source | Why we read it | Key takeaways | +|---|---|---| +| **Redis** (`references/redis`) | Gold standard for in-memory KV + AOF/RDB + data structures | dict incremental rehashing, skiplist with `span`, lazy+sampled expiration, RDB framing, AOF rewrite, RESP | +| **SQLite** WAL doc (`sqlite.org/wal.html`) | WAL semantics done right | append-only changes + commit record, checkpoint, single writer / multi reader, read-degradation vs WAL size, `synchronous` FULL/NORMAL | +| **NeDB** (`references/nedb`) | Pure-JS embedded DB — closest to our stack | append-only log + tombstones + last-writer-wins replay, in-memory AVL secondary indexes, temp+rename compaction, single-concurrency executor, torn-tail tolerance | +| **Bitcask** (`references/bitcask`) | Log-structured hash KV — our storage model | entry format, in-memory keydir (key→offset), append+rotate, merge/compaction, hint-file recovery | +| **cstack/db_tutorial** (`references/db_tutorial`) | Tiny SQLite clone (paged B-tree) | page cache, B-tree node split/merge, cursor; also clarifies what a paged engine costs vs a log | + +--- + +## 1. Constraints + +- **Pure Node.js**, zero native addons. Only built-ins: `node:fs`, + `node:worker_threads`, `node:buffer`, `node:net`, etc. +- **Single-writer, many-reader** within one process (like SQLite WAL). No + multi-process locking in v1. +- Embedded library first; optional TCP server later. + +## 2. Storage model: log-structured, in-memory index + +We deliberately choose a **log-structured** engine (Bitcask/Redis-AOF style) +over a **paged B-tree** (SQLite style): + +- The db_tutorial clone shows the real cost of a paged engine: fixed page + frames + cache, in-place overwrite, node split/merge, parent pointers, + separator keys, root special-casing — and that toy *still* has no WAL, fsync + or crash recovery. That is a lot of complexity for our target. +- A log-structured store gives **sequential writes** (the fastest disk access), + trivial **recovery by replay**, and simple crash safety — at the cost of + needing an in-memory index and a compaction strategy, and range scans being + harder (we solve those with secondary indexes + a skiplist). + +So: **writes append to a log; compaction rewrites.** Reads can then be served +in one of two modes. + +We borrow Bitcask's record format, rotation, copy-out merge, and truncate-tail +recovery. In the default `valueMode: 'memory'`, we keep **values in memory** +like Redis (dataset must fit in RAM), which makes reads O(1) with zero disk +I/O. In `valueMode: 'disk'`, we follow Bitcask more closely: the store keeps +only keys/metadata plus a `(file, offset, len)` pointer, and reads value bulk +from the snapshot/WAL with synchronous positioned reads so datasets can exceed +RAM without changing the synchronous public API. `valueMode: 'auto'` chooses +between the two at startup by comparing the current snapshot/WAL size against +`maxMemoryBytes` (falling back to `memory` when no budget is set). + +## 3. On-disk record format (WAL + snapshot) + +Borrowed from Bitcask's entry layout and Redis RDB's typed, length-prefixed +encoding. Every record is a self-contained frame: + +``` + off size field + 0 2 magic = "MD" (0x4D 0x44) + 2 1 type 1=SET, 2=DEL(tombstone) + 3 1 flags reserved + 4 2 keyLen uint16 + 6 4 valLen uint32 (0 for DEL) + 10 4 metaLen uint32 (0 if none) + 14 8 expireAt int64 ms since epoch, 0 = none + 22 keyLen key + 22+k valLen value + 22+k+v metaLen meta optional metadata blob (dt columns, …) + 22+k+v+m 4 crc32 CRC-32 trailer over [type..meta] +``` + +- **Fixed 22-byte header + 4-byte CRC trailer** → a reader computes the full + frame length (`22 + keyLen + valLen + metaLen + 4`) before reading the + payload (mirrors RDB's length-prefixed idea). The layout matches Bitcask's + `[klen][vlen][key][val][crc32]` records, extended with an optional `meta` + blob for top-level datetime columns; CRC-as-trailer verifies a frame in a + single pass. +- **CRC-32** per frame → recovery detects a torn/corrupt tail and stops cleanly. + (NeDB tolerates a torn last line by ignoring it; we enforce integrity with CRC + and truncate at the first bad frame.) +- **Big-endian vs little-endian**: little-endian (matches Node/x86). + +See `src/codec.ts`. + +## 4. WAL + group commit + fsync policies + +Borrowed from Redis AOF and SQLite WAL: + +- Writes are appended to the active WAL **through a buffer**; frames accumulate + and are flushed in batches (**group commit**) → one `write` syscall covers many + ops. This is the single biggest throughput win in pure Node. +- **fsync policy** (configurable, exactly Redis's three modes): + - `always` — `fsync` after every write (safest, slowest). + - `everysec` — flush + `fsync` on a 1s timer (good default; ≤1s loss window). + - `no` — let the OS flush (fastest, may lose seconds on power loss). +- Single-writer: Node's event loop already serializes JS execution, so in-memory + index updates and WAL appends are naturally ordered. We additionally serialize + the async flush so frames never reach disk out of order (NeDB's + concurrency-1 executor is the same idea; we get it for free from the event + loop plus an explicit flush gate). + +See `src/wal.ts`. + +## 5. Snapshot + compaction + +Borrowed from Redis RDB/`BGREWRITEAOF` and Bitcask's merge. Compaction is +**non-blocking for writes**: the live WAL doubles as a Redis-style rewrite +buffer (`aof_rewrite_buf`), so the (slow) snapshot is written while writers +keep appending, and they pause only for a brief final rotation. + +- When the WAL exceeds a size threshold, compact (rewrite state): + 1. **Fence.** Flush the WAL and record `baseOffset = wal.size`. Every write + durable at/before `baseOffset` is already reflected in the store + (`applyOp` runs synchronously in the same tick as `wal.append`, before the + op awaits the WAL write). + 2. **Snapshot.** Write the live store to `db.snapshot.tmp` as a sequence of + SET frames (tombstones dropped), then `fsync`. This phase is + **non-blocking**: writers keep appending to the live WAL and mutating the + store while we iterate. The snapshot need not be point-in-time — the WAL + tail below repairs any fuzziness on replay. + 3. **Pre-copy the tail.** Stream `WAL[baseOffset .. head]` into `db.wal.tmp`, + looping until the remaining delta is small. Also non-blocking: post-fence + writes accumulate in the live WAL instead of a separate in-memory buffer. + 4. **Rotate** (the only blocking phase, and brief): set `_rotateLock` so new + writers park, flush, copy the tiny remaining tail delta, close the old + WAL, `rename` the snapshot into place, `rename` the new WAL into place, + and reopen it. The two renames are ordered **snapshot-first** so a crash + between them pairs the new snapshot with the old full WAL — replaying the + whole old WAL on top of the new snapshot is idempotent for pre-fence + frames and correct for post-fence frames, so the state stays consistent. + Reversing the order would pair an old snapshot with a truncated new WAL + and lose pre-fence data. +- Reads are unaffected throughout. Writes pause only for the rotation critical + section (one flush, a small copy, two renames + two `fsyncDir`s); the bulk + snapshot + tail copy happen concurrently with writes. Recovery is always + `load snapshot + replay WAL`, last-writer-wins, which is exactly what makes + the non-point-in-time snapshot converge to the correct latest state. +- A writer's gate check (`if (_rotateLock) await _rotateLock`) and its + `wal.append()` are in the same synchronous segment, and `_rotateLock` is set + synchronously before the rotation flush — so a writer either enqueued its + frame before the lock (and is drained by the flush) or parks on the lock + without appending. The event loop cannot interleave between the two, which + is why the critical section is quiescent after the flush and cannot deadlock. +- The snapshot encoder runs on the main thread but **yields to the event loop + every N entries** (chunked), so large snapshots stay responsive without a + Worker. Offloading the encode to a `worker_thread` is the planned + optimization; the snapshot is already consistent without it. + +See `src/snapshot.ts`. + +## 6. Recovery + +1. Load the latest `db.snapshot` if present (CRC-checked frames) → rebuild map. +2. Replay the WAL from the snapshot point → apply SET/DEL, last-writer-wins. +3. **Corruption handling** (`recovery` option): + - `resync` (default): a frame that fails CRC is **skipped** and the parser + resynchronizes to the next valid frame (scan for the `MD` magic, re-validate + with CRC). Only the corrupted bytes are lost; everything after the next + valid frame is recovered. Each lost byte range is reported in + `recoveryInfo.corruptRanges`. A torn tail (corruption reaching EOF) is + **truncated**. A random byte sequence passing magic + length + CRC-32 is + ~1/2³², so a revalidated frame is genuine. + - `strict`: stop at the first bad frame and treat the entire tail as lost (the + classic truncate-at-first-error behavior). + +Either way the database opens in a consistent state; at most the last +un-`fsync`'d records or the individually corrupt frames are lost. + +See `src/recovery.ts`. + +## 7. In-memory primary index + +- `Map`. V8's `Map` already gives us, for + free, the equivalent of Redis's **incrementally-rehashed, power-of-two, + load-factor-managed hash table** (ht_table[2] + rehashidx + SipHash). We do + not reimplement it — that is the big win of targeting Node. +- Values are stored as `Buffer` to keep the hot path allocation-light; a higher + layer handles JS-value encoding. + +## 8. Expiration (TTL) + +Directly ported from Redis: + +- **Lazy**: every `get` checks `expireAt` and deletes if past. +- **Active**: a periodic timer samples a bounded number of keys (Redis samples + ~20 per cycle under a CPU time limit) and deletes expired ones — incremental, + non-blocking, friendly to the event loop. +- A small **min-heap** of expirations lets us short-circuit when nothing is due. + +## 9. Secondary indexes + range queries + +- Equality index: `Map>` (NeDB indexes field values, + including per-element for arrays — we support the same). +- Range / order index: a **skip list** per indexed field, copied from Redis's + `zskiplist`: node `{ key, backward, level[]{forward, span} }`, max level 32, + P=0.25. The `span` field gives **O(log N) rank access** and efficient ordered + range scans (`range`, `order by`, `limit`) without a full scan. +- Unique/sparse indexes with rollback on violation (NeDB pattern). + +## 9b. Document model & query layer (MongoDB-like subset) + +Records are `{ key, value, dt1..dtN }`: + +- **key** — string ≤ 128. Kept in a hash `Map` (O(1) point) **and** a + string-ordered skip list (O(log N) range / prefix / ordered scan). +- **value** — any JSON. Queried by a zero-dep path/filter/projection engine + (`getPath`/`match`/`project`) supporting Mongo-like operators + (`$gt $in $regex $contains $and $or …`) and dot/bracket paths. +- **dt columns** — top-level datetime columns (`dt1..dtN`), each a numeric + (epoch-ms) skip list for O(log N) range. Stored in the frame's optional + `meta` blob (`{ dt }`), separate from `value`. + +Plus a **full-text inverted index** with a Latin + CJK unigram/bigram tokenizer +and TF-IDF ranking. The inverted index is **larger-than-RAM**: the bulk (every +`(doc, term)` posting) lives in an on-disk postings file (`db.text-.postings`, +delta+varint compressed, CRC-framed), while only the small term dictionary +(`Map`), per-doc lengths, and `key↔docID` maps stay in +RAM. Writes go to an in-memory `delta` and deletes set a tombstone; `search` +reads each query term's postings from disk (or a small LRU cache), merges the +delta, drops tombstones, and scores — synchronously, so `db.search()` / +`db.query()` keep their sync API. See `src/text-index.ts` and +`src/text-postings.ts`. + +`db.query(q)` composes all four: it intersects candidate key sets from the key +range, dt range, and text search, then applies the value filter, sort, skip, +limit, and projection. Indexed dimensions are fast; an unindexed value filter +degrades to a full scan (same as Mongo without an index). + +Indexes are pure derived state, rebuilt from the store on startup (definitions +persisted in `db.indexes.json` / `db.textindexes.json`). The equality/range/dt/ +compound indexes are in-memory; the full-text index keeps only its small +dictionary + metadata in memory and stores its postings on disk (rewritten from +the store on open and on compaction), so a crash never loses it — it is simply +rebuilt. + +## 10. Concurrency model + +- **Main thread**: all GET/SET/DEL and index maintenance — lock-free by virtue + of the single-threaded event loop. Snapshot encoding is chunked + yielding so + it does not starve other work. +- **Single writer** at process scope, like SQLite WAL. No multi-process locking + in v1. +- A future `worker_thread` can offload snapshot encoding / compression / heavy + CRC if needed (Node's analog of Redis's `fork()`/COW). + +## 11. Optional server + +A small **RESP-like** TCP server (length-prefixed, CRLF-framed, binary-safe — +trivial per Redis's `rio.c`) so existing Redis clients can talk to minidb. + +## 12. Module map + +``` +src/ + index.ts public API: open/get/set/del/mget/mset/expire/range/... + codec.ts frame encode/decode + CRC streaming parser [done] + crc32.ts table-based CRC-32 [done] + wal.ts buffered append + group commit + fsync policy + store.ts in-memory primary index + TTL (lazy + active) + snapshot.ts chunked snapshot writer (yields to event loop) + recovery.ts load snapshot + replay WAL + truncate torn tail + compaction.ts threshold checks + rewrite/rotate trigger + skiplist.ts comparator-driven skip list (string + numeric) with span + index-manager.ts value-field secondary indexes (equality + range) + compound-index.ts compound indexes (groupBy + orderBy, multiple dt columns) + dt-index.ts datetime-column range indexes (per-column skip list) + query.ts value path / Mongo-like filter / projection / sort + text-index.ts full-text index (CJK n-gram + TF-IDF): in-RAM dictionary + delta + tombstones + text-postings.ts on-disk postings file (delta+varint + CRC) for the text index + lockfile.ts exclusive file lock + stale-lock recovery + read-only + server.ts optional RESP TCP server (redis-cli compatible) +``` + +## 13. Roadmap + +1. **MVP**: Map index + get/set/del + WAL (`everysec`) + recovery replay. +2. **Durability**: snapshot + WAL rewrite/compaction (Worker). +3. **Queries**: secondary indexes + skiplist range. +4. **Server**: RESP-like protocol. +5. **Extras**: eviction (LRU/LFU), compression, sharding. diff --git a/packages/minidb/README.md b/packages/minidb/README.md new file mode 100644 index 000000000..adb55ea97 --- /dev/null +++ b/packages/minidb/README.md @@ -0,0 +1,448 @@ +# minidb + +A **pure-Node.js** (zero native addons) embedded key-value database that mixes +ideas from **Redis** (in-memory KV speed, data structures, AOF-style rewrite) +and **SQLite** (durable single-file persistence, WAL, indexed queries). +Written in **TypeScript**, with strict types and zero runtime dependencies. + +> Built by studying the real source code of Redis, SQLite, NeDB, Bitcask, and a +> tiny SQLite clone — see [`DESIGN_NOTES.md`](./DESIGN_NOTES.md) for what each +> one taught us. + +## Features + +- **O(1) GET/SET** from an in-memory index (`Map`), values as bytes. +- **Optional larger-than-RAM values**: `valueMode: 'disk'` keeps only value + pointers in RAM and reads value bulk from the snapshot/WAL with synchronous + positioned reads. +- **Durable**: append-only **WAL** with group commit + three fsync policies + (`always` / `everysec` / `no`), exactly like Redis AOF. +- **Crash-safe recovery**: CRC-framed records; a torn tail from a crash is + detected and truncated automatically. +- **Snapshots + compaction** (WAL rewrite) bound disk growth and speed up + restart. +- **TTL**: lazy + active expiration (Redis-style, heap-backed). +- **Secondary indexes**: equality and **range** indexes over JSON documents, + backed by a Redis-style **skip list** with O(log N) rank/range; unique and + sparse supported; array fields indexed per element. +- **RESP server** (optional): talk to it with `redis-cli` / `ioredis`. +- **Value codecs**: `buffer`, `string`, or `json`. +- **Zero dependencies**, pure `node:*` built-ins. + +## Install / layout + +Written in **TypeScript**, zero runtime dependencies. Source is in `src/`; build +output goes to `dist/` (generated by `npm run build`). + +``` +src/ + index.ts public API (MiniDb) + types + codec.ts binary frame encode/decode + CRC parser + resync + crc32.ts table-based CRC-32 + wal.ts buffered append + group commit + fsync policies + store.ts in-memory index + TTL (lazy + active) + value refs + value-reader.ts synchronous positioned reads for disk-backed values + snapshot.ts chunked snapshot writer + recovery.ts load snapshot + replay WAL + resync/truncate + compaction.ts WAL rewrite / rotation + skiplist.ts comparator-driven skip list with span + index-manager.ts secondary indexes (equality + range) + dt-index.ts datetime-column indexes + query.ts jq-like value query engine + text-index.ts full-text index (in-RAM dictionary + on-disk postings) + text-postings.ts on-disk postings file for the full-text index + lockfile.ts exclusive file lock + read-only + server.ts optional RESP TCP server +``` + +```bash +npm run build # compile src/ -> dist/ (JS + .d.ts) +npm run typecheck # tsc --noEmit (strict) +``` + +## Quick start (embedded) + +```ts +import { MiniDb } from 'minidb'; + +const db = await MiniDb.open({ + dir: './data', + valueCodec: 'string', // 'buffer' | 'string' | 'json' + fsyncPolicy: 'everysec', // 'always' | 'everysec' | 'no' +}); + +await db.set('hello', 'world'); +db.get('hello'); // 'world' + +await db.set('temp', 'x', { ttl: 5000 }); // expires in 5s +db.ttl('temp'); // remaining ms (-1 none, -2 missing) + +await db.del('hello'); +await db.close(); // flush + fsync + close +``` + +> Within this repo, import from `./src/index.ts` (run with `tsx`). As an installed +> package, import from `'minidb'` (the built `dist/` output). + +Re-open the same `dir` and your data is recovered from the snapshot + WAL. + +### Typed usage + +`MiniDb` is generic over the value type: + +```ts +interface User { name: string; age: number; city: string } +const db = await MiniDb.open({ dir: './data', valueCodec: 'json' }); +await db.set('u1', { name: 'Ann', age: 30, city: 'Paris' }); +const u = db.get('u1'); // User | undefined +``` + +### JSON + secondary indexes + +```js +const db = await MiniDb.open({ dir: './data', valueCodec: 'json' }); + +await db.createIndex('byCity', { field: 'city' }); // equality +await db.createIndex('byAge', { field: 'age', type: 'range' }); // range (skip list) +await db.createIndex('byMail', { field: 'email', unique: true }); // unique + +await db.set('u1', { name: 'Ann', city: 'Paris', age: 30, email: 'a@x.com' }); +await db.set('u2', { name: 'Bob', city: 'Paris', age: 41, email: 'b@x.com' }); + +db.findEq('byCity', 'Paris'); // [{ key:'u1', value:{...} }, { key:'u2', ... }] +db.findRange('byAge', { min: 30, max: 40, count: 10 }); +``` + +Index definitions are persisted and rebuilt from the store on startup. + +## Document model & querying (MongoDB-like subset) + +A record is: + +```js +{ key: 'user:1234', value: { ...any JSON... }, dt1..dtN: } +``` + +- **key** — string ≤ 128 chars, unique, **ordered** (range / prefix / ordered scan). +- **value** — any JSON object (jq-like filter + projection + full-text). +- **dt1..dtN** — any number of top-level **datetime columns**, each indexed for + O(log N) range queries. Pass them as `{ dt: { created: Date.now() } }` (ISO + strings or epoch ms). + +### Key scans (ordered) + +```js +db.scan({ gte: 'user:1', lte: 'user:9', limit: 100 }); // range in key order +db.prefix('user:'); // prefix scan +``` + +### Datetime column queries + +```js +await db.set('a', { n: 1 }, { dt: { created: '2024-01-01' } }); +db.dtColumns(); // ['created'] +db.dtRange('created', { gte: t0, lte: t1 }); // O(log N), [{ key, value, dt, dtValue }] +``` + +### Unified query + +`db.query(q)` composes key range, dt range, full-text, and a value filter: + +```js +db.query({ + key: { prefix: 'post:' }, + dt: { created: { gte: t0, lte: t1 } }, + text: { index: 'body', q: '北京', op: 'AND' }, + filter: { age: { $gte: 18 }, $or: [{ city: 'Paris' }, { city: 'London' }] }, + sort: { age: -1 }, + project: ['name', 'age'], + skip: 0, limit: 20, +}); +``` + +Filter operators: `$eq $ne $gt $gte $lt $lte $in $nin $regex $exists $contains +$type`, plus `$and $or $nor $not`. Paths use dot/bracket notation +(`"address.city"`, `"tags[0]"`). + +> Indexed dimensions (key / dt / text / a `createIndex`'d field) are fast; +> `db.query()` automatically uses matching equality/range value indexes for simple +> top-level predicates (including `$and`), while a value `filter` with no matching +> index falls back to a full collection scan, exactly like MongoDB without an index. + +### Full-text search + +```js +await db.createTextIndex('body', { fields: ['bio'] }); // fields optional (default: all strings) +db.search('body', 'hello 世界', { op: 'AND' }); // [{ key, value, score }] +``` + +Latin words + CJK unigram/bigram tokenization (no dictionary, zero deps), with +TF-IDF ranking. + +## API + +| Method | Description | +|---|---| +| `MiniDb.open({ dir, valueCodec?, valueMode?, fsyncPolicy?, compactThresholdBytes?, autoCompact?, activeExpireIntervalMs?, recovery?, readOnly?, onLockFail?, maxMemoryBytes?, maxMemoryPolicy? })` | Open / create a database | +| `MiniDb.restore(srcDir, destDir, opts?)` | Restore a `db.backup()` directory and open it | +| `MiniDb.openOrRebuild(opts, { onRebuild? })` | Open; on corruption, discard + reopen empty (cache use). Never deletes a live-locked db | +| `get(key)` | Decoded value or `undefined` | +| `set(key, value, { ttl? })` | Set; `ttl` in ms. Resolves per fsync policy | +| `del(key)` | Delete; returns `true` if it existed | +| `has(key)`, `size` | Membership / live key count | +| `mget([keys])`, `mset([[k,v],...])` | Batch read / write | +| `batch([{op:'set'|'del', key, value?, ttl?, dt?}])` | Atomically commit multiple ops (all-or-nothing) | +| `expire(key, ttlMs)`, `ttl(key)` | Set / read TTL | +| `createIndex(name, { field, type?, unique?, sparse? })` | Secondary index (json codec) | +| `dropIndex(name)`, `listIndexes()` | Manage indexes | +| `findEq(name, value)`, `findRange(name, opts)` | Query value indexes | +| `createCompoundIndex(name, { groupBy, orderBy, orderType? })` | Compound index (group + order, e.g. workspace + updatedAt) | +| `compoundRange(name, groupValue, opts)` | Ordered range within a group, O(log N + limit), no full sort | +| `dropCompoundIndex(name)`, `listCompoundIndexes()` | Manage compound indexes | +| `scan({ gte, gt, lte, lt, limit, reverse })` | Ordered key range scan | +| `prefix(p, limit?)` | Key prefix scan | +| `dtColumns()`, `dtRange(col, opts)` | Datetime column range query | +| `query({ key, dt, text, filter, project, sort, skip, limit })` | Unified Mongo-like query | +| `createTextIndex(name, { fields? })`, `dropTextIndex(name)` | Full-text index | +| `search(name, q, { op?, limit? })` | Full-text search | +| `compact()` | Force a snapshot + WAL rewrite now | +| `backup(destDir, { compact? })` | Write a consistent online backup directory | +| `close()` | Flush, fsync, close | + +### Durability (`fsyncPolicy`) + +| Policy | Guarantees | Relative speed | +|---|---|---| +| `always` | fsync after every flush | slowest (safest) | +| `everysec` | fsync once per second (default) | fast, ≤1s loss window | +| `no` | OS decides when to flush | fastest, may lose seconds on power loss | + +### Value storage mode (`valueMode`) + +`valueMode` controls where value bulk lives: + +```js +const db = await MiniDb.open({ + dir: './data', + valueCodec: 'json', + valueMode: 'memory', // 'memory' | 'disk' | 'auto' +}); +``` + +- `memory` (default): values are kept in RAM, Redis-style. Reads are memory-bound. +- `disk`: values stay inline in the durable snapshot/WAL, while RAM keeps only + keys, metadata, indexes, and small `{ file, off, len }` value pointers. Reads + use synchronous positioned reads, so the public API stays synchronous. This + allows value bulk to exceed RAM, at the cost of disk reads on cold `get()` + paths. +- `auto`: at startup, compare the current `db.snapshot` + `db.wal` size with + `maxMemoryBytes`. If the persisted files exceed the budget, open as `disk`; + otherwise open as `memory`. If `maxMemoryBytes` is not set, `auto` falls back + to `memory`. + +### Memory limits + +With `valueMode: 'memory'` the budget bounds keys plus values. With +`valueMode: 'disk'` it bounds the in-RAM key/metadata/index footprint, not the +value bulk. You can bound writes with an approximate memory budget: + +```js +const db = await MiniDb.open({ + dir: './data', + valueCodec: 'json', + maxMemoryBytes: 512 * 1024 * 1024, + maxMemoryPolicy: 'reject', // or 'evict-lru' +}); +``` + +`reject` throws when a write would exceed the budget; `evict-lru` durably deletes +least-recently-used keys first. In `valueMode: 'disk'`, `evict-lru` trims keys to +bound the metadata/index footprint; it does not delete value bytes from disk +because values are already stored outside RAM. `db.stats.evictions` and +`db.stats.maxMemoryRejections` track the behavior. + +### Online backup / restore + +```js +await db.backup('./backup'); +const restored = await MiniDb.restore('./backup', './restored', { valueCodec: 'json' }); +``` + +`backup()` flushes the WAL, optionally compacts, and copies the snapshot, WAL, +index definitions, and text postings while writers are briefly parked, producing +a consistent directory that `restore()` can reopen. + +### Write throughput & SSD endurance + +MiniDb's write path is append-only and sequential, which is already +SSD-friendly. A few choices have an outsized effect on write amplification +and flash wear: + +- **Use batch writes.** `set()` resolves after its own flush, so + `for (const x of xs) await db.set(...)` issues one flush (and, under + `fsyncPolicy: 'always'`, one fsync) per key. Coalesce writes with + `db.batch([...])` / `db.mset([[k, v], ...])` (single atomic WAL frame) or + `await Promise.all(xs.map(x => db.set(x)))` (group commit collapses the + tick into one `writev` + one fsync). +- **Keep the default `fsyncPolicy: 'everysec'`.** It bounds fsyncs to ~1/s. + `always` is the most durable but the hardest on flash (one fsync per + flush); reserve it for data that must survive any crash. +- **Tune `compactThresholdBytes` for large datasets.** Compaction rewrites + all live data, so steady-state write amplification is roughly + `1 + liveDataBytes / compactThresholdBytes`. Raising the threshold (e.g. + 256 MiB–1 GiB) trades a larger WAL / longer recovery for less rewrite. +- **Observe it via `db.stats`.** `walBytesWritten`, `walFsyncs`, + `snapshotBytesWritten`, `compactions`, `evictions`, `maxMemoryRejections`, and + `queryIndexHits` let you measure real write volume, fsync rate, memory + pressure, and index usage under your workload. + +## RESP server (redis-cli compatible) + +```bash +npm run server -- --dir ./data --port 6379 +# or: node --import tsx src/server.ts --dir ./data --port 6379 +``` + +Then in another shell: + +```bash +redis-cli -p 6379 SET foo bar +redis-cli -p 6379 GET foo +``` + +Supported commands: `PING ECHO GET SET DEL EXISTS MGET MSET TTL DBSIZE COMPACT INFO QUIT`. + +## Benchmarks + +Measured on Node v24, 100-byte values (`npm run bench`, `N=100000`): + +| Operation | Throughput | +|---|---| +| Raw `Map` set (baseline) | ~8.6 M ops/s | +| Raw `Map` get (baseline) | ~20 M ops/s | +| **DB get** (in-memory) | **~8.0 M ops/s** | +| **DB set, fsync=everysec** (concurrent, group commit) | **~725 k ops/s** | +| **DB set, fsync=no** (concurrent, group commit) | **~396 k ops/s** | +| DB set, fsync=always (sequential, 1 fsync/op) | ~328 ops/s | +| Compact snapshot of 100k keys | ~69 ms (12 MiB) | + +Reads are memory-bound and track the raw `Map` closely. Writes are +disk-bound; **group commit** makes concurrent writes very fast, while +`synchronous` (`always`) writes pay the fsync cost one would expect from any +database. Numbers vary by machine/disk. + +Query benchmarks (N=50k docs, 200 iters each, `node bench/query.js`): + +| Query | Throughput | +|---|---| +| dt range (small result) | ~183 k ops/s | +| key prefix scan (~100 rows) | ~23 k ops/s | +| full-text search (latin / CJK) | ~350–490 ops/s | +| value filter, **no index** (full scan of 50k) | ~23 ops/s | + +Indexed dimensions (key / dt / text) are fast; an unindexed value filter scans +the whole collection — create a secondary index (`createIndex`) for hot value +predicates. + +For **ordered pagination within a group** (e.g. "sessions in a workspace ordered +by updatedAt"), use a **compound index** instead of fetching the whole group and +sorting in memory. On 10k sessions in one workspace, paginating with +`compoundRange` is ~20–40× faster than "fetch-all + sort", and stays +sub-millisecond at any offset. + +## Testing + +Three layers of tests, run with the built-in `node:test` runner (no deps): + +```bash +npm test # unit tests (fast) +npm run test:e2e # end-to-end stability suite +npm run test:all # both +``` + +The **unit tests** (`test/*.test.js`) cover each module: frame codec/CRC, WAL +group commit, store TTL, snapshot/compaction, recovery truncation, skip list, +secondary/full-text indexes, and the RESP server. + +The **E2E stability suite** (`test/e2e/*.test.js`) covers crash-safety and +long-run behavior: + +| File | What it verifies | +|---|---| +| `fuzz-model.test.js` | thousands of random ops match a reference model (seeded, reproducible) | +| `crash-recovery.test.js` | `kill -9` mid-write and mid-compaction → recovery is always consistent | +| `index-consistency.test.js` | key/dt/secondary/full-text indexes never drift from the store | +| `compaction-race.test.js` | heavy concurrent writes during compaction lose nothing | +| `recovery-matrix.test.js` | WAL corruption at head/mid/tail under `resync` vs `strict` | +| `durability.test.js` | `always`/`everysec`/`no` close-durability + many open/close cycles | +| `boundary.test.js` | key-length limits, large values, many keys, empty db | +| `soak.test.js` | sustained ops + heap stability (opt-in: `SOAK=30 npm run test:e2e`) | + +## Design in one paragraph + +Log-structured engine: all writes append to a CRC-framed **WAL** (group +committed, configurable fsync); all reads hit an in-memory `Map`. When the WAL +grows past a threshold, a **snapshot** of the live keys is written and the WAL +is rotated — **non-blocking**: writers keep appending to the WAL (which doubles +as a Redis-style rewrite buffer) while the snapshot is written, and pause only +for a brief final rotation. Recovery loads the latest +snapshot then replays the WAL, truncating any torn tail. Secondary indexes use +a Redis-style skip list for range queries. See +[`DESIGN_NOTES.md`](./DESIGN_NOTES.md) for the full rationale and the +source-code study behind each choice. + +## Concurrency & multi-process + +minidb is **single-writer**. Opening a directory for writing acquires an exclusive +lock file (`db.lock`); a second writer is rejected with a `LockError`. A lock is +taken over only when its owner PID is dead (stale-lock recovery), never merely +because it is old. + +```js +// second process: throws LockError +await MiniDb.open({ dir: './data' }); + +// degrade to read-only instead of throwing +const ro = await MiniDb.open({ dir: './data', onLockFail: 'readonly' }); +ro.get('k'); // ok (point-in-time view) +await ro.set('k'); // throws "read-only mode" + +// open read-only alongside a writer +const r = await MiniDb.open({ dir: './data', readOnly: true }); +``` + +For many clients, run the **RESP server** (single minidb process, many TCP +clients) — that is the intended concurrent-access model, like Redis. + +For a **rebuildable cache**, use `openOrRebuild`: a corrupt cache is discarded +and reopened empty, while a live-locked db is never destroyed: + +```js +const db = await MiniDb.openOrRebuild( + { dir: cacheDir, valueCodec: 'json' }, + { onRebuild: (err) => log.warn('cache rebuilt:', err.message) }, +); +``` + +## Caveats / roadmap + +- In the default `valueMode: 'memory'`, the dataset must fit in RAM Redis-style. + Use `valueMode: 'disk'` for larger-than-RAM value bulk; cold reads then perform + synchronous positioned reads against the snapshot/WAL. +- Full-text index postings are stored on disk (larger-than-RAM); only the term + dictionary and per-doc metadata stay in memory. Postings are rebuilt from the + store on open and on compaction. Search reads postings synchronously, so a + cold, very large postings list can briefly block the event loop. +- Compaction is **non-blocking for writes**: the WAL itself acts as a + `BGREWRITEAOF`-style rewrite buffer, so the (slow) snapshot is written while + writers keep appending. Writes pause only for a brief final rotation (a + flush, a small tail copy, and two atomic renames). +- Snapshot encoding runs on the main thread (chunked + yielding); offloading to + a `worker_thread` is a planned optimization. +- Single process / single writer. + +## Credits + +Design distilled from reading: Redis (`references/redis`), the SQLite WAL +paper, NeDB (`references/nedb`), Bitcask (`references/bitcask`), and the +cstack SQLite tutorial (`references/db_tutorial`). diff --git a/packages/minidb/bench/bench.ts b/packages/minidb/bench/bench.ts new file mode 100644 index 000000000..918b05db9 --- /dev/null +++ b/packages/minidb/bench/bench.ts @@ -0,0 +1,130 @@ +// bench/bench.js +// +// Throughput / latency micro-benchmarks for MiniDb. +// +// Run: npm run bench (or: node bench/bench.js) + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +const fmt = (n) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); +const ops = (n, ms) => `${fmt((n / ms) * 1000)} ops/s`; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-bench-')); +} + +async function bench(label, fn) { + // warm-up + a couple of GCs if available + if (global.gc) { + global.gc(); + } + const t0 = performance.now(); + await fn(); + const ms = performance.now() - t0; + console.log(` ${label.padEnd(46)} ${ms.toFixed(1).padStart(8)} ms`); + return ms; +} + +async function main() { + const VALUE = 'x'.repeat(100); // 100-byte values + const N = Number(process.env.N || 200_000); + const NSMALL = Number(process.env.NSMALL || 3_000); + + console.log(`\nminidb benchmark (N=${fmt(N)}, value=${VALUE.length}B, node ${process.version})\n`); + + // --- baseline: raw JS Map ---------------------------------------------- + { + const m = new Map(); + const ms = await bench('baseline: raw Map set (in-memory)', () => { + for (let i = 0; i < N; i++) m.set(`k${i}`, VALUE); + }); + console.log(` -> ${ops(N, ms)}`); + const ms2 = await bench('baseline: raw Map get (in-memory)', () => { + let s = 0; + for (let i = 0; i < N; i++) if (m.get(`k${i}`)) s++; + return s; + }); + console.log(` -> ${ops(N, ms2)}`); + } + + // --- DB writes, fsyncPolicy = no (fastest on-disk path) ----------------- + { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no', autoCompact: false }); + const ms = await bench('DB set concurrent, fsync=no (group commit)', async () => { + const p = []; + for (let i = 0; i < N; i++) p.push(db.set(`k${i}`, VALUE)); + await Promise.all(p); + }); + console.log(` -> ${ops(N, ms)}`); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } + + // --- DB writes, fsyncPolicy = everysec --------------------------------- + { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'everysec', autoCompact: false }); + const ms = await bench('DB set concurrent, fsync=everysec', async () => { + const p = []; + for (let i = 0; i < N; i++) p.push(db.set(`k${i}`, VALUE)); + await Promise.all(p); + }); + console.log(` -> ${ops(N, ms)}`); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } + + // --- DB writes, sequential await-each, fsync=always (worst case) ------- + { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'always', autoCompact: false }); + const ms = await bench(`DB set sequential, fsync=always (N=${fmt(NSMALL)})`, async () => { + for (let i = 0; i < NSMALL; i++) await db.set(`k${i}`, VALUE); + }); + console.log(` -> ${ops(NSMALL, ms)}`); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } + + // --- DB reads (in-memory) ---------------------------------------------- + { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no', autoCompact: false }); + const p = []; + for (let i = 0; i < N; i++) p.push(db.set(`k${i}`, VALUE)); + await Promise.all(p); + const ms = await bench('DB get (in-memory, after load)', () => { + let s = 0; + for (let i = 0; i < N; i++) if (db.get(`k${i}`)) s++; + return s; + }); + console.log(` -> ${ops(N, ms)}`); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } + + // --- compaction -------------------------------------------------------- + { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no', autoCompact: false }); + const p = []; + for (let i = 0; i < N; i++) p.push(db.set(`k${i}`, VALUE)); + await Promise.all(p); + const ms = await bench(`compact snapshot of ${fmt(N)} keys`, () => db.compact()); + const snap = await fs.stat(path.join(dir, 'db.snapshot')); + console.log(` -> ${(snap.size / 1024 / 1024).toFixed(2)} MiB snapshot in ${ms.toFixed(0)} ms`); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } + + console.log('\ndone.\n'); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/import-kimi-code.ts b/packages/minidb/bench/import-kimi-code.ts new file mode 100644 index 000000000..6bcff92f1 --- /dev/null +++ b/packages/minidb/bench/import-kimi-code.ts @@ -0,0 +1,247 @@ +// bench/import-kimi-code.js +// +// Import all workspaces + sessions from ~/.kimi-code into minidb and build a +// full-text index over the session content, then measure import + search speed. +// +// Run: node bench/import-kimi-code.js [--data ~/.kimi-code] [--out ] + +import fs from 'node:fs/promises'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { MiniDb } from '../src/index.js'; + +const argv = process.argv.slice(2); +const arg = (name, def) => { + const i = argv.indexOf(`--${name}`); + return i === -1 ? def : argv[i + 1]; +}; +const DATA = path.resolve(arg('data', path.join(os.homedir(), '.kimi-code'))); +const OUT = path.resolve(arg('out', path.join(os.tmpdir(), 'minidb-kimi-code-' + Date.now()))); +const FULL = argv.includes('--full'); // also index full tool results (stress test) + +const fmt = (n) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); +const mib = (n) => (n / 1024 / 1024).toFixed(1) + ' MiB'; + +function extractWireText(wirePath, full = false) { + // Pull searchable text out of the wire protocol: + // - user + assistant message text (context.append_message) + // - tool-call intent: tool name + concise args (command/pattern/path/...) + // - (full mode) tool-result output as well (stress test; noisy) + const ARG_FIELDS = ['command', 'pattern', 'path', 'description', 'query', 'prompt', 'file_path']; + let raw; + try { + raw = readFileSync(wirePath, 'utf8'); + } catch { + return { text: '', messages: 0 }; + } + const parts = []; + let messages = 0; + for (const line of raw.split('\n')) { + if (!line) continue; + let o; + try { + o = JSON.parse(line); + } catch { + continue; + } + if (o.type === 'context.append_message' && o.message && o.message.content) { + let got = false; + for (const c of o.message.content) { + if (c && c.type === 'text' && typeof c.text === 'string') { + parts.push(c.text); + got = true; + } + } + if (got) messages++; + } else if ( + o.type === 'context.append_loop_event' && + o.event && + o.event.type === 'tool.call' + ) { + const e = o.event; + const bits = [e.name]; + for (const k of ARG_FIELDS) { + const v = e.args && e.args[k]; + if (typeof v === 'string' && v) bits.push(v.length > 2000 ? v.slice(0, 2000) : v); + } + parts.push(bits.join(' ')); + } else if ( + full && + o.type === 'context.append_loop_event' && + o.event && + o.event.type === 'tool.result' + ) { + const r = o.event.result; + let out = ''; + if (typeof r === 'string') out = r; + else if (r && typeof r.output === 'string') out = r.output; + else if (r && typeof r.content === 'string') out = r.content; + if (out) parts.push(out.length > 5000 ? out.slice(0, 5000) : out); + } + } + return { text: parts.join('\n'), messages }; +} + +async function dirSize(dir) { + let total = 0; + async function walk(d) { + let ents; + try { + ents = await fs.readdir(d, { withFileTypes: true }); + } catch { + return; + } + for (const e of ents) { + const p = path.join(d, e.name); + if (e.isDirectory()) await walk(p); + else { + try { + total += (await fs.stat(p)).size; + } catch {} + } + } + } + await walk(dir); + return total; +} + +async function main() { + console.log(`data: ${DATA}`); + console.log(`out: ${OUT}`); + await fs.rm(OUT, { recursive: true, force: true }); + + // workspaces + const wsRaw = JSON.parse(readFileSync(path.join(DATA, 'workspaces.json'), 'utf8')); + const workspaces = wsRaw.workspaces || wsRaw; + + // session index + const lines = readFileSync(path.join(DATA, 'session_index.jsonl'), 'utf8').trim().split('\n'); + + const db = await MiniDb.open({ + dir: OUT, + valueCodec: 'json', + fsyncPolicy: 'no', + autoCompact: false, + }); + await db.createTextIndex('body', { fields: ['text'] }); + await db.createIndex('byWorkspace', { field: 'workspaceName' }); + + const t0 = performance.now(); + let imported = 0; + let skipped = 0; + let totalTextBytes = 0; + let totalMessages = 0; + let last = performance.now(); + + for (const line of lines) { + let meta; + try { + meta = JSON.parse(line); + } catch { + continue; + } + const { sessionId, sessionDir, workDir } = meta; + const wirePath = path.join(sessionDir, 'agents', 'main', 'wire.jsonl'); + if (!existsSync(wirePath)) { + skipped++; + continue; + } + + let state = {}; + try { + state = JSON.parse(readFileSync(path.join(sessionDir, 'state.json'), 'utf8')); + } catch {} + + const { text, messages } = extractWireText(wirePath, FULL); + totalTextBytes += Buffer.byteLength(text, 'utf8'); + totalMessages += messages; + + const wsId = path.basename(path.dirname(sessionDir)); // / + const ws = workspaces[wsId] || {}; + const doc = { + title: state.title || '', + workspaceId: wsId, + workspaceName: ws.name || '', + workDir: workDir || '', + text: (state.title ? state.title + '\n' : '') + text, + messageCount: messages, + }; + const updated = state.updatedAt ? Date.parse(state.updatedAt) : 0; + const created = state.createdAt ? Date.parse(state.createdAt) : 0; + + await db.set(sessionId, doc, { dt: { updated, created } }); + imported++; + + const now = performance.now(); + if (now - last > 1000) { + const rate = (imported / (now - t0)) * 1000; + process.stdout.write(`\r imported ${fmt(imported)} (${fmt(rate | 0)} sess/s, ${mib(totalTextBytes)} text)`); + last = now; + } + } + const importMs = performance.now() - t0; + process.stdout.write('\n'); + + // force a compaction so the on-disk size is the compact snapshot size + const ct0 = performance.now(); + await db.compact(); + const compactMs = performance.now() - ct0; + + const sz = await dirSize(OUT); + + console.log(`\n=== import done ===`); + console.log(` sessions imported: ${fmt(imported)} (skipped ${skipped})`); + console.log(` messages indexed : ${fmt(totalMessages)}`); + console.log(` text indexed : ${mib(totalTextBytes)}`); + console.log(` import time : ${(importMs / 1000).toFixed(1)} s (${fmt((imported / importMs) * 1000 | 0)} sess/s, ${mib(totalTextBytes / (importMs / 1000))}/s text)`); + console.log(` compact time : ${compactMs.toFixed(0)} ms`); + console.log(` db size on disk : ${mib(sz)} (${(sz / totalTextBytes).toFixed(2)}x raw text)`); + console.log(` postings terms : ${fmt(db.text.get('body').postings.size)}`); + console.log(` indexed docs : ${db.text.get('body').N}`); + if (global.gc) global.gc(); + console.log(` heap used : ${mib(process.memoryUsage().heapUsed)}`); + + // ---- sample searches ---- + const queries = ['lark-approval', 'database compaction', '北京', 'Redis 持久化', 'worktree init', 'nonexistentxyz123']; + console.log(`\n=== sample searches ===`); + for (const q of queries) { + const s0 = performance.now(); + const res = db.search('body', q, { limit: 5 }); + const ms = performance.now() - s0; + console.log(` "${q}" -> ${res.length} hits in ${ms.toFixed(1)} ms`); + for (const r of res.slice(0, 3)) { + const title = (r.value && r.value.title) || ''; + console.log(` [${r.score.toFixed(3)}] ${r.value.workspaceName} :: ${title.slice(0, 60)}`); + } + } + + // combined query: text + dt range + const q0 = performance.now(); + const recent = db.query({ + text: { index: 'body', q: 'database' }, + sort: { 'workspaceName': 1 }, + limit: 5, + project: ['title', 'workspaceName'], + }); + console.log(`\n composed query (text "database" + sort + project) -> ${recent.length} in ${(performance.now() - q0).toFixed(1)} ms`); + + // dt range: sessions updated in the last 7 days + const week = Date.now() - 7 * 864e5; + const d0 = performance.now(); + const recentDt = db.dtRange('updated', { gte: week, limit: 10 }); + console.log(` dt range (updated in last 7d) -> ${recentDt.length} shown in ${(performance.now() - d0).toFixed(2)} ms`); + + // secondary index lookup by workspace + const w0 = performance.now(); + const byWs = db.findEq('byWorkspace', 'kimi-code-dev-1'); + console.log(` index lookup (workspace=kimi-code-dev-1) -> ${byWs.length} in ${(performance.now() - w0).toFixed(2)} ms`); + + await db.close(); + console.log(`\ndone. db at: ${OUT}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/measure-session-memory.ts b/packages/minidb/bench/measure-session-memory.ts new file mode 100644 index 000000000..7729c4fc1 --- /dev/null +++ b/packages/minidb/bench/measure-session-memory.ts @@ -0,0 +1,231 @@ +// bench/measure-session-memory.ts +// +// Measure per-record memory of the Session scenario (bench/session-store.ts) so +// we can estimate the largest dataset the current all-in-RAM structure can hold. +// +// It builds N synthetic SessionDoc-shaped records with ~TEXT bytes of searchable +// text each, then adds the SAME indexes session-store creates, one by one, +// measuring heap/external/rss after each step. The per-step deltas attribute +// memory to each component (base store+value, dt indexes, equality indexes, +// compound indexes, full-text index). +// +// Run: +// node --import tsx --expose-gc --max-old-space-size=12288 \ +// bench/measure-session-memory.ts [N=30000] [TEXT=4000] + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +const N = Number(process.argv[2] ?? 30000); +const TEXT = Number(process.argv[3] ?? 4000); + +const gc = (): void => { + if (global.gc) { + global.gc(); + global.gc(); + } +}; +const mib = (n: number): string => (n / 1048576).toFixed(1) + ' MiB'; +const kb = (n: number): string => (n / 1024).toFixed(1) + ' KiB'; +const fmt = (n: number): string => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); + +interface Snap { + heap: number; + ext: number; + rss: number; +} +const snap = (): Snap => { + gc(); + const m = process.memoryUsage(); + return { heap: m.heapUsed, ext: m.external, rss: m.rss }; +}; + +// --- realistic-ish text generation ---------------------------------------- +// A bounded vocabulary so terms repeat across documents (like real text), but +// large enough that the global term set is non-trivial. Latin words + CJK +// unigrams/bigrams mirror src/text-index.ts's tokenizer. +const LATIN: string[] = Array.from({ length: 12000 }, (_, i) => 'w' + i.toString(36)); +const CJK_CHARS = '的一是不了在人有我他这个们中来上大为和国地到以说时要就出会可也你对生能而子那得于着下自之年过发后作里用道行所然家种事成方多经么去法学如都同现当没动面起看定天分还进好小部其些主样理心本前开但因只从想实日军者意无力它与长把机十民第公此已工使情明性知全三又关点正业外将两高间由问很最重并物手应战取向头文体政美相见被利什二等产或新己制身果加西斯月话合回特代内信表化老给世位次度门任常先海通教儿原东声提立及比员解水名真论处走义各入几口认条平系气题活尔更别打女变四神总何电数安少报才结反受目太量再感建务做接必场件计管期市直德资命山金指克许统区保至队形社便空决治展科司五基眼书非则听白却界达光放强即像难且权思王象完设式色路记南品住告类求据程北死张该交规万取拉格望觉术领共确传师观清今切院让识候带导争运笑飞风步改收根干造言联组革济众集商亲极九装研视林究越断数据库索引缓存事务日志压缩快照恢复线程队列服务请求响应配置部署监控告警容器镜像仓库分支合并发布版本接口协议编码解码序列化哈希令牌鉴权会话工作区工具调用参数结果错误异常超时重试熔断降级限流分页排序过滤投影聚合统计排名相关度'; + +function makeText(targetChars: number): string { + const parts: string[] = []; + let n = 0; + while (n < targetChars) { + if (Math.random() < 0.55) { + const t = LATIN[(Math.random() * LATIN.length) | 0]!; + parts.push(t); + n += t.length + 1; + } else { + const len = 1 + ((Math.random() * 6) | 0); + const start = (Math.random() * (CJK_CHARS.length - len)) | 0; + const t = CJK_CHARS.slice(start, start + len); + parts.push(t); + n += t.length + 1; + } + } + return parts.join(' ').slice(0, targetChars); +} + +interface SessionDoc { + workspaceId: string; + workspaceName: string; + workDir: string; + title: string; + lastPrompt: string; + text: string; + sessionDir: string; + messageCount: number; +} + +const NWS = 200; // distinct workspaces (low cardinality -> small equality index) + +async function main(): Promise { + if (!global.gc) { + console.error('run with --expose-gc for stable measurements'); + process.exit(1); + } + const dir = path.join(os.tmpdir(), 'minidb-mem-' + Date.now()); + await fs.rm(dir, { recursive: true, force: true }); + + const db = await MiniDb.open({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + autoCompact: false, + activeExpireIntervalMs: 0, // disable timer; we only measure + }); + + console.log(`scenario: ${fmt(N)} sessions, ~${fmt(TEXT)} chars text each`); + console.log(`out: ${dir}\n`); + + const base = snap(); + + // Insert data WITH dt (so dt indexes exist) but BEFORE any secondary index. + const t0 = performance.now(); + const CHUNK = 500; + let totalEncoded = 0; + for (let i = 0; i < N; i += CHUNK) { + const ops = []; + const end = Math.min(i + CHUNK, N); + for (let j = i; j < end; j++) { + const ws = j % NWS; + const text = makeText(TEXT); + const doc: SessionDoc = { + workspaceId: 'ws' + ws, + workspaceName: 'workspace-' + ws, + workDir: '/home/user/proj-' + (j % 1000), + title: 'session title ' + j + ' ' + LATIN[j % LATIN.length], + lastPrompt: 'please refactor the ' + LATIN[(j * 7) % LATIN.length] + ' module', + text, + sessionDir: '/x/ws' + ws + '/sess' + j, + messageCount: (j % 50) + 1, + }; + totalEncoded += Buffer.byteLength(JSON.stringify(doc), 'utf8'); + ops.push({ + op: 'set' as const, + key: 'sess:' + j, + value: doc, + dt: { updatedAt: 1700000000000 + j * 1000, createdAt: 1699000000000 + j * 1000 }, + }); + } + await db.batch(ops); + } + const insertMs = performance.now() - t0; + + const afterData = snap(); + // Base store + key skiplist + ttl heap + value buffers + dt indexes(updatedAt,createdAt) + + await db.createIndex('byWorkspace', { field: 'workspaceId' }); + const afterEqWs = snap(); + + await db.createIndex('byWorkDir', { field: 'workDir' }); + const afterEqWd = snap(); + + await db.createCompoundIndex('byWsUpdated', { groupBy: 'workspaceId', orderBy: 'updatedAt' }); + const afterCmpUpd = snap(); + + await db.createCompoundIndex('byWsCreated', { groupBy: 'workspaceId', orderBy: 'createdAt' }); + const afterCmpCrt = snap(); + + await db.createTextIndex('body', { fields: ['text'] }); + const afterText = snap(); + + // text index internals (larger-than-RAM: postings are on disk; the in-memory + // dictionary maps term -> { off, len, df }). + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const ti = (db as any).text.get('body'); + const postingsTerms = ti.termCount() as number; + let postingsEntries = 0; + for (const e of ti.postings.values()) postingsEntries += (e as { df: number }).df; + const tfDocs = ti.N as number; + + console.log('--- ingest ---'); + console.log(` insert time : ${(insertMs / 1000).toFixed(2)} s (${fmt((N / insertMs) * 1000 | 0)} sess/s)`); + console.log(` avg encoded doc : ${fmt((totalEncoded / N) | 0)} bytes (JSON of value)`); + console.log(` total encoded value: ${mib(totalEncoded)}`); + + const rows: [string, Snap][] = [ + ['empty db (after open)', base], + ['+ data: store + value buf + key skiplist + dt(updatedAt,createdAt)', afterData], + ['+ eq byWorkspace (low cardinality)', afterEqWs], + ['+ eq byWorkDir (1000 distinct)', afterEqWd], + ['+ compound byWsUpdated (skiplist/group)', afterCmpUpd], + ['+ compound byWsCreated (skiplist/group)', afterCmpCrt], + ['+ text body (postings + tf + docLen)', afterText], + ]; + + console.log('\n--- retained memory by stage ---'); + console.log(' stage heap external rss heap/doc'); + for (const [name, s] of rows) { + const perDoc = s.heap / N; + console.log( + ` ${name.padEnd(46)} ${mib(s.heap).padStart(9)} ${mib(s.ext).padStart(10)} ${mib(s.rss).padStart(9)} ${fmt(perDoc | 0).padStart(8)} B`, + ); + } + + console.log('\n--- per-component deltas (heap) ---'); + const deltas: [string, number][] = [ + ['base: store+value+key-skiplist+dt(2 cols)', afterData.heap - base.heap], + ['eq byWorkspace', afterEqWs.heap - afterData.heap], + ['eq byWorkDir', afterEqWd.heap - afterEqWs.heap], + ['compound byWsUpdated', afterCmpUpd.heap - afterEqWd.heap], + ['compound byWsCreated', afterCmpCrt.heap - afterCmpUpd.heap], + ['text body', afterText.heap - afterCmpCrt.heap], + ]; + let sum = 0; + for (const [name, d] of deltas) { + sum += d; + console.log(` ${name.padEnd(46)} ${mib(d).padStart(10)} ${fmt((d / N) | 0).padStart(7)} B/doc ${((d / afterText.heap) * 100).toFixed(1).padStart(5)}%`); + } + console.log(` ${'external (value buffers, off-heap)'.padEnd(46)} ${mib(afterText.ext).padStart(10)} ${fmt((afterText.ext / N) | 0).padStart(7)} B/doc`); + + console.log('\n--- text index shape ---'); + console.log(` indexed docs (N) : ${fmt(tfDocs)}`); + console.log(` unique terms : ${fmt(postingsTerms)}`); + console.log(` postings entries : ${fmt(postingsEntries)} (${fmt(postingsEntries / N)} per doc)`); + + // ---- capacity projection ---- + const heapPerDoc = afterText.heap / N; + const extPerDoc = afterText.ext / N; + const totalPerDoc = heapPerDoc + extPerDoc; + console.log('\n--- capacity projection (linear extrapolation) ---'); + console.log(` on-heap per session : ${fmt(heapPerDoc | 0)} B`); + console.log(` external per session: ${fmt(extPerDoc | 0)} B`); + console.log(` total per session : ${fmt(totalPerDoc | 0)} B (heap + external)`); + for (const budget of [1, 2, 4, 8, 16, 32]) { + const bytes = budget * 1024 * 1024 * 1024; + const maxSessions = Math.floor(bytes / totalPerDoc); + const maxText = (maxSessions * (totalEncoded / N)) / 1048576; + console.log(` RAM ${String(budget).padStart(2)} GiB -> ~${fmt(maxSessions).padStart(12)} sessions (~${fmt(maxText | 0).padStart(8)} MiB text)`); + } + + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/query.ts b/packages/minidb/bench/query.ts new file mode 100644 index 000000000..3cbf98ce8 --- /dev/null +++ b/packages/minidb/bench/query.ts @@ -0,0 +1,97 @@ +// bench/query.js +// +// Query micro-benchmarks: key prefix scan, dt range, value filter, full-text. +// Run: node bench/query.js + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +const fmt = (n) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); +const ops = (n, ms) => `${fmt((n / ms) * 1000)} ops/s`; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-qbench-')); +} + +async function bench(label, fn, iters = 1) { + const t0 = performance.now(); + let r; + for (let i = 0; i < iters; i++) r = await fn(); + const ms = performance.now() - t0; + return { ms, r }; +} + +async function main() { + const N = Number(process.env.N || 50_000); + const ITERS = Number(process.env.ITERS || 200); + console.log(`\nminidb query benchmark (N=${fmt(N)} docs, ${ITERS} iters each, node ${process.version})\n`); + + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + await db.createTextIndex('body'); + + const base = Date.parse('2024-01-01'); + const bulk = []; + for (let i = 0; i < N; i++) { + bulk.push( + db.set( + `user:${String(i).padStart(6, '0')}`, + { + age: 18 + (i % 60), + city: ['Paris', 'London', 'Tokyo', 'Beijing'][i % 4], + bio: i % 3 === 0 ? '我住在北京，喜欢编程和数据库' : 'hello world from nodejs database engine', + }, + { dt: { created: base + i * 1000 } }, + ), + ); + } + await Promise.all(bulk); + + // key prefix scan + let r = await bench('key prefix scan "user:0001.."', () => db.prefix('user:0001'), ITERS); + console.log(` key prefix scan (user:0001*)`.padEnd(42), `${r.ms.toFixed(1)} ms total`, `-> ${ops(ITERS, r.ms)}`, `(~${r.r.length} rows)`); + + // key range + r = await bench('key range', () => db.scan({ gte: 'user:001', lte: 'user:002' }), ITERS); + console.log(` key range [user:001, user:002]`.padEnd(42), `${r.ms.toFixed(1)} ms total`, `-> ${ops(ITERS, r.ms)}`, `(~${r.r.length} rows)`); + + // dt range + r = await bench('dt range', () => db.dtRange('created', { gte: base + 10000, lte: base + 20000 }), ITERS); + console.log(` dt range (10 docs)`.padEnd(42), `${r.ms.toFixed(1)} ms total`, `-> ${ops(ITERS, r.ms)}`); + + // value filter (full scan + match) + r = await bench('value filter', () => db.query({ filter: { city: 'Paris', age: { $gte: 30 } } }), ITERS); + console.log(` value filter (city=Paris & age>=30)`.padEnd(42), `${r.ms.toFixed(1)} ms total`, `-> ${ops(ITERS, r.ms)}`, `(~${r.r.length} rows)`); + + // full-text search + r = await bench('text search latin', () => db.search('body', 'hello'), ITERS); + console.log(` text search "hello"`.padEnd(42), `${r.ms.toFixed(1)} ms total`, `-> ${ops(ITERS, r.ms)}`, `(~${r.r.length} rows)`); + + r = await bench('text search cjk', () => db.search('body', '北京'), ITERS); + console.log(` text search "北京"`.padEnd(42), `${r.ms.toFixed(1)} ms total`, `-> ${ops(ITERS, r.ms)}`, `(~${r.r.length} rows)`); + + // composed query + r = await bench( + 'composed', + () => + db.query({ + dt: { created: { gte: base, lte: base + N * 1000 } }, + filter: { city: 'Beijing' }, + sort: { age: -1 }, + limit: 10, + }), + ITERS, + ); + console.log(` composed (dt + filter + sort + limit)`.padEnd(42), `${r.ms.toFixed(1)} ms total`, `-> ${ops(ITERS, r.ms)}`); + + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + console.log('\ndone.\n'); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/search-kimi-code.ts b/packages/minidb/bench/search-kimi-code.ts new file mode 100644 index 000000000..d3e60c261 --- /dev/null +++ b/packages/minidb/bench/search-kimi-code.ts @@ -0,0 +1,124 @@ +// bench/search-kimi-code.js +// +// Import ~/.kimi-code sessions (useful extraction) and full-text-search for a +// query, printing hits with context snippets. +// +// Run: node bench/search-kimi-code.js [--full] + +import fs from 'node:fs/promises'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { MiniDb } from '../src/index.js'; + +const argv = process.argv.slice(2); +const FULL = argv.includes('--full'); +const query = argv.find((a) => !a.startsWith('--')); +if (!query) { + console.error('usage: node bench/search-kimi-code.js [--full]'); + process.exit(1); +} + +const DATA = path.join(os.homedir(), '.kimi-code'); +const ARG_FIELDS = ['command', 'pattern', 'path', 'description', 'query', 'prompt', 'file_path']; + +function extractWireText(wirePath, full) { + let raw; + try { + raw = readFileSync(wirePath, 'utf8'); + } catch { + return ''; + } + const parts = []; + for (const line of raw.split('\n')) { + if (!line) continue; + let o; + try { + o = JSON.parse(line); + } catch { + continue; + } + if (o.type === 'context.append_message' && o.message && o.message.content) { + for (const c of o.message.content) + if (c && c.type === 'text' && typeof c.text === 'string') parts.push(c.text); + } else if (o.type === 'context.append_loop_event' && o.event && o.event.type === 'tool.call') { + const e = o.event; + const bits = [e.name]; + for (const k of ARG_FIELDS) { + const v = e.args && e.args[k]; + if (typeof v === 'string' && v) bits.push(v.length > 2000 ? v.slice(0, 2000) : v); + } + parts.push(bits.join(' ')); + } else if (full && o.type === 'context.append_loop_event' && o.event && o.event.type === 'tool.result') { + const r = o.event.result; + const out = typeof r === 'string' ? r : r && (r.output || r.content); + if (typeof out === 'string' && out) parts.push(out.length > 5000 ? out.slice(0, 5000) : out); + } + } + return parts.join('\n'); +} + +function snippet(text, q, radius = 60) { + const i = text.indexOf(q); + if (i === -1) return text.slice(0, radius * 2).replace(/\s+/g, ' ') + '…'; + const s = Math.max(0, i - radius); + const e = Math.min(text.length, i + q.length + radius); + return (s > 0 ? '…' : '') + text.slice(s, e).replace(/\s+/g, ' ') + (e < text.length ? '…' : ''); +} + +async function main() { + const OUT = path.join(os.tmpdir(), 'minidb-search-' + Date.now()); + await fs.rm(OUT, { recursive: true, force: true }); + + const workspaces = JSON.parse(readFileSync(path.join(DATA, 'workspaces.json'), 'utf8')).workspaces || {}; + const lines = readFileSync(path.join(DATA, 'session_index.jsonl'), 'utf8').trim().split('\n'); + + const db = await MiniDb.open({ dir: OUT, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + await db.createTextIndex('body', { fields: ['text'] }); + + const t0 = performance.now(); + let n = 0; + for (const line of lines) { + let meta; + try { + meta = JSON.parse(line); + } catch { + continue; + } + const wirePath = path.join(meta.sessionDir, 'agents', 'main', 'wire.jsonl'); + if (!existsSync(wirePath)) continue; + let state = {}; + try { + state = JSON.parse(readFileSync(path.join(meta.sessionDir, 'state.json'), 'utf8')); + } catch {} + const wsId = path.basename(path.dirname(meta.sessionDir)); + const ws = workspaces[wsId] || {}; + const text = (state.title ? state.title + '\n' : '') + extractWireText(wirePath, FULL); + await db.set(meta.sessionId, { + title: state.title || '', + workspaceName: ws.name || '', + workDir: meta.workDir || '', + text, + }); + n++; + } + const importMs = performance.now() - t0; + + const s0 = performance.now(); + const res = db.search('body', query, { limit: 10 }); + const ms = performance.now() - s0; + + console.log(`indexed ${n} sessions in ${(importMs / 1000).toFixed(1)}s; search "${query}" -> ${res.length} hits in ${ms.toFixed(1)}ms\n`); + for (const r of res) { + console.log(`[${r.score.toFixed(3)}] ${r.value.workspaceName} :: ${r.value.title}`); + console.log(` ${snippet(r.value.text, query)}`); + console.log(` ${r.key}`); + console.log(); + } + await db.close(); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/session-store-demo.ts b/packages/minidb/bench/session-store-demo.ts new file mode 100644 index 000000000..b7eee6459 --- /dev/null +++ b/packages/minidb/bench/session-store-demo.ts @@ -0,0 +1,76 @@ +// bench/session-store-demo.ts +// +// Demonstrate the 4 core queries against ~/.kimi-code with timing. +// Run: node --import tsx bench/session-store-demo.ts + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { SessionStore } from './session-store.js'; + +const HOME = path.join(os.homedir(), '.kimi-code'); +const OUT = path.join(os.tmpdir(), 'minidb-session-store-' + Date.now()); + +const ms = (t: number) => t.toFixed(2) + 'ms'; + +async function main() { + await fs.rm(OUT, { recursive: true, force: true }); + console.log(`out: ${OUT}`); + + const store = await SessionStore.open(OUT); + const t0 = performance.now(); + const stats = await store.ingestKimiCode(HOME); + const importMs = performance.now() - t0; + console.log( + `\ningested ${stats.workspaces} workspaces, ${stats.sessions} sessions (${(stats.textBytes / 1024 / 1024).toFixed(1)} MiB text) in ${ms(importMs)}`, + ); + + // 1. list workspaces + let t = performance.now(); + const ws = store.listWorkspaces({ limit: 10 }); + console.log(`\n[1] listWorkspaces -> ${ws.items.length} in ${ms(performance.now() - t)}`); + for (const w of ws.items) console.log(` ${w.id} ${w.name}`); + + if (ws.items.length === 0) return; + const wsId = ws.items[0]!.id; + + // 2. list sessions in workspace (page 1 + 2) + t = performance.now(); + const p1 = store.listSessions(wsId, { limit: 5, offset: 0 }); + const p2 = store.listSessions(wsId, { limit: 5, offset: 5 }); + console.log(`\n[2] listSessions(${wsId}) page1=${p1.items.length} page2=${p2.items.length} in ${ms(performance.now() - t)}`); + for (const s of p1.items) console.log(` ${new Date(s.updatedAt ?? 0).toISOString().slice(0, 10)} ${s.title.slice(0, 50)}`); + + // 3. precise get + if (p1.items.length) { + const sid = p1.items[0]!.sessionId; + t = performance.now(); + const s = store.getSession(sid); + console.log(`\n[3] getSession(${sid}) in ${ms(performance.now() - t)}`); + console.log(` title : ${s?.title}`); + console.log(` updatedAt : ${s?.updatedAt ? new Date(s.updatedAt).toISOString() : '-'}`); + console.log(` wirePath : ${s?.wirePath}`); + console.log(` text chars: ${s?.text.length}`); + } + + // 4. fuzzy search + for (const q of ['database compaction', 'lark-approval', 'Redis 持久化']) { + t = performance.now(); + const hits = store.search(q, { limit: 3 }); + console.log(`\n[4] search("${q}") -> ${hits.length} in ${ms(performance.now() - t)}`); + for (const h of hits) console.log(` [${h.score.toFixed(3)}] ${h.workspaceName} :: ${h.title.slice(0, 50)}`); + } + + // 4b. search scoped to a workspace + t = performance.now(); + const scoped = store.search('database', { workspaceId: wsId, limit: 3 }); + console.log(`\n[4b] search("database", workspace=${wsId}) -> ${scoped.length} in ${ms(performance.now() - t)}`); + + await store.db.close(); + console.log('\ndone.'); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/session-store.ts b/packages/minidb/bench/session-store.ts new file mode 100644 index 000000000..8d8830f12 --- /dev/null +++ b/packages/minidb/bench/session-store.ts @@ -0,0 +1,275 @@ +// bench/session-store.ts +// +// Schema + query layer for the kimi-code session store, built on minidb. +// +// Two logical "tables" via key prefixes in one db: +// ws: -> WorkspaceDoc +// sess: -> SessionDoc +// +// Indexes on the session docs: +// byWorkspace (equality, workspaceId) -> list sessions in a workspace +// byWorkDir (equality, workDir) -> list sessions for a cwd +// body (full-text on `text`) -> fuzzy search title/tool_call/content +// dt.updatedAt / dt.createdAt -> time-ordered listing + range +// +// The full wire.jsonl is NOT stored (it's large and is the source of truth on +// disk); we store its path plus the extracted searchable text. + +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +export interface WorkspaceDoc { + name: string; + root: string; +} + +export interface SessionDoc { + workspaceId: string; + workspaceName: string; + workDir: string; + title: string; + lastPrompt: string; + text: string; // title + messages + tool_call intents (searchable) + sessionDir: string; + messageCount: number; +} + +export interface Page { + items: T[]; + hasMore: boolean; + nextOffset: number | null; +} + +export interface SessionHit extends SessionDoc { + sessionId: string; + updatedAt?: number; + createdAt?: number; +} + +const ARG_FIELDS = ['command', 'pattern', 'path', 'description', 'query', 'prompt', 'file_path']; + +function extractWireText(wirePath: string): { text: string; messages: number } { + let raw: string; + try { + raw = readFileSync(wirePath, 'utf8'); + } catch { + return { text: '', messages: 0 }; + } + const parts: string[] = []; + let messages = 0; + for (const line of raw.split('\n')) { + if (!line) continue; + let o: { type?: string; message?: { content?: { type?: string; text?: string }[] }; event?: { type?: string; name?: string; args?: Record } }; + try { + o = JSON.parse(line) as typeof o; + } catch { + continue; + } + if (o.type === 'context.append_message' && o.message?.content) { + let got = false; + for (const c of o.message.content) { + if (c?.type === 'text' && typeof c.text === 'string') { + parts.push(c.text); + got = true; + } + } + if (got) messages++; + } else if (o.type === 'context.append_loop_event' && o.event?.type === 'tool.call') { + const e = o.event; + const bits = [e.name ?? '']; + for (const k of ARG_FIELDS) { + const v = e.args?.[k]; + if (typeof v === 'string' && v) bits.push(v.length > 2000 ? v.slice(0, 2000) : v); + } + parts.push(bits.join(' ')); + } + } + return { text: parts.join('\n'), messages }; +} + +export class SessionStore { + private constructor(public db: MiniDb) {} + + static async open(dir: string): Promise { + const db = await MiniDb.open({ dir, valueCodec: 'json', onLockFail: 'readonly' }); + // create indexes idempotently (ignore "already exists") + for (const mk of [ + () => db.createIndex('byWorkspace', { field: 'workspaceId' }), + () => db.createIndex('byWorkDir', { field: 'workDir' }), + () => db.createCompoundIndex('byWsUpdated', { groupBy: 'workspaceId', orderBy: 'updatedAt' }), + () => db.createCompoundIndex('byWsCreated', { groupBy: 'workspaceId', orderBy: 'createdAt' }), + () => db.createTextIndex('body', { fields: ['text'] }), + ]) { + try { + await mk(); + } catch { + /* already exists */ + } + } + return new SessionStore(db); + } + + // ---- ingest ------------------------------------------------------------- + + async ingestKimiCode(homeDir: string): Promise<{ workspaces: number; sessions: number; textBytes: number }> { + const wsRaw = JSON.parse(readFileSync(path.join(homeDir, 'workspaces.json'), 'utf8')) as { + workspaces?: Record; + }; + const workspaces = wsRaw.workspaces ?? {}; + const lines = readFileSync(path.join(homeDir, 'session_index.jsonl'), 'utf8').trim().split('\n'); + + let wsCount = 0; + let sessCount = 0; + let textBytes = 0; + const batch: { op: 'set'; key: string; value: unknown; dt?: Record }[] = []; + + // workspaces + for (const [id, ws] of Object.entries(workspaces)) { + batch.push({ + op: 'set', + key: 'ws:' + id, + value: { name: ws.name, root: ws.root } satisfies WorkspaceDoc, + dt: { + lastOpenedAt: ws.last_opened_at ? Date.parse(ws.last_opened_at) : 0, + createdAt: ws.created_at ? Date.parse(ws.created_at) : 0, + }, + }); + wsCount++; + } + + // sessions + for (const line of lines) { + let meta: { sessionId: string; sessionDir: string; workDir: string }; + try { + meta = JSON.parse(line) as typeof meta; + } catch { + continue; + } + const wirePath = path.join(meta.sessionDir, 'agents', 'main', 'wire.jsonl'); + if (!existsSync(wirePath)) continue; + + let state: { title?: string; lastPrompt?: string; createdAt?: string; updatedAt?: string } = {}; + try { + state = JSON.parse(readFileSync(path.join(meta.sessionDir, 'state.json'), 'utf8')) as typeof state; + } catch { + /* no state.json */ + } + + const { text, messages } = extractWireText(wirePath); + const wsId = path.basename(path.dirname(meta.sessionDir)); + const ws = workspaces[wsId]; + const doc: SessionDoc = { + workspaceId: wsId, + workspaceName: ws?.name ?? '', + workDir: meta.workDir, + title: state.title ?? '', + lastPrompt: state.lastPrompt ?? '', + text: (state.title ? state.title + '\n' : '') + text, + sessionDir: meta.sessionDir, + messageCount: messages, + }; + textBytes += Buffer.byteLength(doc.text, 'utf8'); + batch.push({ + op: 'set', + key: 'sess:' + meta.sessionId, + value: doc, + dt: { + updatedAt: state.updatedAt ? Date.parse(state.updatedAt) : 0, + createdAt: state.createdAt ? Date.parse(state.createdAt) : 0, + }, + }); + sessCount++; + } + + // chunk the batch into reasonable groups + const CHUNK = 500; + for (let i = 0; i < batch.length; i += CHUNK) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await this.db.batch(batch.slice(i, i + CHUNK) as any); + } + return { workspaces: wsCount, sessions: sessCount, textBytes }; + } + + // ---- 1. list workspaces (paginated, by lastOpenedAt desc) --------------- + + listWorkspaces({ limit = 20, offset = 0 }: { limit?: number; offset?: number } = {}): Page { + const all = this.db + .prefix('ws:') + .map((r) => ({ id: r.key.slice(3), ...(r.value as WorkspaceDoc), dt: r.dt })); + all.sort((a, b) => (b.dt?.lastOpenedAt ?? 0) - (a.dt?.lastOpenedAt ?? 0)); + const items = all.slice(offset, offset + limit).map(({ id, name, root }) => ({ id, name, root })); + return { items, hasMore: offset + limit < all.length, nextOffset: offset + limit < all.length ? offset + limit : null }; + } + + // ---- 2. list sessions in a workspace (paginated, by updatedAt desc) ----- + + listSessions(workspaceId: string, { limit = 20, offset = 0 }: { limit?: number; offset?: number } = {}): Page { + // O(log N + limit): the compound index is already ordered by updatedAt. + const page = this.db.compoundRange('byWsUpdated', workspaceId, { reverse: true, offset, limit }); + const items: SessionHit[] = []; + for (const r of page) { + const rec = this.db.getRecord(r.key); + items.push({ + sessionId: r.key.slice(5), + ...(r.value as SessionDoc), + updatedAt: rec?.dt?.updatedAt, + createdAt: rec?.dt?.createdAt, + }); + } + // hasMore: peek one more + const peek = this.db.compoundRange('byWsUpdated', workspaceId, { reverse: true, offset: offset + limit, limit: 1 }); + return { items, hasMore: peek.length > 0, nextOffset: peek.length > 0 ? offset + limit : null }; + } + + // ---- 3. precise get session + metadata + wire path + time --------------- + + getSession(sessionId: string): (SessionHit & { wirePath: string }) | null { + const rec = this.db.getRecord('sess:' + sessionId); + if (!rec) return null; + const doc = rec.value as SessionDoc; + return { + sessionId, + ...doc, + updatedAt: rec.dt?.updatedAt, + createdAt: rec.dt?.createdAt, + wirePath: path.join(doc.sessionDir, 'agents', 'main', 'wire.jsonl'), + }; + } + + readWire(sessionId: string): string | null { + const s = this.getSession(sessionId); + if (!s) return null; + try { + return readFileSync(s.wirePath, 'utf8'); + } catch { + return null; + } + } + + // ---- 4. fuzzy search title / tool_call / content ------------------------ + + search(q: string, { workspaceId, limit = 20 }: { workspaceId?: string; limit?: number } = {}): (SessionHit & { score: number })[] { + if (workspaceId) { + // text search intersected with workspace filter, ordered by updatedAt + return this.db + .query({ + text: { index: 'body', q }, + filter: { workspaceId }, + sort: { updatedAt: -1 }, + limit, + }) + .map((r) => ({ sessionId: r.key.slice(5), ...(r.value as SessionDoc), updatedAt: r.dt?.updatedAt, createdAt: r.dt?.createdAt, score: 0 })); + } + return this.db.search('body', q, { limit }).map((r) => { + const rec = this.db.getRecord(r.key); + return { + sessionId: r.key.slice(5), + ...(r.value as SessionDoc), + updatedAt: rec?.dt?.updatedAt, + createdAt: rec?.dt?.createdAt, + score: r.score, + }; + }); + } +} diff --git a/packages/minidb/package.json b/packages/minidb/package.json new file mode 100644 index 000000000..b98bc8913 --- /dev/null +++ b/packages/minidb/package.json @@ -0,0 +1,48 @@ +{ + "name": "@moonshot-ai/minidb", + "version": "0.1.0", + "private": true, + "description": "A pure-Node.js embedded key-value database mixing Redis-style in-memory KV with SQLite-style durable persistence (WAL + snapshot).", + "license": "MIT", + "author": "Moonshot AI", + "homepage": "https://github.com/MoonshotAI/kimi-code/tree/main/packages/minidb#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/MoonshotAI/kimi-code.git", + "directory": "packages/minidb" + }, + "bugs": { + "url": "https://github.com/MoonshotAI/kimi-code/issues" + }, + "keywords": [ + "kimi", + "database", + "key-value", + "embedded", + "wal", + "snapshot" + ], + "files": [ + "dist" + ], + "type": "module", + "imports": { + "#/*": [ + "./src/*.ts", + "./src/*/index.ts" + ] + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "build": "tsdown", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "bench": "node --import tsx bench/bench.ts", + "clean": "rm -rf dist" + } +} diff --git a/packages/minidb/src/codec.ts b/packages/minidb/src/codec.ts new file mode 100644 index 000000000..77488af4a --- /dev/null +++ b/packages/minidb/src/codec.ts @@ -0,0 +1,532 @@ +// src/codec.ts +// +// Binary record format shared by the WAL and the snapshot. +// +// Frame (little-endian): +// +// off size field +// 0 2 magic = 0x4D 0x44 ("MD") — sync marker +// 2 1 type — 1 = SET, 2 = DEL (tombstone), 3 = BATCH +// 3 1 flags — reserved (0) +// 4 2 keyLen — uint16, key length in bytes (max 64 KiB) +// 6 4 valLen — uint32, value length in bytes (0 for DEL) +// 10 4 metaLen — uint32, optional metadata length in bytes (0 if none) +// 14 8 expireAt — int64, ms since epoch; 0 = no expiry +// 22 keyLen key +// 22+k valLen value +// 22+k+v metaLen meta — optional metadata blob (used for dt columns, etc.) +// 22+k+v+m 4 crc32 — CRC-32 TRAILER over [type .. meta] +// +// The fixed-size header (22 bytes) lets a reader compute the full frame length +// (22 + keyLen + valLen + metaLen + 4) before reading the payload. + +import fs from 'node:fs'; +import { crc32 } from './crc32.js'; + +export const MAGIC = Buffer.from([0x4d, 0x44]); // "MD" +export const TYPE_SET = 1; +export const TYPE_DEL = 2; +export const TYPE_BATCH = 3; +export const HEADER_SIZE = 22; // bytes before the payload (key) +export const CRC_SIZE = 4; +export const MAX_KEY_LEN = 0xffff; // uint16 +export const MAX_VAL_LEN = 0xffffffff; // uint32 + +/** A decoded record frame. */ +export interface Frame { + type: number; + key: Buffer; + value: Buffer; + meta: Buffer | null; + expireAt: number; +} + +export interface EncodeFrameInput { + type: number; + key: Buffer; + value?: Buffer | null; + meta?: Buffer | null; + expireAt?: number | bigint; +} + +/** A single op inside a BATCH frame body. */ +export interface BatchOp { + type: number; + key: Buffer; + value: Buffer | null; + meta: Buffer | null; + expireAt: number; +} + +export interface ParseResult { + frames: Frame[]; + corruptRanges: [number, number][]; + eofOffset: number; +} + +/** A frame scanned for recovery in valueMode:'disk'. The value is reported as an + * absolute file offset/length instead of being copied into memory. */ +export interface FrameRef { + type: number; + key: Buffer; + meta: Buffer | null; + expireAt: number; + frameOff: number; + valueOff: number; + valLen: number; + frameLen: number; +} + +export interface ScanFrameRefsResult { + frames: FrameRef[]; + corruptRanges: [number, number][]; + eofOffset: number; +} + +/** A BATCH sub-op scanned for recovery in valueMode:'disk'. */ +export interface BatchOpRef { + type: number; + key: Buffer; + meta: Buffer | null; + expireAt: number; + valueOff: number; + valLen: number; +} + +export class CorruptFrameError extends Error { + readonly offset: number; + constructor(message: string, offset: number) { + super(message); + this.name = 'CorruptFrameError'; + this.offset = offset; // absolute byte offset in the stream where the bad frame starts + } +} + +const EMPTY: Buffer = Buffer.alloc(0); + +/** + * Encode one record into a single Buffer. + */ +export function encodeFrame({ + type, + key, + value = null, + meta = null, + expireAt = 0, +}: EncodeFrameInput): Buffer { + if (!Buffer.isBuffer(key)) throw new TypeError('key must be a Buffer'); + if (key.length > MAX_KEY_LEN) throw new RangeError('key too large'); + const val: Buffer = value ?? EMPTY; + const met: Buffer = meta ?? EMPTY; + if (type === TYPE_SET && !Buffer.isBuffer(val)) throw new TypeError('value must be a Buffer for SET'); + if (!Buffer.isBuffer(met)) throw new TypeError('meta must be a Buffer'); + if (val.length > MAX_VAL_LEN) throw new RangeError('value too large'); + if (met.length > MAX_VAL_LEN) throw new RangeError('meta too large'); + + const frame = Buffer.allocUnsafe(HEADER_SIZE + key.length + val.length + met.length + CRC_SIZE); + + let o = 0; + MAGIC.copy(frame, o); o += 2; + frame.writeUInt8(type, o); o += 1; + frame.writeUInt8(0, o); o += 1; // flags + frame.writeUInt16LE(key.length, o); o += 2; + frame.writeUInt32LE(val.length, o); o += 4; + frame.writeUInt32LE(met.length, o); o += 4; + frame.writeBigInt64LE(BigInt(expireAt ?? 0), o); o += 8; + key.copy(frame, o); o += key.length; + val.copy(frame, o); o += val.length; + met.copy(frame, o); o += met.length; + + // CRC trailer over everything after magic, before the crc field. + const c = crc32(frame.subarray(2, o)); + frame.writeUInt32LE(c, o); + return frame; +} + +/** + * Encode a list of ops into a batch body (used as the `value` of a TYPE_BATCH + * frame). The whole body is protected by the outer frame's CRC, so a batch is + * one atomic unit: it either applies fully or is skipped entirely on recovery. + * + * Body layout: + * count(2) | [ op(1) | keyLen(2) | valLen(4) | metaLen(4) | expireAt(8) | + * key | value | meta ] ... + */ +const SUB_HEADER = 1 + 2 + 4 + 4 + 8; + +export function encodeBatchOps(ops: BatchOp[]): Buffer { + let total = 2; + for (const op of ops) { + total += SUB_HEADER + op.key.length + (op.value ? op.value.length : 0) + (op.meta ? op.meta.length : 0); + } + const body = Buffer.allocUnsafe(total); + let o = 0; + body.writeUInt16LE(ops.length, o); o += 2; + for (const op of ops) { + const key = op.key; + const val: Buffer = op.value ?? EMPTY; + const met: Buffer = op.meta ?? EMPTY; + body.writeUInt8(op.type, o); o += 1; + body.writeUInt16LE(key.length, o); o += 2; + body.writeUInt32LE(val.length, o); o += 4; + body.writeUInt32LE(met.length, o); o += 4; + body.writeBigInt64LE(BigInt(op.expireAt ?? 0), o); o += 8; + key.copy(body, o); o += key.length; + val.copy(body, o); o += val.length; + met.copy(body, o); o += met.length; + } + return body; +} + +export function decodeBatchOps(body: Buffer): BatchOp[] { + const ops: BatchOp[] = []; + let o = 0; + if (body.length < 2) return ops; + const count = body.readUInt16LE(o); o += 2; + for (let i = 0; i < count; i++) { + if (o + SUB_HEADER > body.length) throw new RangeError('batch op header truncated'); + const type = body.readUInt8(o); o += 1; + const keyLen = body.readUInt16LE(o); o += 2; + const valLen = body.readUInt32LE(o); o += 4; + const metaLen = body.readUInt32LE(o); o += 4; + const expireAt = Number(body.readBigInt64LE(o)); o += 8; + if (o + keyLen + valLen + metaLen > body.length) throw new RangeError('batch op payload truncated'); + const key = Buffer.from(body.subarray(o, o + keyLen)); o += keyLen; + const value = Buffer.from(body.subarray(o, o + valLen)); o += valLen; + const meta = metaLen ? Buffer.from(body.subarray(o, o + metaLen)) : null; o += metaLen; + ops.push({ type, key, value, meta, expireAt }); + } + return ops; +} + +/** + * Streaming frame parser. Feed it arbitrary chunks (e.g. from a file read + * stream); it yields whole frames and buffers partial trailing bytes for the + * next feed(). Tracks absolute stream offset so a corrupt frame can be located + * and the file truncated there. + */ +export class FrameParser { + private pending: Buffer = EMPTY; + private offset = 0; // absolute offset of the next byte to be consumed + + *feed(chunk: Buffer): Generator { + let buf: Buffer = this.pending.length ? Buffer.concat([this.pending, chunk]) : chunk; + let pos = 0; + + while (true) { + if (buf.length - pos < HEADER_SIZE) break; + + if (buf[pos] !== MAGIC[0] || buf[pos + 1] !== MAGIC[1]) { + const next = buf.indexOf(MAGIC, pos + 1); + if (next === -1) throw new CorruptFrameError('magic not found', this.offset + pos); + pos = next; + continue; + } + + const type = buf.readUInt8(pos + 2); + const keyLen = buf.readUInt16LE(pos + 4); + const valLen = buf.readUInt32LE(pos + 6); + const metaLen = buf.readUInt32LE(pos + 10); + const frameLen = HEADER_SIZE + keyLen + valLen + metaLen + CRC_SIZE; + + if (buf.length - pos < frameLen) break; // incomplete payload/crc, wait for more + + const storedCrc = buf.readUInt32LE(pos + frameLen - CRC_SIZE); + const computedCrc = crc32(buf.subarray(pos + 2, pos + frameLen - CRC_SIZE)); + if (storedCrc !== computedCrc) { + throw new CorruptFrameError(`crc mismatch at offset ${this.offset + pos}`, this.offset + pos); + } + + const expireAt = Number(buf.readBigInt64LE(pos + 14)); + const keyStart = pos + HEADER_SIZE; + const key = buf.subarray(keyStart, keyStart + keyLen); + const value = buf.subarray(keyStart + keyLen, keyStart + keyLen + valLen); + const metaStart = keyStart + keyLen + valLen; + const meta = metaLen ? buf.subarray(metaStart, metaStart + metaLen) : null; + + yield { + type, + key: Buffer.from(key), + value: Buffer.from(value), + meta: meta ? Buffer.from(meta) : null, + expireAt, + }; + + pos += frameLen; + this.offset += frameLen; + } + + this.pending = pos < buf.length ? Buffer.from(buf.subarray(pos)) : EMPTY; + } + + /** + * Signal end-of-stream. If any bytes are still buffered (a partial frame), + * they are a torn tail left by a crash: throw CorruptFrameError at the offset + * where valid data ends, so recovery can truncate the file there. Returns the + * clean EOF offset (total valid bytes) when there is no leftover. + */ + finish(): number { + if (this.pending.length > 0) { + const off = this.offset; + const n = this.pending.length; + this.pending = EMPTY; + throw new CorruptFrameError(`torn tail: ${n} trailing byte(s)`, off); + } + return this.offset; + } +} + +/** + * Try to read and validate one frame at `pos`. + * @returns the parsed frame + its byte length, or null when there is no valid, + * complete frame at `pos` (no magic, incomplete, insane length, or CRC mismatch). + */ +function readFrameAt(buf: Buffer, pos: number): { frame: Frame; frameLen: number } | null { + if (buf.length - pos < HEADER_SIZE) return null; + if (buf[pos] !== MAGIC[0] || buf[pos + 1] !== MAGIC[1]) return null; + const keyLen = buf.readUInt16LE(pos + 4); + const valLen = buf.readUInt32LE(pos + 6); + const metaLen = buf.readUInt32LE(pos + 10); + if (keyLen > MAX_KEY_LEN) return null; + const frameLen = HEADER_SIZE + keyLen + valLen + metaLen + CRC_SIZE; + if (frameLen < HEADER_SIZE + CRC_SIZE) return null; // length overflow + if (buf.length - pos < frameLen) return null; // incomplete + const stored = buf.readUInt32LE(pos + frameLen - CRC_SIZE); + const computed = crc32(buf.subarray(pos + 2, pos + frameLen - CRC_SIZE)); + if (stored !== computed) return null; // bad crc + + const expireAt = Number(buf.readBigInt64LE(pos + 14)); + const keyStart = pos + HEADER_SIZE; + const key = buf.subarray(keyStart, keyStart + keyLen); + const value = buf.subarray(keyStart + keyLen, keyStart + keyLen + valLen); + const metaStart = keyStart + keyLen + valLen; + const meta = metaLen ? buf.subarray(metaStart, metaStart + metaLen) : null; + return { + frame: { + type: buf.readUInt8(pos + 2), + key: Buffer.from(key), + value: Buffer.from(value), + meta: meta ? Buffer.from(meta) : null, + expireAt, + }, + frameLen, + }; +} + +const CRC_CHUNK = 1 << 20; +const MAGIC_SCAN_CHUNK = 1 << 20; + +function readExactSync(fd: number, buf: Buffer, pos: number): void { + let got = 0; + while (got < buf.length) { + const r = fs.readSync(fd, buf, got, buf.length - got, pos + got); + if (r === 0) throw new Error('codec: short read past EOF'); + got += r; + } +} + +function readFrameRefAt(fd: number, pos: number, size: number): FrameRef | null { + if (size - pos < HEADER_SIZE) return null; + const header = Buffer.allocUnsafe(HEADER_SIZE); + readExactSync(fd, header, pos); + if (header[0] !== MAGIC[0] || header[1] !== MAGIC[1]) return null; + + const type = header.readUInt8(2); + const keyLen = header.readUInt16LE(4); + const valLen = header.readUInt32LE(6); + const metaLen = header.readUInt32LE(10); + if (keyLen > MAX_KEY_LEN) return null; + const frameLen = HEADER_SIZE + keyLen + valLen + metaLen + CRC_SIZE; + if (frameLen < HEADER_SIZE + CRC_SIZE) return null; // length overflow + if (size - pos < frameLen) return null; // incomplete + + let crc = 0; + let crcPos = pos + 2; + let crcLeft = frameLen - CRC_SIZE - 2; + while (crcLeft > 0) { + const len = Math.min(CRC_CHUNK, crcLeft); + const buf = Buffer.allocUnsafe(len); + readExactSync(fd, buf, crcPos); + crc = crc32(buf, crc); + crcPos += len; + crcLeft -= len; + } + const storedCrcBuf = Buffer.allocUnsafe(CRC_SIZE); + readExactSync(fd, storedCrcBuf, pos + frameLen - CRC_SIZE); + if (storedCrcBuf.readUInt32LE(0) !== crc) return null; + + const keyStart = pos + HEADER_SIZE; + const valueOff = keyStart + keyLen; + const metaStart = valueOff + valLen; + const key = Buffer.allocUnsafe(keyLen); + if (keyLen) readExactSync(fd, key, keyStart); + let meta: Buffer | null = null; + if (metaLen) { + meta = Buffer.allocUnsafe(metaLen); + readExactSync(fd, meta, metaStart); + } + + return { + type, + key, + meta, + expireAt: Number(header.readBigInt64LE(14)), + frameOff: pos, + valueOff, + valLen, + frameLen, + }; +} + +function findMagicSync(fd: number, start: number, size: number): number { + const buf = Buffer.allocUnsafe(MAGIC_SCAN_CHUNK); + let pos = start; + while (pos < size) { + const len = Math.min(MAGIC_SCAN_CHUNK, size - pos); + const n = fs.readSync(fd, buf, 0, len, pos); + if (n === 0) return -1; + const idx = buf.subarray(0, n).indexOf(MAGIC); + if (idx >= 0) return pos + idx; + if (n < MAGIC.length) break; + pos += n - (MAGIC.length - 1); + } + return -1; +} + +/** Scan an open snapshot/WAL fd into frame refs without copying values. */ +export function scanFrameRefsFd( + fd: number, + { onCorrupt = 'resync' }: { onCorrupt?: 'resync' | 'strict' } = {}, +): ScanFrameRefsResult { + const size = fs.fstatSync(fd).size; + const frames: FrameRef[] = []; + const corruptRanges: [number, number][] = []; + let pos = 0; + + while (pos < size) { + const r = readFrameRefAt(fd, pos, size); + if (r) { + frames.push(r); + pos += r.frameLen; + continue; + } + + if (onCorrupt === 'strict') { + corruptRanges.push([pos, size]); + break; + } + + const badStart = pos; + let resume = -1; + let scan = pos + 1; + while (scan < size - 1) { + scan = findMagicSync(fd, scan, size); + if (scan === -1) break; + if (readFrameRefAt(fd, scan, size)) { + resume = scan; + break; + } + scan++; + } + corruptRanges.push([badStart, resume === -1 ? size : resume]); + if (resume === -1) break; + pos = resume; + } + + return { frames, corruptRanges, eofOffset: pos }; +} + +/** Scan a snapshot/WAL file into frame refs without copying values. */ +export function scanFrameRefsFile( + filePath: string, + opts: { onCorrupt?: 'resync' | 'strict' } = {}, +): ScanFrameRefsResult { + const fd = fs.openSync(filePath, 'r'); + try { + return scanFrameRefsFd(fd, opts); + } finally { + fs.closeSync(fd); + } +} + +/** Scan BATCH body op refs without copying op values. `bodyOff` is the absolute + * file offset where the BATCH body (the outer frame's value) starts. */ +export function scanBatchOpRefs(body: Buffer, bodyOff: number): BatchOpRef[] { + const ops: BatchOpRef[] = []; + let o = 0; + if (body.length < 2) return ops; + const count = body.readUInt16LE(o); + o += 2; + for (let i = 0; i < count; i++) { + if (o + SUB_HEADER > body.length) throw new RangeError('batch op header truncated'); + const type = body.readUInt8(o); + o += 1; + const keyLen = body.readUInt16LE(o); + o += 2; + const valLen = body.readUInt32LE(o); + o += 4; + const metaLen = body.readUInt32LE(o); + o += 4; + const expireAt = Number(body.readBigInt64LE(o)); + o += 8; + if (o + keyLen + valLen + metaLen > body.length) throw new RangeError('batch op payload truncated'); + const key = Buffer.from(body.subarray(o, o + keyLen)); + const valueOff = bodyOff + o + keyLen; + o += keyLen + valLen; + const meta = metaLen ? Buffer.from(body.subarray(o, o + metaLen)) : null; + o += metaLen; + ops.push({ type, key, valueOff, valLen, meta, expireAt }); + } + return ops; +} + +/** + * Parse a complete buffer into frames, with configurable corruption handling. + * + * - onCorrupt = 'resync' (default): a bad/incomplete frame is skipped and the + * parser resynchronizes to the next valid frame. Only the corrupted bytes are + * lost; everything after the next valid frame is recovered. + * - onCorrupt = 'strict': stop at the first bad frame and treat the entire tail + * as lost. Frames before the first bad frame are kept. + */ +export function parseBuffer( + buf: Buffer, + { onCorrupt = 'resync' }: { onCorrupt?: 'resync' | 'strict' } = {}, +): ParseResult { + const frames: Frame[] = []; + const corruptRanges: [number, number][] = []; + let pos = 0; + + while (pos < buf.length) { + const r = readFrameAt(buf, pos); + if (r) { + frames.push(r.frame); + pos += r.frameLen; + continue; + } + + if (onCorrupt === 'strict') { + corruptRanges.push([pos, buf.length]); + break; + } + + // Resync: scan forward for the next frame that validates. + const badStart = pos; + let resume = -1; + let scan = pos + 1; + while (scan < buf.length - 1) { + scan = buf.indexOf(MAGIC, scan); + if (scan === -1) break; + if (readFrameAt(buf, scan)) { + resume = scan; + break; + } + scan++; + } + corruptRanges.push([badStart, resume === -1 ? buf.length : resume]); + if (resume === -1) break; + pos = resume; + } + + return { frames, corruptRanges, eofOffset: pos }; +} diff --git a/packages/minidb/src/compaction.ts b/packages/minidb/src/compaction.ts new file mode 100644 index 000000000..88db531b8 --- /dev/null +++ b/packages/minidb/src/compaction.ts @@ -0,0 +1,235 @@ +// src/compaction.ts +// +// WAL compaction (a.k.a. snapshot + rewrite). +// +// This is a NON-BLOCKING variant, modelled on Redis's BGREWRITEAOF and +// Bitcask's merge: while the (potentially large) snapshot is being written, +// writers keep appending to the live WAL — the WAL itself acts as the +// "rewrite buffer". Writes are blocked only for a short *rotation* critical +// section at the very end (a flush + a tiny tail copy + two renames). +// +// Phases: +// 1. fence — flush the WAL, record baseOffset = wal.size. Every write +// durable at/before baseOffset is already reflected in the +// store (applyOp runs synchronously before the WAL write is +// awaited). +// 2. snapshot — writeSnapshot(store, tmp). NON-BLOCKING. Writers keep +// appending to the WAL and mutating the store while we +// iterate. The snapshot need NOT be point-in-time: the WAL +// tail copied below is replayed last-writer-wins on top of +// it, repairing any fuzziness. +// 2.5 pre-copy — stream WAL[baseOffset .. head] into db.wal.tmp, draining +// the bulk of the post-fence tail. NON-BLOCKING. Loops +// until the remaining delta is small, so the critical +// section stays bounded. +// 3. rotation — SHORT BLOCKING critical section: set _rotateLock so new +// writers park, flush, copy the tiny remaining tail delta, +// close the old WAL, rename snapshot then WAL (crash-safe +// order), reopen the new WAL. +// 4. bookkeeping — stats + onCompacted() (rebuild derived text postings). +// +// Crash safety: recovery is `load db.snapshot` + `replay db.wal`, last-writer +// wins. We rename the snapshot BEFORE the WAL. If a crash lands between the two +// renames, the new snapshot is paired with the old full WAL — replaying the +// whole old WAL on top of the new snapshot is idempotent for pre-fence frames +// and correct for post-fence frames, so the state is still consistent. The +// reverse order (WAL first) would pair an old snapshot with a truncated new WAL +// and lose pre-fence data. + +import fs from 'node:fs/promises'; +import type { FileHandle } from 'node:fs/promises'; +import path from 'node:path'; +import { WAL } from './wal.js'; +import { writeSnapshot } from './snapshot.js'; +import type { Store, ValueLoc } from './store.js'; +import type { FsyncPolicy } from './wal.js'; + +/** Structural interface of the bits compaction needs from a MiniDb. */ +export interface CompactionTarget { + dir: string; + walPath: string; + fsyncPolicy: FsyncPolicy; + store: Store; + wal: WAL; + compactThresholdBytes: number; + compacting: boolean; + _compactDone: Promise | null; + /** Set only during the short rotation critical section; writers park on it. + * Null outside rotation, so the snapshot phase is fully non-blocking. */ + _rotateLock: Promise | null; + lastCompactError: unknown; + stats: { compactions: number; walBytesWritten: number; walFsyncs: number; snapshotBytesWritten: number }; + /** Reader for disk-backed values; reopened after snapshot/WAL rotation so + * remapped value pointers read from the new files. */ + valueReader?: { reopenBoth(): void }; + /** Optional hook invoked after the snapshot + WAL rotation succeeds, so the + * owner can rewrite derived on-disk state (e.g. text postings) against the + * new live set. */ + onCompacted?: () => void; +} + +export function shouldCompact(db: CompactionTarget): boolean { + return Boolean(db.wal && db.wal.size >= db.compactThresholdBytes); +} + +const COPY_CHUNK = 1 << 20; // 1 MiB read/write coalescing +// A post-fence WAL delta at or below this size is cheap enough to copy inside +// the rotation critical section, so the pre-copy loop stops draining. +const SMALL_DELTA = 64 * 1024; // 64 KiB + +export async function fsyncDir(dir: string): Promise { + let fh: FileHandle | null = null; + try { + fh = await fs.open(dir, 'r'); + await fh.sync(); + } catch { + /* best-effort */ + } finally { + if (fh) await fh.close().catch(() => {}); + } +} + +/** Stream src[start:end] into dst, fsync'ing dst before returning. With + * `append: true` the bytes are appended to an existing dst; otherwise dst is + * created/truncated. Uses its own file handles, independent of the WAL's + * append handle, so it is safe to read the live WAL while writers append. A + * zero-length range still creates/truncates dst (so the new WAL file exists + * even when there is no post-fence tail). */ +export async function copyFileRange( + srcPath: string, + dstPath: string, + start: number, + end: number, + opts: { append?: boolean } = {}, +): Promise { + if (end < start) throw new RangeError(`copyFileRange: end (${end}) < start (${start})`); + const dst = await fs.open(dstPath, opts.append ? 'a' : 'w'); + try { + if (end > start) { + const src = await fs.open(srcPath, 'r'); + try { + const buf = Buffer.allocUnsafe(COPY_CHUNK); + let pos = start; + while (pos < end) { + const len = Math.min(buf.length, end - pos); + const { bytesRead } = await src.read(buf, 0, len, pos); + if (bytesRead === 0) break; // reached EOF earlier than expected + let written = 0; + while (written < bytesRead) { + const { bytesWritten } = await dst.write(buf, written, bytesRead - written); + if (bytesWritten === 0) throw new Error('copyFileRange: write made no progress (short write)'); + written += bytesWritten; + } + pos += bytesRead; + } + } finally { + await src.close().catch(() => {}); + } + } + await dst.sync(); + } finally { + await dst.close().catch(() => {}); + } +} + +export async function compact(db: CompactionTarget): Promise { + if (db.compacting) return db._compactDone ?? undefined; + + db.compacting = true; + db._compactDone = (async () => { + try { + await runCompaction(db); + db.stats.compactions++; + db.lastCompactError = null; + db.onCompacted?.(); + } catch (err) { + db.lastCompactError = err; + throw err; + } finally { + db.compacting = false; + // A failed rotation must not leave writers parked forever. + db._rotateLock = null; + } + })(); + return db._compactDone; +} + +async function runCompaction(db: CompactionTarget): Promise { + const tmp = path.join(db.dir, 'db.snapshot.tmp'); + const snap = path.join(db.dir, 'db.snapshot'); + const walTmp = path.join(db.dir, 'db.wal.tmp'); + + // Phase 1: fence. Every write durable at/before baseOffset is already + // reflected in the store, because applyOp() runs synchronously in the same + // tick as wal.append(), before the op awaits the WAL write. + await db.wal.flush(); + const baseOffset = db.wal.size; + + // Phase 2: snapshot. NON-BLOCKING — writers keep appending to the WAL and + // mutating the store while we iterate. Fuzziness is repaired by the tail. + const snapRes = await writeSnapshot(db.store, tmp); + db.stats.snapshotBytesWritten += snapRes.bytes; + + // Phase 2.5: pre-copy the post-fence WAL tail into db.wal.tmp. NON-BLOCKING. + // Drain until the remaining delta is small enough to copy inside the critical + // section. Each iteration flushes to get a stable `head`, then copies the + // bytes that landed since the previous iteration. The loop converges because + // sequential copy is far faster than fsync-bound writes; if writes ever + // outrun the copy, the critical section simply absorbs a larger delta (still + // correct, just a longer pause — no worse than the old stop-the-world). + let copiedUpTo = baseOffset; + let appended = false; + for (;;) { + await db.wal.flush(); + const head = db.wal.size; + if (head - copiedUpTo <= SMALL_DELTA) break; + await copyFileRange(db.walPath, walTmp, copiedUpTo, head, { append: appended }); + appended = true; + copiedUpTo = head; + } + + // Phase 3: rotation. SHORT BLOCKING critical section. + // + // Setting _rotateLock is synchronous and happens-before the flush below. A + // writer's gate check (`if (_rotateLock) await _rotateLock`) and its + // wal.append() are in the same synchronous segment, so either the writer saw + // the old (null) lock and already enqueued its frame (→ drained by flush), or + // it sees the new lock and parks without appending. The event loop cannot + // interleave between the two, so after flush() the queue is quiescent and + // endOffset is final. + let releaseRotation!: () => void; + db._rotateLock = new Promise((resolve) => { + releaseRotation = resolve; + }); + try { + await db.wal.flush(); + const endOffset = db.wal.size; + await copyFileRange(db.walPath, walTmp, copiedUpTo, endOffset, { append: appended }); + + await db.wal.close(); + + // Snapshot first, then WAL — see the crash-safety note in the file header. + await fs.rename(tmp, snap); + await fsyncDir(db.dir); + await fs.rename(walTmp, db.walPath); + await fsyncDir(db.dir); + + db.wal = new WAL(db.walPath, { fsyncPolicy: db.fsyncPolicy, stats: db.stats }); + await db.wal.open(); + + // Remap disk-backed value pointers to the new snapshot/WAL files. Do this in + // the same synchronous segment as the fd reopen, so synchronous readers can + // never observe a new pointer against an old fd or vice versa. + const snapLocs = snapRes.locs; + db.store.remapLocs((k: string, loc: ValueLoc) => { + if (loc.file === 'wal' && loc.off >= baseOffset) { + return { file: 'wal', off: loc.off - baseOffset, len: loc.len }; + } + return snapLocs.get(k); + }); + db.valueReader?.reopenBoth(); + } finally { + releaseRotation(); + db._rotateLock = null; + } +} diff --git a/packages/minidb/src/compound-index.ts b/packages/minidb/src/compound-index.ts new file mode 100644 index 000000000..6e3edbfae --- /dev/null +++ b/packages/minidb/src/compound-index.ts @@ -0,0 +1,152 @@ +// src/compound-index.ts +// +// Native compound indexes: a group key + an order key, e.g. +// (workspaceId, updatedAt), (workspaceId, createdAt), (workspaceId, archivedAt) +// +// For each group value we keep a SkipList ordered by the order key, so +// "group = X ORDER BY order" is a range scan — O(log N + limit), no full sort. +// Multiple order columns (multiple dt) are supported by creating one compound +// index per order column. + +import { SkipList, cmpNumber, cmpString } from './skiplist.js'; +import type { Comparator, RangeOptions } from './skiplist.js'; + +export type OrderType = 'number' | 'string'; + +export interface CompoundIndexDef { + /** value field path used as the group key (e.g. "workspaceId") */ + groupBy: string; + /** order key: a value field path OR a dt column name (dt takes precedence) */ + orderBy: string; + orderType?: OrderType; +} + +export interface CompoundIndexInfo { + name: string; + groupBy: string; + orderBy: string; + orderType: OrderType; +} + +interface CompoundEntry { + def: Required; + cmp: Comparator; + groups: Map>; // groupValue -> ordered pks + byPk: Map; // pk -> current placement +} + +function getPath(doc: unknown, path: string): unknown { + return path.split('.').reduce((o, k) => (o == null ? undefined : (o as Record)[k]), doc); +} + +export class CompoundIndexManager { + readonly indexes = new Map(); + + create(name: string, def: CompoundIndexDef): void { + if (this.indexes.has(name)) throw new Error(`compound index "${name}" already exists`); + const orderType = def.orderType ?? 'number'; + const full: Required = { groupBy: def.groupBy, orderBy: def.orderBy, orderType }; + const cmp = orderType === 'string' ? (cmpString as Comparator) : (cmpNumber as Comparator); + this.indexes.set(name, { def: full, cmp, groups: new Map(), byPk: new Map() }); + } + + drop(name: string): boolean { + return this.indexes.delete(name); + } + + list(): CompoundIndexInfo[] { + return [...this.indexes.entries()].map(([name, e]) => ({ + name, + groupBy: e.def.groupBy, + orderBy: e.def.orderBy, + orderType: e.def.orderType, + })); + } + + private groupOf(entry: CompoundEntry, group: unknown): SkipList { + let list = entry.groups.get(group); + if (!list) { + list = new SkipList({ compareKey: entry.cmp, compareVal: cmpString }); + entry.groups.set(group, list); + } + return list; + } + + private extract(entry: CompoundEntry, doc: unknown, dt: Record | null): { group: unknown; order: unknown } { + const group = getPath(doc, entry.def.groupBy); + const order = + dt && entry.def.orderBy in dt ? (dt as Record)[entry.def.orderBy] : getPath(doc, entry.def.orderBy); + return { group, order }; + } + + private validOrder(entry: CompoundEntry, order: unknown): boolean { + if (entry.def.orderType === 'number') return typeof order === 'number' && Number.isFinite(order); + return typeof order === 'string'; + } + + /** Add/update a document across all compound indexes. */ + add(pk: string, doc: unknown, dt: Record | null): void { + for (const entry of this.indexes.values()) { + const { group, order } = this.extract(entry, doc, dt); + const prev = entry.byPk.get(pk); + const valid = group !== undefined && group !== null && this.validOrder(entry, order); + + // No-op when placement is unchanged. Without this guard, re-setting a key + // with the same group+order inserted a duplicate skiplist node (and a + // later delete left a phantom entry behind). + if (prev && valid && prev.group === group && prev.order === order) continue; + + if (prev) { + const oldList = entry.groups.get(prev.group); + if (oldList) { + oldList.delete(prev.order, pk); + if (oldList.length === 0) entry.groups.delete(prev.group); + } + } + + if (valid) { + this.groupOf(entry, group).insert(order, pk); + entry.byPk.set(pk, { group, order }); + } else { + entry.byPk.delete(pk); + } + } + } + + remove(pk: string, doc?: unknown, dt?: Record | null): void { + for (const entry of this.indexes.values()) { + const prev = entry.byPk.get(pk); + if (prev) { + const oldList = entry.groups.get(prev.group); + if (oldList) oldList.delete(prev.order, pk); + entry.byPk.delete(pk); + } + } + } + + /** Range over a group, ordered by the order key. */ + range( + name: string, + groupValue: unknown, + opts: RangeOptions & { limit?: number } = {}, + ): { key: string; orderValue: unknown }[] { + const entry = this.indexes.get(name); + if (!entry) throw new Error(`no such compound index: ${name}`); + const list = entry.groups.get(groupValue); + if (!list) return []; + return list + .range({ ...opts, count: opts.limit ?? opts.count }) + .map((n) => ({ key: n.val, orderValue: n.key })); + } + + /** Rebuild from entries of { key, value, dt }. */ + rebuild(entries: Iterable<{ key: string | Buffer; value: unknown; dt?: Record | null }>): void { + for (const entry of this.indexes.values()) { + entry.groups.clear(); + entry.byPk.clear(); + } + for (const { key, value, dt } of entries) { + this.add(typeof key === 'string' ? key : Buffer.from(key).toString('binary'), value, dt ?? null); + } + } +} diff --git a/packages/minidb/src/crc32.ts b/packages/minidb/src/crc32.ts new file mode 100644 index 000000000..2e7d452b2 --- /dev/null +++ b/packages/minidb/src/crc32.ts @@ -0,0 +1,47 @@ +// src/crc32.ts +// +// Table-based CRC-32 (IEEE 802.3, polynomial 0xEDB88320, reflected). +// This matches the algorithm used by Node's zlib, PNG, gzip, etc. +// +// Used by the WAL and snapshot frames to detect torn/corrupted records on +// recovery. Pure JS, no native deps. The 256-entry table is built lazily once +// and reused, so hot-path calls are just a handful of XOR/shifts per byte. + +const POLY = 0xedb88320; +let TABLE: Uint32Array | null = null; + +function buildTable(): Uint32Array { + const t = new Uint32Array(256); + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) { + c = c & 1 ? POLY ^ (c >>> 1) : c >>> 1; + } + t[n] = c >>> 0; + } + return t; +} + +/** + * Compute / continue a CRC-32 over `buf`. + * + * @param buf bytes to checksum + * @param prev previous crc value (for streaming / incremental use) + * @returns unsigned 32-bit crc + */ +export function crc32(buf: Buffer | Uint8Array, prev = 0): number { + if (TABLE === null) TABLE = buildTable(); + let c = prev ^ 0xffffffff; + for (let i = 0; i < buf.length; i++) { + c = TABLE[(c ^ buf[i]!) & 0xff]! ^ (c >>> 8); + } + return (c ^ 0xffffffff) >>> 0; +} + +// Exposed for tests only. +export const _private = { + buildTable, + get table(): Uint32Array | null { + return TABLE; + }, +}; diff --git a/packages/minidb/src/dt-index.ts b/packages/minidb/src/dt-index.ts new file mode 100644 index 000000000..8b8e4685c --- /dev/null +++ b/packages/minidb/src/dt-index.ts @@ -0,0 +1,95 @@ +// src/dt-index.ts +// +// Ordered indexes over declared datetime columns (dt1..dtN). Each column is a +// SkipList ordered by epoch-ms (numeric) with the record key as tie-break, giving +// O(log N) range / rank on every dt column. Pure in-memory derived state; rebuilt +// from the store on startup. + +import { SkipList, cmpNumber, cmpString } from './skiplist.js'; +import type { RangeEntry } from './skiplist.js'; + +interface DtColumn { + list: SkipList; + byKey: Map; +} + +export interface DtRangeEntry { + key: string; + value: number; +} + +export class DtIndex { + private readonly cols = new Map(); // col -> column + private readonly byKey = new Map>(); // key -> { col: ms } + + private col(name: string): DtColumn { + let c = this.cols.get(name); + if (!c) { + c = { list: new SkipList({ compareKey: cmpNumber, compareVal: cmpString }), byKey: new Map() }; + this.cols.set(name, c); + } + return c; + } + + /** Set/replace the dt columns for a key. dt = { col: ms } or null. */ + set(key: string, dt: Record | null): void { + const old = this.byKey.get(key) ?? {}; + const next = dt ?? {}; + + for (const col of Object.keys(old)) { + if (!(col in next) || old[col] !== next[col]) { + const c = this.cols.get(col); + if (c) { + c.list.delete(old[col]!, key); + c.byKey.delete(key); + if (c.byKey.size === 0) this.cols.delete(col); + } + } + } + for (const col of Object.keys(next)) { + const ms = next[col]!; + if (typeof ms !== 'number' || !Number.isFinite(ms)) continue; + if (old[col] === ms) continue; + const c = this.col(col); + c.list.insert(ms, key); + c.byKey.set(key, ms); + } + + if (Object.keys(next).length) this.byKey.set(key, { ...next }); + else this.byKey.delete(key); + } + + del(key: string): void { + const old = this.byKey.get(key); + if (!old) return; + for (const col of Object.keys(old)) { + const c = this.cols.get(col); + if (c) { + c.list.delete(old[col]!, key); + c.byKey.delete(key); + if (c.byKey.size === 0) this.cols.delete(col); + } + } + this.byKey.delete(key); + } + + /** Range over a dt column. */ + range(col: string, opts: Parameters['range']>[0] = {}): DtRangeEntry[] { + const c = this.cols.get(col); + if (!c) return []; + return c.list.range(opts).map((n: RangeEntry) => ({ key: n.val, value: n.key })); + } + + columns(): string[] { + return [...this.cols.keys()]; + } + + /** Rebuild from an iterator of { key, dt }. */ + rebuild(entries: Iterable<{ key: string; dt: Record | null | undefined }>): void { + this.cols.clear(); + this.byKey.clear(); + for (const { key, dt } of entries) { + if (dt) this.set(key, dt); + } + } +} diff --git a/packages/minidb/src/index-manager.ts b/packages/minidb/src/index-manager.ts new file mode 100644 index 000000000..409e1d9ef --- /dev/null +++ b/packages/minidb/src/index-manager.ts @@ -0,0 +1,308 @@ +// src/index-manager.ts +// +// Secondary indexes over JSON documents. Indexes are pure in-memory derived +// state (the WAL/store is the source of truth); they are rebuilt from the store +// on startup. + +import { SkipList, cmpNumber, cmpString } from './skiplist.js'; +import type { RangeOptions } from './skiplist.js'; + +export type IndexType = 'equality' | 'range'; + +export interface IndexDef { + field: string; + type?: IndexType; + unique?: boolean; + sparse?: boolean; +} + +export interface IndexInfo { + name: string; + field: string; + type: IndexType; + unique: boolean; + sparse: boolean; +} + +interface EqIndex { + name: string; + field: string; + type: 'equality'; + unique: boolean; + sparse: boolean; + map: Map>; + byPk: Map; +} + +interface RangeIndex { + name: string; + field: string; + type: 'range'; + unique: boolean; + sparse: boolean; + list: SkipList; + byPk: Map; // array fields index every element +} + +type AnyIndex = EqIndex | RangeIndex; + +export class UniqueViolationError extends Error { + constructor(index: string, value: unknown) { + super(`unique index "${index}" violation on value ${JSON.stringify(value)}`); + this.name = 'UniqueViolationError'; + } +} + +function getField(doc: unknown, path: string): unknown { + return path.split('.').reduce((o, k) => (o == null ? undefined : (o as Record)[k]), doc); +} + +function stableStringify(v: unknown): string { + if (v === null || typeof v !== 'object') return JSON.stringify(v); + if (Array.isArray(v)) return `[${v.map(stableStringify).join(',')}]`; + const keys = Object.keys(v as Record).sort(); + return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify((v as Record)[k])}`).join(',')}}`; +} + +function scalarKey(v: unknown): string { + const t = typeof v; + if (t === 'string' || t === 'number' || t === 'boolean') return `${t}:${v}`; + // Canonicalize property order so {a:1,b:2} and {b:2,a:1} hash to the same + // key; JSON.stringify alone preserves insertion order and would split them. + return `json:${stableStringify(v)}`; +} + +function flatten(value: unknown): unknown[] { + return Array.isArray(value) ? value : [value]; +} + +export class IndexManager { + readonly indexes = new Map(); + + create(name: string, { field, type = 'equality', unique = false, sparse = true }: IndexDef = {} as IndexDef): AnyIndex { + if (!field) throw new TypeError('index requires a field'); + if (this.indexes.has(name)) throw new Error(`index "${name}" already exists`); + const idx: AnyIndex = + type === 'range' + ? { + name, + field, + type, + unique, + sparse, + list: new SkipList({ compareKey: cmpNumber, compareVal: cmpString }), + byPk: new Map(), + } + : { name, field, type, unique, sparse, map: new Map(), byPk: new Map() }; + this.indexes.set(name, idx); + return idx; + } + + drop(name: string): boolean { + return this.indexes.delete(name); + } + + get(name: string): AnyIndex { + const idx = this.indexes.get(name); + if (!idx) throw new Error(`no such index: ${name}`); + return idx; + } + + list(): IndexInfo[] { + return [...this.indexes.values()].map(({ name, field, type, unique, sparse }) => ({ + name, + field, + type, + unique, + sparse, + })); + } + + /** Throw a UniqueViolationError if adding `doc` for `pk` would violate a unique index. */ + checkUnique(pk: string, doc: unknown): void { + for (const idx of this.indexes.values()) { + if (!idx.unique) continue; + const value = getField(doc, idx.field); + if (value === undefined && idx.sparse) continue; + for (const v of flatten(value)) { + if (idx.type === 'range') { + if (typeof v !== 'number' || !Number.isFinite(v)) continue; + const hit = idx.list.range({ gte: v, lte: v, count: 1 }); + if (hit.length && hit[0]!.val !== pk) throw new UniqueViolationError(idx.name, v); + } else { + const set = idx.map.get(scalarKey(v)); + if (set && (set.size > 1 || (set.size === 1 && !set.has(pk)))) { + throw new UniqueViolationError(idx.name, v); + } + } + } + } + } + + /** + * Validate unique constraints for a batch of ops by computing the index state + * AFTER the whole batch and checking it for collisions. The check is + * order-independent, so valid transformations like swapping a unique value + * between two keys, or deleting one key and reusing its value in another, + * are accepted (their final state is still unique). + * + * `ops` is the full op list (set AND del); the last op per key wins. + */ + checkUniqueBatch(ops: readonly { pk: string; op: 'set' | 'del'; doc: unknown }[]): void { + const lastOp = new Map(); + for (const o of ops) lastOp.set(o.pk, o); + const touched = new Set(lastOp.keys()); + + for (const idx of this.indexes.values()) { + if (!idx.unique) continue; + if (idx.type === 'range') { + const owner = new Map(); + for (const [pk, vals] of idx.byPk) { + if (touched.has(pk)) continue; + for (const v of vals) owner.set(v, pk); + } + for (const [pk, o] of lastOp) { + if (o.op === 'del') continue; + for (const v of flatten(getField(o.doc, idx.field))) { + if (typeof v !== 'number' || !Number.isFinite(v)) continue; + const prev = owner.get(v); + if (prev !== undefined && prev !== pk) throw new UniqueViolationError(idx.name, v); + owner.set(v, pk); + } + } + } else { + const owner = new Map(); + for (const [sk, set] of idx.map) { + for (const pk of set) if (!touched.has(pk)) owner.set(sk, pk); + } + for (const [pk, o] of lastOp) { + if (o.op === 'del') continue; + const value = getField(o.doc, idx.field); + if (value === undefined && idx.sparse) continue; + for (const v of flatten(value)) { + const sk = scalarKey(v); + const prev = owner.get(sk); + if (prev !== undefined && prev !== pk) throw new UniqueViolationError(idx.name, v); + owner.set(sk, pk); + } + } + } + } + } + + /** + * Verify that an already-built unique index contains no duplicate values. + * Used when creating a unique index over pre-existing data: if the data + * already violates the constraint, the index must not be created. + */ + assertUniqueValid(name: string): void { + const idx = this.get(name); + if (!idx.unique) return; + if (idx.type === 'range') { + const owner = new Map(); + for (const [pk, vals] of idx.byPk) { + for (const v of vals) { + const prev = owner.get(v); + if (prev !== undefined && prev !== pk) throw new UniqueViolationError(idx.name, v); + owner.set(v, pk); + } + } + } else { + for (const [, set] of idx.map) { + if (set.size > 1) { + const sample = [...set][0]; + throw new UniqueViolationError(idx.name, `${set.size} keys (e.g. ${sample})`); + } + } + } + } + + add(pk: string, doc: unknown): void { + for (const idx of this.indexes.values()) { + const value = getField(doc, idx.field); + if (value === undefined && idx.sparse) continue; + if (idx.type === 'range') { + // Index each distinct numeric element once. Without the de-dupe, an + // array like [10, 10, 10] would insert three (10, pk) nodes and the + // same key would be reported three times by findRange. + const vals = [...new Set(flatten(value).filter((v): v is number => typeof v === 'number' && Number.isFinite(v)))]; + if (vals.length === 0) continue; + for (const v of vals) idx.list.insert(v, pk); + idx.byPk.set(pk, vals); + } else { + const keys: string[] = []; + for (const v of flatten(value)) { + const sk = scalarKey(v); + let set = idx.map.get(sk); + if (!set) idx.map.set(sk, (set = new Set())); + set.add(pk); + keys.push(sk); + } + idx.byPk.set(pk, keys); + } + } + } + + remove(pk: string, _doc: unknown): void { + for (const idx of this.indexes.values()) { + if (idx.type === 'range') { + const old = idx.byPk.get(pk); + if (old) { + for (const v of old) idx.list.delete(v, pk); + idx.byPk.delete(pk); + } + } else { + const keys = idx.byPk.get(pk); + if (keys) { + for (const sk of keys) { + const set = idx.map.get(sk); + if (set) { + set.delete(pk); + if (set.size === 0) idx.map.delete(sk); + } + } + idx.byPk.delete(pk); + } + } + } + } + + findEq(name: string, value: unknown): string[] { + const idx = this.get(name); + if (idx.type !== 'equality') throw new Error(`index "${name}" is not an equality index`); + const set = idx.map.get(scalarKey(value)); + return set ? [...set] : []; + } + + findRange( + name: string, + opts: { min?: number; max?: number; minExclusive?: boolean; maxExclusive?: boolean; offset?: number; count?: number; reverse?: boolean } = {}, + ): { pk: string; value: number }[] { + const idx = this.get(name); + if (idx.type !== 'range') throw new Error(`index "${name}" is not a range index`); + const r: RangeOptions = {}; + if (opts.min !== undefined) (opts.minExclusive ? (r.gt = opts.min) : (r.gte = opts.min)); + if (opts.max !== undefined) (opts.maxExclusive ? (r.lt = opts.max) : (r.lte = opts.max)); + if (opts.offset) r.offset = opts.offset; + if (opts.count !== undefined) r.count = opts.count; + if (opts.reverse) r.reverse = true; + return idx.list.range(r).map((n) => ({ pk: n.val, value: n.key })); + } + + /** Rebuild all indexes from an iterator of { key, value } (value = decoded doc). */ + rebuild(entries: Iterable<{ key: string | Buffer; value: unknown }>): void { + for (const idx of this.indexes.values()) { + if (idx.type === 'range') { + idx.list = new SkipList({ compareKey: cmpNumber, compareVal: cmpString }); + idx.byPk.clear(); + } else { + idx.map.clear(); + idx.byPk.clear(); + } + } + for (const { key, value } of entries) { + const pk = typeof key === 'string' ? key : Buffer.from(key).toString('binary'); + if (value && typeof value === 'object') this.add(pk, value); + } + } +} diff --git a/packages/minidb/src/index.ts b/packages/minidb/src/index.ts new file mode 100644 index 000000000..2a2bd956e --- /dev/null +++ b/packages/minidb/src/index.ts @@ -0,0 +1,1269 @@ +// src/index.ts +// +// MiniDb: the public embedded API. Ties together the in-memory Store (with its +// ordered key index), the WAL, recovery, compaction, dt-column indexes, value +// secondary indexes, and full-text indexes. +// +// Document model: +// { key: string(<=128), value: , dt1..dtN: } + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { Store } from './store.js'; +import type { StoreRecord, ValueLoc, ValueRef } from './store.js'; +import { WAL } from './wal.js'; +import { ValueReader } from './value-reader.js'; +import { recover } from './recovery.js'; +import { compact, shouldCompact } from './compaction.js'; +import { IndexManager, UniqueViolationError } from './index-manager.js'; +import { DtIndex } from './dt-index.js'; +import { TextIndex } from './text-index.js'; +import { CompoundIndexManager } from './compound-index.js'; +import { getPath, match, project } from './query.js'; +import { LockFile, LockError } from './lockfile.js'; +import { encodeFrame, encodeBatchOps, scanBatchOpRefs, HEADER_SIZE, TYPE_SET, TYPE_DEL, TYPE_BATCH } from './codec.js'; +import type { BatchOp as EncodedBatchOp } from './codec.js'; +import type { FsyncPolicy } from './wal.js'; +import type { RecoveryMode, RecoveryInfo, ValueMode } from './recovery.js'; +import type { IndexDef, IndexInfo } from './index-manager.js'; +import type { CompoundIndexDef, CompoundIndexInfo } from './compound-index.js'; +import type { DtRangeEntry } from './dt-index.js'; +import type { RangeOptions } from './skiplist.js'; + +export { UniqueViolationError } from './index-manager.js'; +export { LockError } from './lockfile.js'; +export type { RecoveryInfo } from './recovery.js'; +export type { IndexDef, IndexInfo, IndexType } from './index-manager.js'; +export type { CompoundIndexDef, CompoundIndexInfo } from './compound-index.js'; + +export type ValueCodecName = 'buffer' | 'string' | 'json'; + +export interface ValueCodec { + encode(v: V): Buffer; + decode(b: Buffer): V; +} + +const BUFFER: ValueCodec = { + encode: (v) => { + if (!Buffer.isBuffer(v)) throw new TypeError('value must be a Buffer (use valueCodec: "string" or "json")'); + return v; + }, + // Return a copy so a caller mutating the result cannot corrupt the stored + // value (the store keeps the same Buffer reference internally). + decode: (b) => Buffer.from(b), +}; +const STRING: ValueCodec = { + encode: (v) => Buffer.from(String(v), 'utf8'), + decode: (b) => b.toString('utf8'), +}; +const JSON_CODEC: ValueCodec = { + encode: (v) => Buffer.from(JSON.stringify(v), 'utf8'), + decode: (b) => JSON.parse(b.toString('utf8')), +}; +const CODECS: Record> = { buffer: BUFFER, string: STRING, json: JSON_CODEC }; +const MAX_KEY_LEN = 128; + +function toBuf(key: string | Buffer): Buffer { + return Buffer.isBuffer(key) ? key : Buffer.from(String(key), 'utf8'); +} +// Canonical byte-string form of a key: each char's code unit equals one byte of +// the key's UTF-8 encoding. The store and every derived index key their maps by +// this string, so a string key and the Buffer of its UTF-8 bytes (which is what +// the WAL/snapshot store) map to the same entry. Without this, a multi-byte +// (non-ASCII) string key is stored under one name (UTF-8 bytes, via the Buffer +// path) but looked up under another (the raw UTF-16 string), so get/del/scan and +// every index miss it. +function toKStr(key: string | Buffer): string { + return typeof key === 'string' ? Buffer.from(key, 'utf8').toString('binary') : key.toString('binary'); +} +// Inverse of toKStr: turn a canonical byte-string back into the original UTF-8 +// string for keys returned to callers (scan / findEq / dtRange / ...). +function fromKStr(k: string): string { + return Buffer.from(k, 'binary').toString('utf8'); +} +// Canonicalize the string bounds of a range scan so they compare correctly +// against the canonically-keyed ordered index. +function canonRange(opts: RangeOptions): RangeOptions { + const out: RangeOptions = { ...opts }; + if (out.gte !== undefined) out.gte = toKStr(out.gte); + if (out.gt !== undefined) out.gt = toKStr(out.gt); + if (out.lte !== undefined) out.lte = toKStr(out.lte); + if (out.lt !== undefined) out.lt = toKStr(out.lt); + return out; +} +function normDt(dt?: Record | null): Record | null { + if (!dt) return null; + const out: Record = {}; + for (const [k, v] of Object.entries(dt)) { + const ms = typeof v === 'number' ? v : Date.parse(v); + if (Number.isFinite(ms)) out[k] = ms; + } + return Object.keys(out).length ? out : null; +} + +export type ValueModeSetting = ValueMode | 'auto'; + +async function fileSize(file: string): Promise { + try { + return (await fs.stat(file)).size; + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return 0; + throw e; + } +} + +async function resolveValueMode(mode: ValueModeSetting, dir: string, maxMemoryBytes: number | null): Promise { + if (mode !== 'auto') return mode; + if (maxMemoryBytes === null) return 'memory'; + const total = (await fileSize(path.join(dir, 'db.snapshot'))) + (await fileSize(path.join(dir, 'db.wal'))); + return total > maxMemoryBytes ? 'disk' : 'memory'; +} + +export interface OpenOptions { + dir: string; + valueCodec?: ValueCodecName; + fsyncPolicy?: FsyncPolicy; + compactThresholdBytes?: number; + autoCompact?: boolean; + activeExpireIntervalMs?: number; + recovery?: RecoveryMode; + readOnly?: boolean; + onLockFail?: 'readonly'; + /** Where to keep value bulk. 'memory' keeps values in RAM; 'disk' keeps only + * value pointers in RAM and reads values from the snapshot/WAL on demand. */ + valueMode?: ValueModeSetting; + /** Approximate memory budget for stored keys/values. Undefined disables it. */ + maxMemoryBytes?: number; + /** What to do when a write would exceed maxMemoryBytes. */ + maxMemoryPolicy?: 'reject' | 'evict-lru'; +} + +export interface RestoreOptions extends Omit { + /** Overwrite an existing destination directory. */ + force?: boolean; +} + +export interface SetOptions { + ttl?: number; + dt?: Record; +} + +export type BatchInputOp = + | { op: 'set'; key: string; value: V; ttl?: number; dt?: Record } + | { op: 'del'; key: string }; + +export interface DocRecord { + key: string; + value: V; + dt?: Record; +} + +export interface ScanEntry extends DocRecord {} + +export interface QueryOptions { + key?: string | (RangeOptions & { prefix?: string }); + dt?: Record>; + text?: { index: string; q: string; op?: 'AND' | 'OR'; limit?: number }; + filter?: Record; + project?: readonly string[]; + sort?: Record; + skip?: number; + limit?: number; +} + +interface PreparedOp { + type: number; + key: Buffer; + value: Buffer | null; + meta: Buffer | null; + expireAt: number; + dtNorm: Record | null; + pk: string; + valueDecoded: V | undefined; + loc?: ValueLoc; +} + +export class MiniDb { + dir!: string; + walPath!: string; + private indexPath!: string; + private textIndexPath!: string; + private compoundIndexPath!: string; + store!: Store; + wal!: WAL; + valueReader?: ValueReader; + valueMode: ValueMode = 'memory'; + readonly indexes = new IndexManager(); + readonly dt = new DtIndex(); + readonly compound = new CompoundIndexManager(); + private readonly text = new Map(); + private textDefs: { name: string; fields: readonly string[] | null }[] = []; + + private codec!: ValueCodec; + private codecName: ValueCodecName = 'buffer'; + fsyncPolicy: FsyncPolicy = 'everysec'; + private closed = false; + recoveryInfo: RecoveryInfo | null = null; + readOnly = false; + private lock: LockFile | null = null; + + compactThresholdBytes = 64 * 1024 * 1024; + autoCompact = true; + compacting = false; + _compactDone: Promise | null = null; + /** Set only during compaction's short rotation critical section; writers park + * on it (see the write-op gate). Null the rest of the time, so the snapshot + * phase of compaction is fully non-blocking. */ + _rotateLock: Promise | null = null; + lastCompactError: unknown = null; + maxMemoryBytes: number | null = null; + maxMemoryPolicy: 'reject' | 'evict-lru' = 'reject'; + private access = new Map(); // pk -> last access seq, for LRU eviction + private accessSeq = 0; + private uniqueWriteLock: Promise = Promise.resolve(); + readonly stats = { + compactions: 0, + walBytesWritten: 0, + walFsyncs: 0, + snapshotBytesWritten: 0, + evictions: 0, + maxMemoryRejections: 0, + queryIndexHits: 0, + }; + + /** Hook called by compaction after the store snapshot + WAL are rotated, so + * derived on-disk state (text postings) can be rewritten against the new + * live set. Structural part of the CompactionTarget interface. */ + onCompacted = (): void => { + this.rebuildTextPostings(); + }; + + static async open(opts: OpenOptions): Promise> { + if (!opts || !opts.dir) throw new TypeError('MiniDb.open: opts.dir is required'); + const db = new MiniDb(); + db.dir = opts.dir; + db.walPath = path.join(db.dir, 'db.wal'); + db.indexPath = path.join(db.dir, 'db.indexes.json'); + db.textIndexPath = path.join(db.dir, 'db.textindexes.json'); + db.compoundIndexPath = path.join(db.dir, 'db.compound-indexes.json'); + db.fsyncPolicy = opts.fsyncPolicy ?? 'everysec'; + db.codecName = opts.valueCodec ?? 'buffer'; + db.codec = CODECS[db.codecName] as ValueCodec; + const valueMode: ValueModeSetting = opts.valueMode ?? 'memory'; + if (valueMode !== 'memory' && valueMode !== 'disk' && valueMode !== 'auto') { + throw new RangeError(`unknown valueMode: ${String(valueMode)}`); + } + db.compactThresholdBytes = opts.compactThresholdBytes ?? db.compactThresholdBytes; + db.autoCompact = opts.autoCompact ?? true; + db.maxMemoryBytes = opts.maxMemoryBytes ?? null; + db.maxMemoryPolicy = opts.maxMemoryPolicy ?? 'reject'; + if (db.maxMemoryBytes !== null && (!Number.isFinite(db.maxMemoryBytes) || db.maxMemoryBytes <= 0)) { + throw new RangeError('maxMemoryBytes must be a positive finite number'); + } + + await fs.mkdir(db.dir, { recursive: true }); + db.valueMode = await resolveValueMode(valueMode, db.dir, db.maxMemoryBytes); + + db.readOnly = !!opts.readOnly; + if (!db.readOnly) { + db.lock = new LockFile(path.join(db.dir, 'db.lock')); + const got = await db.lock.acquire(); + if (!got) { + if (opts.onLockFail === 'readonly') { + db.readOnly = true; + db.lock = null; + } else { + throw new LockError(`database is locked by another process: ${db.dir}`); + } + } + } + + db.store = new Store({ + activeExpireIntervalMs: opts.activeExpireIntervalMs ?? 100, + onExpire: (k, rec) => db.onStoreExpire(k, rec), + readValue: (loc) => { + if (!db.valueReader) throw new Error('ValueReader is not open'); + return db.valueReader.read(loc); + }, + }); + try { + db.wal = new WAL(db.walPath, { fsyncPolicy: db.fsyncPolicy, stats: db.stats }); + await db.wal.open(); + + db.recoveryInfo = await recover({ + dir: db.dir, + store: db.store, + mode: opts.recovery ?? 'resync', + truncate: !db.readOnly, + valueMode: db.valueMode, + }); + db.valueReader = new ValueReader(db.dir); + db.valueReader.open(); + db.seedAccessFromStore(); + + await db.loadIndexDefinitions(); + await db.loadCompoundIndexDefinitions(); + await db.loadTextIndexDefinitions(); + db.rebuildAllIndexes(); + + if (db.autoCompact && shouldCompact(db)) await compact(db); + } catch (err) { + // Release every resource acquired so far: an open that fails after the + // WAL/store are set up must not leak a file handle or keep the everysec / + // active-expire timers running. + if (db.wal) await db.wal.close().catch(() => {}); + db.valueReader?.close(); + db.store?.close(); + if (db.lock) { + await db.lock.release().catch(() => {}); + db.lock = null; + } + throw err; + } + return db; + } + + /** + * Open a database, and if opening fails due to corruption (not due to a live + * lock), delete the directory and open a fresh empty database. Recommended for + * a rebuildable cache. A live lock is rethrown. + */ + static async openOrRebuild( + opts: OpenOptions, + hooks: { onRebuild?: (err: unknown) => void } = {}, + ): Promise> { + try { + return await MiniDb.open(opts); + } catch (err) { + if (err instanceof LockError || (err as { code?: string }).code === 'ELOCKED') throw err; + // Only rebuild on errors that indicate unrecoverable/corrupt state (e.g. + // malformed index-definition JSON). Transient I/O errors (EACCES, ENOSPC, + // EIO, EMFILE, …) are rethrown so a cache opener never destroys data + // because of a recoverable system error. + const rebuildable = err instanceof SyntaxError || (err as { name?: string }).name === 'CorruptFrameError'; + if (!rebuildable) throw err; + if (hooks.onRebuild) hooks.onRebuild(err); + await fs.rm(opts.dir, { recursive: true, force: true }); + return MiniDb.open(opts); + } + } + + private encode(v: V): Buffer { + return this.codec.encode(v); + } + private decode(b: Buffer | undefined): V | undefined { + return b === undefined ? undefined : this.codec.decode(b); + } + private pk(key: string | Buffer): string { + return toKStr(key); + } + private indexable(v: unknown): v is Record { + return v !== null && typeof v === 'object'; + } + + private *liveRecords(): Generator<{ key: Buffer; value: V | undefined; dt: Record | null }> { + for (const { key, value, dt } of this.store.entries()) { + yield { key, value: this.decode(value), dt }; + } + } + + /** Live indexable records with canonical keys, for (re)building text indexes. */ + private *textRecords(): Generator<{ key: string; value: unknown }> { + for (const { key, value } of this.liveRecords()) { + if (this.indexable(value)) yield { key: this.pk(key), value }; + } + } + + /** On-disk postings file path for a text index (name sanitized for the fs). */ + private textPostingsPath(name: string): string { + const safe = name.replace(/[^a-zA-Z0-9_.-]/g, '_'); + return path.join(this.dir, `db.text-${safe}.postings`); + } + + /** Rebuild every text index's on-disk postings from the live Store. Drops + * the in-memory delta + tombstones and reclaims orphaned postings records. + * Invoked after compaction (postings are pure derived state, so this is + * only for space/latency, never for correctness). */ + private rebuildTextPostings(): void { + for (const ti of this.text.values()) ti.build(this.textRecords()); + } + + private rebuildAllIndexes(): void { + this.indexes.rebuild(this._liveRecordsRaw()); + this.dt.rebuild([...this.liveRecords()].map(({ key, dt }) => ({ key: this.pk(key), dt }))); + this.compound.rebuild(this.liveRecords()); + for (const [, ti] of this.text) ti.build(this.textRecords()); + } + + private *_liveRecordsRaw(): Generator<{ key: Buffer; value: unknown }> { + for (const { key, value } of this.store.entries()) { + yield { key, value: this.decode(value) }; + } + } + + private async loadIndexDefinitions(): Promise { + try { + const raw = await fs.readFile(this.indexPath, 'utf8'); + for (const d of JSON.parse(raw) as (IndexInfo & IndexDef)[]) this.indexes.create(d.name, d); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + } + } + private async persistIndexDefinitions(): Promise { + await fs.writeFile(this.indexPath, JSON.stringify(this.indexes.list()), 'utf8'); + } + private async loadTextIndexDefinitions(): Promise { + try { + const raw = await fs.readFile(this.textIndexPath, 'utf8'); + this.textDefs = JSON.parse(raw) as { name: string; fields: readonly string[] | null }[]; + for (const d of this.textDefs) { + this.text.set( + d.name, + new TextIndex({ + fields: d.fields, + // A read-only opener must not write to a live writer's postings file; + // it keeps the base postings in memory instead. + postingsPath: this.readOnly ? undefined : this.textPostingsPath(d.name), + }), + ); + } + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + } + } + private async persistTextIndexDefinitions(): Promise { + await fs.writeFile(this.textIndexPath, JSON.stringify(this.textDefs), 'utf8'); + } + private async loadCompoundIndexDefinitions(): Promise { + try { + const raw = await fs.readFile(this.compoundIndexPath, 'utf8'); + for (const d of JSON.parse(raw) as (CompoundIndexInfo & { name: string })[]) { + this.compound.create(d.name, { groupBy: d.groupBy, orderBy: d.orderBy, orderType: d.orderType }); + } + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + } + } + private async persistCompoundIndexDefinitions(): Promise { + await fs.writeFile(this.compoundIndexPath, JSON.stringify(this.compound.list()), 'utf8'); + } + + /** Drop every derived index entry for a key that just expired in the Store. */ + private onStoreExpire(k: string, _rec: StoreRecord): void { + this.access.delete(k); + this.dt.del(k); + this.compound.remove(k); + if (this.indexes.indexes.size) this.indexes.remove(k, undefined); + for (const ti of this.text.values()) ti.remove(k); + } + + private maybeAutoCompact(): void { + if (this.autoCompact && !this.compacting && shouldCompact(this)) compact(this).catch(() => {}); + } + + private hasUniqueIndexes(): boolean { + for (const idx of this.indexes.indexes.values()) if (idx.unique) return true; + return false; + } + + private async withUniqueWriteLock(fn: () => Promise): Promise { + const prev = this.uniqueWriteLock; + let release!: () => void; + this.uniqueWriteLock = new Promise((resolve) => { + release = resolve; + }); + await prev; + try { + return await fn(); + } finally { + release(); + } + } + + private touchAccess(pk: string): void { + this.access.set(pk, ++this.accessSeq); + } + + private seedAccessFromStore(): void { + this.access.clear(); + for (const [k] of this.store.map) this.touchAccess(k); + } + + private projectedBytesForOps(ops: readonly PreparedOp[]): number { + const considered = new Map(); + let projected = this.store.bytes; + for (const op of ops) { + const cur = considered.has(op.pk) ? considered.get(op.pk)! : this.store.recordBytes(op.pk); + projected -= cur; + const next = + op.type === TYPE_SET + ? this.store.estimateSetBytes(op.key, op.value!, op.dtNorm, { countValue: this.valueMode === 'memory' }) + : 0; + projected += next; + considered.set(op.pk, next); + } + return projected; + } + + private pickEvictionVictim(skip: Set): string | undefined { + let best: string | undefined; + let bestSeq = Infinity; + for (const [k, seq] of this.access) { + if (skip.has(k) || !this.store.map.has(k)) continue; + if (seq < bestSeq) { + best = k; + bestSeq = seq; + } + } + if (best) return best; + for (const [k] of this.store.map) if (!skip.has(k)) return k; + return undefined; + } + + private async evictKey(pk: string): Promise { + const bytes = this.store.recordBytes(pk); + if (!bytes) return; + const op = this.prepareDel(Buffer.from(pk, 'binary')); + const appended = this.wal.append(encodeFrame({ type: TYPE_DEL, key: op.key })); + const prev = this.applyOp(op); + try { + await appended; + this.stats.evictions++; + } catch (e) { + this.restoreKey(op.pk, prev); + throw e; + } + } + + private async ensureMemoryFor(ops: readonly PreparedOp[]): Promise { + if (this.maxMemoryBytes === null) return; + this.store.reapExpired(); + let projected = this.projectedBytesForOps(ops); + if (projected <= this.maxMemoryBytes) return; + + if (this.maxMemoryPolicy === 'evict-lru') { + const skip = new Set(ops.map((o) => o.pk)); + while (projected > this.maxMemoryBytes) { + const victim = this.pickEvictionVictim(skip); + if (!victim) break; + projected -= this.store.recordBytes(victim); + await this.evictKey(victim); + } + } + + if (projected > this.maxMemoryBytes) { + this.stats.maxMemoryRejections++; + throw new Error(`maxMemory exceeded: projected ${projected} bytes > ${this.maxMemoryBytes} bytes`); + } + } + + private checkKey(key: string | Buffer): void { + const len = typeof key === 'string' ? key.length : Buffer.from(key).length; + if (len > MAX_KEY_LEN) throw new RangeError(`key too long (>${MAX_KEY_LEN})`); + if ((typeof key === 'string' && key.length === 0) || (Buffer.isBuffer(key) && key.length === 0)) { + throw new RangeError('key must be non-empty'); + } + } + + // ---- KV API ------------------------------------------------------------- + + get(key: string | Buffer): V | undefined { + this.ensureOpen(); + const k = toKStr(key); + const v = this.store.get(k); + if (v !== undefined) this.touchAccess(k); + return this.decode(v); + } + + getRecord(key: string | Buffer): DocRecord | undefined { + this.ensureOpen(); + const k = toKStr(key); + const value = this.store.get(k); + if (value === undefined) return undefined; + const r = this.store.map.get(k); + this.touchAccess(k); + return { key: fromKStr(this.pk(key)), value: this.decode(value)!, dt: r?.dt ?? undefined }; + } + + async set(key: string | Buffer, value: V, { ttl, dt }: SetOptions = {}): Promise { + this.ensureOpen(); + this.ensureWritable(); + this.checkKey(key); + if (this._rotateLock) await this._rotateLock; + const op = this.prepareSet(key, value, { ttl, dt }); + await this.ensureMemoryFor([op]); + + const commit = async (): Promise => { + if (this.indexes.indexes.size && this.indexable(value)) this.indexes.checkUnique(op.pk, value); + const frame = encodeFrame({ type: TYPE_SET, key: op.key, value: op.value, meta: op.meta, expireAt: op.expireAt }); + const appended = this.wal.appendLoc(frame); + if (this.valueMode === 'disk') { + op.loc = { file: 'wal', off: appended.offset + HEADER_SIZE + op.key.length, len: op.value!.length }; + } + // Apply in the SAME synchronous tick as the WAL append, so a concurrent + // compaction always snapshots the post-write state. If the WAL write + // ultimately fails, roll the store + derived indexes back to the pre-op + // record so in-memory state never diverges from what is durable. + const prev = this.applyOp(op); + try { + await appended.done; + } catch (e) { + this.restoreKey(op.pk, prev); + throw e; + } + this.maybeAutoCompact(); + }; + + if (this.hasUniqueIndexes()) await this.withUniqueWriteLock(commit); + else await commit(); + } + + async del(key: string | Buffer): Promise { + this.ensureOpen(); + this.ensureWritable(); + if (this._rotateLock) await this._rotateLock; + const existed = this.store.get(toKStr(key)) !== undefined; + if (!existed) return false; + const op = this.prepareDel(key); + await this.ensureMemoryFor([op]); + const appended = this.wal.append(encodeFrame({ type: TYPE_DEL, key: op.key })); + const prev = this.applyOp(op); + try { + await appended; + } catch (e) { + this.restoreKey(op.pk, prev); + throw e; + } + this.maybeAutoCompact(); + return true; + } + + /** Atomically apply a batch of operations (all-or-nothing). */ + async batch(ops: readonly BatchInputOp[]): Promise { + this.ensureOpen(); + this.ensureWritable(); + if (this._rotateLock) await this._rotateLock; + if (!ops || ops.length === 0) return; + const prepared = ops.map((o) => this.prepareOp(o)); + await this.ensureMemoryFor(prepared); + + const commit = async (): Promise => { + if (this.indexes.indexes.size) { + this.indexes.checkUniqueBatch( + prepared.map((o) => ({ + pk: o.pk, + op: o.type === TYPE_DEL ? ('del' as const) : ('set' as const), + doc: o.valueDecoded, + })), + ); + } + const body = encodeBatchOps( + prepared.map((op) => ({ type: op.type, key: op.key, value: op.value, meta: op.meta, expireAt: op.expireAt })), + ); + const frame = encodeFrame({ type: TYPE_BATCH, key: Buffer.alloc(0), value: body }); + const appended = this.wal.appendLoc(frame); + if (this.valueMode === 'disk') { + const bodyOff = appended.offset + HEADER_SIZE; + const opRefs = scanBatchOpRefs(body, 0); + for (let i = 0; i < prepared.length; i++) { + const op = prepared[i]!; + const ref = opRefs[i]; + if (op.type === TYPE_SET && ref) op.loc = { file: 'wal', off: bodyOff + ref.valueOff, len: ref.valLen }; + } + } + // Capture each key's pre-batch record (first applyOp per key) so the whole + // batch can be rolled back if the WAL write fails, preserving atomicity. + const prevs = new Map(); + for (const op of prepared) { + const prev = this.applyOp(op); + if (!prevs.has(op.pk)) prevs.set(op.pk, prev); + } + try { + await appended.done; + } catch (e) { + for (const [pk, prev] of prevs) this.restoreKey(pk, prev); + throw e; + } + this.maybeAutoCompact(); + }; + + if (this.hasUniqueIndexes()) await this.withUniqueWriteLock(commit); + else await commit(); + } + + private prepareOp(o: BatchInputOp): PreparedOp { + if (o.op === 'set') return this.prepareSet(o.key, o.value, { ttl: o.ttl, dt: o.dt }); + if (o.op === 'del') return this.prepareDel(o.key); + throw new TypeError(`unknown batch op: ${(o as { op: string }).op}`); + } + + private prepareSet(key: string | Buffer, value: V, { ttl, dt }: SetOptions = {}): PreparedOp { + this.checkKey(key); + const pk = this.pk(key); + const dtNorm = normDt(dt); + // A TTL is encoded as an int64 in the frame, so it must be a finite integer + // of milliseconds. A fractional TTL is floored; a non-finite one + // (NaN / ±Infinity) is rejected up front instead of exploding inside the + // frame encoder with an opaque "cannot convert to BigInt" error. ttl 0 (or + // omitted) keeps the existing "no expiry" semantics. + if (ttl !== undefined && !Number.isFinite(ttl)) throw new RangeError('ttl must be a finite number of milliseconds'); + const expireAt = ttl ? Date.now() + Math.floor(ttl) : 0; + const vbuf = this.encode(value); + const meta = dtNorm ? Buffer.from(JSON.stringify({ dt: dtNorm })) : null; + return { type: TYPE_SET, key: toBuf(key), value: vbuf, meta, expireAt, dtNorm, pk, valueDecoded: value }; + } + + private prepareDel(key: string | Buffer): PreparedOp { + this.checkKey(key); + return { type: TYPE_DEL, key: toBuf(key), value: null, meta: null, expireAt: 0, dtNorm: null, pk: this.pk(key), valueDecoded: undefined }; + } + + /** Apply a prepared op to the store + derived indexes. Returns the key's + * pre-op logical record so the caller can roll back on WAL failure. */ + private applyOp(op: PreparedOp): StoreRecord | undefined { + const oldBuf = this.store.get(op.pk); + const prev = oldBuf !== undefined ? this.store.map.get(op.pk) : undefined; + const oldDoc = oldBuf !== undefined ? this.decode(oldBuf) : undefined; + if (op.type === TYPE_SET) { + if (this.valueMode === 'disk' && op.loc) { + const ref: ValueRef = { kind: 'disk', loc: op.loc }; + this.store.setRef(op.key, ref, op.expireAt, op.dtNorm); + } else { + this.store.set(op.key, op.value!, op.expireAt, op.dtNorm); + } + this.dt.set(op.pk, op.dtNorm); + this.compound.add(op.pk, op.valueDecoded, op.dtNorm); + if (this.indexes.indexes.size) { + if (this.indexable(oldDoc)) this.indexes.remove(op.pk, oldDoc); + if (this.indexable(op.valueDecoded)) this.indexes.add(op.pk, op.valueDecoded); + } + for (const ti of this.text.values()) { + if (this.indexable(op.valueDecoded)) ti.add(op.pk, op.valueDecoded); + else ti.remove(op.pk); + } + } else if (op.type === TYPE_DEL) { + const existed = this.store.del(op.key); + if (existed) { + this.access.delete(op.pk); + this.dt.del(op.pk); + this.compound.remove(op.pk); + if (this.indexes.indexes.size && this.indexable(oldDoc)) this.indexes.remove(op.pk, oldDoc); + for (const ti of this.text.values()) ti.remove(op.pk); + } + } + if (op.type === TYPE_SET) this.touchAccess(op.pk); + return prev; + } + + /** Roll a key back to its pre-op record across the store and every derived + * index. Used when a WAL write fails after applyOp already mutated state. */ + private restoreKey(pk: string, prev: StoreRecord | undefined): void { + if (this.indexes.indexes.size) this.indexes.remove(pk, undefined); + for (const ti of this.text.values()) ti.remove(pk); + this.dt.del(pk); + this.compound.remove(pk); + if (prev === undefined) { + this.store.del(pk); + this.access.delete(pk); + return; + } + this.store.setRef(pk, prev.ref, prev.expireAt, prev.dt); + this.touchAccess(pk); + const doc = this.decode(this.store.get(pk)); + this.dt.set(pk, prev.dt); + this.compound.add(pk, doc, prev.dt); + if (this.indexable(doc)) this.indexes.add(pk, doc); + for (const ti of this.text.values()) { + if (this.indexable(doc)) ti.add(pk, doc); + } + } + + has(key: string | Buffer): boolean { + this.ensureOpen(); + return this.store.has(toKStr(key)); + } + get size(): number { + return this.store.size; + } + async mset(entries: readonly (readonly [string, V])[]): Promise { + if (!entries.length) return; + await this.batch(entries.map(([key, value]) => ({ op: 'set' as const, key, value }))); + } + mget(keys: readonly string[]): (V | undefined)[] { + return keys.map((k) => this.get(k)); + } + + async expire(key: string | Buffer, ttlMs: number): Promise { + this.ensureOpen(); + this.ensureWritable(); + if (this._rotateLock) await this._rotateLock; + const k = toKStr(key); + const cur = this.store.getRecord(k); + if (cur === undefined) return false; + // Same validation as set(): the TTL is stored as an int64, so it must be a + // finite integer of milliseconds (fractional values are floored). + if (!Number.isFinite(ttlMs)) throw new RangeError('ttl must be a finite number of milliseconds'); + const expireAt = Date.now() + Math.floor(ttlMs); + const curValue = this.store.get(k); + if (curValue === undefined) return false; + const meta = cur.dt ? Buffer.from(JSON.stringify({ dt: cur.dt })) : null; + const keyBuf = toBuf(key); + const frame = encodeFrame({ type: TYPE_SET, key: keyBuf, value: curValue, meta, expireAt }); + const appended = this.wal.appendLoc(frame); + if (this.valueMode === 'disk') { + const ref: ValueRef = { kind: 'disk', loc: { file: 'wal', off: appended.offset + HEADER_SIZE + keyBuf.length, len: curValue.length } }; + this.store.setRef(k, ref, expireAt, cur.dt); + } else { + this.store.set(k, curValue, expireAt, cur.dt); + } + try { + await appended.done; + } catch (e) { + // Value/dt are unchanged (only the TTL moved), so restoring the store + // record is enough; derived indexes were never touched. + this.store.setRef(k, cur.ref, cur.expireAt, cur.dt); + throw e; + } + this.maybeAutoCompact(); + return true; + } + + ttl(key: string | Buffer): number { + this.ensureOpen(); + const r = this.store.map.get(toKStr(key)); + if (!r) return -2; + if (!r.expireAt) return -1; + const left = r.expireAt - Date.now(); + return left > 0 ? left : -2; + } + + // ---- key-ordered scans -------------------------------------------------- + + scan(opts: RangeOptions = {}): ScanEntry[] { + this.ensureOpen(); + const count = (opts as { limit?: number }).limit ?? Infinity; + const out: ScanEntry[] = []; + for (const r of this.store.scan({ ...canonRange(opts), count })) { + out.push({ key: r.key.toString(), value: this.decode(r.value)!, dt: r.dt ?? undefined }); + } + return out; + } + + prefix(p: string, limit = Infinity): ScanEntry[] { + this.ensureOpen(); + const out: ScanEntry[] = []; + for (const r of this.store.prefix(toKStr(p), limit)) { + out.push({ key: r.key.toString(), value: this.decode(r.value)!, dt: r.dt ?? undefined }); + } + return out; + } + + // ---- dt column queries -------------------------------------------------- + + dtColumns(): string[] { + return this.dt.columns(); + } + + dtRange(col: string, opts: RangeOptions & { limit?: number } = {}): (ScanEntry & { dtValue: number })[] { + this.ensureOpen(); + const rows = this.dt.range(col, { ...opts, count: opts.limit ?? opts.count }); + const out: (ScanEntry & { dtValue: number })[] = []; + for (const { key, value: dtValue } of rows as DtRangeEntry[]) { + const value = this.store.get(key); + if (value === undefined) continue; + const r = this.store.map.get(key); + out.push({ key: fromKStr(key), value: this.decode(value)!, dt: r?.dt ?? undefined, dtValue }); + } + return out; + } + + // ---- value secondary indexes ------------------------------------------- + + async createIndex(name: string, opts: IndexDef): Promise { + this.ensureOpen(); + this.ensureWritable(); + if (this.codecName !== 'json') throw new Error('secondary indexes require valueCodec: "json"'); + this.indexes.create(name, opts); + this.indexes.rebuild(this._liveRecordsRaw()); + try { + // A unique index must not be created over data that already violates it. + this.indexes.assertUniqueValid(name); + } catch (e) { + this.indexes.drop(name); + this.indexes.rebuild(this._liveRecordsRaw()); + throw e; + } + await this.persistIndexDefinitions(); + } + async dropIndex(name: string): Promise { + this.ensureOpen(); + this.ensureWritable(); + const ok = this.indexes.drop(name); + await this.persistIndexDefinitions(); + return ok; + } + listIndexes(): IndexInfo[] { + return this.indexes.list(); + } + findEq(name: string, value: unknown): { key: string; value: V | undefined }[] { + this.ensureOpen(); + return this.indexes + .findEq(name, value) + .map((pk) => ({ key: fromKStr(pk), value: this.decode(this.store.get(pk)) })) + .filter((r): r is { key: string; value: V } => r.value !== undefined); + } + findRange(name: string, opts: Parameters[1]): { key: string; value: V | undefined; field: number }[] { + this.ensureOpen(); + return this.indexes + .findRange(name, opts) + .map(({ pk, value }) => ({ key: fromKStr(pk), value: this.decode(this.store.get(pk)), field: value })) + .filter((r): r is { key: string; value: V; field: number } => r.value !== undefined); + } + + // ---- compound indexes (groupBy + orderBy) ------------------------------- + + async createCompoundIndex(name: string, def: CompoundIndexDef): Promise { + this.ensureOpen(); + this.ensureWritable(); + if (this.codecName !== 'json') throw new Error('compound indexes require valueCodec: "json"'); + this.compound.create(name, def); + this.compound.rebuild(this.liveRecords()); + await this.persistCompoundIndexDefinitions(); + } + + async dropCompoundIndex(name: string): Promise { + this.ensureOpen(); + this.ensureWritable(); + const ok = this.compound.drop(name); + await this.persistCompoundIndexDefinitions(); + return ok; + } + + listCompoundIndexes(): CompoundIndexInfo[] { + return this.compound.list(); + } + + /** + * Ordered range within a group, e.g. "sessions in workspace X ordered by + * updatedAt". O(log N + limit) — no full sort. + */ + compoundRange( + name: string, + groupValue: unknown, + opts: RangeOptions & { limit?: number } = {}, + ): { key: string; value: V | undefined; orderValue: unknown }[] { + this.ensureOpen(); + return this.compound + .range(name, groupValue, opts) + .map(({ key, orderValue }) => ({ + key: fromKStr(key), + value: this.decode(this.store.get(key)), + orderValue, + })) + .filter((r): r is { key: string; value: V; orderValue: unknown } => r.value !== undefined); + } + + // ---- full-text search --------------------------------------------------- + + async createTextIndex(name: string, { fields }: { fields?: readonly string[] } = {}): Promise { + this.ensureOpen(); + this.ensureWritable(); + if (this.codecName !== 'json') throw new Error('text indexes require valueCodec: "json"'); + if (this.text.has(name)) throw new Error(`text index "${name}" already exists`); + const ti = new TextIndex({ fields, postingsPath: this.textPostingsPath(name) }); + this.text.set(name, ti); + const def = { name, fields: fields ?? null }; + this.textDefs.push(def); + ti.build(this.textRecords()); + await this.persistTextIndexDefinitions(); + } + async dropTextIndex(name: string): Promise { + this.ensureOpen(); + this.ensureWritable(); + const ti = this.text.get(name); + const ok = this.text.delete(name); + if (ti) { + ti.close(); + await fs.rm(this.textPostingsPath(name), { force: true }).catch(() => {}); + } + this.textDefs = this.textDefs.filter((d) => d.name !== name); + await this.persistTextIndexDefinitions(); + return ok; + } + + search(name: string, q: string, opts: { op?: 'AND' | 'OR'; limit?: number } = {}): { key: string; value: V | undefined; score: number }[] { + this.ensureOpen(); + const ti = this.text.get(name); + if (!ti) throw new Error(`no such text index: ${name}`); + return ti + .search(q, opts) + .map(({ key, score }) => ({ key: fromKStr(key), value: this.decode(this.store.get(key)), score })) + .filter((r): r is { key: string; value: V; score: number } => r.value !== undefined); + } + + private indexPredicates(filter?: Record): { field: string; cond: unknown }[] { + if (!filter || typeof filter !== 'object') return []; + const out: { field: string; cond: unknown }[] = []; + for (const [key, cond] of Object.entries(filter)) { + if (key === '$and' && Array.isArray(cond)) { + for (const f of cond) { + if (f && typeof f === 'object') { + for (const [k, c] of Object.entries(f)) { + if (!k.startsWith('$')) out.push({ field: k, cond: c }); + } + } + } + } else if (!key.startsWith('$')) { + out.push({ field: key, cond }); + } + } + return out; + } + + private candidateKeysForPredicate(field: string, cond: unknown): Set | null { + if (this.codecName !== 'json' || !this.indexes.indexes.size) return null; + const indexes = this.indexes.list().filter((i) => i.field === field); + if (!indexes.length) return null; + + const isOpObj = cond !== null && typeof cond === 'object' && !(cond instanceof RegExp); + const ops = isOpObj ? (cond as Record) : null; + + const eqIndex = indexes.find((i) => i.type === 'equality'); + if (eqIndex) { + if (!isOpObj) return new Set(this.indexes.findEq(eqIndex.name, cond)); + if (ops && Object.keys(ops).length === 1 && '$eq' in ops) { + return new Set(this.indexes.findEq(eqIndex.name, ops['$eq'])); + } + if (ops && Array.isArray(ops['$in'])) { + const set = new Set(); + for (const v of ops['$in']) for (const pk of this.indexes.findEq(eqIndex.name, v)) set.add(pk); + return set; + } + } + + const rangeIndex = indexes.find((i) => i.type === 'range'); + if (rangeIndex && ops) { + const opts: { min?: number; max?: number; minExclusive?: boolean; maxExclusive?: boolean } = {}; + if (typeof ops['$gte'] === 'number') opts.min = ops['$gte']; + if (typeof ops['$gt'] === 'number') { + opts.min = ops['$gt']; + opts.minExclusive = true; + } + if (typeof ops['$lte'] === 'number') opts.max = ops['$lte']; + if (typeof ops['$lt'] === 'number') { + opts.max = ops['$lt']; + opts.maxExclusive = true; + } + if (opts.min !== undefined || opts.max !== undefined) { + return new Set(this.indexes.findRange(rangeIndex.name, opts).map((r) => r.pk)); + } + } + return null; + } + + private indexedCandidateKeys(filter?: Record): string[] | null { + let candidates: Set | null = null; + for (const p of this.indexPredicates(filter)) { + const set = this.candidateKeysForPredicate(p.field, p.cond); + if (!set) continue; + if (candidates) { + const next = new Set(); + for (const k of candidates) if (set.has(k)) next.add(k); + candidates = next; + } else { + candidates = set; + } + } + if (!candidates) return null; + this.stats.queryIndexHits++; + return [...candidates]; + } + + // ---- unified query ------------------------------------------------------ + + query(q: QueryOptions = {}): ScanEntry[] { + this.ensureOpen(); + + let keys: string[] | null = null; + if (typeof q.key === 'string') { + keys = [toKStr(q.key)]; + } else if (q.key && typeof q.key === 'object') { + if ((q.key as { prefix?: string }).prefix) keys = this.prefix((q.key as { prefix: string }).prefix).map((r) => toKStr(r.key)); + else { + const opts: RangeOptions = {}; + for (const b of ['gte', 'gt', 'lte', 'lt'] as const) + if ((q.key as Record)[b] !== undefined) opts[b] = (q.key as Record)[b] as string; + keys = this.scan(opts).map((r) => toKStr(r.key)); + } + } + + if (q.dt) { + for (const [col, cond] of Object.entries(q.dt)) { + const set = new Set(this.dt.range(col, cond).map((r) => r.key)); + keys = keys === null ? [...set] : keys.filter((k) => set.has(k)); + } + } + + let textOrder: { key: string; score: number }[] | null = null; + if (q.text) { + const ti = this.text.get(q.text.index); + if (!ti) throw new Error(`no such text index: ${q.text.index}`); + const hits = ti.search(q.text.q, { op: q.text.op, limit: q.text.limit ?? 1_000_000 }); + textOrder = hits; + const set = new Set(hits.map((h) => h.key)); + keys = keys === null ? hits.map((h) => h.key) : keys.filter((k) => set.has(k)); + } + + const indexed = this.indexedCandidateKeys(q.filter); + if (indexed) { + const set = new Set(indexed); + keys = keys === null ? indexed : keys.filter((k) => set.has(k)); + } + + if (keys === null) keys = this.scan().map((r) => toKStr(r.key)); + + const docs: ScanEntry[] = []; + for (const k of keys) { + const buf = this.store.get(k); + if (buf === undefined) continue; + const r = this.store.map.get(k); + const value = this.decode(buf)!; + if (q.filter && !match(value, q.filter)) continue; + docs.push({ key: k, value, dt: r?.dt ?? undefined }); + } + + if (textOrder && !q.sort) { + const rank = new Map(textOrder.map((h, i) => [h.key, i])); + docs.sort((a, b) => (rank.get(a.key) ?? 1e9) - (rank.get(b.key) ?? 1e9)); + } + + if (q.sort) { + const entries = Object.entries(q.sort); + docs.sort((a, b) => { + for (const [p, dir] of entries) { + const av = getPath(a.value, p) as number | string; + const bv = getPath(b.value, p) as number | string; + const c = av < bv ? -1 : av > bv ? 1 : 0; + if (c !== 0) return dir < 0 ? -c : c; + } + return 0; + }); + } + + const skip = q.skip ?? 0; + const limit = q.limit === undefined ? Infinity : q.limit; + const sliced = skip || limit !== Infinity ? docs.slice(skip, skip + limit) : docs; + + if (q.project) { + return sliced.map((d) => ({ key: fromKStr(d.key), value: project(d.value, q.project) as V, dt: d.dt })); + } + return sliced.map((d) => ({ ...d, key: fromKStr(d.key) })); + } + + private async persistentFiles(): Promise { + const names = await fs.readdir(this.dir); + return names.filter((n) => + /^db\.(snapshot|wal|indexes\.json|compound-indexes\.json|textindexes\.json)$/.test(n) || + /^db\.text-.*\.postings$/.test(n), + ); + } + + private async copyIfExists(name: string, destDir: string): Promise { + try { + await fs.copyFile(path.join(this.dir, name), path.join(destDir, name)); + return true; + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw e; + } + } + + /** Write a consistent online backup of this database directory. */ + async backup(destDir: string, opts: { compact?: boolean } = {}): Promise { + this.ensureOpen(); + if (!destDir) throw new TypeError('backup: destDir is required'); + if (this.compacting) await this._compactDone; + if (opts.compact !== false && !this.readOnly) await this.compact(); + if (this.compacting) await this._compactDone; + + let releaseRotation!: () => void; + this._rotateLock = new Promise((resolve) => { + releaseRotation = resolve; + }); + try { + await this.wal.flush(); + await fs.mkdir(destDir, { recursive: true }); + const files = await this.persistentFiles(); + const copied: string[] = []; + for (const name of files) if (await this.copyIfExists(name, destDir)) copied.push(name); + await fs.writeFile( + path.join(destDir, 'backup.manifest.json'), + JSON.stringify({ version: 1, createdAt: Date.now(), files: copied }, null, 2), + 'utf8', + ); + } finally { + releaseRotation(); + this._rotateLock = null; + } + } + + /** Restore a backup directory into destDir and open it. */ + static async restore(srcDir: string, destDir: string, opts: RestoreOptions = {}): Promise> { + if (!srcDir) throw new TypeError('restore: srcDir is required'); + if (!destDir) throw new TypeError('restore: destDir is required'); + const { force, ...openOpts } = opts; + + if (force) { + await fs.rm(destDir, { recursive: true, force: true }); + } else { + try { + const existing = await fs.readdir(destDir); + if (existing.length) throw new Error(`restore destination is not empty: ${destDir}`); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + } + } + await fs.mkdir(destDir, { recursive: true }); + + const names = await fs.readdir(srcDir); + for (const name of names) { + if ( + /^db\.(snapshot|wal|indexes\.json|compound-indexes\.json|textindexes\.json)$/.test(name) || + /^db\.text-.*\.postings$/.test(name) || + name === 'backup.manifest.json' + ) { + await fs.copyFile(path.join(srcDir, name), path.join(destDir, name)); + } + } + return MiniDb.open({ ...openOpts, dir: destDir }); + } + + // ---- maintenance -------------------------------------------------------- + + async compact(): Promise { + this.ensureOpen(); + this.ensureWritable(); + await compact(this); + } + + async close(): Promise { + if (this.closed) return; + if (this.compacting) await this._compactDone; + this.closed = true; + for (const ti of this.text.values()) ti.close(); + this.store.close(); + this.valueReader?.close(); + await this.wal.close(); + if (this.lock) { + await this.lock.release(); + this.lock = null; + } + } + + private ensureOpen(): void { + if (this.closed) throw new Error('MiniDb is closed'); + } + private ensureWritable(): void { + if (this.readOnly) throw new Error('MiniDb is open in read-only mode'); + } +} diff --git a/packages/minidb/src/lockfile.ts b/packages/minidb/src/lockfile.ts new file mode 100644 index 000000000..9fd624e47 --- /dev/null +++ b/packages/minidb/src/lockfile.ts @@ -0,0 +1,100 @@ +// src/lockfile.ts +// +// A small exclusive file lock using O_EXCL creation. Used to prevent two +// processes from opening the same database directory for writing (which would +// corrupt it). A lock is considered stale and is taken over only when the +// recorded owner PID is no longer alive — never merely because it is old. + +import fs from 'node:fs/promises'; +import { unlinkSync } from 'node:fs'; + +export class LockError extends Error { + readonly code = 'ELOCKED'; + constructor(message: string) { + super(message); + this.name = 'LockError'; + } +} + +function pidAlive(pid: unknown): boolean { + if (!pid || typeof pid !== 'number') return false; + try { + process.kill(pid, 0); + return true; + } catch (e) { + return (e as NodeJS.ErrnoException).code === 'EPERM'; + } +} + +// Track held locks so we can release them on process exit as a safety net. +const HELD = new Set(); +let exitHooked = false; +function hookExit(): void { + if (exitHooked) return; + exitHooked = true; + process.on('beforeExit', () => { + for (const lock of HELD) lock.releaseSync(); + }); +} + +export class LockFile { + readonly path: string; + held = false; + + constructor(path: string) { + this.path = path; + } + + /** Try to acquire the lock. Returns true if acquired, false if held by a live process. */ + async acquire(): Promise { + try { + const fh = await fs.open(this.path, 'wx'); // O_CREAT | O_EXCL | O_WRONLY + await fh.writeFile(JSON.stringify({ pid: process.pid, ts: Date.now() })); + await fh.close(); + this.markHeld(); + return true; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e; + } + if (await this.isStale()) { + await fs.unlink(this.path).catch(() => {}); + return this.acquire(); + } + return false; + } + + private async isStale(): Promise { + try { + const raw = await fs.readFile(this.path, 'utf8'); + const { pid } = JSON.parse(raw) as { pid?: number }; + return !pidAlive(pid); + } catch { + return true; + } + } + + private markHeld(): void { + this.held = true; + HELD.add(this); + hookExit(); + } + + async release(): Promise { + if (!this.held) return; + await fs.unlink(this.path).catch(() => {}); + this.held = false; + HELD.delete(this); + } + + /** Best-effort sync release for the exit hook. */ + releaseSync(): void { + if (!this.held) return; + try { + unlinkSync(this.path); + } catch { + /* ignore */ + } + this.held = false; + HELD.delete(this); + } +} diff --git a/packages/minidb/src/query.ts b/packages/minidb/src/query.ts new file mode 100644 index 000000000..1bd20f1fc --- /dev/null +++ b/packages/minidb/src/query.ts @@ -0,0 +1,153 @@ +// src/query.ts +// +// A small, zero-dependency "jq / MongoDB-ish" query engine over JSON documents: +// - path get/set/projection (dot + bracket index: "a.b[0].c") +// - filter predicates (Mongo-like: { age: { $gt: 18 }, $or: [...] }) +// - sort / skip / limit + +export type Doc = unknown; + +type Path = string | readonly (string | number)[]; + +function tokenizePath(path: Path): (string | number)[] { + if (Array.isArray(path)) return [...path]; + const tokens: (string | number)[] = []; + for (const seg of String(path).split('.')) { + let s = seg; + while (s.length) { + const m = s.match(/^([^[]*)\[(\d+)\](.*)$/); + if (m) { + if (m[1]) tokens.push(m[1]); + tokens.push(Number(m[2])); + s = m[3]!; + } else { + tokens.push(s); + s = ''; + } + } + } + return tokens; +} + +export function getPath(doc: Doc, path: Path): unknown { + let cur: unknown = doc; + for (const t of tokenizePath(path)) { + if (cur == null) return undefined; + cur = (cur as Record)[t]; + } + return cur; +} + +export function setPath(obj: Doc, path: Path, value: unknown): Doc { + const tokens = tokenizePath(path); + let cur = obj as Record; + for (let i = 0; i < tokens.length - 1; i++) { + const t = tokens[i]!; + if (cur[t] == null || typeof cur[t] !== 'object') { + cur[t] = typeof tokens[i + 1] === 'number' ? [] : {}; + } + cur = cur[t] as Record; + } + cur[tokens[tokens.length - 1]!] = value; + return obj; +} + +/** Keep only the given paths (inclusion). Returns a new object. */ +export function project(doc: Doc, paths?: readonly string[]): Doc { + if (!paths || !paths.length) return doc; + const out: Record = {}; + for (const p of paths) { + const v = getPath(doc, p); + if (v !== undefined) setPath(out, p, v); + } + return out; +} + +// --- filter -------------------------------------------------------------- + +type Cond = unknown; + +function matchCond(val: unknown, cond: Cond): boolean { + if (cond === null || typeof cond !== 'object' || cond instanceof RegExp) { + if (cond instanceof RegExp) { + // A caller-supplied RegExp with the global/sticky flag is stateful: + // .test() advances lastIndex. Reset it so every document is tested from + // the start instead of alternating match/miss across documents. + cond.lastIndex = 0; + return typeof val === 'string' && cond.test(val); + } + return val === cond; + } + for (const op of Object.keys(cond as Record)) { + const arg = (cond as Record)[op]; + switch (op) { + case '$eq': + if (val !== arg) return false; + break; + case '$ne': + if (val === arg) return false; + break; + case '$gt': + if (!((val as number) > (arg as number))) return false; + break; + case '$gte': + if (!((val as number) >= (arg as number))) return false; + break; + case '$lt': + if (!((val as number) < (arg as number))) return false; + break; + case '$lte': + if (!((val as number) <= (arg as number))) return false; + break; + case '$in': + if (!Array.isArray(arg) || !arg.includes(val)) return false; + break; + case '$nin': + if (!Array.isArray(arg) || arg.includes(val)) return false; + break; + case '$regex': { + const re = + arg instanceof RegExp ? arg : Array.isArray(arg) ? new RegExp(arg[0] as string, arg[1] as string | undefined) : new RegExp(arg as string); + if (typeof val !== 'string') return false; + // Reset a stateful (global/sticky) RegExp so a reused instance does not + // carry lastIndex over from the previous document. + re.lastIndex = 0; + if (!re.test(val)) return false; + break; + } + case '$exists': + if ((val !== undefined) !== !!arg) return false; + break; + case '$contains': + if (!Array.isArray(val) || !val.includes(arg)) return false; + break; + case '$type': + if (typeof val !== arg) return false; + break; + default: + return false; + } + } + return true; +} + +/** Does `doc` satisfy the Mongo-like `filter`? */ +export function match(doc: Doc, filter?: Record | null): boolean { + if (!filter || Object.keys(filter).length === 0) return true; + for (const key of Object.keys(filter)) { + const cond = filter[key]; + if (key === '$and') { + if (!Array.isArray(cond) || !cond.every((f) => match(doc, f as Record))) return false; + } else if (key === '$or') { + if (!Array.isArray(cond) || !cond.some((f) => match(doc, f as Record))) return false; + } else if (key === '$nor') { + if (!Array.isArray(cond) || cond.some((f) => match(doc, f as Record))) return false; + } else if (key === '$not') { + if (match(doc, cond as Record)) return false; + } else { + if (!matchCond(getPath(doc, key), cond)) return false; + } + } + return true; +} + diff --git a/packages/minidb/src/recovery.ts b/packages/minidb/src/recovery.ts new file mode 100644 index 000000000..c25064ed8 --- /dev/null +++ b/packages/minidb/src/recovery.ts @@ -0,0 +1,171 @@ +// src/recovery.ts +// +// Startup recovery: load the latest snapshot (if any) then replay the WAL on +// top, last-writer-wins. In valueMode:'disk' recovery scans frames without +// copying values and stores { file, off, len } pointers instead. + +import fs from 'node:fs/promises'; +import fsSync from 'node:fs'; +import path from 'node:path'; +import { scanFrameRefsFd, scanBatchOpRefs, TYPE_SET, TYPE_DEL, TYPE_BATCH } from './codec.js'; +import type { FrameRef } from './codec.js'; +import type { Store, ValueLoc, ValueRef } from './store.js'; + +export type RecoveryMode = 'resync' | 'strict'; +export type ValueMode = 'memory' | 'disk'; + +export interface RecoveryInfo { + snapshotFrames: number; + walFrames: number; + truncatedWal: boolean; + corruptRanges: [number, number][]; + snapshotCorruptRanges: [number, number][]; + lostBytes: number; +} + +function readAtSync(fd: number, off: number, len: number): Buffer { + if (len === 0) return Buffer.alloc(0); + const buf = Buffer.allocUnsafe(len); + let got = 0; + while (got < len) { + const r = fsSync.readSync(fd, buf, got, len - got, off + got); + if (r === 0) throw new Error('recovery: short read past EOF'); + got += r; + } + return buf; +} + +function parseMeta(meta: Buffer | null): Record | null { + if (!meta) return null; + const parsed = JSON.parse(meta.toString('utf8')) as { dt?: Record }; + return parsed.dt ?? null; +} + +function applySetRef( + f: { key: Buffer; valueOff: number; valLen: number; meta: Buffer | null; expireAt: number }, + file: ValueLoc['file'], + fd: number, + store: Store, + valueMode: ValueMode, +): void { + // A record whose TTL already elapsed while the db was closed must not be + // replayed as a live key: that would make `size` count a key scan/get hide, + // and would rebuild indexes without it (inconsistency). + // + // We must also actively DROP the key: an expired SET is still the *latest* + // write for its key. If an older live value for the same key was already + // loaded from the snapshot (or an earlier WAL frame), simply skipping this + // frame would leave that stale value behind — resurrecting a key the most + // recent write had already expired. Deleting it preserves last-writer-wins + // semantics: a later op (another SET, or a DEL) will re-establish the key if + // needed; otherwise the key stays gone, as its expired TTL dictates. + if (f.expireAt && f.expireAt <= Date.now()) { + store.del(f.key); + return; + } + const dt = parseMeta(f.meta); + if (valueMode === 'disk') { + const ref: ValueRef = { kind: 'disk', loc: { file, off: f.valueOff, len: f.valLen } }; + store.setRef(f.key, ref, f.expireAt, dt); + } else { + store.set(f.key, readAtSync(fd, f.valueOff, f.valLen), f.expireAt, dt); + } +} + +function applyBatchRef( + f: FrameRef, + file: ValueLoc['file'], + fd: number, + store: Store, + valueMode: ValueMode, +): void { + let ops; + try { + ops = scanBatchOpRefs(readAtSync(fd, f.valueOff, f.valLen), f.valueOff); + } catch { + // A malformed body with a valid outer CRC can only come from an encoder + // bug. Skip the whole batch rather than half-apply it, preserving the + // all-or-nothing guarantee. + return; + } + for (const op of ops) { + if (op.type === TYPE_SET) applySetRef(op, file, fd, store, valueMode); + else if (op.type === TYPE_DEL) store.del(op.key); + } +} + +function applyFrames(frames: FrameRef[], file: ValueLoc['file'], fd: number, store: Store, valueMode: ValueMode): void { + for (const f of frames) { + if (f.type === TYPE_SET) applySetRef(f, file, fd, store, valueMode); + else if (f.type === TYPE_DEL) store.del(f.key); + else if (f.type === TYPE_BATCH) applyBatchRef(f, file, fd, store, valueMode); + } +} + +export async function recover({ + dir, + store, + mode = 'resync', + truncate = true, + valueMode = 'memory', +}: { + dir: string; + store: Store; + mode?: RecoveryMode; + truncate?: boolean; + valueMode?: ValueMode; +}): Promise { + const snapPath = path.join(dir, 'db.snapshot'); + const walPath = path.join(dir, 'db.wal'); + + let snapshotFrames = 0; + let snapshotCorrupt: [number, number][] = []; + if (fsSync.existsSync(snapPath)) { + const fd = fsSync.openSync(snapPath, 'r'); + try { + const r = scanFrameRefsFd(fd, { onCorrupt: mode }); + applyFrames(r.frames, 'snapshot', fd, store, valueMode); + snapshotFrames = r.frames.length; + snapshotCorrupt = r.corruptRanges; + } finally { + fsSync.closeSync(fd); + } + } + + let walFrames = 0; + let walCorrupt: [number, number][] = []; + let truncatedWal = false; + if (fsSync.existsSync(walPath)) { + const fd = fsSync.openSync(walPath, 'r'); + let walSize = 0; + try { + walSize = fsSync.fstatSync(fd).size; + const r = scanFrameRefsFd(fd, { onCorrupt: mode }); + applyFrames(r.frames, 'wal', fd, store, valueMode); + walFrames = r.frames.length; + walCorrupt = r.corruptRanges; + const last = r.corruptRanges[r.corruptRanges.length - 1]; + if (last && last[1] === walSize) { + // A torn/corrupt tail is normally truncated so the next writer appends + // cleanly. In read-only mode (truncate = false) we must never mutate the + // database files: a read-only opener racing a live writer could otherwise + // observe a momentarily-incomplete tail and destroy live data. + if (truncate) { + await fs.truncate(walPath, last[0]); + truncatedWal = true; + } + } + } finally { + fsSync.closeSync(fd); + } + } + + return { + snapshotFrames, + walFrames, + truncatedWal, + corruptRanges: walCorrupt, + snapshotCorruptRanges: snapshotCorrupt, + lostBytes: [...walCorrupt, ...snapshotCorrupt].reduce((a, [s, e]) => a + (e - s), 0), + }; +} diff --git a/packages/minidb/src/server.ts b/packages/minidb/src/server.ts new file mode 100644 index 000000000..7ac24e53b --- /dev/null +++ b/packages/minidb/src/server.ts @@ -0,0 +1,210 @@ +// src/server.ts +// +// A minimal RESP (REdis Serialization Protocol) TCP front-end for MiniDb, so +// existing Redis clients (redis-cli, ioredis, ...) can talk to it. + +import net from 'node:net'; +import type { Socket } from 'node:net'; +import { MiniDb } from './index.js'; + +const CRLF = '\r\n'; +const NIL = `$-1${CRLF}`; + +const reply = { + ok: () => `+OK${CRLF}`, + pong: () => `+PONG${CRLF}`, + int: (n: number) => `:${n}${CRLF}`, + err: (m: string) => `-ERR ${m}${CRLF}`, + // Bulk replies carry raw bytes. Build a Buffer so non-ASCII / binary values + // are written verbatim instead of being re-encoded as UTF-8 (which corrupted + // them and desynced the protocol when `socket.write(string)` defaulted to + // utf8). + bulk: (v: unknown): Buffer => { + if (v === undefined || v === null) return Buffer.from(NIL); + const b = Buffer.isBuffer(v) ? v : Buffer.from(String(v)); + return Buffer.concat([Buffer.from(`$${b.length}${CRLF}`), b, Buffer.from(CRLF)]); + }, + array: (items: unknown[]): Buffer => { + const parts: Buffer[] = [Buffer.from(`*${items.length}${CRLF}`)]; + for (const it of items) parts.push(reply.bulk(it)); + return Buffer.concat(parts); + }, +}; + +class RespParser { + private buf: Buffer = Buffer.alloc(0); + private readonly maxBuf: number; + + constructor({ maxBuf = 64 * 1024 * 1024 }: { maxBuf?: number } = {}) { + this.maxBuf = maxBuf; + } + + *feed(chunk: Buffer): Generator { + this.buf = this.buf.length ? Buffer.concat([this.buf, chunk]) : chunk; + if (this.buf.length > this.maxBuf) { + throw new Error(`RESP request too large (>${this.maxBuf} bytes)`); + } + while (this.buf.length) { + const parsed = this.tryParse(); + if (!parsed) break; + yield parsed; + } + } + + private tryParse(): Buffer[] | null { + if (this.buf[0] !== 0x2a /* '*' */) { + const idx = this.buf.indexOf(CRLF); + if (idx === -1) return null; + const line = this.buf.subarray(0, idx).toString(); + this.buf = this.buf.subarray(idx + 2); + return line.split(' ').filter(Boolean).map((s) => Buffer.from(s)); + } + + let pos = 1; + let end = this.buf.indexOf(CRLF, pos); + if (end === -1) return null; + const argc = Number(this.buf.subarray(pos, end).toString()); + pos = end + 2; + + const args: Buffer[] = []; + for (let i = 0; i < argc; i++) { + if (pos >= this.buf.length || this.buf[pos] !== 0x24 /* '$' */) return null; + pos++; + end = this.buf.indexOf(CRLF, pos); + if (end === -1) return null; + const len = Number(this.buf.subarray(pos, end).toString()); + pos = end + 2; + if (this.buf.length - pos < len + 2) return null; + args.push(this.buf.subarray(pos, pos + len)); + pos += len + 2; + } + this.buf = this.buf.subarray(pos); + return args; + } +} + +async function handle(db: MiniDb, args: Buffer[]): Promise { + const cmd = args[0]!.toString().toUpperCase(); + const S = (i: number): string | undefined => (args[i] === undefined ? undefined : args[i]!.toString()); + + switch (cmd) { + case 'PING': + return args[1] ? reply.bulk(S(1)) : reply.pong(); + case 'ECHO': + return reply.bulk(S(1)); + case 'GET': { + const v = db.get(S(1)!); + return reply.bulk(v === undefined ? null : v); + } + case 'SET': { + const key = S(1)!; + const val = S(2)!; + let ttl: number | undefined; + for (let i = 3; i < args.length; i++) { + const opt = S(i)!.toUpperCase(); + if (opt === 'EX') ttl = Number(S(++i)) * 1000; + else if (opt === 'PX') ttl = Number(S(++i)); + } + await db.set(key, val, ttl ? { ttl } : {}); + return reply.ok(); + } + case 'DEL': { + let n = 0; + for (let i = 1; i < args.length; i++) if (await db.del(S(i)!)) n++; + return reply.int(n); + } + case 'EXISTS': + return reply.int(db.has(S(1)!) ? 1 : 0); + case 'MGET': { + const out: unknown[] = []; + for (let i = 1; i < args.length; i++) { + const v = db.get(S(i)!); + out.push(v === undefined ? null : v); + } + return reply.array(out); + } + case 'MSET': { + const entries: (readonly [string, string])[] = []; + for (let i = 1; i + 1 < args.length; i += 2) entries.push([S(i)!, S(i + 1)!]); + await db.mset(entries); // atomic batch (single WAL frame), like Redis MSET + return reply.ok(); + } + case 'TTL': + return reply.int(Math.trunc(db.ttl(S(1)!) / 1000)); + case 'DBSIZE': + return reply.int(db.size); + case 'COMPACT': + await db.compact(); + return reply.ok(); + case 'INFO': + return reply.bulk(`minidb_version:0.0.1${CRLF}keys:${db.size}${CRLF}compactions:${db.stats.compactions}${CRLF}`); + case 'QUIT': + return null; + default: + return reply.err(`unknown command '${cmd}'`); + } +} + +export interface ServerOptions { + dir: string; + port?: number; + host?: string; + fsyncPolicy?: 'always' | 'everysec' | 'no'; +} + +export interface ServerHandle { + server: net.Server; + db: MiniDb; + close: () => Promise; + port: number; + host: string; +} + +export async function startServer({ dir, port = 6379, host = '127.0.0.1', fsyncPolicy = 'everysec' }: ServerOptions): Promise { + const db = (await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy })) as MiniDb; + const server = net.createServer((socket: Socket) => { + const parser = new RespParser(); + socket.on('data', (chunk: Buffer) => { + (async () => { + try { + for (const args of parser.feed(chunk)) { + const res = await handle(db, args); + if (res === null) { + socket.end(); + return; + } + socket.write(res); + } + } catch (e) { + socket.write(reply.err((e as Error).message)); + } + })(); + }); + }); + + await new Promise((resolve) => server.listen(port, host, resolve)); + const actualPort = (server.address() as net.AddressInfo).port; + + const close = async (): Promise => { + server.close(); + await db.close(); + }; + process.on('SIGINT', () => { + void close().then(() => process.exit(0)); + }); + return { server, db, close, port: actualPort, host }; +} + +// Run directly: node --import tsx src/server.ts --dir ./data --port 6379 +if (import.meta.url === `file://${process.argv[1]}`) { + const argv = process.argv.slice(2); + const arg = (name: string, def: string): string => { + const i = argv.indexOf(`--${name}`); + return i === -1 ? def : argv[i + 1]!; + }; + const dir = arg('dir', './data'); + const port = Number(arg('port', '6379')); + const fsyncPolicy = arg('fsync', 'everysec') as 'always' | 'everysec' | 'no'; + const { host, port: p } = await startServer({ dir, port, fsyncPolicy }); + console.log(`minidb RESP server listening on ${host}:${p} (dir=${dir}, fsync=${fsyncPolicy})`); +} diff --git a/packages/minidb/src/skiplist.ts b/packages/minidb/src/skiplist.ts new file mode 100644 index 000000000..43501cbb6 --- /dev/null +++ b/packages/minidb/src/skiplist.ts @@ -0,0 +1,262 @@ +// src/skiplist.ts +// +// A comparator-driven skip list (Redis zskiplist, generalized). Nodes are +// ordered by a sort `key` with a deterministic tie-break on `val`; the `span` +// field on each level gives O(log N) rank access and efficient range scans. + +const MAX_LEVEL = 32; +const P = 0.25; + +export type Comparator = (a: T, b: T) => number; + +export const cmpNumber: Comparator = (a, b) => a - b; +export const cmpString: Comparator = (a, b) => (a < b ? -1 : a > b ? 1 : 0); + +function randomLevel(): number { + let lvl = 1; + while (Math.random() < P && lvl < MAX_LEVEL) lvl++; + return lvl; +} + +interface Level { + forward: SkipNode | null; + span: number; +} + +class SkipNode { + key: K; + val: V; + backward: SkipNode | null = null; + level: Level[]; + + constructor(key: K, val: V, level: number) { + this.key = key; + this.val = val; + this.level = Array.from({ length: level }, () => ({ forward: null, span: 0 })); + } +} + +export interface SkipListOptions { + compareKey?: Comparator; + compareVal?: Comparator; +} + +export interface RangeOptions { + gte?: K; + gt?: K; + lte?: K; + lt?: K; + offset?: number; + count?: number; + reverse?: boolean; +} + +export interface RangeEntry { + key: K; + val: V; +} + +export class SkipList { + private readonly cmpK: Comparator; + private readonly cmpV: Comparator; + private readonly header: SkipNode; + private tail: SkipNode | null = null; + length = 0; + private level = 1; + + constructor(opts: SkipListOptions = {}) { + this.cmpK = opts.compareKey ?? (cmpNumber as unknown as Comparator); + this.cmpV = opts.compareVal ?? (cmpString as unknown as Comparator); + this.header = new SkipNode(undefined as unknown as K, undefined as unknown as V, MAX_LEVEL); + } + + private nodeLess(a: SkipNode, b: { key: K; val: V }): boolean { + const c = this.cmpK(a.key, b.key); + return c < 0 || (c === 0 && this.cmpV(a.val, b.val) < 0); + } + + insert(key: K, val: V): SkipNode { + const update = new Array>(MAX_LEVEL); + const rank = new Array(MAX_LEVEL).fill(0); + let x: SkipNode = this.header; + const target = { key, val }; + + for (let i = this.level - 1; i >= 0; i--) { + rank[i] = i === this.level - 1 ? 0 : rank[i + 1]!; + let f = x.level[i]!.forward; + while (f && this.nodeLess(f, target)) { + rank[i]! += x.level[i]!.span; + x = f; + f = x.level[i]!.forward; + } + update[i] = x; + } + + let lvl = randomLevel(); + if (lvl > this.level) { + for (let i = this.level; i < lvl; i++) { + rank[i] = 0; + update[i] = this.header; + update[i]!.level[i]!.span = this.length; + } + this.level = lvl; + } + + x = new SkipNode(key, val, lvl); + for (let i = 0; i < lvl; i++) { + x.level[i]!.forward = update[i]!.level[i]!.forward; + update[i]!.level[i]!.forward = x; + x.level[i]!.span = update[i]!.level[i]!.span - (rank[0]! - rank[i]!); + update[i]!.level[i]!.span = rank[0]! - rank[i]! + 1; + } + for (let i = lvl; i < this.level; i++) update[i]!.level[i]!.span++; + + x.backward = update[0] === this.header ? null : update[0]!; + if (x.level[0]!.forward) x.level[0]!.forward.backward = x; + else this.tail = x; + this.length++; + return x; + } + + private deleteNode(x: SkipNode, update: SkipNode[]): void { + for (let i = 0; i < this.level; i++) { + if (update[i]!.level[i]!.forward === x) { + update[i]!.level[i]!.span += x.level[i]!.span - 1; + update[i]!.level[i]!.forward = x.level[i]!.forward; + } else { + update[i]!.level[i]!.span--; + } + } + if (x.level[0]!.forward) x.level[0]!.forward.backward = x.backward; + else this.tail = x.backward; + while (this.level > 1 && this.header.level[this.level - 1]!.forward === null) this.level--; + this.length--; + } + + delete(key: K, val: V): boolean { + const update = new Array>(MAX_LEVEL); + let x: SkipNode = this.header; + const target = { key, val }; + for (let i = this.level - 1; i >= 0; i--) { + let f = x.level[i]!.forward; + while (f && this.nodeLess(f, target)) { + x = f; + f = x.level[i]!.forward; + } + update[i] = x; + } + const last = x.level[0]!.forward; + if (last && this.cmpK(last.key, key) === 0 && this.cmpV(last.val, val) === 0) { + this.deleteNode(last, update); + return true; + } + return false; + } + + /** First node with key >= bound (or > bound if strict). */ + lowerBound(bound: K, { strict = false }: { strict?: boolean } = {}): SkipNode | null { + let x: SkipNode = this.header; + for (let i = this.level - 1; i >= 0; i--) { + let f: SkipNode | null; + while ( + (f = x.level[i]!.forward) && + (strict ? this.cmpK(f.key, bound) <= 0 : this.cmpK(f.key, bound) < 0) + ) { + x = f; + } + } + return x.level[0]!.forward; + } + + /** 0-based rank of (key, val), or null if absent. */ + getRank(key: K, val: V): number | null { + let x: SkipNode = this.header; + let rank = 0; + const target = { key, val }; + for (let i = this.level - 1; i >= 0; i--) { + let f = x.level[i]!.forward; + while (f && (this.nodeLess(f, target) || (this.cmpK(f.key, key) === 0 && this.cmpV(f.val, val) === 0))) { + rank += x.level[i]!.span; + x = f; + f = x.level[i]!.forward; + } + } + if (x !== this.header && this.cmpK(x.key, key) === 0 && this.cmpV(x.val, val) === 0) return rank - 1; + return null; + } + + /** Node at 0-based rank, or null. */ + getByRank(rank: number): RangeEntry | null { + if (rank < 0 || rank >= this.length) return null; + const target = rank + 1; + let x: SkipNode = this.header; + let traversed = 0; + for (let i = this.level - 1; i >= 0; i--) { + let f = x.level[i]!.forward; + while (f && traversed + x.level[i]!.span <= target) { + traversed += x.level[i]!.span; + x = f; + f = x.level[i]!.forward; + } + if (traversed === target) return { key: x.key, val: x.val }; + } + return null; + } + + /** Range scan. */ + range(opts: RangeOptions = {}): RangeEntry[] { + let offset = opts.offset ?? 0; + let count = opts.count ?? Infinity; + const out: RangeEntry[] = []; + + if (opts.reverse) { + let x: SkipNode | null; + if (opts.lte !== undefined) { + const after = this.lowerBound(opts.lte, { strict: true }); + x = after ? after.backward : this.tail; + } else if (opts.lt !== undefined) { + const after = this.lowerBound(opts.lt, { strict: false }); + x = after ? after.backward : this.tail; + } else { + x = this.tail; + } + while (x) { + if (opts.gte !== undefined && this.cmpK(x.key, opts.gte) < 0) break; + if (opts.gt !== undefined && this.cmpK(x.key, opts.gt) <= 0) break; + if (offset > 0) offset--; + else if (count > 0) { + out.push({ key: x.key, val: x.val }); + count--; + } else break; + x = x.backward; + } + return out; + } + + const hasLower = opts.gte !== undefined || opts.gt !== undefined; + let x = hasLower + ? this.lowerBound(opts.gte !== undefined ? opts.gte : (opts.gt as K), { strict: opts.gt !== undefined }) + : this.header.level[0]!.forward; + while (x) { + if (opts.lte !== undefined && this.cmpK(x.key, opts.lte) > 0) break; + if (opts.lt !== undefined && this.cmpK(x.key, opts.lt) >= 0) break; + if (offset > 0) offset--; + else if (count > 0) { + out.push({ key: x.key, val: x.val }); + count--; + } else break; + x = x.level[0]!.forward; + } + return out; + } + + toArray(): RangeEntry[] { + const out: RangeEntry[] = []; + let x = this.header.level[0]!.forward; + while (x) { + out.push({ key: x.key, val: x.val }); + x = x.level[0]!.forward; + } + return out; + } +} diff --git a/packages/minidb/src/snapshot.ts b/packages/minidb/src/snapshot.ts new file mode 100644 index 000000000..ff5f8a510 --- /dev/null +++ b/packages/minidb/src/snapshot.ts @@ -0,0 +1,88 @@ +// src/snapshot.ts +// +// Write a point-in-time snapshot of the live Store to a temp file as a sequence +// of SET frames (tombstones dropped — only live keys are emitted). The caller is +// responsible for the atomic rename + WAL rotation. +// +// We yield to the event loop every `yieldEvery` entries so a large snapshot does +// not starve other work. + +import fs from 'node:fs/promises'; +import type { FileHandle } from 'node:fs/promises'; +import { encodeFrame, HEADER_SIZE, TYPE_SET } from './codec.js'; +import type { Store, ValueLoc } from './store.js'; + +const yieldToLoop = (): Promise => new Promise((r) => setImmediate(r)); +const FLUSH_BYTES = 1 << 20; // coalesce into ~1 MiB writev batches + +export interface SnapshotResult { + count: number; + bytes: number; + locs: Map; +} + +export async function writeSnapshot( + store: Store, + tmpPath: string, + opts: { yieldEvery?: number } = {}, +): Promise { + const yieldEvery = opts.yieldEvery ?? 2000; + const fh: FileHandle = await fs.open(tmpPath, 'w'); + let count = 0; + let bytes = 0; + let batch: Buffer[] = []; + let batchBytes = 0; + const locs = new Map(); + + const flushBatch = async (): Promise => { + if (batch.length === 0) return; + // writev(2) may short-write (signal interruption, RLIMIT_FSIZE, …). Loop + // until every byte is on the kernel side; otherwise the snapshot would be + // silently truncated and later renamed over the good one. + let bufs = batch; + let off = 0; // byte offset within bufs[0] + while (bufs.length > 0) { + const toWrite = off > 0 ? [bufs[0]!.subarray(off), ...bufs.slice(1)] : bufs; + const { bytesWritten } = await fh.writev(toWrite); + if (bytesWritten === 0) throw new Error('snapshot writev made no progress (short write)'); + bytes += bytesWritten; + let rem = bytesWritten; + while (rem > 0 && bufs.length > 0) { + const left = bufs[0]!.length - off; + if (rem < left) { + off += rem; + rem = 0; + } else { + rem -= left; + bufs.shift(); + off = 0; + } + } + } + batch = []; + batchBytes = 0; + }; + + try { + for (const { key, value, expireAt, dt } of store.entries()) { + const meta = dt ? Buffer.from(JSON.stringify({ dt })) : null; + const frame = encodeFrame({ type: TYPE_SET, key, value, expireAt, meta }); + const frameOff = bytes + batchBytes; + locs.set(key.toString('binary'), { + file: 'snapshot', + off: frameOff + HEADER_SIZE + key.length, + len: value.length, + }); + batch.push(frame); + batchBytes += frame.length; + count++; + if (batchBytes >= FLUSH_BYTES) await flushBatch(); + if (count % yieldEvery === 0) await yieldToLoop(); + } + await flushBatch(); + await fh.sync(); + } finally { + await fh.close(); + } + return { count, bytes, locs }; +} diff --git a/packages/minidb/src/store.ts b/packages/minidb/src/store.ts new file mode 100644 index 000000000..ed04eafda --- /dev/null +++ b/packages/minidb/src/store.ts @@ -0,0 +1,317 @@ +// src/store.ts +// +// In-memory primary index with TTL and an ordered key index. Values may be held +// either inline in memory or as a pointer into the on-disk snapshot/WAL; in the +// latter case the caller injects a synchronous positioned reader. + +import { SkipList, cmpString } from './skiplist.js'; +import type { RangeEntry, RangeOptions } from './skiplist.js'; + +export interface ValueLoc { + file: 'snapshot' | 'wal'; + off: number; + len: number; +} + +export type ValueRef = { kind: 'memory'; value: Buffer } | { kind: 'disk'; loc: ValueLoc }; + +export interface StoreRecord { + ref: ValueRef; + expireAt: number; + seq: number; + dt: Record | null; +} + +export interface StoreEntry { + key: Buffer; + value: Buffer; + expireAt: number; + dt: Record | null; +} + +export type ValueReader = (loc: ValueLoc) => Buffer; + +const toKStr = (key: string | Buffer): string => + typeof key === 'string' ? key : Buffer.from(key).toString('binary'); +const fromKStr = (kstr: string): Buffer => Buffer.from(kstr, 'binary'); +const DISK_REF_BYTES = 64; + +interface HeapEntry { + t: number; + k: string; + seq: number; +} + +class MinHeap { + private a: HeapEntry[] = []; + get size(): number { + return this.a.length; + } + clear(): void { + this.a = []; + } + peek(): HeapEntry | undefined { + return this.a[0]; + } + push(item: HeapEntry): void { + const a = this.a; + a.push(item); + let i = a.length - 1; + while (i > 0) { + const p = (i - 1) >> 1; + if (a[p]!.t <= a[i]!.t) break; + [a[p], a[i]] = [a[i]!, a[p]!]; + i = p; + } + } + pop(): HeapEntry | undefined { + const a = this.a; + const top = a[0]; + const last = a.pop(); + if (a.length && last !== undefined) { + a[0] = last; + let i = 0; + while (true) { + let s = i; + const l = 2 * i + 1; + const r = l + 1; + if (l < a.length && a[l]!.t < a[s]!.t) s = l; + if (r < a.length && a[r]!.t < a[s]!.t) s = r; + if (s === i) break; + [a[s], a[i]] = [a[i]!, a[s]!]; + i = s; + } + } + return top; + } +} + +export interface StoreOptions { + activeExpireIntervalMs?: number; + activeExpireMaxPerTick?: number; + /** Read a value back from disk for disk-backed records. Required whenever a + * StoreRecord may hold a disk ref. */ + readValue?: ValueReader; + /** Called after a key is removed due to TTL expiration (lazy or active), + * so the owner can drop derived state (secondary/dt/text indexes). Not + * called for explicit del(), which the owner handles itself. */ + onExpire?: (key: string, record: StoreRecord) => void; +} + +export class Store { + readonly map = new Map(); // kstr -> record + private readonly order = new SkipList({ compareKey: cmpString }); // kstr ordered + private readonly heap = new MinHeap(); + private seq = 0; + /** Approximate bytes held by live + expired-not-yet-reaped records. In + * valueMode:'disk' this counts keys/metadata/refs, not the value bulk. */ + bytes = 0; + private readonly maxPerTick: number; + private timer: ReturnType | null = null; + private readonly onExpire?: (key: string, record: StoreRecord) => void; + private readonly readValue?: ValueReader; + + constructor(opts: StoreOptions = {}) { + this.maxPerTick = opts.activeExpireMaxPerTick ?? 100; + this.onExpire = opts.onExpire; + this.readValue = opts.readValue; + const interval = opts.activeExpireIntervalMs ?? 100; + this.timer = interval > 0 ? setInterval(() => this.activeExpire(), interval) : null; + this.timer?.unref?.(); + } + + get size(): number { + // Count only logically-live keys: expired-but-not-yet-reaped entries stay + // in the map until lazy/active expiration removes them, so map.size would + // otherwise over-report. + let n = 0; + const now = Date.now(); + for (const [, r] of this.map) if (!r.expireAt || r.expireAt > now) n++; + return n; + } + + private metaBytes(dt: Record | null): number { + return dt ? Buffer.byteLength(JSON.stringify({ dt }), 'utf8') : 0; + } + + private refBytes(ref: ValueRef): number { + return ref.kind === 'memory' ? ref.value.length : DISK_REF_BYTES; + } + + private cloneRef(ref: ValueRef): ValueRef { + return ref.kind === 'memory' ? { kind: 'memory', value: Buffer.from(ref.value) } : { kind: 'disk', loc: { ...ref.loc } }; + } + + private materialize(ref: ValueRef): Buffer { + if (ref.kind === 'memory') return ref.value; + if (!this.readValue) throw new Error('Store cannot read disk-backed value without a ValueReader'); + return this.readValue(ref.loc); + } + + /** Approximate bytes used by one record, matching the bytes tracked on set(). */ + recordBytes(k: string): number { + const r = this.map.get(k); + return r ? Buffer.byteLength(k, 'binary') + this.refBytes(r.ref) + this.metaBytes(r.dt) : 0; + } + + /** Approximate bytes a SET would store for this key/value/dt. Pass + * countValue:false for valueMode:'disk', where only key/metadata/ref bytes + * stay in RAM and the value bulk lives in the snapshot/WAL. */ + estimateSetBytes( + key: string | Buffer, + value: Buffer, + dt: Record | null, + opts: { countValue?: boolean } = {}, + ): number { + const k = toKStr(key); + const countValue = opts.countValue ?? true; + return Buffer.byteLength(k, 'binary') + (countValue ? value.length : DISK_REF_BYTES) + this.metaBytes(dt); + } + + private remove(k: string): boolean { + const r = this.map.get(k); + const ok = this.map.delete(k); + if (ok) { + if (r) this.bytes -= Buffer.byteLength(k, 'binary') + this.refBytes(r.ref) + this.metaBytes(r.dt); + this.order.delete(k, k); + } + return ok; + } + + /** Remove a key that has expired and notify the owner so derived indexes + * stay in sync. */ + private expireKey(k: string, rec: StoreRecord): void { + if (this.remove(k)) this.onExpire?.(k, rec); + } + + set(key: string | Buffer, value: Buffer, expireAt = 0, dt: Record | null = null): void { + this.setRef(key, { kind: 'memory', value: Buffer.from(value) }, expireAt, dt); + } + + setRef(key: string | Buffer, ref: ValueRef, expireAt = 0, dt: Record | null = null): void { + const k = toKStr(key); + const existed = this.map.has(k); + if (existed) this.bytes -= this.recordBytes(k); + const seq = ++this.seq; + const stored = this.cloneRef(ref); + this.map.set(k, { ref: stored, expireAt: expireAt || 0, seq, dt }); + this.bytes += Buffer.byteLength(k, 'binary') + this.refBytes(stored) + this.metaBytes(dt); + if (!existed) this.order.insert(k, k); + if (expireAt) { + this.heap.push({ t: expireAt, k, seq }); + // Overwriting a TTL key leaves a stale heap entry that is only reaped + // when its (possibly far-future) timestamp passes, so the heap can grow + // without bound under frequent TTL updates. Rebuild it once stale entries + // clearly dominate the live set, bounding memory at ~2x live TTL keys. + if (this.heap.size > this.map.size * 2 + 64) this.rebuildHeap(); + } + } + + private rebuildHeap(): void { + this.heap.clear(); + for (const [k, r] of this.map) { + if (r.expireAt) this.heap.push({ t: r.expireAt, k, seq: r.seq }); + } + } + + get(key: string | Buffer): Buffer | undefined { + const k = toKStr(key); + const r = this.map.get(k); + if (!r) return undefined; + if (r.expireAt && r.expireAt <= Date.now()) { + this.expireKey(k, r); // lazy expiration + return undefined; + } + return this.materialize(r.ref); + } + + /** Read the full raw record. Like get(), an expired record is lazily reaped + * here (notifying the owner so derived indexes stay in sync) rather than + * being left behind as a ghost that read paths such as scan/query/dtRange + * would otherwise skip-but-not-clean. The returned record is raw: its ref may + * point at disk rather than holding a Buffer. */ + getRecord(key: string | Buffer): StoreRecord | undefined { + const k = toKStr(key); + const r = this.map.get(k); + if (!r) return undefined; + if (r.expireAt && r.expireAt <= Date.now()) { + this.expireKey(k, r); // lazy expiration (same as get) + return undefined; + } + return r; + } + + del(key: string | Buffer): boolean { + return this.remove(toKStr(key)); + } + + has(key: string | Buffer): boolean { + return this.get(key) !== undefined; + } + + *entries(): Generator { + const now = Date.now(); + for (const [k, r] of this.map) { + if (r.expireAt && r.expireAt <= now) continue; + yield { key: fromKStr(k), value: this.materialize(r.ref), expireAt: r.expireAt, dt: r.dt }; + } + } + + /** Ordered scan over keys. */ + *scan(opts: RangeOptions = {}): Generator { + for (const n of this.order.range(opts) as Iterable>) { + const r = this.getRecord(n.key); + if (!r) continue; + yield { key: fromKStr(n.key), value: this.materialize(r.ref), expireAt: r.expireAt, dt: r.dt }; + } + } + + /** Prefix scan over keys. */ + *prefix(p: string, limit = Infinity): Generator { + const pk = toKStr(p); + yield* this.scan({ gte: pk, lt: pk + '\uffff', count: limit }); + } + + /** Rewrite disk-backed value locations after compaction rotates the + * snapshot/WAL files. Memory refs are left untouched. */ + remapLocs(remap: (k: string, loc: ValueLoc, rec: StoreRecord) => ValueLoc | undefined): void { + for (const [k, r] of this.map) { + if (r.ref.kind !== 'disk') continue; + const next = remap(k, r.ref.loc, r); + if (next) r.ref = { kind: 'disk', loc: { ...next } }; + } + } + + private activeExpire(): void { + const now = Date.now(); + let n = 0; + while (n++ < this.maxPerTick && this.heap.size && this.heap.peek()!.t <= now) { + const e = this.heap.pop()!; + const r = this.map.get(e.k); + if (r && r.seq === e.seq && r.expireAt && r.expireAt <= now) { + this.expireKey(e.k, r); + } + } + } + + /** Synchronously reap every expired record. Returns the number removed. */ + reapExpired(): number { + const now = Date.now(); + let n = 0; + for (const [k, r] of this.map) { + if (r.expireAt && r.expireAt <= now) { + this.expireKey(k, r); + n++; + } + } + return n; + } + + /** Stop the active-expiration timer. */ + close(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } +} diff --git a/packages/minidb/src/text-index.ts b/packages/minidb/src/text-index.ts new file mode 100644 index 000000000..f95fb9f69 --- /dev/null +++ b/packages/minidb/src/text-index.ts @@ -0,0 +1,316 @@ +// src/text-index.ts +// +// Full-text inverted index, larger-than-RAM. +// +// - Tokenizer: Latin/number words + CJK unigrams & bigrams (no dictionary, +// zero dependencies, works for Chinese without a segmenter). +// - Storage: the bulk (every (doc, term) pair) lives in an on-disk postings +// file (see text-postings.ts). Only the small term dictionary, per-doc +// lengths, and key<->docID maps stay in RAM. +// - Writes: appended to an in-memory `delta`; deletes set a tombstone in +// `removed`. No disk I/O on the write path. +// - Reads: `search` reads each query term's postings from disk (or a small +// LRU cache), merges the in-memory `delta`, drops tombstones, and scores +// by TF-IDF. Synchronous by design so db.search()/db.query() keep their +// synchronous API. +// - Durability: the postings file is a pure derived cache of the Store; it is +// rebuilt from the Store on open and on compaction. The Store (snapshot + +// WAL) is the source of truth, so a crash never loses postings — they are +// simply rebuilt. + +import { getPath } from './query.js'; +import { PostingsFile } from './text-postings.js'; +import type { PostingEntry } from './text-postings.js'; + +const LATIN = /[a-z0-9]+/g; +const CJK = /[\u3400-\u9fff\u3040-\u30ff\uff00-\uffef]+/g; + +/** Tokenize text into terms (lowercased latin words + CJK uni/bigrams). */ +export function tokenize(str: unknown): string[] { + const s = String(str).toLowerCase(); + const terms: string[] = []; + const latin = s.match(LATIN); + if (latin) terms.push(...latin); + const runs = s.match(CJK) ?? []; + for (const r of runs) { + for (let i = 0; i < r.length; i++) { + terms.push(r[i]!); + if (i + 1 < r.length) terms.push(r[i]! + r[i + 1]!); + } + } + return terms; +} + +function stringLeaves(obj: unknown, acc: string[] = []): string[] { + if (obj == null) return acc; + if (typeof obj === 'string') { + acc.push(obj); + return acc; + } + if (typeof obj !== 'object') return acc; + for (const v of Object.values(obj as Record)) stringLeaves(v, acc); + return acc; +} + +export interface TextIndexOptions { + fields?: readonly string[] | null; + /** Path to the postings file. If omitted, the index keeps its base postings + * in memory instead of on disk (used by read-only openers, which must not + * write to a live writer's directory). */ + postingsPath?: string; + /** Max number of decoded postings lists to keep in the LRU cache (hot + * terms). 0 disables caching. */ + cacheTerms?: number; +} + +export interface SearchHit { + key: string; + score: number; +} + +export interface SearchOptions { + op?: 'AND' | 'OR'; + limit?: number; +} + +const EMPTY_MAP: ReadonlyMap = new Map(); + +export class TextIndex { + private readonly fields: readonly string[] | null; + private readonly path: string | null; + private readonly cacheTerms: number; + + // Term dictionary (the in-RAM "postings" map): term -> pointer into the + // postings file. Public so callers can read the unique-term count via .size. + readonly postings = new Map(); + + // Per-doc state (residual in-RAM floor: one entry per indexed doc). + private readonly docLen = new Map(); // docID -> token count + private readonly keys: (string | undefined)[] = []; // docID -> key + private readonly keyToId = new Map(); // key -> docID + + // Write buffer (in-RAM only; folded into queries, dropped on build). + private readonly delta = new Map>(); // term -> (docID -> freq) + private deltaCount = 0; + private readonly removed = new Set(); // tombstoned docIDs + + // Memory-base mode (no postingsPath): base postings kept in RAM. + private memBase: Map> | null = null; + + // Disk-base mode. + private pf: PostingsFile | null = null; + + // LRU cache of decoded base postings: term -> [docID, freq][] + private readonly cache = new Map(); + + /** Number of live indexed documents. */ + N = 0; + + constructor(opts: TextIndexOptions = {}) { + this.fields = opts.fields ?? null; + this.path = opts.postingsPath ?? null; + this.cacheTerms = opts.cacheTerms ?? 1024; + if (!this.path) this.memBase = new Map(); + } + + private extract(doc: unknown): string { + if (this.fields && this.fields.length) { + return this.fields + .map((f) => getPath(doc, f)) + .filter((v): v is string => typeof v === 'string') + .join(' '); + } + return stringLeaves(doc).join(' '); + } + + /** Number of distinct terms currently indexed (base + delta). */ + termCount(): number { + if (this.memBase) { + // include terms that only exist in delta + let n = this.memBase.size; + for (const t of this.delta.keys()) if (!this.memBase.has(t)) n++; + return n; + } + let n = this.postings.size; + for (const t of this.delta.keys()) if (!this.postings.has(t)) n++; + return n; + } + + /** + * Rebuild the index from scratch over `entries` (the live Store view). + * Assigns fresh dense docIDs, writes a new postings file (disk mode) or + * replaces the in-memory base (memory mode), and clears the delta + + * tombstones. Called on open and on compaction. + */ + build(entries: Iterable<{ key: string; value: unknown }>): void { + this.postings.clear(); + this.docLen.clear(); + this.keys.length = 0; + this.keyToId.clear(); + this.delta.clear(); + this.deltaCount = 0; + this.removed.clear(); + this.cache.clear(); + this.N = 0; + + const agg = new Map>(); // term -> (docID -> freq) + for (const { key, value } of entries) { + const docID = this.keys.length; + this.keys.push(key); + this.keyToId.set(key, docID); + const tokens = tokenize(this.extract(value)); + const counts = new Map(); + for (const t of tokens) counts.set(t, (counts.get(t) ?? 0) + 1); + for (const [t, c] of counts) { + let m = agg.get(t); + if (!m) agg.set(t, (m = new Map())); + m.set(docID, c); // docIDs increase monotonically -> insertion order is sorted + } + this.docLen.set(docID, tokens.length); + this.N++; + } + + if (this.path) { + // Disk mode: rewrite the postings file, then reopen for reads. + if (this.pf) { + this.pf.close(); + this.pf = null; + } + const dict = PostingsFile.rebuildSync(this.path, aggToSorted(agg)); + for (const [t, e] of dict) this.postings.set(t, e); + this.pf = PostingsFile.open(this.path); + } else { + // Memory mode. + this.memBase = agg; + } + } + + /** Add or replace a document. Overwrites tombstone the old docID. */ + add(key: string, doc: unknown): void { + if (this.keyToId.has(key)) this.remove(key); + const docID = this.keys.length; + this.keys.push(key); + this.keyToId.set(key, docID); + const tokens = tokenize(this.extract(doc)); + const counts = new Map(); + for (const t of tokens) counts.set(t, (counts.get(t) ?? 0) + 1); + for (const [t, c] of counts) { + let m = this.delta.get(t); + if (!m) this.delta.set(t, (m = new Map())); + m.set(docID, c); + this.deltaCount++; + } + this.docLen.set(docID, tokens.length); + this.N++; + } + + /** Remove a document by key (tombstone its docID). */ + remove(key: string): void { + const id = this.keyToId.get(key); + if (id === undefined) return; + this.removed.add(id); + this.keyToId.delete(key); + this.keys[id] = undefined; + this.docLen.delete(id); + for (const m of this.delta.values()) if (m.delete(id)) this.deltaCount--; + this.N--; + } + + /** Decoded base postings for a term (disk, cached; or memory). May still + * contain tombstoned docIDs — callers filter via `removed`. */ + private readBase(term: string): ReadonlyMap { + if (this.memBase) return this.memBase.get(term) ?? EMPTY_MAP; + + let arr = this.cache.get(term); + if (arr) { + // LRU touch: move to most-recent. + this.cache.delete(term); + this.cache.set(term, arr); + } else { + const entry = this.postings.get(term); + arr = entry && this.pf ? this.pf.read(entry) : []; + if (this.cacheTerms > 0) { + this.cache.set(term, arr); + if (this.cache.size > this.cacheTerms) { + const oldest = this.cache.keys().next().value as string; + this.cache.delete(oldest); + } + } + } + const m = new Map(); + for (const [id, f] of arr) m.set(id, f); + return m; + } + + /** Live postings for a term = (base ∪ delta) minus tombstones. */ + private livePostings(term: string): Map { + const out = new Map(); + for (const [id, f] of this.readBase(term)) if (!this.removed.has(id)) out.set(id, f); + const d = this.delta.get(term); + if (d) for (const [id, f] of d) if (!this.removed.has(id)) out.set(id, f); + return out; + } + + private idf(df: number): number { + return Math.log(1 + this.N / (df || 1)); + } + + search(query: string, opts: SearchOptions = {}): SearchHit[] { + const qtokens = [...new Set(tokenize(query))]; + if (!qtokens.length) return []; + const op = opts.op ?? 'AND'; + const limit = opts.limit ?? 50; + + const termMaps = new Map>(); + for (const t of qtokens) termMaps.set(t, this.livePostings(t)); + + let candidates: Set; + if (op === 'OR') { + candidates = new Set(); + for (const m of termMaps.values()) for (const id of m.keys()) candidates.add(id); + } else { + const lists = [...termMaps.values()]; + if (lists.some((m) => m.size === 0)) return []; + lists.sort((a, b) => a.size - b.size); + candidates = new Set(lists[0]!.keys()); + for (let i = 1; i < lists.length && candidates.size; i++) { + for (const id of candidates) if (!lists[i]!.has(id)) candidates.delete(id); + } + } + + const scored: SearchHit[] = []; + for (const id of candidates) { + const len = this.docLen.get(id) ?? 1; + let score = 0; + for (const t of qtokens) { + const f = termMaps.get(t)!.get(id) ?? 0; + if (f) score += (f / len) * this.idf(termMaps.get(t)!.size); + } + if (score > 0) { + const key = this.keys[id]; + if (key !== undefined) scored.push({ key, score }); + } + } + scored.sort((a, b) => b.score - a.score); + return scored.slice(0, limit); + } + + /** Close the underlying postings file. */ + close(): void { + if (this.pf) { + this.pf.close(); + this.pf = null; + } + } +} + +/** Yield `{ term, entries }` with entries sorted by docID ascending (they are + * already in insertion order, which equals ascending docID during build). */ +function* aggToSorted( + agg: Map>, +): Generator<{ term: string; entries: readonly (readonly [number, number])[] }> { + for (const [term, m] of agg) { + const entries = [...m]; + yield { term, entries }; + } +} diff --git a/packages/minidb/src/text-postings.ts b/packages/minidb/src/text-postings.ts new file mode 100644 index 000000000..ab1ff8707 --- /dev/null +++ b/packages/minidb/src/text-postings.ts @@ -0,0 +1,241 @@ +// src/text-postings.ts +// +// On-disk postings storage for the full-text index (larger-than-RAM). +// +// Each term's postings list (sorted `(docID, freq)` pairs) is stored as one +// append-only record in a flat file. The in-memory term dictionary maps +// `term -> { off, len, df }` and points at these records, so the bulk of the +// index (every (doc, term) pair) lives on disk and is read on demand, while +// only the small dictionary stays in RAM. +// +// Record frame (little-endian), CRC-verified: +// off size field +// 0 2 termLen uint16 +// 2 t term utf8 +// 2+t 4 df uint32 (document frequency at build time) +// 6+t 4 payloadLen uint32 +// 10+t p payload delta+varint encoded (docID, freq) pairs +// 10+t+p 4 crc32 over [termLen .. payload] +// +// Payload encoding: docIDs are sorted ascending and delta-encoded; both the +// deltas and the freqs are varint-coded (LEB128, unsigned). This is pure JS +// and gives ~5-10x compression for dense docID ranges. + +import fs from 'node:fs'; +import path from 'node:path'; +import { crc32 } from './crc32.js'; + +const HEADER_LEN = 2 + 4 + 4; // termLen + df + payloadLen (term is variable) +const CRC_LEN = 4; + +// ---- varint (unsigned LEB128, uint32) ------------------------------------ + +function encodeVarintInto(n: number, out: number[]): void { + n >>>= 0; + while (n >= 0x80) { + out.push((n & 0x7f) | 0x80); + n >>>= 7; + } + out.push(n); +} + +function decodeVarint(buf: Buffer, cur: { i: number }): number { + let r = 0; + let shift = 0; + for (;;) { + const b = buf[cur.i++]; + if (b === undefined) throw new Error('postings: truncated varint'); + r |= (b & 0x7f) << shift; + if ((b & 0x80) === 0) return r >>> 0; + shift += 7; + if (shift > 35) throw new Error('postings: varint too long'); + } +} + +// ---- posting list codec --------------------------------------------------- + +/** Encode a sorted (by docID asc) list of [docID, freq] pairs. */ +export function encodePostingList(entries: readonly (readonly [number, number])[]): Buffer { + const bytes: number[] = []; + encodeVarintInto(entries.length, bytes); + let prev = 0; + for (const [docID, freq] of entries) { + encodeVarintInto(docID - prev, bytes); + encodeVarintInto(freq, bytes); + prev = docID; + } + return Buffer.from(bytes); +} + +/** Decode a payload back into [docID, freq] pairs (ascending docID). */ +export function decodePostingList(buf: Buffer): [number, number][] { + const cur = { i: 0 }; + const count = decodeVarint(buf, cur); + const out: [number, number][] = new Array(count); + let prev = 0; + for (let k = 0; k < count; k++) { + const d = decodeVarint(buf, cur); + const freq = decodeVarint(buf, cur); + prev += d; + out[k] = [prev, freq]; + } + return out; +} + +// ---- record frame codec --------------------------------------------------- + +export interface DecodedRecord { + term: string; + df: number; + payload: Buffer; +} + +/** Encode one term's record frame (with CRC trailer). */ +export function encodeRecord(term: string, df: number, payload: Buffer): Buffer { + const termBuf = Buffer.from(term, 'utf8'); + if (termBuf.length > 0xffff) throw new RangeError('postings: term too long'); + const bodyLen = HEADER_LEN + termBuf.length + payload.length; + const body = Buffer.alloc(bodyLen); + let o = 0; + body.writeUInt16LE(termBuf.length, o); + o += 2; + termBuf.copy(body, o); + o += termBuf.length; + body.writeUInt32LE(df >>> 0, o); + o += 4; + body.writeUInt32LE(payload.length, o); + o += 4; + payload.copy(body, o); + const crc = crc32(body); + const out = Buffer.alloc(bodyLen + CRC_LEN); + body.copy(out, 0); + out.writeUInt32LE(crc >>> 0, bodyLen); + return out; +} + +/** Decode + CRC-verify a record frame. */ +export function decodeRecord(buf: Buffer): DecodedRecord { + if (buf.length < HEADER_LEN + CRC_LEN) throw new Error('postings: record too short'); + const stored = buf.readUInt32LE(buf.length - CRC_LEN); + const calc = crc32(buf.subarray(0, buf.length - CRC_LEN)); + if (stored !== calc) throw new Error('postings: record crc mismatch'); + let o = 0; + const termLen = buf.readUInt16LE(o); + o += 2; + if (o + termLen + 4 + 4 > buf.length - CRC_LEN) throw new Error('postings: record term length out of bounds'); + const term = buf.toString('utf8', o, o + termLen); + o += termLen; + const df = buf.readUInt32LE(o); + o += 4; + const payloadLen = buf.readUInt32LE(o); + o += 4; + if (o + payloadLen > buf.length - CRC_LEN) throw new Error('postings: record payload length out of bounds'); + const payload = buf.subarray(o, o + payloadLen); + return { term, df, payload }; +} + +// ---- postings file -------------------------------------------------------- + +/** A pointer to one term's record in the postings file. */ +export interface PostingEntry { + off: number; + len: number; + df: number; +} + +/** + * Append-only postings file with synchronous positioned reads. Synchronous I/O + * is deliberate: `TextIndex.search()` is synchronous (so `db.search()` / + * `db.query()` keep their sync API), and hot records are served from the OS + * page cache or the in-memory LRU cache anyway. + */ +export class PostingsFile { + private fd: number | null = null; + + private constructor(readonly path: string) {} + + /** + * Open an existing postings file for positioned reads. Throws if the file is + * missing — callers treat a missing file as an empty index. Read-only: the + * file is only ever rewritten wholesale by {@link rebuildSync}, so the fd + * stays valid until the next rebuild (which the caller must close + reopen). + */ + static open(filePath: string): PostingsFile { + const pf = new PostingsFile(filePath); + pf.fd = fs.openSync(filePath, 'r'); + return pf; + } + + get open(): boolean { + return this.fd !== null; + } + + /** Read + decode one term's postings record by dictionary pointer. */ + read(entry: PostingEntry): [number, number][] { + if (this.fd === null) throw new Error('postings file is closed'); + const buf = Buffer.alloc(entry.len); + let got = 0; + while (got < entry.len) { + const r = fs.readSync(this.fd, buf, got, entry.len - got, entry.off + got); + if (r === 0) throw new Error('postings: short read past EOF'); + got += r; + } + const rec = decodeRecord(buf); + return decodePostingList(rec.payload); + } + + close(): void { + if (this.fd !== null) { + fs.closeSync(this.fd); + this.fd = null; + } + } + + /** + * Build a fresh postings file from an iterator of `{ term, entries }` + * (entries must be sorted by docID asc). Writes to `.tmp`, fsyncs, and + * atomically renames over ``. Returns the new term dictionary. The old + * file (if any) is replaced only after the new one is fully durable, so a + * crash mid-build leaves the previous file intact. + */ + static rebuildSync( + filePath: string, + iter: Iterable<{ term: string; entries: readonly (readonly [number, number])[] }>, + ): Map { + const tmp = filePath + '.tmp'; + const fd = fs.openSync(tmp, 'w'); + const dict = new Map(); + let off = 0; + try { + for (const { term, entries } of iter) { + if (entries.length === 0) continue; + const payload = encodePostingList(entries); + const rec = encodeRecord(term, entries.length, payload); + let written = 0; + while (written < rec.length) { + const w = fs.writeSync(fd, rec, written, rec.length - written, off + written); + if (w === 0) throw new Error('postings: rebuild write made no progress'); + written += w; + } + dict.set(term, { off, len: rec.length, df: entries.length }); + off += rec.length; + } + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + fs.renameSync(tmp, filePath); + // Best-effort directory fsync so the rename survives a crash. + try { + const dfd = fs.openSync(path.dirname(filePath), 'r'); + try { + fs.fsyncSync(dfd); + } finally { + fs.closeSync(dfd); + } + } catch { + /* some platforms disallow fsync on a directory */ + } + return dict; + } +} diff --git a/packages/minidb/src/value-reader.ts b/packages/minidb/src/value-reader.ts new file mode 100644 index 000000000..4377fd9cd --- /dev/null +++ b/packages/minidb/src/value-reader.ts @@ -0,0 +1,87 @@ +// src/value-reader.ts +// +// Synchronous positioned reader for disk-backed KV values. Values live inline in +// the existing db.snapshot / db.wal frames; StoreRecord only keeps a small +// { file, off, len } pointer. Synchronous reads keep the public KV API +// synchronous, mirroring the full-text postings file design. + +import fs from 'node:fs'; +import path from 'node:path'; +import type { ValueLoc } from './store.js'; + +export class ValueReader { + readonly snapshotPath: string; + readonly walPath: string; + private snapshotFd: number | null = null; + private walFd: number | null = null; + + constructor(dir: string) { + this.snapshotPath = path.join(dir, 'db.snapshot'); + this.walPath = path.join(dir, 'db.wal'); + } + + open(): void { + this.snapshotFd = this.openIfExists(this.snapshotPath); + this.walFd = this.openIfExists(this.walPath); + } + + private openIfExists(file: string): number | null { + try { + return fs.openSync(file, 'r'); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw e; + } + } + + private fdFor(loc: ValueLoc): number { + const fd = loc.file === 'snapshot' ? this.snapshotFd : this.walFd; + if (fd === null) throw new Error(`value reader: ${loc.file} file is not open`); + return fd; + } + + read(loc: ValueLoc): Buffer { + if (loc.len === 0) return Buffer.alloc(0); + const fd = this.fdFor(loc); + const buf = Buffer.allocUnsafe(loc.len); + let got = 0; + while (got < loc.len) { + const r = fs.readSync(fd, buf, got, loc.len - got, loc.off + got); + if (r === 0) throw new Error(`value reader: short read from ${loc.file} at ${loc.off + got}`); + got += r; + } + return buf; + } + + reopenSnapshot(): void { + if (this.snapshotFd !== null) { + fs.closeSync(this.snapshotFd); + this.snapshotFd = null; + } + this.snapshotFd = this.openIfExists(this.snapshotPath); + } + + reopenWal(): void { + if (this.walFd !== null) { + fs.closeSync(this.walFd); + this.walFd = null; + } + this.walFd = this.openIfExists(this.walPath); + } + + reopenBoth(): void { + this.reopenSnapshot(); + this.reopenWal(); + } + + close(): void { + if (this.snapshotFd !== null) { + fs.closeSync(this.snapshotFd); + this.snapshotFd = null; + } + if (this.walFd !== null) { + fs.closeSync(this.walFd); + this.walFd = null; + } + } +} diff --git a/packages/minidb/src/wal.ts b/packages/minidb/src/wal.ts new file mode 100644 index 000000000..d5b9f0ecb --- /dev/null +++ b/packages/minidb/src/wal.ts @@ -0,0 +1,186 @@ +// src/wal.ts +// +// Write-ahead log: buffered, append-only, group-committed, with three fsync +// policies matching Redis AOF. +// +// 'always' — write + fsync for every flush (safest, slowest) +// 'everysec' — write every flush; fsync on a 1s timer (default; ≤1s loss window) +// 'no' — write only; let the OS flush (fastest, may lose seconds) +// +// Group commit: all append() calls within a tick are coalesced into a single +// writev(2) syscall on the next macrotask. Only one flush is ever in flight, so +// frames reach disk strictly in append order (single-writer, like SQLite WAL). + +import fs from 'node:fs/promises'; +import type { FileHandle } from 'node:fs/promises'; + +export type FsyncPolicy = 'always' | 'everysec' | 'no'; + +const POLICIES = new Set(['always', 'everysec', 'no']); + +interface PendingWrite { + buf: Buffer; + resolve: () => void; + reject: (err: unknown) => void; +} + +export interface WALOptions { + fsyncPolicy?: FsyncPolicy; + syncIntervalMs?: number; + /** Optional sink for cumulative write/fsync counters. Owned by MiniDb so the + * counts survive WAL rotation during compaction (which replaces the WAL). */ + stats?: { walBytesWritten: number; walFsyncs: number }; +} + +export class WAL { + readonly path: string; + private readonly policy: FsyncPolicy; + private readonly syncIntervalMs: number; + + private fh: FileHandle | null = null; + size = 0; // bytes on disk (best-effort; updated after each write) + private nextOffset = 0; // logical next append offset, including queued/in-flight frames + private queue: PendingWrite[] = []; + private queuedBytes = 0; + private flushing = false; + private inflight: Promise | null = null; + private scheduled = false; + private timer: ReturnType | null = null; + private closed = false; + private readonly stats: { walBytesWritten: number; walFsyncs: number } | null; + + constructor(path: string, opts: WALOptions = {}) { + const policy = opts.fsyncPolicy ?? 'everysec'; + if (!POLICIES.has(policy)) throw new RangeError(`unknown fsyncPolicy: ${policy}`); + this.path = path; + this.policy = policy; + this.syncIntervalMs = opts.syncIntervalMs ?? 1000; + this.stats = opts.stats ?? null; + } + + async open(): Promise { + if (this.fh) return; + this.fh = await fs.open(this.path, 'a'); // create + append at EOF + const st = await this.fh.stat(); + this.size = st.size; + this.nextOffset = st.size; + if (this.policy === 'everysec') { + this.timer = setInterval(() => { + this.sync().catch(() => {}); + }, this.syncIntervalMs); + this.timer.unref?.(); + } + } + + /** Append one frame and return its predicted absolute file offset. The offset + * is known synchronously because frames are flushed strictly in append order; + * callers may use it immediately as a value pointer before `done` resolves. */ + appendLoc(frame: Buffer): { offset: number; done: Promise } { + if (this.closed) return { offset: -1, done: Promise.reject(new Error('WAL is closed')) }; + if (!Buffer.isBuffer(frame)) return { offset: -1, done: Promise.reject(new TypeError('frame must be a Buffer')) }; + const offset = this.nextOffset; + this.nextOffset += frame.length; + const done = new Promise((resolve, reject) => { + this.queue.push({ buf: frame, resolve, reject }); + this.queuedBytes += frame.length; + if (!this.flushing && !this.scheduled) { + this.scheduled = true; + setImmediate(() => this.flushBatch()); + } + }); + return { offset, done }; + } + + /** Append one frame. Resolves once written to the OS page cache; for + * fsyncPolicy 'always' it additionally waits for fsync. */ + append(frame: Buffer): Promise { + return this.appendLoc(frame).done; + } + + private async flushBatch(): Promise { + this.scheduled = false; + if (this.flushing) return this.inflight; + if (this.queue.length === 0) return null; + + this.flushing = true; + const run = async () => { + const batch = this.queue; + this.queue = []; + this.queuedBytes = 0; + // writev(2) may short-write (signal interruption, RLIMIT_FSIZE, …). Retry + // until the whole batch lands so a partial write never rejects frames + // whose in-memory side effects were already applied. Only a real I/O + // error (or zero progress) rejects the batch. + let bufs = batch.map((b) => b.buf); + let off = 0; // byte offset within bufs[0] + try { + while (bufs.length > 0) { + const toWrite = off > 0 ? [bufs[0]!.subarray(off), ...bufs.slice(1)] : bufs; + const { bytesWritten } = await this.fh!.writev(toWrite); + if (bytesWritten === 0) throw new Error('WAL writev made no progress (short write)'); + this.size += bytesWritten; + if (this.stats) this.stats.walBytesWritten += bytesWritten; + let rem = bytesWritten; + while (rem > 0 && bufs.length > 0) { + const left = bufs[0]!.length - off; + if (rem < left) { + off += rem; + rem = 0; + } else { + rem -= left; + bufs.shift(); + off = 0; + } + } + } + if (this.policy === 'always') await this.sync(); + for (const b of batch) b.resolve(); + } catch (err) { + for (const b of batch) b.reject(err); + } finally { + this.flushing = false; + this.inflight = null; + if (this.queue.length > 0 && !this.closed) { + this.scheduled = true; + setImmediate(() => this.flushBatch()); + } + } + }; + this.inflight = run(); + return this.inflight; + } + + /** Force an fsync of the underlying file. */ + async sync(): Promise { + if (this.fh) { + await this.fh.sync(); + if (this.stats) this.stats.walFsyncs++; + } + } + + /** Flush buffered frames to the OS (without necessarily fsync'ing). + * Loops until everything queued up to now has been flushed: an earlier + * version only awaited the in-flight batch and could return while newer + * frames were still queued, which let compaction truncate un-flushed data. */ + async flush(): Promise { + while (this.queue.length > 0 || this.inflight) { + if (this.inflight) await this.inflight; + if (this.queue.length > 0) await this.flushBatch(); + } + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + await this.flush(); + if (this.fh) { + await this.sync(); + await this.fh.close(); + this.fh = null; + } + } +} diff --git a/packages/minidb/test/batch.test.ts b/packages/minidb/test/batch.test.ts new file mode 100644 index 000000000..a03e4aed7 --- /dev/null +++ b/packages/minidb/test/batch.test.ts @@ -0,0 +1,112 @@ +// test/batch.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-batch-')); +} + +test('batch applies multiple ops atomically', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + try { + await db.batch([ + { op: 'set', key: 'a', value: { n: 1 } }, + { op: 'set', key: 'b', value: { n: 2 } }, + { op: 'del', key: 'a' }, + ]); + assert.equal(db.get('a'), undefined); + assert.deepEqual(db.get('b'), { n: 2 }); + assert.equal(db.size, 1); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('batch with dt + indexes updates everything', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byCity', { field: 'city' }); + try { + await db.batch([ + { op: 'set', key: 'u1', value: { city: 'Paris' }, dt: { created: 100 } }, + { op: 'set', key: 'u2', value: { city: 'Paris' }, dt: { created: 200 } }, + ]); + assert.deepEqual(db.findEq('byCity', 'Paris').map((r) => r.key).sort(), ['u1', 'u2']); + assert.deepEqual(db.dtRange('created', { gte: 100 }).map((r) => r.key).sort(), ['u1', 'u2']); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('batch unique violation rejects the whole batch', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byEmail', { field: 'email', unique: true }); + await db.set('a', { email: 'x@y.com' }); + try { + await assert.rejects( + () => db.batch([{ op: 'set', key: 'b', value: { email: 'ok@y.com' } }, { op: 'set', key: 'c', value: { email: 'x@y.com' } }]), + /unique/, + ); + // neither op applied + assert.equal(db.get('b'), undefined); + assert.equal(db.get('c'), undefined); + assert.equal(db.size, 1); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('batch is atomic on recovery: a corrupt batch frame skips the whole batch', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'always', autoCompact: false }); + await db.set('before', { v: 0 }); + await db.batch([ + { op: 'set', key: 'x', value: { v: 1 } }, + { op: 'set', key: 'y', value: { v: 2 } }, + ]); + await db.set('after', { v: 3 }); + await db.close(); + + // Corrupt the batch frame (the 2nd frame, between 'before' and 'after'). + const wal = path.join(dir, 'db.wal'); + const buf = await fs.readFile(wal); + const magic = Buffer.from([0x4d, 0x44]); + const second = buf.indexOf(magic, buf.indexOf(magic) + 1); // start of batch frame + buf[second + 30] ^= 0xff; // flip a byte inside the batch body -> bad crc + await fs.writeFile(wal, buf); + + db = await MiniDb.open({ dir, valueCodec: 'json', recovery: 'resync' }); + assert.equal(db.get('before').v, 0, 'before survives'); + // the batch is either fully skipped or (if resync lands inside) neither x nor y appears + // — but it must NEVER be the case that only one of x/y is present. + const x = db.get('x'); + const y = db.get('y'); + assert.ok(!(x !== undefined && y === undefined), 'batch must not be half-applied (x only)'); + assert.ok(!(x === undefined && y !== undefined), 'batch must not be half-applied (y only)'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('empty batch is a no-op', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + try { + await db.batch([]); + assert.equal(db.size, 0); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/codec.test.ts b/packages/minidb/test/codec.test.ts new file mode 100644 index 000000000..c930abef1 --- /dev/null +++ b/packages/minidb/test/codec.test.ts @@ -0,0 +1,100 @@ +// test/codec.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { + encodeFrame, + FrameParser, + CorruptFrameError, + TYPE_SET, + TYPE_DEL, + HEADER_SIZE, +} from '../src/codec.js'; + +const B = (s) => Buffer.from(s); + +test('round-trip SET', () => { + const f = encodeFrame({ type: TYPE_SET, key: B('foo'), value: B('bar'), expireAt: 123 }); + const p = new FrameParser(); + const out = [...p.feed(f)]; + assert.equal(out.length, 1); + assert.equal(out[0].type, TYPE_SET); + assert.equal(out[0].key.toString(), 'foo'); + assert.equal(out[0].value.toString(), 'bar'); + assert.equal(out[0].expireAt, 123); +}); + +test('round-trip SET with meta', () => { + const meta = Buffer.from(JSON.stringify({ dt: { dt1: 123 } })); + const f = encodeFrame({ type: TYPE_SET, key: B('d'), value: B('v'), meta }); + const out = [...new FrameParser().feed(f)]; + assert.equal(out[0].key.toString(), 'd'); + assert.deepEqual(JSON.parse(out[0].meta.toString()), { dt: { dt1: 123 } }); +}); + +test('frame without meta yields meta=null', () => { + const f = encodeFrame({ type: TYPE_SET, key: B('d'), value: B('v') }); + const out = [...new FrameParser().feed(f)]; + assert.equal(out[0].meta, null); +}); + +test('round-trip DEL tombstone (empty value)', () => { + const f = encodeFrame({ type: TYPE_DEL, key: B('gone') }); + const out = [...new FrameParser().feed(f)]; + assert.equal(out[0].type, TYPE_DEL); + assert.equal(out[0].key.toString(), 'gone'); + assert.equal(out[0].value.length, 0); +}); + +test('multiple frames in one chunk', () => { + const a = encodeFrame({ type: TYPE_SET, key: B('a'), value: B('1') }); + const b = encodeFrame({ type: TYPE_SET, key: B('b'), value: B('2') }); + const out = [...new FrameParser().feed(Buffer.concat([a, b]))]; + assert.equal(out.map((r) => r.key.toString()).join(','), 'a,b'); +}); + +test('partial feed across chunks still parses', () => { + const f = encodeFrame({ type: TYPE_SET, key: B('split'), value: B('x'.repeat(50)) }); + const p = new FrameParser(); + const cut = 7; + const out1 = [...p.feed(f.subarray(0, cut))]; + assert.equal(out1.length, 0); + const out2 = [...p.feed(f.subarray(cut))]; + assert.equal(out2.length, 1); + assert.equal(out2[0].key.toString(), 'split'); + assert.equal(out2[0].value.length, 50); +}); + +test('crc mismatch throws CorruptFrameError with offset', () => { + const f = encodeFrame({ type: TYPE_SET, key: B('bad'), value: B('data') }); + const corrupted = Buffer.from(f); + corrupted[HEADER_SIZE + 1] ^= 0xff; // flip a byte inside the payload + assert.throws(() => [...new FrameParser().feed(corrupted)], (e) => { + assert.ok(e instanceof CorruptFrameError); + assert.equal(e.offset, 0); + return true; + }); +}); + +test('finish() reports a torn trailing partial frame at the valid-data offset', () => { + const p = new FrameParser(); + const good = encodeFrame({ type: TYPE_SET, key: B('ok'), value: B('v') }); + [...p.feed(good)]; + [...p.feed(Buffer.from([0x4d, 0x44, 0x01]))]; // magic + type: incomplete header + assert.throws( + () => p.finish(), + (e) => e instanceof CorruptFrameError && e.offset === good.length, + ); +}); + +test('finish() returns the clean EOF offset when there is no leftover', () => { + const p = new FrameParser(); + const a = encodeFrame({ type: TYPE_SET, key: B('a'), value: B('1') }); + const b = encodeFrame({ type: TYPE_SET, key: B('b'), value: B('2') }); + [...p.feed(Buffer.concat([a, b]))]; + assert.equal(p.finish(), a.length + b.length); +}); + +test('frame length = header + payload + crc trailer', () => { + const f = encodeFrame({ type: TYPE_SET, key: B('k'), value: B('v') }); + assert.equal(f.length, HEADER_SIZE + 1 + 1 + 4); +}); diff --git a/packages/minidb/test/compaction-fault.test.ts b/packages/minidb/test/compaction-fault.test.ts new file mode 100644 index 000000000..79a57ca17 --- /dev/null +++ b/packages/minidb/test/compaction-fault.test.ts @@ -0,0 +1,102 @@ +// test/compaction-fault.test.ts +// +// Fault-injection tests for the compaction helpers using `vi.doMock` to replace +// node:fs/promises. These exercise branches that real filesystems essentially +// never produce (a write that returns 0 bytes, a close/sync that throws). +// +// NOTE: each test resets the module registry and re-mocks node:fs/promises so a +// fresh import of compaction.ts picks up that test's mocked fs. + +import assert from 'node:assert/strict'; +import { afterEach, test, vi } from 'vitest'; + +interface MockHandle { + read?: (...args: unknown[]) => Promise<{ bytesRead: number }>; + write?: (...args: unknown[]) => Promise<{ bytesWritten: number }>; + sync?: () => Promise; + close?: () => Promise; +} + +// Replace node:fs/promises with a module whose only export is `open`. compaction.ts +// consumes it via a default import (`import fs from 'node:fs/promises'`), so the +// mock also exposes the same handle set as its `default` export. +function mockFsPromises(open: (path: string, flags?: string) => Promise): void { + const exports = { open }; + vi.doMock('node:fs/promises', () => ({ ...exports, default: exports })); +} + +afterEach(() => { + vi.doUnmock('node:fs/promises'); + vi.resetModules(); +}); + +test('copyFileRange throws when the destination short-writes (bytesWritten === 0)', async () => { + mockFsPromises(async (_p, flags) => { + if (flags === 'r') { + // Source claims to have produced bytes so the loop reaches write(). + return { read: async () => ({ bytesRead: 16 }), close: async () => {} }; + } + // Destination makes no progress → the short-write guard fires. + return { write: async () => ({ bytesWritten: 0 }), sync: async () => {}, close: async () => {} }; + }); + const { copyFileRange } = await import('../src/compaction.js'); + await assert.rejects(() => copyFileRange('/tmp/src', '/tmp/dst', 0, 16), /short write/); +}); + +test('copyFileRange tolerates a source close() failure (best-effort close)', async () => { + mockFsPromises(async (_p, flags) => { + if (flags === 'r') { + return { + read: async (buf: unknown) => { + (buf as Buffer)[0] = 0xab; + return { bytesRead: 1 }; + }, + close: async () => { + throw new Error('source close failed'); + }, + }; + } + return { + write: async (_b: unknown, _o: unknown, len: number) => ({ bytesWritten: len }), + sync: async () => {}, + close: async () => {}, + }; + }); + const { copyFileRange } = await import('../src/compaction.js'); + // The source close failure is swallowed; the copy itself succeeds. + await copyFileRange('/tmp/src', '/tmp/dst', 0, 1); +}); + +test('copyFileRange tolerates a destination close() failure (best-effort close)', async () => { + mockFsPromises(async (_p, flags) => { + if (flags === 'r') { + return { read: async () => ({ bytesRead: 0 }), close: async () => {} }; + } + return { + write: async (_b: unknown, _o: unknown, len: number) => ({ bytesWritten: len }), + sync: async () => {}, + close: async () => { + throw new Error('dest close failed'); + }, + }; + }); + const { copyFileRange } = await import('../src/compaction.js'); + // Empty range (start===end) → no writes, only a dst.sync() then a failing + // dst.close() that must be swallowed. + await copyFileRange('/tmp/src', '/tmp/dst', 0, 0); +}); + +test('fsyncDir swallows a sync() failure and still closes the handle', async () => { + let closed = false; + mockFsPromises(async () => ({ + sync: async () => { + throw new Error('sync failed'); + }, + close: async () => { + closed = true; + }, + })); + const { fsyncDir } = await import('../src/compaction.js'); + await fsyncDir('/tmp/whatever'); + assert.equal(closed, true, 'close() is called even after sync() throws'); +}); diff --git a/packages/minidb/test/compaction-internal.test.ts b/packages/minidb/test/compaction-internal.test.ts new file mode 100644 index 000000000..7a8426458 --- /dev/null +++ b/packages/minidb/test/compaction-internal.test.ts @@ -0,0 +1,156 @@ +// test/compaction-internal.test.ts +// +// White-box boundary tests for the compaction helpers (copyFileRange / +// fsyncDir) that are hard to reach through the public MiniDb API. Both are +// exported from src/compaction.ts for this purpose but are NOT re-exported +// from the package entry point. +// +// Fault-injection tests (mocked fs) live in compaction-fault.test.ts so that +// their dynamic imports + query-string cache busting do not distort the +// coverage report of this file (Node's coverage keys scripts by URL). + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { copyFileRange, fsyncDir } from '../src/compaction.js'; + +async function tmpDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-copy-')); +} + +// --- copyFileRange: happy path ------------------------------------------------ + +test('copyFileRange copies an arbitrary byte range verbatim', async () => { + const dir = await tmpDir(); + try { + const src = path.join(dir, 'src'); + const dst = path.join(dir, 'dst'); + const data = Buffer.alloc(8192); + for (let i = 0; i < data.length; i++) data[i] = (i * 7) % 256; + await fs.writeFile(src, data); + + await copyFileRange(src, dst, 1000, 6000); + + const out = await fs.readFile(dst); + assert.equal(out.length, 5000); + assert.ok(out.equals(data.subarray(1000, 6000))); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('copyFileRange copies a range larger than one internal chunk', async () => { + // COPY_CHUNK is 1 MiB; copy 2.5 MiB to exercise the multi-chunk loop. + const dir = await tmpDir(); + try { + const src = path.join(dir, 'src'); + const dst = path.join(dir, 'dst'); + const size = (5 << 20) / 2; // 2.5 MiB + const data = Buffer.alloc(size); + for (let i = 0; i < data.length; i++) data[i] = i % 251; + await fs.writeFile(src, data); + + await copyFileRange(src, dst, 0, data.length); + + const out = await fs.readFile(dst); + assert.equal(out.length, data.length); + assert.ok(out.equals(data)); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- copyFileRange: EOF / empty-range boundaries ----------------------------- + +test('copyFileRange stops at EOF when end exceeds the source size', async () => { + // Exercises the `bytesRead === 0` break: the source is shorter than the + // requested range, so read() returns 0 and the loop terminates early. + const dir = await tmpDir(); + try { + const src = path.join(dir, 'src'); + const dst = path.join(dir, 'dst'); + await fs.writeFile(src, Buffer.from('hello')); // 5 bytes + + await copyFileRange(src, dst, 0, 1000); // ask for far more than exists + + const out = await fs.readFile(dst); + assert.ok(out.equals(Buffer.from('hello'))); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('copyFileRange with start===end creates an empty file in create mode', async () => { + const dir = await tmpDir(); + try { + const src = path.join(dir, 'src'); + const dst = path.join(dir, 'dst'); + await fs.writeFile(src, Buffer.from('data')); + await fs.writeFile(dst, Buffer.from('preexisting-to-be-truncated')); + + await copyFileRange(src, dst, 2, 2); // empty range, 'w' truncates + + const out = await fs.readFile(dst); + assert.equal(out.length, 0); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('copyFileRange with start===end in append mode leaves existing content intact', async () => { + const dir = await tmpDir(); + try { + const src = path.join(dir, 'src'); + const dst = path.join(dir, 'dst'); + await fs.writeFile(src, Buffer.from('source')); + await fs.writeFile(dst, Buffer.from('existing')); + + await copyFileRange(src, dst, 3, 3, { append: true }); + + const out = await fs.readFile(dst); + assert.ok(out.equals(Buffer.from('existing'))); // unchanged + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('copyFileRange in append mode appends the range to existing content', async () => { + const dir = await tmpDir(); + try { + const src = path.join(dir, 'src'); + const dst = path.join(dir, 'dst'); + await fs.writeFile(src, Buffer.from('0123456789')); + await fs.writeFile(dst, Buffer.from('abc')); + + await copyFileRange(src, dst, 2, 5, { append: true }); + + const out = await fs.readFile(dst); + assert.ok(out.equals(Buffer.from('abc234'))); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('copyFileRange rejects end < start', async () => { + await assert.rejects(() => copyFileRange('/tmp/a', '/tmp/b', 10, 5), /end \(5\) < start \(10\)/); +}); + +// --- fsyncDir ---------------------------------------------------------------- + +test('fsyncDir syncs an existing directory without throwing', async () => { + const dir = await tmpDir(); + try { + await fsyncDir(dir); // should not throw + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('fsyncDir swallows errors when the directory cannot be opened', async () => { + // A non-existent path makes fs.open reject → the catch branch runs and the + // finally closes nothing (fh is null). Best-effort semantics. + const bogus = path.join(os.tmpdir(), `minidb-no-such-dir-${Date.now()}`); + await fsyncDir(bogus); // must not throw +}); diff --git a/packages/minidb/test/compaction.test.ts b/packages/minidb/test/compaction.test.ts new file mode 100644 index 000000000..29c92a94d --- /dev/null +++ b/packages/minidb/test/compaction.test.ts @@ -0,0 +1,166 @@ +// test/compaction.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-compact-')); +} + +test('manual compact writes a snapshot, shrinks the WAL, and keeps data', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + for (let i = 0; i < 500; i++) await db.set(`k${i}`, `value-${i}`); + const walBefore = (await fs.stat(path.join(dir, 'db.wal'))).size; + await db.compact(); + assert.equal(db.stats.compactions, 1); + + const snap = await fs.stat(path.join(dir, 'db.snapshot')); + const walAfter = (await fs.stat(path.join(dir, 'db.wal'))).size; + assert.ok(snap.size > 0, 'snapshot file exists and is non-empty'); + assert.ok(walAfter < walBefore, 'WAL shrank after compaction'); + assert.equal(db.size, 500); + await db.close(); + + // Recovery should load snapshot + (small) WAL and restore everything. + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.recoveryInfo.snapshotFrames, 500); + assert.equal(db.size, 500); + assert.equal(db.get('k0'), 'value-0'); + assert.equal(db.get('k499'), 'value-499'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('a write issued during compaction is preserved', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + for (let i = 0; i < 200; i++) await db.set(`k${i}`, `v${i}`); + + const compactP = db.compact(); + const setP = db.set('during', 'hello'); // guard should queue behind compaction + await Promise.all([compactP, setP]); + + assert.equal(db.get('during'), 'hello'); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, 201); + assert.equal(db.get('during'), 'hello'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('auto-compaction triggers when the WAL crosses the threshold', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ + dir, + valueCodec: 'string', + fsyncPolicy: 'no', + compactThresholdBytes: 1024, // 1 KiB + }); + for (let i = 0; i < 200; i++) await db.set(`k${i}`, `v${i}`.padEnd(50, 'x')); + // Allow the background compaction to finish. + if (db.compacting) await db._compactDone; + assert.ok(db.stats.compactions >= 1, 'at least one auto-compaction ran'); + assert.equal(db.size, 200); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('del-then-compact drops tombstoned keys from the snapshot', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + await db.set('a', '1'); + await db.set('b', '2'); + await db.del('a'); + await db.compact(); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.recoveryInfo.snapshotFrames, 1); // only 'b' survived + assert.equal(db.get('a'), undefined); + assert.equal(db.get('b'), '2'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('concurrent SET/UPDATE/DEL during compaction survive recovery', async () => { + // Exercises the fuzzy-snapshot + WAL-tail-replay convergence: writes that + // land while the snapshot is being written must all be reflected after a + // reopen, with last-writer-wins semantics. + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + const N = 10000; + for (let i = 0; i < N; i++) await db.set('k' + i, 'v' + i); + + // Even keys are updated, keys == 1 (mod 4) are deleted, keys == 3 (mod 4) + // are left untouched, and a batch of new keys is added — all racing the + // in-progress snapshot. + let deleted = 0; + const M = 2000; + const compactP = db.compact(); + const ops: Promise[] = []; + for (let i = 0; i < N; i++) { + if (i % 2 === 0) ops.push(db.set('k' + i, 'updated-' + i)); + else if (i % 4 === 1) { + deleted++; + ops.push(db.del('k' + i)); + } + } + for (let i = 0; i < M; i++) ops.push(db.set('new' + i, 'n' + i)); + await Promise.all(ops); + await compactP; + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + for (let i = 0; i < N; i++) { + if (i % 2 === 0) assert.equal(db.get('k' + i), 'updated-' + i, `updated k${i}`); + else if (i % 4 === 1) assert.equal(db.get('k' + i), undefined, `deleted k${i}`); + else assert.equal(db.get('k' + i), 'v' + i, `unchanged k${i}`); + } + for (let i = 0; i < M; i++) assert.equal(db.get('new' + i), 'n' + i, `new${i}`); + assert.equal(db.size, N - deleted + M); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('compaction with no concurrent writes produces an empty WAL tail', async () => { + // When nothing is written during compaction, the post-fence WAL tail is empty + // and the new WAL should be zero-length (or near-zero). Data still survives. + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + for (let i = 0; i < 300; i++) await db.set('k' + i, 'v' + i); + await db.compact(); + const walSize = (await fs.stat(path.join(dir, 'db.wal'))).size; + assert.equal(walSize, 0, 'WAL tail is empty when no writes raced compaction'); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, 300); + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get('k299'), 'v299'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/compound-index.test.ts b/packages/minidb/test/compound-index.test.ts new file mode 100644 index 000000000..50c296b4c --- /dev/null +++ b/packages/minidb/test/compound-index.test.ts @@ -0,0 +1,98 @@ +// test/compound-index.test.ts +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-compound-')); +} + +test('compound index orders sessions within a workspace by updatedAt', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createCompoundIndex('byWsUpdated', { groupBy: 'workspaceId', orderBy: 'updatedAt' }); + try { + await db.set('a', { workspaceId: 'W1', title: 'a' }, { dt: { updatedAt: 300 } }); + await db.set('b', { workspaceId: 'W1', title: 'b' }, { dt: { updatedAt: 100 } }); + await db.set('c', { workspaceId: 'W1', title: 'c' }, { dt: { updatedAt: 200 } }); + await db.set('d', { workspaceId: 'W2', title: 'd' }, { dt: { updatedAt: 500 } }); + + const asc = db.compoundRange('byWsUpdated', 'W1', { count: 10 }); + assert.deepEqual(asc.map((r) => r.key), ['b', 'c', 'a']); + + const desc = db.compoundRange('byWsUpdated', 'W1', { reverse: true, count: 10 }); + assert.deepEqual(desc.map((r) => r.key), ['a', 'c', 'b']); + + // pagination + const page = db.compoundRange('byWsUpdated', 'W1', { reverse: true, offset: 1, count: 1 }); + assert.deepEqual(page.map((r) => r.key), ['c']); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('multiple dt columns each get their own compound index', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createCompoundIndex('byWsUpdated', { groupBy: 'workspaceId', orderBy: 'updatedAt' }); + await db.createCompoundIndex('byWsCreated', { groupBy: 'workspaceId', orderBy: 'createdAt' }); + try { + await db.set('a', { workspaceId: 'W1' }, { dt: { updatedAt: 300, createdAt: 10 } }); + await db.set('b', { workspaceId: 'W1' }, { dt: { updatedAt: 100, createdAt: 30 } }); + await db.set('c', { workspaceId: 'W1' }, { dt: { updatedAt: 200, createdAt: 20 } }); + + assert.deepEqual(db.compoundRange('byWsUpdated', 'W1').map((r) => r.key), ['b', 'c', 'a']); + assert.deepEqual(db.compoundRange('byWsCreated', 'W1').map((r) => r.key), ['a', 'c', 'b']); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('updating the order key moves the entry', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createCompoundIndex('byWsUpdated', { groupBy: 'workspaceId', orderBy: 'updatedAt' }); + try { + await db.set('a', { workspaceId: 'W1' }, { dt: { updatedAt: 100 } }); + await db.set('b', { workspaceId: 'W1' }, { dt: { updatedAt: 200 } }); + assert.deepEqual(db.compoundRange('byWsUpdated', 'W1').map((r) => r.key), ['a', 'b']); + // bump 'a' to the top + await db.set('a', { workspaceId: 'W1' }, { dt: { updatedAt: 999 } }); + assert.deepEqual(db.compoundRange('byWsUpdated', 'W1').map((r) => r.key), ['b', 'a']); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('compound index persists and rebuilds across reopen', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createCompoundIndex('byWsUpdated', { groupBy: 'workspaceId', orderBy: 'updatedAt' }); + await db.set('a', { workspaceId: 'W1' }, { dt: { updatedAt: 200 } }); + await db.set('b', { workspaceId: 'W1' }, { dt: { updatedAt: 100 } }); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.deepEqual(db.listCompoundIndexes().map((i) => i.name), ['byWsUpdated']); + assert.deepEqual(db.compoundRange('byWsUpdated', 'W1').map((r) => r.key), ['b', 'a']); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('delete removes from the compound index', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createCompoundIndex('byWsUpdated', { groupBy: 'workspaceId', orderBy: 'updatedAt' }); + await db.set('a', { workspaceId: 'W1' }, { dt: { updatedAt: 100 } }); + await db.set('b', { workspaceId: 'W1' }, { dt: { updatedAt: 200 } }); + await db.del('a'); + assert.deepEqual(db.compoundRange('byWsUpdated', 'W1').map((r) => r.key), ['b']); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); diff --git a/packages/minidb/test/crc32.test.ts b/packages/minidb/test/crc32.test.ts new file mode 100644 index 000000000..fc7b3a5b4 --- /dev/null +++ b/packages/minidb/test/crc32.test.ts @@ -0,0 +1,21 @@ +// test/crc32.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { crc32 } from '../src/crc32.js'; + +test('crc32 known vectors', () => { + // CRC-32/ISO-HDLC of "123456789" is 0xCBF43926 (the canonical check value). + assert.equal(crc32(Buffer.from('123456789')).toString(16), 'cbf43926'); + assert.equal(crc32(Buffer.from('')).toString(16), '0'); + assert.equal(crc32(Buffer.from('a')).toString(16), 'e8b7be43'); +}); + +test('crc32 is incremental', () => { + const data = Buffer.from('hello world from minidb'); + const whole = crc32(data); + let running = 0; + for (let i = 0; i < data.length; i += 5) { + running = crc32(data.subarray(i, Math.min(i + 5, data.length)), running); + } + assert.equal(running, whole); +}); diff --git a/packages/minidb/test/db.test.ts b/packages/minidb/test/db.test.ts new file mode 100644 index 000000000..14f455919 --- /dev/null +++ b/packages/minidb/test/db.test.ts @@ -0,0 +1,324 @@ +// test/db.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; +import { encodeFrame, TYPE_SET } from '../src/codec.js'; + +const B = (s) => Buffer.from(s); + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-db-')); +} + +test('set/get persists across reopen (string codec)', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'everysec' }); + await db.set('a', '1'); + await db.set('b', '2'); + assert.equal(db.get('a'), '1'); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.get('a'), '1'); + assert.equal(db.get('b'), '2'); + assert.equal(db.size, 2); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('del persists across reopen', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + await db.set('x', '1'); + await db.set('y', '2'); + await db.del('x'); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.get('x'), undefined); + assert.equal(db.get('y'), '2'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('json codec round-trips values', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.set('obj', { a: 1, b: [2, 3] }); + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.deepEqual(db.get('obj'), { a: 1, b: [2, 3] }); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('ttl / expire', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); + await db.set('t', 'v', { ttl: 50 }); + assert.ok(db.ttl('t') > 0 && db.ttl('t') <= 50); + assert.equal(db.ttl('nope'), -2); + await db.expire('t', 1000); + assert.ok(db.ttl('t') > 50); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('recovery truncates a torn WAL tail and keeps valid data', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + for (let i = 0; i < 100; i++) await db.set(`k${i}`, `v${i}`); + await db.close(); + + // Append a half-written frame to simulate a crash mid-write. + const walPath = path.join(dir, 'db.wal'); + const partial = encodeFrame({ type: TYPE_SET, key: B('torn'), value: B('x'.repeat(200)) }).subarray(0, 11); + await fs.appendFile(walPath, partial); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.recoveryInfo.truncatedWal, true); + assert.equal(db.size, 100); + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get('k99'), 'v99'); + await db.close(); + + // The torn tail is gone for good. + const reopened = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(reopened.recoveryInfo.truncatedWal, false); + assert.equal(reopened.size, 100); + await reopened.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('maxMemory reject policy blocks writes over budget', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string', maxMemoryBytes: 50, maxMemoryPolicy: 'reject' }); + await db.set('a', '1234567890'); + await db.set('b', '1234567890'); + await db.set('c', '1234567890'); + await db.set('d', '1234567890'); + await assert.rejects(() => db.set('e', '1234567890'), /maxMemory exceeded/); + assert.equal(db.get('e'), undefined); + assert.ok(db.stats.maxMemoryRejections >= 1); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('maxMemory evict-lru evicts old keys to make room', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string', maxMemoryBytes: 50, maxMemoryPolicy: 'evict-lru' }); + await db.set('a', '1234567890'); + await db.set('b', '1234567890'); + await db.set('c', '1234567890'); + await db.set('d', '1234567890'); + assert.equal(db.get('a'), '1234567890'); // touch a; b becomes the LRU victim + await db.set('e', '1234567890'); + assert.equal(db.get('a'), '1234567890'); + assert.equal(db.get('b'), undefined); + assert.equal(db.get('e'), '1234567890'); + assert.ok(db.stats.evictions >= 1); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('backup + restore preserves data, indexes, and text search', async () => { + const dir = await tmpDir(); + const backupDir = await tmpDir(); + const restoreDir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byCity', { field: 'city' }); + await db.createTextIndex('body', { fields: ['body'] }); + await db.set('a', { city: 'Paris', body: 'hello world' }); + await db.set('b', { city: 'London', body: 'hello London' }); + await db.backup(backupDir); + await db.close(); + + const restored = await MiniDb.restore(backupDir, restoreDir, { valueCodec: 'json' }); + assert.deepEqual(restored.get('a'), { city: 'Paris', body: 'hello world' }); + assert.deepEqual(restored.findEq('byCity', 'London').map((r) => r.key), ['b']); + assert.deepEqual(restored.search('body', 'hello').map((r) => r.key).sort(), ['a', 'b']); + await restored.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + await fs.rm(backupDir, { recursive: true, force: true }); + await fs.rm(restoreDir, { recursive: true, force: true }); + } +}); + +test('valueMode disk stores value pointers and reads from WAL', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk', fsyncPolicy: 'no' }); + const big = 'x'.repeat(1000); + await db.set('a', big); + const rec = db.store.map.get('a'); + assert.equal(rec?.ref.kind, 'disk'); + assert.equal(rec?.ref.kind === 'disk' && rec.ref.loc.file, 'wal'); + assert.equal(db.get('a'), big); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk' }); + const rec2 = db.store.map.get('a'); + assert.equal(rec2?.ref.kind, 'disk'); + assert.equal(db.get('a'), big); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('valueMode disk batch stores value pointers and survives reopen', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk', fsyncPolicy: 'no' }); + await db.batch([ + { op: 'set', key: 'a', value: '1'.repeat(100) }, + { op: 'set', key: 'b', value: '2'.repeat(100) }, + ]); + assert.equal(db.store.map.get('a')?.ref.kind, 'disk'); + assert.equal(db.store.map.get('b')?.ref.kind, 'disk'); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk' }); + assert.equal(db.get('a'), '1'.repeat(100)); + assert.equal(db.get('b'), '2'.repeat(100)); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('valueMode disk compaction remaps pointers to the snapshot', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk', fsyncPolicy: 'no' }); + await db.set('a', '1'.repeat(1000)); + await db.set('b', '2'.repeat(1000)); + await db.compact(); + const rec = db.store.map.get('a'); + assert.equal(rec?.ref.kind, 'disk'); + assert.equal(rec?.ref.kind === 'disk' && rec.ref.loc.file, 'snapshot'); + assert.equal(db.get('a'), '1'.repeat(1000)); + assert.equal(db.get('b'), '2'.repeat(1000)); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk' }); + assert.equal(db.get('a'), '1'.repeat(1000)); + assert.equal(db.get('b'), '2'.repeat(1000)); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('valueMode disk maxMemory excludes value bulk', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ + dir, + valueCodec: 'string', + valueMode: 'disk', + maxMemoryBytes: 200, + maxMemoryPolicy: 'reject', + fsyncPolicy: 'no', + }); + await db.set('a', 'x'.repeat(1000)); + await db.set('b', 'y'.repeat(1000)); + await db.set('c', 'z'.repeat(1000)); + assert.equal(db.get('a'), 'x'.repeat(1000)); + assert.ok(db.store.bytes < 200); + await assert.rejects(() => db.set('d', 'w'.repeat(1000)), /maxMemory exceeded/); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('valueMode auto selects disk when persisted files exceed maxMemoryBytes', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + await db.set('a', 'x'.repeat(1000)); + await db.close(); + + const walSize = (await fs.stat(path.join(dir, 'db.wal'))).size; + db = await MiniDb.open({ + dir, + valueCodec: 'string', + valueMode: 'auto', + maxMemoryBytes: Math.max(1, walSize - 1), + fsyncPolicy: 'no', + }); + assert.equal(db.valueMode, 'disk'); + assert.equal(db.store.map.get('a')?.ref.kind, 'disk'); + assert.equal(db.get('a'), 'x'.repeat(1000)); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('valueMode auto selects memory when persisted files fit maxMemoryBytes', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + await db.set('a', 'x'.repeat(1000)); + await db.close(); + + const walSize = (await fs.stat(path.join(dir, 'db.wal'))).size; + db = await MiniDb.open({ + dir, + valueCodec: 'string', + valueMode: 'auto', + maxMemoryBytes: walSize + 1000, + fsyncPolicy: 'no', + }); + assert.equal(db.valueMode, 'memory'); + assert.equal(db.store.map.get('a')?.ref.kind, 'memory'); + assert.equal(db.get('a'), 'x'.repeat(1000)); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('valueMode auto without maxMemoryBytes defaults to memory', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + await db.set('a', 'x'.repeat(1000)); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'auto', fsyncPolicy: 'no' }); + assert.equal(db.valueMode, 'memory'); + assert.equal(db.store.map.get('a')?.ref.kind, 'memory'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/defense.test.ts b/packages/minidb/test/defense.test.ts new file mode 100644 index 000000000..db2fb9373 --- /dev/null +++ b/packages/minidb/test/defense.test.ts @@ -0,0 +1,247 @@ +// Exercises defensive input/state-validation branches that are reachable +// through the public/direct API but were not covered by the functional tests. +// Fault-injection-only branches (writev short-write, fsync failure, >64MB +// RESP payload, cross-user EPERM, process-exit hook) are intentionally not +// covered here — see the coverage summary in the commit message. +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import net from 'node:net'; +import { WAL } from '../src/wal.js'; +import { encodeFrame, decodeBatchOps, TYPE_SET } from '../src/codec.js'; +import { MiniDb } from '../src/index.js'; +import { LockFile } from '../src/lockfile.js'; +import { startServer } from '../src/server.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-defense-')); +} + +const B = (s: string) => Buffer.from(s); + +// --- codec.encodeFrame input validation ------------------------------------ + +test('encodeFrame rejects a non-buffer key', () => { + assert.throws(() => encodeFrame({ type: TYPE_SET, key: 'x' as unknown as Buffer, value: B('v') }), /key must be a Buffer/); +}); + +test('encodeFrame rejects an oversized key', () => { + const big = Buffer.alloc(0xffff + 1); + assert.throws(() => encodeFrame({ type: TYPE_SET, key: big, value: B('v') }), /key too large/); +}); + +test('encodeFrame rejects a SET with a non-buffer value', () => { + assert.throws( + () => encodeFrame({ type: TYPE_SET, key: B('k'), value: 'not-a-buffer' as unknown as Buffer }), + /value must be a Buffer for SET/, + ); +}); + +test('encodeFrame rejects a non-buffer meta', () => { + assert.throws( + () => encodeFrame({ type: TYPE_SET, key: B('k'), value: B('v'), meta: 'x' as unknown as Buffer }), + /meta must be a Buffer/, + ); +}); + +// --- codec.decodeBatchOps bounds checks ------------------------------------ + +test('decodeBatchOps returns [] for a body shorter than the count field', () => { + assert.deepEqual(decodeBatchOps(Buffer.alloc(0)), []); + assert.deepEqual(decodeBatchOps(Buffer.alloc(1)), []); +}); + +test('decodeBatchOps throws on a truncated op header', () => { + // count = 1 but no op bytes follow. + const body = Buffer.alloc(2); + body.writeUInt16LE(1, 0); + assert.throws(() => decodeBatchOps(body), /batch op header truncated/); +}); + +test('decodeBatchOps throws on a truncated op payload', () => { + // Build one op header claiming a 100-byte key, then cut the body short. + const header = Buffer.alloc(1 + 2 + 4 + 4 + 8); + let o = 0; + header.writeUInt8(TYPE_SET, o); o += 1; + header.writeUInt16LE(100, o); o += 2; + header.writeUInt32LE(0, o); o += 4; + header.writeUInt32LE(0, o); o += 4; + header.writeBigInt64LE(0n, o); + const body = Buffer.concat([Buffer.from([1, 0]), header]); // count=1 + header, no payload + assert.throws(() => decodeBatchOps(body), /batch op payload truncated/); +}); + +// --- WAL input/state validation -------------------------------------------- + +test('WAL constructor rejects an unknown fsyncPolicy', () => { + assert.throws(() => new WAL('/tmp/x', { fsyncPolicy: 'bogus' as never }), /unknown fsyncPolicy/); +}); + +test('WAL append rejects a non-buffer frame', async () => { + const dir = await tmpDir(); + const wal = new WAL(path.join(dir, 'db.wal'), { fsyncPolicy: 'no' }); + await wal.open(); + try { + await assert.rejects(() => wal.append('x' as unknown as Buffer), /frame must be a Buffer/); + } finally { + await wal.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('WAL append after close rejects', async () => { + const dir = await tmpDir(); + const wal = new WAL(path.join(dir, 'db.wal'), { fsyncPolicy: 'no' }); + await wal.open(); + await wal.close(); + await assert.rejects( + () => wal.append(encodeFrame({ type: TYPE_SET, key: B('a'), value: B('1') })), + /WAL is closed/, + ); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('WAL open and close are idempotent', async () => { + const dir = await tmpDir(); + const wal = new WAL(path.join(dir, 'db.wal'), { fsyncPolicy: 'everysec' }); + await wal.open(); + await wal.open(); // second open is a no-op + await wal.close(); + await wal.close(); // second close is a no-op + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- LockFile stale / corrupt handling ------------------------------------- + +test('LockFile.acquire returns false when a live process holds the lock', async () => { + const dir = await tmpDir(); + const p = path.join(dir, 'db.lock'); + const a = new LockFile(p); + assert.equal(await a.acquire(), true); + const b = new LockFile(p); + assert.equal(await b.acquire(), false); // same PID is alive -> not stale + await a.release(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('a corrupt lock file is treated as stale and taken over', async () => { + const dir = await tmpDir(); + await fs.writeFile(path.join(dir, 'db.lock'), 'not-json'); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + await db.set('a', '1'); + assert.equal(db.get('a'), '1'); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('a lock file with a non-numeric pid is treated as stale', async () => { + const dir = await tmpDir(); + await fs.writeFile(path.join(dir, 'db.lock'), JSON.stringify({ pid: 'abc' })); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.readOnly, false); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('LockFile release/releaseSync are no-ops when not held', async () => { + const dir = await tmpDir(); + const lock = new LockFile(path.join(dir, 'db.lock')); + await assert.doesNotReject(() => lock.release()); + assert.doesNotThrow(() => lock.releaseSync()); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- snapshot / compaction edge cases -------------------------------------- + +test('compacting an empty database produces an empty snapshot and keeps data intact', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + await db.compact(); // empty store -> flushBatch early-return + assert.equal(db.stats.compactions, 1); + const snap = await fs.stat(path.join(dir, 'db.snapshot')); + assert.equal(snap.size, 0); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, 0); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('a second concurrent compact() reuses the in-flight compaction', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + for (let i = 0; i < 100; i++) await db.set(`k${i}`, `v${i}`); + const p1 = db.compact(); + const p2 = db.compact(); // compacting === true -> early-returns, reusing the in-flight work + // (compact is `async`, so each call wraps the shared _compactDone in a fresh + // promise; the guard is what prevents a second compaction from running.) + await Promise.all([p1, p2]); + assert.equal(db.stats.compactions, 1); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- key validation -------------------------------------------------------- + +test('set rejects an empty key', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + await assert.rejects(() => db.set('', 'v'), /key must be non-empty/); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('batch rejects an empty key', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + await assert.rejects(() => db.batch([{ op: 'set', key: '', value: 'v' }]), /key must be non-empty/); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- index lookup errors --------------------------------------------------- + +test('findEq / findRange on a missing index throw', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.throws(() => db.findEq('nope', 'x'), /no such index/); + assert.throws(() => db.findRange('nope', {}), /no such index/); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('compoundRange on a missing index throws', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.throws(() => db.compoundRange('nope', 'g'), /no such compound index/); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- RESP error reply ------------------------------------------------------ + +test('RESP server replies with -ERR when a command throws', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await new Promise((resolve, reject) => { + const s = net.connect(srv.port, '127.0.0.1'); + s.once('connect', () => resolve(s)); + s.once('error', reject); + }); + const r = await new Promise((resolve) => { + sock.once('data', (d) => resolve(d.toString())); + // 129-byte key exceeds MAX_KEY_LEN -> db.set throws -> server replies -ERR. + const key = 'x'.repeat(129); + sock.write(`*3\r\n$3\r\nSET\r\n$${key.length}\r\n${key}\r\n$1\r\nv\r\n`); + }); + assert.ok(r.startsWith('-ERR'), r); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/degrade.test.ts b/packages/minidb/test/degrade.test.ts new file mode 100644 index 000000000..7420b14fc --- /dev/null +++ b/packages/minidb/test/degrade.test.ts @@ -0,0 +1,59 @@ +// test/degrade.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; +import { LockError } from '../src/lockfile.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-degrade-')); +} + +test('openOrRebuild opens a healthy db normally', async () => { + const dir = await tmpDir(); + const db = await MiniDb.openOrRebuild({ dir, valueCodec: 'string' }); + await db.set('a', '1'); + assert.equal(db.get('a'), '1'); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('openOrRebuild discards a corrupt db and starts fresh', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byX', { field: 'x' }); + await db.set('important', { x: 1 }); + await db.close(); + + // Corrupt the index-definitions file so open() throws during load. + await fs.writeFile(path.join(dir, 'db.indexes.json'), '{ not valid json'); + + let rebuilt = null; + db = await MiniDb.openOrRebuild( + { dir, valueCodec: 'json' }, + { onRebuild: (e) => (rebuilt = e) }, + ); + assert.ok(rebuilt instanceof Error, 'onRebuild called with the original error'); + assert.equal(db.size, 0, 'fresh empty db after rebuild'); + // data is gone (rebuild semantics) and the db is usable again + await db.set('fresh', { v: 1 }); + assert.deepEqual(db.get('fresh'), { v: 1 }); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('openOrRebuild does NOT delete a live-locked db', async () => { + const dir = await tmpDir(); + const db1 = await MiniDb.open({ dir, valueCodec: 'string' }); + await db1.set('a', '1'); + try { + await assert.rejects(() => MiniDb.openOrRebuild({ dir, valueCodec: 'string' }), LockError); + // data still there + assert.ok(await fs.stat(path.join(dir, 'db.wal'))); + } finally { + await db1.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/e2e/boundary.test.ts b/packages/minidb/test/e2e/boundary.test.ts new file mode 100644 index 000000000..711d58d22 --- /dev/null +++ b/packages/minidb/test/e2e/boundary.test.ts @@ -0,0 +1,115 @@ +// test/e2e/boundary.test.js +// +// Boundary / resource edge cases. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { MiniDb } from '../../src/index.js'; +import { tmpDir, rmrf } from './helpers/tmp.js'; + +test('boundary: key length limits', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + await db.set('a'.repeat(128), 'ok'); // exactly at limit + assert.equal(db.get('a'.repeat(128)), 'ok'); + await assert.rejects(() => db.set('a'.repeat(129), 'no'), /key too long/); + await assert.rejects(() => db.set('', 'no'), /non-empty/); + } finally { + await db.close(); + await rmrf(dir); + } +}); + +test('boundary: large value round-trips and recovers', async () => { + const dir = await tmpDir(); + const big = Buffer.alloc(5 * 1024 * 1024, 0xab); // 5 MiB + big[0] = 0x01; + big[big.length - 1] = 0x02; + let db = await MiniDb.open({ dir, valueCodec: 'buffer', fsyncPolicy: 'always' }); + try { + await db.set('big', big); + const got = db.get('big'); + assert.equal(got.length, big.length); + assert.equal(got[0], 0x01); + assert.equal(got[big.length - 1], 0x02); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'buffer' }); + const got2 = db.get('big'); + assert.equal(got2.length, big.length); + assert.equal(got2[0], 0x01); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}); + +test('boundary: many keys survive compaction + recovery', async () => { + const dir = await tmpDir(); + const N = 20000; + let db = await MiniDb.open({ + dir, + valueCodec: 'string', + fsyncPolicy: 'no', + compactThresholdBytes: 64 * 1024, + }); + try { + const ops = []; + for (let i = 0; i < N; i++) ops.push(db.set('k' + i, 'v' + i)); + await Promise.all(ops); + if (db.compacting) await db._compactDone; + assert.equal(db.size, N); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, N); + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get('k' + (N - 1)), 'v' + (N - 1)); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}); + +test('boundary: empty db open/close/reopen', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'string' }); + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, 0); + assert.equal(db.get('nope'), undefined); + await db.close(); + await rmrf(dir); +}); + +test('boundary: overwrite same key many times keeps size 1', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + for (let i = 0; i < 1000; i++) await db.set('same', 'v' + i); + assert.equal(db.size, 1); + assert.equal(db.get('same'), 'v999'); + } finally { + await db.close(); + await rmrf(dir); + } +}); + +test('boundary: empty / unicode / binary values', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + await db.set('empty', ''); + await db.set('unicode', '你好 🌍'); + assert.equal(db.get('empty'), ''); + assert.equal(db.get('unicode'), '你好 🌍'); + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.get('empty'), ''); + assert.equal(db.get('unicode'), '你好 🌍'); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}); diff --git a/packages/minidb/test/e2e/compaction-race.test.ts b/packages/minidb/test/e2e/compaction-race.test.ts new file mode 100644 index 000000000..58cc7e06f --- /dev/null +++ b/packages/minidb/test/e2e/compaction-race.test.ts @@ -0,0 +1,185 @@ +// test/e2e/compaction-race.test.js +// +// Stress the stop-the-world compaction guard: heavy concurrent writes while +// compaction is triggered frequently (both auto and manual) must not lose any +// write, must not throw, and must leave a recoverable database. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { MiniDb } from '../../src/index.js'; +import { tmpDir, rmrf } from './helpers/tmp.js'; + +test('compaction-race: concurrent writes + frequent compaction lose nothing', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + compactThresholdBytes: 2048, // tiny -> compaction triggers a lot + }); + const N = 1000; + const written = new Map(); + try { + const ops = []; + for (let i = 0; i < N; i++) { + const k = 'k' + i; + const v = { i, pad: 'x'.repeat(30) }; + written.set(k, v); + ops.push(db.set(k, v)); + if (i % 100 === 0) ops.push(db.compact().then(() => {})); // manual compaction, concurrent + } + await Promise.all(ops); + if (db.compacting) await db._compactDone; + + for (const [k, v] of written) assert.deepEqual(db.get(k), v, `get(${k})`); + assert.equal(db.size, N); + const compactionsRan = db.stats.compactions; + + // recoverable + still correct + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'json' }); + for (const [k, v] of written) assert.deepEqual(db.get(k), v, `reopen get(${k})`); + assert.equal(db.size, N); + assert.ok(compactionsRan >= 1, 'at least one compaction ran'); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}); + +test('compaction-race: reads remain available during compaction', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + compactThresholdBytes: 4096, + }); + try { + for (let i = 0; i < 200; i++) await db.set('k' + i, { i }); + // trigger compaction but don't await; reads should still work immediately + const cp = db.compact(); + for (let i = 0; i < 200; i++) { + const v = db.get('k' + i); + assert.deepEqual(v, { i }, `read during compaction k${i}`); + } + await cp; + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}); + +test('compaction-race: snapshot phase does not block writes', async () => { + // A write issued while a large snapshot is being written must complete + // BEFORE the whole compaction finishes — i.e. the snapshot phase is + // non-blocking. writeSnapshot yields to the event loop every chunk, so the + // write's WAL append is serviced in a yield gap, well ahead of the snapshot + // completing. We race the first write against the compaction rather than + // measuring absolute timing, so the assertion stays robust under CPU/IO load + // (e.g. when the e2e files run concurrently). + const dir = await tmpDir(); + const db = await MiniDb.open({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + compactThresholdBytes: 1 << 30, // drive compaction manually + }); + try { + const N = 150_000; + for (let i = 0; i < N; i++) await db.set('k' + i, { i, pad: 'x'.repeat(500) }); + + const cp = db.compact(); + const first = db.set('w0', { i: 0 }); + const winner = await Promise.race([ + cp.then(() => 'compact' as const), + first.then(() => 'write' as const), + ]); + assert.equal( + winner, + 'write', + 'a write issued during compaction completes before compaction finishes (non-blocking snapshot)', + ); + await first; + + const M = 500; + for (let i = 1; i < M; i++) await db.set('w' + i, { i }); + await cp; + + for (let i = 0; i < M; i++) assert.deepEqual(db.get('w' + i), { i }, `get(w${i})`); + assert.equal(db.size, N + M); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}, 60_000); + +test('compaction-race: heavy writes during compaction grow a WAL tail that survives recovery', async () => { + // Sustained writes during compaction force the pre-copy loop to drain a real + // WAL tail; the tail must be replayed on top of the snapshot after a reopen. + const dir = await tmpDir(); + let db = await MiniDb.open({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + compactThresholdBytes: 1 << 30, + }); + try { + const N = 20_000; + for (let i = 0; i < N; i++) await db.set('k' + i, { i }); + + const cp = db.compact(); + const M = 5000; + const writes: Promise[] = []; + for (let i = 0; i < M; i++) writes.push(db.set('k' + i, { i, bumped: true })); + await Promise.all(writes); + await cp; + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'json' }); + for (let i = 0; i < M; i++) assert.deepEqual(db.get('k' + i), { i, bumped: true }, `bumped k${i}`); + for (let i = M; i < N; i++) assert.deepEqual(db.get('k' + i), { i }, `untouched k${i}`); + assert.equal(db.size, N); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}); + +test('compaction-race: valueMode disk preserves concurrent writes and remaps pointers', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ + dir, + valueCodec: 'json', + valueMode: 'disk', + fsyncPolicy: 'no', + compactThresholdBytes: 2048, + }); + const N = 300; + const written = new Map(); + try { + const ops = []; + for (let i = 0; i < N; i++) { + const k = 'k' + i; + const v = { i, pad: 'x'.repeat(100) }; + written.set(k, v); + ops.push(db.set(k, v)); + if (i % 50 === 0) ops.push(db.compact().then(() => {})); + } + await Promise.all(ops); + if (db.compacting) await db._compactDone; + + for (const [k, v] of written) assert.deepEqual(db.get(k), v, `get(${k})`); + assert.equal(db.size, N); + const sawDiskRef = [...db.store.map.values()].some((r) => r.ref.kind === 'disk'); + assert.ok(sawDiskRef, 'expected disk-backed value refs after compaction'); + + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'json', valueMode: 'disk' }); + for (const [k, v] of written) assert.deepEqual(db.get(k), v, `reopen get(${k})`); + assert.equal(db.size, N); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}); diff --git a/packages/minidb/test/e2e/crash-recovery.test.ts b/packages/minidb/test/e2e/crash-recovery.test.ts new file mode 100644 index 000000000..bdd1335b9 --- /dev/null +++ b/packages/minidb/test/e2e/crash-recovery.test.ts @@ -0,0 +1,84 @@ +// test/e2e/crash-recovery.test.js +// +// Crash injection: spawn a child that writes keys sequentially with +// fsyncPolicy='always', kill -9 it at a random time, then reopen and verify: +// - the database always opens (never corrupted); +// - recovered keys form a contiguous prefix k0..kM with no gaps; +// - every recovered key's value is correct. +// This is the durability contract for sequential 'always' writes. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { MiniDb } from '../../src/index.js'; +import { tmpDir, rmrf } from './helpers/tmp.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const WRITER = path.join(__dirname, 'helpers', 'crash-writer.ts'); + +function crashWriter(dir: string, compactEvery: number, runMs: number): Promise { + return new Promise((resolve) => { + const child = spawn(process.execPath, ['--import', 'tsx', WRITER, dir, String(compactEvery)], { + stdio: ['ignore', 'pipe', 'ignore'], + }); + let killTimer: ReturnType | null = null; + // Wait until the child signals progress (>=25 keys durable) before starting + // the kill clock, so the test is robust even under heavy CPU contention. + child.stdout.on('data', () => { + if (!killTimer) killTimer = setTimeout(() => child.kill('SIGKILL'), runMs); + }); + const safety = setTimeout(() => child.kill('SIGKILL'), 8000); // should not happen + child.on('exit', () => { + if (killTimer) clearTimeout(killTimer); + clearTimeout(safety); + resolve(); + }); + }); +} + +async function verifyContiguous(dir, label) { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + let last = -1; + for (let i = 0; i < 1_000_000; i++) { + const v = db.get('k' + i); + if (v === undefined) { + last = i - 1; + break; + } + assert.equal(v.i, i, `${label}: value mismatch at k${i}`); + } + await db.close(); + return last; // highest contiguous index +} + +test('crash-recovery: kill mid-write, recovery yields a contiguous correct prefix', async () => { + const runs = 8; + for (let r = 0; r < runs; r++) { + const dir = await tmpDir(); + try { + const runMs = Math.floor(Math.random() * 150); + await crashWriter(dir, 0, runMs); + const last = await verifyContiguous(dir, `run${r}`); + assert.ok(last >= 2, `run${r}: expected several durable keys, got up to k${last}`); + } finally { + await rmrf(dir); + } + } +}); + +test('crash-recovery: kill during compaction, still consistent', async () => { + const runs = 4; + for (let r = 0; r < runs; r++) { + const dir = await tmpDir(); + try { + const runMs = Math.floor(Math.random() * 150); + await crashWriter(dir, 40, runMs); // compact every 40 writes + const last = await verifyContiguous(dir, `compact-run${r}`); + assert.ok(last >= 2, `compact-run${r}: expected several durable keys, got up to k${last}`); + } finally { + await rmrf(dir); + } + } +}); diff --git a/packages/minidb/test/e2e/durability.test.ts b/packages/minidb/test/e2e/durability.test.ts new file mode 100644 index 000000000..1903806ee --- /dev/null +++ b/packages/minidb/test/e2e/durability.test.ts @@ -0,0 +1,61 @@ +// test/e2e/durability.test.js +// +// Durability semantics for the three fsync policies and close(). + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { MiniDb } from '../../src/index.js'; +import { tmpDir, rmrf } from './helpers/tmp.js'; + +for (const policy of ['always', 'everysec', 'no']) { + test(`durability: close() persists all writes (fsyncPolicy=${policy})`, async () => { + const dir = await tmpDir(); + const N = 200; + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: policy }); + try { + for (let i = 0; i < N; i++) await db.set('k' + i, 'v' + i); + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, N); + for (let i = 0; i < N; i++) assert.equal(db.get('k' + i), 'v' + i); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } + }); +} + +test("durability: 'always' writes each frame to the file before set() resolves", async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'always' }); + const walPath = path.join(dir, 'db.wal'); + try { + for (let i = 0; i < 20; i++) { + await db.set('k' + i, 'v' + i); + const buf = await fs.readFile(walPath); + assert.ok(buf.includes(Buffer.from('k' + i)), `frame for k${i} already in WAL`); + } + } finally { + await db.close(); + await rmrf(dir); + } +}); + +test('durability: data survives many open/close cycles', async () => { + const dir = await tmpDir(); + try { + for (let cycle = 0; cycle < 20; cycle++) { + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'everysec' }); + await db.set('cycle' + cycle, 'v' + cycle); + await db.close(); + } + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, 20); + for (let cycle = 0; cycle < 20; cycle++) assert.equal(db.get('cycle' + cycle), 'v' + cycle); + await db.close(); + } finally { + await rmrf(dir); + } +}); diff --git a/packages/minidb/test/e2e/fuzz-model.test.ts b/packages/minidb/test/e2e/fuzz-model.test.ts new file mode 100644 index 000000000..0c12b9128 --- /dev/null +++ b/packages/minidb/test/e2e/fuzz-model.test.ts @@ -0,0 +1,66 @@ +// test/e2e/fuzz-model.test.js +// +// Model-based fuzz: feed identical random op sequences to MiniDb and a tiny +// reference model, and assert they always agree. Seeded so any failure is +// reproducible by re-running with the same seed. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { MiniDb } from '../../src/index.js'; +import { Model } from './helpers/model.js'; +import { mulberry32, randInt, pick } from './helpers/prng.js'; +import { tmpDir, rmrf } from './helpers/tmp.js'; + +const KEYS = Array.from({ length: 24 }, (_, i) => 'k' + i); +const VALUES = [{ a: 1 }, { b: [1, 2, 3] }, { s: 'hello' }, { n: 42 }, null, { nested: { x: 1, y: [2] } }]; + +async function runSeed(seed, steps) { + const rng = mulberry32(seed >>> 0); + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + const model = new Model(); + const ctx = () => `seed=${seed} step`; + try { + for (let i = 0; i < steps; i++) { + const op = randInt(rng, 5); + const key = pick(rng, KEYS); + + if (op === 0) { + // set (no expiry, or far-future expiry so nothing auto-expires mid-run) + const value = pick(rng, VALUES); + const ttl = rng() < 0.2 ? 60000 : 0; + await db.set(key, value, ttl ? { ttl } : {}); + model.set(key, value, ttl); + } else if (op === 1) { + await db.del(key); + model.del(key); + } else if (op === 2) { + assert.deepEqual(db.get(key), model.get(key), `${ctx()} ${i}: get(${key})`); + } else if (op === 3) { + const ttl = 60000; + const a = await db.expire(key, ttl); + const e = model.expire(key, ttl); + assert.equal(a, e, `${ctx()} ${i}: expire(${key})`); + } else { + // reopen, then compare everything + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + for (const k of KEYS) { + assert.deepEqual(db.get(k), model.get(k), `${ctx()} ${i}: after reopen get(${k})`); + } + } + } + // final full compare + for (const k of KEYS) assert.deepEqual(db.get(k), model.get(k), `final get(${k}) seed=${seed}`); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +} + +test('fuzz-model: random op sequences match a reference model (many seeds)', async () => { + const seeds = [1, 2, 3, 42, 12345, 99999, 0xdeadbeef, 0xc0ffee, 777, 20240625]; + for (const seed of seeds) { + await runSeed(seed, 500); + } +}, 60_000); diff --git a/packages/minidb/test/e2e/helpers/crash-writer.ts b/packages/minidb/test/e2e/helpers/crash-writer.ts new file mode 100644 index 000000000..3e62496bf --- /dev/null +++ b/packages/minidb/test/e2e/helpers/crash-writer.ts @@ -0,0 +1,36 @@ +// test/e2e/helpers/crash-writer.js +// Child process: writes keys k0,k1,k2,... sequentially with fsyncPolicy='always' +// (so every *completed* set is durable), optionally compacting every N writes. +// Runs until killed by the parent (crash injection). +// +// node crash-writer.js [compactEvery] + +import { MiniDb } from '../../../src/index.js'; + +const dir = process.argv[2]; +const compactEvery = Number(process.argv[3] || 0); +if (!dir) { + console.error('usage: crash-writer.js [compactEvery]'); + process.exit(2); +} + +const db = await MiniDb.open({ + dir, + valueCodec: 'json', + fsyncPolicy: 'always', + autoCompact: false, +}); + +let i = 0; +for (;;) { + await db.set('k' + i, { i, pad: 'x'.repeat(40) }); + i++; + if (compactEvery && i % compactEvery === 0) { + try { + await db.compact(); + } catch { + /* compaction may race with the kill; ignore */ + } + } + if (i % 25 === 0) console.log(String(i)); +} diff --git a/packages/minidb/test/e2e/helpers/model.ts b/packages/minidb/test/e2e/helpers/model.ts new file mode 100644 index 000000000..b8e80017c --- /dev/null +++ b/packages/minidb/test/e2e/helpers/model.ts @@ -0,0 +1,52 @@ +// test/e2e/helpers/model.js +// A minimal reference model of the KV semantics, used by the model-based fuzz +// test. It mirrors what MiniDb should do with plain JS structures. + +const clone = (v) => (v === undefined ? undefined : structuredClone(v)); + +export class Model { + constructor() { + this.map = new Map(); // key -> { value, expireAt } + } + + _purge(k, r) { + if (r.expireAt && r.expireAt <= Date.now()) { + this.map.delete(k); + return true; + } + return false; + } + + set(key, value, ttl) { + this.map.set(key, { value: clone(value), expireAt: ttl ? Date.now() + ttl : 0 }); + } + + get(key) { + const r = this.map.get(key); + if (!r) return undefined; + if (this._purge(key, r)) return undefined; + return r.value; + } + + del(key) { + return this.map.delete(key); + } + + expire(key, ttl) { + const r = this.map.get(key); + if (!r || this._purge(key, r)) return false; + r.expireAt = Date.now() + ttl; + return true; + } + + has(key) { + return this.get(key) !== undefined; + } + + // Live (non-expired) keys at this instant. + liveKeys() { + const out = []; + for (const k of this.map.keys()) if (this.get(k) !== undefined) out.push(k); + return out.sort(); + } +} diff --git a/packages/minidb/test/e2e/helpers/prng.ts b/packages/minidb/test/e2e/helpers/prng.ts new file mode 100644 index 000000000..efcf7322a --- /dev/null +++ b/packages/minidb/test/e2e/helpers/prng.ts @@ -0,0 +1,33 @@ +// test/e2e/helpers/prng.js +// Deterministic seeded PRNG so a failing fuzz run can be reproduced exactly. + +export function mulberry32(seed) { + let a = seed >>> 0; + return function () { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +export function randInt(rng, n) { + return Math.floor(rng() * n); +} + +export function pick(rng, arr) { + return arr[Math.floor(rng() * arr.length)]; +} + +// string hash -> 32-bit seed +export function seedFromString(s) { + let h = 1779033703 ^ s.length; + for (let i = 0; i < s.length; i++) { + h = Math.imul(h ^ s.charCodeAt(i), 3432918353); + h = (h << 13) | (h >>> 19); + } + h = Math.imul(h ^ (h >>> 16), 2246822507); + h = Math.imul(h ^ (h >>> 13), 3266489909); + return (h ^= h >>> 16) >>> 0; +} diff --git a/packages/minidb/test/e2e/helpers/tmp.ts b/packages/minidb/test/e2e/helpers/tmp.ts new file mode 100644 index 000000000..5b76f97bd --- /dev/null +++ b/packages/minidb/test/e2e/helpers/tmp.ts @@ -0,0 +1,20 @@ +// test/e2e/helpers/tmp.js +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +export async function tmpDir(prefix = 'minidb-e2e-') { + return fs.mkdtemp(path.join(os.tmpdir(), prefix)); +} + +export async function rmrf(dir) { + await fs.rm(dir, { recursive: true, force: true }); +} + +export async function fileSize(p) { + try { + return (await fs.stat(p)).size; + } catch { + return 0; + } +} diff --git a/packages/minidb/test/e2e/index-consistency.test.ts b/packages/minidb/test/e2e/index-consistency.test.ts new file mode 100644 index 000000000..b83bbbcb3 --- /dev/null +++ b/packages/minidb/test/e2e/index-consistency.test.ts @@ -0,0 +1,92 @@ +// test/e2e/index-consistency.test.js +// +// Verify that all derived indexes (key order, dt, value secondary, full-text) +// stay consistent with the primary store under random operations, including +// after an index rebuild on reopen. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { MiniDb } from '../../src/index.js'; +import { UniqueViolationError } from '../../src/index-manager.js'; +import { mulberry32, randInt, pick } from './helpers/prng.js'; +import { tmpDir, rmrf } from './helpers/tmp.js'; + +const CITIES = ['Paris', 'London', 'Tokyo', 'Beijing']; +const BIOS = ['hello world', 'foo bar baz', '我住在北京 喜欢编程', 'database engine rocks', 'lark approval skill']; + +function randomDoc(rng) { + return { + city: pick(rng, CITIES), + age: randInt(rng, 60), + email: 'u' + randInt(rng, 100000) + '@x.com', + bio: pick(rng, BIOS), + }; +} + +test('index-consistency: indexes stay consistent with the store under random ops', async () => { + const rng = mulberry32(0x5eed1234); + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + await db.createIndex('byCity', { field: 'city' }); + await db.createIndex('byAge', { field: 'age', type: 'range' }); + await db.createIndex('byEmail', { field: 'email', unique: true }); + await db.createTextIndex('body', { fields: ['bio'] }); + + const live = new Map(); // key -> doc (reference of live docs) + try { + for (let i = 0; i < 800; i++) { + const key = 'u' + randInt(rng, 80); + if (rng() < 0.75) { + const doc = randomDoc(rng); + try { + await db.set(key, doc, { dt: { created: randInt(rng, 1_000_000) } }); + live.set(key, doc); + } catch (e) { + if (!(e instanceof UniqueViolationError)) throw e; + // unique-email collision: write rejected, live unchanged + } + } else { + await db.del(key); + live.delete(key); + } + } + + const expectedKeys = [...live.keys()].sort(); + + // 1) key order index + assert.deepEqual(db.scan().map((r) => r.key), expectedKeys, 'key order scan'); + + // 2) equality index + for (const city of CITIES) { + const fromIdx = db.findEq('byCity', city).map((r) => r.key).sort(); + const expected = [...live.entries()].filter(([, d]) => d.city === city).map(([k]) => k).sort(); + assert.deepEqual(fromIdx, expected, `byCity ${city}`); + } + + // 3) range index + const [min, max] = [20, 40]; + const fromRange = db.findRange('byAge', { min, max }).map((r) => r.key).sort(); + const expectedRange = [...live.entries()].filter(([, d]) => d.age >= min && d.age <= max).map(([k]) => k).sort(); + assert.deepEqual(fromRange, expectedRange, 'byAge range'); + + // 4) dt index + assert.deepEqual(db.dtRange('created', { gte: 0 }).map((r) => r.key).sort(), expectedKeys, 'dt created all'); + + // 5) text index: search results == docs whose bio contains the term + const term = '北京'; + const hits = db.search('body', term, { limit: 1000 }).map((r) => r.key).sort(); + const expectedHits = [...live.entries()].filter(([, d]) => d.bio.includes(term)).map(([k]) => k).sort(); + assert.deepEqual(hits, expectedHits, 'text search 北京'); + + // 6) after rebuild on reopen, indexes still match + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + const fromIdx2 = db.findEq('byCity', 'Paris').map((r) => r.key).sort(); + const expected2 = [...live.entries()].filter(([, d]) => d.city === 'Paris').map(([k]) => k).sort(); + assert.deepEqual(fromIdx2, expected2, 'byCity Paris after rebuild'); + assert.deepEqual(db.scan().map((r) => r.key), expectedKeys, 'key order after rebuild'); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}); diff --git a/packages/minidb/test/e2e/recovery-matrix.test.ts b/packages/minidb/test/e2e/recovery-matrix.test.ts new file mode 100644 index 000000000..62965b75a --- /dev/null +++ b/packages/minidb/test/e2e/recovery-matrix.test.ts @@ -0,0 +1,132 @@ +// test/e2e/recovery-matrix.test.js +// +// Matrix of recovery scenarios: WAL corruption at head/mid/tail under +// 'resync' vs 'strict', and snapshot + WAL combinations. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { MiniDb } from '../../src/index.js'; +import { HEADER_SIZE, CRC_SIZE } from '../../src/codec.js'; +import { tmpDir, rmrf } from './helpers/tmp.js'; + +// key 'kN'(2B) + value 'vN'(2B), no meta -> 22+2+2+0+4 = 30 bytes / frame +const FRAME = HEADER_SIZE + 2 + 2 + 0 + CRC_SIZE; + +async function writeTen(dir) { + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'always', autoCompact: false }); + for (let i = 0; i < 10; i++) await db.set('k' + i, 'v' + i); + await db.close(); +} + +async function corruptWalFrame(dir, frameIndex) { + const walPath = path.join(dir, 'db.wal'); + const buf = await fs.readFile(walPath); + buf[frameIndex * FRAME + HEADER_SIZE + 2] ^= 0xff; // flip a value byte -> bad crc + await fs.writeFile(walPath, buf); +} + +const POS = { head: 0, mid: 5, tail: 9 }; + +for (const mode of ['resync', 'strict']) { + for (const [where, idx] of Object.entries(POS)) { + test(`recovery-matrix: WAL corrupt at ${where}, mode=${mode}`, async () => { + const dir = await tmpDir(); + try { + await writeTen(dir); + await corruptWalFrame(dir, idx); + const db = await MiniDb.open({ dir, valueCodec: 'string', recovery: mode }); + const present = new Set(Array.from({ length: 10 }, (_, i) => 'k' + i).filter((k) => db.get(k) !== undefined)); + + if (where === 'tail') { + // last frame bad -> truncated, everything else recovered + assert.equal(db.recoveryInfo.truncatedWal, true); + assert.equal(present.size, 9); + assert.ok(!present.has('k9')); + } else if (mode === 'resync') { + // only the bad frame lost + assert.equal(present.size, 9); + assert.ok(!present.has('k' + idx)); + } else { + // strict: everything from the bad frame onward lost + for (let i = 0; i < 10; i++) { + if (i < idx) assert.ok(present.has('k' + i), `k${i} should survive`); + else assert.ok(!present.has('k' + i), `k${i} should be lost (strict)`); + } + } + await db.close(); + } finally { + await rmrf(dir); + } + }); + } +} + +test('recovery-matrix: clean WAL recovers everything (both modes)', async () => { + for (const mode of ['resync', 'strict']) { + const dir = await tmpDir(); + try { + await writeTen(dir); + const db = await MiniDb.open({ dir, valueCodec: 'string', recovery: mode }); + assert.equal(db.size, 10); + assert.equal(db.recoveryInfo.lostBytes, 0); + await db.close(); + } finally { + await rmrf(dir); + } + } +}); + +test('recovery-matrix: snapshot present, empty WAL', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'always', autoCompact: false }); + for (let i = 0; i < 10; i++) await db.set('k' + i, 'v' + i); + await db.compact(); + await db.close(); + const db2 = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db2.recoveryInfo.snapshotFrames, 10); + assert.equal(db2.recoveryInfo.walFrames, 0); + assert.equal(db2.size, 10); + await db2.close(); + } finally { + await rmrf(dir); + } +}); + +test('recovery-matrix: snapshot + clean WAL', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'always', autoCompact: false }); + for (let i = 0; i < 10; i++) await db.set('k' + i, 'v' + i); + await db.compact(); + for (let i = 0; i < 5; i++) await db.set('a' + i, 'b' + i); // new WAL writes + await db.close(); + const db2 = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db2.size, 15); + await db2.close(); + } finally { + await rmrf(dir); + } +}); + +test('recovery-matrix: snapshot + corrupt WAL mid (resync keeps snapshot + surviving WAL)', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'always', autoCompact: false }); + for (let i = 0; i < 10; i++) await db.set('k' + i, 'v' + i); + await db.compact(); + for (let i = 0; i < 5; i++) await db.set('a' + i, 'b' + i); // 5 new WAL frames + await db.close(); + await corruptWalFrame(dir, 2); // corrupt 'a2' in the new WAL + const db2 = await MiniDb.open({ dir, valueCodec: 'string', recovery: 'resync' }); + assert.equal(db2.get('a2'), undefined, 'corrupt WAL frame lost'); + assert.equal(db2.get('a0'), 'b0'); + assert.equal(db2.get('a4'), 'b4'); + assert.equal(db2.size, 14); // 10 snapshot + 4 surviving WAL + await db2.close(); + } finally { + await rmrf(dir); + } +}); diff --git a/packages/minidb/test/e2e/soak.test.ts b/packages/minidb/test/e2e/soak.test.ts new file mode 100644 index 000000000..63226c251 --- /dev/null +++ b/packages/minidb/test/e2e/soak.test.ts @@ -0,0 +1,70 @@ +// test/e2e/soak.test.js +// +// Long-running soak: sustained random set/del/compact/reopen to catch memory +// leaks and slow degradation. Opt-in via SOAK= because it is slow. +// +// SOAK=30 node --test --expose-gc test/e2e/soak.test.js + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { MiniDb } from '../../src/index.js'; +import { Model } from './helpers/model.js'; +import { mulberry32, pick } from './helpers/prng.js'; +import { tmpDir, rmrf } from './helpers/tmp.js'; + +const SOAK = Number(process.env.SOAK || 0); + +test('soak: sustained random ops do not leak memory or corrupt', { skip: !SOAK }, async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + compactThresholdBytes: 256 * 1024, + }); + const KEYS = Array.from({ length: 500 }, (_, i) => 'k' + i); + const rng = mulberry32(0x50ac1234); + const model = new Model(); + const deadline = Date.now() + SOAK * 1000; + let ops = 0; + let firstHeap = 0; + try { + while (Date.now() < deadline) { + for (let j = 0; j < 200; j++) { + const key = pick(rng, KEYS); + if (rng() < 0.7) { + const v = { i: ops, pad: 'x'.repeat(50) }; + await db.set(key, v); + model.set(key, v); + } else { + await db.del(key); + model.del(key); + } + ops++; + } + if (ops % 2000 === 0) { + if (global.gc) global.gc(); + const h = process.memoryUsage().heapUsed; + if (!firstHeap) firstHeap = h; + } + if (ops % 5000 === 0) await db.compact().catch(() => {}); + } + + if (global.gc) global.gc(); + const finalHeap = process.memoryUsage().heapUsed; + const mib = (n) => (n / 1024 / 1024).toFixed(1); + console.log(` soak: ${ops} ops, heap ${mib(firstHeap)} -> ${mib(finalHeap)} MiB`); + assert.ok( + finalHeap < firstHeap * 3 + 50 * 1024 * 1024, + `heap grew too much: ${firstHeap} -> ${finalHeap}`, + ); + + // integrity after reopen + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'json' }); + for (const k of KEYS) assert.deepEqual(db.get(k), model.get(k), `soak reopen ${k}`); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } +}); diff --git a/packages/minidb/test/indexes-extra.test.ts b/packages/minidb/test/indexes-extra.test.ts new file mode 100644 index 000000000..f9c0f9aae --- /dev/null +++ b/packages/minidb/test/indexes-extra.test.ts @@ -0,0 +1,120 @@ +// Covers the secondary-index paths not exercised by indexes.test.ts: drop, +// error branches, unique range indexes, and findRange options. +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb, UniqueViolationError } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-idx2-')); +} + +test('dropIndex removes the index and reports status', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byCity', { field: 'city' }); + assert.equal(db.listIndexes().length, 1); + assert.equal(await db.dropIndex('byCity'), true); + assert.equal(db.listIndexes().length, 0); + // Dropping a missing index returns false. + assert.equal(await db.dropIndex('byCity'), false); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('createIndex validates field and duplicate names', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await assert.rejects(() => db.createIndex('bad', {} as never), /requires a field/); + await db.createIndex('byCity', { field: 'city' }); + await assert.rejects(() => db.createIndex('byCity', { field: 'city' }), /already exists/); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('findEq / findRange reject the wrong index type', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('eq', { field: 'n' }); + await db.createIndex('rg', { field: 'n', type: 'range' }); + await db.set('a', { n: 1 }); + assert.throws(() => db.findEq('rg', 1), /not an equality index/); + assert.throws(() => db.findRange('eq', {}), /not a range index/); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('findRange supports exclusive bounds, offset and reverse', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byAge', { field: 'age', type: 'range' }); + for (let i = 1; i <= 5; i++) await db.set(`p${i}`, { age: i * 10 }); + + // Exclusive on both ends: (20, 50) => 30, 40. + assert.deepEqual( + db.findRange('byAge', { min: 20, max: 50, minExclusive: true, maxExclusive: true }).map((r) => r.field), + [30, 40], + ); + // Offset skips the first matches. + assert.deepEqual(db.findRange('byAge', { offset: 1, count: 2 }).map((r) => r.field), [20, 30]); + // Reverse ordering. + assert.deepEqual(db.findRange('byAge', { reverse: true, count: 2 }).map((r) => r.field), [50, 40]); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('unique range index rejects duplicate numeric values', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byScore', { field: 'score', type: 'range', unique: true }); + await db.set('a', { score: 10 }); + await assert.rejects(() => db.set('b', { score: 10 }), UniqueViolationError); + // Re-setting the same key with the same value is still allowed. + await db.set('a', { score: 10 }); + assert.equal(db.size, 1); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('sparse index skips records missing the field', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byCity', { field: 'city' }); // sparse by default + await db.set('a', { city: 'Paris' }); + await db.set('b', { name: 'no-city' }); + assert.deepEqual(db.findEq('byCity', 'Paris').map((r) => r.key), ['a']); + assert.equal(db.size, 2); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('secondary indexes require the json codec', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + await assert.rejects(() => db.createIndex('x', { field: 'n' }), /require valueCodec: "json"/); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/indexes.test.ts b/packages/minidb/test/indexes.test.ts new file mode 100644 index 000000000..1fbef4f41 --- /dev/null +++ b/packages/minidb/test/indexes.test.ts @@ -0,0 +1,108 @@ +// test/indexes.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; +import { UniqueViolationError } from '../src/index-manager.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-idx-')); +} + +test('equality index findEq', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byCity', { field: 'city' }); + await db.set('u1', { name: 'Ann', city: 'Paris' }); + await db.set('u2', { name: 'Bob', city: 'Paris' }); + await db.set('u3', { name: 'Eve', city: 'London' }); + const res = db.findEq('byCity', 'Paris').map((r) => r.key).sort(); + assert.deepEqual(res, ['u1', 'u2']); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('range index findRange with bounds + limit', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byAge', { field: 'age', type: 'range' }); + for (let i = 1; i <= 10; i++) await db.set(`p${i}`, { age: i * 10 }); + const ages = db.findRange('byAge', { min: 30, max: 70, count: 3 }).map((r) => r.field); + assert.deepEqual(ages, [30, 40, 50]); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('unique index rejects duplicates', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byEmail', { field: 'email', unique: true }); + await db.set('a', { email: 'x@y.com' }); + await assert.rejects(() => db.set('b', { email: 'x@y.com' }), UniqueViolationError); + // Re-setting the same key with the same value is allowed. + await db.set('a', { email: 'x@y.com' }); + assert.equal(db.size, 1); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('index is updated on delete', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byCity', { field: 'city' }); + await db.set('u1', { city: 'Paris' }); + await db.del('u1'); + assert.deepEqual(db.findEq('byCity', 'Paris'), []); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('array field is indexed per element', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byTag', { field: 'tags' }); + await db.set('a', { tags: ['red', 'blue'] }); + await db.set('b', { tags: ['blue', 'green'] }); + assert.deepEqual(db.findEq('byTag', 'red').map((r) => r.key), ['a']); + assert.deepEqual(db.findEq('byTag', 'blue').map((r) => r.key).sort(), ['a', 'b']); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('index definitions + data rebuild across reopen', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byAge', { field: 'age', type: 'range' }); + for (let i = 1; i <= 5; i++) await db.set(`p${i}`, { age: i }); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.deepEqual(db.listIndexes().map((i) => i.name), ['byAge']); + assert.deepEqual(db.findRange('byAge', { min: 2, max: 4 }).map((r) => r.key), [ + 'p2', + 'p3', + 'p4', + ]); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/lock.test.ts b/packages/minidb/test/lock.test.ts new file mode 100644 index 000000000..3e5c22b83 --- /dev/null +++ b/packages/minidb/test/lock.test.ts @@ -0,0 +1,75 @@ +// test/lock.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; +import { LockError } from '../src/lockfile.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-lock-')); +} + +test('a second writer on the same dir is rejected with LockError', async () => { + const dir = await tmpDir(); + const db1 = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + await assert.rejects(() => MiniDb.open({ dir, valueCodec: 'string' }), LockError); + } finally { + await db1.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('lock is released on close, allowing another writer', async () => { + const dir = await tmpDir(); + const db1 = await MiniDb.open({ dir, valueCodec: 'string' }); + await db1.set('a', '1'); + await db1.close(); + + const db2 = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db2.get('a'), '1'); + await db2.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('readOnly open succeeds alongside a writer and rejects writes', async () => { + const dir = await tmpDir(); + const db1 = await MiniDb.open({ dir, valueCodec: 'string' }); + await db1.set('a', '1'); + try { + const ro = await MiniDb.open({ dir, valueCodec: 'string', readOnly: true }); + assert.equal(ro.readOnly, true); + assert.equal(ro.get('a'), '1'); + await assert.rejects(() => ro.set('b', '2'), /read-only/); + await ro.close(); + } finally { + await db1.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test("onLockFail: 'readonly' degrades instead of throwing", async () => { + const dir = await tmpDir(); + const db1 = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + const db2 = await MiniDb.open({ dir, valueCodec: 'string', onLockFail: 'readonly' }); + assert.equal(db2.readOnly, true); + await db2.close(); + } finally { + await db1.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('a stale lock (dead PID) is taken over', async () => { + const dir = await tmpDir(); + await fs.writeFile(path.join(dir, 'db.lock'), JSON.stringify({ pid: 999999, ts: Date.now() })); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.readOnly, false); + await db.set('a', '1'); + assert.equal(db.get('a'), '1'); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); diff --git a/packages/minidb/test/query-engine.test.ts b/packages/minidb/test/query-engine.test.ts new file mode 100644 index 000000000..c5975e963 --- /dev/null +++ b/packages/minidb/test/query-engine.test.ts @@ -0,0 +1,192 @@ +// Drives the query engine through the real db.query() path to cover the +// operators and path handling that the higher-level tests do not exercise. +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-qe-')); +} + +async function seed() { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.set('a', { name: 'Ann', age: 30, city: 'Paris', tags: ['x', 'y'], score: 9.5, active: true }); + await db.set('b', { name: 'Bob', age: 17, city: 'London', tags: ['y'], score: 4.2, active: false }); + await db.set('c', { name: 'Eve', age: 25, city: 'Paris', tags: ['z'], score: 7.0 }); + await db.set('d', { name: 'Max', age: 40, city: 'Berlin', tags: [], score: 8.1, active: true }); + return { dir, db }; +} + +test('comparison operators $eq $ne $gt $gte $lt $lte', async () => { + const { dir, db } = await seed(); + try { + assert.deepEqual(db.query({ filter: { age: { $eq: 25 } } }).map((r) => r.key), ['c']); + assert.deepEqual(db.query({ filter: { age: { $ne: 25 } } }).map((r) => r.key).sort(), ['a', 'b', 'd']); + assert.deepEqual(db.query({ filter: { age: { $gte: 30 } } }).map((r) => r.key).sort(), ['a', 'd']); + assert.deepEqual(db.query({ filter: { age: { $lte: 25 } } }).map((r) => r.key).sort(), ['b', 'c']); + // Combine two operators in one cond object (both must hold). + assert.deepEqual( + db.query({ filter: { age: { $gt: 17, $lt: 30 } } }).map((r) => r.key), + ['c'], + ); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('$in / $nin membership', async () => { + const { dir, db } = await seed(); + try { + assert.deepEqual(db.query({ filter: { city: { $in: ['Paris', 'Berlin'] } } }).map((r) => r.key).sort(), [ + 'a', + 'c', + 'd', + ]); + assert.deepEqual(db.query({ filter: { city: { $nin: ['Paris'] } } }).map((r) => r.key).sort(), ['b', 'd']); + // Non-array argument never matches. + assert.deepEqual(db.query({ filter: { city: { $in: 'Paris' as unknown as string[] } } }).map((r) => r.key), []); + assert.deepEqual(db.query({ filter: { city: { $nin: 'Paris' as unknown as string[] } } }).map((r) => r.key), []); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('$exists and $type', async () => { + const { dir, db } = await seed(); + try { + // 'active' is set on a, b, d but not c. + assert.deepEqual(db.query({ filter: { active: { $exists: true } } }).map((r) => r.key).sort(), ['a', 'b', 'd']); + assert.deepEqual(db.query({ filter: { active: { $exists: false } } }).map((r) => r.key), ['c']); + assert.deepEqual(db.query({ filter: { name: { $type: 'string' } } }).map((r) => r.key).sort(), [ + 'a', + 'b', + 'c', + 'd', + ]); + assert.deepEqual(db.query({ filter: { age: { $type: 'string' } } }).map((r) => r.key), []); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('$regex with string, [pattern, flags], RegExp and stateful global RegExp', async () => { + const { dir, db } = await seed(); + try { + assert.deepEqual(db.query({ filter: { name: { $regex: '^A' } } }).map((r) => r.key), ['a']); + // Tuple form supplies flags. + assert.deepEqual( + db.query({ filter: { name: { $regex: ['^a', 'i'] as unknown as string } } }).map((r) => r.key), + ['a'], + ); + // RegExp instance form. + assert.deepEqual(db.query({ filter: { name: { $regex: /^B/ } } }).map((r) => r.key), ['b']); + // A global RegExp is stateful; the engine must reset lastIndex so every + // document is tested from the start. + const g = /^E/g; + assert.deepEqual(db.query({ filter: { name: { $regex: g } } }).map((r) => r.key), ['c']); + // Non-string field never matches a regex. + assert.deepEqual(db.query({ filter: { age: { $regex: '3' } } }).map((r) => r.key), []); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('scalar RegExp value resets lastIndex between documents', async () => { + const { dir, db } = await seed(); + try { + const g = /^A/g; + assert.deepEqual(db.query({ filter: { name: g } }).map((r) => r.key), ['a']); + // Non-string value against a scalar RegExp never matches. + assert.deepEqual(db.query({ filter: { age: /^3/ } }).map((r) => r.key), []); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('logical operators $and $or $nor $not', async () => { + const { dir, db } = await seed(); + try { + assert.deepEqual( + db.query({ filter: { $and: [{ city: 'Paris' }, { age: { $gt: 20 } }] } }).map((r) => r.key).sort(), + ['a', 'c'], + ); + assert.deepEqual( + db.query({ filter: { $or: [{ name: 'Ann' }, { name: 'Bob' }] } }).map((r) => r.key).sort(), + ['a', 'b'], + ); + // $nor: matches documents that satisfy NONE of the branches. + assert.deepEqual( + db.query({ filter: { $nor: [{ city: 'Paris' }, { city: 'London' }] } }).map((r) => r.key), + ['d'], + ); + assert.deepEqual(db.query({ filter: { $not: { city: 'Paris' } } }).map((r) => r.key).sort(), ['b', 'd']); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('unknown operator and non-array logical args never match', async () => { + const { dir, db } = await seed(); + try { + // Unknown operator falls through to the default case (no match). + assert.deepEqual(db.query({ filter: { age: { $bogus: 1 } as never } }).map((r) => r.key), []); + // $and/$or/$nor with a non-array argument are treated as non-matching. + assert.deepEqual(db.query({ filter: { $and: { city: 'Paris' } as never } }).map((r) => r.key), []); + assert.deepEqual(db.query({ filter: { $or: { city: 'Paris' } as never } }).map((r) => r.key), []); + assert.deepEqual(db.query({ filter: { $nor: { city: 'Paris' } as never } }).map((r) => r.key), []); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('nested dot + bracket paths in filter and projection', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.set('p1', { user: { name: 'Ann', addr: { zip: '75001' } }, items: [{ sku: 'A' }, { sku: 'B' }] }); + await db.set('p2', { user: { name: 'Bob', addr: { zip: '12345' } }, items: [{ sku: 'A' }] }); + await db.set('p3', { user: { name: 'Eve' }, items: [] }); + + assert.deepEqual(db.query({ filter: { 'user.name': 'Bob' } }).map((r) => r.key), ['p2']); + assert.deepEqual(db.query({ filter: { 'user.addr.zip': '75001' } }).map((r) => r.key), ['p1']); + // Bracket index into an array element. + assert.deepEqual(db.query({ filter: { 'items[1].sku': 'B' } }).map((r) => r.key), ['p1']); + // Missing intermediate path does not match. + assert.deepEqual(db.query({ filter: { 'user.addr.zip': 'x' } }).map((r) => r.key).length, 0); + + // Nested projection rebuilds the nested shape in the output. + const projected = db.query({ filter: { 'user.name': 'Ann' }, project: ['user.addr.zip', 'items[0].sku'] }); + assert.deepEqual(projected[0]!.value, { user: { addr: { zip: '75001' } }, items: [{ sku: 'A' }] }); + + // Projecting a missing path drops it; empty project list returns the doc. + assert.deepEqual(db.query({ project: [] })[0]!.value, (await db.get('p1'))); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('sort ascending/descending, skip and limit', async () => { + const { dir, db } = await seed(); + try { + assert.deepEqual(db.query({ sort: { age: 1 } }).map((r) => r.key), ['b', 'c', 'a', 'd']); + assert.deepEqual(db.query({ sort: { age: -1 } }).map((r) => r.key), ['d', 'a', 'c', 'b']); + assert.deepEqual(db.query({ sort: { age: 1 }, skip: 1, limit: 2 }).map((r) => r.key), ['c', 'a']); + // No filter / sort returns every document. + assert.equal(db.query().length, 4); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/query.test.ts b/packages/minidb/test/query.test.ts new file mode 100644 index 000000000..0dbab6e07 --- /dev/null +++ b/packages/minidb/test/query.test.ts @@ -0,0 +1,189 @@ +// test/query.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-query-')); +} + +test('key range + prefix scan', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + for (const k of ['user:3', 'user:1', 'user:2', 'order:1']) await db.set(k, k); + assert.deepEqual(db.scan({ gte: 'user:', lte: 'user:~' }).map((r) => r.key), [ + 'user:1', + 'user:2', + 'user:3', + ]); + assert.deepEqual(db.prefix('user:').map((r) => r.key), ['user:1', 'user:2', 'user:3']); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('dt columns: set, range query, persist across reopen', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'json' }); + const jan = Date.parse('2024-01-01'); + const mar = Date.parse('2024-03-01'); + const jun = Date.parse('2024-06-01'); + await db.set('a', { n: 1 }, { dt: { created: jan } }); + await db.set('b', { n: 2 }, { dt: { created: mar } }); + await db.set('c', { n: 3 }, { dt: { created: jun } }); + + assert.deepEqual(db.dtColumns().sort(), ['created']); + const rows = db.dtRange('created', { gte: jan, lte: mar }); + assert.deepEqual(rows.map((r) => r.key), ['a', 'b']); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.deepEqual(db.dtRange('created', { gt: mar }).map((r) => r.key), ['c']); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('value filter (Mongo-like) with operators', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.set('a', { name: 'Ann', age: 30, tags: ['x', 'y'] }); + await db.set('b', { name: 'Bob', age: 17, tags: ['y'] }); + await db.set('c', { name: 'Eve', age: 25, tags: ['z'] }); + + assert.deepEqual( + db.query({ filter: { age: { $gt: 18 } } }).map((r) => r.key).sort(), + ['a', 'c'], + ); + assert.deepEqual( + db.query({ filter: { tags: { $contains: 'y' } } }).map((r) => r.key).sort(), + ['a', 'b'], + ); + assert.deepEqual( + db.query({ filter: { $or: [{ age: { $lt: 18 } }, { name: 'Eve' }] } }).map((r) => r.key).sort(), + ['b', 'c'], + ); + assert.deepEqual( + db.query({ filter: { name: { $regex: '^A' } } }).map((r) => r.key), + ['a'], + ); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('query composes dt range + value filter + sort + limit + project', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + for (let i = 1; i <= 10; i++) { + await db.set(`p${i}`, { age: i * 5, city: i % 2 ? 'Paris' : 'London' }, { dt: { ts: i * 1000 } }); + } + const res = db.query({ + dt: { ts: { gte: 3000, lte: 8000 } }, + filter: { city: 'Paris' }, + sort: { age: -1 }, + limit: 2, + project: ['age'], + }); + // ts in [3000,8000] => i=3..8; Paris (odd i) => 3,5,7; sort age desc => 7(35),5(25) + assert.deepEqual(res.map((r) => r.key), ['p7', 'p5']); + assert.deepEqual(res[0].value, { age: 35 }); // projected + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('full-text search: latin + CJK', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createTextIndex('bio', { fields: ['bio'] }); + await db.set('a', { bio: 'hello world from London' }); + await db.set('b', { bio: '我住在北京，喜欢编程' }); + await db.set('c', { bio: '我在上海写代码' }); + + const latin = db.search('bio', 'hello').map((r) => r.key); + assert.deepEqual(latin, ['a']); + + const cjk = db.search('bio', '北京').map((r) => r.key); + assert.deepEqual(cjk, ['b']); + + const or = db.search('bio', '北京 上海', { op: 'OR' }).map((r) => r.key).sort(); + assert.deepEqual(or, ['b', 'c']); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('text index persists + rebuilds across reopen', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createTextIndex('bio', { fields: ['bio'] }); + await db.set('a', { bio: '我爱北京天安门' }); + await db.set('b', { bio: '今天天气不错' }); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.deepEqual(db.search('bio', '北京').map((r) => r.key), ['a']); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('query composes key prefix + text + filter', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createTextIndex('body'); + await db.set('post:1', { title: 'Redis 持久化详解', tag: 'db' }); + await db.set('post:2', { title: 'Node 事件循环', tag: 'js' }); + await db.set('note:1', { title: 'Redis 笔记', tag: 'db' }); + + const res = db.query({ + key: { prefix: 'post:' }, + text: { index: 'body', q: 'Redis' }, + filter: { tag: 'db' }, + }); + assert.deepEqual(res.map((r) => r.key), ['post:1']); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('query uses value indexes for equality/range filters', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byCity', { field: 'city' }); + await db.createIndex('byAge', { field: 'age', type: 'range' }); + await db.set('a', { city: 'Paris', age: 30 }); + await db.set('b', { city: 'Paris', age: 17 }); + await db.set('c', { city: 'London', age: 41 }); + + assert.deepEqual(db.query({ filter: { city: 'Paris' } }).map((r) => r.key).sort(), ['a', 'b']); + assert.deepEqual(db.query({ filter: { age: { $gte: 30 } } }).map((r) => r.key).sort(), ['a', 'c']); + assert.deepEqual( + db.query({ filter: { $and: [{ city: 'Paris' }, { age: { $gte: 18 } }] } }).map((r) => r.key), + ['a'], + ); + assert.ok(db.stats.queryIndexHits >= 3); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/recovery.test.ts b/packages/minidb/test/recovery.test.ts new file mode 100644 index 000000000..68c88c430 --- /dev/null +++ b/packages/minidb/test/recovery.test.ts @@ -0,0 +1,111 @@ +// test/recovery.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; +import { HEADER_SIZE, CRC_SIZE } from '../src/codec.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-recover-')); +} + +// Each record: key='kN'(2B), value='vN'(2B), no meta -> 22+2+2+0+4 = 30 bytes +const FRAME = HEADER_SIZE + 2 + 2 + 0 + CRC_SIZE; + +async function writeFive(dir) { + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'always', autoCompact: false }); + for (let i = 0; i < 5; i++) await db.set(`k${i}`, `v${i}`); + await db.close(); +} + +test('resync: a single corrupt frame mid-file only loses that frame', async () => { + const dir = await tmpDir(); + try { + await writeFive(dir); + const walPath = path.join(dir, 'db.wal'); + const buf = await fs.readFile(walPath); + assert.equal(buf.length, FRAME * 5); + + // Corrupt k2's value (frame at offset 2*FRAME; value starts after header+key). + buf[2 * FRAME + HEADER_SIZE + 2] ^= 0xff; + await fs.writeFile(walPath, buf); + + const db = await MiniDb.open({ dir, valueCodec: 'string', recovery: 'resync' }); + assert.equal(db.recoveryInfo.corruptRanges.length, 1); + assert.deepEqual(db.recoveryInfo.corruptRanges[0], [2 * FRAME, 3 * FRAME]); + assert.equal(db.recoveryInfo.lostBytes, FRAME); + + // k2 lost, everything else recovered. + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get('k1'), 'v1'); + assert.equal(db.get('k2'), undefined); + assert.equal(db.get('k3'), 'v3'); + assert.equal(db.get('k4'), 'v4'); + assert.equal(db.size, 4); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('resync: multiple corrupt frames are each skipped', async () => { + const dir = await tmpDir(); + try { + await writeFive(dir); + const walPath = path.join(dir, 'db.wal'); + const buf = await fs.readFile(walPath); + buf[1 * FRAME + HEADER_SIZE + 2] ^= 0xff; // k1 + buf[3 * FRAME + HEADER_SIZE + 2] ^= 0xff; // k3 + await fs.writeFile(walPath, buf); + + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.recoveryInfo.corruptRanges.length, 2); + assert.deepEqual( + [...new Set(['k0', 'k1', 'k2', 'k3', 'k4'].filter((k) => db.get(k) !== undefined))].sort(), + ['k0', 'k2', 'k4'], + ); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('resync: torn tail is still truncated', async () => { + const dir = await tmpDir(); + try { + await writeFive(dir); + const walPath = path.join(dir, 'db.wal'); + const valid = await fs.readFile(walPath); + // append a half-written frame + await fs.appendFile(walPath, valid.subarray(0, 11)); + + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.recoveryInfo.truncatedWal, true); + assert.equal(db.size, 5); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('strict mode truncates at the first bad frame', async () => { + const dir = await tmpDir(); + try { + await writeFive(dir); + const walPath = path.join(dir, 'db.wal'); + const buf = await fs.readFile(walPath); + buf[2 * FRAME + HEADER_SIZE + 2] ^= 0xff; // corrupt k2 + await fs.writeFile(walPath, buf); + + const db = await MiniDb.open({ dir, valueCodec: 'string', recovery: 'strict' }); + // strict recovers k0,k1 then stops at k2; k3,k4 are NOT recovered. + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get('k1'), 'v1'); + assert.equal(db.size, 2); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/review-fixes.test.ts b/packages/minidb/test/review-fixes.test.ts new file mode 100644 index 000000000..0b761bab3 --- /dev/null +++ b/packages/minidb/test/review-fixes.test.ts @@ -0,0 +1,164 @@ +// Regression tests for the deep-review fixes. +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; +import { WAL } from '../src/wal.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-fix-')); +} + +// --- P0: WAL.flush() must drain frames queued behind an in-flight batch ----- + +test('WAL.flush() drains frames queued behind an in-flight batch', async () => { + const dir = await tmpDir(); + try { + const wal = new WAL(path.join(dir, 'a.wal'), { fsyncPolicy: 'always' }); + await wal.open(); + const big = Buffer.alloc(1024 * 1024, 0x61); + const pA = wal.append(big); + await new Promise((r) => setImmediate(r)); + await new Promise((r) => setImmediate(r)); + const pB = wal.append(Buffer.from('B')); + await wal.flush(); + const pending = (wal as unknown as { queue: unknown[] }).queue.length; + await wal.close(); + await pA; + await pB; + assert.equal(pending, 0, 'flush() must leave nothing queued'); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- P0: compaction must not lose concurrent writes (clean restart) --------- + +for (const policy of ['always', 'everysec', 'no'] as const) { + test(`compact + concurrent writes survive clean close+reopen (fsync=${policy})`, async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ + dir, + valueCodec: 'string', + fsyncPolicy: policy, + compactThresholdBytes: 1, + autoCompact: false, + }); + for (let i = 0; i < 50; i++) await db.set(`seed${i}`, 'x'.repeat(64)); + + const N = 500; + const big = 'y'.repeat(4096); + const writes: Promise[] = []; + for (let i = 0; i < N; i++) writes.push(db.set(`live${i}`, big)); + await Promise.all([db.compact(), ...writes]); + await db.close(); + + const db2 = await MiniDb.open({ dir, valueCodec: 'string' }); + const lost: string[] = []; + for (let i = 0; i < N; i++) if (db2.get(`live${i}`) !== big) lost.push(`live${i}`); + await db2.close(); + assert.deepEqual(lost, [], `lost ${lost.length}/${N} keys: ${lost.slice(0, 5).join(',')}`); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); +} + +// --- P0: TTL expiration must drop derived index entries --------------------- + +test('expired keys are removed from secondary indexes', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json', activeExpireIntervalMs: 20 }); + await db.createIndex('byCity', { field: 'city' }); + await db.set('u1', { city: 'Paris' }, { ttl: 30 }); + await new Promise((r) => setTimeout(r, 120)); + assert.equal(db.get('u1'), undefined); + assert.deepEqual(db.findEq('byCity', 'Paris'), []); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('expired keys are removed from the full-text index', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json', activeExpireIntervalMs: 20 }); + await db.createTextIndex('body'); + await db.set('p1', { bio: 'hello world' }, { ttl: 30 }); + await new Promise((r) => setTimeout(r, 120)); + assert.deepEqual(db.search('body', 'hello'), []); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- P1: batch() must enforce unique indexes within the batch --------------- + +test('batch() rejects intra-batch unique violations', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byMail', { field: 'email', unique: true }); + await assert.rejects( + db.batch([ + { op: 'set', key: 'a', value: { email: 'dup@x.com' } }, + { op: 'set', key: 'b', value: { email: 'dup@x.com' } }, + ]), + /unique/i, + ); + assert.equal(db.get('a'), undefined, 'nothing committed on failure'); + assert.equal(db.get('b'), undefined); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- P1: recovery must drop records whose TTL already elapsed --------------- + +test('recovery drops expired records (size consistent with scan)', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); + await db.set('ephemeral', 'v', { ttl: 1 }); + await db.set('stable', 'ok'); + await new Promise((r) => setTimeout(r, 20)); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); + assert.equal(db.size, 1); + assert.equal(db.scan().length, 1); + assert.equal(db.get('ephemeral'), undefined); + assert.equal(db.get('stable'), 'ok'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- concurrent sets keep the unique index consistent ---------------------- + +test('concurrent sets cannot both commit the same unique value', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byMail', { field: 'email', unique: true }); + const results = await Promise.allSettled([ + db.set('a', { email: 'same@x.com' }), + db.set('b', { email: 'same@x.com' }), + ]); + const committed = results.filter((r) => r.status === 'fulfilled').length; + const hits = db.findEq('byMail', 'same@x.com'); + await db.close(); + assert.ok(committed <= 1, `both committed: ${JSON.stringify(hits)}`); + assert.ok(hits.length <= 1, `unique violated: ${JSON.stringify(hits)}`); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/review-round2.test.ts b/packages/minidb/test/review-round2.test.ts new file mode 100644 index 000000000..ff29d813e --- /dev/null +++ b/packages/minidb/test/review-round2.test.ts @@ -0,0 +1,376 @@ +// Regression tests for the second deep-review round (compound-index dup, +// snapshot/WAL short-write, rollback on WAL failure, RESP binary, buffer +// aliasing, batch unique swap/del+set, unique-on-existing-dups, batch key +// length, TTL heap growth, size vs expiry, range-array, dtColumns, MSET +// atomicity, openOrRebuild scope, open-failure cleanup). +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import net from 'node:net'; +import { MiniDb } from '../src/index.js'; +import { startServer } from '../src/server.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-r2-')); +} + +// --- #1 compound index: re-setting same group+order must not duplicate ------ + +test('compound index: re-set with same group+order does not duplicate', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createCompoundIndex('byWs', { groupBy: 'workspaceId', orderBy: 'updatedAt' }); + try { + await db.set('a', { workspaceId: 'W1' }, { dt: { updatedAt: 100 } }); + await db.set('a', { workspaceId: 'W1' }, { dt: { updatedAt: 100 } }); + await db.set('a', { workspaceId: 'W1' }, { dt: { updatedAt: 100 } }); + assert.deepEqual(db.compoundRange('byWs', 'W1').map((r) => r.key), ['a']); + await db.del('a'); + assert.deepEqual(db.compoundRange('byWs', 'W1'), []); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('compound index: becoming invalid removes the entry cleanly', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createCompoundIndex('byWs', { groupBy: 'workspaceId', orderBy: 'updatedAt' }); + try { + await db.set('a', { workspaceId: 'W1' }, { dt: { updatedAt: 100 } }); + await db.set('a', { other: 1 }); // no workspaceId -> removed from index + assert.deepEqual(db.compoundRange('byWs', 'W1'), []); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- #2 / #3 WAL + snapshot tolerate fragmented (short) writes -------------- + +test('WAL and snapshot tolerate short writes (fragmented writev)', async () => { + const dir = await tmpDir(); + // Force every writev to write at most 11 bytes, so each frame lands in many + // short writes. The retry loops must still deliver every byte. + const probe = await fs.open(path.join(dir, '_probe'), 'w'); + const proto = Object.getPrototypeOf(probe) as { writev: (...a: unknown[]) => Promise }; + await probe.close(); + await fs.rm(path.join(dir, '_probe'), { force: true }); + const orig = proto.writev; + const CAP = 11; + proto.writev = async function (buffers: ReadonlyArray<{ length: number; subarray: (a: number, b: number) => unknown }>, position?: number | null) { + const first = buffers[0]; + if (!first) return orig.call(this, buffers, position); + const n = Math.min(first.length, CAP); + return orig.call(this, [first.subarray(0, n)], position); + }; + try { + const db = await MiniDb.open({ dir, valueCodec: 'string', compactThresholdBytes: 1, autoCompact: false }); + for (let i = 0; i < 30; i++) await db.set(`k${i}`, `value-${i}-${'x'.repeat(40)}`); + await db.compact(); + await db.set('after', 'tail'); + await db.close(); + + const db2 = await MiniDb.open({ dir, valueCodec: 'string' }); + for (let i = 0; i < 30; i++) assert.equal(db2.get(`k${i}`), `value-${i}-${'x'.repeat(40)}`); + assert.equal(db2.get('after'), 'tail'); + await db2.close(); + } finally { + proto.writev = orig; + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- #3 rollback on WAL write failure --------------------------------------- + +test('set rolls back store + indexes when the WAL write fails', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'always' }); + await db.createIndex('byCity', { field: 'city' }); + await db.set('old', { city: 'Paris' }); + + const fh = (db as unknown as { wal: { fh: { writev: (...a: unknown[]) => Promise } } }).wal.fh; + const orig = fh.writev.bind(fh); + let boom = true; + fh.writev = async (...a: unknown[]) => { + if (boom) { + boom = false; + throw new Error('injected WAL failure'); + } + return orig(...a); + }; + + await assert.rejects(db.set('new', { city: 'London' }), /injected/); + assert.equal(db.get('new'), undefined, 'failed set must not be visible'); + assert.deepEqual(db.findEq('byCity', 'London'), [], 'index must not contain rolled-back key'); + // Further writes still work after the injected failure clears. + boom = false; + await db.set('ok', { city: 'Berlin' }); + assert.equal(db.get('ok')?.city, 'Berlin'); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('batch rolls back atomically when the WAL write fails', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'always' }); + await db.set('a', { n: 1 }); + + const fh = (db as unknown as { wal: { fh: { writev: (...a: unknown[]) => Promise } } }).wal.fh; + const orig = fh.writev.bind(fh); + let boom = true; + fh.writev = async (...a: unknown[]) => { + if (boom) { + boom = false; + throw new Error('injected WAL failure'); + } + return orig(...a); + }; + + await assert.rejects( + db.batch([ + { op: 'set', key: 'a', value: { n: 99 } }, + { op: 'set', key: 'b', value: { n: 2 } }, + ]), + /injected/, + ); + assert.deepEqual(db.get('a'), { n: 1 }, 'a must be restored to pre-batch value'); + assert.equal(db.get('b'), undefined, 'b must not exist'); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #4 RESP server is binary-safe for non-ASCII values --------------------- + +test('RESP GET returns correct UTF-8 bulk for non-ASCII values', async () => { + const dir = await tmpDir(); + const { port, close } = await startServer({ dir, port: 0 }); + try { + const raw = await new Promise((resolve, reject) => { + const sock = net.connect(port, '127.0.0.1'); + const chunks: Buffer[] = []; + sock.on('data', (c) => chunks.push(c)); + sock.on('connect', () => { + const v = Buffer.from('北京', 'utf8'); + const set = `*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$${v.length}\r\n`; + sock.write(Buffer.concat([Buffer.from(set, 'binary'), v, Buffer.from('\r\n')])); + setTimeout(() => sock.write('GET k\r\n'), 50); + setTimeout(() => sock.end(), 150); + }); + sock.on('end', () => resolve(Buffer.concat(chunks))); + sock.on('error', reject); + }); + const expected = Buffer.concat([Buffer.from('$6\r\n', 'binary'), Buffer.from('北京', 'utf8'), Buffer.from('\r\n', 'binary')]); + assert.ok(raw.includes(expected), `expected bulk reply, got ${JSON.stringify(raw.toString('binary'))}`); + } finally { + await close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- #5 buffer codec get returns an independent copy ------------------------ + +test('buffer codec: mutating a returned value does not corrupt the store', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'buffer' }); + await db.set('k', Buffer.from('hello')); + const b = db.get('k')!; + b[0] = 0xff; + assert.equal(db.get('k')![0], 0x68, 'stored value must be unchanged'); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #6 batch unique: swaps and del+set are valid --------------------------- + +test('batch allows swapping a unique value between two keys', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byMail', { field: 'email', unique: true }); + await db.set('u1', { email: 'a@x.com' }); + await db.set('u2', { email: 'b@x.com' }); + await db.batch([ + { op: 'set', key: 'u1', value: { email: 'b@x.com' } }, + { op: 'set', key: 'u2', value: { email: 'a@x.com' } }, + ]); + assert.equal(db.get('u1')?.email, 'b@x.com'); + assert.equal(db.get('u2')?.email, 'a@x.com'); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('batch allows del(u1) + set(u2) reusing u1 unique value', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byMail', { field: 'email', unique: true }); + await db.set('u1', { email: 'x@y.com' }); + await db.batch([ + { op: 'del', key: 'u1' }, + { op: 'set', key: 'u2', value: { email: 'x@y.com' } }, + ]); + assert.equal(db.get('u1'), undefined); + assert.equal(db.get('u2')?.email, 'x@y.com'); + assert.deepEqual(db.findEq('byMail', 'x@y.com').map((r) => r.key), ['u2']); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +test('batch still rejects genuine unique violations', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byMail', { field: 'email', unique: true }); + await db.set('u1', { email: 'x@y.com' }); + await assert.rejects(db.batch([{ op: 'set', key: 'u2', value: { email: 'x@y.com' } }]), /unique/i); + assert.equal(db.get('u2'), undefined); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #7 createIndex(unique) rejects existing duplicates --------------------- + +test('createIndex(unique) rejects existing duplicate data', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.set('a', { email: 'dup@x.com' }); + await db.set('b', { email: 'dup@x.com' }); + await assert.rejects(db.createIndex('byMail', { field: 'email', unique: true }), /unique/i); + assert.equal(db.listIndexes().length, 0, 'index must not persist after failed create'); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #8 batch/mset enforce key length --------------------------------------- + +test('batch rejects keys longer than 128 chars', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + const longKey = 'k'.repeat(200); + await assert.rejects(db.batch([{ op: 'set', key: longKey, value: { v: 1 } }]), /key too long/i); + assert.equal(db.get(longKey), undefined); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #9 TTL heap does not grow unboundedly ---------------------------------- + +test('repeated TTL updates on one key do not bloat the heap', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); + for (let i = 0; i < 5000; i++) await db.set('k', 'v', { ttl: 1_000_000 }); + const heapSize = (db as unknown as { store: { heap: { size: number } } }).store.heap.size; + assert.ok(heapSize < 100, `heap should stay small, got ${heapSize}`); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #10 size excludes expired keys ----------------------------------------- + +test('size excludes expired-but-not-yet-reaped keys', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); + await db.set('e', 'v', { ttl: 1 }); + await db.set('s', 'ok'); + await new Promise((r) => setTimeout(r, 10)); + assert.equal(db.size, 1); + assert.equal(db.scan().length, 1); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #11 range index indexes array fields per element ----------------------- + +test('range index indexes array fields per element', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byScore', { field: 'scores', type: 'range' }); + await db.set('a', { scores: [10, 20, 30] }); + await db.set('b', { scores: [25] }); + const r = db.findRange('byScore', { min: 0, max: 100 }); + assert.equal(r.length, 4, `expected 4 indexed elements, got ${r.length}`); + assert.deepEqual(db.findRange('byScore', { min: 20, max: 20 }).map((x) => x.key), ['a']); + // update removes old elements + await db.set('a', { scores: [99] }); + assert.deepEqual(db.findRange('byScore', { min: 10, max: 30 }).map((x) => x.key), ['b']); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #12 dtColumns drops emptied columns ------------------------------------ + +test('dtColumns drops columns that no record has anymore', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.set('a', { v: 1 }, { dt: { created: 100 } }); + await db.del('a'); + assert.deepEqual(db.dtColumns(), []); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #13 RESP MSET sets all keys (atomic via batch) ------------------------- + +test('RESP MSET sets all keys', async () => { + const dir = await tmpDir(); + const { port, close } = await startServer({ dir, port: 0 }); + try { + const raw = await new Promise((resolve, reject) => { + const sock = net.connect(port, '127.0.0.1'); + const chunks: Buffer[] = []; + sock.on('data', (c) => chunks.push(c)); + sock.on('connect', () => { + sock.write('MSET a 1 b 2 c 3\r\n'); + setTimeout(() => sock.write('GET a\r\nGET b\r\nGET c\r\n'), 40); + setTimeout(() => sock.end(), 140); + }); + sock.on('end', () => resolve(Buffer.concat(chunks))); + sock.on('error', reject); + }); + const s = raw.toString('binary'); + assert.ok(s.includes('+OK')); + assert.ok(s.includes('$1\r\n1\r\n')); + assert.ok(s.includes('$1\r\n2\r\n')); + assert.ok(s.includes('$1\r\n3\r\n')); + } finally { + await close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- #14 openOrRebuild only rebuilds on corruption -------------------------- + +test('openOrRebuild rebuilds on corrupt index-definition JSON', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.set('k', { v: 1 }); + await db.close(); + await fs.writeFile(path.join(dir, 'db.indexes.json'), '{ not valid json', 'utf8'); + + let rebuilt = false; + const db2 = await MiniDb.openOrRebuild({ dir, valueCodec: 'json' }, { onRebuild: () => (rebuilt = true) }); + assert.ok(rebuilt, 'onRebuild should be called'); + assert.equal(db2.get('k'), undefined, 'rebuilt db is empty'); + await db2.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +// --- #15 open failure releases resources (lock usable again) ---------------- + +test('open failure on corrupt index JSON releases the lock', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.set('k', { v: 1 }); + await db.close(); + await fs.writeFile(path.join(dir, 'db.indexes.json'), '{ not valid json', 'utf8'); + + await assert.rejects(MiniDb.open({ dir, valueCodec: 'json' }), /JSON|Unexpected/i); + // Lock must be released so a subsequent open can proceed. Remove the corrupt + // definition file so recovery can continue from the snapshot/WAL. + await fs.rm(path.join(dir, 'db.indexes.json'), { force: true }); + const db2 = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.equal(db2.get('k')?.v, 1, 'data intact after failed open'); + await db2.close(); + await fs.rm(dir, { recursive: true, force: true }); +}); diff --git a/packages/minidb/test/review-round3.test.ts b/packages/minidb/test/review-round3.test.ts new file mode 100644 index 000000000..5d95e200c --- /dev/null +++ b/packages/minidb/test/review-round3.test.ts @@ -0,0 +1,241 @@ +// Regression tests for the third deep-review round: +// A) Non-ASCII (multi-byte UTF-8) string keys must work on every surface. +// B) readOnly open must never mutate the database files (no WAL truncation). +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-r3-')); +} + +// --- A) Non-ASCII string keys ---------------------------------------------- + +test('non-ASCII key: set/get/has works (live, no restart)', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + await db.set('é', 'accent'); + await db.set('北京', 'cjk'); + await db.set('key🎉', 'emoji'); + assert.equal(db.get('é'), 'accent'); + assert.equal(db.get('北京'), 'cjk'); + assert.equal(db.get('key🎉'), 'emoji'); + assert.equal(db.has('é'), true); + assert.equal(db.has('missing-é'), false); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key: del removes it and reports existence', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + await db.set('café', 'x'); + assert.equal(await db.del('café'), true, 'del must report the key existed'); + assert.equal(db.get('café'), undefined); + assert.equal(db.has('café'), false); + assert.equal(await db.del('café'), false, 'second del reports missing'); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key survives close + reopen (WAL recovery)', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'string' }); + await db.set('北京', 'v1'); + await db.set('上海', 'v2'); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + assert.equal(db.get('北京'), 'v1'); + assert.equal(db.get('上海'), 'v2'); + assert.equal(db.has('北京'), true); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key survives compaction (snapshot recovery)', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ + dir, + valueCodec: 'string', + compactThresholdBytes: 1, + autoCompact: false, + }); + await db.set('城市', 'Beijing'); + await db.set('要删除', 'gone'); + await db.del('要删除'); + await db.compact(); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + assert.equal(db.get('城市'), 'Beijing'); + assert.equal(db.get('要删除'), undefined, 'tombstone must survive compaction'); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key: scan returns the original key and value', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + await db.set('b-ascii', '1'); + await db.set('a-é', '2'); + await db.set('c-北京', '3'); + const keys = db.scan().map((r) => r.key); + assert.ok(keys.includes('a-é'), 'scan must include the accented key'); + assert.ok(keys.includes('c-北京'), 'scan must include the CJK key'); + assert.equal(db.get('a-é'), '2'); + assert.equal(db.get('c-北京'), '3'); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key: prefix scan matches', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + await db.set('用户:1', 'a'); + await db.set('用户:2', 'b'); + await db.set('other:1', 'c'); + const keys = db.prefix('用户:').map((r) => r.key).sort(); + assert.deepEqual(keys, ['用户:1', '用户:2']); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key: secondary equality index returns key and value', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byCity', { field: 'city' }); + try { + await db.set('用户1', { city: 'Paris', n: 1 }); + await db.set('用户2', { city: 'Paris', n: 2 }); + const r = db.findEq('byCity', 'Paris').sort((a, b) => (a.key < b.key ? -1 : 1)); + assert.deepEqual(r.map((x) => x.key), ['用户1', '用户2']); + assert.deepEqual(r.map((x) => (x.value as { n: number }).n), [1, 2]); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key: secondary index is consistent after recovery', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byCity', { field: 'city' }); + await db.set('用户1', { city: 'Paris' }); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'json' }); + try { + const r = db.findEq('byCity', 'Paris'); + assert.equal(r.length, 1); + assert.equal(r[0]!.key, '用户1'); + assert.deepEqual(r[0]!.value, { city: 'Paris' }); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key: range index, dt index, compound index, text index', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byAge', { field: 'age', type: 'range' }); + await db.createCompoundIndex('byWs', { groupBy: 'ws', orderBy: 'created' }); + await db.createTextIndex('body', { fields: ['bio'] }); + try { + await db.set('用户A', { age: 30, ws: 'W1', bio: 'hello world' }, { dt: { created: 100 } }); + await db.set('用户B', { age: 40, ws: 'W1', bio: 'hello there' }, { dt: { created: 200 } }); + + assert.deepEqual(db.findRange('byAge', { min: 25, max: 35 }).map((r) => r.key), ['用户A']); + assert.deepEqual(db.dtRange('created', { gte: 100, lte: 100 }).map((r) => r.key), ['用户A']); + assert.deepEqual(db.compoundRange('byWs', 'W1').map((r) => r.key), ['用户A', '用户B']); + assert.deepEqual(db.search('body', 'world').map((r) => r.key), ['用户A']); + // values resolve too + assert.equal(db.findRange('byAge', { min: 25, max: 35 })[0]!.value?.age, 30); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key: unified query by exact key and by prefix', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + try { + await db.set('post:北京', { tag: 'a' }); + await db.set('post:上海', { tag: 'b' }); + assert.deepEqual(db.query({ key: 'post:北京' }).map((r) => r.key), ['post:北京']); + const pref = db.query({ key: { prefix: 'post:' } }).map((r) => r.key).sort(); + assert.deepEqual(pref, ['post:上海', 'post:北京']); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('non-ASCII key: batch set + del survives recovery', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'string' }); + await db.batch([ + { op: 'set', key: '批1', value: 'one' }, + { op: 'set', key: '批2', value: 'two' }, + ]); + await db.batch([{ op: 'del', key: '批1' }]); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + assert.equal(db.get('批1'), undefined); + assert.equal(db.get('批2'), 'two'); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- B) readOnly must not mutate the database ------------------------------ + +test('readOnly open does not truncate a torn WAL tail', async () => { + const dir = await tmpDir(); + const walPath = path.join(dir, 'db.wal'); + + // Build a db with one valid record, then append a torn (partial) frame. + let db = await MiniDb.open({ dir, valueCodec: 'string' }); + await db.set('a', '1'); + await db.close(); + + const torn = Buffer.from([0x4d, 0x44, 0x01, 0x00, 0x01, 0x00]); // magic + partial header + await fs.appendFile(walPath, torn); + const sizeBefore = (await fs.stat(walPath)).size; + + // Opening read-only must NOT mutate the file, even though the tail is torn. + db = await MiniDb.open({ dir, valueCodec: 'string', readOnly: true }); + try { + assert.equal(db.get('a'), '1', 'valid record is still readable'); + } finally { + await db.close(); + } + const sizeAfter = (await fs.stat(walPath)).size; + assert.equal(sizeAfter, sizeBefore, 'readOnly open must not truncate the WAL'); +}); diff --git a/packages/minidb/test/review-round4.test.ts b/packages/minidb/test/review-round4.test.ts new file mode 100644 index 000000000..fd235e647 --- /dev/null +++ b/packages/minidb/test/review-round4.test.ts @@ -0,0 +1,152 @@ +// Regression tests for the fourth deep-review round: +// A) Text AND search must be empty when any query term is absent. +// B) Query $regex must not be fooled by stateful (global/sticky) RegExp. +// C) Range index must not duplicate a key for repeated array elements. +// D) Equality index must match objects regardless of property order. +// E) Index query methods must exclude expired keys (no ghost entries). +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-r4-')); +} + +// --- A) Text AND search: every term must be present ------------------------- + +test('text AND search is empty when a query term is absent', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createTextIndex('bio', { fields: ['bio'] }); + try { + await db.set('a', { bio: 'hello world' }); + await db.set('b', { bio: 'hello there' }); + // 'zzznope' is in no document, so a true AND must yield nothing. + assert.deepEqual(db.search('bio', 'hello zzznope', { op: 'AND' }).map((r) => r.key), []); + assert.deepEqual(db.search('bio', 'zzznope hello', { op: 'AND' }).map((r) => r.key), []); + // Default operator is AND. + assert.deepEqual(db.search('bio', 'hello zzznope').map((r) => r.key), []); + // Sanity: when every term is present the intersection is correct. + assert.deepEqual(db.search('bio', 'hello world', { op: 'AND' }).map((r) => r.key), ['a']); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- B) Query $regex: stateful RegExp must match every document -------------- + +test('query $regex with a global RegExp matches every document', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + try { + for (const k of ['a', 'b', 'c', 'd']) await db.set(k, { name: 'ab' }); + // /a/g advances lastIndex on every .test(); without resetting it would + // alternate match/miss across documents and return only ~2 of 4. + assert.equal(db.query({ filter: { name: { $regex: /a/g } } }).length, 4); + // Same bug via the top-level RegExp shorthand. + assert.equal(db.query({ filter: { name: /a/g } }).length, 4); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- C) Range index: repeated array elements index once ---------------------- + +test('range index dedupes repeated array elements', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byScore', { field: 'scores', type: 'range' }); + try { + await db.set('a', { scores: [10, 10, 10] }); + assert.deepEqual(db.findRange('byScore', { min: 10, max: 10 }).map((r) => r.key), ['a']); + assert.equal(db.findRange('byScore', { min: 0, max: 100 }).length, 1); + // Updating the array must not leave stale duplicate nodes behind. + await db.set('a', { scores: [20, 20] }); + assert.deepEqual(db.findRange('byScore', { min: 10, max: 10 }).map((r) => r.key), []); + assert.deepEqual(db.findRange('byScore', { min: 20, max: 20 }).map((r) => r.key), ['a']); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- D) Equality index: object keys ignore property order -------------------- + +test('equality index matches objects regardless of key order', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byMeta', { field: 'meta' }); + try { + await db.set('k1', { meta: { a: 1, b: 2 } }); + assert.deepEqual(db.findEq('byMeta', { b: 2, a: 1 }).map((r) => r.key), ['k1']); + // Nested objects must also be canonicalized. + await db.set('k2', { meta: { nested: { x: 1, y: 2 } } }); + assert.deepEqual(db.findEq('byMeta', { nested: { y: 2, x: 1 } }).map((r) => r.key), ['k2']); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- E) Index query methods must not return expired (ghost) entries ---------- + +test('findEq excludes expired keys', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', activeExpireIntervalMs: 0 }); + await db.createIndex('byCity', { field: 'city' }); + try { + await db.set('g', { city: 'Paris' }, { ttl: 1 }); + await new Promise((r) => setTimeout(r, 10)); + assert.deepEqual(db.findEq('byCity', 'Paris'), []); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('findRange excludes expired keys', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', activeExpireIntervalMs: 0 }); + await db.createIndex('byAge', { field: 'age', type: 'range' }); + try { + await db.set('g', { age: 30 }, { ttl: 1 }); + await new Promise((r) => setTimeout(r, 10)); + assert.deepEqual(db.findRange('byAge', { min: 0, max: 100 }), []); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('search excludes expired keys', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', activeExpireIntervalMs: 0 }); + await db.createTextIndex('body', { fields: ['bio'] }); + try { + await db.set('g', { bio: 'hello world' }, { ttl: 1 }); + await new Promise((r) => setTimeout(r, 10)); + assert.deepEqual(db.search('body', 'hello'), []); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('compoundRange excludes expired keys', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', activeExpireIntervalMs: 0 }); + await db.createCompoundIndex('byWs', { groupBy: 'ws', orderBy: 'ts' }); + try { + await db.set('g', { ws: 'W1' }, { ttl: 1, dt: { ts: 100 } }); + await new Promise((r) => setTimeout(r, 10)); + assert.deepEqual(db.compoundRange('byWs', 'W1'), []); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/review-round5.test.ts b/packages/minidb/test/review-round5.test.ts new file mode 100644 index 000000000..34da36d73 --- /dev/null +++ b/packages/minidb/test/review-round5.test.ts @@ -0,0 +1,123 @@ +// Regression tests for the fifth deep-review round: +// A) Recovery must not let an expired overwrite resurrect an older value. +// B) Observing an expired key via scan/query/dtRange must reap it from the +// derived indexes (dt / secondary), so they cannot leak ghost entries. +// C) A non-integer / non-finite TTL must not explode with a BigInt error. +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-r5-')); +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +// --- A) Recovery must not resurrect an older value ------------------------- + +test('recovery: expired overwrite does not resurrect older value (WAL only)', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); + await db.set('k', 'v1'); // no ttl + await db.set('k', 'v2', { ttl: 1 }); // overwrites with a ttl that will expire + await sleep(20); // v2 expires while open (not reaped: activeExpire off, no get) + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); + try { + assert.equal(db.get('k'), undefined, 'the expired v2 overwrite must not resurrect v1'); + assert.equal(db.size, 0, 'no live key should remain'); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('recovery: expired overwrite does not resurrect a snapshotted value', async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ + dir, + valueCodec: 'string', + activeExpireIntervalMs: 0, + compactThresholdBytes: 1, + autoCompact: false, + }); + await db.set('k', 'v1'); + await db.compact(); // snapshot now holds k=v1; WAL is truncated + await db.set('k', 'v2', { ttl: 1 }); // lives only in the post-compaction WAL + await sleep(20); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); + try { + assert.equal(db.get('k'), undefined, 'expired WAL overwrite must not resurrect the snapshot value'); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- B) Derived indexes must not leak expired keys ------------------------- + +test('expired key is reaped from the dt index when observed via scan', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', activeExpireIntervalMs: 0 }); + try { + await db.set('g', { v: 1 }, { ttl: 1, dt: { created: 100 } }); + await sleep(20); + db.scan(); // non-get read path: must reap the expired key + derived indexes + assert.deepEqual(db.dt.columns(), [], 'dt index must drop a column whose only key expired'); + assert.deepEqual(db.dtRange('created', { gte: 0, lte: 1000 }), []); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('expired key is reaped from a secondary index when observed via scan', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', activeExpireIntervalMs: 0 }); + await db.createIndex('byCity', { field: 'city' }); + try { + await db.set('g', { city: 'Paris' }, { ttl: 1 }); + await sleep(20); + db.scan(); // reap via the getRecord read path + // Inspect the raw index manager (no store.get, so no self-healing here): + assert.deepEqual(db.indexes.findEq('byCity', 'Paris'), [], 'secondary index must not retain the expired key'); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// --- C) TTL validation ------------------------------------------------------ + +test('set with a fractional ttl does not throw (rounded to integer ms)', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); + try { + await db.set('k', 'v', { ttl: 200.9 }); // floor to 200ms; must not throw a BigInt error + const left = db.ttl('k'); + assert.ok(left > 0 && left <= 201, `ttl should be ~200ms, got ${left}`); + assert.equal(db.get('k'), 'v'); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('set with a non-finite ttl is rejected with a clear error and stores nothing', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string' }); + try { + await assert.rejects(db.set('a', 'v', { ttl: Infinity }), /ttl/i); + await assert.rejects(db.set('b', 'v', { ttl: NaN }), /ttl/i); + assert.equal(db.get('a'), undefined, 'rejected set must not be stored'); + assert.equal(db.get('b'), undefined); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/server-extra.test.ts b/packages/minidb/test/server-extra.test.ts new file mode 100644 index 000000000..d391723d9 --- /dev/null +++ b/packages/minidb/test/server-extra.test.ts @@ -0,0 +1,140 @@ +// Covers the RESP commands and parser paths not exercised by server.test.ts. +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import net from 'node:net'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { startServer } from '../src/server.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-srv2-')); +} + +function encode(...args: string[]) { + let s = `*${args.length}\r\n`; + for (const a of args) { + const b = Buffer.from(a); + s += `$${b.length}\r\n${a}\r\n`; + } + return s; +} + +function connect(port: number): Promise { + return new Promise((resolve, reject) => { + const sock = net.connect(port, '127.0.0.1'); + sock.once('connect', () => resolve(sock)); + sock.once('error', reject); + }); +} + +function send(sock: net.Socket, cmd: string | Buffer): Promise { + return new Promise((resolve) => { + sock.once('data', (d) => resolve(d.toString())); + sock.write(cmd); + }); +} + +// Send a command and resolve when the server closes the connection (QUIT). +function sendUntilClose(sock: net.Socket, cmd: string | Buffer): Promise { + return new Promise((resolve) => { + sock.once('close', () => resolve()); + sock.write(cmd); + }); +} + +test('RESP: ECHO and PING with argument', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + assert.equal(await send(sock, encode('ECHO', 'hello')), '$5\r\nhello\r\n'); + assert.equal(await send(sock, encode('PING', 'hi')), '$2\r\nhi\r\n'); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('RESP: EXISTS / MSET / MGET / TTL', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + assert.equal(await send(sock, encode('MSET', 'a', '1', 'b', '2')), '+OK\r\n'); + assert.equal(await send(sock, encode('EXISTS', 'a')), ':1\r\n'); + assert.equal(await send(sock, encode('EXISTS', 'z')), ':0\r\n'); + assert.equal(await send(sock, encode('MGET', 'a', 'b')), '*2\r\n$1\r\n1\r\n$1\r\n2\r\n'); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('RESP: SET with EX / PX sets a TTL', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + assert.equal(await send(sock, encode('SET', 'ex', 'v', 'EX', '10')), '+OK\r\n'); + const ex = Number((await send(sock, encode('TTL', 'ex'))).slice(1)); + assert.ok(ex > 0 && ex <= 10, `EX ttl=${ex}`); + + assert.equal(await send(sock, encode('SET', 'px', 'v', 'PX', '5000')), '+OK\r\n'); + const px = Number((await send(sock, encode('TTL', 'px'))).slice(1)); + assert.ok(px > 0 && px <= 5, `PX ttl=${px}`); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('RESP: INFO and COMPACT', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + await send(sock, encode('SET', 'k', 'v')); + const info = await send(sock, encode('INFO')); + assert.ok(info.includes('minidb_version:0.0.1'), info); + assert.ok(info.includes('keys:1'), info); + assert.equal(await send(sock, encode('COMPACT')), '+OK\r\n'); + const info2 = await send(sock, encode('INFO')); + assert.ok(info2.includes('compactions:1'), info2); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('RESP: QUIT closes the connection', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + await sendUntilClose(sock, encode('QUIT')); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('RESP: inline (non-array) command path', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + // Redis inline protocol: a bare line of space-separated tokens. + assert.equal(await send(sock, 'PING\r\n'), '+PONG\r\n'); + assert.equal(await send(sock, 'SET foo bar\r\n'), '+OK\r\n'); + assert.equal(await send(sock, 'GET foo\r\n'), '$3\r\nbar\r\n'); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/server.test.ts b/packages/minidb/test/server.test.ts new file mode 100644 index 000000000..ef922b963 --- /dev/null +++ b/packages/minidb/test/server.test.ts @@ -0,0 +1,85 @@ +// test/server.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import net from 'node:net'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { startServer } from '../src/server.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-srv-')); +} + +function encode(...args) { + let s = `*${args.length}\r\n`; + for (const a of args) { + const b = Buffer.from(a); + s += `$${b.length}\r\n${a}\r\n`; + } + return s; +} + +function connect(port) { + return new Promise((resolve, reject) => { + const sock = net.connect(port, '127.0.0.1'); + sock.once('connect', () => resolve(sock)); + sock.once('error', reject); + }); +} + +// Send one command and resolve with the next response chunk. +function send(sock, cmd) { + return new Promise((resolve) => { + sock.once('data', (d) => resolve(d.toString())); + sock.write(cmd); + }); +} + +test('RESP server: PING / SET / GET', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + assert.equal(await send(sock, encode('PING')), '+PONG\r\n'); + assert.equal(await send(sock, encode('SET', 'foo', 'bar')), '+OK\r\n'); + assert.equal(await send(sock, encode('GET', 'foo')), '$3\r\nbar\r\n'); + assert.equal(await send(sock, encode('GET', 'missing')), '$-1\r\n'); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('RESP server: MGET / DEL / DBSIZE', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + await send(sock, encode('SET', 'a', '1')); + await send(sock, encode('SET', 'b', '2')); + assert.equal(await send(sock, encode('MGET', 'a', 'b', 'z')), '*3\r\n$1\r\n1\r\n$1\r\n2\r\n$-1\r\n'); + assert.equal(await send(sock, encode('DBSIZE')), ':2\r\n'); + assert.equal(await send(sock, encode('DEL', 'a')), ':1\r\n'); + assert.equal(await send(sock, encode('DBSIZE')), ':1\r\n'); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('RESP server: unknown command returns an error', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + const r = await send(sock, encode('NOPE')); + assert.ok(r.startsWith('-ERR')); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/skiplist.test.ts b/packages/minidb/test/skiplist.test.ts new file mode 100644 index 000000000..8dffc355c --- /dev/null +++ b/packages/minidb/test/skiplist.test.ts @@ -0,0 +1,98 @@ +// test/skiplist.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { SkipList, cmpNumber, cmpString } from '../src/skiplist.js'; + +test('numeric: insert keeps order by (key, val)', () => { + const sl = new SkipList(); // numeric key, string val tie-break + sl.insert(3, 'c'); + sl.insert(1, 'a'); + sl.insert(2, 'b'); + sl.insert(2, 'a'); + assert.deepEqual( + sl.toArray().map((n) => `${n.key}:${n.val}`), + ['1:a', '2:a', '2:b', '3:c'], + ); + assert.equal(sl.length, 4); +}); + +test('getRank is 0-based and correct', () => { + const sl = new SkipList(); + ['a', 'b', 'c', 'd', 'e'].forEach((m, i) => sl.insert(i, m)); + assert.equal(sl.getRank(0, 'a'), 0); + assert.equal(sl.getRank(2, 'c'), 2); + assert.equal(sl.getRank(4, 'e'), 4); + assert.equal(sl.getRank(9, 'z'), null); +}); + +test('getByRank returns the node at rank', () => { + const sl = new SkipList(); + for (let i = 0; i < 100; i++) sl.insert(i, `m${i}`); + assert.deepEqual(sl.getByRank(0), { key: 0, val: 'm0' }); + assert.deepEqual(sl.getByRank(50), { key: 50, val: 'm50' }); + assert.deepEqual(sl.getByRank(99), { key: 99, val: 'm99' }); + assert.equal(sl.getByRank(100), null); +}); + +test('range with gte/lte/gt/lt/offset/count', () => { + const sl = new SkipList(); + for (let i = 1; i <= 10; i++) sl.insert(i, `v${i}`); + assert.deepEqual(sl.range({ gte: 3, lte: 7 }).map((n) => n.key), [3, 4, 5, 6, 7]); + assert.deepEqual(sl.range({ gt: 3, lt: 7 }).map((n) => n.key), [4, 5, 6]); + assert.deepEqual(sl.range({ gte: 1, lte: 10, offset: 2, count: 3 }).map((n) => n.key), [3, 4, 5]); +}); + +test('reverse range', () => { + const sl = new SkipList(); + for (let i = 1; i <= 10; i++) sl.insert(i, `v${i}`); + assert.deepEqual(sl.range({ gte: 3, lte: 7, reverse: true }).map((n) => n.key), [7, 6, 5, 4, 3]); +}); + +test('delete removes and keeps ranks consistent', () => { + const sl = new SkipList(); + for (let i = 0; i < 50; i++) sl.insert(i, `m${i}`); + assert.ok(sl.delete(25, 'm25')); + assert.ok(!sl.delete(25, 'm25')); + assert.equal(sl.length, 49); + assert.equal(sl.getRank(26, 'm26'), 25); + assert.deepEqual(sl.getByRank(25), { key: 26, val: 'm26' }); +}); + +test('string comparator orders keys and supports prefix scans', () => { + const sl = new SkipList({ compareKey: cmpString }); + ['user:3', 'user:1', 'user:2', 'order:1', 'user:10'].forEach((k) => sl.insert(k, k)); + assert.deepEqual( + sl.toArray().map((n) => n.key), + ['order:1', 'user:1', 'user:10', 'user:2', 'user:3'], + ); + // prefix scan via range [prefix, prefix+\uFFFF] + const pref = 'user:'; + const res = sl.range({ gte: pref, lt: pref + '\uffff' }).map((n) => n.key); + assert.deepEqual(res, ['user:1', 'user:10', 'user:2', 'user:3']); +}); + +test('matches a sorted reference under random operations', () => { + const sl = new SkipList(); + const ref = new Map(); // val -> key + for (let i = 0; i < 2000; i++) { + const val = `k${(Math.random() * 200) | 0}`; + if (Math.random() < 0.7 || ref.size === 0) { + const key = (Math.random() * 1000) | 0; + const old = ref.get(val); + if (old !== undefined) sl.delete(old, val); + ref.set(val, key); + sl.insert(key, val); + } else { + const key = ref.get(val); + if (key !== undefined) { + sl.delete(key, val); + ref.delete(val); + } + } + } + const sorted = [...ref.entries()] + .map(([val, key]) => ({ key, val })) + .sort((a, b) => (a.key - b.key) || (a.val < b.val ? -1 : a.val > b.val ? 1 : 0)); + assert.deepEqual(sl.toArray(), sorted); + assert.equal(sl.length, sorted.length); +}); diff --git a/packages/minidb/test/stats.test.ts b/packages/minidb/test/stats.test.ts new file mode 100644 index 000000000..6cb3636a0 --- /dev/null +++ b/packages/minidb/test/stats.test.ts @@ -0,0 +1,93 @@ +// Verifies the SSD-pressure instrumentation on db.stats: WAL bytes written, +// fsync count, snapshot bytes written, and compaction count. +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-stats-')); +} + +// Frame overhead is 22 (header) + 4 (crc) = 26 bytes, plus key + value. +const FRAME_OVERHEAD = 26; + +test('walBytesWritten accumulates the exact frame bytes written', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + try { + assert.equal(db.stats.walBytesWritten, 0); + let expected = 0; + for (let i = 0; i < 10; i++) { + const k = `k${i}`; + const v = `v${i}`; + await db.set(k, v); + expected += FRAME_OVERHEAD + Buffer.byteLength(k) + Buffer.byteLength(v); + } + assert.equal(db.stats.walBytesWritten, expected); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test("walFsyncs counts one fsync per flush under fsyncPolicy 'always'", async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'always' }); + try { + const N = 5; + for (let i = 0; i < N; i++) await db.set(`k${i}`, `v${i}`); + // Sequential awaited sets each flush + fsync independently. + assert.ok(db.stats.walFsyncs >= N, `expected >= ${N} fsyncs, got ${db.stats.walFsyncs}`); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test("walFsyncs stays low under fsyncPolicy 'no'", async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + try { + for (let i = 0; i < 50; i++) await db.set(`k${i}`, `v${i}`); + // 'no' never fsyncs on the write path (only on close). + assert.equal(db.stats.walFsyncs, 0); + } finally { + await db.close(); // close fsyncs once, counted after our assertion above + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('snapshotBytesWritten and compactions update on compact', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + try { + assert.equal(db.stats.snapshotBytesWritten, 0); + assert.equal(db.stats.compactions, 0); + for (let i = 0; i < 100; i++) await db.set(`k${i}`, `value-${i}`); + await db.compact(); + assert.equal(db.stats.compactions, 1); + assert.ok(db.stats.snapshotBytesWritten > 0, 'snapshot wrote some bytes'); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('batch writes a single frame (lower write amplification than per-key sets)', async () => { + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + try { + const ops = Array.from({ length: 10 }, (_, i) => ({ op: 'set' as const, key: `b${i}`, value: `v${i}` })); + await db.batch(ops); + // One TYPE_BATCH frame: 26 + (2-byte count + sum of per-op subframes). + // Just assert it is far less than 10 individual SET frames would be. + const individual = 10 * (FRAME_OVERHEAD + 2 + 2); // ~10 * 30 = 300 + assert.ok(db.stats.walBytesWritten < individual, `batch wrote ${db.stats.walBytesWritten} < ${individual}`); + } finally { + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/store.test.ts b/packages/minidb/test/store.test.ts new file mode 100644 index 000000000..706ae1ab7 --- /dev/null +++ b/packages/minidb/test/store.test.ts @@ -0,0 +1,105 @@ +// test/store.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { Store } from '../src/store.js'; + +const B = (s) => Buffer.from(s); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +test('basic set/get/del/has/size', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + s.set('a', B('1')); + s.set('b', B('2')); + assert.equal(s.get('a').toString(), '1'); + assert.equal(s.size, 2); + assert.ok(s.has('a')); + assert.ok(s.del('a')); + assert.ok(!s.has('a')); + assert.equal(s.size, 1); +}); + +test('keys are binary-safe (Buffer keys, arbitrary bytes)', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + const k = Buffer.from([0x00, 0xff, 0x10, 0x00]); + s.set(k, B('bin')); + assert.equal(s.get(Buffer.from([0x00, 0xff, 0x10, 0x00])).toString(), 'bin'); + assert.equal(s.size, 1); +}); + +test('overwrite updates value without growing size', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + s.set('x', B('1')); + s.set('x', B('2')); + assert.equal(s.size, 1); + assert.equal(s.get('x').toString(), '2'); +}); + +test('lazy expiration on get', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + s.set('soon', B('gone'), Date.now() - 1); // already expired + assert.equal(s.get('soon'), undefined); + assert.equal(s.size, 0); +}); + +test('active expiration sweep removes due keys', async () => { + const s = new Store({ activeExpireIntervalMs: 10, activeExpireMaxPerTick: 100 }); + s.set('e1', B('v'), Date.now() + 20); + s.set('e2', B('v'), Date.now() + 20); + s.set('keep', B('v')); // no ttl + assert.equal(s.map.size, 3); + await sleep(80); // several sweep ticks + assert.equal(s.map.size, 1); + assert.equal(s.get('keep').toString(), 'v'); + s.close(); +}); + +test('overwriting a TTL key clears the old expiration (stale heap entry skipped)', async () => { + const s = new Store({ activeExpireIntervalMs: 10, activeExpireMaxPerTick: 100 }); + s.set('k', B('tmp'), Date.now() + 20); // expires soon + s.set('k', B('persist')); // overwritten with NO ttl + await sleep(80); + assert.equal(s.get('k').toString(), 'persist'); + s.close(); +}); + +test('entries() yields only live entries', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + s.set('a', B('1')); + s.set('b', B('2'), Date.now() - 1); // expired + const rows = [...s.entries()]; + assert.equal(rows.length, 1); + assert.equal(rows[0].key.toString(), 'a'); +}); + +test('ordered scan over keys (range)', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + ['c', 'a', 'd', 'b'].forEach((k) => s.set(k, B(k))); + assert.deepEqual( + [...s.scan({ gte: 'b', lte: 'd' })].map((r) => r.key.toString()), + ['b', 'c', 'd'], + ); +}); + +test('prefix scan over keys', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + ['user:1', 'user:2', 'user:10', 'order:1'].forEach((k) => s.set(k, B(k))); + assert.deepEqual( + [...s.prefix('user:')].map((r) => r.key.toString()), + ['user:1', 'user:10', 'user:2'], + ); +}); + +test('stores dt per record and exposes via getRecord', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + s.set('k', B('v'), 0, { dt1: 1000, dt2: 2000 }); + const r = s.getRecord('k'); + assert.deepEqual(r.dt, { dt1: 1000, dt2: 2000 }); + assert.equal([...s.entries()][0].dt.dt1, 1000); +}); + +test('delete removes key from the ordered index', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + ['a', 'b', 'c'].forEach((k) => s.set(k, B(k))); + s.del('b'); + assert.deepEqual([...s.scan()].map((r) => r.key.toString()), ['a', 'c']); +}); diff --git a/packages/minidb/test/text-index.test.ts b/packages/minidb/test/text-index.test.ts new file mode 100644 index 000000000..a2ae5054c --- /dev/null +++ b/packages/minidb/test/text-index.test.ts @@ -0,0 +1,277 @@ +// test/text-index.test.ts +// +// Larger-than-RAM full-text index: postings codec, on-disk postings file, and +// the TextIndex (delta + tombstones + disk-backed base + cache). + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import fssync from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; +import { TextIndex } from '../src/text-index.js'; +import { + encodePostingList, + decodePostingList, + encodeRecord, + decodeRecord, + PostingsFile, +} from '../src/text-postings.js'; + +async function tmpDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-text-')); +} + +// ---- codec --------------------------------------------------------------- + +test('postings codec: roundtrip + delta compression', () => { + const entries: [number, number][] = [ + [0, 1], + [3, 2], + [10, 5], + [1000, 1], + [40000, 7], + ]; + const enc = encodePostingList(entries); + assert.deepEqual(decodePostingList(enc), entries); + // delta+varint should beat a naive 8 bytes/pair for dense-ish ids. + assert.ok(enc.length < entries.length * 8, `expected compression, got ${enc.length} bytes`); +}); + +test('postings codec: empty list', () => { + assert.deepEqual(decodePostingList(encodePostingList([])), []); +}); + +test('record frame: CRC detects corruption', () => { + const payload = encodePostingList([ + [1, 1], + [2, 3], + ]); + const rec = encodeRecord('hello', 2, payload); + const good = decodeRecord(rec); + assert.equal(good.term, 'hello'); + assert.equal(good.df, 2); + assert.deepEqual(decodePostingList(good.payload), [ + [1, 1], + [2, 3], + ]); + + const bad = Buffer.from(rec); + bad[2] ^= 0xff; // flip a byte inside the term + assert.throws(() => decodeRecord(bad), /crc mismatch/); +}); + +// ---- PostingsFile -------------------------------------------------------- + +test('PostingsFile: rebuild + positioned read', async () => { + const dir = await tmpDir(); + try { + const p = path.join(dir, 'x.postings'); + const dict = PostingsFile.rebuildSync(p, [ + { + term: 'hello', + entries: [ + [0, 1], + [5, 2], + [9, 1], + ], + }, + { + term: '北京', + entries: [ + [1, 3], + [2, 1], + ], + }, + { term: 'empty', entries: [] }, // must be skipped + ]); + assert.equal(dict.size, 2); + assert.ok(!dict.has('empty')); + + const pf = PostingsFile.open(p); + assert.deepEqual(pf.read(dict.get('hello')!), [ + [0, 1], + [5, 2], + [9, 1], + ]); + assert.deepEqual(pf.read(dict.get('北京')!), [ + [1, 3], + [2, 1], + ]); + pf.close(); + + // rebuild is atomic: a second rebuild replaces the file and dict. + const dict2 = PostingsFile.rebuildSync(p, [{ term: 'only', entries: [[7, 1]] }]); + assert.equal(dict2.size, 1); + const pf2 = PostingsFile.open(p); + assert.deepEqual(pf2.read(dict2.get('only')!), [[7, 1]]); + pf2.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('PostingsFile: corrupt record throws on read', async () => { + const dir = await tmpDir(); + try { + const p = path.join(dir, 'x.postings'); + const dict = PostingsFile.rebuildSync(p, [{ term: 'a', entries: [[1, 1]] }]); + // flip a byte in the file payload + const e = dict.get('a')!; + const fd = fssync.openSync(p, 'r+'); + fssync.writeSync(fd, Buffer.from([0xff]), 0, 1, e.off + Math.floor(e.len / 2)); + fssync.closeSync(fd); + + const pf = PostingsFile.open(p); + assert.throws(() => pf.read(e), /crc mismatch/); + pf.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +// ---- TextIndex (direct, disk-backed) ------------------------------------- + +test('TextIndex: add + search (AND/OR) disk-backed', async () => { + const dir = await tmpDir(); + try { + const ti = new TextIndex({ postingsPath: path.join(dir, 't.postings') }); + ti.add('a', { bio: 'hello world from London' }); + ti.add('b', { bio: '我住在北京，喜欢编程' }); + ti.add('c', { bio: '我在上海写代码' }); + + assert.deepEqual(ti.search('hello').map((h) => h.key), ['a']); + assert.deepEqual(ti.search('北京').map((h) => h.key), ['b']); + assert.deepEqual(ti.search('北京 上海', { op: 'OR' }).map((h) => h.key).sort(), ['b', 'c']); + // AND across two terms only present together in 'b' + assert.deepEqual(ti.search('北京 编程').map((h) => h.key), ['b']); + ti.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('TextIndex: overwrite tombstones old postings', async () => { + const dir = await tmpDir(); + try { + const ti = new TextIndex({ postingsPath: path.join(dir, 't.postings') }); + ti.add('a', { bio: 'hello world' }); + assert.deepEqual(ti.search('hello').map((h) => h.key), ['a']); + // overwrite 'a' with different text -> old 'hello' posting must be gone + ti.add('a', { bio: 'goodbye world' }); + assert.deepEqual(ti.search('hello').map((h) => h.key), []); + assert.deepEqual(ti.search('goodbye').map((h) => h.key), ['a']); + assert.equal(ti.N, 1); + ti.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('TextIndex: remove deletes postings', async () => { + const dir = await tmpDir(); + try { + const ti = new TextIndex({ postingsPath: path.join(dir, 't.postings') }); + ti.add('a', { bio: 'hello world' }); + ti.add('b', { bio: 'hello there' }); + ti.remove('a'); + assert.deepEqual(ti.search('hello').map((h) => h.key), ['b']); + assert.equal(ti.N, 1); + ti.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('TextIndex: build persists to disk + merges delta after build', async () => { + const dir = await tmpDir(); + try { + const p = path.join(dir, 't.postings'); + const ti = new TextIndex({ postingsPath: p }); + ti.build([ + { key: 'a', value: { bio: 'hello world' } }, + { key: 'b', value: { bio: '我住在北京' } }, + ]); + assert.ok(fssync.existsSync(p), 'postings file should exist after build'); + assert.deepEqual(ti.search('hello').map((h) => h.key), ['a']); + + // new writes after build go to the in-memory delta and are still found + ti.add('c', { bio: 'hello from c' }); + assert.deepEqual(ti.search('hello').map((h) => h.key).sort(), ['a', 'c']); + ti.close(); + + // a fresh TextIndex over the same file sees the base but not the lost delta + // (delta is volatile by design; the db rebuilds from the Store on open). + const ti2 = new TextIndex({ postingsPath: p }); + // rebuild base from the file's perspective by re-reading the same entries + ti2.build([ + { key: 'a', value: { bio: 'hello world' } }, + { key: 'b', value: { bio: '我住在北京' } }, + ]); + assert.deepEqual(ti2.search('hello').map((h) => h.key), ['a']); + ti2.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('TextIndex: memory-only mode (no postingsPath)', () => { + const ti = new TextIndex(); // memory base + ti.add('a', { bio: 'hello world' }); + ti.add('b', { bio: '北京天安门' }); + assert.deepEqual(ti.search('hello').map((h) => h.key), ['a']); + assert.deepEqual(ti.search('北京').map((h) => h.key), ['b']); + assert.equal(ti.termCount() > 0, true); + ti.close(); +}); + +// ---- through MiniDb ------------------------------------------------------ + +test('MiniDb: text postings written to disk, search survives reopen', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createTextIndex('bio', { fields: ['bio'] }); + await db.set('a', { bio: '我爱北京天安门' }); + await db.set('b', { bio: '今天天气不错' }); + await db.close(); + + assert.ok(fssync.existsSync(path.join(dir, 'db.text-bio.postings')), 'postings file exists'); + + db = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.deepEqual(db.search('bio', '北京').map((r) => r.key), ['a']); + // overwrite then reopen -> tombstone must not resurrect old text + await db.set('a', { bio: '我爱上海' }); + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'json' }); + assert.deepEqual(db.search('bio', '北京').map((r) => r.key), []); + assert.deepEqual(db.search('bio', '上海').map((r) => r.key), ['a']); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('MiniDb: compaction rebuilds postings (file reclaimed)', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json', autoCompact: false }); + await db.createTextIndex('bio', { fields: ['bio'] }); + for (let i = 0; i < 50; i++) await db.set('k' + i, { bio: 'hello world ' + i }); + const p = path.join(dir, 'db.text-bio.postings'); + assert.ok(fssync.existsSync(p)); + // overwrite everything to create tombstones, then add more (delta grows) + for (let i = 0; i < 50; i++) await db.set('k' + i, { bio: 'goodbye world ' + i }); + for (let i = 50; i < 80; i++) await db.set('k' + i, { bio: 'hello again ' + i }); + + await db.compact(); // should rebuild postings from the live store + + // after compaction the postings reflect the latest values only + assert.equal(db.search('bio', 'hello').length, 30); // k50..k79 + assert.equal(db.search('bio', 'goodbye').length, 50); // k0..k49 + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/wal.test.ts b/packages/minidb/test/wal.test.ts new file mode 100644 index 000000000..b4929369e --- /dev/null +++ b/packages/minidb/test/wal.test.ts @@ -0,0 +1,115 @@ +// test/wal.test.js +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { WAL } from '../src/wal.js'; +import { encodeFrame, FrameParser, CorruptFrameError, TYPE_SET, TYPE_DEL } from '../src/codec.js'; + +const B = (s) => Buffer.from(s); + +async function tmpWalPath() { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-wal-')); + return { dir, file: path.join(dir, 'db.wal') }; +} + +function parseAll(buf) { + return [...new FrameParser().feed(buf)]; +} + +test('append then read back preserves frames and order', async () => { + const { dir, file } = await tmpWalPath(); + try { + const wal = new WAL(file, { fsyncPolicy: 'everysec' }); + await wal.open(); + await wal.append(encodeFrame({ type: TYPE_SET, key: B('a'), value: B('1') })); + await wal.append(encodeFrame({ type: TYPE_SET, key: B('b'), value: B('2') })); + await wal.append(encodeFrame({ type: TYPE_DEL, key: B('a') })); + await wal.close(); + + const frames = parseAll(await fs.readFile(file)); + assert.equal(frames.length, 3); + assert.equal(frames[0].key.toString(), 'a'); + assert.equal(frames[1].key.toString(), 'b'); + assert.equal(frames[2].type, TYPE_DEL); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('group commit: many concurrent appends all land in order', async () => { + const { dir, file } = await tmpWalPath(); + try { + const wal = new WAL(file, { fsyncPolicy: 'no' }); + await wal.open(); + const N = 1000; + const ops = []; + for (let i = 0; i < N; i++) { + ops.push(wal.append(encodeFrame({ type: TYPE_SET, key: B(`k${i}`), value: B(`${i}`) }))); + } + await Promise.all(ops); + await wal.close(); + + const frames = parseAll(await fs.readFile(file)); + assert.equal(frames.length, N); + // Spot-check ordering. + assert.equal(frames[0].key.toString(), 'k0'); + assert.equal(frames[N - 1].key.toString(), `k${N - 1}`); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test("fsyncPolicy 'always' works", async () => { + const { dir, file } = await tmpWalPath(); + try { + const wal = new WAL(file, { fsyncPolicy: 'always' }); + await wal.open(); + await wal.append(encodeFrame({ type: TYPE_SET, key: B('durable'), value: B('yes') })); + await wal.close(); + const frames = parseAll(await fs.readFile(file)); + assert.equal(frames[0].key.toString(), 'durable'); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('recovery truncates a torn/corrupt tail at the error offset', async () => { + const { dir, file } = await tmpWalPath(); + try { + const wal = new WAL(file, { fsyncPolicy: 'no' }); + await wal.open(); + const N = 50; + for (let i = 0; i < N; i++) { + await wal.append(encodeFrame({ type: TYPE_SET, key: B(`key${i}`), value: B('v') })); + } + await wal.close(); + + const validSize = (await fs.stat(file)).size; + // Simulate a crash that left a half-written frame at the end. + const partial = encodeFrame({ type: TYPE_SET, key: B('torn'), value: B('x'.repeat(100)) }).subarray(0, 13); + await fs.appendFile(file, partial); + + const buf = await fs.readFile(file); + const parser = new FrameParser(); + const frames = []; + let err = null; + try { + for (const f of parser.feed(buf)) frames.push(f); + parser.finish(); + } catch (e) { + err = e; + } + assert.ok(err instanceof CorruptFrameError, 'expected a corrupt-frame error'); + assert.equal(err.offset, validSize, 'error offset should equal end of valid data'); + assert.equal(frames.length, N, 'all valid frames before the tail are recovered'); + + // Truncate at the error offset and verify the file is clean again. + await fs.truncate(file, err.offset); + const after = parseAll(await fs.readFile(file)); + assert.equal(after.length, N); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/tsconfig.json b/packages/minidb/tsconfig.json new file mode 100644 index 000000000..8e90bc90a --- /dev/null +++ b/packages/minidb/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": {}, + "include": ["src"] +} diff --git a/packages/minidb/tsdown.config.ts b/packages/minidb/tsdown.config.ts new file mode 100644 index 000000000..b27e00afd --- /dev/null +++ b/packages/minidb/tsdown.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'tsdown'; + +export default defineConfig({ + entry: ['./src/index.ts'], + format: ['esm'], + dts: false, + outDir: 'dist', + clean: true, + deps: { + alwaysBundle: [/^@moonshot-ai\//], + neverBundle: [], + }, +}); diff --git a/packages/minidb/vitest.config.ts b/packages/minidb/vitest.config.ts new file mode 100644 index 000000000..db8268b3b --- /dev/null +++ b/packages/minidb/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + name: 'minidb', + include: ['test/**/*.test.ts'], + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 71bce0d7f..3cf58ebb1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -547,6 +547,8 @@ importers: specifier: workspace:^ version: link:../kaos + packages/minidb: {} + packages/node-sdk: dependencies: '@antfu/utils':