From 37b2c46691ebea1f2e8e0928caa4db89496380fc Mon Sep 17 00:00:00 2001 From: "haozhe.yang" Date: Thu, 2 Jul 2026 01:07:19 +0800 Subject: [PATCH] perf(minidb): add dt-ordered limit fast path for queries - add SkipList.iterate and DtIndex.iterate: lazy range scans that let a caller stop early without materializing the whole range - add IndexManager.hasEq for O(1) equality-index membership checks that avoid materializing the full posting list - add MiniDb.tryDtOrderedLimit (with cheapEqChecks): for a query bounded by a single dt column whose result order is that column, walk the dt skiplist in order and stop at limit, pre-filtering cheap equality predicates before decode; wire it into query() - fix text-index tokenization overflowing the call stack on large docs by loop-pushing instead of spreading the match array - add bench/* benchmarks and tests for the fast path and skiplist iteration --- packages/minidb/bench/baseline.ts | 169 +++++++++++++++++ packages/minidb/bench/message-composed.ts | 197 +++++++++++++++++++ packages/minidb/bench/message-range.ts | 180 ++++++++++++++++++ packages/minidb/bench/search-baseline.ts | 218 ++++++++++++++++++++++ packages/minidb/bench/session-children.ts | 214 +++++++++++++++++++++ packages/minidb/src/dt-index.ts | 11 ++ packages/minidb/src/index-manager.ts | 9 + packages/minidb/src/index.ts | 92 +++++++++ packages/minidb/src/skiplist.ts | 47 +++++ packages/minidb/src/text-index.ts | 4 +- packages/minidb/test/query.test.ts | 70 +++++++ packages/minidb/test/skiplist.test.ts | 32 ++++ 12 files changed, 1242 insertions(+), 1 deletion(-) create mode 100644 packages/minidb/bench/baseline.ts create mode 100644 packages/minidb/bench/message-composed.ts create mode 100644 packages/minidb/bench/message-range.ts create mode 100644 packages/minidb/bench/search-baseline.ts create mode 100644 packages/minidb/bench/session-children.ts diff --git a/packages/minidb/bench/baseline.ts b/packages/minidb/bench/baseline.ts new file mode 100644 index 000000000..b760c6313 --- /dev/null +++ b/packages/minidb/bench/baseline.ts @@ -0,0 +1,169 @@ +// bench/baseline.ts +// +// "Without minidb" baseline for bench.ts. Reproduces the same durable KV +// operations using only a JS Map + raw node:fs, so the timings isolate what +// minidb's machinery (group commit, binary codec + CRC, in-memory store, +// snapshot format) costs over a hand-written equivalent. +// +// Conventions mirror bench.ts so the two can be compared line by line: +// - same N / NSMALL / 100B value +// - three fsync policies: no / everysec (1s timer) / always (per-write fsync) +// - concurrent set = fire all appends then Promise.all (file opened with +// O_APPEND, so each write lands atomically at EOF, like minidb's WAL fd) +// - snapshot = serialize the whole map to JSON and fsync it +// +// Run: node --import tsx bench/baseline.ts + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); +const ops = (n: number, ms: number) => `${fmt((n / ms) * 1000)} ops/s`; + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'baseline-bench-')); +} + +async function bench(label: string, fn: () => Promise | unknown) { + if (global.gc) global.gc(); + const t0 = performance.now(); + await fn(); + const ms = performance.now() - t0; + console.log(` ${label.padEnd(46)} ${ms.toFixed(1).padStart(8)} ms`); + return ms; +} + +async function main() { + const VALUE = 'x'.repeat(100); // 100-byte values + const N = Number(process.env.N || 200_000); + const NSMALL = Number(process.env.NSMALL || 3_000); + + console.log(`\nbaseline (no minidb) (N=${fmt(N)}, value=${VALUE.length}B, node ${process.version})\n`); + + // --- baseline: raw JS Map (in-memory) --------------------------------- + { + const m = new Map(); + const ms = await bench('baseline: raw Map set (in-memory)', () => { + for (let i = 0; i < N; i++) m.set(`k${i}`, VALUE); + }); + console.log(` -> ${ops(N, ms)}`); + const ms2 = await bench('baseline: raw Map get (in-memory)', () => { + let s = 0; + for (let i = 0; i < N; i++) if (m.get(`k${i}`)) s++; + return s; + }); + console.log(` -> ${ops(N, ms2)}`); + } + + // --- naive durable writes, fsync=no ----------------------------------- + // One JSONL record per set, O_APPEND, no fsync (rely on OS flush). + { + const dir = await tmpDir(); + const file = path.join(dir, 'log.jsonl'); + const m = new Map(); + const fh = await fs.open(file, 'a'); + let bytes = 0; + const ms = await bench('naive set concurrent, fsync=no (O_APPEND)', async () => { + const p: Promise[] = []; + for (let i = 0; i < N; i++) { + const k = `k${i}`; + m.set(k, VALUE); + const line = Buffer.from(JSON.stringify({ k, v: VALUE }) + '\n'); + bytes += line.length; + p.push(fh.write(line)); + } + await Promise.all(p); + }); + await fh.close(); + console.log(` -> ${ops(N, ms)} (${(bytes / 1024 / 1024).toFixed(2)} MiB written)`); + await fs.rm(dir, { recursive: true, force: true }); + } + + // --- naive durable writes, fsync=everysec ----------------------------- + { + const dir = await tmpDir(); + const file = path.join(dir, 'log.jsonl'); + const m = new Map(); + const fh = await fs.open(file, 'a'); + const timer = setInterval(() => { + fh.sync().catch(() => {}); + }, 1000); + timer.unref?.(); + let bytes = 0; + const ms = await bench('naive set concurrent, fsync=everysec', async () => { + const p: Promise[] = []; + for (let i = 0; i < N; i++) { + const k = `k${i}`; + m.set(k, VALUE); + const line = Buffer.from(JSON.stringify({ k, v: VALUE }) + '\n'); + bytes += line.length; + p.push(fh.write(line)); + } + await Promise.all(p); + }); + clearInterval(timer); + await fh.sync(); + await fh.close(); + console.log(` -> ${ops(N, ms)} (${(bytes / 1024 / 1024).toFixed(2)} MiB written)`); + await fs.rm(dir, { recursive: true, force: true }); + } + + // --- naive durable writes, sequential fsync=always -------------------- + { + const dir = await tmpDir(); + const file = path.join(dir, 'log.jsonl'); + const m = new Map(); + const fh = await fs.open(file, 'a'); + const ms = await bench(`naive set sequential, fsync=always (N=${fmt(NSMALL)})`, async () => { + for (let i = 0; i < NSMALL; i++) { + const k = `k${i}`; + m.set(k, VALUE); + await fh.write(Buffer.from(JSON.stringify({ k, v: VALUE }) + '\n')); + await fh.sync(); + } + }); + await fh.close(); + console.log(` -> ${ops(NSMALL, ms)}`); + await fs.rm(dir, { recursive: true, force: true }); + } + + // --- naive reads (in-memory Map after load) --------------------------- + { + const m = new Map(); + for (let i = 0; i < N; i++) m.set(`k${i}`, VALUE); + const ms = await bench('naive get (in-memory Map)', () => { + let s = 0; + for (let i = 0; i < N; i++) if (m.get(`k${i}`)) s++; + return s; + }); + console.log(` -> ${ops(N, ms)}`); + } + + // --- naive snapshot --------------------------------------------------- + // Dump the whole map to one JSON file + fsync (the hand-written equivalent + // of minidb's compact()). + { + const dir = await tmpDir(); + const file = path.join(dir, 'snapshot.json'); + const m = new Map(); + for (let i = 0; i < N; i++) m.set(`k${i}`, VALUE); + const ms = await bench(`naive JSON snapshot of ${fmt(N)} keys`, async () => { + const body = JSON.stringify(Object.fromEntries(m)); + await fs.writeFile(file, body); + const fh = await fs.open(file, 'r'); + await fh.sync(); + await fh.close(); + }); + const snap = await fs.stat(file); + console.log(` -> ${(snap.size / 1024 / 1024).toFixed(2)} MiB snapshot in ${ms.toFixed(0)} ms`); + await fs.rm(dir, { recursive: true, force: true }); + } + + console.log('\ndone.\n'); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/message-composed.ts b/packages/minidb/bench/message-composed.ts new file mode 100644 index 000000000..6e8efa6f6 --- /dev/null +++ b/packages/minidb/bench/message-composed.ts @@ -0,0 +1,197 @@ +// bench/message-composed.ts +// +// The exact scenario the user asked about: +// index ALL conversational messages (user prompts + assistant text), then +// answer: "user messages within a time window". +// +// predicate: role === 'user' AND ts >= since (last 7 days) +// +// Three result shapes, each tells a different story: +// A. top-50 recent (LIMIT) — index should win (key-level intersect + +// decode bounded by limit) +// B. ALL matches (no limit) — naive wins (minidb decodes every match) +// C. COUNT matches — naive wins big (minidb has no count +// path; it materializes everything then +// takes .length). Motivates a count/keys +// API. +// +// Run: node --import tsx bench/message-composed.ts + +import fs from 'node:fs/promises'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { MiniDb } from '../src/index.js'; + +const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); +const LIMIT = 50; +const WINDOW_DAYS = 30; +const SINCE_DAYS = 7; + +interface Msg { + id: string; + ts: number; + role: 'user' | 'assistant'; + text: string; +} + +function loadAllMessages(): Msg[] { + const DATA = path.join(os.homedir(), '.kimi-code'); + const lines = readFileSync(path.join(DATA, 'session_index.jsonl'), 'utf8').trim().split('\n'); + const out: Msg[] = []; + for (const line of lines) { + let meta: any; + try { + meta = JSON.parse(line); + } catch { + continue; + } + const wire = path.join(meta.sessionDir, 'agents', 'main', 'wire.jsonl'); + if (!existsSync(wire)) continue; + let raw: string; + try { + raw = readFileSync(wire, 'utf8'); + } catch { + continue; + } + for (const ln of raw.split('\n')) { + if (!ln) continue; + let o: any; + try { + o = JSON.parse(ln); + } catch { + continue; + } + const ts = typeof o.time === 'number' ? o.time : 0; + if (o.type === 'context.append_message' && o.message) { + let text = ''; + for (const c of o.message.content || []) + if (c && c.type === 'text' && typeof c.text === 'string') text += c.text; + out.push({ id: `${meta.sessionId}:u${out.length}`, ts, role: 'user', text }); + } else if ( + o.type === 'context.append_loop_event' && + o.event && + o.event.type === 'content.part' && + o.event.part && + o.event.part.type === 'text' && + typeof o.event.part.text === 'string' + ) { + out.push({ id: `${meta.sessionId}:a${out.length}`, ts, role: 'assistant', text: o.event.part.text }); + } + } + } + return out; +} + +function scaleTo(real: Msg[], n: number): Msg[] { + const now = Date.now(); + const span = WINDOW_DAYS * 864e5; + const out: Msg[] = new Array(n); + for (let i = 0; i < n; i++) { + const r = real[i % real.length]!; + out[i] = { id: `m${i}`, ts: now - Math.floor((i / n) * span), role: r.role, text: r.text }; + } + return out; +} + +async function buildMinidb(msgs: Msg[]) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-mc-')); + const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + await db.createIndex('byRole', { field: 'role' }); + const t0 = performance.now(); + const CHUNK = 5000; + for (let s = 0; s < msgs.length; s += CHUNK) { + const p: Promise[] = []; + for (let i = s; i < Math.min(s + CHUNK, msgs.length); i++) { + const m = msgs[i]!; + p.push(db.set(m.id, { role: m.role, text: m.text }, { dt: { ts: m.ts } })); + } + await Promise.all(p); + } + return { db, buildMs: performance.now() - t0, dir }; +} + +function med(fn: () => void, runs = 9): number { + const t: number[] = []; + for (let i = 0; i < runs; i++) { + const t0 = performance.now(); + fn(); + t.push(performance.now() - t0); + } + t.sort((a, b) => a - b); + return t[(runs / 2) | 0]!; +} + +async function runCase(label: string, msgs: Msg[]) { + const since = Date.now() - SINCE_DAYS * 864e5; + const userTotal = msgs.filter((m) => m.role === 'user').length; + const { db, buildMs, dir } = await buildMinidb(msgs); + + // A. top-50 recent user in window + let aHits = 0; + const aDb = med(() => { + aHits = db.query({ + dt: { ts: { gte: since } }, + filter: { role: 'user' }, + sort: { ts: -1 }, + limit: LIMIT, + }).length; + }); + const aNaive = med(() => { + aHits = msgs.filter((m) => m.role === 'user' && m.ts >= since).sort((a, b) => b.ts - a.ts).slice(0, LIMIT).length; + }); + + // B. ALL user in window (no limit) + let bDbHits = 0; + const bDb = med(() => { + bDbHits = db.query({ dt: { ts: { gte: since } }, filter: { role: 'user' } }).length; + }); + let bNaiveHits = 0; + const bNaive = med(() => { + bNaiveHits = msgs.filter((m) => m.role === 'user' && m.ts >= since).length; + }); + + // C. COUNT user in window (minidb has no count path -> materialize then .length) + const cDb = med(() => { + db.query({ dt: { ts: { gte: since } }, filter: { role: 'user' } }); + }); + let cNaive = 0; + const cNaiveMs = med(() => { + let c = 0; + for (const m of msgs) if (m.role === 'user' && m.ts >= since) c++; + cNaive = c; + }); + + if (global.gc) global.gc(); + const heap = process.memoryUsage().heapUsed; + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + + console.log(`\n--- ${label}: N=${fmt(msgs.length)} msgs (user=${fmt(userTotal)}) ---`); + console.log(` build: minidb ${(buildMs / 1000).toFixed(2)}s | heap ${(heap / 1024 / 1024).toFixed(0)} MiB`); + console.log( + ` A top-${LIMIT}: minidb ${aDb.toFixed(3)} ms vs naive ${aNaive.toFixed(3)} ms -> ${(aNaive / aDb).toFixed(1)}x (hits ${aHits})`, + ); + console.log( + ` B ALL: minidb ${bDb.toFixed(3)} ms vs naive ${bNaive.toFixed(3)} ms -> ${(bNaive / bDb).toFixed(2)}x (hits ${fmt(bDbHits)}/${fmt(bNaiveHits)})`, + ); + console.log( + ` C COUNT: minidb ${cDb.toFixed(3)} ms vs naive ${cNaiveMs.toFixed(3)} ms -> ${(cNaiveMs / cDb).toFixed(2)}x (count ${fmt(cNaive)})`, + ); +} + +async function main() { + const real = loadAllMessages(); + const byRole = real.reduce((a: any, m) => ((a[m.role] = (a[m.role] || 0) + 1), a), {}); + console.log(`loaded ${fmt(real.length)} real messages: ${JSON.stringify(byRole)}`); + for (const n of [real.length, 100_000, 1_000_000]) { + const msgs = n === real.length ? real : scaleTo(real, n); + await runCase(n === real.length ? 'real data' : 'scaled', msgs); + } + console.log('\ndone.'); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/message-range.ts b/packages/minidb/bench/message-range.ts new file mode 100644 index 000000000..8903f7ded --- /dev/null +++ b/packages/minidb/bench/message-range.ts @@ -0,0 +1,180 @@ +// bench/message-range.ts +// +// Benchmarks the "show me the user's most recent messages" query: +// predicate: role === 'user' AND ts >= since (last 7 days) +// order: ts DESC +// limit: 50 (the typical "recent N" UI pattern) +// +// This is a message-granularity range query with a SMALL LIMIT, which is a +// different shape from the session-level benches: +// - import-kimi-code / search-baseline index whole sessions (one doc/session). +// - the metadata lookups (workspace lookup, dt range over sessions) return +// ALL matches, so at small N a linear filter beats the index. +// Here, an ordered dt index can walk just `limit` entries (O(log N + limit)) +// instead of scanning + sorting the whole set (O(N log M)). We measure at the +// real scale (~7.2k user messages) and at scaled-up sizes to find the +// crossover where the index starts winning. +// +// Run: node --import tsx bench/message-range.ts + +import fs from 'node:fs/promises'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import { MiniDb } from '../src/index.js'; + +const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); +const LIMIT = 50; +const WINDOW_DAYS = 30; // replicated timestamps are spread over the last 30 days +const SINCE_DAYS = 7; // "recent" = last 7 days + +interface Msg { + id: string; + ts: number; + role: string; + text: string; +} + +function loadRealMessages(): Msg[] { + const DATA = path.join(os.homedir(), '.kimi-code'); + const lines = readFileSync(path.join(DATA, 'session_index.jsonl'), 'utf8').trim().split('\n'); + const out: Msg[] = []; + for (const line of lines) { + let meta: any; + try { + meta = JSON.parse(line); + } catch { + continue; + } + const wire = path.join(meta.sessionDir, 'agents', 'main', 'wire.jsonl'); + if (!existsSync(wire)) continue; + let raw: string; + try { + raw = readFileSync(wire, 'utf8'); + } catch { + continue; + } + for (const ln of raw.split('\n')) { + if (!ln) continue; + let o: any; + try { + o = JSON.parse(ln); + } catch { + continue; + } + if (o.type !== 'context.append_message' || !o.message) continue; + const m = o.message; + const ts = typeof o.time === 'number' ? o.time : 0; + let text = ''; + for (const c of m.content || []) + if (c && c.type === 'text' && typeof c.text === 'string') text += c.text; + out.push({ id: `${meta.sessionId}:${out.length}`, ts, role: m.role || 'user', text }); + } + } + return out; +} + +// Replicate the real messages to N, spreading timestamps uniformly over the +// last WINDOW_DAYS so the "last SINCE_DAYS" match fraction stays ~constant. +function scaleTo(real: Msg[], n: number): Msg[] { + const now = Date.now(); + const span = WINDOW_DAYS * 864e5; + const out: Msg[] = new Array(n); + for (let i = 0; i < n; i++) { + const r = real[i % real.length]!; + out[i] = { + id: `m${i}`, + ts: now - Math.floor((i / n) * span), + role: r.role, + text: r.text, + }; + } + return out; +} + +async function buildMinidb(msgs: Msg[]): Promise<{ db: any; buildMs: number; dir: string }> { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-msg-')); + const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + const t0 = performance.now(); + // concurrent insert in chunks to avoid one giant promise array + const CHUNK = 5000; + for (let s = 0; s < msgs.length; s += CHUNK) { + const p: Promise[] = []; + for (let i = s; i < Math.min(s + CHUNK, msgs.length); i++) { + const m = msgs[i]!; + p.push(db.set(m.id, { role: m.role, text: m.text }, { dt: { ts: m.ts } })); + } + await Promise.all(p); + } + return { db, buildMs: performance.now() - t0, dir }; +} + +function med(fn: () => void, runs = 9): number { + const t: number[] = []; + for (let i = 0; i < runs; i++) { + const t0 = performance.now(); + fn(); + t.push(performance.now() - t0); + } + t.sort((a, b) => a - b); + return t[(runs / 2) | 0]!; +} + +async function runCase(label: string, msgs: Msg[]) { + const since = Date.now() - SINCE_DAYS * 864e5; + + // ---- minidb dt index ---- + const { db, buildMs, dir } = await buildMinidb(msgs); + let indexHits = 0; + const indexMs = med(() => { + const res = db.dtRange('ts', { gte: since, reverse: true, limit: LIMIT }); + indexHits = res.length; + }); + // variant: return ALL matches (no limit), to mirror the earlier metadata case + let allHits = 0; + const indexAllMs = med(() => { + const res = db.dtRange('ts', { gte: since }); + allHits = res.length; + }); + if (global.gc) global.gc(); + const heap = process.memoryUsage().heapUsed; + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + + // ---- naive baseline ---- + let naiveHits = 0; + const naiveMs = med(() => { + const res = msgs.filter((m) => m.ts >= since).sort((a, b) => b.ts - a.ts).slice(0, LIMIT); + naiveHits = res.length; + }); + let naiveAllHits = 0; + const naiveAllMs = med(() => { + const res = msgs.filter((m) => m.ts >= since); + naiveAllHits = res.length; + }); + + console.log(`\n--- ${label}: N=${fmt(msgs.length)} messages ---`); + console.log(` build: minidb ${(buildMs / 1000).toFixed(2)}s | heap ${(heap / 1024 / 1024).toFixed(0)} MiB`); + console.log( + ` top-${LIMIT} recent: minidb dtRange ${indexMs.toFixed(3)} ms (${indexHits}) vs naive filter+sort ${naiveMs.toFixed(3)} ms (${naiveHits}) -> ${(naiveMs / indexMs).toFixed(1)}x`, + ); + console.log( + ` ALL matches: minidb dtRange ${indexAllMs.toFixed(3)} ms (${fmt(allHits)}) vs naive filter ${naiveAllMs.toFixed(3)} ms (${fmt(naiveAllHits)}) -> ${(naiveAllMs / indexAllMs).toFixed(1)}x`, + ); +} + +async function main() { + const real = loadRealMessages(); + console.log(`loaded ${fmt(real.length)} real user messages`); + const sizes = [real.length, 100_000, 1_000_000]; + for (const n of sizes) { + const msgs = n === real.length ? real : scaleTo(real, n); + await runCase(n === real.length ? 'real data' : `scaled`, msgs); + } + console.log('\ndone.'); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/search-baseline.ts b/packages/minidb/bench/search-baseline.ts new file mode 100644 index 000000000..800dc1542 --- /dev/null +++ b/packages/minidb/bench/search-baseline.ts @@ -0,0 +1,218 @@ +// bench/search-baseline.ts +// +// "Without minidb" search baseline over the SAME real ~/.kimi-code sessions +// used by import-kimi-code.ts. Loads every session's extracted text into an +// in-memory array, then answers the same queries with a naive full scan +// (substring token-AND for text, linear Array.filter for secondary index / dt +// range). Paired with import-kimi-code.ts, this shows what minidb's indexes +// buy you on real data. +// +// Run: node --import tsx bench/search-baseline.ts [--data ~/.kimi-code] + +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; + +const argv = process.argv.slice(2); +const arg = (name: string, def: string) => { + const i = argv.indexOf(`--${name}`); + return i === -1 ? def : argv[i + 1]!; +}; +const DATA = path.resolve(arg('data', path.join(os.homedir(), '.kimi-code'))); +const FULL = argv.includes('--full'); // also index full tool results (matches import-kimi-code --full) + +const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); +const mib = (n: number) => (n / 1024 / 1024).toFixed(1) + ' MiB'; + +interface Doc { + sessionId: string; + title: string; + workspaceName: string; + text: string; + lower: string; // lowercased text, for naive substring search + updated: number; + created: number; +} + +const ARG_FIELDS = ['command', 'pattern', 'path', 'description', 'query', 'prompt', 'file_path']; + +function extractWireText(wirePath: string, full: boolean): string { + let raw: string; + try { + raw = readFileSync(wirePath, 'utf8'); + } catch { + return ''; + } + const parts: string[] = []; + for (const line of raw.split('\n')) { + if (!line) continue; + let o: any; + try { + o = JSON.parse(line); + } catch { + continue; + } + if (o.type === 'context.append_message' && o.message && o.message.content) { + for (const c of o.message.content) + if (c && c.type === 'text' && typeof c.text === 'string') parts.push(c.text); + } else if (o.type === 'context.append_loop_event' && o.event && o.event.type === 'tool.call') { + const e = o.event; + const bits: string[] = [e.name]; + for (const k of ARG_FIELDS) { + const v = e.args && e.args[k]; + if (typeof v === 'string' && v) bits.push(v.length > 2000 ? v.slice(0, 2000) : v); + } + parts.push(bits.join(' ')); + } else if (full && o.type === 'context.append_loop_event' && o.event && o.event.type === 'tool.result') { + const r = o.event.result; + let out = ''; + if (typeof r === 'string') out = r; + else if (r && typeof r.output === 'string') out = r.output; + else if (r && typeof r.content === 'string') out = r.content; + if (out) parts.push(out.length > 5000 ? out.slice(0, 5000) : out); + } + } + return parts.join('\n'); +} + +function tokenize(s: string): string[] { + return s + .toLowerCase() + .split(/[\s,，。.!?！?、;；:：()\[\]{}"'<>/\\|@#%^&*+=~`\-_]+/) + .filter((t) => t.length > 0); +} + +// Naive full-text: a doc matches if every query token appears as a substring +// (AND semantics). Score = total occurrences of matched tokens. Returns top N. +function naiveTextSearch(docs: Doc[], q: string, limit: number) { + const toks = tokenize(q); + if (toks.length === 0) return []; + const hits: { doc: Doc; score: number }[] = []; + for (const d of docs) { + let score = 0; + let ok = true; + for (const t of toks) { + let idx = -1; + let cnt = 0; + while ((idx = d.lower.indexOf(t, idx + 1)) !== -1) cnt++; + if (cnt === 0) { + ok = false; + break; + } + score += cnt; + } + if (ok) hits.push({ doc: d, score }); + } + hits.sort((a, b) => b.score - a.score); + return hits.slice(0, limit); +} + +// median of N runs +function med(fn: () => T, runs = 7): { value: T; ms: number } { + const times: number[] = []; + let value!: T; + for (let i = 0; i < runs; i++) { + const t0 = performance.now(); + value = fn(); + times.push(performance.now() - t0); + } + times.sort((a, b) => a - b); + return { value, ms: times[(runs / 2) | 0]! }; +} + +async function main() { + console.log(`data: ${DATA}`); + + const wsRaw = JSON.parse(readFileSync(path.join(DATA, 'workspaces.json'), 'utf8')); + const workspaces = wsRaw.workspaces || wsRaw; + const lines = readFileSync(path.join(DATA, 'session_index.jsonl'), 'utf8').trim().split('\n'); + + const t0 = performance.now(); + const docs: Doc[] = []; + let totalTextBytes = 0; + let skipped = 0; + for (const line of lines) { + let meta: any; + try { + meta = JSON.parse(line); + } catch { + continue; + } + const { sessionId, sessionDir } = meta; + const wirePath = path.join(sessionDir, 'agents', 'main', 'wire.jsonl'); + if (!existsSync(wirePath)) { + skipped++; + continue; + } + let state: any = {}; + try { + state = JSON.parse(readFileSync(path.join(sessionDir, 'state.json'), 'utf8')); + } catch {} + const body = extractWireText(wirePath, FULL); + const text = (state.title ? state.title + '\n' : '') + body; + totalTextBytes += Buffer.byteLength(text, 'utf8'); + const wsId = path.basename(path.dirname(sessionDir)); + const ws = workspaces[wsId] || {}; + docs.push({ + sessionId, + title: state.title || '', + workspaceName: ws.name || '', + text, + lower: text.toLowerCase(), + updated: state.updatedAt ? Date.parse(state.updatedAt) : 0, + created: state.createdAt ? Date.parse(state.createdAt) : 0, + }); + } + const loadMs = performance.now() - t0; + if (global.gc) global.gc(); + console.log(`\n=== loaded (no index) ===`); + console.log(` sessions loaded: ${fmt(docs.length)} (skipped ${skipped})`); + console.log(` text loaded : ${mib(totalTextBytes)}`); + console.log(` load+parse time: ${(loadMs / 1000).toFixed(1)} s`); + console.log(` heap used : ${mib(process.memoryUsage().heapUsed)}`); + + // ---- naive full-text searches ---- + const queries = ['lark-approval', 'database compaction', '北京', 'Redis 持久化', 'worktree init', 'nonexistentxyz123']; + console.log(`\n=== naive full-text search (full scan, median of 7) ===`); + for (const q of queries) { + const { value: res, ms } = med(() => naiveTextSearch(docs, q, 5)); + console.log(` "${q}" -> ${res.length} hits in ${ms.toFixed(2)} ms`); + for (const r of res.slice(0, 3)) { + console.log(` [${r.score}] ${r.doc.workspaceName} :: ${r.doc.title.slice(0, 60)}`); + } + } + + // ---- naive composed query: text includes 'database', then sort ---- + { + const { value: res, ms } = med(() => { + const hit = docs.filter((d) => d.lower.includes('database')); + hit.sort((a, b) => a.workspaceName.localeCompare(b.workspaceName)); + return hit.slice(0, 5); + }); + console.log(`\n naive composed (text "database" + sort) -> ${res.length} in ${ms.toFixed(2)} ms`); + } + + // ---- naive dt range: updated in last 7 days ---- + { + const week = Date.now() - 7 * 864e5; + const { value: res, ms } = med(() => { + const hit = docs.filter((d) => d.updated >= week); + hit.sort((a, b) => b.updated - a.updated); + return hit.slice(0, 10); + }); + console.log(` naive dt range (updated in last 7d) -> ${res.length} shown in ${ms.toFixed(2)} ms`); + } + + // ---- naive secondary index lookup: workspaceName ---- + { + const { value: res, ms } = med(() => docs.filter((d) => d.workspaceName === 'kimi-code-dev-1')); + console.log(` naive lookup (workspace=kimi-code-dev-1) -> ${res.length} in ${ms.toFixed(2)} ms`); + } + + console.log(`\ndone.`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/session-children.ts b/packages/minidb/bench/session-children.ts new file mode 100644 index 000000000..9ded55ff7 --- /dev/null +++ b/packages/minidb/bench/session-children.ts @@ -0,0 +1,214 @@ +// bench/session-children.ts +// +// Benchmark for nested session "children" lookups. +// +// Scenario +// -------- +// Sessions form a tree (root -> child -> grandchild -> ...). Each session +// stores its OWN id as the key (`sess:`) and the parent link in the +// value (`value.metadata.parent_session_id`). The relationship is NOT encoded +// in the key, because session ids are long and a materialized path would blow +// the 128-char key cap and duplicate the whole ancestor chain on every leaf. +// +// We compare two ways to answer "list the direct children of session X": +// +// 1. indexed — compound index { groupBy: 'metadata.parent_session_id', +// orderBy: 'updatedAt' } + db.compoundRange('byParent', X, ...) +// -> O(log N + fanout) +// +// 2. scan — the legacy approach: db.scan() everything, filter by +// metadata.parent_session_id === X, sort by updatedAt desc. +// -> O(N log N) +// +// Run +// --- +// node --import tsx bench/session-children.ts +// +// Knobs (env): +// TOTAL total sessions (default 1_000) +// FANOUT children per parent (default 30) +// MAX_DEPTH deepest generation, root=0 (default 3) +// ITERS query iterations to avg (default 500) +// +// With TOTAL=1000, FANOUT=30, MAX_DEPTH=3 the tree is exactly: +// depth0: 1, depth1: 30, depth2: 900, depth3: 69 (1000 total) + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MiniDb } from '../src/index.js'; + +const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); +const ops = (n: number, ms: number) => `${fmt((n / ms) * 1000)} ops/s`; + +const TOTAL = Number(process.env.TOTAL || 1_000); +const FANOUT = Number(process.env.FANOUT || 30); +const MAX_DEPTH = Number(process.env.MAX_DEPTH || 3); +const ITERS = Number(process.env.ITERS || 500); + +interface Node { + id: string; + parentId: string | null; + depth: number; + updatedAt: number; +} + +// Long-ish session ids: `sess_` + 40 hex chars (45 chars total). +const makeId = (i: number) => `sess_${i.toString(16).padStart(40, '0')}`; + +function buildTree(total: number, fanout: number, maxDepth: number): Node[] { + const nodes: Node[] = []; + const base = Date.parse('2024-01-01'); + let counter = 0; + const root: Node = { id: makeId(counter++), parentId: null, depth: 0, updatedAt: base + counter }; + nodes.push(root); + const queue: Node[] = [root]; + while (queue.length && nodes.length < total) { + const parent = queue.shift()!; + if (parent.depth >= maxDepth) continue; + for (let i = 0; i < fanout && nodes.length < total; i++) { + const child: Node = { + id: makeId(counter++), + parentId: parent.id, + depth: parent.depth + 1, + updatedAt: base + counter, + }; + nodes.push(child); + queue.push(child); + } + } + return nodes; +} + +async function tmpDir() { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-children-')); +} + +async function timeIt(fn: () => T | Promise, iters: number): Promise<{ ms: number; last: T }> { + let last!: T; + const t0 = performance.now(); + for (let i = 0; i < iters; i++) last = await fn(); + return { ms: performance.now() - t0, last }; +} + +async function main() { + console.log( + `\nminidb session-children benchmark (TOTAL=${fmt(TOTAL)}, FANOUT=${FANOUT}, MAX_DEPTH=${MAX_DEPTH}, ITERS=${ITERS}, node ${process.version})\n`, + ); + + // ---- build the tree ----------------------------------------------------- + const nodes = buildTree(TOTAL, FANOUT, MAX_DEPTH); + const byDepth = new Map(); + const childrenOf = new Map(); + for (const n of nodes) { + byDepth.set(n.depth, (byDepth.get(n.depth) ?? 0) + 1); + const arr = childrenOf.get(n.parentId) ?? []; + arr.push(n.id); + childrenOf.set(n.parentId, arr); + } + const maxDepth = Math.max(...byDepth.keys()); + console.log(` tree: ${fmt(nodes.length)} sessions, max depth ${maxDepth}`); + for (const [d, c] of [...byDepth.entries()].sort((a, b) => a[0] - b[0])) { + console.log(` depth ${d}: ${fmt(c)} sessions`); + } + + // Pick a parent at each depth that actually has children, for depth + // coverage in the query benchmark. + const parentAtDepth = new Map(); + for (const n of nodes) { + if (!parentAtDepth.has(n.depth) && (childrenOf.get(n.id)?.length ?? 0) > 0) parentAtDepth.set(n.depth, n); + } + + // ---- open + ingest ------------------------------------------------------ + const dir = await tmpDir(); + const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); + + const tIngest = performance.now(); + const CHUNK = 500; + for (let i = 0; i < nodes.length; i += CHUNK) { + const slice = nodes.slice(i, i + CHUNK); + await db.batch( + slice.map((n) => ({ + op: 'set' as const, + key: n.id, + value: { + title: `session ${n.id}`, + workspaceId: 'ws_bench', + metadata: { + // Only real children carry parent_session_id (+ kind). Roots do + // not, so they stay out of the byParent index entirely. + ...(n.parentId ? { parent_session_id: n.parentId, child_session_kind: 'child' } : {}), + }, + }, + dt: { updatedAt: n.updatedAt }, + })), + ); + } + const ingestMs = performance.now() - tIngest; + console.log(`\n ingest ${fmt(nodes.length)} sessions (batch)`.padEnd(46), `${ingestMs.toFixed(1).padStart(8)} ms`, `-> ${ops(nodes.length, ingestMs)}`); + + // ---- index build (the thing that makes children fast) ------------------- + const tIdx = performance.now(); + await db.createCompoundIndex('byParent', { + groupBy: 'metadata.parent_session_id', + orderBy: 'updatedAt', + }); + const idxMs = performance.now() - tIdx; + console.log(` createCompoundIndex('byParent') + rebuild`.padEnd(46), `${idxMs.toFixed(1).padStart(8)} ms`); + + // ---- query benchmarks --------------------------------------------------- + const PAGE = 20; + console.log(`\n listChildren(parent) page_size=${PAGE}, averaged over ${ITERS} iters:\n`); + + for (const [depth, parent] of [...parentAtDepth.entries()].sort((a, b) => a[0] - b[0])) { + const fanout = childrenOf.get(parent.id)!.length; + + // indexed: O(log N + fanout) + const idx = await timeIt( + () => db.compoundRange('byParent', parent.id, { reverse: true, limit: PAGE }), + ITERS, + ); + // scan (legacy): O(N log N) + const scan = await timeIt(() => { + const all = db.scan(); + const filtered = all.filter( + (r) => + (r.value as { metadata?: Record })?.metadata?.['parent_session_id'] === parent.id && + (r.value as { metadata?: Record })?.metadata?.['child_session_kind'] === 'child', + ); + filtered.sort((a, b) => (b.dt?.updatedAt ?? 0) - (a.dt?.updatedAt ?? 0)); + return filtered.slice(0, PAGE); + }, ITERS); + + const idxAvg = idx.ms / ITERS; + const scanAvg = scan.ms / ITERS; + console.log( + ` depth ${depth} parent (fanout=${fmt(fanout)}):`.padEnd(38), + `indexed ${idxAvg.toFixed(3)} ms/op`.padEnd(22), + `scan ${scanAvg.toFixed(3)} ms/op`.padEnd(20), + `speedup x${(scanAvg / idxAvg).toFixed(1)}`, + ); + } + + // ---- correctness spot-check --------------------------------------------- + const root = nodes[0]!; + const idxChildren = db.compoundRange('byParent', root.id, { reverse: true, limit: 10_000 }).map((r) => r.key); + const expected = new Set(childrenOf.get(root.id)!); + const ok = idxChildren.length === expected.size && idxChildren.every((k) => expected.has(k)); + console.log(`\n correctness: indexed children of root == expected ${ok ? 'OK' : 'MISMATCH'} (${idxChildren.length}/${expected.size})`); + + // ---- storage snapshot --------------------------------------------------- + await db.compact(); + const snap = await fs.stat(path.join(dir, 'db.snapshot')); + const heapMiB = (process.memoryUsage().heapUsed / 1024 / 1024).toFixed(1); + console.log(` storage: ${(snap.size / 1024 / 1024).toFixed(2)} MiB snapshot, heap ${heapMiB} MiB`); + + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); + console.log('\ndone.\n'); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/src/dt-index.ts b/packages/minidb/src/dt-index.ts index 8b8e4685c..d01ba7fd8 100644 --- a/packages/minidb/src/dt-index.ts +++ b/packages/minidb/src/dt-index.ts @@ -80,6 +80,17 @@ export class DtIndex { return c.list.range(opts).map((n: RangeEntry) => ({ key: n.val, value: n.key })); } + /** Lazy range over a dt column; yields { key: recordKey, value: ts } like + * range() but lets the caller stop early without materializing everything. */ + *iterate( + col: string, + opts: Parameters['iterate']>[0] = {}, + ): Generator { + const c = this.cols.get(col); + if (!c) return; + for (const n of c.list.iterate(opts)) yield { key: n.val, value: n.key }; + } + columns(): string[] { return [...this.cols.keys()]; } diff --git a/packages/minidb/src/index-manager.ts b/packages/minidb/src/index-manager.ts index 409e1d9ef..6c005d11c 100644 --- a/packages/minidb/src/index-manager.ts +++ b/packages/minidb/src/index-manager.ts @@ -274,6 +274,15 @@ export class IndexManager { return set ? [...set] : []; } + /** O(1) membership test: is `pk` indexed under `value` on this equality + * index? Avoids materializing the full posting list like findEq does. */ + hasEq(name: string, value: unknown, pk: string): boolean { + const idx = this.get(name); + if (idx.type !== 'equality') throw new Error(`index "${name}" is not an equality index`); + const set = idx.map.get(scalarKey(value)); + return !!set && set.has(pk); + } + findRange( name: string, opts: { min?: number; max?: number; minExclusive?: boolean; maxExclusive?: boolean; offset?: number; count?: number; reverse?: boolean } = {}, diff --git a/packages/minidb/src/index.ts b/packages/minidb/src/index.ts index 2a2bd956e..8f091c577 100644 --- a/packages/minidb/src/index.ts +++ b/packages/minidb/src/index.ts @@ -1079,10 +1079,102 @@ export class MiniDb { return [...candidates]; } + // Extract simple equality predicates (top-level or inside $and) that are + // backed by an equality index, for use as a cheap per-candidate pre-filter. + // Only direct equality and {$eq: x} qualify; $in / range / non-indexed fields + // are left to the full match() after decode. + private cheapEqChecks(filter?: Record): { name: string; value: unknown }[] { + const out: { name: string; value: unknown }[] = []; + if (!filter || typeof filter !== 'object' || !this.indexes.indexes.size) return out; + for (const { field, cond } of this.indexPredicates(filter)) { + const idx = this.indexes.list().find((i) => i.field === field && i.type === 'equality'); + if (!idx) continue; + if (cond !== null && typeof cond === 'object' && !(cond instanceof RegExp)) { + const ops = cond as Record; + if (Object.keys(ops).length === 1 && '$eq' in ops) out.push({ name: idx.name, value: ops['$eq'] }); + } else { + out.push({ name: idx.name, value: cond }); + } + } + return out; + } + + // Fast path: a query bounded by a single dt column whose result order is that + // dt column can walk the dt skiplist in order and stop as soon as `limit` + // qualifying rows are found, instead of materializing + decoding + sorting the + // whole candidate set. Returns null when the query is not eligible (caller + // falls back to the general path). Kept conservative so results match exactly. + private tryDtOrderedLimit(q: QueryOptions): ScanEntry[] | null { + if (q.text) return null; // text has its own ranking + if (q.key !== undefined) return null; + if (q.limit === undefined) return null; // unbounded -> full return, no win + if (!q.dt) return null; + const dtCols = Object.keys(q.dt); + if (dtCols.length !== 1) return null; + const col = dtCols[0]!; + const cond = q.dt[col]!; + + // Result order must be the dt column's order. + let reverse = false; + if (q.sort) { + const entries = Object.entries(q.sort); + if (entries.length !== 1) return null; + const [sortKey, dir] = entries[0]!; + if (sortKey !== col) return null; + reverse = dir < 0; + } + + const limit = q.limit; + const skip = q.skip ?? 0; + // Only the range bounds are honored here; ignore any stray count/offset a + // caller put on the dt cond so they cannot truncate the walk prematurely. + const iterOpts: RangeOptions = { reverse }; + if (cond.gte !== undefined) iterOpts.gte = cond.gte; + if (cond.gt !== undefined) iterOpts.gt = cond.gt; + if (cond.lte !== undefined) iterOpts.lte = cond.lte; + if (cond.lt !== undefined) iterOpts.lt = cond.lt; + + // Cheap key-level pre-filter (no decode, no full-set materialization) for + // simple equality predicates that have an equality index. + const eqChecks = this.cheapEqChecks(q.filter); + + const out: { key: string; value: V; dt: Record | undefined }[] = []; + let skipped = 0; + for (const { key: kstr } of this.dt.iterate(col, iterOpts)) { + let rejected = false; + for (const c of eqChecks) { + if (!this.indexes.hasEq(c.name, c.value, kstr)) { + rejected = true; + break; + } + } + if (rejected) continue; + const buf = this.store.get(kstr); + if (buf === undefined) continue; + const r = this.store.map.get(kstr); + const value = this.decode(buf)!; + if (q.filter && !match(value, q.filter)) continue; + if (skipped < skip) { + skipped++; + continue; + } + out.push({ key: kstr, value, dt: r?.dt ?? undefined }); + if (out.length >= limit) break; + } + + return out.map((d) => ({ + key: fromKStr(d.key), + value: q.project ? (project(d.value, q.project) as V) : d.value, + dt: d.dt, + })); + } + // ---- unified query ------------------------------------------------------ query(q: QueryOptions = {}): ScanEntry[] { this.ensureOpen(); + const fast = this.tryDtOrderedLimit(q); + if (fast !== null) return fast; let keys: string[] | null = null; if (typeof q.key === 'string') { diff --git a/packages/minidb/src/skiplist.ts b/packages/minidb/src/skiplist.ts index 43501cbb6..5fd4426db 100644 --- a/packages/minidb/src/skiplist.ts +++ b/packages/minidb/src/skiplist.ts @@ -250,6 +250,53 @@ export class SkipList { return out; } + /** Lazy range scan. Same bounds/offset/count/reverse semantics as range(), + * but yields entries one by one so a caller can stop early without + * materializing the whole range. */ + *iterate(opts: RangeOptions = {}): Generator> { + let offset = opts.offset ?? 0; + let count = opts.count ?? Infinity; + + if (opts.reverse) { + let x: SkipNode | null; + if (opts.lte !== undefined) { + const after = this.lowerBound(opts.lte, { strict: true }); + x = after ? after.backward : this.tail; + } else if (opts.lt !== undefined) { + const after = this.lowerBound(opts.lt, { strict: false }); + x = after ? after.backward : this.tail; + } else { + x = this.tail; + } + while (x) { + if (opts.gte !== undefined && this.cmpK(x.key, opts.gte) < 0) break; + if (opts.gt !== undefined && this.cmpK(x.key, opts.gt) <= 0) break; + if (offset > 0) offset--; + else if (count > 0) { + yield { key: x.key, val: x.val }; + count--; + } else break; + x = x.backward; + } + return; + } + + const hasLower = opts.gte !== undefined || opts.gt !== undefined; + let x = hasLower + ? this.lowerBound(opts.gte !== undefined ? opts.gte : (opts.gt as K), { strict: opts.gt !== undefined }) + : this.header.level[0]!.forward; + while (x) { + if (opts.lte !== undefined && this.cmpK(x.key, opts.lte) > 0) break; + if (opts.lt !== undefined && this.cmpK(x.key, opts.lt) >= 0) break; + if (offset > 0) offset--; + else if (count > 0) { + yield { key: x.key, val: x.val }; + count--; + } else break; + x = x.level[0]!.forward; + } + } + toArray(): RangeEntry[] { const out: RangeEntry[] = []; let x = this.header.level[0]!.forward; diff --git a/packages/minidb/src/text-index.ts b/packages/minidb/src/text-index.ts index f95fb9f69..dcb4b9de6 100644 --- a/packages/minidb/src/text-index.ts +++ b/packages/minidb/src/text-index.ts @@ -30,7 +30,9 @@ export function tokenize(str: unknown): string[] { const s = String(str).toLowerCase(); const terms: string[] = []; const latin = s.match(LATIN); - if (latin) terms.push(...latin); + // Loop-push instead of `terms.push(...latin)`: spreading a large match array + // (hundreds of thousands of tokens from a big doc) overflows the call stack. + if (latin) for (const t of latin) terms.push(t); const runs = s.match(CJK) ?? []; for (const r of runs) { for (let i = 0; i < r.length; i++) { diff --git a/packages/minidb/test/query.test.ts b/packages/minidb/test/query.test.ts index 0dbab6e07..629780386 100644 --- a/packages/minidb/test/query.test.ts +++ b/packages/minidb/test/query.test.ts @@ -104,6 +104,76 @@ test('query composes dt range + value filter + sort + limit + project', async () } }); +test('dt-ordered limit fast path matches a reference (no ties)', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byRole', { field: 'role' }); + const docs: { key: string; role: string; n: number; ts: number }[] = []; + for (let i = 1; i <= 20; i++) { + const ts = i * 100; // unique + const role = i % 2 ? 'user' : 'assistant'; + await db.set(`d${i}`, { role, n: i }, { dt: { ts } }); + docs.push({ key: `d${i}`, role, n: i, ts }); + } + const gte = 200; + const lte = 1500; + + const refDesc = docs + .filter((d) => d.role === 'user' && d.ts >= gte && d.ts <= lte) + .sort((a, b) => b.ts - a.ts); + + // sort by dt desc + limit -> fast path + assert.deepEqual( + db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, sort: { ts: -1 }, limit: 3 }).map((r) => r.key), + refDesc.slice(0, 3).map((d) => d.key), + ); + // ascending + skip + limit -> fast path + const refAsc = [...refDesc].reverse(); + assert.deepEqual( + db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, sort: { ts: 1 }, skip: 1, limit: 2 }).map((r) => r.key), + refAsc.slice(1, 3).map((d) => d.key), + ); + // no sort -> defaults to dt ascending, still fast path + assert.deepEqual( + db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, limit: 2 }).map((r) => r.key), + refAsc.slice(0, 2).map((d) => d.key), + ); + // project still applies on the fast path + const proj = db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, sort: { ts: -1 }, limit: 1, project: ['n'] }); + assert.deepEqual(proj[0], { key: refDesc[0]!.key, value: { n: refDesc[0]!.n }, dt: { ts: refDesc[0]!.ts } }); + // sort by a non-dt field -> falls back to general path (still correct) + assert.deepEqual( + db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, sort: { n: -1 }, limit: 3 }).map((r) => r.key), + refDesc.slice(0, 3).map((d) => d.key), // n == i order matches ts here + ); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('dt-ordered limit fast path orders equal ts by key (tie-break)', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + // several docs share the same ts; SkipList tie-break is by record key + for (const k of ['t3', 't1', 't4', 't2']) await db.set(k, { v: k }, { dt: { ts: 1000 } }); + // descending -> key desc; ascending -> key asc + assert.deepEqual( + db.query({ dt: { ts: { gte: 1000, lte: 1000 } }, sort: { ts: -1 }, limit: 4 }).map((r) => r.key), + ['t4', 't3', 't2', 't1'], + ); + assert.deepEqual( + db.query({ dt: { ts: { gte: 1000, lte: 1000 } }, sort: { ts: 1 }, limit: 2 }).map((r) => r.key), + ['t1', 't2'], + ); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + test('full-text search: latin + CJK', async () => { const dir = await tmpDir(); try { diff --git a/packages/minidb/test/skiplist.test.ts b/packages/minidb/test/skiplist.test.ts index 8dffc355c..0b9043562 100644 --- a/packages/minidb/test/skiplist.test.ts +++ b/packages/minidb/test/skiplist.test.ts @@ -71,6 +71,38 @@ test('string comparator orders keys and supports prefix scans', () => { assert.deepEqual(res, ['user:1', 'user:10', 'user:2', 'user:3']); }); +test('iterate() yields the same sequence as range()', () => { + const sl = new SkipList(); + for (let i = 1; i <= 20; i++) sl.insert(i, `v${i}`); + const cases = [ + {}, + { gte: 3, lte: 7 }, + { gt: 3, lt: 7 }, + { gte: 1, lte: 20, offset: 2, count: 5 }, + { gte: 5 }, + { lte: 5 }, + { gte: 3, lte: 7, reverse: true }, + { reverse: true, offset: 2, count: 4 }, + { gt: 3, lt: 7, reverse: true }, + { gte: 100 }, // empty + { lte: 0 }, // empty + ]; + for (const opts of cases) { + assert.deepEqual([...sl.iterate(opts)], sl.range(opts), `opts=${JSON.stringify(opts)}`); + } +}); + +test('iterate() stops early (lazy)', () => { + const sl = new SkipList(); + for (let i = 1; i <= 1000; i++) sl.insert(i, `v${i}`); + let seen = 0; + for (const _ of sl.iterate({ gte: 1 })) { + seen++; + if (seen === 3) break; + } + assert.equal(seen, 3); +}); + test('matches a sorted reference under random operations', () => { const sl = new SkipList(); const ref = new Map(); // val -> key