mirror of
https://github.com/MoonshotAI/kimi-code.git
synced 2026-07-10 01:39:25 +00:00
perf(minidb): add dt-ordered limit fast path for queries
- add SkipList.iterate and DtIndex.iterate: lazy range scans that let a caller stop early without materializing the whole range - add IndexManager.hasEq for O(1) equality-index membership checks that avoid materializing the full posting list - add MiniDb.tryDtOrderedLimit (with cheapEqChecks): for a query bounded by a single dt column whose result order is that column, walk the dt skiplist in order and stop at limit, pre-filtering cheap equality predicates before decode; wire it into query() - fix text-index tokenization overflowing the call stack on large docs by loop-pushing instead of spreading the match array - add bench/* benchmarks and tests for the fast path and skiplist iteration
This commit is contained in:
haozhe.yang
parent
61ef35b302
commit
37b2c46691
12 changed files with 1242 additions and 1 deletions
169
packages/minidb/bench/baseline.ts
Normal file
169
packages/minidb/bench/baseline.ts
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
// bench/baseline.ts
|
||||
//
|
||||
// "Without minidb" baseline for bench.ts. Reproduces the same durable KV
|
||||
// operations using only a JS Map + raw node:fs, so the timings isolate what
|
||||
// minidb's machinery (group commit, binary codec + CRC, in-memory store,
|
||||
// snapshot format) costs over a hand-written equivalent.
|
||||
//
|
||||
// Conventions mirror bench.ts so the two can be compared line by line:
|
||||
// - same N / NSMALL / 100B value
|
||||
// - three fsync policies: no / everysec (1s timer) / always (per-write fsync)
|
||||
// - concurrent set = fire all appends then Promise.all (file opened with
|
||||
// O_APPEND, so each write lands atomically at EOF, like minidb's WAL fd)
|
||||
// - snapshot = serialize the whole map to JSON and fsync it
|
||||
//
|
||||
// Run: node --import tsx bench/baseline.ts
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||
const ops = (n: number, ms: number) => `${fmt((n / ms) * 1000)} ops/s`;
|
||||
|
||||
async function tmpDir() {
|
||||
return fs.mkdtemp(path.join(os.tmpdir(), 'baseline-bench-'));
|
||||
}
|
||||
|
||||
async function bench(label: string, fn: () => Promise<unknown> | unknown) {
|
||||
if (global.gc) global.gc();
|
||||
const t0 = performance.now();
|
||||
await fn();
|
||||
const ms = performance.now() - t0;
|
||||
console.log(` ${label.padEnd(46)} ${ms.toFixed(1).padStart(8)} ms`);
|
||||
return ms;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const VALUE = 'x'.repeat(100); // 100-byte values
|
||||
const N = Number(process.env.N || 200_000);
|
||||
const NSMALL = Number(process.env.NSMALL || 3_000);
|
||||
|
||||
console.log(`\nbaseline (no minidb) (N=${fmt(N)}, value=${VALUE.length}B, node ${process.version})\n`);
|
||||
|
||||
// --- baseline: raw JS Map (in-memory) ---------------------------------
|
||||
{
|
||||
const m = new Map<string, string>();
|
||||
const ms = await bench('baseline: raw Map set (in-memory)', () => {
|
||||
for (let i = 0; i < N; i++) m.set(`k${i}`, VALUE);
|
||||
});
|
||||
console.log(` -> ${ops(N, ms)}`);
|
||||
const ms2 = await bench('baseline: raw Map get (in-memory)', () => {
|
||||
let s = 0;
|
||||
for (let i = 0; i < N; i++) if (m.get(`k${i}`)) s++;
|
||||
return s;
|
||||
});
|
||||
console.log(` -> ${ops(N, ms2)}`);
|
||||
}
|
||||
|
||||
// --- naive durable writes, fsync=no -----------------------------------
|
||||
// One JSONL record per set, O_APPEND, no fsync (rely on OS flush).
|
||||
{
|
||||
const dir = await tmpDir();
|
||||
const file = path.join(dir, 'log.jsonl');
|
||||
const m = new Map<string, string>();
|
||||
const fh = await fs.open(file, 'a');
|
||||
let bytes = 0;
|
||||
const ms = await bench('naive set concurrent, fsync=no (O_APPEND)', async () => {
|
||||
const p: Promise<unknown>[] = [];
|
||||
for (let i = 0; i < N; i++) {
|
||||
const k = `k${i}`;
|
||||
m.set(k, VALUE);
|
||||
const line = Buffer.from(JSON.stringify({ k, v: VALUE }) + '\n');
|
||||
bytes += line.length;
|
||||
p.push(fh.write(line));
|
||||
}
|
||||
await Promise.all(p);
|
||||
});
|
||||
await fh.close();
|
||||
console.log(` -> ${ops(N, ms)} (${(bytes / 1024 / 1024).toFixed(2)} MiB written)`);
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// --- naive durable writes, fsync=everysec -----------------------------
|
||||
{
|
||||
const dir = await tmpDir();
|
||||
const file = path.join(dir, 'log.jsonl');
|
||||
const m = new Map<string, string>();
|
||||
const fh = await fs.open(file, 'a');
|
||||
const timer = setInterval(() => {
|
||||
fh.sync().catch(() => {});
|
||||
}, 1000);
|
||||
timer.unref?.();
|
||||
let bytes = 0;
|
||||
const ms = await bench('naive set concurrent, fsync=everysec', async () => {
|
||||
const p: Promise<unknown>[] = [];
|
||||
for (let i = 0; i < N; i++) {
|
||||
const k = `k${i}`;
|
||||
m.set(k, VALUE);
|
||||
const line = Buffer.from(JSON.stringify({ k, v: VALUE }) + '\n');
|
||||
bytes += line.length;
|
||||
p.push(fh.write(line));
|
||||
}
|
||||
await Promise.all(p);
|
||||
});
|
||||
clearInterval(timer);
|
||||
await fh.sync();
|
||||
await fh.close();
|
||||
console.log(` -> ${ops(N, ms)} (${(bytes / 1024 / 1024).toFixed(2)} MiB written)`);
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// --- naive durable writes, sequential fsync=always --------------------
|
||||
{
|
||||
const dir = await tmpDir();
|
||||
const file = path.join(dir, 'log.jsonl');
|
||||
const m = new Map<string, string>();
|
||||
const fh = await fs.open(file, 'a');
|
||||
const ms = await bench(`naive set sequential, fsync=always (N=${fmt(NSMALL)})`, async () => {
|
||||
for (let i = 0; i < NSMALL; i++) {
|
||||
const k = `k${i}`;
|
||||
m.set(k, VALUE);
|
||||
await fh.write(Buffer.from(JSON.stringify({ k, v: VALUE }) + '\n'));
|
||||
await fh.sync();
|
||||
}
|
||||
});
|
||||
await fh.close();
|
||||
console.log(` -> ${ops(NSMALL, ms)}`);
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// --- naive reads (in-memory Map after load) ---------------------------
|
||||
{
|
||||
const m = new Map<string, string>();
|
||||
for (let i = 0; i < N; i++) m.set(`k${i}`, VALUE);
|
||||
const ms = await bench('naive get (in-memory Map)', () => {
|
||||
let s = 0;
|
||||
for (let i = 0; i < N; i++) if (m.get(`k${i}`)) s++;
|
||||
return s;
|
||||
});
|
||||
console.log(` -> ${ops(N, ms)}`);
|
||||
}
|
||||
|
||||
// --- naive snapshot ---------------------------------------------------
|
||||
// Dump the whole map to one JSON file + fsync (the hand-written equivalent
|
||||
// of minidb's compact()).
|
||||
{
|
||||
const dir = await tmpDir();
|
||||
const file = path.join(dir, 'snapshot.json');
|
||||
const m = new Map<string, string>();
|
||||
for (let i = 0; i < N; i++) m.set(`k${i}`, VALUE);
|
||||
const ms = await bench(`naive JSON snapshot of ${fmt(N)} keys`, async () => {
|
||||
const body = JSON.stringify(Object.fromEntries(m));
|
||||
await fs.writeFile(file, body);
|
||||
const fh = await fs.open(file, 'r');
|
||||
await fh.sync();
|
||||
await fh.close();
|
||||
});
|
||||
const snap = await fs.stat(file);
|
||||
console.log(` -> ${(snap.size / 1024 / 1024).toFixed(2)} MiB snapshot in ${ms.toFixed(0)} ms`);
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
console.log('\ndone.\n');
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
197
packages/minidb/bench/message-composed.ts
Normal file
197
packages/minidb/bench/message-composed.ts
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
// bench/message-composed.ts
|
||||
//
|
||||
// The exact scenario the user asked about:
|
||||
// index ALL conversational messages (user prompts + assistant text), then
|
||||
// answer: "user messages within a time window".
|
||||
//
|
||||
// predicate: role === 'user' AND ts >= since (last 7 days)
|
||||
//
|
||||
// Three result shapes, each tells a different story:
|
||||
// A. top-50 recent (LIMIT) — index should win (key-level intersect +
|
||||
// decode bounded by limit)
|
||||
// B. ALL matches (no limit) — naive wins (minidb decodes every match)
|
||||
// C. COUNT matches — naive wins big (minidb has no count
|
||||
// path; it materializes everything then
|
||||
// takes .length). Motivates a count/keys
|
||||
// API.
|
||||
//
|
||||
// Run: node --import tsx bench/message-composed.ts
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { MiniDb } from '../src/index.js';
|
||||
|
||||
const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||
const LIMIT = 50;
|
||||
const WINDOW_DAYS = 30;
|
||||
const SINCE_DAYS = 7;
|
||||
|
||||
interface Msg {
|
||||
id: string;
|
||||
ts: number;
|
||||
role: 'user' | 'assistant';
|
||||
text: string;
|
||||
}
|
||||
|
||||
function loadAllMessages(): Msg[] {
|
||||
const DATA = path.join(os.homedir(), '.kimi-code');
|
||||
const lines = readFileSync(path.join(DATA, 'session_index.jsonl'), 'utf8').trim().split('\n');
|
||||
const out: Msg[] = [];
|
||||
for (const line of lines) {
|
||||
let meta: any;
|
||||
try {
|
||||
meta = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const wire = path.join(meta.sessionDir, 'agents', 'main', 'wire.jsonl');
|
||||
if (!existsSync(wire)) continue;
|
||||
let raw: string;
|
||||
try {
|
||||
raw = readFileSync(wire, 'utf8');
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const ln of raw.split('\n')) {
|
||||
if (!ln) continue;
|
||||
let o: any;
|
||||
try {
|
||||
o = JSON.parse(ln);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const ts = typeof o.time === 'number' ? o.time : 0;
|
||||
if (o.type === 'context.append_message' && o.message) {
|
||||
let text = '';
|
||||
for (const c of o.message.content || [])
|
||||
if (c && c.type === 'text' && typeof c.text === 'string') text += c.text;
|
||||
out.push({ id: `${meta.sessionId}:u${out.length}`, ts, role: 'user', text });
|
||||
} else if (
|
||||
o.type === 'context.append_loop_event' &&
|
||||
o.event &&
|
||||
o.event.type === 'content.part' &&
|
||||
o.event.part &&
|
||||
o.event.part.type === 'text' &&
|
||||
typeof o.event.part.text === 'string'
|
||||
) {
|
||||
out.push({ id: `${meta.sessionId}:a${out.length}`, ts, role: 'assistant', text: o.event.part.text });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function scaleTo(real: Msg[], n: number): Msg[] {
|
||||
const now = Date.now();
|
||||
const span = WINDOW_DAYS * 864e5;
|
||||
const out: Msg[] = new Array(n);
|
||||
for (let i = 0; i < n; i++) {
|
||||
const r = real[i % real.length]!;
|
||||
out[i] = { id: `m${i}`, ts: now - Math.floor((i / n) * span), role: r.role, text: r.text };
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function buildMinidb(msgs: Msg[]) {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-mc-'));
|
||||
const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false });
|
||||
await db.createIndex('byRole', { field: 'role' });
|
||||
const t0 = performance.now();
|
||||
const CHUNK = 5000;
|
||||
for (let s = 0; s < msgs.length; s += CHUNK) {
|
||||
const p: Promise<unknown>[] = [];
|
||||
for (let i = s; i < Math.min(s + CHUNK, msgs.length); i++) {
|
||||
const m = msgs[i]!;
|
||||
p.push(db.set(m.id, { role: m.role, text: m.text }, { dt: { ts: m.ts } }));
|
||||
}
|
||||
await Promise.all(p);
|
||||
}
|
||||
return { db, buildMs: performance.now() - t0, dir };
|
||||
}
|
||||
|
||||
function med(fn: () => void, runs = 9): number {
|
||||
const t: number[] = [];
|
||||
for (let i = 0; i < runs; i++) {
|
||||
const t0 = performance.now();
|
||||
fn();
|
||||
t.push(performance.now() - t0);
|
||||
}
|
||||
t.sort((a, b) => a - b);
|
||||
return t[(runs / 2) | 0]!;
|
||||
}
|
||||
|
||||
async function runCase(label: string, msgs: Msg[]) {
|
||||
const since = Date.now() - SINCE_DAYS * 864e5;
|
||||
const userTotal = msgs.filter((m) => m.role === 'user').length;
|
||||
const { db, buildMs, dir } = await buildMinidb(msgs);
|
||||
|
||||
// A. top-50 recent user in window
|
||||
let aHits = 0;
|
||||
const aDb = med(() => {
|
||||
aHits = db.query({
|
||||
dt: { ts: { gte: since } },
|
||||
filter: { role: 'user' },
|
||||
sort: { ts: -1 },
|
||||
limit: LIMIT,
|
||||
}).length;
|
||||
});
|
||||
const aNaive = med(() => {
|
||||
aHits = msgs.filter((m) => m.role === 'user' && m.ts >= since).sort((a, b) => b.ts - a.ts).slice(0, LIMIT).length;
|
||||
});
|
||||
|
||||
// B. ALL user in window (no limit)
|
||||
let bDbHits = 0;
|
||||
const bDb = med(() => {
|
||||
bDbHits = db.query({ dt: { ts: { gte: since } }, filter: { role: 'user' } }).length;
|
||||
});
|
||||
let bNaiveHits = 0;
|
||||
const bNaive = med(() => {
|
||||
bNaiveHits = msgs.filter((m) => m.role === 'user' && m.ts >= since).length;
|
||||
});
|
||||
|
||||
// C. COUNT user in window (minidb has no count path -> materialize then .length)
|
||||
const cDb = med(() => {
|
||||
db.query({ dt: { ts: { gte: since } }, filter: { role: 'user' } });
|
||||
});
|
||||
let cNaive = 0;
|
||||
const cNaiveMs = med(() => {
|
||||
let c = 0;
|
||||
for (const m of msgs) if (m.role === 'user' && m.ts >= since) c++;
|
||||
cNaive = c;
|
||||
});
|
||||
|
||||
if (global.gc) global.gc();
|
||||
const heap = process.memoryUsage().heapUsed;
|
||||
await db.close();
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
|
||||
console.log(`\n--- ${label}: N=${fmt(msgs.length)} msgs (user=${fmt(userTotal)}) ---`);
|
||||
console.log(` build: minidb ${(buildMs / 1000).toFixed(2)}s | heap ${(heap / 1024 / 1024).toFixed(0)} MiB`);
|
||||
console.log(
|
||||
` A top-${LIMIT}: minidb ${aDb.toFixed(3)} ms vs naive ${aNaive.toFixed(3)} ms -> ${(aNaive / aDb).toFixed(1)}x (hits ${aHits})`,
|
||||
);
|
||||
console.log(
|
||||
` B ALL: minidb ${bDb.toFixed(3)} ms vs naive ${bNaive.toFixed(3)} ms -> ${(bNaive / bDb).toFixed(2)}x (hits ${fmt(bDbHits)}/${fmt(bNaiveHits)})`,
|
||||
);
|
||||
console.log(
|
||||
` C COUNT: minidb ${cDb.toFixed(3)} ms vs naive ${cNaiveMs.toFixed(3)} ms -> ${(cNaiveMs / cDb).toFixed(2)}x (count ${fmt(cNaive)})`,
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const real = loadAllMessages();
|
||||
const byRole = real.reduce((a: any, m) => ((a[m.role] = (a[m.role] || 0) + 1), a), {});
|
||||
console.log(`loaded ${fmt(real.length)} real messages: ${JSON.stringify(byRole)}`);
|
||||
for (const n of [real.length, 100_000, 1_000_000]) {
|
||||
const msgs = n === real.length ? real : scaleTo(real, n);
|
||||
await runCase(n === real.length ? 'real data' : 'scaled', msgs);
|
||||
}
|
||||
console.log('\ndone.');
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
180
packages/minidb/bench/message-range.ts
Normal file
180
packages/minidb/bench/message-range.ts
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
// bench/message-range.ts
|
||||
//
|
||||
// Benchmarks the "show me the user's most recent messages" query:
|
||||
// predicate: role === 'user' AND ts >= since (last 7 days)
|
||||
// order: ts DESC
|
||||
// limit: 50 (the typical "recent N" UI pattern)
|
||||
//
|
||||
// This is a message-granularity range query with a SMALL LIMIT, which is a
|
||||
// different shape from the session-level benches:
|
||||
// - import-kimi-code / search-baseline index whole sessions (one doc/session).
|
||||
// - the metadata lookups (workspace lookup, dt range over sessions) return
|
||||
// ALL matches, so at small N a linear filter beats the index.
|
||||
// Here, an ordered dt index can walk just `limit` entries (O(log N + limit))
|
||||
// instead of scanning + sorting the whole set (O(N log M)). We measure at the
|
||||
// real scale (~7.2k user messages) and at scaled-up sizes to find the
|
||||
// crossover where the index starts winning.
|
||||
//
|
||||
// Run: node --import tsx bench/message-range.ts
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { MiniDb } from '../src/index.js';
|
||||
|
||||
const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||
const LIMIT = 50;
|
||||
const WINDOW_DAYS = 30; // replicated timestamps are spread over the last 30 days
|
||||
const SINCE_DAYS = 7; // "recent" = last 7 days
|
||||
|
||||
interface Msg {
|
||||
id: string;
|
||||
ts: number;
|
||||
role: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
function loadRealMessages(): Msg[] {
|
||||
const DATA = path.join(os.homedir(), '.kimi-code');
|
||||
const lines = readFileSync(path.join(DATA, 'session_index.jsonl'), 'utf8').trim().split('\n');
|
||||
const out: Msg[] = [];
|
||||
for (const line of lines) {
|
||||
let meta: any;
|
||||
try {
|
||||
meta = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const wire = path.join(meta.sessionDir, 'agents', 'main', 'wire.jsonl');
|
||||
if (!existsSync(wire)) continue;
|
||||
let raw: string;
|
||||
try {
|
||||
raw = readFileSync(wire, 'utf8');
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const ln of raw.split('\n')) {
|
||||
if (!ln) continue;
|
||||
let o: any;
|
||||
try {
|
||||
o = JSON.parse(ln);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (o.type !== 'context.append_message' || !o.message) continue;
|
||||
const m = o.message;
|
||||
const ts = typeof o.time === 'number' ? o.time : 0;
|
||||
let text = '';
|
||||
for (const c of m.content || [])
|
||||
if (c && c.type === 'text' && typeof c.text === 'string') text += c.text;
|
||||
out.push({ id: `${meta.sessionId}:${out.length}`, ts, role: m.role || 'user', text });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Replicate the real messages to N, spreading timestamps uniformly over the
|
||||
// last WINDOW_DAYS so the "last SINCE_DAYS" match fraction stays ~constant.
|
||||
function scaleTo(real: Msg[], n: number): Msg[] {
|
||||
const now = Date.now();
|
||||
const span = WINDOW_DAYS * 864e5;
|
||||
const out: Msg[] = new Array(n);
|
||||
for (let i = 0; i < n; i++) {
|
||||
const r = real[i % real.length]!;
|
||||
out[i] = {
|
||||
id: `m${i}`,
|
||||
ts: now - Math.floor((i / n) * span),
|
||||
role: r.role,
|
||||
text: r.text,
|
||||
};
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function buildMinidb(msgs: Msg[]): Promise<{ db: any; buildMs: number; dir: string }> {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-msg-'));
|
||||
const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false });
|
||||
const t0 = performance.now();
|
||||
// concurrent insert in chunks to avoid one giant promise array
|
||||
const CHUNK = 5000;
|
||||
for (let s = 0; s < msgs.length; s += CHUNK) {
|
||||
const p: Promise<unknown>[] = [];
|
||||
for (let i = s; i < Math.min(s + CHUNK, msgs.length); i++) {
|
||||
const m = msgs[i]!;
|
||||
p.push(db.set(m.id, { role: m.role, text: m.text }, { dt: { ts: m.ts } }));
|
||||
}
|
||||
await Promise.all(p);
|
||||
}
|
||||
return { db, buildMs: performance.now() - t0, dir };
|
||||
}
|
||||
|
||||
function med(fn: () => void, runs = 9): number {
|
||||
const t: number[] = [];
|
||||
for (let i = 0; i < runs; i++) {
|
||||
const t0 = performance.now();
|
||||
fn();
|
||||
t.push(performance.now() - t0);
|
||||
}
|
||||
t.sort((a, b) => a - b);
|
||||
return t[(runs / 2) | 0]!;
|
||||
}
|
||||
|
||||
async function runCase(label: string, msgs: Msg[]) {
|
||||
const since = Date.now() - SINCE_DAYS * 864e5;
|
||||
|
||||
// ---- minidb dt index ----
|
||||
const { db, buildMs, dir } = await buildMinidb(msgs);
|
||||
let indexHits = 0;
|
||||
const indexMs = med(() => {
|
||||
const res = db.dtRange('ts', { gte: since, reverse: true, limit: LIMIT });
|
||||
indexHits = res.length;
|
||||
});
|
||||
// variant: return ALL matches (no limit), to mirror the earlier metadata case
|
||||
let allHits = 0;
|
||||
const indexAllMs = med(() => {
|
||||
const res = db.dtRange('ts', { gte: since });
|
||||
allHits = res.length;
|
||||
});
|
||||
if (global.gc) global.gc();
|
||||
const heap = process.memoryUsage().heapUsed;
|
||||
await db.close();
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
|
||||
// ---- naive baseline ----
|
||||
let naiveHits = 0;
|
||||
const naiveMs = med(() => {
|
||||
const res = msgs.filter((m) => m.ts >= since).sort((a, b) => b.ts - a.ts).slice(0, LIMIT);
|
||||
naiveHits = res.length;
|
||||
});
|
||||
let naiveAllHits = 0;
|
||||
const naiveAllMs = med(() => {
|
||||
const res = msgs.filter((m) => m.ts >= since);
|
||||
naiveAllHits = res.length;
|
||||
});
|
||||
|
||||
console.log(`\n--- ${label}: N=${fmt(msgs.length)} messages ---`);
|
||||
console.log(` build: minidb ${(buildMs / 1000).toFixed(2)}s | heap ${(heap / 1024 / 1024).toFixed(0)} MiB`);
|
||||
console.log(
|
||||
` top-${LIMIT} recent: minidb dtRange ${indexMs.toFixed(3)} ms (${indexHits}) vs naive filter+sort ${naiveMs.toFixed(3)} ms (${naiveHits}) -> ${(naiveMs / indexMs).toFixed(1)}x`,
|
||||
);
|
||||
console.log(
|
||||
` ALL matches: minidb dtRange ${indexAllMs.toFixed(3)} ms (${fmt(allHits)}) vs naive filter ${naiveAllMs.toFixed(3)} ms (${fmt(naiveAllHits)}) -> ${(naiveAllMs / indexAllMs).toFixed(1)}x`,
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const real = loadRealMessages();
|
||||
console.log(`loaded ${fmt(real.length)} real user messages`);
|
||||
const sizes = [real.length, 100_000, 1_000_000];
|
||||
for (const n of sizes) {
|
||||
const msgs = n === real.length ? real : scaleTo(real, n);
|
||||
await runCase(n === real.length ? 'real data' : `scaled`, msgs);
|
||||
}
|
||||
console.log('\ndone.');
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
218
packages/minidb/bench/search-baseline.ts
Normal file
218
packages/minidb/bench/search-baseline.ts
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
// bench/search-baseline.ts
|
||||
//
|
||||
// "Without minidb" search baseline over the SAME real ~/.kimi-code sessions
|
||||
// used by import-kimi-code.ts. Loads every session's extracted text into an
|
||||
// in-memory array, then answers the same queries with a naive full scan
|
||||
// (substring token-AND for text, linear Array.filter for secondary index / dt
|
||||
// range). Paired with import-kimi-code.ts, this shows what minidb's indexes
|
||||
// buy you on real data.
|
||||
//
|
||||
// Run: node --import tsx bench/search-baseline.ts [--data ~/.kimi-code]
|
||||
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const arg = (name: string, def: string) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i === -1 ? def : argv[i + 1]!;
|
||||
};
|
||||
const DATA = path.resolve(arg('data', path.join(os.homedir(), '.kimi-code')));
|
||||
const FULL = argv.includes('--full'); // also index full tool results (matches import-kimi-code --full)
|
||||
|
||||
const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||
const mib = (n: number) => (n / 1024 / 1024).toFixed(1) + ' MiB';
|
||||
|
||||
interface Doc {
|
||||
sessionId: string;
|
||||
title: string;
|
||||
workspaceName: string;
|
||||
text: string;
|
||||
lower: string; // lowercased text, for naive substring search
|
||||
updated: number;
|
||||
created: number;
|
||||
}
|
||||
|
||||
const ARG_FIELDS = ['command', 'pattern', 'path', 'description', 'query', 'prompt', 'file_path'];
|
||||
|
||||
function extractWireText(wirePath: string, full: boolean): string {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = readFileSync(wirePath, 'utf8');
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
const parts: string[] = [];
|
||||
for (const line of raw.split('\n')) {
|
||||
if (!line) continue;
|
||||
let o: any;
|
||||
try {
|
||||
o = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (o.type === 'context.append_message' && o.message && o.message.content) {
|
||||
for (const c of o.message.content)
|
||||
if (c && c.type === 'text' && typeof c.text === 'string') parts.push(c.text);
|
||||
} else if (o.type === 'context.append_loop_event' && o.event && o.event.type === 'tool.call') {
|
||||
const e = o.event;
|
||||
const bits: string[] = [e.name];
|
||||
for (const k of ARG_FIELDS) {
|
||||
const v = e.args && e.args[k];
|
||||
if (typeof v === 'string' && v) bits.push(v.length > 2000 ? v.slice(0, 2000) : v);
|
||||
}
|
||||
parts.push(bits.join(' '));
|
||||
} else if (full && o.type === 'context.append_loop_event' && o.event && o.event.type === 'tool.result') {
|
||||
const r = o.event.result;
|
||||
let out = '';
|
||||
if (typeof r === 'string') out = r;
|
||||
else if (r && typeof r.output === 'string') out = r.output;
|
||||
else if (r && typeof r.content === 'string') out = r.content;
|
||||
if (out) parts.push(out.length > 5000 ? out.slice(0, 5000) : out);
|
||||
}
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
function tokenize(s: string): string[] {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.split(/[\s,，。.!?！?、;；:：()\[\]{}"'<>/\\|@#%^&*+=~`\-_]+/)
|
||||
.filter((t) => t.length > 0);
|
||||
}
|
||||
|
||||
// Naive full-text: a doc matches if every query token appears as a substring
|
||||
// (AND semantics). Score = total occurrences of matched tokens. Returns top N.
|
||||
function naiveTextSearch(docs: Doc[], q: string, limit: number) {
|
||||
const toks = tokenize(q);
|
||||
if (toks.length === 0) return [];
|
||||
const hits: { doc: Doc; score: number }[] = [];
|
||||
for (const d of docs) {
|
||||
let score = 0;
|
||||
let ok = true;
|
||||
for (const t of toks) {
|
||||
let idx = -1;
|
||||
let cnt = 0;
|
||||
while ((idx = d.lower.indexOf(t, idx + 1)) !== -1) cnt++;
|
||||
if (cnt === 0) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
score += cnt;
|
||||
}
|
||||
if (ok) hits.push({ doc: d, score });
|
||||
}
|
||||
hits.sort((a, b) => b.score - a.score);
|
||||
return hits.slice(0, limit);
|
||||
}
|
||||
|
||||
// median of N runs
|
||||
function med<T>(fn: () => T, runs = 7): { value: T; ms: number } {
|
||||
const times: number[] = [];
|
||||
let value!: T;
|
||||
for (let i = 0; i < runs; i++) {
|
||||
const t0 = performance.now();
|
||||
value = fn();
|
||||
times.push(performance.now() - t0);
|
||||
}
|
||||
times.sort((a, b) => a - b);
|
||||
return { value, ms: times[(runs / 2) | 0]! };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log(`data: ${DATA}`);
|
||||
|
||||
const wsRaw = JSON.parse(readFileSync(path.join(DATA, 'workspaces.json'), 'utf8'));
|
||||
const workspaces = wsRaw.workspaces || wsRaw;
|
||||
const lines = readFileSync(path.join(DATA, 'session_index.jsonl'), 'utf8').trim().split('\n');
|
||||
|
||||
const t0 = performance.now();
|
||||
const docs: Doc[] = [];
|
||||
let totalTextBytes = 0;
|
||||
let skipped = 0;
|
||||
for (const line of lines) {
|
||||
let meta: any;
|
||||
try {
|
||||
meta = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const { sessionId, sessionDir } = meta;
|
||||
const wirePath = path.join(sessionDir, 'agents', 'main', 'wire.jsonl');
|
||||
if (!existsSync(wirePath)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
let state: any = {};
|
||||
try {
|
||||
state = JSON.parse(readFileSync(path.join(sessionDir, 'state.json'), 'utf8'));
|
||||
} catch {}
|
||||
const body = extractWireText(wirePath, FULL);
|
||||
const text = (state.title ? state.title + '\n' : '') + body;
|
||||
totalTextBytes += Buffer.byteLength(text, 'utf8');
|
||||
const wsId = path.basename(path.dirname(sessionDir));
|
||||
const ws = workspaces[wsId] || {};
|
||||
docs.push({
|
||||
sessionId,
|
||||
title: state.title || '',
|
||||
workspaceName: ws.name || '',
|
||||
text,
|
||||
lower: text.toLowerCase(),
|
||||
updated: state.updatedAt ? Date.parse(state.updatedAt) : 0,
|
||||
created: state.createdAt ? Date.parse(state.createdAt) : 0,
|
||||
});
|
||||
}
|
||||
const loadMs = performance.now() - t0;
|
||||
if (global.gc) global.gc();
|
||||
console.log(`\n=== loaded (no index) ===`);
|
||||
console.log(` sessions loaded: ${fmt(docs.length)} (skipped ${skipped})`);
|
||||
console.log(` text loaded : ${mib(totalTextBytes)}`);
|
||||
console.log(` load+parse time: ${(loadMs / 1000).toFixed(1)} s`);
|
||||
console.log(` heap used : ${mib(process.memoryUsage().heapUsed)}`);
|
||||
|
||||
// ---- naive full-text searches ----
|
||||
const queries = ['lark-approval', 'database compaction', '北京', 'Redis 持久化', 'worktree init', 'nonexistentxyz123'];
|
||||
console.log(`\n=== naive full-text search (full scan, median of 7) ===`);
|
||||
for (const q of queries) {
|
||||
const { value: res, ms } = med(() => naiveTextSearch(docs, q, 5));
|
||||
console.log(` "${q}" -> ${res.length} hits in ${ms.toFixed(2)} ms`);
|
||||
for (const r of res.slice(0, 3)) {
|
||||
console.log(` [${r.score}] ${r.doc.workspaceName} :: ${r.doc.title.slice(0, 60)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- naive composed query: text includes 'database', then sort ----
|
||||
{
|
||||
const { value: res, ms } = med(() => {
|
||||
const hit = docs.filter((d) => d.lower.includes('database'));
|
||||
hit.sort((a, b) => a.workspaceName.localeCompare(b.workspaceName));
|
||||
return hit.slice(0, 5);
|
||||
});
|
||||
console.log(`\n naive composed (text "database" + sort) -> ${res.length} in ${ms.toFixed(2)} ms`);
|
||||
}
|
||||
|
||||
// ---- naive dt range: updated in last 7 days ----
|
||||
{
|
||||
const week = Date.now() - 7 * 864e5;
|
||||
const { value: res, ms } = med(() => {
|
||||
const hit = docs.filter((d) => d.updated >= week);
|
||||
hit.sort((a, b) => b.updated - a.updated);
|
||||
return hit.slice(0, 10);
|
||||
});
|
||||
console.log(` naive dt range (updated in last 7d) -> ${res.length} shown in ${ms.toFixed(2)} ms`);
|
||||
}
|
||||
|
||||
// ---- naive secondary index lookup: workspaceName ----
|
||||
{
|
||||
const { value: res, ms } = med(() => docs.filter((d) => d.workspaceName === 'kimi-code-dev-1'));
|
||||
console.log(` naive lookup (workspace=kimi-code-dev-1) -> ${res.length} in ${ms.toFixed(2)} ms`);
|
||||
}
|
||||
|
||||
console.log(`\ndone.`);
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
214
packages/minidb/bench/session-children.ts
Normal file
214
packages/minidb/bench/session-children.ts
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
// bench/session-children.ts
|
||||
//
|
||||
// Benchmark for nested session "children" lookups.
|
||||
//
|
||||
// Scenario
|
||||
// --------
|
||||
// Sessions form a tree (root -> child -> grandchild -> ...). Each session
|
||||
// stores its OWN id as the key (`sess:<long-id>`) and the parent link in the
|
||||
// value (`value.metadata.parent_session_id`). The relationship is NOT encoded
|
||||
// in the key, because session ids are long and a materialized path would blow
|
||||
// the 128-char key cap and duplicate the whole ancestor chain on every leaf.
|
||||
//
|
||||
// We compare two ways to answer "list the direct children of session X":
|
||||
//
|
||||
// 1. indexed — compound index { groupBy: 'metadata.parent_session_id',
|
||||
// orderBy: 'updatedAt' } + db.compoundRange('byParent', X, ...)
|
||||
// -> O(log N + fanout)
|
||||
//
|
||||
// 2. scan — the legacy approach: db.scan() everything, filter by
|
||||
// metadata.parent_session_id === X, sort by updatedAt desc.
|
||||
// -> O(N log N)
|
||||
//
|
||||
// Run
|
||||
// ---
|
||||
// node --import tsx bench/session-children.ts
|
||||
//
|
||||
// Knobs (env):
|
||||
// TOTAL total sessions (default 1_000)
|
||||
// FANOUT children per parent (default 30)
|
||||
// MAX_DEPTH deepest generation, root=0 (default 3)
|
||||
// ITERS query iterations to avg (default 500)
|
||||
//
|
||||
// With TOTAL=1000, FANOUT=30, MAX_DEPTH=3 the tree is exactly:
|
||||
// depth0: 1, depth1: 30, depth2: 900, depth3: 69 (1000 total)
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { MiniDb } from '../src/index.js';
|
||||
|
||||
const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 });
|
||||
const ops = (n: number, ms: number) => `${fmt((n / ms) * 1000)} ops/s`;
|
||||
|
||||
const TOTAL = Number(process.env.TOTAL || 1_000);
|
||||
const FANOUT = Number(process.env.FANOUT || 30);
|
||||
const MAX_DEPTH = Number(process.env.MAX_DEPTH || 3);
|
||||
const ITERS = Number(process.env.ITERS || 500);
|
||||
|
||||
interface Node {
|
||||
id: string;
|
||||
parentId: string | null;
|
||||
depth: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
// Long-ish session ids: `sess_` + 40 hex chars (45 chars total).
|
||||
const makeId = (i: number) => `sess_${i.toString(16).padStart(40, '0')}`;
|
||||
|
||||
function buildTree(total: number, fanout: number, maxDepth: number): Node[] {
|
||||
const nodes: Node[] = [];
|
||||
const base = Date.parse('2024-01-01');
|
||||
let counter = 0;
|
||||
const root: Node = { id: makeId(counter++), parentId: null, depth: 0, updatedAt: base + counter };
|
||||
nodes.push(root);
|
||||
const queue: Node[] = [root];
|
||||
while (queue.length && nodes.length < total) {
|
||||
const parent = queue.shift()!;
|
||||
if (parent.depth >= maxDepth) continue;
|
||||
for (let i = 0; i < fanout && nodes.length < total; i++) {
|
||||
const child: Node = {
|
||||
id: makeId(counter++),
|
||||
parentId: parent.id,
|
||||
depth: parent.depth + 1,
|
||||
updatedAt: base + counter,
|
||||
};
|
||||
nodes.push(child);
|
||||
queue.push(child);
|
||||
}
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
async function tmpDir() {
|
||||
return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-children-'));
|
||||
}
|
||||
|
||||
async function timeIt<T>(fn: () => T | Promise<T>, iters: number): Promise<{ ms: number; last: T }> {
|
||||
let last!: T;
|
||||
const t0 = performance.now();
|
||||
for (let i = 0; i < iters; i++) last = await fn();
|
||||
return { ms: performance.now() - t0, last };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log(
|
||||
`\nminidb session-children benchmark (TOTAL=${fmt(TOTAL)}, FANOUT=${FANOUT}, MAX_DEPTH=${MAX_DEPTH}, ITERS=${ITERS}, node ${process.version})\n`,
|
||||
);
|
||||
|
||||
// ---- build the tree -----------------------------------------------------
|
||||
const nodes = buildTree(TOTAL, FANOUT, MAX_DEPTH);
|
||||
const byDepth = new Map<number, number>();
|
||||
const childrenOf = new Map<string | null, string[]>();
|
||||
for (const n of nodes) {
|
||||
byDepth.set(n.depth, (byDepth.get(n.depth) ?? 0) + 1);
|
||||
const arr = childrenOf.get(n.parentId) ?? [];
|
||||
arr.push(n.id);
|
||||
childrenOf.set(n.parentId, arr);
|
||||
}
|
||||
const maxDepth = Math.max(...byDepth.keys());
|
||||
console.log(` tree: ${fmt(nodes.length)} sessions, max depth ${maxDepth}`);
|
||||
for (const [d, c] of [...byDepth.entries()].sort((a, b) => a[0] - b[0])) {
|
||||
console.log(` depth ${d}: ${fmt(c)} sessions`);
|
||||
}
|
||||
|
||||
// Pick a parent at each depth that actually has children, for depth
|
||||
// coverage in the query benchmark.
|
||||
const parentAtDepth = new Map<number, Node>();
|
||||
for (const n of nodes) {
|
||||
if (!parentAtDepth.has(n.depth) && (childrenOf.get(n.id)?.length ?? 0) > 0) parentAtDepth.set(n.depth, n);
|
||||
}
|
||||
|
||||
// ---- open + ingest ------------------------------------------------------
|
||||
const dir = await tmpDir();
|
||||
const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false });
|
||||
|
||||
const tIngest = performance.now();
|
||||
const CHUNK = 500;
|
||||
for (let i = 0; i < nodes.length; i += CHUNK) {
|
||||
const slice = nodes.slice(i, i + CHUNK);
|
||||
await db.batch(
|
||||
slice.map((n) => ({
|
||||
op: 'set' as const,
|
||||
key: n.id,
|
||||
value: {
|
||||
title: `session ${n.id}`,
|
||||
workspaceId: 'ws_bench',
|
||||
metadata: {
|
||||
// Only real children carry parent_session_id (+ kind). Roots do
|
||||
// not, so they stay out of the byParent index entirely.
|
||||
...(n.parentId ? { parent_session_id: n.parentId, child_session_kind: 'child' } : {}),
|
||||
},
|
||||
},
|
||||
dt: { updatedAt: n.updatedAt },
|
||||
})),
|
||||
);
|
||||
}
|
||||
const ingestMs = performance.now() - tIngest;
|
||||
console.log(`\n ingest ${fmt(nodes.length)} sessions (batch)`.padEnd(46), `${ingestMs.toFixed(1).padStart(8)} ms`, `-> ${ops(nodes.length, ingestMs)}`);
|
||||
|
||||
// ---- index build (the thing that makes children fast) -------------------
|
||||
const tIdx = performance.now();
|
||||
await db.createCompoundIndex('byParent', {
|
||||
groupBy: 'metadata.parent_session_id',
|
||||
orderBy: 'updatedAt',
|
||||
});
|
||||
const idxMs = performance.now() - tIdx;
|
||||
console.log(` createCompoundIndex('byParent') + rebuild`.padEnd(46), `${idxMs.toFixed(1).padStart(8)} ms`);
|
||||
|
||||
// ---- query benchmarks ---------------------------------------------------
|
||||
const PAGE = 20;
|
||||
console.log(`\n listChildren(parent) page_size=${PAGE}, averaged over ${ITERS} iters:\n`);
|
||||
|
||||
for (const [depth, parent] of [...parentAtDepth.entries()].sort((a, b) => a[0] - b[0])) {
|
||||
const fanout = childrenOf.get(parent.id)!.length;
|
||||
|
||||
// indexed: O(log N + fanout)
|
||||
const idx = await timeIt(
|
||||
() => db.compoundRange('byParent', parent.id, { reverse: true, limit: PAGE }),
|
||||
ITERS,
|
||||
);
|
||||
// scan (legacy): O(N log N)
|
||||
const scan = await timeIt(() => {
|
||||
const all = db.scan();
|
||||
const filtered = all.filter(
|
||||
(r) =>
|
||||
(r.value as { metadata?: Record<string, unknown> })?.metadata?.['parent_session_id'] === parent.id &&
|
||||
(r.value as { metadata?: Record<string, unknown> })?.metadata?.['child_session_kind'] === 'child',
|
||||
);
|
||||
filtered.sort((a, b) => (b.dt?.updatedAt ?? 0) - (a.dt?.updatedAt ?? 0));
|
||||
return filtered.slice(0, PAGE);
|
||||
}, ITERS);
|
||||
|
||||
const idxAvg = idx.ms / ITERS;
|
||||
const scanAvg = scan.ms / ITERS;
|
||||
console.log(
|
||||
` depth ${depth} parent (fanout=${fmt(fanout)}):`.padEnd(38),
|
||||
`indexed ${idxAvg.toFixed(3)} ms/op`.padEnd(22),
|
||||
`scan ${scanAvg.toFixed(3)} ms/op`.padEnd(20),
|
||||
`speedup x${(scanAvg / idxAvg).toFixed(1)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// ---- correctness spot-check ---------------------------------------------
|
||||
const root = nodes[0]!;
|
||||
const idxChildren = db.compoundRange('byParent', root.id, { reverse: true, limit: 10_000 }).map((r) => r.key);
|
||||
const expected = new Set(childrenOf.get(root.id)!);
|
||||
const ok = idxChildren.length === expected.size && idxChildren.every((k) => expected.has(k));
|
||||
console.log(`\n correctness: indexed children of root == expected ${ok ? 'OK' : 'MISMATCH'} (${idxChildren.length}/${expected.size})`);
|
||||
|
||||
// ---- storage snapshot ---------------------------------------------------
|
||||
await db.compact();
|
||||
const snap = await fs.stat(path.join(dir, 'db.snapshot'));
|
||||
const heapMiB = (process.memoryUsage().heapUsed / 1024 / 1024).toFixed(1);
|
||||
console.log(` storage: ${(snap.size / 1024 / 1024).toFixed(2)} MiB snapshot, heap ${heapMiB} MiB`);
|
||||
|
||||
await db.close();
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
console.log('\ndone.\n');
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -80,6 +80,17 @@ export class DtIndex {
|
|||
return c.list.range(opts).map((n: RangeEntry<number, string>) => ({ key: n.val, value: n.key }));
|
||||
}
|
||||
|
||||
/** Lazy range over a dt column; yields { key: recordKey, value: ts } like
|
||||
* range() but lets the caller stop early without materializing everything. */
|
||||
*iterate(
|
||||
col: string,
|
||||
opts: Parameters<SkipList<number, string>['iterate']>[0] = {},
|
||||
): Generator<DtRangeEntry> {
|
||||
const c = this.cols.get(col);
|
||||
if (!c) return;
|
||||
for (const n of c.list.iterate(opts)) yield { key: n.val, value: n.key };
|
||||
}
|
||||
|
||||
columns(): string[] {
|
||||
return [...this.cols.keys()];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -274,6 +274,15 @@ export class IndexManager {
|
|||
return set ? [...set] : [];
|
||||
}
|
||||
|
||||
/** O(1) membership test: is `pk` indexed under `value` on this equality
|
||||
* index? Avoids materializing the full posting list like findEq does. */
|
||||
hasEq(name: string, value: unknown, pk: string): boolean {
|
||||
const idx = this.get(name);
|
||||
if (idx.type !== 'equality') throw new Error(`index "${name}" is not an equality index`);
|
||||
const set = idx.map.get(scalarKey(value));
|
||||
return !!set && set.has(pk);
|
||||
}
|
||||
|
||||
findRange(
|
||||
name: string,
|
||||
opts: { min?: number; max?: number; minExclusive?: boolean; maxExclusive?: boolean; offset?: number; count?: number; reverse?: boolean } = {},
|
||||
|
|
|
|||
|
|
@ -1079,10 +1079,102 @@ export class MiniDb<V = unknown> {
|
|||
return [...candidates];
|
||||
}
|
||||
|
||||
// Extract simple equality predicates (top-level or inside $and) that are
|
||||
// backed by an equality index, for use as a cheap per-candidate pre-filter.
|
||||
// Only direct equality and {$eq: x} qualify; $in / range / non-indexed fields
|
||||
// are left to the full match() after decode.
|
||||
private cheapEqChecks(filter?: Record<string, unknown>): { name: string; value: unknown }[] {
|
||||
const out: { name: string; value: unknown }[] = [];
|
||||
if (!filter || typeof filter !== 'object' || !this.indexes.indexes.size) return out;
|
||||
for (const { field, cond } of this.indexPredicates(filter)) {
|
||||
const idx = this.indexes.list().find((i) => i.field === field && i.type === 'equality');
|
||||
if (!idx) continue;
|
||||
if (cond !== null && typeof cond === 'object' && !(cond instanceof RegExp)) {
|
||||
const ops = cond as Record<string, unknown>;
|
||||
if (Object.keys(ops).length === 1 && '$eq' in ops) out.push({ name: idx.name, value: ops['$eq'] });
|
||||
} else {
|
||||
out.push({ name: idx.name, value: cond });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Fast path: a query bounded by a single dt column whose result order is that
|
||||
// dt column can walk the dt skiplist in order and stop as soon as `limit`
|
||||
// qualifying rows are found, instead of materializing + decoding + sorting the
|
||||
// whole candidate set. Returns null when the query is not eligible (caller
|
||||
// falls back to the general path). Kept conservative so results match exactly.
|
||||
private tryDtOrderedLimit(q: QueryOptions): ScanEntry<V>[] | null {
|
||||
if (q.text) return null; // text has its own ranking
|
||||
if (q.key !== undefined) return null;
|
||||
if (q.limit === undefined) return null; // unbounded -> full return, no win
|
||||
if (!q.dt) return null;
|
||||
const dtCols = Object.keys(q.dt);
|
||||
if (dtCols.length !== 1) return null;
|
||||
const col = dtCols[0]!;
|
||||
const cond = q.dt[col]!;
|
||||
|
||||
// Result order must be the dt column's order.
|
||||
let reverse = false;
|
||||
if (q.sort) {
|
||||
const entries = Object.entries(q.sort);
|
||||
if (entries.length !== 1) return null;
|
||||
const [sortKey, dir] = entries[0]!;
|
||||
if (sortKey !== col) return null;
|
||||
reverse = dir < 0;
|
||||
}
|
||||
|
||||
const limit = q.limit;
|
||||
const skip = q.skip ?? 0;
|
||||
// Only the range bounds are honored here; ignore any stray count/offset a
|
||||
// caller put on the dt cond so they cannot truncate the walk prematurely.
|
||||
const iterOpts: RangeOptions<number> = { reverse };
|
||||
if (cond.gte !== undefined) iterOpts.gte = cond.gte;
|
||||
if (cond.gt !== undefined) iterOpts.gt = cond.gt;
|
||||
if (cond.lte !== undefined) iterOpts.lte = cond.lte;
|
||||
if (cond.lt !== undefined) iterOpts.lt = cond.lt;
|
||||
|
||||
// Cheap key-level pre-filter (no decode, no full-set materialization) for
|
||||
// simple equality predicates that have an equality index.
|
||||
const eqChecks = this.cheapEqChecks(q.filter);
|
||||
|
||||
const out: { key: string; value: V; dt: Record<string, number> | undefined }[] = [];
|
||||
let skipped = 0;
|
||||
for (const { key: kstr } of this.dt.iterate(col, iterOpts)) {
|
||||
let rejected = false;
|
||||
for (const c of eqChecks) {
|
||||
if (!this.indexes.hasEq(c.name, c.value, kstr)) {
|
||||
rejected = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (rejected) continue;
|
||||
const buf = this.store.get(kstr);
|
||||
if (buf === undefined) continue;
|
||||
const r = this.store.map.get(kstr);
|
||||
const value = this.decode(buf)!;
|
||||
if (q.filter && !match(value, q.filter)) continue;
|
||||
if (skipped < skip) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
out.push({ key: kstr, value, dt: r?.dt ?? undefined });
|
||||
if (out.length >= limit) break;
|
||||
}
|
||||
|
||||
return out.map((d) => ({
|
||||
key: fromKStr(d.key),
|
||||
value: q.project ? (project(d.value, q.project) as V) : d.value,
|
||||
dt: d.dt,
|
||||
}));
|
||||
}
|
||||
|
||||
// ---- unified query ------------------------------------------------------
|
||||
|
||||
query(q: QueryOptions = {}): ScanEntry<V>[] {
|
||||
this.ensureOpen();
|
||||
const fast = this.tryDtOrderedLimit(q);
|
||||
if (fast !== null) return fast;
|
||||
|
||||
let keys: string[] | null = null;
|
||||
if (typeof q.key === 'string') {
|
||||
|
|
|
|||
|
|
@ -250,6 +250,53 @@ export class SkipList<K = number, V = string> {
|
|||
return out;
|
||||
}
|
||||
|
||||
/** Lazy range scan. Same bounds/offset/count/reverse semantics as range(),
|
||||
* but yields entries one by one so a caller can stop early without
|
||||
* materializing the whole range. */
|
||||
*iterate(opts: RangeOptions<K> = {}): Generator<RangeEntry<K, V>> {
|
||||
let offset = opts.offset ?? 0;
|
||||
let count = opts.count ?? Infinity;
|
||||
|
||||
if (opts.reverse) {
|
||||
let x: SkipNode<K, V> | null;
|
||||
if (opts.lte !== undefined) {
|
||||
const after = this.lowerBound(opts.lte, { strict: true });
|
||||
x = after ? after.backward : this.tail;
|
||||
} else if (opts.lt !== undefined) {
|
||||
const after = this.lowerBound(opts.lt, { strict: false });
|
||||
x = after ? after.backward : this.tail;
|
||||
} else {
|
||||
x = this.tail;
|
||||
}
|
||||
while (x) {
|
||||
if (opts.gte !== undefined && this.cmpK(x.key, opts.gte) < 0) break;
|
||||
if (opts.gt !== undefined && this.cmpK(x.key, opts.gt) <= 0) break;
|
||||
if (offset > 0) offset--;
|
||||
else if (count > 0) {
|
||||
yield { key: x.key, val: x.val };
|
||||
count--;
|
||||
} else break;
|
||||
x = x.backward;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const hasLower = opts.gte !== undefined || opts.gt !== undefined;
|
||||
let x = hasLower
|
||||
? this.lowerBound(opts.gte !== undefined ? opts.gte : (opts.gt as K), { strict: opts.gt !== undefined })
|
||||
: this.header.level[0]!.forward;
|
||||
while (x) {
|
||||
if (opts.lte !== undefined && this.cmpK(x.key, opts.lte) > 0) break;
|
||||
if (opts.lt !== undefined && this.cmpK(x.key, opts.lt) >= 0) break;
|
||||
if (offset > 0) offset--;
|
||||
else if (count > 0) {
|
||||
yield { key: x.key, val: x.val };
|
||||
count--;
|
||||
} else break;
|
||||
x = x.level[0]!.forward;
|
||||
}
|
||||
}
|
||||
|
||||
toArray(): RangeEntry<K, V>[] {
|
||||
const out: RangeEntry<K, V>[] = [];
|
||||
let x = this.header.level[0]!.forward;
|
||||
|
|
|
|||
|
|
@ -30,7 +30,9 @@ export function tokenize(str: unknown): string[] {
|
|||
const s = String(str).toLowerCase();
|
||||
const terms: string[] = [];
|
||||
const latin = s.match(LATIN);
|
||||
if (latin) terms.push(...latin);
|
||||
// Loop-push instead of `terms.push(...latin)`: spreading a large match array
|
||||
// (hundreds of thousands of tokens from a big doc) overflows the call stack.
|
||||
if (latin) for (const t of latin) terms.push(t);
|
||||
const runs = s.match(CJK) ?? [];
|
||||
for (const r of runs) {
|
||||
for (let i = 0; i < r.length; i++) {
|
||||
|
|
|
|||
|
|
@ -104,6 +104,76 @@ test('query composes dt range + value filter + sort + limit + project', async ()
|
|||
}
|
||||
});
|
||||
|
||||
test('dt-ordered limit fast path matches a reference (no ties)', async () => {
|
||||
const dir = await tmpDir();
|
||||
try {
|
||||
const db = await MiniDb.open({ dir, valueCodec: 'json' });
|
||||
await db.createIndex('byRole', { field: 'role' });
|
||||
const docs: { key: string; role: string; n: number; ts: number }[] = [];
|
||||
for (let i = 1; i <= 20; i++) {
|
||||
const ts = i * 100; // unique
|
||||
const role = i % 2 ? 'user' : 'assistant';
|
||||
await db.set(`d${i}`, { role, n: i }, { dt: { ts } });
|
||||
docs.push({ key: `d${i}`, role, n: i, ts });
|
||||
}
|
||||
const gte = 200;
|
||||
const lte = 1500;
|
||||
|
||||
const refDesc = docs
|
||||
.filter((d) => d.role === 'user' && d.ts >= gte && d.ts <= lte)
|
||||
.sort((a, b) => b.ts - a.ts);
|
||||
|
||||
// sort by dt desc + limit -> fast path
|
||||
assert.deepEqual(
|
||||
db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, sort: { ts: -1 }, limit: 3 }).map((r) => r.key),
|
||||
refDesc.slice(0, 3).map((d) => d.key),
|
||||
);
|
||||
// ascending + skip + limit -> fast path
|
||||
const refAsc = [...refDesc].reverse();
|
||||
assert.deepEqual(
|
||||
db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, sort: { ts: 1 }, skip: 1, limit: 2 }).map((r) => r.key),
|
||||
refAsc.slice(1, 3).map((d) => d.key),
|
||||
);
|
||||
// no sort -> defaults to dt ascending, still fast path
|
||||
assert.deepEqual(
|
||||
db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, limit: 2 }).map((r) => r.key),
|
||||
refAsc.slice(0, 2).map((d) => d.key),
|
||||
);
|
||||
// project still applies on the fast path
|
||||
const proj = db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, sort: { ts: -1 }, limit: 1, project: ['n'] });
|
||||
assert.deepEqual(proj[0], { key: refDesc[0]!.key, value: { n: refDesc[0]!.n }, dt: { ts: refDesc[0]!.ts } });
|
||||
// sort by a non-dt field -> falls back to general path (still correct)
|
||||
assert.deepEqual(
|
||||
db.query({ dt: { ts: { gte, lte } }, filter: { role: 'user' }, sort: { n: -1 }, limit: 3 }).map((r) => r.key),
|
||||
refDesc.slice(0, 3).map((d) => d.key), // n == i order matches ts here
|
||||
);
|
||||
await db.close();
|
||||
} finally {
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('dt-ordered limit fast path orders equal ts by key (tie-break)', async () => {
|
||||
const dir = await tmpDir();
|
||||
try {
|
||||
const db = await MiniDb.open({ dir, valueCodec: 'json' });
|
||||
// several docs share the same ts; SkipList tie-break is by record key
|
||||
for (const k of ['t3', 't1', 't4', 't2']) await db.set(k, { v: k }, { dt: { ts: 1000 } });
|
||||
// descending -> key desc; ascending -> key asc
|
||||
assert.deepEqual(
|
||||
db.query({ dt: { ts: { gte: 1000, lte: 1000 } }, sort: { ts: -1 }, limit: 4 }).map((r) => r.key),
|
||||
['t4', 't3', 't2', 't1'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
db.query({ dt: { ts: { gte: 1000, lte: 1000 } }, sort: { ts: 1 }, limit: 2 }).map((r) => r.key),
|
||||
['t1', 't2'],
|
||||
);
|
||||
await db.close();
|
||||
} finally {
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('full-text search: latin + CJK', async () => {
|
||||
const dir = await tmpDir();
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -71,6 +71,38 @@ test('string comparator orders keys and supports prefix scans', () => {
|
|||
assert.deepEqual(res, ['user:1', 'user:10', 'user:2', 'user:3']);
|
||||
});
|
||||
|
||||
test('iterate() yields the same sequence as range()', () => {
|
||||
const sl = new SkipList();
|
||||
for (let i = 1; i <= 20; i++) sl.insert(i, `v${i}`);
|
||||
const cases = [
|
||||
{},
|
||||
{ gte: 3, lte: 7 },
|
||||
{ gt: 3, lt: 7 },
|
||||
{ gte: 1, lte: 20, offset: 2, count: 5 },
|
||||
{ gte: 5 },
|
||||
{ lte: 5 },
|
||||
{ gte: 3, lte: 7, reverse: true },
|
||||
{ reverse: true, offset: 2, count: 4 },
|
||||
{ gt: 3, lt: 7, reverse: true },
|
||||
{ gte: 100 }, // empty
|
||||
{ lte: 0 }, // empty
|
||||
];
|
||||
for (const opts of cases) {
|
||||
assert.deepEqual([...sl.iterate(opts)], sl.range(opts), `opts=${JSON.stringify(opts)}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('iterate() stops early (lazy)', () => {
|
||||
const sl = new SkipList();
|
||||
for (let i = 1; i <= 1000; i++) sl.insert(i, `v${i}`);
|
||||
let seen = 0;
|
||||
for (const _ of sl.iterate({ gte: 1 })) {
|
||||
seen++;
|
||||
if (seen === 3) break;
|
||||
}
|
||||
assert.equal(seen, 3);
|
||||
});
|
||||
|
||||
test('matches a sorted reference under random operations', () => {
|
||||
const sl = new SkipList();
|
||||
const ref = new Map(); // val -> key
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue