fix(rvf): size branch membership by vector IDs

This commit is contained in:
ruvnet 2026-07-28 00:31:28 -04:00
parent 86da05db61
commit eebea1e50e
12 changed files with 95 additions and 16 deletions

2
Cargo.lock generated
View file

@ -11498,7 +11498,7 @@ dependencies = [
[[package]]
name = "rvf-runtime"
version = "0.3.1"
version = "0.3.2"
dependencies = [
"crc32fast",
"rvf-index",

2
crates/rvf/Cargo.lock generated
View file

@ -1775,7 +1775,7 @@ dependencies = [
[[package]]
name = "rvf-runtime"
version = "0.3.1"
version = "0.3.2"
dependencies = [
"crc32fast",
"rand",

View file

@ -15,7 +15,7 @@ rust-version = "1.87"
crate-type = ["cdylib"]
[dependencies]
rvf-runtime = { version = "0.3.1", path = "../rvf-runtime" }
rvf-runtime = { version = "0.3.2", path = "../rvf-runtime" }
rvf-types = { version = "0.2.0", path = "../rvf-types" }
napi = { version = "2", features = ["async"] }
napi-derive = "2"

View file

@ -1,6 +1,6 @@
{
"name": "@ruvector/rvf-node-darwin-arm64",
"version": "0.2.1",
"version": "0.2.2",
"cpu": [
"arm64"
],

View file

@ -1,6 +1,6 @@
{
"name": "@ruvector/rvf-node-darwin-x64",
"version": "0.2.1",
"version": "0.2.2",
"cpu": [
"x64"
],

View file

@ -1,6 +1,6 @@
{
"name": "@ruvector/rvf-node-linux-arm64-gnu",
"version": "0.2.1",
"version": "0.2.2",
"cpu": [
"arm64"
],

View file

@ -1,6 +1,6 @@
{
"name": "@ruvector/rvf-node-linux-x64-gnu",
"version": "0.2.1",
"version": "0.2.2",
"cpu": [
"x64"
],

View file

@ -1,6 +1,6 @@
{
"name": "@ruvector/rvf-node-win32-x64-msvc",
"version": "0.2.1",
"version": "0.2.2",
"cpu": [
"x64"
],

View file

@ -1,6 +1,6 @@
{
"name": "@ruvector/rvf-node",
"version": "0.2.2",
"version": "0.2.3",
"description": "RuVector Format Node.js native bindings",
"main": "index.js",
"types": "index.d.ts",
@ -28,10 +28,10 @@
"@napi-rs/cli": "^2.18.0"
},
"optionalDependencies": {
"@ruvector/rvf-node-win32-x64-msvc": "0.2.1",
"@ruvector/rvf-node-darwin-x64": "0.2.1",
"@ruvector/rvf-node-darwin-arm64": "0.2.1",
"@ruvector/rvf-node-linux-x64-gnu": "0.2.1",
"@ruvector/rvf-node-linux-arm64-gnu": "0.2.1"
"@ruvector/rvf-node-win32-x64-msvc": "0.2.2",
"@ruvector/rvf-node-darwin-x64": "0.2.2",
"@ruvector/rvf-node-darwin-arm64": "0.2.2",
"@ruvector/rvf-node-linux-x64-gnu": "0.2.2",
"@ruvector/rvf-node-linux-arm64-gnu": "0.2.2"
}
}

View file

@ -1,6 +1,6 @@
[package]
name = "rvf-runtime"
version = "0.3.1"
version = "0.3.2"
edition = "2021"
description = "RuVector Format runtime -- RvfStore API, compaction, and streaming I/O"
license = "MIT OR Apache-2.0"

View file

@ -44,6 +44,9 @@ fn err(code: ErrorCode) -> RvfError {
const COW_STATE_FIXED_SIZE: usize = 64 + 4 + 1 + 4 + 4 + 4;
const MAX_COW_LINEAGE_DEPTH: u32 = 1024;
/// Bound dense membership bitmaps so a sparse, attacker-controlled vector ID
/// cannot turn branch creation into an unbounded allocation.
const MAX_MEMBERSHIP_FILTER_BYTES: u64 = 64 * 1024 * 1024;
fn relative_parent_reference(child_path: &Path, parent_path: &Path) -> Result<PathBuf, RvfError> {
let child_dir = child_path
@ -2265,6 +2268,30 @@ impl RvfStore {
.checked_next_power_of_two()
.ok_or_else(|| err(ErrorCode::CowMapCorrupt))?;
let total_vecs = self.vectors.len() as u64;
// MembershipFilter is keyed by vector ID, not by slab ordinal. Sizing
// it from `len()` silently drops ID == len (the normal Node string-ID
// case, which starts at 1) and every sparse ID. Size from the highest
// live ID instead, while bounding the dense bitmap to fail closed
// rather than risk an allocation DoS.
let membership_capacity = self
.vectors
.ids()
.copied()
.max()
.map(|max_id| {
max_id
.checked_add(1)
.ok_or_else(|| err(ErrorCode::MembershipInvalid))
})
.transpose()?
.unwrap_or(0);
let membership_bytes = membership_capacity
.div_ceil(64)
.checked_mul(8)
.ok_or_else(|| err(ErrorCode::MembershipInvalid))?;
if membership_bytes > MAX_MEMBERSHIP_FILTER_BYTES {
return Err(err(ErrorCode::MembershipInvalid));
}
let cluster_count = if vectors_per_cluster > 0 {
total_vecs.div_ceil(vectors_per_cluster as u64) as u32
} else {
@ -2287,7 +2314,7 @@ impl RvfStore {
));
// Initialize membership filter with all parent vectors visible
let mut filter = MembershipFilter::new_include(total_vecs);
let mut filter = MembershipFilter::new_include(membership_capacity);
for &vid in self.vectors.ids() {
if !self.deletion_bitmap.is_deleted(vid) {
filter.add(vid);

View file

@ -500,6 +500,58 @@ fn branch_query_reads_through_to_parent() {
);
}
/// Membership bitmaps are keyed by vector ID, so count-based capacity drops
/// normal one-based and sparse IDs even though the vectors exist.
#[test]
fn branch_query_reads_one_based_and_sparse_parent_ids() {
let dir = TempDir::new().unwrap();
let base_path = dir.path().join("base_sparse.rvf");
let child_path = dir.path().join("child_sparse.rvf");
let dim: u16 = 4;
let one_based = vec![1.0, 0.0, 0.0, 0.0];
let sparse = vec![0.0, 1.0, 0.0, 0.0];
let mut base = RvfStore::create(&base_path, make_options(dim)).unwrap();
base.ingest_batch(
&[one_based.as_slice(), sparse.as_slice()],
&[1, 10_000],
None,
)
.unwrap();
base.freeze().unwrap();
let child = base.branch(&child_path).unwrap();
let opts = QueryOptions::default();
assert_eq!(child.query(&one_based, 1, &opts).unwrap()[0].id, 1);
assert_eq!(child.query(&sparse, 1, &opts).unwrap()[0].id, 10_000);
child.close().unwrap();
base.close().unwrap();
}
/// A hostile sparse ID must fail branch creation instead of overflowing or
/// attempting an unbounded dense-bitmap allocation.
#[test]
fn branch_rejects_unbounded_membership_capacity() {
let dir = TempDir::new().unwrap();
let base_path = dir.path().join("base_hostile_id.rvf");
let child_path = dir.path().join("child_hostile_id.rvf");
let vector = vec![1.0, 0.0, 0.0, 0.0];
let mut base = RvfStore::create(&base_path, make_options(4)).unwrap();
base.ingest_batch(&[vector.as_slice()], &[u64::MAX], None)
.unwrap();
base.freeze().unwrap();
assert!(matches!(
base.branch(&child_path),
Err(rvf_types::RvfError::Code(
rvf_types::ErrorCode::MembershipInvalid
))
));
base.close().unwrap();
}
// ===========================================================================
// TEST 10: branch_state_survives_reopen_and_relocation
// ===========================================================================