mirror of
https://github.com/ruvnet/RuVector.git
synced 2026-07-31 20:14:01 +00:00
fix(rvf): size branch membership by vector IDs
This commit is contained in:
parent
86da05db61
commit
eebea1e50e
12 changed files with 95 additions and 16 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -11498,7 +11498,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rvf-runtime"
|
||||
version = "0.3.1"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"rvf-index",
|
||||
|
|
|
|||
2
crates/rvf/Cargo.lock
generated
2
crates/rvf/Cargo.lock
generated
|
|
@ -1775,7 +1775,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rvf-runtime"
|
||||
version = "0.3.1"
|
||||
version = "0.3.2"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"rand",
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ rust-version = "1.87"
|
|||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
rvf-runtime = { version = "0.3.1", path = "../rvf-runtime" }
|
||||
rvf-runtime = { version = "0.3.2", path = "../rvf-runtime" }
|
||||
rvf-types = { version = "0.2.0", path = "../rvf-types" }
|
||||
napi = { version = "2", features = ["async"] }
|
||||
napi-derive = "2"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@ruvector/rvf-node-darwin-arm64",
|
||||
"version": "0.2.1",
|
||||
"version": "0.2.2",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@ruvector/rvf-node-darwin-x64",
|
||||
"version": "0.2.1",
|
||||
"version": "0.2.2",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@ruvector/rvf-node-linux-arm64-gnu",
|
||||
"version": "0.2.1",
|
||||
"version": "0.2.2",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@ruvector/rvf-node-linux-x64-gnu",
|
||||
"version": "0.2.1",
|
||||
"version": "0.2.2",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@ruvector/rvf-node-win32-x64-msvc",
|
||||
"version": "0.2.1",
|
||||
"version": "0.2.2",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@ruvector/rvf-node",
|
||||
"version": "0.2.2",
|
||||
"version": "0.2.3",
|
||||
"description": "RuVector Format Node.js native bindings",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
|
@ -28,10 +28,10 @@
|
|||
"@napi-rs/cli": "^2.18.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@ruvector/rvf-node-win32-x64-msvc": "0.2.1",
|
||||
"@ruvector/rvf-node-darwin-x64": "0.2.1",
|
||||
"@ruvector/rvf-node-darwin-arm64": "0.2.1",
|
||||
"@ruvector/rvf-node-linux-x64-gnu": "0.2.1",
|
||||
"@ruvector/rvf-node-linux-arm64-gnu": "0.2.1"
|
||||
"@ruvector/rvf-node-win32-x64-msvc": "0.2.2",
|
||||
"@ruvector/rvf-node-darwin-x64": "0.2.2",
|
||||
"@ruvector/rvf-node-darwin-arm64": "0.2.2",
|
||||
"@ruvector/rvf-node-linux-x64-gnu": "0.2.2",
|
||||
"@ruvector/rvf-node-linux-arm64-gnu": "0.2.2"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "rvf-runtime"
|
||||
version = "0.3.1"
|
||||
version = "0.3.2"
|
||||
edition = "2021"
|
||||
description = "RuVector Format runtime -- RvfStore API, compaction, and streaming I/O"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
|
|
|||
|
|
@ -44,6 +44,9 @@ fn err(code: ErrorCode) -> RvfError {
|
|||
|
||||
const COW_STATE_FIXED_SIZE: usize = 64 + 4 + 1 + 4 + 4 + 4;
|
||||
const MAX_COW_LINEAGE_DEPTH: u32 = 1024;
|
||||
/// Bound dense membership bitmaps so a sparse, attacker-controlled vector ID
|
||||
/// cannot turn branch creation into an unbounded allocation.
|
||||
const MAX_MEMBERSHIP_FILTER_BYTES: u64 = 64 * 1024 * 1024;
|
||||
|
||||
fn relative_parent_reference(child_path: &Path, parent_path: &Path) -> Result<PathBuf, RvfError> {
|
||||
let child_dir = child_path
|
||||
|
|
@ -2265,6 +2268,30 @@ impl RvfStore {
|
|||
.checked_next_power_of_two()
|
||||
.ok_or_else(|| err(ErrorCode::CowMapCorrupt))?;
|
||||
let total_vecs = self.vectors.len() as u64;
|
||||
// MembershipFilter is keyed by vector ID, not by slab ordinal. Sizing
|
||||
// it from `len()` silently drops ID == len (the normal Node string-ID
|
||||
// case, which starts at 1) and every sparse ID. Size from the highest
|
||||
// live ID instead, while bounding the dense bitmap to fail closed
|
||||
// rather than risk an allocation DoS.
|
||||
let membership_capacity = self
|
||||
.vectors
|
||||
.ids()
|
||||
.copied()
|
||||
.max()
|
||||
.map(|max_id| {
|
||||
max_id
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| err(ErrorCode::MembershipInvalid))
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or(0);
|
||||
let membership_bytes = membership_capacity
|
||||
.div_ceil(64)
|
||||
.checked_mul(8)
|
||||
.ok_or_else(|| err(ErrorCode::MembershipInvalid))?;
|
||||
if membership_bytes > MAX_MEMBERSHIP_FILTER_BYTES {
|
||||
return Err(err(ErrorCode::MembershipInvalid));
|
||||
}
|
||||
let cluster_count = if vectors_per_cluster > 0 {
|
||||
total_vecs.div_ceil(vectors_per_cluster as u64) as u32
|
||||
} else {
|
||||
|
|
@ -2287,7 +2314,7 @@ impl RvfStore {
|
|||
));
|
||||
|
||||
// Initialize membership filter with all parent vectors visible
|
||||
let mut filter = MembershipFilter::new_include(total_vecs);
|
||||
let mut filter = MembershipFilter::new_include(membership_capacity);
|
||||
for &vid in self.vectors.ids() {
|
||||
if !self.deletion_bitmap.is_deleted(vid) {
|
||||
filter.add(vid);
|
||||
|
|
|
|||
|
|
@ -500,6 +500,58 @@ fn branch_query_reads_through_to_parent() {
|
|||
);
|
||||
}
|
||||
|
||||
/// Membership bitmaps are keyed by vector ID, so count-based capacity drops
|
||||
/// normal one-based and sparse IDs even though the vectors exist.
|
||||
#[test]
|
||||
fn branch_query_reads_one_based_and_sparse_parent_ids() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let base_path = dir.path().join("base_sparse.rvf");
|
||||
let child_path = dir.path().join("child_sparse.rvf");
|
||||
let dim: u16 = 4;
|
||||
let one_based = vec![1.0, 0.0, 0.0, 0.0];
|
||||
let sparse = vec![0.0, 1.0, 0.0, 0.0];
|
||||
|
||||
let mut base = RvfStore::create(&base_path, make_options(dim)).unwrap();
|
||||
base.ingest_batch(
|
||||
&[one_based.as_slice(), sparse.as_slice()],
|
||||
&[1, 10_000],
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
base.freeze().unwrap();
|
||||
|
||||
let child = base.branch(&child_path).unwrap();
|
||||
let opts = QueryOptions::default();
|
||||
assert_eq!(child.query(&one_based, 1, &opts).unwrap()[0].id, 1);
|
||||
assert_eq!(child.query(&sparse, 1, &opts).unwrap()[0].id, 10_000);
|
||||
|
||||
child.close().unwrap();
|
||||
base.close().unwrap();
|
||||
}
|
||||
|
||||
/// A hostile sparse ID must fail branch creation instead of overflowing or
|
||||
/// attempting an unbounded dense-bitmap allocation.
|
||||
#[test]
|
||||
fn branch_rejects_unbounded_membership_capacity() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let base_path = dir.path().join("base_hostile_id.rvf");
|
||||
let child_path = dir.path().join("child_hostile_id.rvf");
|
||||
let vector = vec![1.0, 0.0, 0.0, 0.0];
|
||||
|
||||
let mut base = RvfStore::create(&base_path, make_options(4)).unwrap();
|
||||
base.ingest_batch(&[vector.as_slice()], &[u64::MAX], None)
|
||||
.unwrap();
|
||||
base.freeze().unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
base.branch(&child_path),
|
||||
Err(rvf_types::RvfError::Code(
|
||||
rvf_types::ErrorCode::MembershipInvalid
|
||||
))
|
||||
));
|
||||
base.close().unwrap();
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// TEST 10: branch_state_survives_reopen_and_relocation
|
||||
// ===========================================================================
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue