mirror of
https://github.com/ruvnet/RuVector.git
synced 2026-05-23 04:27:11 +00:00
* feat(ruvector): implement missing capabilities (ADR-143) - speculativeEmbed: real FNV-1a hash embedding (128-dim) from file content - ragRetrieve: cosine similarity on embeddings + TF-IDF keyword fallback - contextRank: TF-IDF weighted scoring instead of raw keyword matching - Remove false DiskANN claim (will implement as Rust crate next) Co-Authored-By: claude-flow <ruv@ruv.net> * feat(diskann): Vamana graph + PQ — SSD-friendly billion-scale ANN (ADR-143) New Rust crate: ruvector-diskann Core algorithm (NeurIPS 2019 DiskANN paper): - Vamana graph with α-robust pruning (bounded out-degree R) - k-means++ seeded Product Quantization (M subspaces, 256 centroids) - Asymmetric PQ distance tables for fast candidate filtering - Two-phase search: PQ-filtered beam search → exact re-ranking - Memory-mapped persistence (mmap vectors + binary graph) Performance characteristics: - L2-squared distance with 8-wide loop unrolling (auto-vectorized) - Greedy beam search with bounded visited set - Save/load with flat binary format (mmap-friendly) 9 tests passing: distance, PQ train/encode, Vamana build/search, bounded degree, full index CRUD, PQ-accelerated search, save/load. Co-Authored-By: claude-flow <ruv@ruv.net> * feat(diskann): NAPI-RS bindings + npm package + 14 tests passing Rust core (ruvector-diskann): - 4-accumulator L2 distance for ILP optimization - Recall@10 = 1.000 on 2K vectors - Search latency: 90µs (5K vectors, 128d, k=10) - 14 tests: distance, PQ, Vamana, recall, scale, edge cases NAPI-RS bindings (ruvector-diskann-node): - Sync + async build/search - Batch insert (flat Float32Array) - Save/load, delete, count - Thread-safe via parking_lot::RwLock npm package (@ruvector/diskann): - Platform-specific loader (linux/darwin/win) - TypeScript declarations - Node.js test passing Co-Authored-By: claude-flow <ruv@ruv.net> * ci(diskann): add cross-platform build + publish workflow 5 targets: linux-x64, linux-arm64, darwin-x64, darwin-arm64, win32-x64 Co-Authored-By: claude-flow <ruv@ruv.net> * perf(diskann): FlatVectors + VisitedSet + ILP + optional SIMD/GPU Optimizations applied: - FlatVectors: contiguous f32 slab (eliminates Vec<Vec> indirection) - VisitedSet: O(1) clear via generation counter (replaces HashSet) - 4-accumulator ILP for L2 distance (auto-vectorized) - Flat PQ distance table (cache-line friendly) - Parallel medoid finding via rayon - Zero-copy save (write flat slab directly) - Optional simsimd feature for hardware NEON/AVX2/AVX-512 - Optional gpu feature with Metal/CUDA/Vulkan dispatch stubs Results (5K vectors, 128d): - Search: 90µs → 55µs (1.6x faster) - Build: 6.9s → 6.2s (10% faster) - Recall@10: 0.998 (maintained) - 17 tests passing Co-Authored-By: claude-flow <ruv@ruv.net> --------- Co-authored-by: Reuven <cohen@ruv-mac-mini.local>
188 lines
6 KiB
YAML
188 lines
6 KiB
YAML
name: Build DiskANN Native Modules
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- 'crates/ruvector-diskann/**'
|
|
- 'crates/ruvector-diskann-node/**'
|
|
- 'npm/packages/diskann/**'
|
|
- '.github/workflows/build-diskann.yml'
|
|
pull_request:
|
|
branches: [main]
|
|
paths:
|
|
- 'crates/ruvector-diskann/**'
|
|
- 'crates/ruvector-diskann-node/**'
|
|
- 'npm/packages/diskann/**'
|
|
workflow_dispatch:
|
|
inputs:
|
|
publish:
|
|
description: 'Publish to npm after build'
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
|
|
jobs:
|
|
build:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
settings:
|
|
- host: ubuntu-22.04
|
|
target: x86_64-unknown-linux-gnu
|
|
platform: linux-x64-gnu
|
|
- host: ubuntu-22.04
|
|
target: aarch64-unknown-linux-gnu
|
|
platform: linux-arm64-gnu
|
|
- host: macos-14
|
|
target: x86_64-apple-darwin
|
|
platform: darwin-x64
|
|
- host: macos-14
|
|
target: aarch64-apple-darwin
|
|
platform: darwin-arm64
|
|
- host: windows-2022
|
|
target: x86_64-pc-windows-msvc
|
|
platform: win32-x64-msvc
|
|
|
|
name: Build DiskANN ${{ matrix.settings.platform }}
|
|
runs-on: ${{ matrix.settings.host }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '18'
|
|
|
|
- name: Setup Rust
|
|
uses: dtolnay/rust-toolchain@stable
|
|
with:
|
|
targets: ${{ matrix.settings.target }}
|
|
|
|
- name: Cache Rust
|
|
uses: Swatinem/rust-cache@v2
|
|
with:
|
|
key: diskann-${{ matrix.settings.target }}
|
|
|
|
- name: Install cross-compilation tools (Linux ARM64)
|
|
if: matrix.settings.platform == 'linux-arm64-gnu'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
|
|
|
|
- name: Install dependencies
|
|
working-directory: npm/packages/diskann
|
|
run: npm install --ignore-scripts --omit=optional --force
|
|
|
|
- name: Build native module
|
|
working-directory: npm/packages/diskann
|
|
run: npx napi build --platform --release --cargo-cwd ../../../crates/ruvector-diskann-node --target ${{ matrix.settings.target }}
|
|
env:
|
|
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
|
|
|
|
- name: Prepare artifact
|
|
shell: bash
|
|
run: |
|
|
mkdir -p diskann-artifacts/${{ matrix.settings.platform }}
|
|
NODE_FILE=$(find npm/packages/diskann -name "*.node" -type f | head -1)
|
|
if [ -z "$NODE_FILE" ]; then
|
|
echo "ERROR: No .node file found"
|
|
exit 1
|
|
fi
|
|
echo "Found: $NODE_FILE"
|
|
cp -v "$NODE_FILE" "diskann-artifacts/${{ matrix.settings.platform }}/"
|
|
|
|
- name: Test native module (native platform only)
|
|
if: |
|
|
(matrix.settings.platform == 'linux-x64-gnu') ||
|
|
(matrix.settings.platform == 'darwin-arm64') ||
|
|
(matrix.settings.platform == 'win32-x64-msvc')
|
|
continue-on-error: true
|
|
working-directory: npm/packages/diskann
|
|
run: npm test
|
|
|
|
- name: Upload artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: diskann-${{ matrix.settings.platform }}
|
|
path: diskann-artifacts/${{ matrix.settings.platform }}/*.node
|
|
if-no-files-found: error
|
|
|
|
publish:
|
|
name: Publish DiskANN Platform Packages
|
|
runs-on: ubuntu-22.04
|
|
needs: build
|
|
if: inputs.publish == true || startsWith(github.ref, 'refs/tags/v')
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '18'
|
|
registry-url: 'https://registry.npmjs.org'
|
|
|
|
- name: Download all artifacts
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: artifacts
|
|
|
|
- name: Create and publish platform packages
|
|
env:
|
|
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
run: |
|
|
VERSION=$(node -p "require('./npm/packages/diskann/package.json').version")
|
|
echo "Publishing version: $VERSION"
|
|
|
|
for dir in artifacts/diskann-*/; do
|
|
platform=$(basename "$dir" | sed 's/diskann-//')
|
|
NODE_FILE=$(find "$dir" -name "*.node" | head -1)
|
|
if [ -z "$NODE_FILE" ]; then continue; fi
|
|
|
|
echo "=== Publishing @ruvector/diskann-${platform}@${VERSION} ==="
|
|
PKG_DIR="npm-pkg/diskann-${platform}"
|
|
mkdir -p "$PKG_DIR"
|
|
|
|
case "$platform" in
|
|
linux-x64-gnu) OS="linux"; CPU="x64"; LIBC='"libc": ["glibc"],' ;;
|
|
linux-arm64-gnu) OS="linux"; CPU="arm64"; LIBC='"libc": ["glibc"],' ;;
|
|
darwin-x64) OS="darwin"; CPU="x64"; LIBC="" ;;
|
|
darwin-arm64) OS="darwin"; CPU="arm64"; LIBC="" ;;
|
|
win32-x64-msvc) OS="win32"; CPU="x64"; LIBC="" ;;
|
|
esac
|
|
|
|
NODE_NAME="ruvector-diskann.${platform}.node"
|
|
cp "$NODE_FILE" "$PKG_DIR/$NODE_NAME"
|
|
|
|
cat > "$PKG_DIR/package.json" << EOF
|
|
{
|
|
"name": "@ruvector/diskann-${platform}",
|
|
"version": "${VERSION}",
|
|
"os": ["${OS}"],
|
|
"cpu": ["${CPU}"],
|
|
${LIBC}
|
|
"main": "${NODE_NAME}",
|
|
"files": ["${NODE_NAME}"],
|
|
"description": "DiskANN native bindings - ${platform}",
|
|
"license": "MIT",
|
|
"repository": {"type": "git", "url": "https://github.com/ruvnet/ruvector"},
|
|
"engines": {"node": ">= 18"},
|
|
"publishConfig": {"access": "public"}
|
|
}
|
|
EOF
|
|
|
|
cd "$PKG_DIR"
|
|
npm publish --access public || echo "Failed to publish @ruvector/diskann-${platform}"
|
|
cd ../..
|
|
done
|
|
|
|
- name: Publish main package
|
|
working-directory: npm/packages/diskann
|
|
env:
|
|
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
run: npm publish --access public || echo "Package may already exist"
|