diff --git a/.github/workflows/build-diskann.yml b/.github/workflows/build-diskann.yml new file mode 100644 index 000000000..2cd815c4c --- /dev/null +++ b/.github/workflows/build-diskann.yml @@ -0,0 +1,188 @@ +name: Build DiskANN Native Modules + +on: + push: + branches: [main] + paths: + - 'crates/ruvector-diskann/**' + - 'crates/ruvector-diskann-node/**' + - 'npm/packages/diskann/**' + - '.github/workflows/build-diskann.yml' + pull_request: + branches: [main] + paths: + - 'crates/ruvector-diskann/**' + - 'crates/ruvector-diskann-node/**' + - 'npm/packages/diskann/**' + workflow_dispatch: + inputs: + publish: + description: 'Publish to npm after build' + required: false + type: boolean + default: false + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + strategy: + fail-fast: false + matrix: + settings: + - host: ubuntu-22.04 + target: x86_64-unknown-linux-gnu + platform: linux-x64-gnu + - host: ubuntu-22.04 + target: aarch64-unknown-linux-gnu + platform: linux-arm64-gnu + - host: macos-14 + target: x86_64-apple-darwin + platform: darwin-x64 + - host: macos-14 + target: aarch64-apple-darwin + platform: darwin-arm64 + - host: windows-2022 + target: x86_64-pc-windows-msvc + platform: win32-x64-msvc + + name: Build DiskANN ${{ matrix.settings.platform }} + runs-on: ${{ matrix.settings.host }} + + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.settings.target }} + + - name: Cache Rust + uses: Swatinem/rust-cache@v2 + with: + key: diskann-${{ matrix.settings.target }} + + - name: Install cross-compilation tools (Linux ARM64) + if: matrix.settings.platform == 'linux-arm64-gnu' + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + + - name: Install dependencies + working-directory: npm/packages/diskann + run: npm install --ignore-scripts --omit=optional --force + + - name: Build native module + working-directory: npm/packages/diskann + run: npx napi build --platform --release --cargo-cwd ../../../crates/ruvector-diskann-node --target ${{ matrix.settings.target }} + env: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc + + - name: Prepare artifact + shell: bash + run: | + mkdir -p diskann-artifacts/${{ matrix.settings.platform }} + NODE_FILE=$(find npm/packages/diskann -name "*.node" -type f | head -1) + if [ -z "$NODE_FILE" ]; then + echo "ERROR: No .node file found" + exit 1 + fi + echo "Found: $NODE_FILE" + cp -v "$NODE_FILE" "diskann-artifacts/${{ matrix.settings.platform }}/" + + - name: Test native module (native platform only) + if: | + (matrix.settings.platform == 'linux-x64-gnu') || + (matrix.settings.platform == 'darwin-arm64') || + (matrix.settings.platform == 'win32-x64-msvc') + continue-on-error: true + working-directory: npm/packages/diskann + run: npm test + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: diskann-${{ matrix.settings.platform }} + path: diskann-artifacts/${{ matrix.settings.platform }}/*.node + if-no-files-found: error + + publish: + name: Publish DiskANN Platform Packages + runs-on: ubuntu-22.04 + needs: build + if: inputs.publish == true || startsWith(github.ref, 'refs/tags/v') + + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + registry-url: 'https://registry.npmjs.org' + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Create and publish platform packages + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + VERSION=$(node -p "require('./npm/packages/diskann/package.json').version") + echo "Publishing version: $VERSION" + + for dir in artifacts/diskann-*/; do + platform=$(basename "$dir" | sed 's/diskann-//') + NODE_FILE=$(find "$dir" -name "*.node" | head -1) + if [ -z "$NODE_FILE" ]; then continue; fi + + echo "=== Publishing @ruvector/diskann-${platform}@${VERSION} ===" + PKG_DIR="npm-pkg/diskann-${platform}" + mkdir -p "$PKG_DIR" + + case "$platform" in + linux-x64-gnu) OS="linux"; CPU="x64"; LIBC='"libc": ["glibc"],' ;; + linux-arm64-gnu) OS="linux"; CPU="arm64"; LIBC='"libc": ["glibc"],' ;; + darwin-x64) OS="darwin"; CPU="x64"; LIBC="" ;; + darwin-arm64) OS="darwin"; CPU="arm64"; LIBC="" ;; + win32-x64-msvc) OS="win32"; CPU="x64"; LIBC="" ;; + esac + + NODE_NAME="ruvector-diskann.${platform}.node" + cp "$NODE_FILE" "$PKG_DIR/$NODE_NAME" + + cat > "$PKG_DIR/package.json" << EOF + { + "name": "@ruvector/diskann-${platform}", + "version": "${VERSION}", + "os": ["${OS}"], + "cpu": ["${CPU}"], + ${LIBC} + "main": "${NODE_NAME}", + "files": ["${NODE_NAME}"], + "description": "DiskANN native bindings - ${platform}", + "license": "MIT", + "repository": {"type": "git", "url": "https://github.com/ruvnet/ruvector"}, + "engines": {"node": ">= 18"}, + "publishConfig": {"access": "public"} + } + EOF + + cd "$PKG_DIR" + npm publish --access public || echo "Failed to publish @ruvector/diskann-${platform}" + cd ../.. + done + + - name: Publish main package + working-directory: npm/packages/diskann + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public || echo "Package may already exist" diff --git a/Cargo.lock b/Cargo.lock index d4920f1f5..0e181a744 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9101,6 +9101,36 @@ dependencies = [ "wasm-bindgen-test", ] +[[package]] +name = "ruvector-diskann" +version = "2.1.0" +dependencies = [ + "bincode 2.0.1", + "bytemuck", + "memmap2", + "parking_lot 0.12.5", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "ruvector-diskann-node" +version = "2.1.0" +dependencies = [ + "napi", + "napi-build", + "napi-derive", + "parking_lot 0.12.5", + "ruvector-diskann", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "ruvector-dither" version = "0.1.0"