ruvector/crates/ruvector-bench/scripts/run_all_benchmarks.sh

#!/bin/bash
# Run complete Ruvector benchmark suite

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BENCH_DIR="$(dirname "$SCRIPT_DIR")"
OUTPUT_DIR="${BENCH_DIR}/bench_results"

echo "╔════════════════════════════════════════╗"
echo "║   Ruvector Benchmark Suite Runner     ║"
echo "╚════════════════════════════════════════╝"
echo ""

# Create output directory
mkdir -p "$OUTPUT_DIR"

# Parse arguments
QUICK_MODE=false
PROFILE=false

while [[ $# -gt 0 ]]; do
    case $1 in
        --quick)
            QUICK_MODE=true
            shift
            ;;
        --profile)
            PROFILE=true
            shift
            ;;
        *)
            echo "Unknown option: $1"
            echo "Usage: $0 [--quick] [--profile]"
            exit 1
            ;;
    esac
done

# Set benchmark parameters based on mode
if [ "$QUICK_MODE" = true ]; then
    echo "Running in QUICK mode (reduced dataset sizes)..."
    VECTORS=10000
    QUERIES=500
else
    echo "Running in FULL mode (standard dataset sizes)..."
    VECTORS=100000
    QUERIES=1000
fi

echo "Output directory: $OUTPUT_DIR"
echo ""

# Build benchmarks
echo "═══════════════════════════════════════════════════════════════"
echo "Building benchmark suite..."
echo "═══════════════════════════════════════════════════════════════"
cd "$BENCH_DIR"
cargo build --release
echo "✓ Build complete"
echo ""

# Run ANN Benchmarks
echo "═══════════════════════════════════════════════════════════════"
echo "1. ANN Benchmarks (SIFT/GIST/Deep1M compatibility)"
echo "═══════════════════════════════════════════════════════════════"
cargo run --release --bin ann-benchmark -- \
    --dataset synthetic \
    --num-vectors $VECTORS \
    --queries $QUERIES \
    --dimensions 128 \
    --output "$OUTPUT_DIR"
echo ""

# Run AgenticDB Benchmarks
echo "═══════════════════════════════════════════════════════════════"
echo "2. AgenticDB Workload Benchmarks"
echo "═══════════════════════════════════════════════════════════════"
cargo run --release --bin agenticdb-benchmark -- \
    --episodes $VECTORS \
    --skills $(($VECTORS / 10)) \
    --queries $QUERIES \
    --output "$OUTPUT_DIR"
echo ""

# Run Latency Benchmarks
echo "═══════════════════════════════════════════════════════════════"
echo "3. Latency Profiling"
echo "═══════════════════════════════════════════════════════════════"
cargo run --release --bin latency-benchmark -- \
    --num-vectors $(($VECTORS / 2)) \
    --queries $QUERIES \
    --dimensions 384 \
    --threads "1,4,8" \
    --output "$OUTPUT_DIR"
echo ""

# Run Memory Benchmarks
echo "═══════════════════════════════════════════════════════════════"
echo "4. Memory Profiling"
echo "═══════════════════════════════════════════════════════════════"
if [ "$QUICK_MODE" = true ]; then
    SCALES="1000,10000"
else
    SCALES="1000,10000,100000"
fi

cargo run --release --bin memory-benchmark -- \
    --dimensions 384 \
    --scales "$SCALES" \
    --output "$OUTPUT_DIR"
echo ""

# Run Comparison Benchmarks
echo "═══════════════════════════════════════════════════════════════"
echo "5. Cross-System Comparison"
echo "═══════════════════════════════════════════════════════════════"
cargo run --release --bin comparison-benchmark -- \
    --num-vectors $(($VECTORS / 2)) \
    --queries $QUERIES \
    --dimensions 384 \
    --output "$OUTPUT_DIR"
echo ""

# Run Profiling (optional)
if [ "$PROFILE" = true ]; then
    echo "═══════════════════════════════════════════════════════════════"
    echo "6. Performance Profiling with Flamegraph"
    echo "═══════════════════════════════════════════════════════════════"
    cargo run --release --features profiling --bin profiling-benchmark -- \
        --num-vectors $(($VECTORS / 2)) \
        --queries $QUERIES \
        --dimensions 384 \
        --flamegraph \
        --output "$OUTPUT_DIR/profiling"
    echo ""
fi

# Generate summary report
echo "═══════════════════════════════════════════════════════════════"
echo "Generating Summary Report"
echo "═══════════════════════════════════════════════════════════════"

SUMMARY_FILE="$OUTPUT_DIR/SUMMARY.md"

cat > "$SUMMARY_FILE" << EOF
# Ruvector Benchmark Results Summary

**Generated:** $(date)
**Mode:** $([ "$QUICK_MODE" = true ] && echo "Quick" || echo "Full")

## Configuration
- Vectors: $VECTORS
- Queries: $QUERIES
- Profiling: $([ "$PROFILE" = true ] && echo "Enabled" || echo "Disabled")

## Results Location
All benchmark results are saved in: \`$OUTPUT_DIR\`

## Available Reports

### 1. ANN Benchmarks
- JSON: \`ann_benchmark.json\`
- CSV: \`ann_benchmark.csv\`
- Report: \`ann_benchmark.md\`

### 2. AgenticDB Workloads
- JSON: \`agenticdb_benchmark.json\`
- CSV: \`agenticdb_benchmark.csv\`
- Report: \`agenticdb_benchmark.md\`

### 3. Latency Profiling
- JSON: \`latency_benchmark.json\`
- CSV: \`latency_benchmark.csv\`
- Report: \`latency_benchmark.md\`

### 4. Memory Profiling
- JSON: \`memory_benchmark.json\`
- CSV: \`memory_benchmark.csv\`
- Report: \`memory_benchmark.md\`

### 5. System Comparison
- JSON: \`comparison_benchmark.json\`
- CSV: \`comparison_benchmark.csv\`
- Report: \`comparison_benchmark.md\`

EOF

if [ "$PROFILE" = true ]; then
    cat >> "$SUMMARY_FILE" << EOF

### 6. Performance Profiling
- Flamegraph: \`profiling/flamegraph.svg\`
- Profile: \`profiling/profile.txt\`

EOF
fi

cat >> "$SUMMARY_FILE" << EOF

## Quick Analysis

To view individual benchmark reports, use:
\`\`\`bash
cat $OUTPUT_DIR/ann_benchmark.md
cat $OUTPUT_DIR/agenticdb_benchmark.md
cat $OUTPUT_DIR/latency_benchmark.md
cat $OUTPUT_DIR/memory_benchmark.md
cat $OUTPUT_DIR/comparison_benchmark.md
\`\`\`

To view CSV data for analysis:
\`\`\`bash
column -t -s, $OUTPUT_DIR/ann_benchmark.csv | less -S
\`\`\`

EOF

echo "✓ Summary report generated: $SUMMARY_FILE"
echo ""

echo "════════════════════════════════════════════════════════════════"
echo "✓ All benchmarks complete!"
echo "════════════════════════════════════════════════════════════════"
echo ""
echo "Results saved to: $OUTPUT_DIR"
echo "Summary report: $SUMMARY_FILE"
echo ""
echo "View results:"
echo "  cat $SUMMARY_FILE"
echo "  cat $OUTPUT_DIR/*.md"
echo ""

# Display quick stats if available
if [ -f "$OUTPUT_DIR/comparison_benchmark.csv" ]; then
    echo "Quick Performance Summary:"
    echo "─────────────────────────────────────────"
    grep "ruvector_optimized" "$OUTPUT_DIR/comparison_benchmark.csv" | \
        awk -F',' '{printf "  Optimized QPS: %s\n  Latency p50: %sms\n  Latency p99: %sms\n", $7, $8, $10}'
    echo ""
fi

echo "To run again:"
echo "  ./scripts/run_all_benchmarks.sh           # Full benchmarks"
echo "  ./scripts/run_all_benchmarks.sh --quick   # Quick mode"
echo "  ./scripts/run_all_benchmarks.sh --profile # With profiling"