mirror of
https://github.com/mindverse/Second-Me.git
synced 2026-08-17 12:43:19 +00:00
053090937d
* Add CUDA support - CUDA detection - Memory handling - Ollama model release after training * Fix logging issue added cuda support flag so log accurately reflected cuda toggle * Update llama.cpp rebuild Changed llama.cpp to only check if cuda support is enabled and if so rebuild during the first build rather than each run * Improved vram management Enabled memory pinning and optimizer state offload * Fix CUDA check rewrote llama.cpp rebuild logic, added manual y/n toggle if user wants to enable cuda support * Added fast restart and fixed CUDA check command Added make docker-restart-backend-fast to restart the backend and reflect code changes without causing a full llama.cpp rebuild Fixed make docker-check-cuda command to correctly reflect cuda support * Added docker-compose.gpu.yml Added docker-compose.gpu.yml to fix error on machines without nvidia gpu and made sure "\n" is added before .env modification * Fixed cuda toggle Last push accidentally broke cuda toggle * Code review fixes Fixed errors resulting from removed code: - Added return save_path to end of save_hf_model function - Rolled back download_file_with_progress function * Update Makefile Use cuda by default when using docker-restart-backend-fast * Minor cleanup Removed unnecessary makefile command and fixed gpu logging * Delete .gpu_selected * Simplified cuda training code - Removed dtype setting to let torch automatically handle it - Removed vram logging - Removed Unnecessary/old comments * Fixed gpu/cpu selection Made "make docker-use-gpu/cpu" command work with .gpu_selected flag and changed "make docker-restart-backend-fast" command to respect flag instead of always using gpu * Fix Ollama embedding error Added custom exception class for Ollama embeddings, which seemed to be returning keyword arguments while the Python exception class only accepts positional ones * Fixed model selection & memory error Fixed training defaulting to 0.5B model regardless of selection and fixed "free(): double free detected in tcache 2" error caused by cuda flag being passed incorrectly
57 lines
No EOL
2.3 KiB
Bash
57 lines
No EOL
2.3 KiB
Bash
#!/bin/bash
|
|
# Helper script to check if GPU support is available at runtime
|
|
|
|
echo "=== GPU Support Check ==="
|
|
|
|
# Check if llama-server binary exists and is linked to CUDA libraries
|
|
if [ -f "/app/llama.cpp/build/bin/llama-server" ]; then
|
|
echo "llama-server binary found, checking for CUDA linkage..."
|
|
CUDA_LIBS=$(ldd /app/llama.cpp/build/bin/llama-server | grep -i "cuda\|nvidia")
|
|
|
|
if [ -n "$CUDA_LIBS" ]; then
|
|
echo "✅ llama-server is built with CUDA support:"
|
|
echo "$CUDA_LIBS"
|
|
echo "GPU acceleration is available"
|
|
|
|
# Check for GPU optimization marker file (optional, not required)
|
|
GPU_MARKER_FILE="/app/data/gpu_optimized.json"
|
|
if [ -f "$GPU_MARKER_FILE" ]; then
|
|
GPU_OPTIMIZED=$(grep -o '"gpu_optimized": *true' "$GPU_MARKER_FILE" || echo "false")
|
|
OPTIMIZED_DATE=$(grep -o '"optimized_on": *"[^"]*"' "$GPU_MARKER_FILE" | cut -d'"' -f4)
|
|
|
|
if [[ "$GPU_OPTIMIZED" == *"true"* ]]; then
|
|
echo "📝 GPU-optimized build marker found (built on: $OPTIMIZED_DATE)"
|
|
else
|
|
echo "📝 GPU marker file found but not marked as optimized (built on: $OPTIMIZED_DATE)"
|
|
fi
|
|
else
|
|
echo "📝 No GPU optimization marker file found, but CUDA support is detected in binary"
|
|
fi
|
|
|
|
# Check if NVIDIA GPU is accessible at runtime
|
|
if nvidia-smi &>/dev/null; then
|
|
echo "🔍 NVIDIA GPU is available at runtime"
|
|
echo "=== GPU ACCELERATION IS READY TO USE ==="
|
|
exit 0
|
|
else
|
|
echo "⚠️ WARNING: llama-server has CUDA support, but NVIDIA GPU is not accessible"
|
|
echo "Check that Docker is running with GPU access (--gpus all)"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "❌ llama-server is not linked with CUDA libraries"
|
|
echo "Container was built without CUDA support"
|
|
fi
|
|
else
|
|
echo "❌ llama-server binary not found at /app/llama.cpp/build/bin/llama-server"
|
|
fi
|
|
|
|
# Final check for GPU hardware
|
|
if nvidia-smi &>/dev/null; then
|
|
echo "🔍 NVIDIA GPU is available at runtime, but llama-server doesn't support CUDA"
|
|
echo "To enable GPU support, rebuild using: make docker-up (and select CUDA support when prompted)"
|
|
exit 1
|
|
else
|
|
echo "❌ No NVIDIA GPU detected at runtime"
|
|
exit 1
|
|
fi |