From b6bfab128f3b39c117f3e65aedd06a7475d5566d Mon Sep 17 00:00:00 2001
From: henk717 <henk@henk.tech>
Date: Fri, 3 May 2024 11:12:57 +0200
Subject: [PATCH] CUDA 12 CI (#815)

* Allow KCPP_CUDA to specify CUDA version

* CUDA 12 CI Linux

* CUDA 12 CI

* Fix KCPP_CUDA indent

* KCPP_CUDA ENV Fix

StackOverflow is bad for advice sometimes....

* Lowcase cuda on output filename

* Strip . from filename output
---
 .../kcpp-build-release-linux-cuda12.yaml      | 33 ++++++++++++++++++
 .../kcpp-build-release-win-cuda12.yaml        | 34 +++++++++++++++++++
 koboldcpp.sh                                  | 21 +++++++++---
 3 files changed, 83 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/kcpp-build-release-linux-cuda12.yaml
 create mode 100644 .github/workflows/kcpp-build-release-win-cuda12.yaml

diff --git a/.github/workflows/kcpp-build-release-linux-cuda12.yaml b/.github/workflows/kcpp-build-release-linux-cuda12.yaml
new file mode 100644
index 000000000..0cbeaf9a2
--- /dev/null
+++ b/.github/workflows/kcpp-build-release-linux-cuda12.yaml
@@ -0,0 +1,33 @@
+name: Koboldcpp Builder Linux CUDA12
+
+on: workflow_dispatch
+env:
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  KCPP_CUDA: 12.1.0
+
+jobs:
+  linux:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v3
+        with:
+          ref: concedo_experimental
+
+      - name: Dependencies
+        id: depends
+        run: |
+          sudo apt-get update
+          sudo apt-get install git curl bzip2
+
+      - name: Build
+        id: make_build
+        run: |
+          ./koboldcpp.sh dist
+
+      - name: Save artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: kcpp_linux_binary
+          path: dist/
diff --git a/.github/workflows/kcpp-build-release-win-cuda12.yaml b/.github/workflows/kcpp-build-release-win-cuda12.yaml
new file mode 100644
index 000000000..ed9065d93
--- /dev/null
+++ b/.github/workflows/kcpp-build-release-win-cuda12.yaml
@@ -0,0 +1,34 @@
+name: Koboldcpp Builder Windows CUDA12
+
+on: workflow_dispatch
+env:
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+
+jobs:
+  windows:
+    runs-on: windows-2019
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v3
+        with:
+          ref: concedo_experimental
+
+      - uses: Jimver/cuda-toolkit@v0.2.11
+        id: cuda-toolkit
+        with:
+          cuda: '12.1.0'
+        
+      - name: Build
+        id: cmake_build
+        run: |
+          mkdir build
+          cd build
+          cmake .. -DLLAMA_CUBLAS=ON -DCMAKE_SYSTEM_VERSION="10.0.19041.0"
+          cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS}
+
+      - name: Save artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: kcpp_windows_cuda_binary
+          path: build/bin/Release/
diff --git a/koboldcpp.sh b/koboldcpp.sh
index b2923ec4f..587270d2a 100755
--- a/koboldcpp.sh
+++ b/koboldcpp.sh
@@ -1,13 +1,24 @@
 #!/bin/bash
+
 if [ ! -f "bin/micromamba" ]; then
 	curl -Ls https://anaconda.org/conda-forge/micromamba/1.5.3/download/linux-64/micromamba-1.5.3-0.tar.bz2 | tar -xvj bin/micromamba
 fi
 
 if [[ ! -f "conda/envs/linux/bin/python" || $1 == "rebuild" ]]; then
-	bin/micromamba create --no-shortcuts -r conda -n linux -f environment.yaml -y
-	bin/micromamba create --no-shortcuts -r conda -n linux -f environment.yaml -y
+	cp environment.yaml environment.tmp.yaml
+	if [ -n "$KCPP_CUDA" ]; then
+		sed -i -e "s/nvidia\/label\/cuda-11.5.0/nvidia\/label\/cuda-$KCPP_CUDA/g" environment.tmp.yaml
+	else
+		KCPP_CUDA=11.5.0
+	fi
+	bin/micromamba create --no-shortcuts -r conda -n linux -f environment.tmp.yaml -y
+	bin/micromamba create --no-shortcuts -r conda -n linux -f environment.tmp.yaml -y
 	bin/micromamba run -r conda -n linux make clean
+	echo $KCPP_CUDA > conda/envs/linux/cudaver
+	echo rm environment.tmp.yaml
 fi
+KCPP_CUDA=$(<conda/envs/linux/cudaver)
+KCPP_CUDAAPPEND=-cuda${KCPP_CUDA//.}$KCPP_APPEND
 
 bin/micromamba run -r conda -n linux make LLAMA_VULKAN=1 LLAMA_OPENBLAS=1 LLAMA_CLBLAST=1 LLAMA_CUBLAS=1 LLAMA_PORTABLE=1 LLAMA_ADD_CONDA_PATHS=1
 
@@ -15,9 +26,9 @@ if [[ $1 == "rebuild" ]]; then
 	echo Rebuild complete, you can now try to launch Koboldcpp.
 elif [[ $1 == "dist" ]]; then
 	bin/micromamba remove -r conda -n linux --force ocl-icd -y
-	bin/micromamba run -r conda -n linux pyinstaller --noconfirm --onefile --collect-all customtkinter --add-data='./koboldcpp_default.so:.' --add-data='./koboldcpp_cublas.so:.' --add-data='./koboldcpp_vulkan.so:.' --add-data='./koboldcpp_openblas.so:.' --add-data='./koboldcpp_clblast.so:.' --add-data "./koboldcpp_failsafe.so:." --add-data "./koboldcpp_noavx2.so:." --add-data "./koboldcpp_clblast_noavx2.so:." --add-data "./koboldcpp_vulkan_noavx2.so:." --add-data "./koboldcpp_vulkan.so:." --add-data='./klite.embd:.' --add-data='./kcpp_docs.embd:.' --add-data='./rwkv_vocab.embd:.' --add-data='./rwkv_world_vocab.embd:.' --clean --console koboldcpp.py -n "koboldcpp-linux-x64"
-	bin/micromamba run -r conda -n linux pyinstaller --noconfirm --onefile --collect-all customtkinter --add-data='./koboldcpp_default.so:.' --add-data='./koboldcpp_openblas.so:.' --add-data='./koboldcpp_vulkan.so:.' --add-data='./koboldcpp_clblast.so:.' --add-data "./koboldcpp_failsafe.so:." --add-data "./koboldcpp_noavx2.so:." --add-data "./koboldcpp_clblast_noavx2.so:." --add-data "./koboldcpp_vulkan_noavx2.so:." --add-data "./koboldcpp_vulkan.so:." --add-data='./klite.embd:.' --add-data='./kcpp_docs.embd:.' --add-data='./rwkv_vocab.embd:.' --add-data='./rwkv_world_vocab.embd:.' --clean --console koboldcpp.py -n "koboldcpp-linux-x64-nocuda"
+	bin/micromamba run -r conda -n linux pyinstaller --noconfirm --onefile --collect-all customtkinter --add-data='./koboldcpp_default.so:.' --add-data='./koboldcpp_cublas.so:.' --add-data='./koboldcpp_vulkan.so:.' --add-data='./koboldcpp_openblas.so:.' --add-data='./koboldcpp_clblast.so:.' --add-data "./koboldcpp_failsafe.so:." --add-data "./koboldcpp_noavx2.so:." --add-data "./koboldcpp_clblast_noavx2.so:." --add-data "./koboldcpp_vulkan_noavx2.so:." --add-data "./koboldcpp_vulkan.so:." --add-data='./klite.embd:.' --add-data='./kcpp_docs.embd:.' --add-data='./rwkv_vocab.embd:.' --add-data='./rwkv_world_vocab.embd:.' --clean --console koboldcpp.py -n "koboldcpp-linux-x64$KCPP_CUDAAPPEND"
+	bin/micromamba run -r conda -n linux pyinstaller --noconfirm --onefile --collect-all customtkinter --add-data='./koboldcpp_default.so:.' --add-data='./koboldcpp_openblas.so:.' --add-data='./koboldcpp_vulkan.so:.' --add-data='./koboldcpp_clblast.so:.' --add-data "./koboldcpp_failsafe.so:." --add-data "./koboldcpp_noavx2.so:." --add-data "./koboldcpp_clblast_noavx2.so:." --add-data "./koboldcpp_vulkan_noavx2.so:." --add-data "./koboldcpp_vulkan.so:." --add-data='./klite.embd:.' --add-data='./kcpp_docs.embd:.' --add-data='./rwkv_vocab.embd:.' --add-data='./rwkv_world_vocab.embd:.' --clean --console koboldcpp.py -n "koboldcpp-linux-x64-nocuda$KCPP_APPEND"
 	bin/micromamba install -r conda -n linux ocl-icd -c conda-forge -y
- else
+else
 	bin/micromamba run -r conda -n linux python koboldcpp.py $*
 fi