vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-31 01:54:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Breaking change: unify the windows and linux build flags.

Browse source 
To do a full build on windows you now need LLAMA_PORTABLE=1 LLAMA_VULKAN=1 LLAMA_CLBLAST=1
This commit is contained in:
Concedo 2024-12-23 22:35:54 +08:00
parent 5f8f483fae
commit 2a890ec25a
7 changed files with 31 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

34
.github/workflows/kcpp-build-release-win-cuda.yaml vendored
View file

@ -1,34 +0,0 @@
name: Koboldcpp Windows CUDA
on: workflow_dispatch
env:
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
jobs:
windows:
runs-on: windows-2019
steps:
- name: Clone
id: checkout
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref || github.ref_name }}
- uses: Jimver/cuda-toolkit@v0.2.15
id: cuda-toolkit
with:
cuda: '11.4.4'
- name: Build
id: cmake_build
run: |
mkdir build
cd build
cmake .. -DLLAMA_CUBLAS=ON -DCMAKE_SYSTEM_VERSION="10.0.19041.0"
cmake --build . --config Release -j 2
- name: Save artifact
uses: actions/upload-artifact@v3
with:
name: kcpp_windows_cuda_binary
path: build/bin/Release/

34
.github/workflows/kcpp-build-release-win-cuda12.yaml vendored
View file

@ -1,34 +0,0 @@
name: Koboldcpp Windows CUDA12
on: workflow_dispatch
env:
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
jobs:
windows:
runs-on: windows-2019
steps:
- name: Clone
id: checkout
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref || github.ref_name }}
- uses: Jimver/cuda-toolkit@v0.2.15
id: cuda-toolkit
with:
cuda: '12.1.0'
- name: Build
id: cmake_build
run: |
mkdir build
cd build
cmake .. -DLLAMA_CUBLAS=ON -DCMAKE_SYSTEM_VERSION="10.0.19041.0"
cmake --build . --config Release -j 2
- name: Save artifact
uses: actions/upload-artifact@v3
with:
name: kcpp_windows_cuda_binary
path: build/bin/Release/

2
.github/workflows/kcpp-build-release-win-full-cu12.yaml vendored
View file

@ -45,7 +45,7 @@ jobs:
- name: Build Non-CUDA
id: make_build
run: |
make -j ${env:NUMBER_OF_PROCESSORS}
make LLAMA_CLBLAST=1 LLAMA_VULKAN=1 LLAMA_PORTABLE=1 -j ${env:NUMBER_OF_PROCESSORS}
- uses: Jimver/cuda-toolkit@v0.2.15
id: cuda-toolkit

2
.github/workflows/kcpp-build-release-win-full.yaml vendored
View file

@ -45,7 +45,7 @@ jobs:
- name: Build Non-CUDA
id: make_build
run: |
make -j ${env:NUMBER_OF_PROCESSORS}
make LLAMA_CLBLAST=1 LLAMA_VULKAN=1 LLAMA_PORTABLE=1 -j ${env:NUMBER_OF_PROCESSORS}
- uses: Jimver/cuda-toolkit@v0.2.15
id: cuda-toolkit

2
.github/workflows/kcpp-build-release-win-oldcpu-full.yaml vendored
View file

@ -45,7 +45,7 @@ jobs:
- name: Build Non-CUDA
id: make_build
run: |
make -j ${env:NUMBER_OF_PROCESSORS} LLAMA_NOAVX2=1
make LLAMA_CLBLAST=1 LLAMA_VULKAN=1 LLAMA_PORTABLE=1 -j ${env:NUMBER_OF_PROCESSORS} LLAMA_NOAVX2=1
- uses: Jimver/cuda-toolkit@v0.2.15
id: cuda-toolkit

37
Makefile
View file

@ -5,9 +5,6 @@
default: koboldcpp_default koboldcpp_failsafe koboldcpp_noavx2 koboldcpp_clblast koboldcpp_clblast_noavx2 koboldcpp_cublas koboldcpp_hipblas koboldcpp_vulkan koboldcpp_vulkan_noavx2 finishedmsg
tools: quantize_gpt2 quantize_gptj quantize_gguf quantize_neox quantize_mpt quantize_clip whispermain sdmain gguf-split
dev: koboldcpp_default
dev2: koboldcpp_clblast
dev3: koboldcpp_vulkan finishedmsg
ifndef UNAME_S
UNAME_S := $(shell uname -s)
@ -147,6 +144,7 @@ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64))
# Use all CPU extensions that are available:
# old library NEEDS mf16c to work. so we must build with it. new one doesnt
ifeq ($(OS),Windows_NT)
ifdef LLAMA_PORTABLE
CFLAGS +=
NONECFLAGS +=
SIMPLECFLAGS += -mavx -msse3
@ -155,8 +153,10 @@ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64))
else
FULLCFLAGS += -mavx2 -msse3 -mfma -mf16c -mavx
endif
else
CFLAGS += -march=native -mtune=native
endif
else
# if not on windows, they are clearly building it themselves, so lets just use whatever is supported
ifdef LLAMA_PORTABLE
CFLAGS +=
NONECFLAGS +=
@ -373,10 +373,17 @@ NOTIFY_MSG =
ifeq ($(OS),Windows_NT)
DEFAULT_BUILD = $(CXX) $(CXXFLAGS) $^ -shared -o $@.dll $(LDFLAGS)
ifdef LLAMA_PORTABLE
FAILSAFE_BUILD = $(CXX) $(CXXFLAGS) $^ -shared -o $@.dll $(LDFLAGS)
NOAVX2_BUILD = $(CXX) $(CXXFLAGS) $^ -shared -o $@.dll $(LDFLAGS)
endif
ifdef LLAMA_CLBLAST
CLBLAST_BUILD = $(CXX) $(CXXFLAGS) $^ lib/OpenCL.lib lib/clblast.lib -shared -o $@.dll $(LDFLAGS)
endif
ifdef LLAMA_VULKAN
VULKAN_BUILD = $(CXX) $(CXXFLAGS) $^ lib/vulkan-1.lib -shared -o $@.dll $(LDFLAGS)
endif
ifdef LLAMA_CUBLAS
CUBLAS_BUILD = $(CXX) $(CXXFLAGS) $(CUBLAS_FLAGS) $^ -shared -o $@.dll $(CUBLASLD_FLAGS) $(LDFLAGS)
@ -409,18 +416,18 @@ else
ifdef LLAMA_VULKAN
VULKAN_BUILD = $(CXX) $(CXXFLAGS) $^ -lvulkan -shared -o $@.so $(LDFLAGS)
endif
endif
ifndef LLAMA_CLBLAST
ifndef LLAMA_CUBLAS
ifndef LLAMA_HIPBLAS
ifndef LLAMA_VULKAN
ifndef LLAMA_METAL
NOTIFY_MSG = @echo -e '\n***\nYou did a basic CPU build. For faster speeds, consider installing and linking a GPU BLAS library. For example, set LLAMA_VULKAN=1 to compile with Vulkan support. Read the KoboldCpp Wiki for more information. This is just a reminder, not an error.\n***\n'
endif
endif
endif
endif
endif
ifndef LLAMA_CLBLAST
ifndef LLAMA_CUBLAS
ifndef LLAMA_HIPBLAS
ifndef LLAMA_VULKAN
ifndef LLAMA_METAL
NOTIFY_MSG = @echo -e '\n***\nYou did a basic CPU build. For faster speeds, consider installing and linking a GPU BLAS library. For example, set LLAMA_CLBLAST=1 LLAMA_VULKAN=1 to compile with Vulkan and CLBlast support. Add LLAMA_PORTABLE=1 to make a sharable build that other devices can use. Read the KoboldCpp Wiki for more information. This is just a reminder, not an error.\n***\n'
endif
endif
endif
endif
endif

7
README.md
View file

@ -83,13 +83,16 @@ when you can't use the precompiled binary directly, we provide an automated buil
- For Debian: Install `libclblast-dev`.
- You can attempt a CuBLAS build with `LLAMA_CUBLAS=1`, (or `LLAMA_HIPBLAS=1` for AMD). You will need CUDA Toolkit installed. Some have also reported success with the CMake file, though that is more for windows.
- For a full featured build (all backends), do `make LLAMA_CLBLAST=1 LLAMA_CUBLAS=1 LLAMA_VULKAN=1`. (Note that `LLAMA_CUBLAS=1` will not work on windows, you need visual studio)
- To make your build sharable and capable of working on other devices, you must use `LLAMA_PORTABLE=1`
- After all binaries are built, you can run the python script with the command `koboldcpp.py [ggml_model.gguf] [port]`
### Compiling on Windows
- You're encouraged to use the .exe released, but if you want to compile your binaries from source at Windows, the easiest way is:
- Get the latest release of w64devkit (https://github.com/skeeto/w64devkit). Be sure to use the "vanilla one", not i686 or other different stuff. If you try they will conflit with the precompiled libs!
- Clone the repo with `git clone https://github.com/LostRuins/koboldcpp.git`
- Make sure you are using the w64devkit integrated terminal, then run `make` at the KoboldCpp source folder. This will create the .dll files.
- Make sure you are using the w64devkit integrated terminal, then run `make` at the KoboldCpp source folder. This will create the .dll files for a pure CPU native build.
- For a full featured build (all backends), do `make LLAMA_CLBLAST=1 LLAMA_VULKAN=1`. (Note that `LLAMA_CUBLAS=1` will not work on windows, you need visual studio)
- To make your build sharable and capable of working on other devices, you must use `LLAMA_PORTABLE=1`
- If you want to generate the .exe file, make sure you have the python module PyInstaller installed with pip (`pip install PyInstaller`). Then run the script `make_pyinstaller.bat`
- The koboldcpp.exe file will be at your dist folder.
- **Building with CUDA**: Visual Studio, CMake and CUDA Toolkit is required. Clone the repo, then open the CMake file and compile it in Visual Studio. Copy the `koboldcpp_cublas.dll` generated into the same directory as the `koboldcpp.py` file. If you are bundling executables, you may need to include CUDA dynamic libraries (such as `cublasLt64_11.dll` and `cublas64_11.dll`) in order for the executable to work correctly on a different PC.
@ -104,6 +107,7 @@ when you can't use the precompiled binary directly, we provide an automated buil
- You can compile your binaries from source. You can clone the repo with `git clone https://github.com/LostRuins/koboldcpp.git`
- A makefile is provided, simply run `make`.
- If you want Metal GPU support, instead run `make LLAMA_METAL=1`, note that MacOS metal libraries need to be installed.
- To make your build sharable and capable of working on other devices, you must use `LLAMA_PORTABLE=1`
- After all binaries are built, you can run the python script with the command `koboldcpp.py --model [ggml_model.gguf]` (and add `--gpulayers (number of layer)` if you wish to offload layers to GPU).
### Compiling on Android (Termux Installation)
@ -114,6 +118,7 @@ when you can't use the precompiled binary directly, we provide an automated buil
- Clone the repo `git clone https://github.com/LostRuins/koboldcpp.git`
- Navigate to the koboldcpp folder `cd koboldcpp`
- Build the project `make`
- To make your build sharable and capable of working on other devices, you must use `LLAMA_PORTABLE=1`, this disables usage of ARM instrinsics.
- Grab a small GGUF model, such as `wget https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf`
- Start the python server `python koboldcpp.py --model KobbleTiny-Q4_K.gguf`
- Connect to `http://localhost:5001` on your mobile browser