mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2025-09-11 09:34:37 +00:00
updated colab
This commit is contained in:
Concedo
parent
911da8765f
commit
1012281320
1 changed files with 134 additions and 128 deletions
124
colab.ipynb
124
colab.ipynb
|
|
@ -46,65 +46,71 @@
|
||||||
},
|
},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"#@title <b>v-- Enter your model below and then click this to start Koboldcpp</b>\r\n",
|
"#@title <b>v-- Enter your model below and then click this to start Koboldcpp</b>\n",
|
||||||
"\r\n",
|
"\n",
|
||||||
"Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\"]{allow-input: true}\r\n",
|
"Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\"]{allow-input: true}\n",
|
||||||
"Layers = 99 #@param [99]{allow-input: true}\r\n",
|
"Layers = 99 #@param [99]{allow-input: true}\n",
|
||||||
"ContextSize = 4096 #@param [4096,8192] {allow-input: true}\r\n",
|
"ContextSize = 4096 #@param [4096,8192] {allow-input: true}\n",
|
||||||
"FlashAttention = True #@param {type:\"boolean\"}\r\n",
|
"FlashAttention = True #@param {type:\"boolean\"}\n",
|
||||||
"FACommand = \"\"\r\n",
|
"Multiplayer = False #@param {type:\"boolean\"}\n",
|
||||||
"#@markdown <hr>\r\n",
|
"FACommand = \"\"\n",
|
||||||
"LoadVisionMMProjector = False #@param {type:\"boolean\"}\r\n",
|
"MPCommand = \"\"\n",
|
||||||
"Mmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\" #@param [\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\r\n",
|
"#@markdown <hr>\n",
|
||||||
"VCommand = \"\"\r\n",
|
"LoadVisionMMProjector = False #@param {type:\"boolean\"}\n",
|
||||||
"#@markdown <hr>\r\n",
|
"Mmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\" #@param [\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\n",
|
||||||
"LoadImgModel = False #@param {type:\"boolean\"}\r\n",
|
"VCommand = \"\"\n",
|
||||||
"ImgModel = \"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\"]{allow-input: true}\r\n",
|
"#@markdown <hr>\n",
|
||||||
"SCommand = \"\"\r\n",
|
"LoadImgModel = False #@param {type:\"boolean\"}\n",
|
||||||
"#@markdown <hr>\r\n",
|
"ImgModel = \"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\"]{allow-input: true}\n",
|
||||||
"LoadSpeechModel = False #@param {type:\"boolean\"}\r\n",
|
"SCommand = \"\"\n",
|
||||||
"SpeechModel = \"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\" #@param [\"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\"]{allow-input: true}\r\n",
|
"#@markdown <hr>\n",
|
||||||
"WCommand = \"\"\r\n",
|
"LoadSpeechModel = False #@param {type:\"boolean\"}\n",
|
||||||
"\r\n",
|
"SpeechModel = \"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\" #@param [\"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\"]{allow-input: true}\n",
|
||||||
"import os\r\n",
|
"WCommand = \"\"\n",
|
||||||
"if not os.path.isfile(\"/opt/bin/nvidia-smi\"):\r\n",
|
"\n",
|
||||||
" raise RuntimeError(\"⚠️Colab did not give you a GPU due to usage limits, this can take a few hours before they let you back in. Check out https://lite.koboldai.net for a free alternative (that does not provide an API link but can load KoboldAI saves and chat cards) or subscribe to Colab Pro for immediate access.⚠️\")\r\n",
|
"import os\n",
|
||||||
"\r\n",
|
"if not os.path.isfile(\"/opt/bin/nvidia-smi\"):\n",
|
||||||
"%cd /content\r\n",
|
" raise RuntimeError(\"⚠️Colab did not give you a GPU due to usage limits, this can take a few hours before they let you back in. Check out https://lite.koboldai.net for a free alternative (that does not provide an API link but can load KoboldAI saves and chat cards) or subscribe to Colab Pro for immediate access.⚠️\")\n",
|
||||||
"if Mmproj and LoadVisionMMProjector:\r\n",
|
"\n",
|
||||||
" VCommand = \"--mmproj vmodel.gguf\"\r\n",
|
"%cd /content\n",
|
||||||
"else:\r\n",
|
"if Mmproj and LoadVisionMMProjector:\n",
|
||||||
" SCommand = \"\"\r\n",
|
" VCommand = \"--mmproj vmodel.gguf\"\n",
|
||||||
"if ImgModel and LoadImgModel:\r\n",
|
"else:\n",
|
||||||
" SCommand = \"--sdmodel imodel.gguf --sdthreads 4 --sdquant --sdclamped\"\r\n",
|
" SCommand = \"\"\n",
|
||||||
"else:\r\n",
|
"if ImgModel and LoadImgModel:\n",
|
||||||
" SCommand = \"\"\r\n",
|
" SCommand = \"--sdmodel imodel.gguf --sdthreads 4 --sdquant --sdclamped\"\n",
|
||||||
"if SpeechModel and LoadSpeechModel:\r\n",
|
"else:\n",
|
||||||
" WCommand = \"--whispermodel wmodel.bin\"\r\n",
|
" SCommand = \"\"\n",
|
||||||
"else:\r\n",
|
"if SpeechModel and LoadSpeechModel:\n",
|
||||||
" WCommand = \"\"\r\n",
|
" WCommand = \"--whispermodel wmodel.bin\"\n",
|
||||||
"if FlashAttention:\r\n",
|
"else:\n",
|
||||||
" FACommand = \"--flashattention\"\r\n",
|
" WCommand = \"\"\n",
|
||||||
"else:\r\n",
|
"if FlashAttention:\n",
|
||||||
" FACommand = \"\"\r\n",
|
" FACommand = \"--flashattention\"\n",
|
||||||
"\r\n",
|
"else:\n",
|
||||||
"!echo Downloading KoboldCpp, please wait...\r\n",
|
" FACommand = \"\"\n",
|
||||||
"!wget -O dlfile.tmp https://kcpplinux.concedo.workers.dev && mv dlfile.tmp koboldcpp_linux\r\n",
|
"if Multiplayer:\n",
|
||||||
"!test -f koboldcpp_linux && echo Download Successful || echo Download Failed\r\n",
|
" MPCommand = \"--multiplayer\"\n",
|
||||||
"!chmod +x ./koboldcpp_linux\r\n",
|
"else:\n",
|
||||||
"!apt update\r\n",
|
" MPCommand = \"\"\n",
|
||||||
"!apt install aria2 -y\r\n",
|
"\n",
|
||||||
"# simple fix for a common URL mistake\r\n",
|
"!echo Downloading KoboldCpp, please wait...\n",
|
||||||
"if \"https://huggingface.co/\" in Model and \"/blob/main/\" in Model: \r\n",
|
"!wget -O dlfile.tmp https://kcpplinux.concedo.workers.dev && mv dlfile.tmp koboldcpp_linux\n",
|
||||||
" Model = Model.replace(\"/blob/main/\", \"/resolve/main/\")\r\n",
|
"!test -f koboldcpp_linux && echo Download Successful || echo Download Failed\n",
|
||||||
"!aria2c -x 10 -o model.gguf --summary-interval=5 --download-result=default --allow-overwrite=true --file-allocation=none $Model\r\n",
|
"!chmod +x ./koboldcpp_linux\n",
|
||||||
"if VCommand:\r\n",
|
"!apt update\n",
|
||||||
" !aria2c -x 10 -o vmodel.gguf --summary-interval=5 --download-result=default --allow-overwrite=true --file-allocation=none $Mmproj\r\n",
|
"!apt install aria2 -y\n",
|
||||||
"if SCommand:\r\n",
|
"# simple fix for a common URL mistake\n",
|
||||||
" !aria2c -x 10 -o imodel.gguf --summary-interval=5 --download-result=default --allow-overwrite=true --file-allocation=none $ImgModel\r\n",
|
"if \"https://huggingface.co/\" in Model and \"/blob/main/\" in Model:\n",
|
||||||
"if WCommand:\r\n",
|
" Model = Model.replace(\"/blob/main/\", \"/resolve/main/\")\n",
|
||||||
" !aria2c -x 10 -o wmodel.bin --summary-interval=5 --download-result=default --allow-overwrite=true --file-allocation=none $SpeechModel\r\n",
|
"!aria2c -x 10 -o model.gguf --summary-interval=5 --download-result=default --allow-overwrite=true --file-allocation=none $Model\n",
|
||||||
"!./koboldcpp_linux model.gguf --usecublas 0 mmq --multiuser --gpulayers $Layers --contextsize $ContextSize --quiet --remotetunnel $FACommand $VCommand $SCommand $WCommand\r\n"
|
"if VCommand:\n",
|
||||||
|
" !aria2c -x 10 -o vmodel.gguf --summary-interval=5 --download-result=default --allow-overwrite=true --file-allocation=none $Mmproj\n",
|
||||||
|
"if SCommand:\n",
|
||||||
|
" !aria2c -x 10 -o imodel.gguf --summary-interval=5 --download-result=default --allow-overwrite=true --file-allocation=none $ImgModel\n",
|
||||||
|
"if WCommand:\n",
|
||||||
|
" !aria2c -x 10 -o wmodel.bin --summary-interval=5 --download-result=default --allow-overwrite=true --file-allocation=none $SpeechModel\n",
|
||||||
|
"!./koboldcpp_linux model.gguf --usecublas 0 mmq --multiuser --gpulayers $Layers --contextsize $ContextSize --websearch --quiet --remotetunnel $FACommand $MPCommand $VCommand $SCommand $WCommand\n"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue