mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-05-08 18:30:50 +00:00
deduplicate repeated statements in colab, minor refactgor
This commit is contained in:
Concedo
parent
bd0d6c2da5
commit
df30473716
1 changed files with 58 additions and 82 deletions
140
colab.ipynb
140
colab.ipynb
|
|
@ -52,7 +52,7 @@
|
|||
"#@markdown You will need to restart KoboldCpp with the play button when you make changes to these settings.\n",
|
||||
"Model = \"Tiefighter 13B\" # @param [\"=== KoboldAI Models ===\",\"Tiefighter 13B\",\"Estopia 13B\",\"Bookadventures 8B (Writing)\",\"=== Roleplay Models ===\",\"Cydonia 24B\",\"Broken Tutu 24B\",\"Dans PersonalityEngine 24B\",\"==== Adventure Models ====\",\"Harbinger 24B (Instruct)\",\"Muse 12B (Instruct)\",\"=== Image Recognition Models ===\",\"Qwen 3 VL\",\"=== Corporate Models ===\",\"Mistral Small 24B\",\"GPT OSS 20B\",\"Gemma 3n E4B it\",\"GLM4 9b chat\",\"=== Classic Models ===\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\"] {\"allow-input\":true}\n",
|
||||
"Layers = 99 #@param [99]{allow-input: true}\n",
|
||||
"ContextSize = 4096 #@param [4096,8192] {allow-input: true}\n",
|
||||
"ContextSize = \"Default\" #@param [\"Default\",\"4096\",\"8192\"] {allow-input: true}\n",
|
||||
"FlashAttention = True #@param {type:\"boolean\"}\n",
|
||||
"Multiplayer = False #@param {type:\"boolean\"}\n",
|
||||
"DeleteExistingModels = True #@param {type:\"boolean\"}\n",
|
||||
|
|
@ -88,100 +88,76 @@
|
|||
"#@markdown Only select the following box if regular cloudflare tunnel fails to work. It will generate an inferior localtunnel tunnel, which you can use after entering a password.\n",
|
||||
"MakeLocalTunnelFallback = False #@param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"CustomCtxSize = \"\"\n",
|
||||
"CustomFlashAttention = True\n",
|
||||
"CustomMmproj = \"\"\n",
|
||||
"Customized = False\n",
|
||||
"if Model == \"Tiefighter 13B\":\n",
|
||||
" Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"4096\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = True\n",
|
||||
" Mmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\"\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\"\n",
|
||||
" CustomCtxSize = \"4096\"\n",
|
||||
" CustomMmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\"\n",
|
||||
"if Model == \"Estopia 13B\":\n",
|
||||
" Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"4096\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = True\n",
|
||||
" Mmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\"\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\"\n",
|
||||
" CustomCtxSize = \"4096\"\n",
|
||||
" CustomMmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\"\n",
|
||||
"if Model == \"Bookadventures 8B (Writing)\":\n",
|
||||
" Model = \"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"65536\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\"\n",
|
||||
" CustomCtxSize = \"65536\"\n",
|
||||
"if Model == \"Cydonia 24B\":\n",
|
||||
" Model = \"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"12288\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\"\n",
|
||||
" CustomCtxSize = \"12288\"\n",
|
||||
"if Model == \"Broken Tutu 24B\":\n",
|
||||
" Model = \"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"12288\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\"\n",
|
||||
" CustomCtxSize = \"12288\"\n",
|
||||
"if Model == \"Dans PersonalityEngine 24B\":\n",
|
||||
" Model = \"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"12288\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\"\n",
|
||||
" CustomCtxSize = \"12288\"\n",
|
||||
"if Model == \"Harbinger 24B (Instruct)\":\n",
|
||||
" Model = \"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"12288\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\"\n",
|
||||
" CustomCtxSize = \"12288\"\n",
|
||||
"if Model == \"Muse 12B (Instruct)\":\n",
|
||||
" Model = \"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"49152\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\"\n",
|
||||
" CustomCtxSize = \"49152\"\n",
|
||||
"if Model == \"Qwen 3 VL\":\n",
|
||||
" Model = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\"\n",
|
||||
" LoadVisionMMProjector = True\n",
|
||||
" Mmproj = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"32768\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\"\n",
|
||||
" CustomMmproj = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\"\n",
|
||||
" CustomCtxSize = \"32768\"\n",
|
||||
"if Model == \"Mistral Small 24B\":\n",
|
||||
" Model = \"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"12288\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\"\n",
|
||||
" CustomCtxSize = \"12288\"\n",
|
||||
"if Model == \"GPT OSS 20B\":\n",
|
||||
" Model = \"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"32768\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\"\n",
|
||||
" CustomCtxSize = \"32768\"\n",
|
||||
"if Model == \"Gemma 3n E4B it\":\n",
|
||||
" Model = \"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf?download=true\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"65536\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf?download=true\"\n",
|
||||
" CustomCtxSize = \"65536\"\n",
|
||||
"if Model == \"GLM4 9b chat\":\n",
|
||||
" Model = \"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf?download=true\"\n",
|
||||
" if not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" ContextSize = \"65536\"\n",
|
||||
" FlashAttention = True\n",
|
||||
" LoadVisionMMProjector = False\n",
|
||||
" Customized = True\n",
|
||||
" Model = \"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf?download=true\"\n",
|
||||
" CustomCtxSize = \"65536\"\n",
|
||||
"\n",
|
||||
"if Customized and not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel, LoadVisionMMProjector)):\n",
|
||||
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
|
||||
" FlashAttention = CustomFlashAttention\n",
|
||||
" if CustomMmproj!=\"\":\n",
|
||||
" Mmproj = CustomMmproj\n",
|
||||
" LoadVisionMMProjector = True\n",
|
||||
" if ContextSize==\"Default\" and CustomCtxSize!=\"\":\n",
|
||||
" ContextSize = CustomCtxSize\n",
|
||||
"if ContextSize==\"Default\":\n",
|
||||
" ContextSize = \"4096\"\n",
|
||||
"\n",
|
||||
"import os, glob\n",
|
||||
"if not os.path.isfile(\"/opt/bin/nvidia-smi\"):\n",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue