koboldcpp/colab.ipynb

277 lines
17 KiB
Text
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "view-in-github"
},
"source": [
"<a href=\"https://colab.research.google.com/github/LostRuins/koboldcpp/blob/concedo/colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "2FCn5tmpn3UV"
},
"source": [
"## Welcome to the Official KoboldCpp Colab Notebook\n",
"It's really easy to get started. Just press the two **Play** buttons below, and then connect to the **Cloudflare URL** shown at the end.\n",
"You can select a model from the dropdown, or enter a **custom URL** to a GGUF model (Example: `https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf`)\n",
"\n",
"Named models in the Model selector automatically use the optimal context settings as long as you do not load additional models such as an image generation model.\n",
"\n",
"**Keep this page open and occationally check for captcha's so that your AI is not shut down**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "QNaj3u0jn3UW"
},
"outputs": [],
"source": [
"#@title v-- Tap this if you play on Mobile { display-mode: \"form\" }\n",
"%%html\n",
"<b>Ensure this audio is playing to prevent colab from shutting down, then start KoboldCpp below</b><br/>\n",
"<audio autoplay=\"\" src=\"https://raw.githubusercontent.com/KoboldAI/KoboldAI-Client/main/colab/silence.m4a\" loop controls>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "uJS9i_Dltv8Y"
},
"outputs": [],
"source": [
"# @title <b>v-- Enter your model below and then click this to start Koboldcpp</b> {\"display-mode\":\"form\"}\n",
"#@markdown You will need to restart KoboldCpp with the play button when you make changes to these settings.\n",
"Model = \"Tiefighter 13B\" # @param [\"=== KoboldAI Models ===\",\"Tiefighter 13B\",\"Estopia 13B\",\"Bookadventures 8B (Writing)\",\"=== Roleplay Models ===\",\"Cydonia 24B\",\"Broken Tutu 24B\",\"Dans PersonalityEngine 24B\",\"==== Adventure Models ====\",\"Harbinger 24B (Instruct)\",\"Muse 12B (Instruct)\",\"=== Image Recognition Models ===\",\"Qwen 3 VL\",\"=== Corporate Models ===\",\"Mistral Small 24B\",\"GPT OSS 20B\",\"Gemma 3n E4B it\",\"GLM4 9b chat\",\"=== Classic Models ===\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\"] {\"allow-input\":true}\n",
"Layers = 99 #@param [99]{allow-input: true}\n",
"ContextSize = \"Default\" #@param [\"Default\",\"4096\",\"8192\"] {allow-input: true}\n",
"FlashAttention = True #@param {type:\"boolean\"}\n",
"Multiplayer = False #@param {type:\"boolean\"}\n",
"DeleteExistingModels = True #@param {type:\"boolean\"}\n",
"AlwaysUseManualSettings = False #@param {type:\"boolean\"}\n",
"FACommand = \"\"\n",
"MPCommand = \"\"\n",
"#@markdown <hr>\n",
"LoadVisionMMProjector = False #@param {type:\"boolean\"}\n",
"Mmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\" #@param [\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/mistral-7b-mmproj-v1.5-Q4_1.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-7b-mmproj-v1.5-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/mmproj/resolve/main/LLaMA3-8B_mmproj-Q4_1.gguf\"]{allow-input: true}\n",
"VCommand = \"\"\n",
"#@markdown <hr>\n",
"LoadImgModel = False #@param {type:\"boolean\"}\n",
"ImgModel = \"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/imgmodel/resolve/main/imgmodel_ftuned_q4_0.gguf\"]{allow-input: true}\n",
"SCommand = \"\"\n",
"#@markdown <hr>\n",
"LoadSpeechModel = False #@param {type:\"boolean\"}\n",
"SpeechModel = \"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\" #@param [\"https://huggingface.co/koboldcpp/whisper/resolve/main/whisper-base.en-q5_1.bin\"]{allow-input: true}\n",
"WCommand = \"\"\n",
"#@markdown <hr>\n",
"LoadTTSModel = False #@param {type:\"boolean\"}\n",
"TTSModel = \"https://huggingface.co/koboldcpp/tts/resolve/main/OuteTTS-0.2-500M-Q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/tts/resolve/main/OuteTTS-0.2-500M-Q4_0.gguf\",\"https://huggingface.co/koboldcpp/tts/resolve/main/Kokoro_no_espeak_Q4.gguf\"]{allow-input: true}\n",
"WavTokModel = \"https://huggingface.co/koboldcpp/tts/resolve/main/WavTokenizer-Large-75-Q4_0.gguf\" #@param [\"https://huggingface.co/koboldcpp/tts/resolve/main/WavTokenizer-Large-75-Q4_0.gguf\"]{allow-input: true}\n",
"TTSCommand = \"\"\n",
"#@markdown <hr>\n",
"LoadEmbeddingsModel = False #@param {type:\"boolean\"}\n",
"EmbeddingsModel = \"https://huggingface.co/yixuan-chia/snowflake-arctic-embed-s-GGUF/resolve/main/snowflake-arctic-embed-s-Q4_0.gguf\" #@param [\"https://huggingface.co/yixuan-chia/snowflake-arctic-embed-s-GGUF/resolve/main/snowflake-arctic-embed-s-Q4_0.gguf\"]{allow-input: true}\n",
"ECommand = \"\"\n",
"#@markdown <hr>\n",
"#@markdown This enables saving stories directly to your google drive. You will have to grant permissions, and then you can access the saves from the \"KoboldCpp Server Storage\" option.\n",
"AllowSaveToGoogleDrive = False #@param {type:\"boolean\"}\n",
"SavGdriveCommand = \"\"\n",
"#@markdown <hr>\n",
"#@markdown Only select the following box if regular cloudflare tunnel fails to work. It will generate an inferior localtunnel tunnel, which you can use after entering a password.\n",
"MakeLocalTunnelFallback = False #@param {type:\"boolean\"}\n",
"\n",
"CustomCtxSize = \"\"\n",
"CustomFlashAttention = True\n",
"CustomMmproj = \"\"\n",
"Customized = False\n",
"if Model == \"Tiefighter 13B\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\"\n",
" CustomCtxSize = \"4096\"\n",
" CustomMmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\"\n",
"if Model == \"Estopia 13B\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\"\n",
" CustomCtxSize = \"4096\"\n",
" CustomMmproj = \"https://huggingface.co/koboldcpp/mmproj/resolve/main/llama-13b-mmproj-v1.5.Q4_1.gguf\"\n",
"if Model == \"Bookadventures 8B (Writing)\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\"\n",
" CustomCtxSize = \"65536\"\n",
"if Model == \"Cydonia 24B\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\"\n",
" CustomCtxSize = \"12288\"\n",
"if Model == \"Broken Tutu 24B\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\"\n",
" CustomCtxSize = \"12288\"\n",
"if Model == \"Dans PersonalityEngine 24B\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\"\n",
" CustomCtxSize = \"12288\"\n",
"if Model == \"Harbinger 24B (Instruct)\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\"\n",
" CustomCtxSize = \"12288\"\n",
"if Model == \"Muse 12B (Instruct)\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\"\n",
" CustomCtxSize = \"49152\"\n",
"if Model == \"Qwen 3 VL\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\"\n",
" CustomMmproj = \"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/mmproj-BF16.gguf\"\n",
" CustomCtxSize = \"32768\"\n",
"if Model == \"Mistral Small 24B\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\"\n",
" CustomCtxSize = \"12288\"\n",
"if Model == \"GPT OSS 20B\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\"\n",
" CustomCtxSize = \"32768\"\n",
"if Model == \"Gemma 3n E4B it\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf?download=true\"\n",
" CustomCtxSize = \"65536\"\n",
"if Model == \"GLM4 9b chat\":\n",
" Customized = True\n",
" Model = \"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf?download=true\"\n",
" CustomCtxSize = \"65536\"\n",
"\n",
"if Customized and not any((AlwaysUseManualSettings, LoadImgModel, LoadTTSModel, LoadSpeechModel, LoadVisionMMProjector)):\n",
" print(\"Overriding model settings to the optimal ones for your chosen model\")\n",
" FlashAttention = CustomFlashAttention\n",
" if CustomMmproj!=\"\":\n",
" Mmproj = CustomMmproj\n",
" LoadVisionMMProjector = True\n",
" if ContextSize==\"Default\" and CustomCtxSize!=\"\":\n",
" ContextSize = CustomCtxSize\n",
"if ContextSize==\"Default\":\n",
" ContextSize = \"4096\"\n",
"\n",
"import os, glob\n",
"if not os.path.isfile(\"/opt/bin/nvidia-smi\"):\n",
" raise RuntimeError(\"⚠Colab did not give you a GPU due to usage limits, this can take a few hours before they let you back in. Check out https://lite.koboldai.net for a free alternative (that does not provide an API link but can load KoboldAI saves and chat cards) or subscribe to Colab Pro for immediate access.⚠️\")\n",
"\n",
"if AllowSaveToGoogleDrive:\n",
" print(\"Attempting to request access to save to your google drive...\")\n",
" try:\n",
" from google.colab import drive\n",
" import os, json\n",
" drive.mount('/content/drive', force_remount=True)\n",
" if not os.path.exists(\"/content/drive/MyDrive\"):\n",
" raise RuntimeError(\"Google Drive mount failed. Please grant permissions and try again.\")\n",
" kcppdir = '/content/drive/MyDrive/koboldcpp_data'\n",
" os.makedirs(kcppdir, exist_ok=True)\n",
" savedatapath = os.path.join(kcppdir, \"koboldcpp_save_db.jsondb\")\n",
" if not os.path.exists(savedatapath):\n",
" settings_data = {}\n",
" with open(savedatapath, \"w\") as json_file:\n",
" json.dump(settings_data, json_file, indent=4)\n",
" print(f\"Created new koboldcpp_save_db.jsondb at {savedatapath}\")\n",
" else:\n",
" print(f\"Loading saved data at {savedatapath}\")\n",
" SavGdriveCommand = f\" --savedatafile {savedatapath}\"\n",
" except Exception as e:\n",
" print(f\"⚠️ Error: {e}\")\n",
" print(\"Please ensure you grant Google Drive permissions and try again.\")\n",
"\n",
"%cd /content\n",
"if Mmproj and LoadVisionMMProjector:\n",
" VCommand = f\"--mmproj {Mmproj}\"\n",
"else:\n",
" SCommand = \"\"\n",
"if ImgModel and LoadImgModel:\n",
" SCommand = f\"--sdmodel {ImgModel} --sdthreads 4 --sdquant --sdclamped\"\n",
"else:\n",
" SCommand = \"\"\n",
"if SpeechModel and LoadSpeechModel:\n",
" WCommand = f\"--whispermodel {SpeechModel}\"\n",
"else:\n",
" WCommand = \"\"\n",
"if TTSModel and WavTokModel and LoadTTSModel:\n",
" TTSCommand = f\"--ttsmodel {TTSModel} --ttswavtokenizer {WavTokModel} --ttsgpu\"\n",
"elif TTSModel and LoadTTSModel:\n",
" TTSCommand = f\"--ttsmodel {TTSModel} --ttsgpu\"\n",
"else:\n",
" TTSCommand = \"\"\n",
"if EmbeddingsModel and LoadEmbeddingsModel:\n",
" ECommand = f\"--embeddingsmodel {EmbeddingsModel}\"\n",
"else:\n",
" ECommand = \"\"\n",
"if FlashAttention:\n",
" FACommand = \"--flashattention\"\n",
"else:\n",
" FACommand = \"\"\n",
"if Multiplayer:\n",
" MPCommand = \"--multiplayer\"\n",
"else:\n",
" MPCommand = \"\"\n",
"\n",
"ModelCommand = f\"--model {Model}\"\n",
"\n",
"if DeleteExistingModels:\n",
" print(\"Deleting all cached models to redownload...\")\n",
" patterns = ['*.gguf', '*.bin', '*.ggml']\n",
" for pattern in patterns:\n",
" for file_path in glob.glob(pattern):\n",
" try:\n",
" os.remove(file_path)\n",
" print(f\"Deleted: {file_path}\")\n",
" except Exception as e:\n",
" print(f\"Failed to delete {file_path}: {e}\")\n",
"\n",
"\n",
"!echo Downloading KoboldCpp, please wait...\n",
"!wget -O dlfile.tmp https://kcpplinux.concedo.workers.dev && mv dlfile.tmp koboldcpp_linux\n",
"!test -f koboldcpp_linux && echo Download Successful || echo Download Failed\n",
"!chmod +x ./koboldcpp_linux\n",
"!apt update\n",
"!apt install aria2 -y\n",
"\n",
"if MakeLocalTunnelFallback:\n",
" import urllib\n",
" print(\"Trying to use LocalTunnel as a fallback tunnel (not so good)...\")\n",
" ltpw = urllib.request.urlopen('https://loca.lt/mytunnelpassword').read().decode('utf8').strip(\"\\n\")\n",
" !nohup npx --yes localtunnel --port 5001 > lt.log 2>&1 &\n",
" !sleep 8\n",
" print(\"=================\")\n",
" print(\"(LocalTunnel Results)\")\n",
" !cat lt.log\n",
" print(f\"Please open the above link, and input the password '{ltpw}'\\nYour KoboldCpp will start shortly...\")\n",
" print(\"=================\")\n",
" !sleep 10\n",
"!./koboldcpp_linux $ModelCommand --usecuda 0 mmq --chatcompletionsadapter AutoGuess --multiuser --gpulayers $Layers --contextsize $ContextSize --websearch --quiet --remotetunnel $FACommand $MPCommand $VCommand $SCommand $WCommand $TTSCommand $ECommand $SavGdriveCommand\n"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"cell_execution_strategy": "setup",
"gpuType": "T4",
"include_colab_link": true,
"private_outputs": true,
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}