mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2025-09-09 16:44:35 +00:00
7d120f2794
-add configurable context size to parameters along with models and layers for ease of use -this can already be done with a simple edit by experienced llm users but new users may not know this is a parameter they should set. Co-authored-by: LostRuins <39025047+LostRuins@users.noreply.github.com>
88 lines
No EOL
3.5 KiB
Text
88 lines
No EOL
3.5 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text",
|
|
"id": "view-in-github"
|
|
},
|
|
"source": [
|
|
"<a href=\"https://colab.research.google.com/github/LostRuins/koboldcpp/blob/concedo/colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Welcome to the Official KoboldCpp Colab Notebook\r\n",
|
|
"It's really easy to get started. Just press the two **Play** buttons below, and then connect to the **Cloudflare URL** shown at the end. \r\n",
|
|
"You can also change the model URL to use any GGUF model from Huggingface."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#@title <-- Tap this if you play on Mobile { display-mode: \"form\" }\r\n",
|
|
"%%html\r\n",
|
|
"<b>Press play on the music player to keep the tab alive, then start KoboldCpp below</b><br/>\r\n",
|
|
"<audio src=\"https://raw.githubusercontent.com/KoboldAI/KoboldAI-Client/main/colab/silence.m4a\" controls>"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"cellView": "form",
|
|
"id": "uJS9i_Dltv8Y"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"#@title <b>v-- Enter your model below and then click this to start Koboldcpp</b>\r\n",
|
|
"\r\n",
|
|
"Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf\" #@param [\"\"]{allow-input: true}\r\n",
|
|
"Layers = 43 #@param [43]{allow-input: true}\r\n",
|
|
"ContextSize = 4096 #@param [4096] {allow-input: true}\r\n",
|
|
"\r\n",
|
|
"%cd /content\r\n",
|
|
"!git clone https://github.com/LostRuins/koboldcpp\r\n",
|
|
"%cd /content/koboldcpp\r\n",
|
|
"kvers = !(cat koboldcpp.py | grep 'KcppVersion = ' | cut -d '\"' -f2)\r\n",
|
|
"kvers = kvers[0]\r\n",
|
|
"!echo Finding prebuilt binary for {kvers}\r\n",
|
|
"!wget -c https://huggingface.co/concedo/koboldcpp/resolve/main/prebuilt_binaries/{kvers}.so\r\n",
|
|
"!test -f {kvers}.so && mv {kvers}.so koboldcpp_cublas.so || echo Prebuilt Binary Does Not Exist\r\n",
|
|
"!test -f koboldcpp_cublas.so && echo Prebuilt Binary Exists || make koboldcpp_cublas LLAMA_CUBLAS=1\r\n",
|
|
"!cp koboldcpp_cublas.so koboldcpp_cublas.dat\r\n",
|
|
"!wget $Model -O model.ggml\r\n",
|
|
"!wget -c https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64\r\n",
|
|
"!chmod +x cloudflared-linux-amd64\r\n",
|
|
"!nohup ./cloudflared-linux-amd64 tunnel --url http://localhost:5001 &\r\n",
|
|
"!sleep 8\r\n",
|
|
"!cat nohup.out\r\n",
|
|
"!python koboldcpp.py model.ggml --usecublas 0 mmq --multiuser --gpulayers $Layers --contextsize $ContextSize --hordeconfig concedo 1 1 --onready \"echo Connect to the link below && cat nohup.out | grep trycloudflare.com && rm nohup.out\"\r\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"accelerator": "GPU",
|
|
"colab": {
|
|
"authorship_tag": "",
|
|
"gpuType": "T4",
|
|
"include_colab_link": true,
|
|
"private_outputs": true,
|
|
"provenance": []
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"name": "python"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
} |