diff --git a/llm_config.py b/llm_config.py index 6544730..5204d35 100644 --- a/llm_config.py +++ b/llm_config.py @@ -2,23 +2,6 @@ LLM_TYPE = "ollama" # Options: 'ollama', 'openai', 'anthropic' -# LLM settings for llama_cpp -MODEL_PATH = "/home/james/llama.cpp/models/gemma-2-9b-it-Q6_K.gguf" # Replace with your llama.cpp models filepath - -LLM_CONFIG_LLAMA_CPP = { - "llm_type": "llama_cpp", - "model_path": MODEL_PATH, - "n_ctx": 20000, # context size - "n_gpu_layers": 0, # number of layers to offload to GPU (-1 for all, 0 for none) - "n_threads": 8, # number of threads to use - "temperature": 0.7, # temperature for sampling - "top_p": 0.9, # top p for sampling - "top_k": 40, # top k for sampling - "repeat_penalty": 1.1, # repeat penalty - "max_tokens": 1024, # max tokens to generate - "stop": ["User:", "\n\n"] # stop sequences -} - # LLM settings for Ollama LLM_CONFIG_OLLAMA = { "llm_type": "ollama",