vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 09:34:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

pilled the new dequants for clblast, fixed some ooms

Browse source
This commit is contained in:
Concedo 2023-04-30 14:15:44 +08:00
parent 0061b90ec6
commit b3315459c7
6 changed files with 242 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

2
otherarch/gptj_v2.cpp
View file

@ -362,7 +362,7 @@ bool gptj_eval(
static size_t buf_size = 256u*1024*1024;
static void * buf = malloc(buf_size);
if (mem_per_token > 0 && mem_per_token*N*1.9 > buf_size) {
if (mem_per_token > 0 && (mem_per_token*N*2 + 16u*1024*1024) > buf_size) {
const size_t buf_size_new = 320u*1024*1024 + 2*(mem_per_token*N); // add 10% to account for ggml object overhead
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);