")).replace(/``(.*?)``/gm,function(e,l){return"
"+r(l).replace(/`/g,"`")+""})).replace(/`(.*?)`/gm,"$1")).replace(/^\>\> (.*$)/gm,"")).replace(/^\> (.*$)/gm,"$1
$1")).replace(/<\/blockquote\>\n
")).replace(/!\[(.*?)\]\((.*?) "(.*?)"\)/gm,'
- $1
- $1
- /gm,"
- ")).replace(/^<[ou]l>
- (.*\%SpcStg\%.*\%SpcEtg\%.*)<\/li><\/[ou]l\>/gm,"$1").replace(/^\s{2,6}[\*+-][ .](.*)/gm,"
- $1
- $1
- /gm,"
- ")).replace(/\*\*\*(\w.*?[^\\])\*\*\*/gm,"$1")).replace(/\*\*(\w.*?[^\\])\*\*/gm,"$1")).replace(/\*(\w.*?[^\\])\*/gm,"$1")).replace(/___(\w.*?[^\\])___/gm,"$1")).replace(/__(\w.*?[^\\])__/gm,"$1")).replace(/~~(\w.*?)~~/gm,"
$1")).replace(/\^\^(\w.*?)\^\^/gm,"$1")).replace(/\{\{(\w.*?)\}\}/gm,"$1")).replace(/^((?:\|[^|\r\n]*[^|\r\n\s]\s*)+\|(?:\r?\n|\r|))+/gm,function (matchedTable){return convertMarkdownTableToHtml(matchedTable);})).replace(/ \n/g,"\n
") + .replace(/^[\*+-][ .](.*)/gm,"- $1
- $1
- /gm,"
- ")).replace(/^<[ou]l>
- (.*\%SpcStg\%.*\%SpcEtg\%.*)<\/li><\/[ou]l\>/gm,"$1").replace(/^\s{2,6}[\*+-][ .](.*)/gm,"
- $1
- $1
- /gm,"
- ")).replace(/\*\*\*(\w.*?[^\\])\*\*\*/gm,"$1")).replace(/\*\*(\w.*?[^\\])\*\*/gm,"$1")).replace(/\*(\w.*?[^\\])\*/gm,"$1")).replace(/___(\w.*?[^\\])___/gm,"$1")).replace(/__(\w.*?[^\\])__/gm,"$1")).replace(/~~(\w.*?)~~/gm,"
$1")).replace(/\^\^(\w.*?)\^\^/gm,"$1")).replace(/\{\{(\w.*?)\}\}/gm,"$1")).replace(/^((?:\|[^|\r\n]*[^|\r\n\s]\s*)+\|(?:\r?\n|\r|))+/gm,function (matchedTable){return convertMarkdownTableToHtml(matchedTable);})).replace(/ \n/g,"\n
") //.replace(/\n\s*\n/g,"\n\n") ).replace(/^ {4,10}(.*)/gm,function(e,l){return"
"})).replace(/^\t(.*)/gm,function(e,l){return""+r(l)+"
"})).replace(/<\/code\><\/pre\>\n"+r(l)+""))>=0;)n=e.indexOf("",a),p+=c(e.substr(0,a))+e.substr(a+6,n>0?n-a-6:mdtext.length),e=e.substr(n+7);return p+c(e)} @@ -8119,7 +8120,7 @@ Current version: 116 localsettings.last_selected_preset = document.getElementById("presets").value; //clean and clamp invalid values - localsettings.max_context_length = cleannum(localsettings.max_context_length, 8, 99999); + localsettings.max_context_length = cleannum(localsettings.max_context_length, 8, 999999); localsettings.max_length = cleannum(localsettings.max_length, 1, (localsettings.max_context_length-1)); localsettings.temperature = cleannum(localsettings.temperature, 0.01, 5); localsettings.rep_pen = cleannum(localsettings.rep_pen, 0.1, 5); diff --git a/koboldcpp.py b/koboldcpp.py index 86c0dc2d1..96e47ba87 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -22,6 +22,14 @@ logit_bias_max = 16 bias_min_value = -100.0 bias_max_value = 100.0 +class logit_bias(ctypes.Structure): + _fields_ = [("token_id", ctypes.c_int32), + ("bias", ctypes.c_float)] + +class token_count_outputs(ctypes.Structure): + _fields_ = [("count", ctypes.c_int), + ("ids", ctypes.POINTER(ctypes.c_int))] + class load_model_inputs(ctypes.Structure): _fields_ = [("threads", ctypes.c_int), ("blasthreads", ctypes.c_int), @@ -49,10 +57,6 @@ class load_model_inputs(ctypes.Structure): ("banned_tokens", ctypes.c_char_p * ban_token_max), ("tensor_split", ctypes.c_float * tensor_split_max)] -class logit_bias(ctypes.Structure): - _fields_ = [("token_id", ctypes.c_int32), - ("bias", ctypes.c_float)] - class generation_inputs(ctypes.Structure): _fields_ = [("seed", ctypes.c_int), ("prompt", ctypes.c_char_p), @@ -103,12 +107,9 @@ class sd_generation_inputs(ctypes.Structure): class sd_generation_outputs(ctypes.Structure): _fields_ = [("status", ctypes.c_int), + ("data_length", ctypes.c_uint), ("data", ctypes.c_char_p)] -class token_count_outputs(ctypes.Structure): - _fields_ = [("count", ctypes.c_int), - ("ids", ctypes.POINTER(ctypes.c_int))] - handle = None def getdirpath(): @@ -273,10 +274,10 @@ def init_library(): handle.abort_generate.restype = ctypes.c_bool handle.token_count.restype = token_count_outputs handle.get_pending_output.restype = ctypes.c_char_p - handle.load_model_sd.argtypes = [sd_load_model_inputs] - handle.load_model_sd.restype = ctypes.c_bool - handle.generate_sd.argtypes = [sd_generation_inputs] - handle.generate_sd.restype = sd_generation_outputs + handle.sd_load_model.argtypes = [sd_load_model_inputs] + handle.sd_load_model.restype = ctypes.c_bool + handle.sd_generate.argtypes = [sd_generation_inputs] + handle.sd_generate.restype = sd_generation_outputs def load_model(model_filename): global args @@ -469,14 +470,29 @@ def generate(prompt, memory="", max_length=32, max_context_length=512, temperatu return outstr -def load_model_sd(model_filename): +def sd_load_model(model_filename): global args inputs = sd_load_model_inputs() inputs.debugmode = args.debugmode inputs.model_filename = model_filename.encode("UTF-8") - ret = handle.load_model_sd(inputs) + ret = handle.sd_load_model(inputs) return ret +def sd_generate(prompt, negative_prompt="", cfg_scale=5, sample_steps=20, seed=-1, sample_method="euler a"): + global maxctx, args, currentusergenkey, totalgens, pendingabortkey + inputs = sd_generation_inputs() + inputs.prompt = prompt.encode("UTF-8") + inputs.negative_prompt = negative_prompt.encode("UTF-8") + inputs.cfg_scale = cfg_scale + inputs.sample_steps = sample_steps + inputs.seed = seed + inputs.sample_method = sample_method.encode("UTF-8") + ret = handle.sd_generate(inputs) + outstr = "" + if ret.status==1: + outstr = ret.data.decode("UTF-8","ignore") + return outstr + def utfprint(str): try: print(str) @@ -2567,7 +2583,7 @@ def main(launch_args,start_server=True): time.sleep(3) sys.exit(2) imgmodel = os.path.abspath(imgmodel) - loadok = load_model_sd(imgmodel) + loadok = sd_load_model(imgmodel) print("Load Image Model OK: " + str(loadok)) if not loadok: exitcounter = 999 diff --git a/model_adapter.h b/model_adapter.h index 75e90434e..51cd75ced 100644 --- a/model_adapter.h +++ b/model_adapter.h @@ -78,7 +78,7 @@ bool gpttype_generate_abort(); const std::string & gpttype_get_pending_output(); std::vectorgpttype_get_token_arr(const std::string & input); -bool sdtype_load_model(const load_sd_model_inputs inputs); +bool sdtype_load_model(const sd_load_model_inputs inputs); sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs); void timer_start(); diff --git a/otherarch/sdcpp/CMakeLists.txt b/otherarch/sdcpp/CMakeLists.txt deleted file mode 100644 index a9bedebef..000000000 --- a/otherarch/sdcpp/CMakeLists.txt +++ /dev/null @@ -1,95 +0,0 @@ -cmake_minimum_required(VERSION 3.12) -project("stable-diffusion") - -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -endif() - -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) - -if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) - set(SD_STANDALONE ON) -else() - set(SD_STANDALONE OFF) -endif() - -# -# Option list -# - -# general -option(SD_CUBLAS "sd: cuda backend" OFF) -option(SD_HIPBLAS "sd: rocm backend" OFF) -option(SD_METAL "sd: metal backend" OFF) -option(SD_FLASH_ATTN "sd: use flash attention for x4 less memory usage" OFF) -option(SD_FAST_SOFTMAX "sd: x1.5 faster softmax, indeterministic (sometimes, same seed don't generate same image), cuda only" OFF) -option(SD_BUILD_SHARED_LIBS "sd: build shared libs" OFF) - -if(SD_CUBLAS) - message("Use CUBLAS as backend stable-diffusion") - set(GGML_CUBLAS ON) - add_definitions(-DSD_USE_CUBLAS) -endif() - -if(SD_METAL) - message("Use Metal as backend stable-diffusion") - set(GGML_METAL ON) - add_definitions(-DSD_USE_METAL) -endif() - -if (SD_HIPBLAS) - message("Use HIPBLAS as backend stable-diffusion") - set(GGML_HIPBLAS ON) - add_definitions(-DSD_USE_CUBLAS) - if(SD_FAST_SOFTMAX) - set(GGML_CUDA_FAST_SOFTMAX ON) - endif() -endif () - -if(SD_FLASH_ATTN) - message("Use Flash Attention for memory optimization") - add_definitions(-DSD_USE_FLASH_ATTENTION) -endif() - -set(SD_LIB stable-diffusion) - -file(GLOB SD_LIB_SOURCES - "*.h" - "*.cpp" - "*.hpp" -) - -# we can get only one share lib -if(SD_BUILD_SHARED_LIBS) - message("Build shared library") - set(BUILD_SHARED_LIBS OFF) - message(${SD_LIB_SOURCES}) - add_library(${SD_LIB} SHARED ${SD_LIB_SOURCES}) - add_definitions(-DSD_BUILD_SHARED_LIB) - target_compile_definitions(${SD_LIB} PRIVATE -DSD_BUILD_DLL) - set(CMAKE_POSITION_INDEPENDENT_CODE ON) -else() - message("Build static library") - add_library(${SD_LIB} STATIC ${SD_LIB_SOURCES}) -endif() - - -set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) - - -# deps -add_subdirectory(ggml) - -add_subdirectory(thirdparty) - -target_link_libraries(${SD_LIB} PUBLIC ggml zip) -target_include_directories(${SD_LIB} PUBLIC . thirdparty) -target_compile_features(${SD_LIB} PUBLIC cxx_std_11) - - -add_subdirectory(examples) - diff --git a/otherarch/sdcpp/sd_adapter.cpp b/otherarch/sdcpp/sdtype_adapter.cpp similarity index 98% rename from otherarch/sdcpp/sd_adapter.cpp rename to otherarch/sdcpp/sdtype_adapter.cpp index 629db9f62..69febaeae 100644 --- a/otherarch/sdcpp/sd_adapter.cpp +++ b/otherarch/sdcpp/sdtype_adapter.cpp @@ -125,7 +125,7 @@ static void sd_logger_callback(enum sd_log_level_t level, const char* log, void* } } -bool sdtype_load_model(const load_sd_model_inputs inputs) { +bool sdtype_load_model(const sd_load_model_inputs inputs) { printf("\nSelected Image Model: %s\n",inputs.model_filename); @@ -174,6 +174,7 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) printf("\nError: KCPP SD is not initialized!\n"); output.data = nullptr; output.status = 0; + output.data_length = 0; return output; } uint8_t * input_image_buffer = NULL; @@ -233,6 +234,7 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) printf("\nKCPP SD generate failed!\n"); output.data = nullptr; output.status = 0; + output.data_length = 0; return output; } @@ -255,5 +257,6 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) output.data = nullptr; output.status = 1; + output.data_length = 0; return output; }