vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 17:44:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

better way to handle termux compatibility (+2 squashed commit)

Browse source 
Squashed commit:

[301986f11] better way to handle termux compatibility

[16b03b225] updated lite
This commit is contained in:
Concedo 2024-12-10 23:01:28 +08:00
parent f6aebb1449
commit 4548d893ee
7 changed files with 58 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

28
Makefile
View file

@ -21,6 +21,10 @@ ifndef UNAME_M
UNAME_M := $(shell uname -m) UNAME_M := $(shell uname -m)
endif endif
ifndef UNAME_O
UNAME_O := $(shell uname -o)
endif
ifneq ($(shell grep -e "Arch Linux" -e "ID_LIKE=arch" /etc/os-release 2>/dev/null),) ifneq ($(shell grep -e "Arch Linux" -e "ID_LIKE=arch" /etc/os-release 2>/dev/null),)
ARCH_ADD = -lcblas ARCH_ADD = -lcblas
endif endif
@ -133,6 +137,9 @@ ifdef LLAMA_PERF
CXXFLAGS += -DGGML_PERF CXXFLAGS += -DGGML_PERF
endif endif
CCV := $(shell $(CC) --version | head -n 1)
CXXV := $(shell $(CXX) --version | head -n 1)
# Architecture specific # Architecture specific
# TODO: probably these flags need to be tweaked on some architectures # TODO: probably these flags need to be tweaked on some architectures
# feel free to update the Makefile for your architecture and send a pull request or issue # feel free to update the Makefile for your architecture and send a pull request or issue
@ -315,9 +322,19 @@ ifneq ($(filter aarch64%,$(UNAME_M)),)
CFLAGS += CFLAGS +=
CXXFLAGS += CXXFLAGS +=
else else
# sve is cooked so we are disabling it # sve is cooked on termux so we are disabling it
CFLAGS += -mcpu=native -DLLAMA_NOSVE ifeq ($(UNAME_O), Android)
CXXFLAGS += -mcpu=native -DLLAMA_NOSVE ifneq ($(findstring clang, $(CCV)), )
CFLAGS += -mcpu=native+nosve
CXXFLAGS += -mcpu=native+nosve
else
CFLAGS += -mcpu=native
CXXFLAGS += -mcpu=native
endif
else
CFLAGS += -mcpu=native
CXXFLAGS += -mcpu=native
endif
endif endif
endif endif
@ -406,17 +423,16 @@ else
endif endif
endif endif
CCV := $(shell $(CC) --version | head -n 1)
CXXV := $(shell $(CXX) --version | head -n 1)
# #
# Print build information # Print build information
# #
$(info I llama.cpp build info: ) $(info I koboldcpp build info: )
$(info I UNAME_S: $(UNAME_S)) $(info I UNAME_S: $(UNAME_S))
$(info I UNAME_P: $(UNAME_P)) $(info I UNAME_P: $(UNAME_P))
$(info I UNAME_M: $(UNAME_M)) $(info I UNAME_M: $(UNAME_M))
$(info I UNAME_O: $(UNAME_O))
$(info I CFLAGS: $(CFLAGS)) $(info I CFLAGS: $(CFLAGS))
$(info I CXXFLAGS: $(CXXFLAGS)) $(info I CXXFLAGS: $(CXXFLAGS))
$(info I LDFLAGS: $(LDFLAGS)) $(info I LDFLAGS: $(LDFLAGS))

4
ggml/src/ggml-cpu/ggml-cpu-aarch64.c
View file

@ -741,7 +741,7 @@ void ggml_gemv_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *
UNUSED(blocklen); UNUSED(blocklen);
#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)
#if defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE) #if defined(__ARM_FEATURE_SVE)
if (ggml_cpu_has_sve() && ggml_cpu_get_sve_cnt() == QK8_0) { if (ggml_cpu_has_sve() && ggml_cpu_get_sve_cnt() == QK8_0) {
const void * b_ptr = vx; const void * b_ptr = vx;
const void * a_ptr = vy; const void * a_ptr = vy;
@ -2081,7 +2081,7 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *
UNUSED(blocklen); UNUSED(blocklen);
#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)
#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) && ! defined(LLAMA_NOSVE) #if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
if (ggml_cpu_has_sve() && ggml_cpu_has_matmul_int8() && ggml_cpu_get_sve_cnt() == QK8_0) { if (ggml_cpu_has_sve() && ggml_cpu_has_matmul_int8() && ggml_cpu_get_sve_cnt() == QK8_0) {
const void * b_ptr = vx; const void * b_ptr = vx;
const void * a_ptr = vy; const void * a_ptr = vy;

2
ggml/src/ggml-cpu/ggml-cpu-impl.h
View file

@ -59,7 +59,7 @@ struct ggml_compute_params {
#endif #endif
#endif #endif
#if defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE) #if defined(__ARM_FEATURE_SVE)
#include <arm_sve.h> #include <arm_sve.h>
#include <sys/prctl.h> #include <sys/prctl.h>
#endif #endif

4
ggml/src/ggml-cpu/ggml-cpu-quants.c
View file

@ -1829,7 +1829,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r
int ib = 0; int ib = 0;
float sumf = 0; float sumf = 0;
#if defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE) #if defined(__ARM_FEATURE_SVE)
svfloat32_t sumv0 = svdup_n_f32(0.0f); svfloat32_t sumv0 = svdup_n_f32(0.0f);
svfloat32_t sumv1 = svdup_n_f32(0.0f); svfloat32_t sumv1 = svdup_n_f32(0.0f);
@ -3419,7 +3419,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r
int ib = 0; int ib = 0;
float sumf = 0; float sumf = 0;
#if defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE) #if defined(__ARM_FEATURE_SVE)
svfloat32_t sumv0 = svdup_n_f32(0.0f); svfloat32_t sumv0 = svdup_n_f32(0.0f);
svfloat32_t sumv1 = svdup_n_f32(0.0f); svfloat32_t sumv1 = svdup_n_f32(0.0f);

10
ggml/src/ggml-cpu/ggml-cpu.c
View file

@ -40,7 +40,7 @@
#include <omp.h> #include <omp.h>
#endif #endif
#if (defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE)) || defined(__ARM_FEATURE_MATMUL_INT8) #if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8)
#undef GGML_USE_LLAMAFILE #undef GGML_USE_LLAMAFILE
#endif #endif
@ -2442,7 +2442,7 @@ static void ggml_init_arm_arch_features(void) {
ggml_arm_arch_features.has_i8mm = !!(hwcap2 & HWCAP2_I8MM); ggml_arm_arch_features.has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
ggml_arm_arch_features.has_sve = !!(hwcap & HWCAP_SVE); ggml_arm_arch_features.has_sve = !!(hwcap & HWCAP_SVE);
#if defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE) #if defined(__ARM_FEATURE_SVE)
ggml_arm_arch_features.sve_cnt = PR_SVE_VL_LEN_MASK & prctl(PR_SVE_GET_VL); ggml_arm_arch_features.sve_cnt = PR_SVE_VL_LEN_MASK & prctl(PR_SVE_GET_VL);
#endif #endif
#elif defined(__APPLE__) #elif defined(__APPLE__)
@ -2479,7 +2479,7 @@ static void ggml_init_arm_arch_features(void) {
ggml_arm_arch_features.has_i8mm = 0; ggml_arm_arch_features.has_i8mm = 0;
#endif #endif
#if defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE) #if defined(__ARM_FEATURE_SVE)
ggml_arm_arch_features.has_sve = 1; ggml_arm_arch_features.has_sve = 1;
ggml_arm_arch_features.sve_cnt = 16; ggml_arm_arch_features.sve_cnt = 16;
#else #else
@ -13946,7 +13946,7 @@ int ggml_cpu_has_dotprod(void) {
} }
int ggml_cpu_has_sve(void) { int ggml_cpu_has_sve(void) {
#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE) #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)
return ggml_arm_arch_features.has_sve; return ggml_arm_arch_features.has_sve;
#else #else
return 0; return 0;
@ -13962,7 +13962,7 @@ int ggml_cpu_has_matmul_int8(void) {
} }
int ggml_cpu_get_sve_cnt(void) { int ggml_cpu_get_sve_cnt(void) {
#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE) #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)
return ggml_arm_arch_features.sve_cnt; return ggml_arm_arch_features.sve_cnt;
#else #else
return 0; return 0;

2
ggml/src/ggml-impl.h
View file

@ -10,7 +10,7 @@
#include <stdint.h> #include <stdint.h>
#include <string.h> #include <string.h>
#if defined(__ARM_FEATURE_SVE) && ! defined(LLAMA_NOSVE) #ifdef __ARM_FEATURE_SVE
#include <arm_sve.h> #include <arm_sve.h>
#endif // __ARM_FEATURE_SVE #endif // __ARM_FEATURE_SVE

42
klite.embd
View file

@ -263,7 +263,7 @@ Current version indicated by LITEVER below.
max-width: 100%; max-width: 100%;
height: auto; height: auto;
} }
.txtchunk, ai_context_koboldlite_internal, #gametext, .chat_received_withd_msg p,.chat_sent_msg p { .txtchunk, #gametext, .chat_received_withd_msg p,.chat_sent_msg p {
white-space: pre-wrap; white-space: pre-wrap;
} }
@ -12169,7 +12169,7 @@ Current version indicated by LITEVER below.
{ {
let newgenlc = newgen.toLowerCase().trim(); let newgenlc = newgen.toLowerCase().trim();
if (newgenlc.startsWith("draw ") || if (newgenlc.startsWith("draw ") ||
newgenlc.match(/\b(?:draw (?:a|an)\b)|(?:draw me (?:a|an)\b)|(?:(?:draw|show me|generate|create|make|illustrate|visualize|produce|give me)(?:\s\w+){0,3}\s(?:image|picture|drawing|photo))/)) newgenlc.match(/\b(?:draw (?:a|an)\b)|(?:draw me (?:a|an)\b)|(?:(?:draw|show me|generate|create|make|illustrate|visualize|produce|give me)(?:\s\w+){0,4}\s(?:image|picture|drawing|photo))/))
{ {
img_gen_trigger_prompt = newgen; img_gen_trigger_prompt = newgen;
doNotGenerate = true; doNotGenerate = true;
@ -16186,11 +16186,11 @@ Current version indicated by LITEVER below.
if(localsettings.opmode==3 && localsettings.gui_type_chat==1) if(localsettings.opmode==3 && localsettings.gui_type_chat==1)
{ {
render_enhanced_chat(textToRender); render_messenger_ui(textToRender);
} }
else else
{ {
document.getElementById("chat_msg_body").innerHTML = render_enhanced_chat_instruct(textToRender,false); document.getElementById("chat_msg_body").innerHTML = render_aesthetic_ui(textToRender,false);
} }
if ((localsettings.opmode == 3 && localsettings.chatopponent != "")||localsettings.opmode == 4||localsettings.opmode==2) { if ((localsettings.opmode == 3 && localsettings.chatopponent != "")||localsettings.opmode == 4||localsettings.opmode==2) {
document.getElementById("cht_inp_bg").classList.add("shorter"); document.getElementById("cht_inp_bg").classList.add("shorter");
@ -16879,7 +16879,7 @@ Current version indicated by LITEVER below.
return chatunits; return chatunits;
} }
function render_enhanced_chat(input) function render_messenger_ui(input)
{ {
var chatbody = document.getElementById("chat_msg_body"); var chatbody = document.getElementById("chat_msg_body");
if(!chatbody) if(!chatbody)
@ -18066,7 +18066,7 @@ Current version indicated by LITEVER below.
} }
} }
function render_enhanced_chat_instruct(input, isPreview) //class suffix string used to prevent defined styles from leaking into global scope function render_aesthetic_ui(input, isPreview) //class suffix string used to prevent defined styles from leaking into global scope
{ {
if(!isPreview) if(!isPreview)
{ {
@ -18078,7 +18078,15 @@ Current version indicated by LITEVER below.
document.getElementById('enhancedchatinterface_inner').style.backgroundColor = null; document.getElementById('enhancedchatinterface_inner').style.backgroundColor = null;
} }
} }
let as = aestheticInstructUISettings;
let classSuffixStr = isPreview ? "prv" : ""; let classSuffixStr = isPreview ? "prv" : "";
let portraitsStyling = // Also, implement portraits as css classes. Now chat entries can reuse them instead of recreating them.
`<style>
.you-portrait-image`+classSuffixStr+` {margin: 10px 6px; background:url(`+ as.you_portrait +`); background-clip: content-box; background-position: 50% 50%; background-size: 100% 100%; background-origin: content-box; background-repeat: no-repeat; border:none;}
.AI-portrait-image`+classSuffixStr+` {margin: 10px 6px; background:url(`+ (as.AI_portrait!="default"?as.AI_portrait:niko_square) +`); background-clip: content-box; background-position: 50% 50%; background-size: 100% 100%; background-origin: content-box; background-repeat: no-repeat; border:none;}
</style>
`;
const contextDict = { sysOpen: '<sys_context_koboldlite_internal>', youOpen: '<user_context_koboldlite_internal>', AIOpen: '<AI_context_koboldlite_internal>', closeTag: '<end_of_context_koboldlite_internal>' } const contextDict = { sysOpen: '<sys_context_koboldlite_internal>', youOpen: '<user_context_koboldlite_internal>', AIOpen: '<AI_context_koboldlite_internal>', closeTag: '<end_of_context_koboldlite_internal>' }
let you = "$UnusedTagMatch$"; let bot = "$UnusedTagMatch$"; // Instruct tags will be used to wrap text in styled bubbles. let you = "$UnusedTagMatch$"; let bot = "$UnusedTagMatch$"; // Instruct tags will be used to wrap text in styled bubbles.
if(localsettings.opmode==3||localsettings.opmode==4) if(localsettings.opmode==3||localsettings.opmode==4)
@ -18087,7 +18095,6 @@ Current version indicated by LITEVER below.
bot = get_instruct_endtag(); bot = get_instruct_endtag();
} }
let as = aestheticInstructUISettings; // ..and use this as shortcut to avoid typing it each time.
if(localsettings.opmode==3) if(localsettings.opmode==3)
{ {
if(!input.startsWith("\n")) if(!input.startsWith("\n"))
@ -18152,12 +18159,6 @@ Current version indicated by LITEVER below.
} }
} }
let portraitsStyling = // Also, implement portraits as css classes. Now chat entries can reuse them instead of recreating them.
`<style>
.you-portrait-image`+classSuffixStr+` {margin: 10px 6px; background:url(`+ as.you_portrait +`); background-clip: content-box; background-position: 50% 50%; background-size: 100% 100%; background-origin: content-box; background-repeat: no-repeat; border:none;}
.AI-portrait-image`+classSuffixStr+` {margin: 10px 6px; background:url(`+ (as.AI_portrait!="default"?as.AI_portrait:niko_square) +`); background-clip: content-box; background-position: 50% 50%; background-size: 100% 100%; background-origin: content-box; background-repeat: no-repeat; border:none;}
</style>
`;
// We'll transform the input to a well-formatted HTML string that'll contain the whole visuals for the Aesthetic Instruct Mode. Effectively we're styling the input. // We'll transform the input to a well-formatted HTML string that'll contain the whole visuals for the Aesthetic Instruct Mode. Effectively we're styling the input.
let noSystemPrompt = input.trim().startsWith(you.trim()) || input.trim().startsWith(bot.trim()); let noSystemPrompt = input.trim().startsWith(you.trim()) || input.trim().startsWith(bot.trim());
@ -18167,6 +18168,13 @@ Current version indicated by LITEVER below.
if (synchro_pending_stream != "" && !isPreview) { if (synchro_pending_stream != "" && !isPreview) {
newbodystr += getStreamingText(); newbodystr += getStreamingText();
} // Add the pending stream if it's needed. This will add any streamed text to a new bubble for the AI. } // Add the pending stream if it's needed. This will add any streamed text to a new bubble for the AI.
else{
let codeblockcount = (newbodystr.match(/```/g) || []).length;
if(codeblockcount>0 && codeblockcount%2!=0 )
{
newbodystr += "```"; //force end code block
}
}
newbodystr += contextDict.closeTag + '</p></div></div>'; // Lastly, append the closing div so our body's raw form is completed. newbodystr += contextDict.closeTag + '</p></div></div>'; // Lastly, append the closing div so our body's raw form is completed.
if (aestheticInstructUISettings.use_markdown) { if (aestheticInstructUISettings.use_markdown) {
@ -18177,8 +18185,8 @@ Current version indicated by LITEVER below.
let internalHTMLparts = []; // We'll cache the embedded HTML parts here to keep them intact. let internalHTMLparts = []; // We'll cache the embedded HTML parts here to keep them intact.
for (let role of aestheticTextStyleRoles) { // ..starting by the "speech" and *actions* for each role. for (let role of aestheticTextStyleRoles) { // ..starting by the "speech" and *actions* for each role.
let styleRole = aestheticInstructUISettings.use_uniform_colors ? 'uniform' : role; // Uniform role is preferred if it's active on the settings. let styleRole = aestheticInstructUISettings.use_uniform_colors ? 'uniform' : role; // Uniform role is preferred if it's active on the settings.
newbodystr = newbodystr.replace(new RegExp(`${contextDict[`${role}Open`]}([^]*?)${contextDict.closeTag}`, 'g'), (match, p) => { newbodystr = newbodystr.replace(new RegExp(`${contextDict[`${role}Open`]}([^]*?)${contextDict.closeTag}`, 'g'), (match, captured) => {
let replacedText = match.replace(/<[^>]*>/g, (htmlPart) => { internalHTMLparts.push(htmlPart); return `<internal_html_${internalHTMLparts.length - 1}>`; }); let replacedText = captured.replace(/<[^>]*>/g, (htmlPart) => { internalHTMLparts.push(htmlPart); return `<internal_html_${internalHTMLparts.length - 1}>`; });
replacedText = replacedText.replace(bold_regex, wrapperSpan(styleRole, 'action')); // Apply the actions style to *actions*. replacedText = replacedText.replace(bold_regex, wrapperSpan(styleRole, 'action')); // Apply the actions style to *actions*.
replacedText = replacedText.replace(italics_regex, wrapperSpan(styleRole, 'action')); // Apply the actions style to *actions*. replacedText = replacedText.replace(italics_regex, wrapperSpan(styleRole, 'action')); // Apply the actions style to *actions*.
replacedText = replacedText.replace(/“(.*?)”/g, wrapperSpan(styleRole, 'speech')); // Apply the speech style to "speech". replacedText = replacedText.replace(/“(.*?)”/g, wrapperSpan(styleRole, 'speech')); // Apply the speech style to "speech".
@ -18187,7 +18195,7 @@ Current version indicated by LITEVER below.
{ {
replacedText = simpleMarkdown(replacedText); replacedText = simpleMarkdown(replacedText);
} }
return replacedText; return `<span>${replacedText}</span>`;
}); });
} }
newbodystr = newbodystr.replace(/<internal_html_(.*?)>/gm, (match, p) => { newbodystr = newbodystr.replace(/<internal_html_(.*?)>/gm, (match, p) => {
@ -18304,7 +18312,7 @@ Current version indicated by LITEVER below.
preview = replaceAll(preview,'\n[USER_REPLY]\n', ""); preview = replaceAll(preview,'\n[USER_REPLY]\n', "");
preview = replaceAll(preview,'\n[AI_REPLY]\n', ""); preview = replaceAll(preview,'\n[AI_REPLY]\n', "");
} }
document.getElementById('aesthetic_text_preview').innerHTML = render_enhanced_chat_instruct(preview,true); document.getElementById('aesthetic_text_preview').innerHTML = render_aesthetic_ui(preview,true);
} }
</script> </script>