Merge branch 'upstream' into concedo_experimental

# Conflicts:
#	examples/server/README.md
#	src/llama-model.cpp

examples/server/webui/src/components/MarkdownDisplay.tsx

@@ -23,6 +23,7 @@ export default function MarkdownDisplay({ content }: { content: string }) {
         button: (props) => (
           <CopyCodeButton {...props} origContent={preprocessedContent} />
         ),
+        // note: do not use "pre", "p" or other basic html elements here, it will cause the node to re-render when the message is being generated (this should be a bug with react-markdown, not sure how to fix it)
       }}
     >
       {preprocessedContent}

examples/server/webui/src/components/SettingDialog.tsx

@@ -3,6 +3,7 @@ import { useAppContext } from '../utils/app.context';
 import { CONFIG_DEFAULT, CONFIG_INFO } from '../Config';
 import { isDev } from '../Config';
 import StorageUtils from '../utils/storage';
+import { isBoolean, isNumeric, isString } from '../utils/misc';
 
 type SettKey = keyof typeof CONFIG_DEFAULT;
 
@@ -52,7 +53,42 @@ export default function SettingDialog({
   };
 
   const handleSave = () => {
-    saveConfig(localConfig);
+    // copy the local config to prevent direct mutation
+    const newConfig: typeof CONFIG_DEFAULT = JSON.parse(
+      JSON.stringify(localConfig)
+    );
+    // validate the config
+    for (const key in newConfig) {
+      const value = newConfig[key as SettKey];
+      const mustBeBoolean = isBoolean(CONFIG_DEFAULT[key as SettKey]);
+      const mustBeString = isString(CONFIG_DEFAULT[key as SettKey]);
+      const mustBeNumeric = isNumeric(CONFIG_DEFAULT[key as SettKey]);
+      if (mustBeString) {
+        if (!isString(value)) {
+          alert(`Value for ${key} must be string`);
+          return;
+        }
+      } else if (mustBeNumeric) {
+        const trimedValue = value.toString().trim();
+        const numVal = Number(trimedValue);
+        if (isNaN(numVal) || !isNumeric(numVal) || trimedValue.length === 0) {
+          alert(`Value for ${key} must be numeric`);
+          return;
+        }
+        // force conversion to number
+        // @ts-expect-error this is safe
+        newConfig[key] = numVal;
+      } else if (mustBeBoolean) {
+        if (!isBoolean(value)) {
+          alert(`Value for ${key} must be boolean`);
+          return;
+        }
+      } else {
+        console.error(`Unknown default type for key ${key}`);
+      }
+    }
+    if (isDev) console.log('Saving config', newConfig);
+    saveConfig(newConfig);
     onClose();
   };
 
@@ -66,6 +102,11 @@ export default function SettingDialog({
     onClose();
   };
 
+  const onChange = (key: SettKey) => (value: string | boolean) => {
+    // note: we do not perform validation here, because we may get incomplete value as user is still typing it
+    setLocalConfig({ ...localConfig, [key]: value });
+  };
+
   return (
     <dialog className={`modal ${show ? 'modal-open' : ''}`}>
       <div className="modal-box">
@@ -79,9 +120,7 @@ export default function SettingDialog({
             configKey="apiKey"
             configDefault={CONFIG_DEFAULT}
             value={localConfig.apiKey}
-            onChange={(value) =>
+            onChange={onChange('apiKey')}
-              setLocalConfig({ ...localConfig, apiKey: value })
-            }
           />
 
           <label className="form-control mb-2">
@@ -92,12 +131,7 @@ export default function SettingDialog({
               className="textarea textarea-bordered h-24"
               placeholder={`Default: ${CONFIG_DEFAULT.systemMessage}`}
               value={localConfig.systemMessage}
-              onChange={(e) =>
+              onChange={(e) => onChange('systemMessage')(e.target.value)}
-                setLocalConfig({
-                  ...localConfig,
-                  systemMessage: e.target.value,
-                })
-              }
             />
           </label>
 
@@ -107,9 +141,7 @@ export default function SettingDialog({
               configKey={key}
               configDefault={CONFIG_DEFAULT}
               value={localConfig[key]}
-              onChange={(value) =>
+              onChange={onChange(key)}
-                setLocalConfig({ ...localConfig, [key]: value })
-              }
             />
           ))}
 
@@ -123,9 +155,7 @@ export default function SettingDialog({
                 configKey="samplers"
                 configDefault={CONFIG_DEFAULT}
                 value={localConfig.samplers}
-                onChange={(value) =>
+                onChange={onChange('samplers')}
-                  setLocalConfig({ ...localConfig, samplers: value })
-                }
               />
               {OTHER_SAMPLER_KEYS.map((key) => (
                 <SettingsModalShortInput
@@ -133,9 +163,7 @@ export default function SettingDialog({
                   configKey={key}
                   configDefault={CONFIG_DEFAULT}
                   value={localConfig[key]}
-                  onChange={(value) =>
+                  onChange={onChange(key)}
-                    setLocalConfig({ ...localConfig, [key]: value })
-                  }
                 />
               ))}
             </div>
@@ -152,9 +180,7 @@ export default function SettingDialog({
                   configKey={key}
                   configDefault={CONFIG_DEFAULT}
                   value={localConfig[key]}
-                  onChange={(value) =>
+                  onChange={onChange(key)}
-                    setLocalConfig({ ...localConfig, [key]: value })
-                  }
                 />
               ))}
             </div>
@@ -171,10 +197,7 @@ export default function SettingDialog({
                   className="checkbox"
                   checked={localConfig.showThoughtInProgress}
                   onChange={(e) =>
-                    setLocalConfig({
+                    onChange('showThoughtInProgress')(e.target.checked)
-                      ...localConfig,
-                      showThoughtInProgress: e.target.checked,
-                    })
                   }
                 />
                 <span className="ml-4">
@@ -187,10 +210,7 @@ export default function SettingDialog({
                   className="checkbox"
                   checked={localConfig.excludeThoughtOnReq}
                   onChange={(e) =>
-                    setLocalConfig({
+                    onChange('excludeThoughtOnReq')(e.target.checked)
-                      ...localConfig,
-                      excludeThoughtOnReq: e.target.checked,
-                    })
                   }
                 />
                 <span className="ml-4">
@@ -220,10 +240,7 @@ export default function SettingDialog({
                   className="checkbox"
                   checked={localConfig.showTokensPerSecond}
                   onChange={(e) =>
-                    setLocalConfig({
+                    onChange('showTokensPerSecond')(e.target.checked)
-                      ...localConfig,
-                      showTokensPerSecond: e.target.checked,
-                    })
                   }
                 />
                 <span className="ml-4">Show tokens per second</span>
@@ -245,9 +262,7 @@ export default function SettingDialog({
                   className="textarea textarea-bordered h-24"
                   placeholder='Example: { "mirostat": 1, "min_p": 0.1 }'
                   value={localConfig.custom}
-                  onChange={(e) =>
+                  onChange={(e) => onChange('custom')(e.target.value)}
-                    setLocalConfig({ ...localConfig, custom: e.target.value })
-                  }
                 />
               </label>
             </div>

ggml/src/ggml-cpu/ggml-cpu.c

@@ -13900,9 +13900,13 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
             tp->ec    = GGML_STATUS_ABORTED;
         }
 
-        ggml_barrier(state->threadpool);
+        if (node_n + 1 < cgraph->n_nodes) {
+            ggml_barrier(state->threadpool);
+        }
     }
 
+    ggml_barrier(state->threadpool);
+
     return 0;
 }
 

ggml/src/ggml-cuda/mma.cuh

@@ -16,7 +16,7 @@
 #include "common.cuh"
 
 
-#if CUDART_VERSION >= 11800
+#if CUDART_VERSION >= 11080
 
 static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {
     int ret = 0;
@@ -50,7 +50,7 @@ static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {
     return ret_low | ret_high;
 }
 
-#endif // CUDART_VERSION >= 11800
+#endif // CUDART_VERSION >= 11080
 
 
 template <typename T>

src/llama-model.cpp

@@ -1280,8 +1280,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
 
     bool use_mmap_buffer = true;
 
-    LLAMA_LOG_INFO("%s: loading model tensors, this can take a while...", __func__);
+    LLAMA_LOG_INFO("%s: loading model tensors, this can take a while... (mmap = %s)\n", __func__, ml.use_mmap ? "true" : "false");
-    // LLAMA_LOG_INFO("%s: loading model tensors, this can take a while... (mmap = %s)\n", __func__, use_mmap_buffer ? "true" : "false");
 
     // build a list of buffer types for the CPU and GPU devices
     pimpl->cpu_buft_list = make_cpu_buft_list(devices);

src/llama.cpp

@@ -9471,7 +9471,6 @@ static struct llama_model * llama_model_load_from_file_impl(
         struct llama_model_params params) {
     ggml_time_init();
 
-
     unsigned cur_percentage = 0;
     if (params.progress_callback == NULL) {
         params.progress_callback_user_data = &cur_percentage;