vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-07-10 01:18:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'upstream' into concedo_experimental

Browse source 
# Conflicts:
#	.github/workflows/release.yml
#	.github/workflows/ui-build-self-hosted.yml
#	.github/workflows/ui-build.yml
#	.github/workflows/ui-publish.yml
#	.github/workflows/ui-self-hosted.yml
#	.github/workflows/ui.yml
#	.gitignore
#	README.md
#	docs/ops.md
#	docs/ops/Vulkan.csv
#	ggml/CMakeLists.txt
#	scripts/sync-ggml.last
#	scripts/sync_vendor.py
#	scripts/ui-assets.cmake
#	tests/test-jinja.cpp
#	tests/test-llama-archs.cpp
This commit is contained in:
Concedo 2026-06-14 11:26:16 +08:00
commit ea0351c71a
117 changed files with 11026 additions and 2390 deletions
Download patch file Download diff file Expand all files Collapse all files

7
common/arg.cpp
View file

@ -2244,6 +2244,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.image_max_tokens = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_IMAGE_MAX_TOKENS"));
add_opt(common_arg(
{"--mtmd-batch-max-tokens"}, "N",
string_format("maximum number of image tokens per batch when encoding images (default: %d)", params.mtmd_batch_max_tokens),
[](common_params & params, int value) {
params.mtmd_batch_max_tokens = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));
if (llama_supports_rpc()) {
add_opt(common_arg(
{"--rpc"}, "SERVERS",

1
common/common.h
View file

@ -576,6 +576,7 @@ struct common_params {
std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
int image_min_tokens = -1;
int image_max_tokens = -1;
int mtmd_batch_max_tokens = 1024;
// finetune
struct lr_opt lr;

35
common/fit.cpp
View file

@ -26,7 +26,7 @@ class common_params_fit_exception : public std::runtime_error {
using std::runtime_error::runtime_error;
};
std::vector<llama_device_memory_data> common_get_device_memory_data(
static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
const char * path_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
@ -150,6 +150,29 @@ std::vector<llama_device_memory_data> common_get_device_memory_data(
return ret;
}
common_device_memory_data_vec common_get_device_memory_data(
const char * path_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
ggml_log_level log_level) {
std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);
common_device_memory_data_vec ret(impl.size());
for (size_t i = 0; i < impl.size(); i++) {
ret[i].total = impl[i].total;
ret[i].free = impl[i].free;
ret[i].model = impl[i].mb.model;
ret[i].context = impl[i].mb.context;
ret[i].compute = impl[i].mb.compute;
}
return ret;
}
static void common_params_fit_impl(
const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
@ -169,7 +192,7 @@ static void common_params_fit_impl(
// step 1: get data for default parameters and check whether any changes are necessary in the first place
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
const dmds_t dmds_full = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
const dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
const size_t nd = devs.size(); // number of devices
std::vector<int64_t> margins; // this function uses int64_t rather than size_t for memory sizes to more conveniently handle deficits
@ -304,7 +327,7 @@ static void common_params_fit_impl(
int64_t sum_projected_used_min_ctx = 0;
cparams->n_ctx = n_ctx_min;
const dmds_t dmds_min_ctx = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
const dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
if (nd == 0) {
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
} else {
@ -482,7 +505,7 @@ static void common_params_fit_impl(
llama_model_params mparams_copy = *mparams;
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
const dmds_t dmd_nl = common_get_device_memory_data(
const dmds_t dmd_nl = common_get_device_memory_data_impl(
path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
LOG_TRC("%s: memory for test allocation by device:\n", func_name);
@ -510,7 +533,7 @@ static void common_params_fit_impl(
mparams->tensor_buft_overrides = tensor_buft_overrides;
LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
const dmds_t dmds_cpu_moe = common_get_device_memory_data(
const dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
for (size_t id = 0; id < nd; id++) {
@ -940,7 +963,7 @@ void common_fit_print(
uint32_t hp_nct = 0; // hparams.n_ctx_train
uint32_t hp_nex = 0; // hparams.n_expert
auto dmd = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
GGML_ASSERT(dmd.size() == devs.size() + 1);
for (size_t id = 0; id < devs.size(); id++) {

56
common/fit.h
View file

@ -1,9 +1,7 @@
#pragma once
#include "ggml.h"
#include "ggml-backend.h"
#include "llama.h"
#include "../src/llama-ext.h"
#include <vector>
@ -18,31 +16,41 @@ enum common_params_fit_status {
// - this function is NOT thread safe because it modifies the global llama logger state
// - only parameters that have the same value as in llama_default_model_params are modified
// with the exception of the context size which is modified if and only if equal to 0
enum common_params_fit_status common_fit_params(
const char * path_model,
struct llama_model_params * mparams,
struct llama_context_params * cparams,
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
size_t * margins, // margins of memory to leave per device in bytes
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
common_params_fit_status common_fit_params(
const char * path_model,
llama_model_params * mparams,
llama_context_params * cparams,
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
size_t * margins, // margins of memory to leave per device in bytes
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
// print estimated memory to stdout
void common_fit_print(
const char * path_model,
struct llama_model_params * mparams,
struct llama_context_params * cparams);
const char * path_model,
llama_model_params * mparams,
llama_context_params * cparams);
void common_memory_breakdown_print(const struct llama_context * ctx);
void common_memory_breakdown_print(const llama_context * ctx);
struct common_device_memory_data {
int64_t total;
int64_t free;
size_t model;
size_t context;
size_t compute;
};
using common_device_memory_data_vec = std::vector<common_device_memory_data>;
// Load a model + context with no_alloc and return the per-device memory breakdown.
std::vector<llama_device_memory_data> common_get_device_memory_data(
const char * path_model,
const struct llama_model_params * mparams,
const struct llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
enum ggml_log_level log_level);
common_device_memory_data_vec common_get_device_memory_data(
const char * path_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
ggml_log_level log_level);

4
common/jinja/runtime.cpp
View file

@ -761,9 +761,9 @@ value member_expression::execute_impl(context & ctx) {
if (is_stmt<slice_expression>(this->property)) {
auto s = cast_stmt<slice_expression>(this->property);
value start_val = s->start_expr ? s->start_expr->execute(ctx) : mk_val<value_int>(0);
value stop_val = s->stop_expr ? s->stop_expr->execute(ctx) : mk_val<value_int>(arr_size);
value step_val = s->step_expr ? s->step_expr->execute(ctx) : mk_val<value_int>(1);
value start_val = s->start_expr ? s->start_expr->execute(ctx) : (step_val->as_int() < 0 ? mk_val<value_int>(arr_size - 1) : mk_val<value_int>(0));
value stop_val = s->stop_expr ? s->stop_expr->execute(ctx) : (step_val->as_int() < 0 ? mk_val<value_int>(-1) : mk_val<value_int>(arr_size));
// translate to function call: obj.slice(start, stop, step)
JJ_DEBUG("Member expression is a slice: start %s, stop %s, step %s",

33
common/jinja/value.cpp
View file

@ -90,14 +90,14 @@ static T slice(const T & array, int64_t start, int64_t stop, int64_t step = 1) {
stop_val = std::min(stop_val, len);
}
} else {
start_val = len - 1;
start_val = start;
if (start_val < 0) {
start_val = std::max(len + start_val, (int64_t)-1);
start_val = std::max(len + start_val, (int64_t)0);
} else {
start_val = std::min(start_val, len - 1);
}
stop_val = -1;
stop_val = stop;
if (stop_val < -1) {
stop_val = std::max(len + stop_val, (int64_t)-1);
} else {
@ -673,6 +673,9 @@ const func_builtins & value_string_t::get_builtins() const {
std::string str = val_input->as_string().str();
// FIXME: Support non-specified delimiter (split on consecutive (no leading or trailing) whitespace)
std::string delim = (args.count() > 1) ? args.get_pos(1)->as_string().str() : " ";
if (delim.empty()) {
throw raised_exception("empty separator");
}
int64_t maxsplit = (args.count() > 2) ? args.get_pos(2)->as_int() : -1;
auto result = mk_val<value_array>();
size_t pos = 0;
@ -697,6 +700,9 @@ const func_builtins & value_string_t::get_builtins() const {
std::string str = val_input->as_string().str();
// FIXME: Support non-specified delimiter (split on consecutive (no leading or trailing) whitespace)
std::string delim = (args.count() > 1) ? args.get_pos(1)->as_string().str() : " ";
if (delim.empty()) {
throw raised_exception("empty separator");
}
int64_t maxsplit = (args.count() > 2) ? args.get_pos(2)->as_int() : -1;
auto result = mk_val<value_array>();
size_t pos = 0;
@ -722,10 +728,23 @@ const func_builtins & value_string_t::get_builtins() const {
if (count > 0) {
throw not_implemented_exception("String replace with count argument not implemented");
}
size_t pos = 0;
while ((pos = str.find(old_str, pos)) != std::string::npos) {
str.replace(pos, old_str.length(), new_str);
pos += new_str.length();
if (old_str != new_str) {
size_t pos = 0;
if (old_str.empty()) {
std::string new_res;
new_res.reserve(str.length() + new_str.length() * (str.length() + 1));
new_res += new_str;
for (const char c : str) {
new_res.push_back(c);
new_res += new_str;
}
str = new_res;
} else {
while ((pos = str.find(old_str, pos)) != std::string::npos) {
str.replace(pos, old_str.length(), new_str);
pos += new_str.length();
}
}
}
auto res = mk_val<value_string>(str);
res->val_str.mark_input_based_on(args.get_pos(0)->val_str);

1
conversion/__init__.py
View file

@ -40,6 +40,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"ChatGLMModel": "chatglm",
"CodeShellForCausalLM": "codeshell",
"CogVLMForCausalLM": "cogvlm",
"Cohere2MoeForCausalLM": "command_r",
"Cohere2ForCausalLM": "command_r",
"CohereForCausalLM": "command_r",
"DbrxForCausalLM": "dbrx",

7
conversion/base.py
View file

@ -1195,7 +1195,7 @@ class TextModel(ModelBase):
self.gguf_writer.add_embedding_length(n_embd)
logger.info(f"gguf: embedding length = {n_embd}")
if (n_ff := self.find_hparam(["intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None:
if (n_ff := self.find_hparam(["prefix_dense_intermediate_size", "intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None:
self.gguf_writer.add_feed_forward_length(n_ff)
logger.info(f"gguf: feed forward length = {n_ff}")
@ -1280,7 +1280,7 @@ class TextModel(ModelBase):
self.gguf_writer.add_expert_group_used_count(n_group_used)
logger.info(f"gguf: expert groups used count = {n_group_used}")
if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func"], optional=True)) is not None:
if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn"], optional=True)) is not None:
if score_func == "sigmoid":
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
elif score_func == "softmax":
@ -1495,6 +1495,9 @@ class TextModel(ModelBase):
if chkhsh == "d772b220ace2baec124bed8cfafce0ead7d6c38a4b65ef11261cf9d5d62246d1":
# ref: https://huggingface.co/CohereLabs/tiny-aya-base
res = "tiny_aya"
if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e":
# ref: https://huggingface.co/CohereLabs/North-Mini-Code-1.0
res = "cohere2moe"
if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":
# ref: https://huggingface.co/Qwen/Qwen1.5-7B
res = "qwen2"

120
conversion/command_r.py
View file

@ -1,5 +1,6 @@
from __future__ import annotations
import re
from typing import Iterable, TYPE_CHECKING
import torch
@ -55,3 +56,122 @@ class Cohere2Model(TextModel):
return
yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Cohere2MoeForCausalLM")
class Cohere2MoeModel(TextModel):
model_arch = gguf.MODEL_ARCH.COHERE2MOE
_n_main_layers: int | None = None
_expert_tensor_re = re.compile(
r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(down_proj|gate_proj|up_proj)\.weight"
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if (n_nextn := int(self.hparams.get("num_nextn_predict_layers", 0) or 0)) > 0 and not self.no_mtp:
self.block_count += n_nextn
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
self._experts: list[dict[str, Tensor]] = [{} for _ in range(self.block_count)]
def _set_vocab_gpt2(self) -> None:
tokens, toktypes, tokpre = self.get_vocab_base()
self.gguf_writer.add_tokenizer_model("gpt2")
self.gguf_writer.add_tokenizer_pre(tokpre)
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(toktypes)
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
special_vocab.add_to_gguf(self.gguf_writer)
def set_gguf_parameters(self):
hparams = self.hparams
expert_intermediate_size = hparams["intermediate_size"]
mlp_layer_types = hparams.get("mlp_layer_types")
n_dense_lead = hparams.get("first_k_dense_replace", 0)
if mlp_layer_types is not None:
n_dense_lead = next((i for i, t in enumerate(mlp_layer_types) if t != "dense"), len(mlp_layer_types))
super().set_gguf_parameters()
self.gguf_writer.add_logit_scale(hparams["logit_scale"])
self.gguf_writer.add_sliding_window(hparams["sliding_window"])
self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]])
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
self.gguf_writer.add_expert_feed_forward_length(expert_intermediate_size)
self.gguf_writer.add_leading_dense_block_count(n_dense_lead)
self.gguf_writer.add_expert_weights_norm(hparams.get("norm_topk_prob", False))
if (num_shared_experts := hparams.get("num_shared_experts", 0)) > 0:
if hparams.get("shared_expert_combination_strategy", "average") != "average":
raise ValueError("Cohere2 MoE only supports average shared expert combination")
self.gguf_writer.add_expert_shared_count(num_shared_experts)
self.gguf_writer.add_expert_shared_feed_forward_length(expert_intermediate_size * num_shared_experts)
if (n_nextn := hparams.get("num_nextn_predict_layers", 0)) > 0 and not self.no_mtp:
self.gguf_writer.add_nextn_predict_layers(n_nextn)
self.gguf_writer.add_rope_dimension_count(hparams["head_dim"])
self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
def index_tensors(self, remote_hf_model_id: str | None = None):
hparams = {**self.hparams, **self.hparams.get("text_config", {})}
self._n_main_layers = hparams.get("num_hidden_layers")
type(self)._n_main_layers = self._n_main_layers
return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
@classmethod
def filter_tensors(cls, item):
if (titem := super().filter_tensors(item)) is None:
return None
name, gen = titem
if cls._n_main_layers is not None:
is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers
if is_mtp and cls.no_mtp:
return None
if cls.mtp_only and not is_mtp and name not in (
"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
):
return None
return name, gen
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.endswith(".bias"):
if torch.any(data_torch != 0):
raise ValueError(f"Bias tensor {name!r} is not zero.")
logger.debug(f"Skipping bias tensor {name!r}.")
return
if (m := self._expert_tensor_re.fullmatch(name)) is not None:
n_experts = self.hparams["num_experts"]
layer_idx = int(m.group(1))
assert bid is None or bid == layer_idx
self._experts[layer_idx][name] = data_torch
expected = {
f"model.layers.{layer_idx}.mlp.experts.{xid}.{w_name}.weight"
for xid in range(n_experts)
for w_name in ("down_proj", "gate_proj", "up_proj")
}
if expected.issubset(self._experts[layer_idx]):
for w_name in ["down_proj", "gate_proj", "up_proj"]:
datas: list[Tensor] = []
for xid in range(n_experts):
ename = f"model.layers.{layer_idx}.mlp.experts.{xid}.{w_name}.weight"
datas.append(self._experts[layer_idx][ename])
del self._experts[layer_idx][ename]
data_torch = torch.stack(datas, dim=0)
merged_name = f"model.layers.{layer_idx}.mlp.experts.{w_name}.weight"
yield from super().modify_tensors(data_torch, merged_name, layer_idx)
return
yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):
super().prepare_tensors()
experts = [k for d in self._experts for k in d.keys()]
if len(experts) > 0:
raise ValueError(f"Unprocessed experts: {experts}")

1
convert_hf_to_gguf_update.py
View file

@ -100,6 +100,7 @@ models = [
{"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", },
{"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },
{"name": "tiny_aya", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/tiny-aya-base", },
{"name": "cohere2moe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0", },
{"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", },
{"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },
{"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", },

180
ggml/src/ggml-vulkan/ggml-vulkan.cpp
View file

@ -839,6 +839,7 @@ struct vk_device_struct {
// [src/dst 0=fp32,1=fp16]
vk_pipeline pipeline_exp[2];
vk_pipeline pipeline_expm1[2];
vk_pipeline pipeline_elu[2];
vk_pipeline pipeline_gelu[2];
vk_pipeline pipeline_gelu_erf[2];
@ -1208,30 +1209,35 @@ struct vk_op_glu_push_constants {
uint32_t mode; // 0: default, 1: swapped, 2: split
float alpha; // for swiglu_oai
float limit;
uint32_t nb00;
uint32_t nb01;
uint32_t nb02;
uint32_t nb03;
uint32_t ne01;
uint32_t ne02;
uint32_t nb10;
uint32_t nb11;
uint32_t nb12;
uint32_t nb13;
uint32_t ne11;
uint32_t ne12;
uint32_t nb20;
uint32_t nb21;
uint32_t nb22;
uint32_t nb23;
uint32_t ne21;
uint32_t ne22;
uint32_t misalign_offsets;
uint32_t ne2_012mp; uint32_t ne2_012L;
uint32_t ne2_01mp; uint32_t ne2_01L;
uint32_t ne2_0mp; uint32_t ne2_0L;
};
static_assert(sizeof(vk_op_glu_push_constants) <= 128, "sizeof(vk_op_glu_push_constants) must be <= 128");
struct vk_op_unary_push_constants {
uint32_t ne;
uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;
uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;
uint32_t misalign_offsets;
float param1; float param2;
uint32_t ne0_012mp; uint32_t ne0_012L;
uint32_t ne0_01mp; uint32_t ne0_01L;
uint32_t ne0_0mp; uint32_t ne0_0L;
uint32_t ne1_012mp; uint32_t ne1_012L;
uint32_t ne1_01mp; uint32_t ne1_01L;
uint32_t ne1_0mp; uint32_t ne1_0L;
float param1; float param2; float param3; float param4;
uint32_t ne0_012mp; uint32_t ne0_01mp; uint32_t ne0_0mp; uint32_t ne0_Ls;
uint32_t ne1_012mp; uint32_t ne1_01mp; uint32_t ne1_0mp; uint32_t ne1_Ls;
};
static_assert(sizeof(vk_op_unary_push_constants) <= 128, "sizeof(vk_op_unary_push_constants) must be <= 128");
@ -1336,6 +1342,10 @@ static void init_fastdiv_values(uint32_t d, uint32_t &mp, uint32_t &L)
mp = (uint32_t)((uint64_t{1} << 32) * ((uint64_t{1} << L) - d) / d + 1);
}
static uint32_t pack_fastdiv_L(uint32_t L0, uint32_t L1, uint32_t L2) {
return L0 | (L1 << 8) | (L2 << 16);
}
template <typename T> void init_pushconst_fastdiv(T &p) {
GGML_UNUSED(p);
static_assert(!std::is_const<T>::value, "unexpected type");
@ -1343,12 +1353,29 @@ template <typename T> void init_pushconst_fastdiv(T &p) {
template <> void init_pushconst_fastdiv(vk_op_unary_push_constants &p) {
// Compute magic values to divide by these six numbers.
init_fastdiv_values(p.ne02*p.ne01*p.ne00, p.ne0_012mp, p.ne0_012L);
init_fastdiv_values(p.ne01*p.ne00, p.ne0_01mp, p.ne0_01L);
init_fastdiv_values(p.ne00, p.ne0_0mp, p.ne0_0L);
init_fastdiv_values(p.ne12*p.ne11*p.ne10, p.ne1_012mp, p.ne1_012L);
init_fastdiv_values(p.ne11*p.ne10, p.ne1_01mp, p.ne1_01L);
init_fastdiv_values(p.ne10, p.ne1_0mp, p.ne1_0L);
uint32_t ne0_012L;
uint32_t ne0_01L;
uint32_t ne0_0L;
uint32_t ne1_012L;
uint32_t ne1_01L;
uint32_t ne1_0L;
init_fastdiv_values(p.ne02*p.ne01*p.ne00, p.ne0_012mp, ne0_012L);
init_fastdiv_values(p.ne01*p.ne00, p.ne0_01mp, ne0_01L);
init_fastdiv_values(p.ne00, p.ne0_0mp, ne0_0L);
init_fastdiv_values(p.ne12*p.ne11*p.ne10, p.ne1_012mp, ne1_012L);
init_fastdiv_values(p.ne11*p.ne10, p.ne1_01mp, ne1_01L);
init_fastdiv_values(p.ne10, p.ne1_0mp, ne1_0L);
p.ne0_Ls = pack_fastdiv_L(ne0_012L, ne0_01L, ne0_0L);
p.ne1_Ls = pack_fastdiv_L(ne1_012L, ne1_01L, ne1_0L);
}
template <> void init_pushconst_fastdiv(vk_op_glu_push_constants &p) {
// GLU linearizes over dst, then uses dst coordinates for src0/src1.
init_fastdiv_values(p.ne22*p.ne21*p.ne20, p.ne2_012mp, p.ne2_012L);
init_fastdiv_values(p.ne21*p.ne20, p.ne2_01mp, p.ne2_01L);
init_fastdiv_values(p.ne20, p.ne2_0mp, p.ne2_0L);
}
struct vk_op_binary_push_constants {
@ -5012,8 +5039,8 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_repeat_i16, "repeat_i16", repeat_i16_len, repeat_i16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
#define CREATE_UNARY(name) \
ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); \
ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
CREATE_UNARY(elu)
CREATE_UNARY(gelu)
@ -5036,6 +5063,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
CREATE_UNARY(trunc)
CREATE_UNARY(sgn)
CREATE_UNARY(exp)
CREATE_UNARY(expm1)
#undef CREATE_UNARY
ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
@ -7773,6 +7801,23 @@ static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, si
if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && src->device->uma) {
GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue.cmd_pool);
ggml_vk_ctx_begin(src->device, subctx);
subctx->s->buffer->buf.pipelineBarrier(
vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost,
{},
{ { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,
vk::AccessFlagBits::eHostRead } },
{}, {});
ggml_vk_ctx_end(subctx);
ggml_vk_submit(subctx, src->device->fence);
VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),
"vk_buffer_read_2d uma waitForFences");
src->device->device.resetFences({ src->device->fence });
ggml_vk_queue_command_pools_cleanup(src->device);
if (width == spitch && width == dpitch) {
memcpy(dst, (const uint8_t *) src->ptr + offset, width * height);
} else {
@ -8207,7 +8252,6 @@ static vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const
static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out) {
VK_LOG_DEBUG("ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), ";
std::cerr << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")");
const int tensor_type_size = ggml_type_size(tensor->type);
const uint32_t ne = ggml_nelements(tensor);
std::array<uint32_t, 3> elements;
@ -8220,14 +8264,11 @@ static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context&
elements = { ne, 1, 1 };
}
vk_op_unary_push_constants pc = {
(uint32_t)ne,
(uint32_t)tensor->ne[0], (uint32_t)tensor->ne[1], (uint32_t)tensor->ne[2], (uint32_t)tensor->ne[3], (uint32_t)tensor->nb[0] / tensor_type_size, (uint32_t)tensor->nb[1] / tensor_type_size, (uint32_t)tensor->nb[2] / tensor_type_size, (uint32_t)tensor->nb[3] / tensor_type_size,
(uint32_t)tensor->ne[0], (uint32_t)tensor->ne[1], (uint32_t)tensor->ne[2], (uint32_t)tensor->ne[3], 1 , (uint32_t)tensor->ne[0] , (uint32_t)(tensor->ne[0] * tensor->ne[1]) , (uint32_t)(tensor->ne[0] * tensor->ne[1] * tensor->ne[2]),
0,
0.0f, 0.0f,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(tensor, tensor, ne);
pc.nb10 = 1;
pc.nb11 = (uint32_t)tensor->ne[0];
pc.nb12 = (uint32_t)(tensor->ne[0] * tensor->ne[1]);
pc.nb13 = (uint32_t)(tensor->ne[0] * tensor->ne[1] * tensor->ne[2]);
init_pushconst_fastdiv(pc);
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { in, out }, pc, elements);
ggml_vk_sync_buffers(ctx, subctx);
@ -8241,7 +8282,6 @@ static void ggml_vk_cpy_to_strided(
uint32_t nb10, uint32_t nb11, uint32_t nb12, uint32_t nb13) {
VK_LOG_DEBUG("ggml_vk_cpy_to_strided((" << tensor << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), ";
std::cerr << "dst_nb=(" << nb10 << ", " << nb11 << ", " << nb12 << ", " << nb13 << "), buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")");
const int tensor_type_size = ggml_type_size(tensor->type);
const uint32_t ne = ggml_nelements(tensor);
std::array<uint32_t, 3> elements;
@ -8254,14 +8294,11 @@ static void ggml_vk_cpy_to_strided(
elements = { ne, 1, 1 };
}
vk_op_unary_push_constants pc = {
(uint32_t)ne,
(uint32_t)tensor->ne[0], (uint32_t)tensor->ne[1], (uint32_t)tensor->ne[2], (uint32_t)tensor->ne[3], (uint32_t)tensor->nb[0] / tensor_type_size, (uint32_t)tensor->nb[1] / tensor_type_size, (uint32_t)tensor->nb[2] / tensor_type_size, (uint32_t)tensor->nb[3] / tensor_type_size,
(uint32_t)tensor->ne[0], (uint32_t)tensor->ne[1], (uint32_t)tensor->ne[2], (uint32_t)tensor->ne[3], nb10, nb11, nb12, nb13,
0,
0.0f, 0.0f,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(tensor, tensor, ne);
pc.nb10 = nb10;
pc.nb11 = nb11;
pc.nb12 = nb12;
pc.nb13 = nb13;
init_pushconst_fastdiv(pc);
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { in, out }, pc, elements);
ggml_vk_sync_buffers(ctx, subctx);
@ -10466,6 +10503,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
switch (ggml_get_unary_op(dst)) {
case GGML_UNARY_OP_EXP:
return ctx->device->pipeline_exp[dst->type == GGML_TYPE_F16];
case GGML_UNARY_OP_EXPM1:
return ctx->device->pipeline_expm1[dst->type == GGML_TYPE_F16];
case GGML_UNARY_OP_ELU:
return ctx->device->pipeline_elu[dst->type == GGML_TYPE_F16];
case GGML_UNARY_OP_SILU:
@ -10864,6 +10903,21 @@ template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk
GGML_UNUSED(src3);
}
template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_glu_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
const uint32_t b_offset = src1 ? get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type) : a_offset;
const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
GGML_ASSERT(a_offset < (1u << 8));
GGML_ASSERT(b_offset < (1u << 8));
GGML_ASSERT(d_offset < (1u << 8));
p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;
GGML_UNUSED(src2);
GGML_UNUSED(src3);
}
template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_sum_rows_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
@ -12213,17 +12267,17 @@ static void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, c
}
static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f, 0.0f, 0.0f });
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, vk_op_unary_push_constants_init(src0, dst));
}
static void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
float * op_params = (float *)dst->op_params;
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY,
{
(uint32_t)ggml_nelements(src0), 0,
op_params[1], op_params[2], op_params[3], op_params[4]
}
);
vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
p.param1 = op_params[1];
p.param2 = op_params[2];
p.param3 = op_params[3];
p.param4 = op_params[4];
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, std::move(p));
}
static void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
@ -12243,6 +12297,9 @@ static void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const
}
const uint32_t mode = split ? 2 : (swapped ? 1 : 0);
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = split ? ggml_type_size(src1->type) : src0_type_size;
const uint32_t dst_type_size = ggml_type_size(dst->type);
ggml_vk_op_f32<vk_op_glu_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_GLU,
{
@ -12252,16 +12309,22 @@ static void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const
mode,
alpha,
limit,
(uint32_t)(src0->nb[1] / src0->nb[0]),
(uint32_t)(src0->nb[2] / src0->nb[0]),
(uint32_t)(src0->nb[3] / src0->nb[0]),
(uint32_t)src0->ne[1],
(uint32_t)src0->ne[2],
(uint32_t)(dst->nb[1] / dst->nb[0]),
(uint32_t)(dst->nb[2] / dst->nb[0]),
(uint32_t)(dst->nb[3] / dst->nb[0]),
(uint32_t)(src0->nb[0] / src0_type_size),
(uint32_t)(src0->nb[1] / src0_type_size),
(uint32_t)(src0->nb[2] / src0_type_size),
(uint32_t)(src0->nb[3] / src0_type_size),
(uint32_t)((split ? src1->nb[0] : src0->nb[0]) / src1_type_size),
(uint32_t)((split ? src1->nb[1] : src0->nb[1]) / src1_type_size),
(uint32_t)((split ? src1->nb[2] : src0->nb[2]) / src1_type_size),
(uint32_t)((split ? src1->nb[3] : src0->nb[3]) / src1_type_size),
(uint32_t)(dst->nb[0] / dst_type_size),
(uint32_t)(dst->nb[1] / dst_type_size),
(uint32_t)(dst->nb[2] / dst_type_size),
(uint32_t)(dst->nb[3] / dst_type_size),
(uint32_t)dst->ne[1],
(uint32_t)dst->ne[2]
(uint32_t)dst->ne[2],
0,
0, 0, 0, 0, 0, 0,
});
}
@ -14264,6 +14327,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr
switch (ggml_get_unary_op(node)) {
case GGML_UNARY_OP_ELU:
case GGML_UNARY_OP_EXP:
case GGML_UNARY_OP_EXPM1:
case GGML_UNARY_OP_SILU:
case GGML_UNARY_OP_GELU:
case GGML_UNARY_OP_GELU_ERF:
@ -16653,6 +16717,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_OP_UNARY:
switch (ggml_get_unary_op(op)) {
case GGML_UNARY_OP_EXP:
case GGML_UNARY_OP_EXPM1:
case GGML_UNARY_OP_ELU:
case GGML_UNARY_OP_GELU:
case GGML_UNARY_OP_GELU_ERF:
@ -16673,8 +16738,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_UNARY_OP_FLOOR:
case GGML_UNARY_OP_TRUNC:
case GGML_UNARY_OP_SGN:
return ggml_is_contiguous(op->src[0]) &&
(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&
(op->src[0]->type == op->type);
default:
@ -16690,7 +16754,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_GLU_OP_GEGLU_QUICK:
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&
(op->src[0]->type == op->type);
(op->src[0]->type == op->type) &&
(!op->src[1] || op->src[1]->type == op->src[0]->type);
default:
return false;
}
@ -17820,6 +17885,9 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph *
case GGML_UNARY_OP_EXP:
tensor_clone = ggml_exp(ggml_ctx, src_clone[0]);
break;
case GGML_UNARY_OP_EXPM1:
tensor_clone = ggml_expm1(ggml_ctx, src_clone[0]);
break;
case GGML_UNARY_OP_ELU:
tensor_clone = ggml_elu(ggml_ctx, src_clone[0]);
break;

21
ggml/src/ggml-vulkan/vulkan-shaders/abs.comp
View file

@ -1,21 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
data_d[i] = D_TYPE(abs(float(data_a[i])));
}

22
ggml/src/ggml-vulkan/vulkan-shaders/ceil.comp
View file

@ -1,22 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float x = float(data_a[i]);
data_d[i] = D_TYPE(ceil(x));
}

6
ggml/src/ggml-vulkan/vulkan-shaders/diag.comp
View file

@ -12,11 +12,11 @@ void main() {
return;
}
const uint i13 = fastdiv(idx, p.ne1_012mp, p.ne1_012L);
const uint i13 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));
const uint i13_offset = i13 * p.ne12*p.ne11*p.ne10;
const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, p.ne1_01L);
const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));
const uint i12_offset = i12*p.ne11*p.ne10;
const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, p.ne1_0L);
const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));
const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
if (i10 == i11) {

27
ggml/src/ggml-vulkan/vulkan-shaders/elu.comp
View file

@ -1,27 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
float x = float(data_a[i]);
if (x < 0.0f) {
x = exp(x) - 1;
}
data_d[i] = D_TYPE(x);
}

20
ggml/src/ggml-vulkan/vulkan-shaders/exp.comp
View file

@ -1,20 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
data_d[i] = D_TYPE(exp(float(data_a[i])));
}

22
ggml/src/ggml-vulkan/vulkan-shaders/floor.comp
View file

@ -1,22 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float x = float(data_a[i]);
data_d[i] = D_TYPE(floor(x));
}

25
ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp
View file

@ -1,25 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const float GELU_COEF_A = 0.044715f;
const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float xi = float(data_a[i]);
const float val = SQRT_2_OVER_PI*xi*(1.0f + GELU_COEF_A*xi*xi);
data_d[i] = D_TYPE(0.5f*xi*(2.0f - 2.0f / (exp(2 * val) + 1)));
}

39
ggml/src/ggml-vulkan/vulkan-shaders/gelu_erf.comp
View file

@ -1,39 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
// based on Abramowitz and Stegun formula 7.1.26 or similar Hastings' approximation
// ref: https://www.johndcook.com/blog/python_erf/
const float p_erf = 0.3275911f;
const float a1_erf = 0.254829592f;
const float a2_erf = -0.284496736f;
const float a3_erf = 1.421413741f;
const float a4_erf = -1.453152027f;
const float a5_erf = 1.061405429f;
const float SQRT_2_INV = 0.70710678118654752440084436210484f;
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float a = float(data_a[i]);
const float a_div_sqr2 = a * SQRT_2_INV;
const float sign_x = sign(a_div_sqr2);
const float x = abs(a_div_sqr2);
const float t = 1.0f / (1.0f + p_erf * x);
const float y = 1.0f - (((((a5_erf * t + a4_erf) * t) + a3_erf) * t + a2_erf) * t + a1_erf) * t * exp(-x * x);
const float erf_approx = sign_x * y;
data_d[i] = D_TYPE(0.5f * a * (1.0f + erf_approx));
}

23
ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp
View file

@ -1,23 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const float GELU_QUICK_COEF = -1.702f;
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float x = float(data_a[i]);
data_d[i] = D_TYPE(x * (1.0f / (1.0f + exp(GELU_QUICK_COEF * x))));
}

40
ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl
View file

@ -7,14 +7,12 @@ layout (push_constant) uniform parameter
uint ne00; uint ne01; uint ne02; uint ne03; uint nb00; uint nb01; uint nb02; uint nb03;
uint ne10; uint ne11; uint ne12; uint ne13; uint nb10; uint nb11; uint nb12; uint nb13;
uint misalign_offsets;
float param1; float param2;
float param1; float param2; float param3; float param4;
uint ne0_012mp; uint ne0_012L;
uint ne0_01mp; uint ne0_01L;
uint ne0_0mp; uint ne0_0L;
uint ne1_012mp; uint ne1_012L;
uint ne1_01mp; uint ne1_01L;
uint ne1_0mp; uint ne1_0L;
// The three L values are packed as bytes to keep this layout under the 128B
// push constant limit while still leaving room for four float parameters.
uint ne0_012mp; uint ne0_01mp; uint ne0_0mp; uint ne0_Ls;
uint ne1_012mp; uint ne1_01mp; uint ne1_0mp; uint ne1_Ls;
} p;
layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
@ -42,42 +40,46 @@ uint fastdiv(uint n, uint mp, uint L) {
return (msbs + n) >> L;
}
uint fastdiv_L(uint packed, uint slot) {
return (packed >> (slot * 8)) & 0x3Fu;
}
uint src0_idx(uint idx) {
const uint i03 = fastdiv(idx, p.ne0_012mp, p.ne0_012L);
const uint i03 = fastdiv(idx, p.ne0_012mp, fastdiv_L(p.ne0_Ls, 0));
const uint i03_offset = i03 * p.ne02*p.ne01*p.ne00;
const uint i02 = fastdiv(idx - i03_offset, p.ne0_01mp, p.ne0_01L);
const uint i02 = fastdiv(idx - i03_offset, p.ne0_01mp, fastdiv_L(p.ne0_Ls, 1));
const uint i02_offset = i02*p.ne01*p.ne00;
const uint i01 = fastdiv(idx - i03_offset - i02_offset, p.ne0_0mp, p.ne0_0L);
const uint i01 = fastdiv(idx - i03_offset - i02_offset, p.ne0_0mp, fastdiv_L(p.ne0_Ls, 2));
const uint i00 = idx - i03_offset - i02_offset - i01*p.ne00;
return i03*p.nb03 + i02*p.nb02 + i01*p.nb01 + i00*p.nb00;
}
uint dst_idx(uint idx) {
const uint i13 = fastdiv(idx, p.ne1_012mp, p.ne1_012L);
const uint i13 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));
const uint i13_offset = i13 * p.ne12*p.ne11*p.ne10;
const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, p.ne1_01L);
const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));
const uint i12_offset = i12*p.ne11*p.ne10;
const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, p.ne1_0L);
const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));
const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
return i13*p.nb13 + i12*p.nb12 + i11*p.nb11 + i10*p.nb10;
}
uint src0_idx_quant(uint idx, uint qk) {
const uint i03 = fastdiv(idx, p.ne0_012mp, p.ne0_012L);
const uint i03 = fastdiv(idx, p.ne0_012mp, fastdiv_L(p.ne0_Ls, 0));
const uint i03_offset = i03 * p.ne02*p.ne01*p.ne00;
const uint i02 = fastdiv(idx - i03_offset, p.ne0_01mp, p.ne0_01L);
const uint i02 = fastdiv(idx - i03_offset, p.ne0_01mp, fastdiv_L(p.ne0_Ls, 1));
const uint i02_offset = i02*p.ne01*p.ne00;
const uint i01 = fastdiv(idx - i03_offset - i02_offset, p.ne0_0mp, p.ne0_0L);
const uint i01 = fastdiv(idx - i03_offset - i02_offset, p.ne0_0mp, fastdiv_L(p.ne0_Ls, 2));
const uint i00 = idx - i03_offset - i02_offset - i01*p.ne00;
return i03*p.nb03 + i02*p.nb02 + i01*p.nb01 + (i00/qk)*p.nb00;
}
uint dst_idx_quant(uint idx, uint qk) {
const uint i13 = fastdiv(idx, p.ne1_012mp, p.ne1_012L);
const uint i13 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));
const uint i13_offset = i13 * p.ne12*p.ne11*p.ne10;
const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, p.ne1_01L);
const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));
const uint i12_offset = i12*p.ne11*p.ne10;
const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, p.ne1_0L);
const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));
const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
return i13*p.nb13 + i12*p.nb12 + i11*p.nb11 + (i10/qk)*p.nb10;
}

27
ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl
View file

@ -15,14 +15,33 @@ layout (push_constant) uniform parameter
uint mode;
float alpha;
float limit;
uint nb00;
uint nb01;
uint nb02;
uint nb03;
uint ne01;
uint ne02;
uint nb10;
uint nb11;
uint nb12;
uint nb13;
uint ne11;
uint ne12;
uint nb20;
uint nb21;
uint nb22;
uint nb23;
uint ne21;
uint ne22;
uint misalign_offsets;
uint ne2_012mp; uint ne2_012L;
uint ne2_01mp; uint ne2_01L;
uint ne2_0mp; uint ne2_0L;
} p;
uint get_aoffset() { return p.misalign_offsets >> 16; }
uint get_boffset() { return (p.misalign_offsets >> 8) & 0xFF; }
uint get_doffset() { return p.misalign_offsets & 0xFF; }
// see init_fastdiv_values in ggml-vulkan.cpp
uint fastdiv(uint n, uint mp, uint L) {
uint msbs, lsbs;
umulExtended(n, mp, msbs, lsbs);
return (msbs + n) >> L;
}

32
ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl
View file

@ -5,35 +5,31 @@ void main() {
return;
}
const uint row = i / p.ne20;
const uint col = i - row * p.ne20;
const uint i23 = fastdiv(i, p.ne2_012mp, p.ne2_012L);
const uint i23_offset = i23 * p.ne22*p.ne21*p.ne20;
const uint i22 = fastdiv(i - i23_offset, p.ne2_01mp, p.ne2_01L);
const uint i22_offset = i22*p.ne21*p.ne20;
const uint i21 = fastdiv(i - i23_offset - i22_offset, p.ne2_0mp, p.ne2_0L);
const uint i20 = i - i23_offset - i22_offset - i21*p.ne20;
const uint i3 = row / (p.ne01 * p.ne02);
const uint i2 = (row % (p.ne01 * p.ne02)) / p.ne01;
const uint i1 = row % p.ne01;
const uint src_idx = i3 * p.nb03 + i2 * p.nb02 + i1 * p.nb01 + col;
const uint dst_i3 = row / (p.ne11 * p.ne12);
const uint dst_i2 = (row % (p.ne11 * p.ne12)) / p.ne11;
const uint dst_i1 = row % p.ne11;
const uint dst_idx = dst_i3 * p.nb13 + dst_i2 * p.nb12 + dst_i1 * p.nb11 + col;
const uint src_idx_a = get_aoffset() + i23 * p.nb03 + i22 * p.nb02 + i21 * p.nb01 + i20 * p.nb00;
const uint src_idx_b = get_boffset() + i23 * p.nb13 + i22 * p.nb12 + i21 * p.nb11 + i20 * p.nb10;
const uint dst_idx = get_doffset() + i23 * p.nb23 + i22 * p.nb22 + i21 * p.nb21 + i20 * p.nb20;
if (p.mode == 0) {
// Default
const uint offset = p.ne00 / 2;
const uint idx = src_idx;
const uint offset = (p.ne00 / 2) * p.nb00;
const uint idx = src_idx_a;
data_d[dst_idx] = D_TYPE(op(float(data_a[idx]), float(data_a[idx + offset])));
} else if (p.mode == 1) {
// Swapped
const uint offset = p.ne00 / 2;
const uint idx = src_idx;
const uint offset = (p.ne00 / 2) * p.nb00;
const uint idx = src_idx_a;
data_d[dst_idx] = D_TYPE(op(float(data_a[idx + offset]), float(data_a[idx])));
} else {
// Split
const uint idx = src_idx;
data_d[dst_idx] = D_TYPE(op(float(data_a[idx]), float(data_b[idx])));
data_d[dst_idx] = D_TYPE(op(float(data_a[src_idx_a]), float(data_b[src_idx_b])));
}
}

22
ggml/src/ggml-vulkan/vulkan-shaders/hardsigmoid.comp
View file

@ -1,22 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float x = float(data_a[i]);
data_d[i] = D_TYPE(min(1.0f, max(0.0f, (x + 3.0f) / 6.0f)));
}

22
ggml/src/ggml-vulkan/vulkan-shaders/hardswish.comp
View file

@ -1,22 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float x = float(data_a[i]);
data_d[i] = D_TYPE(x * min(1.0f, max(0.0f, (x + 3.0f) / 6.0f)));
}

20
ggml/src/ggml-vulkan/vulkan-shaders/neg.comp
View file

@ -1,20 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
data_d[i] = D_TYPE(-float(data_a[i]));
}

21
ggml/src/ggml-vulkan/vulkan-shaders/relu.comp
View file

@ -1,21 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
data_d[i] = D_TYPE(max(float(data_a[i]), 0));
}

6
ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp
View file

@ -13,11 +13,11 @@ void main() {
}
// Destination multi-index (inlined dst_idx)
const uint i13 = fastdiv(idx, p.ne1_012mp, p.ne1_012L);
const uint i13 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));
const uint i13_offset = i13 * p.ne12*p.ne11*p.ne10;
const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, p.ne1_01L);
const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));
const uint i12_offset = i12*p.ne11*p.ne10;
const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, p.ne1_0L);
const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));
const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
const uint d_idx = i13*p.nb13 + i12*p.nb12 + i11*p.nb11 + i10*p.nb10;

6
ggml/src/ggml-vulkan/vulkan-shaders/roll.comp
View file

@ -20,11 +20,11 @@ void main() {
return;
}
const uint i3 = fastdiv(idx, p.ne1_012mp, p.ne1_012L);
const uint i3 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));
const uint i3_offset = i3 * p.ne12*p.ne11*p.ne10;
const uint i2 = fastdiv(idx - i3_offset, p.ne1_01mp, p.ne1_01L);
const uint i2 = fastdiv(idx - i3_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));
const uint i2_offset = i2*p.ne11*p.ne10;
const uint i1 = fastdiv(idx - i3_offset - i2_offset, p.ne1_0mp, p.ne1_0L);
const uint i1 = fastdiv(idx - i3_offset - i2_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));
const uint i0 = idx - i3_offset - i2_offset - i1*p.ne10;
const uint p1 = floatBitsToUint(p.param1);

29
ggml/src/ggml-vulkan/vulkan-shaders/round.comp
View file

@ -1,29 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float x = float(data_a[i]);
float result;
// Round halfway cases away from zero as roundf does.
if (x >= 0.0) {
result = floor(x + 0.5);
} else {
result = ceil(x - 0.5);
}
data_d[i] = D_TYPE(result);
}

21
ggml/src/ggml-vulkan/vulkan-shaders/sgn.comp
View file

@ -1,21 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
data_d[i] = D_TYPE(sign(float(data_a[i])));
}

20
ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp
View file

@ -1,20 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
data_d[i] = D_TYPE(1. / (1 + exp(-1. * float(data_a[i]))));
}

22
ggml/src/ggml-vulkan/vulkan-shaders/silu.comp
View file

@ -1,22 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float xi = float(data_a[i]);
data_d[i] = D_TYPE(xi / (1.0f + exp(-xi)));
}

23
ggml/src/ggml-vulkan/vulkan-shaders/softplus.comp
View file

@ -1,23 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float x = float(data_a[i]);
const float result = (x > 20.0f) ? x : log(1.0f + exp(x));
data_d[i] = D_TYPE(result);
}

22
ggml/src/ggml-vulkan/vulkan-shaders/step.comp
View file

@ -1,22 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float x = float(data_a[i]);
data_d[i] = D_TYPE(x >= 0.0f ? 1.0f : 0.0f);
}

20
ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp
View file

@ -1,20 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
data_d[i] = D_TYPE(1. - 2. / (exp(2.*float(data_a[i])) + 1.));
}

6
ggml/src/ggml-vulkan/vulkan-shaders/tri.comp
View file

@ -17,11 +17,11 @@ void main() {
return;
}
const uint i03 = fastdiv(idx, p.ne0_012mp, p.ne0_012L);
const uint i03 = fastdiv(idx, p.ne0_012mp, fastdiv_L(p.ne0_Ls, 0));
const uint i03_offset = i03 * p.ne02*p.ne01*p.ne00;
const uint i02 = fastdiv(idx - i03_offset, p.ne0_01mp, p.ne0_01L);
const uint i02 = fastdiv(idx - i03_offset, p.ne0_01mp, fastdiv_L(p.ne0_Ls, 1));
const uint i02_offset = i02*p.ne01*p.ne00;
const uint i01 = fastdiv(idx - i03_offset - i02_offset, p.ne0_0mp, p.ne0_0L);
const uint i01 = fastdiv(idx - i03_offset - i02_offset, p.ne0_0mp, fastdiv_L(p.ne0_Ls, 2));
const uint i00 = idx - i03_offset - i02_offset - i01*p.ne00;
int param = floatBitsToInt(p.param1);

22
ggml/src/ggml-vulkan/vulkan-shaders/trunc.comp
View file

@ -1,22 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
const float x = float(data_a[i]);
data_d[i] = D_TYPE(trunc(x));
}

144
ggml/src/ggml-vulkan/vulkan-shaders/unary.comp Normal file
View file

@ -0,0 +1,144 @@
#version 450
#include "types.glsl"
#include "generic_unary_head.glsl"
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
float op_abs(float x) {
return abs(x);
}
float op_sgn(float x) {
return sign(x);
}
float op_neg(float x) {
return -x;
}
float op_step(float x) {
return x >= 0.0f ? 1.0f : 0.0f;
}
float op_tanh(float x) {
return 1.0f - 2.0f / (exp(2.0f*x) + 1.0f);
}
float op_elu(float x) {
return x < 0.0f ? exp(x) - 1.0f : x;
}
float op_relu(float x) {
return max(x, 0.0f);
}
float op_sigmoid(float x) {
return 1.0f / (1.0f + exp(-x));
}
float op_gelu(float x) {
const float GELU_COEF_A = 0.044715f;
const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
const float val = SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x);
return 0.5f*x*(2.0f - 2.0f / (exp(2.0f * val) + 1.0f));
}
float op_gelu_quick(float x) {
const float GELU_QUICK_COEF = -1.702f;
return x * (1.0f / (1.0f + exp(GELU_QUICK_COEF * x)));
}
float op_silu(float x) {
return x / (1.0f + exp(-x));
}
float op_hardswish(float x) {
return x * min(1.0f, max(0.0f, (x + 3.0f) / 6.0f));
}
float op_hardsigmoid(float x) {
return min(1.0f, max(0.0f, (x + 3.0f) / 6.0f));
}
float op_exp(float x) {
return exp(x);
}
float op_expm1(float x) {
// exp(x) - 1 loses many ulps to cancellation near zero. Use a degree-6
// Taylor expansion for |x| <= 1/4: the omitted x^7/5040 term is < 1.3e-8,
// about 0.5 ulp at expm1(0.25), and a host-side f32 model stays within
// 2 ulps over the interval. The first native exp(x)-1 values outside the
// cutoff are about 1 ulp for +0.25 and 2 ulps for -0.25.
if (abs(x) <= 0.25f) {
return x * (1.0f + x * (0.5f + x * ((1.0f/6.0f) + x * ((1.0f/24.0f) + x * ((1.0f/120.0f) + x * (1.0f/720.0f))))));
}
return exp(x) - 1.0f;
}
float op_softplus(float x) {
return (x > 20.0f) ? x : log(1.0f + exp(x));
}
float op_gelu_erf(float a) {
// based on Abramowitz and Stegun formula 7.1.26 or similar Hastings' approximation
const float p_erf = 0.3275911f;
const float a1_erf = 0.254829592f;
const float a2_erf = -0.284496736f;
const float a3_erf = 1.421413741f;
const float a4_erf = -1.453152027f;
const float a5_erf = 1.061405429f;
const float SQRT_2_INV = 0.70710678118654752440084436210484f;
const float a_div_sqr2 = a * SQRT_2_INV;
const float sign_x = sign(a_div_sqr2);
const float x = abs(a_div_sqr2);
const float t = 1.0f / (1.0f + p_erf * x);
const float y = 1.0f - (((((a5_erf * t + a4_erf) * t) + a3_erf) * t + a2_erf) * t + a1_erf) * t * exp(-x * x);
return 0.5f * a * (1.0f + sign_x * y);
}
float op_xielu(float x) {
const float alpha_n = p.param1;
const float alpha_p = p.param2;
const float beta = p.param3;
const float eps = p.param4;
if (x > 0.0f) {
return alpha_p * x * x + beta * x;
}
const float min_x_eps = min(x, eps);
return (op_expm1(min_x_eps) - x) * alpha_n + beta * x;
}
float op_floor(float x) {
return floor(x);
}
float op_ceil(float x) {
return ceil(x);
}
float op_round(float x) {
// Round halfway cases away from zero as roundf does.
return x >= 0.0f ? floor(x + 0.5f) : ceil(x - 0.5f);
}
float op_trunc(float x) {
return trunc(x);
}
void main() {
const uint idx = get_idx();
if (idx >= p.ne) {
return;
}
const uint a_idx = get_aoffset() + src0_idx(idx);
const uint d_idx = get_doffset() + dst_idx(idx);
data_d[d_idx] = D_TYPE(OP(float(data_a[a_idx])));
}

86
ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
View file

@ -894,47 +894,49 @@ void process_shaders() {
string_to_spv("upscale_f32", "upscale.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("exp_f16", "exp.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("exp_f32", "exp.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("exp_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_exp"}});
string_to_spv("exp_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_exp"}});
string_to_spv("expm1_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_expm1"}});
string_to_spv("expm1_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_expm1"}});
string_to_spv("log_f16", "log.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("log_f32", "log.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("gelu_f16", "gelu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("gelu_f32", "gelu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("gelu_erf_f16", "gelu_erf.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("gelu_erf_f32", "gelu_erf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("gelu_quick_f16", "gelu_quick.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("gelu_quick_f32", "gelu_quick.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("silu_f16", "silu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("silu_f32", "silu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("relu_f16", "relu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("relu_f32", "relu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("neg_f16", "neg.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("neg_f32", "neg.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("tanh_f16", "tanh.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("tanh_f32", "tanh.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("sigmoid_f16", "sigmoid.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("sigmoid_f32", "sigmoid.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("hardsigmoid_f16","hardsigmoid.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("hardsigmoid_f32","hardsigmoid.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("hardswish_f16", "hardswish.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("hardswish_f32", "hardswish.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("abs_f16", "abs.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("abs_f32", "abs.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("elu_f16", "elu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("elu_f32", "elu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("xielu_f16", "xielu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("xielu_f32", "xielu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("sgn_f16", "sgn.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("sgn_f32", "sgn.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("gelu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu"}});
string_to_spv("gelu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu"}});
string_to_spv("gelu_erf_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu_erf"}});
string_to_spv("gelu_erf_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu_erf"}});
string_to_spv("gelu_quick_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu_quick"}});
string_to_spv("gelu_quick_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu_quick"}});
string_to_spv("silu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_silu"}});
string_to_spv("silu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_silu"}});
string_to_spv("relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_relu"}});
string_to_spv("relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_relu"}});
string_to_spv("neg_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_neg"}});
string_to_spv("neg_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_neg"}});
string_to_spv("tanh_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_tanh"}});
string_to_spv("tanh_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_tanh"}});
string_to_spv("sigmoid_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sigmoid"}});
string_to_spv("sigmoid_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sigmoid"}});
string_to_spv("hardsigmoid_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_hardsigmoid"}});
string_to_spv("hardsigmoid_f32","unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_hardsigmoid"}});
string_to_spv("hardswish_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_hardswish"}});
string_to_spv("hardswish_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_hardswish"}});
string_to_spv("abs_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_abs"}});
string_to_spv("abs_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_abs"}});
string_to_spv("elu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_elu"}});
string_to_spv("elu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_elu"}});
string_to_spv("xielu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_xielu"}});
string_to_spv("xielu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_xielu"}});
string_to_spv("sgn_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sgn"}});
string_to_spv("sgn_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sgn"}});
string_to_spv("tri_f16", "tri.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("tri_f32", "tri.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("diag_f16", "diag.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("diag_f32", "diag.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("softplus_f16", "softplus.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("softplus_f32", "softplus.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("softplus_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_softplus"}});
string_to_spv("softplus_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_softplus"}});
string_to_spv("add1_f16_f16", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
string_to_spv("add1_f16_f32", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
@ -942,16 +944,16 @@ void process_shaders() {
string_to_spv("arange_f32", "arange.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
string_to_spv("fill_f32", "fill.comp", {{"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
string_to_spv("fill_f16", "fill.comp", {{"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
string_to_spv("step_f16", "step.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("step_f32", "step.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("round_f16", "round.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("round_f32", "round.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("ceil_f16", "ceil.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("ceil_f32", "ceil.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("floor_f16", "floor.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("floor_f32", "floor.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("trunc_f16", "trunc.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("trunc_f32", "trunc.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("step_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_step"}});
string_to_spv("step_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_step"}});
string_to_spv("round_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_round"}});
string_to_spv("round_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_round"}});
string_to_spv("ceil_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_ceil"}});
string_to_spv("ceil_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_ceil"}});
string_to_spv("floor_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_floor"}});
string_to_spv("floor_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_floor"}});
string_to_spv("trunc_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_trunc"}});
string_to_spv("trunc_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_trunc"}});
string_to_spv("geglu_f16", "geglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("geglu_f32", "geglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});

35
ggml/src/ggml-vulkan/vulkan-shaders/xielu.comp
View file

@ -1,35 +0,0 @@
#version 450
#include "generic_head.glsl"
#include "types.glsl"
#extension GL_EXT_control_flow_attributes : enable
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
void main() {
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
if (i >= p.KX) {
return;
}
float x = float(data_a[i]);
float alpha_n = p.param1;
float alpha_p = p.param2;
float beta = p.param3;
float eps = p.param4;
if (x > 0.0f) {
x = alpha_p * x * x + beta * x;
} else {
const float min_x_eps = min(x, eps);
x = (exp(min_x_eps) - 1 - x) * alpha_n + beta * x;
}
data_d[i] = D_TYPE(x);
}

29
gguf-py/gguf/constants.py
View file

@ -457,6 +457,7 @@ class MODEL_ARCH(IntEnum):
XVERSE = auto()
COMMAND_R = auto()
COHERE2 = auto()
COHERE2MOE = auto()
DBRX = auto()
OLMO = auto()
OLMO2 = auto()
@ -1012,6 +1013,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.XVERSE: "xverse",
MODEL_ARCH.COMMAND_R: "command-r",
MODEL_ARCH.COHERE2: "cohere2",
MODEL_ARCH.COHERE2MOE: "cohere2moe",
MODEL_ARCH.DBRX: "dbrx",
MODEL_ARCH.OLMO: "olmo",
MODEL_ARCH.OLMO2: "olmo2",
@ -2872,6 +2874,33 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
],
MODEL_ARCH.COHERE2MOE: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
MODEL_TENSOR.OUTPUT,
MODEL_TENSOR.ATTN_NORM,
MODEL_TENSOR.ATTN_Q,
MODEL_TENSOR.ATTN_K,
MODEL_TENSOR.ATTN_V,
MODEL_TENSOR.ATTN_OUT,
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.FFN_GATE_INP,
MODEL_TENSOR.FFN_GATE_EXP,
MODEL_TENSOR.FFN_GATE_UP_EXP,
MODEL_TENSOR.FFN_DOWN_EXP,
MODEL_TENSOR.FFN_UP_EXP,
MODEL_TENSOR.FFN_GATE_SHEXP,
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
MODEL_TENSOR.NEXTN_EH_PROJ,
MODEL_TENSOR.NEXTN_EMBED_TOKENS,
MODEL_TENSOR.NEXTN_ENORM,
MODEL_TENSOR.NEXTN_HNORM,
MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
],
MODEL_ARCH.DBRX: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,

1
src/llama-arch.cpp
View file

@ -66,6 +66,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_XVERSE, "xverse" },
{ LLM_ARCH_COMMAND_R, "command-r" },
{ LLM_ARCH_COHERE2, "cohere2" },
{ LLM_ARCH_COHERE2MOE, "cohere2moe" },
{ LLM_ARCH_DBRX, "dbrx" },
{ LLM_ARCH_OLMO, "olmo" },
{ LLM_ARCH_OLMO2, "olmo2" },

1
src/llama-arch.h
View file

@ -71,6 +71,7 @@ enum llm_arch {
LLM_ARCH_XVERSE,
LLM_ARCH_COMMAND_R,
LLM_ARCH_COHERE2,
LLM_ARCH_COHERE2MOE,
LLM_ARCH_DBRX,
LLM_ARCH_OLMO,
LLM_ARCH_OLMO2,

1
src/llama-ext.h
View file

@ -2,6 +2,7 @@
// this is a staging header for new llama.cpp API
// breaking changes and C++ are allowed. everything here should be considered WIP
// try as much as possible to not include this header in the rest of the codebase
#include "llama.h"

1
src/llama-model-saver.cpp
View file

@ -18,6 +18,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {
case LLM_ARCH_GEMMA3:
case LLM_ARCH_GEMMA3N:
case LLM_ARCH_COHERE2:
case LLM_ARCH_COHERE2MOE:
case LLM_ARCH_OLMO2:
case LLM_ARCH_BITNET:
case LLM_ARCH_T5:

10
src/llama-model.cpp
View file

@ -52,6 +52,7 @@
#include "models/codeshell.cpp"
#include "models/cogvlm.cpp"
#include "models/cohere2.cpp"
#include "models/cohere2moe.cpp"
#include "models/command-r.cpp"
#include "models/dbrx.cpp"
#include "models/deci.cpp"
@ -292,6 +293,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_command_r(params);
case LLM_ARCH_COHERE2:
return new llama_model_cohere2(params);
case LLM_ARCH_COHERE2MOE:
return new llama_model_cohere2moe(params);
case LLM_ARCH_DBRX:
return new llama_model_dbrx(params);
case LLM_ARCH_OLMO:
@ -1602,9 +1605,12 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
ml.done_getting_tensors();
// Tied NVFP4 output is valid when no separate LM-head scale tensors are present.
// If sidecar scales exist, the output weight must be an actual output tensor.
GGML_ASSERT(!(output && tok_embd &&
strcmp(output->name, tok_embd->name) == 0 &&
output->type == GGML_TYPE_NVFP4));
output->type == GGML_TYPE_NVFP4 &&
(output_s || output_in_s)));
// populate tensors_by_name
for (auto & [_, ctx_ptr] : ml.ctx_map) {
for (auto * cur = ggml_get_first_tensor(ctx_ptr.get()); cur != NULL; cur = ggml_get_next_tensor(ctx_ptr.get(), cur)) {
@ -1979,6 +1985,7 @@ void llama_model::print_info() const {
}
if (arch == LLM_ARCH_MELLUM ||
arch == LLM_ARCH_COHERE2MOE ||
arch == LLM_ARCH_QWEN3MOE ||
arch == LLM_ARCH_OPENAI_MOE ||
arch == LLM_ARCH_QWEN3VLMOE ||
@ -2524,6 +2531,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_XVERSE:
case LLM_ARCH_COMMAND_R:
case LLM_ARCH_COHERE2:
case LLM_ARCH_COHERE2MOE:
case LLM_ARCH_OLMO:
case LLM_ARCH_ARCTIC:
case LLM_ARCH_DEEPSEEK:

6
src/models/cohere2.cpp
View file

@ -122,9 +122,9 @@ llama_model_cohere2::graph::graph(const llama_model & model, const llm_graph_par
// feed-forward network
{
cur = build_ffn(ffn_inp,
model.layers[il].ffn_up, NULL, NULL,
model.layers[il].ffn_gate, NULL, NULL,
model.layers[il].ffn_down, NULL, NULL,
model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,
model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,
model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,
NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
cb(cur, "ffn_out", il);
}

443
src/models/cohere2moe.cpp Normal file
View file

@ -0,0 +1,443 @@
#include "models.h"
void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {
const bool found_norm = ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps, false);
const bool found_norm_rms = ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps, false);
if (!found_norm && !found_norm_rms) {
throw std::runtime_error("missing Cohere2 MoE norm epsilon");
}
if (!found_norm_rms) {
hparams.f_norm_rms_eps = 0.0f;
}
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);
ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);
ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false);
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
}
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
uint32_t swa_period = 4;
if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) {
hparams.set_swa_pattern(swa_period, true);
} else {
ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
}
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
switch (hparams.n_layer()) {
case 49: type = LLM_TYPE_30B_A3B; break;
default: type = LLM_TYPE_UNKNOWN;
}
}
void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) {
LLAMA_LOAD_LOCALS;
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
// Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP
// tensors live in a separate file. Mark MTP tensors NOT_REQUIRED so the
// trunk loads cleanly.
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
// output
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0);
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);
// if output is NULL, init from the input tok embed
if (output == NULL) {
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED);
}
if (n_expert == 0) {
throw std::runtime_error("n_expert must be > 0 for Cohere2Moe");
}
if (n_expert_used == 0) {
throw std::runtime_error("n_expert_used must be > 0 for Cohere2Moe");
}
auto load_block_trunk = [&](int i, int flags) {
auto & layer = layers[i];
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, flags);
create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, flags);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, flags);
if (static_cast<uint32_t>(i) < hparams.n_layer_dense_lead) {
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, flags);
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, flags);
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, flags);
} else {
const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff;
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, flags);
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, flags);
create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, flags);
if (hparams.n_expert_shared > 0) {
const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff_exp * hparams.n_expert_shared;
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), { n_embd, n_ff_shexp }, flags);
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_shexp, n_embd }, flags);
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), { n_embd, n_ff_shexp }, flags);
}
}
};
auto load_block_mtp = [&](int i, int flags) {
auto & layer = layers[i];
// MTP block looks like a full-attention Cohere2 MoE decoder block.
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, flags);
create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, flags);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, flags);
const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff;
// Routed experts
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, flags);
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, flags);
create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, flags);
if (hparams.n_expert_shared > 0) {
const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff_exp * hparams.n_expert_shared;
// Shared experts
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), { n_embd, n_ff_shexp }, flags);
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_shexp, n_embd }, flags);
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), { n_embd, n_ff_shexp }, flags);
}
// NextN-specific tensors that define the MTP block.
layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags);
layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags);
layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), { n_embd }, flags);
layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED);
};
for (int i = 0; i < n_layer; ++i) {
load_block_trunk(i, trunk_flags);
}
// MTP/NextN layers are loaded as extra decoder blocks.
for (int i = n_layer; i < n_layer_all; ++i) {
load_block_mtp(i, mtp_flags);
}
}
std::unique_ptr<llm_graph_context> llama_model_cohere2moe::build_arch_graph(const llm_graph_params & params) const {
if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
return std::make_unique<graph_mtp>(*this, params);
}
return std::make_unique<graph>(*this, params);
}
llama_model_cohere2moe::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
GGML_ASSERT(n_embd_head == n_rot);
const llm_norm_type cohere2moe_norm_type = hparams.f_norm_rms_eps == 0.0f ? LLM_NORM : LLM_NORM_RMS;
const float f_logit_scale = hparams.f_logit_scale;
ggml_tensor * cur;
ggml_tensor * inpL = build_inp_embd(model.tok_embd);
ggml_tensor * inp_pos = build_inp_pos();
auto * inp_attn = build_attn_inp_kv_iswa();
ggml_tensor * inp_out_ids = build_inp_out_ids();
// MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass.
for (int il = 0; il < n_layer; ++il) {
const bool is_swa = hparams.is_swa(il);
// Dense-prefix full-attention layers use RoPE; later layers follow the SWA pattern.
const bool force_rope = static_cast<uint32_t>(il) < hparams.n_layer_dense_lead;
cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, cohere2moe_norm_type, il);
cb(cur, "attn_norm", il);
ggml_tensor * ffn_inp = cur;
{
const auto & layer = model.layers[il];
auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur,
n_embd_head, n_head, n_head_kv, il);
if (is_swa || force_rope) {
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, rope_factors,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
Kcur = ggml_rope_ext(
ctx0, Kcur, inp_pos, rope_factors,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
}
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
layer.wo, layer.wo_b, layer.wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
1.0f / sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);
}
ggml_tensor * attn_out = cur;
const auto & layer = model.layers[il];
if (layer.ffn_gate_inp == nullptr) {
cur = build_ffn(ffn_inp,
layer.ffn_up, nullptr, layer.ffn_up_s,
layer.ffn_gate, nullptr, layer.ffn_gate_s,
layer.ffn_down, nullptr, layer.ffn_down_s,
nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
cb(cur, "ffn_out", il);
} else {
cur = build_moe_ffn(ffn_inp,
layer.ffn_gate_inp,
layer.ffn_up_exps,
layer.ffn_gate_exps,
layer.ffn_down_exps,
nullptr,
n_expert, n_expert_used,
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr, layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
cb(cur, "ffn_moe_out", il);
if (layer.ffn_up_shexp) {
ggml_tensor * ffn_shexp = build_ffn(ffn_inp,
layer.ffn_up_shexp, nullptr, layer.ffn_up_shexp_s,
layer.ffn_gate_shexp, nullptr, layer.ffn_gate_shexp_s,
layer.ffn_down_shexp, nullptr, layer.ffn_down_shexp_s,
nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
cb(ffn_shexp, "ffn_shexp", il);
cur = ggml_add(ctx0, cur, ffn_shexp);
cur = ggml_scale(ctx0, cur, 0.5f);
cb(cur, "ffn_out", il);
}
}
cur = ggml_add(ctx0, cur, inpL);
cur = ggml_add(ctx0, cur, attn_out);
cur = build_cvec(cur, il);
cb(cur, "l_out", il);
inpL = cur;
}
cur = inpL;
cur = build_norm(cur, model.output_norm, nullptr, cohere2moe_norm_type, -1);
cb(cur, "h_nextn", -1);
res->t_h_nextn = cur;
if (!cparams.embeddings_nextn_masked && inp_out_ids) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
}
cb(cur, "result_norm", -1);
res->t_embd = cur;
cur = build_lora_mm(model.output, cur);
if (f_logit_scale) {
cur = ggml_scale(ctx0, cur, f_logit_scale);
}
cb(cur, "result_output", -1);
res->t_logits = cur;
ggml_build_forward_expand(gf, cur);
}
llama_model_cohere2moe::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
GGML_ASSERT(hparams.n_layer_nextn > 0 && "COHERE2MOE MTP requires n_layer_nextn > 0");
GGML_ASSERT(hparams.n_layer_nextn == 1 && "COHERE2MOE MTP currently only supports a single MTP block");
const int64_t n_embd_head = hparams.n_embd_head_v();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
GGML_ASSERT(n_embd_head == n_rot);
const int il = hparams.n_layer();
const auto & layer = model.layers[il];
GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj");
GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm");
GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm");
GGML_ASSERT(layer.ffn_gate_inp && "MTP block missing ffn_gate_inp");
const llm_norm_type cohere2moe_norm_type = hparams.f_norm_rms_eps == 0.0f ? LLM_NORM : LLM_NORM_RMS;
// TODO: extract in a common llm_graph_context::build_inp_embd_h()
auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
ggml_set_input(inp->tokens);
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
ggml_set_input(inp->embd);
// TODO: make static using `ggml_build_forward_select()`
// see llm_graph_context::build_inp_embd() for reference
ggml_tensor * tok_embd;
if (ubatch.token) {
ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
} else {
tok_embd = inp->embd;
}
cb(tok_embd, "mtp_tok_embd", il);
inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
ggml_set_input(inp->h);
ggml_set_name(inp->h, "mtp_h_input");
ggml_tensor * h_embd = inp->h;
res->add_input(std::move(inp));
ggml_tensor * inp_pos = build_inp_pos();
ggml_tensor * inp_out_ids = build_inp_out_ids();
auto * inp_attn = build_attn_inp_kv_iswa();
ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, cohere2moe_norm_type, il);
cb(h_norm, "mtp_hnorm", il);
ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, cohere2moe_norm_type, il);
cb(e_norm, "mtp_enorm", il);
ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0);
cb(concat, "mtp_concat", il);
ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);
cb(cur, "mtp_eh_proj", il);
ggml_tensor * inpL = cur;
cur = build_norm(cur, layer.attn_norm, nullptr, cohere2moe_norm_type, il);
cb(cur, "mtp_attn_norm", il);
ggml_tensor * ffn_inp = cur;
auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, rope_factors,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
Kcur = ggml_rope_ext(
ctx0, Kcur, inp_pos, rope_factors,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Qcur, "mtp_Qcur", il);
cb(Kcur, "mtp_Kcur", il);
cb(Vcur, "mtp_Vcur", il);
cur = build_attn(inp_attn,
layer.wo, layer.wo_b, layer.wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
1.0f / sqrtf(float(n_embd_head)), il);
cb(cur, "mtp_attn_out", il);
ggml_tensor * attn_out = cur;
cur = build_moe_ffn(ffn_inp,
layer.ffn_gate_inp,
layer.ffn_up_exps,
layer.ffn_gate_exps,
layer.ffn_down_exps,
nullptr,
n_expert, n_expert_used,
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr, layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
cb(cur, "mtp_ffn_moe_out", il);
if (layer.ffn_up_shexp) {
ggml_tensor * ffn_shexp = build_ffn(ffn_inp,
layer.ffn_up_shexp, nullptr, layer.ffn_up_shexp_s,
layer.ffn_gate_shexp, nullptr, layer.ffn_gate_shexp_s,
layer.ffn_down_shexp, nullptr, layer.ffn_down_shexp_s,
nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
cb(ffn_shexp, "mtp_ffn_shexp", il);
cur = ggml_add(ctx0, cur, ffn_shexp);
cur = ggml_scale(ctx0, cur, 0.5f);
cb(cur, "mtp_ffn_out", il);
}
cur = ggml_add(ctx0, cur, inpL);
cur = ggml_add(ctx0, cur, attn_out);
cb(cur, "mtp_post_ffn", il);
ggml_tensor * head_norm_w = layer.nextn.shared_head_norm
? layer.nextn.shared_head_norm
: model.output_norm;
GGML_ASSERT(head_norm_w && "COHERE2MOE MTP: missing both nextn.shared_head_norm and output_norm");
cur = build_norm(cur, head_norm_w, nullptr, cohere2moe_norm_type, -1);
cb(cur, "h_nextn", -1);
res->t_h_nextn = cur;
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
cb(cur, "mtp_shared_head_norm", -1);
ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;
GGML_ASSERT(head_w && "COHERE2MOE MTP: missing LM head (nextn.shared_head_head or model.output)");
cur = build_lora_mm(head_w, cur, layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : nullptr);
if (hparams.f_logit_scale) {
cur = ggml_scale(ctx0, cur, hparams.f_logit_scale);
}
cb(cur, "result_output", -1);
res->t_logits = cur;
ggml_build_forward_expand(gf, cur);
}

17
src/models/models.h
View file

@ -937,6 +937,23 @@ struct llama_model_cohere2 : public llama_model_base {
};
struct llama_model_cohere2moe : public llama_model_base {
llama_model_cohere2moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
void load_arch_tensors(llama_model_loader & ml) override;
struct graph : public llm_graph_context {
graph(const llama_model & model, const llm_graph_params & params);
};
struct graph_mtp : public llm_graph_context {
graph_mtp(const llama_model & model, const llm_graph_params & params);
};
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
struct llama_model_dbrx : public llama_model_base {
llama_model_dbrx(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;

4
tools/mtmd/clip-graph.h
View file

@ -54,6 +54,10 @@ struct clip_graph {
virtual ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const;
// TODO: build_mm(w, b, x) to support bias
virtual bool support_batch() const {
return false;
}
//
// utility functions
//

77
tools/mtmd/clip.cpp
View file

@ -219,6 +219,8 @@ struct clip_ctx {
std::map<ggml_backend_dev_t, size_t> mem_usage;
std::map<ggml_backend_dev_t, size_t> mem_compute;
bool support_batch = false;
clip_ctx(clip_context_params & ctx_params) {
flash_attn_type = ctx_params.flash_attn_type;
no_alloc = ctx_params.no_alloc;
@ -362,7 +364,7 @@ ggml_tensor * clip_graph::build_vit(
std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos,
const build_vit_opts & opts
) {
// batch dim: inp is [n_embd, n_pos] (B==1) or [n_embd, n_pos, B] (multi-tile encode)
// batch dim: inp is [n_embd, n_pos, B]
const int64_t B = inp->ne[2];
if (learned_pos_embd) {
@ -910,7 +912,7 @@ ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale
return cur;
}
static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch & imgs) {
static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs) {
const clip_image_f32 & img = *imgs.entries[0];
std::unique_ptr<clip_graph> builder;
@ -1073,7 +1075,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
// TODO [QWEN_VIDEO]: improve this in the future
builder->n_batch = imgs.entries.size();
return builder->build();
return builder;
}
//
@ -2900,7 +2902,7 @@ struct clip_model_loader {
std::vector<support_info_op> ops;
};
static void warmup(clip_ctx & ctx_clip) {
static clip_image_f32_batch get_dummy_batch(clip_ctx & ctx_clip) {
// create a fake batch
const auto & hparams = ctx_clip.model.hparams;
clip_image_f32_batch batch;
@ -2914,6 +2916,20 @@ struct clip_model_loader {
LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size);
}
batch.entries.push_back(std::move(img));
return batch;
}
static void init_ctx(clip_ctx & ctx_clip) {
ctx_clip.buf_compute_meta.resize(ctx_clip.max_nodes * ggml_tensor_overhead() + ggml_graph_overhead());
// check batching support
auto batch = get_dummy_batch(ctx_clip);
auto builder = clip_get_graph_builder(&ctx_clip, batch);
ctx_clip.support_batch = builder->support_batch();
}
static void warmup(clip_ctx & ctx_clip) {
auto batch = get_dummy_batch(ctx_clip);
warmup(ctx_clip, batch);
}
@ -2986,9 +3002,7 @@ struct clip_model_loader {
// only initialize backend buffers, but do not allocate them yet
static support_info_graph reserve_compute_meta(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) {
ctx_clip.buf_compute_meta.resize(ctx_clip.max_nodes * ggml_tensor_overhead() + ggml_graph_overhead());
ggml_cgraph * gf = clip_image_build_graph(&ctx_clip, batch);
ggml_cgraph * gf = clip_get_graph_builder(&ctx_clip, batch)->build();
ggml_backend_sched_reserve(ctx_clip.sched.get(), gf);
ctx_clip.mem_compute.clear();
@ -3151,6 +3165,7 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
ctx_vision = new clip_ctx(ctx_params);
loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION);
loader.load_tensors(*ctx_vision);
loader.init_ctx(*ctx_vision);
if (ctx_params.warmup) {
loader.warmup(*ctx_vision);
}
@ -3164,6 +3179,7 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
ctx_audio = new clip_ctx(ctx_params);
loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO);
loader.load_tensors(*ctx_audio);
loader.init_ctx(*ctx_audio);
if (ctx_params.warmup) {
loader.warmup(*ctx_audio);
}
@ -3565,25 +3581,22 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
return n_patches;
}
bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f32 * img, float * vec) {
bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f32 * img, std::vector<float> & out_vec) {
clip_image_f32_batch imgs;
clip_image_f32_ptr img_copy(clip_image_f32_init());
*img_copy = *img;
imgs.entries.push_back(std::move(img_copy));
return clip_image_batch_encode(ctx, n_threads, &imgs, vec);
return clip_image_batch_encode(ctx, n_threads, &imgs, out_vec);
}
bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs_c_ptr, float * vec) {
bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs_c_ptr, std::vector<float> & out_batch_embd) {
const clip_image_f32_batch & imgs = *imgs_c_ptr;
int n_batch_cur = imgs.entries.size();
// maximum supported batch size, usually == 2 for qwen-vl-based models
int n_batch_max = clip_model_n_batch_max(ctx);
// TODO @ngxson : implement batch size > 1 as a loop
// we don't need true batching support because the cgraph will gonna be big anyway
if (n_batch_cur > n_batch_max) {
// [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames
if (!ctx->support_batch && n_batch_cur > clip_model_n_temporal_merge(ctx)) {
LOG_ERR("%s: batch size %d exceeds maximum supported batch/temporal-merge size %d\n", __func__, n_batch_cur, clip_model_n_temporal_merge(ctx));
return false;
}
@ -3594,7 +3607,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
// build the inference graph
ggml_backend_sched_reset(ctx->sched.get());
ggml_cgraph * gf = clip_image_build_graph(ctx, imgs);
ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs)->build();
ggml_backend_sched_alloc_graph(ctx->sched.get(), gf);
// set inputs
@ -3663,6 +3676,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
const int n = nx * ny;
for (int b = 0; b < n_batch_cur; b++) {
LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny);
const auto & buf = imgs.entries[b]->get_ro_buf();
float * batch_entry = inp_raw.data() + b * (3*n);
for (int y = 0; y < ny; y++) {
@ -4497,7 +4511,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
// the last node is the embedding tensor
ggml_tensor * embeddings = ggml_graph_node(gf, -1);
// sanity check (only support batch size of 1 for now)
// sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one)
const int n_tokens_out = embeddings->ne[1];
const int expected_n_tokens_out = clip_n_output_tokens(ctx, imgs.entries[0].get());
if (n_tokens_out != expected_n_tokens_out) {
@ -4505,16 +4519,26 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
GGML_ABORT("Invalid number of output tokens");
}
// copy the embeddings to the location passed by the user
if (vec != nullptr) {
ggml_backend_tensor_get(embeddings, vec, 0, ggml_nbytes(embeddings));
LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__,
(int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]);
// copy output to user buffer if provided
// if output is empty, skip the copy
if (!out_batch_embd.empty()) {
if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) {
LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings));
GGML_ABORT("Output buffer size mismatch");
}
ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings));
} else {
LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__);
}
// Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set
if (ctx->debug_output_embeddings) {
const int64_t n_embd = embeddings->ne[0];
const int64_t n_tokens = embeddings->ne[1];
std::vector<float> emb_data(n_embd * n_tokens);
std::vector<float> emb_data(ggml_nelements(embeddings));
ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings));
LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
@ -4651,7 +4675,14 @@ bool clip_has_audio_encoder(const struct clip_ctx * ctx) {
return ctx->model.modality == CLIP_MODALITY_AUDIO;
}
int clip_model_n_batch_max(const struct clip_ctx * ctx) {
bool clip_support_batch(const struct clip_ctx * ctx) {
return ctx->support_batch;
}
// TODO @ngxson : this is no longer correct with mtmd_batch API
// this was only meant to be used by qwen-vl-based models, to fuse 2 input images into one (qwen-vl video support)
// this logic should be refactored in near future to distinctly handle "merge frames" and "batching"
int clip_model_n_temporal_merge(const struct clip_ctx * ctx) {
switch (ctx->proj_type()) {
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:

8
tools/mtmd/clip.h
View file

@ -97,8 +97,8 @@ size_t clip_image_f32_batch_nx(const struct clip_image_f32_batch * batch, int id
size_t clip_image_f32_batch_ny(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->ny
struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->data
bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);
bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);
bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, std::vector<float> & out_vec);
bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector<float> & out_batch_embd);
bool clip_is_llava(const struct clip_ctx * ctx);
// note for contributor: this clip_is_(model) pattern is deprecated
@ -107,7 +107,9 @@ bool clip_is_llava(const struct clip_ctx * ctx);
bool clip_has_vision_encoder(const struct clip_ctx * ctx);
bool clip_has_audio_encoder(const struct clip_ctx * ctx);
int clip_model_n_batch_max(const struct clip_ctx * ctx);
bool clip_support_batch(const struct clip_ctx * ctx);
int clip_model_n_temporal_merge(const struct clip_ctx * ctx); // TODO @ngxson : remove, refactor this
std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx);

20
tools/mtmd/models/gemma4v.cpp
View file

@ -10,7 +10,7 @@ ggml_cgraph * clip_graph_gemma4v::build() {
ggml_set_name(inp_raw, "inp_raw_scaled");
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
inp = ggml_reshape_2d(ctx0, inp, n_patches, n_embd);
inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch);
inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
ggml_set_name(inp, "inp");
// note: no patch bias
@ -51,10 +51,11 @@ ggml_cgraph * clip_graph_gemma4v::build() {
// first half
ggml_tensor * first;
{
first = ggml_view_3d(ctx0, cur,
n_dim/2, n_head, n_pos,
first = ggml_view_4d(ctx0, cur,
n_dim/2, n_head, n_pos, n_batch,
cur->nb[1],
cur->nb[2],
cur->nb[3],
0);
first = ggml_rope_ext(
ctx0,
@ -70,10 +71,11 @@ ggml_cgraph * clip_graph_gemma4v::build() {
// second half
ggml_tensor * second;
{
second = ggml_view_3d(ctx0, cur,
n_dim/2, n_head, n_pos,
second = ggml_view_4d(ctx0, cur,
n_dim/2, n_head, n_pos, n_batch,
cur->nb[1],
cur->nb[2],
cur->nb[3],
n_dim/2 * ggml_element_size(cur));
second = ggml_rope_ext(
ctx0,
@ -103,14 +105,14 @@ ggml_cgraph * clip_graph_gemma4v::build() {
const int kernel_size = hparams.n_merge;
GGML_ASSERT(kernel_size > 0);
// [n_embd, n_patches] -> [n_patches_x, n_patches_y, n_embd, 1]
cur = ggml_cont_4d(ctx0, ggml_transpose(ctx0, cur), n_patches_x, n_patches_y, n_embd, 1);
// [n_embd, n_patches] -> [n_patches_x, n_patches_y, n_embd, n_batch]
cur = ggml_cont_4d(ctx0, ggml_transpose(ctx0, cur), n_patches_x, n_patches_y, n_embd, n_batch);
cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG,
kernel_size, kernel_size, kernel_size, kernel_size, 0, 0);
const int out_x = n_patches_x / kernel_size;
const int out_y = n_patches_y / kernel_size;
// [out_x, out_y, n_embd, 1] -> [n_embd, out_x * out_y]
cur = ggml_reshape_3d(ctx0, cur, out_x * out_y, n_embd, 1);
// [out_x, out_y, n_embd, n_batch] -> [n_embd, out_x * out_y, n_batch]
cur = ggml_reshape_3d(ctx0, cur, out_x * out_y, n_embd, n_batch);
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
cur = ggml_scale(ctx0, cur, sqrtf((float)n_embd));
cb(cur, "pooled", -1);

1
tools/mtmd/models/models.h
View file

@ -16,6 +16,7 @@ struct clip_graph_gemma4v : clip_graph {
clip_graph_gemma4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
bool support_batch() const override { return true; }
};
struct clip_graph_gemma4uv : clip_graph {

13
tools/mtmd/mtmd-helper.h
View file

@ -67,8 +67,8 @@ MTMD_API void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * image,
// helper function that automatically:
// 1. run llama_decode() on text chunks
// 2. run mtmd_encode() on image chunks, then mtmd_get_output_embd() and then llama_decode()
// if any of the mtmd_encode() or llama_decode() calls return non-zero, stop and forward the error
// 2. run mtmd_encode_chunk() on image chunks, then mtmd_get_output_embd() and then llama_decode()
// if any of the mtmd_encode_chunk() or llama_decode() calls return non-zero, stop and forward the error
// otherwise, returns 0 on success
// this function is NOT thread-safe
MTMD_API int32_t mtmd_helper_eval_chunks(mtmd_context * ctx,
@ -157,13 +157,16 @@ MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
} // extern "C"
#endif
#ifdef __cplusplus
#include <set>
#include <memory>
namespace mtmd_helper {
//
// C++ wrappers
//
#ifdef __cplusplus
namespace mtmd_helper {
// video-related C++ wrappers
struct mtmd_helper_video_deleter {
void operator()(mtmd_helper_video * val) { mtmd_helper_video_free(val); }

362
tools/mtmd/mtmd.cpp
View file

@ -69,8 +69,8 @@ struct mtmd_bitmap {
return data.size();
}
bool can_batch_with(const mtmd_bitmap & other) const {
// [QWEN_VIDEO] can batch if both are images with same size
bool can_merge_with(const mtmd_bitmap & other) const {
// [QWEN_VIDEO] can (temporal) merge if both are images with same size
return !is_audio && !other.is_audio && nx == other.nx && ny == other.ny;
}
@ -90,12 +90,24 @@ struct mtmd_image_tokens {
uint32_t ny = 0; // number of tokens in y direction
mtmd_pos_type pos = MTMD_POS_TYPE_NORMAL;
uint32_t image_idx = 0; // 0-based position of this image among image chunks in the prompt(used by pos == MTMD_POS_TYPE_HUNYUANVL)
uint32_t n_temporal_merge = 1; // for qwen-vl style temporal merge
uint32_t n_tokens() const {
if (pos == MTMD_POS_TYPE_HUNYUANVL) {
// [BOI] [row0 tokens + newline] ... [row(ny-1) tokens + newline] [EOI]
return (nx + 1) * ny + 2;
}
return nx * ny;
// [QWEN_VIDEO] this logic is quite ugly, it's mostly to make qwen-vl temporal merge work, can be improved in the future
if (batch_f32.entries.size() == 1 || n_temporal_merge == 1) {
return nx * ny;
}
uint32_t nz = batch_f32.entries.size();
// TODO: simplify this by repeating the last frame until it fits the temporal merge
if (nz % n_temporal_merge != 0) {
nz = nz / n_temporal_merge + 1;
} else {
nz = nz / n_temporal_merge;
}
return nx * ny * nz;
}
clip_image_f32_batch batch_f32; // preprocessed image patches
std::string id; // optional user-defined ID, useful for KV cache tracking
@ -110,12 +122,17 @@ struct mtmd_image_tokens {
return false;
}
bool can_batch_with(const mtmd_image_tokens & other) {
return nx == other.nx && ny == other.ny && pos == other.pos;
}
mtmd_image_tokens clone() {
return mtmd_image_tokens{
nx,
ny,
pos,
image_idx,
n_temporal_merge,
batch_f32.clone(),
id
};
@ -153,12 +170,49 @@ struct mtmd_input_chunk {
std::vector<llama_token> tokens_text;
mtmd_image_tokens_ptr tokens_image;
mtmd_audio_tokens_ptr tokens_audio;
bool can_batch_with(const mtmd_input_chunk & other) const {
if (type != other.type) {
return false;
}
if (tokens_image && other.tokens_image) {
return tokens_image->can_batch_with(*other.tokens_image);
}
// TODO: allow batching audio chunks of the same size
return false;
}
bool is_placeholder() const {
if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {
return tokens_image && tokens_image->is_placeholder();
} else if (type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
return tokens_audio && tokens_audio->is_placeholder();
}
return false;
}
};
struct mtmd_input_chunks {
std::vector<mtmd_input_chunk> entries;
};
struct mtmd_batch {
mtmd_context * ctx;
std::vector<const mtmd_input_chunk *> entries;
std::vector<float> output_embd; // aggregated output embedding for the whole batch
mtmd_batch(mtmd_context * ctx): ctx(ctx) {}
int32_t n_tokens() const {
int32_t n = 0;
for (const auto * chunk : entries) {
n += mtmd_input_chunk_get_n_tokens(chunk);
}
return n;
}
};
// slice template, used by some llava-uhd models to correctly place the special tokens around image embeddings
// models not having it (llava-1.6) will process embeddings without any special tokens in-between
enum mtmd_slice_tmpl {
@ -197,6 +251,7 @@ mtmd_context_params mtmd_context_params_default() {
/* image_max_tokens */ -1,
/* cb_eval */ nullptr,
/* cb_eval_user_data */ nullptr,
/* batch_max_tokens */ 1024,
};
return params;
}
@ -204,7 +259,7 @@ mtmd_context_params mtmd_context_params_default() {
struct mtmd_context {
struct clip_ctx * ctx_v; // vision
struct clip_ctx * ctx_a; // audio
std::vector<float> image_embd_v; // image embedding vector
std::vector<float> out_embd; // image embedding vector
bool print_timings;
int n_threads;
@ -239,17 +294,21 @@ struct mtmd_context {
std::unique_ptr<mtmd_audio_preprocessor> audio_preproc;
std::unique_ptr<mtmd_image_preprocessor> image_preproc;
// batching
int32_t batch_max_tokens;
// TODO @ngxson : add timings
mtmd_context(const char * mmproj_fname,
const llama_model * text_model,
const mtmd_context_params & ctx_params,
bool no_alloc = false) :
print_timings(ctx_params.print_timings),
n_threads (ctx_params.n_threads),
media_marker (ctx_params.media_marker),
n_embd_text (text_model ? llama_model_n_embd_inp(text_model) : -1),
vocab (text_model ? llama_model_get_vocab(text_model) : nullptr)
print_timings (ctx_params.print_timings),
n_threads (ctx_params.n_threads),
media_marker (ctx_params.media_marker),
n_embd_text (text_model ? llama_model_n_embd_inp(text_model) : -1),
vocab (text_model ? llama_model_get_vocab(text_model) : nullptr),
batch_max_tokens(ctx_params.batch_max_tokens)
{
if (ctx_params.image_marker != nullptr) {
throw std::runtime_error("custom image_marker is not supported anymore, use media_marker instead");
@ -680,6 +739,16 @@ struct mtmd_context {
return ctx_a ? clip_get_projector_type(ctx_a) : PROJECTOR_TYPE_UNKNOWN;
}
int64_t n_embd_out() const {
if (ctx_v) {
return clip_n_mmproj_embd(ctx_v);
} else if (ctx_a) {
return clip_n_mmproj_embd(ctx_a);
} else {
throw std::runtime_error("no CLIP model loaded");
}
}
~mtmd_context() {
clip_free(ctx_a);
clip_free(ctx_v);
@ -845,7 +914,7 @@ struct mtmd_tokenizer {
// [QWEN_VIDEO] handle frame merging for models that support it (i.e. qwen-vl)
int n_merge_frames = 1;
if (ctx->ctx_v) {
n_merge_frames = clip_model_n_batch_max(ctx->ctx_v);
n_merge_frames = clip_model_n_temporal_merge(ctx->ctx_v);
GGML_ASSERT(n_merge_frames <= 2 && "we only support merging maximum 2 images for now; open an issue if this model supports merging more");
}
@ -860,7 +929,7 @@ struct mtmd_tokenizer {
if (i + 1 < parts.size() && parts[i + 1].bitmap != nullptr) {
const mtmd_bitmap * bm_a = parts[i].bitmap;
const mtmd_bitmap * bm_b = parts[i + 1].bitmap;
if (bm_a->can_batch_with(*bm_b)) {
if (bm_a->can_merge_with(*bm_b)) {
LOG_DBG("%s: merging 2 frames at part index %zu and %zu\n", __func__, i, i + 1);
merged_bitmaps.push_back({bm_a, bm_b});
parts.erase(parts.begin() + i + 1); // collapse the second bitmap part
@ -1103,13 +1172,17 @@ struct mtmd_tokenizer {
size_t n_tokens = 0;
for (const auto & e : batch_f32.entries) {
n_tokens += clip_n_output_tokens(ctx->ctx_v, e.get());
if (clip_model_n_batch_max(ctx->ctx_v) == 2) {
if (clip_model_n_temporal_merge(ctx->ctx_v) == 2) {
// [QWEN_VIDEO] pair input is merged to the same embd, so only count as one image
break;
}
}
mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens);
// [QWEN_VIDEO] improve this in the future
image_tokens->n_temporal_merge = clip_model_n_temporal_merge(ctx->ctx_v);
if (mtmd_decode_use_mrope(ctx)) {
// for Qwen2VL, we need this information for M-RoPE decoding positions
image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, batch_f32.entries[0].get());
@ -1327,60 +1400,18 @@ int32_t mtmd_tokenize(mtmd_context * ctx,
}
}
int32_t mtmd_encode_chunk(mtmd_context * ctx, const mtmd_input_chunk * chunk) {
if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
LOG_WRN("mtmd_encode_chunk has no effect for text chunks\n");
return 0;
} else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {
if (!ctx->ctx_v) {
LOG_ERR("%s: model does not support vision input\n", __func__);
return 1;
}
if (chunk->tokens_image == nullptr) {
LOG_ERR("%s: image tokens are null\n", __func__);
return 1;
}
if (chunk->tokens_image->is_placeholder()) {
LOG_ERR("%s: image tokens batch is placeholder\n", __func__);
return 1;
}
return mtmd_encode(ctx, chunk->tokens_image.get());
} else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
if (!ctx->ctx_a) {
LOG_ERR("%s: model does not support audio input\n", __func__);
return 1;
}
if (chunk->tokens_audio == nullptr) {
LOG_ERR("%s: audio tokens are null\n", __func__);
return 1;
}
if (chunk->tokens_audio->is_placeholder()) {
LOG_ERR("%s: audio tokens batch is placeholder\n", __func__);
return 1;
}
int n_mmproj_embd = ctx->n_embd_text;
ctx->image_embd_v.resize(chunk->tokens_audio->n_tokens * n_mmproj_embd);
bool ok = clip_image_batch_encode(
ctx->ctx_a,
ctx->n_threads,
&chunk->tokens_audio->batch_f32,
ctx->image_embd_v.data());
return ok ? 0 : 1;
}
LOG_ERR("%s: unknown chunk type %d\n", __func__, (int)chunk->type);
return 1;
}
int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) {
static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * image_tokens, std::vector<float> & out_embd) {
clip_ctx * ctx_clip = ctx->ctx_v;
if (!ctx_clip) {
LOG_ERR("%s: this API does not support non-vision input, please use mtmd_encode_chunk instead\n", __func__);
return 1;
}
auto proj_type = clip_get_projector_type(ctx_clip);
int n_mmproj_embd = clip_n_mmproj_embd(ctx_clip);
ctx->image_embd_v.resize(image_tokens->n_tokens() * n_mmproj_embd);
int n_embd_out = ctx->n_embd_out();
auto n_tokens_out = image_tokens->n_tokens();
out_embd.resize((size_t)n_embd_out * n_tokens_out);
bool ok = false;
if (clip_is_llava(ctx_clip)
@ -1400,12 +1431,19 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens)
return 1;
}
int n_tokens_per_image = clip_n_output_tokens(ctx_clip, entries[i].get());
ok = clip_image_encode(
std::vector<float> tmp_embd((size_t)n_tokens_per_image * n_embd_out);
bool ok_i = clip_image_encode(
ctx_clip,
ctx->n_threads,
entries[i].get(),
ctx->image_embd_v.data() + offset);
offset += static_cast<size_t>(n_mmproj_embd) * n_tokens_per_image;
tmp_embd);
if (!ok_i) {
LOG_ERR("%s: failed to encode image %zu\n", __func__, i);
return 1;
}
ok = true;
std::copy(tmp_embd.begin(), tmp_embd.end(), out_embd.begin() + offset);
offset += static_cast<size_t>(n_embd_out) * n_tokens_per_image;
}
} else {
if (image_tokens->is_placeholder()) {
@ -1416,14 +1454,206 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens)
ctx_clip,
ctx->n_threads,
&image_tokens->batch_f32,
ctx->image_embd_v.data());
out_embd);
}
return ok ? 0 : 1;
}
static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector<float> & out_embd) {
if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
LOG_WRN("mtmd_encode_chunk has no effect for text chunks\n");
return 0;
} else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {
if (!ctx->ctx_v) {
LOG_ERR("%s: model does not support vision input\n", __func__);
return 1;
}
if (chunk->tokens_image == nullptr) {
LOG_ERR("%s: image tokens are null\n", __func__);
return 1;
}
if (chunk->tokens_image->is_placeholder()) {
LOG_ERR("%s: image tokens batch is placeholder\n", __func__);
return 1;
}
return mtmd_encode_impl(ctx, chunk->tokens_image.get(), out_embd);
} else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
if (!ctx->ctx_a) {
LOG_ERR("%s: model does not support audio input\n", __func__);
return 1;
}
if (chunk->tokens_audio == nullptr) {
LOG_ERR("%s: audio tokens are null\n", __func__);
return 1;
}
if (chunk->tokens_audio->is_placeholder()) {
LOG_ERR("%s: audio tokens batch is placeholder\n", __func__);
return 1;
}
int n_mmproj_embd = ctx->n_embd_out();
out_embd.resize((size_t)chunk->tokens_audio->n_tokens * n_mmproj_embd);
bool ok = clip_image_batch_encode(
ctx->ctx_a,
ctx->n_threads,
&chunk->tokens_audio->batch_f32,
out_embd);
return ok ? 0 : 1;
}
LOG_ERR("%s: unknown chunk type %d\n", __func__, (int)chunk->type);
return 1;
}
int32_t mtmd_encode_chunk(mtmd_context * ctx, const mtmd_input_chunk * chunk) {
// this is the non-batching version
try {
return mtmd_encode_chunk_impl(ctx, chunk, ctx->out_embd);
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
return 1;
}
}
int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) {
try {
return mtmd_encode_impl(ctx, image_tokens, ctx->out_embd);
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
return 1;
}
}
float * mtmd_get_output_embd(mtmd_context * ctx) {
return ctx->image_embd_v.data();
return ctx->out_embd.data();
}
mtmd_batch * mtmd_batch_init(mtmd_context * ctx) {
return new mtmd_batch(ctx);
}
void mtmd_batch_free(mtmd_batch * batch) {
if (batch) {
delete batch;
}
}
int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk) {
if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
LOG_ERR("%s: text chunk is not supported in batch\n", __func__);
return 1;
}
auto * ctx = batch->ctx->get_clip_ctx(chunk);
if (!ctx) {
LOG_ERR("%s: model does not support input chunk type %d\n", __func__, (int)chunk->type);
return 1;
}
if (batch->entries.empty()) {
// batch must have at least one chunk
batch->entries.push_back(chunk);
return 0;
}
if (!clip_support_batch(ctx)) {
// if no batching support, batch can only have one single chunk
return 2; // "batch too large" error code
}
int32_t new_n_tokens = batch->n_tokens() + (int32_t)mtmd_input_chunk_get_n_tokens(chunk);
if (new_n_tokens > batch->ctx->batch_max_tokens) {
return 2; // "batch too large" error code
}
auto & first_chunk = batch->entries[0];
if (first_chunk->can_batch_with(*chunk)) {
batch->entries.push_back(chunk);
return 0;
}
return 3; // "cannot batch" error code
}
static int32_t mtmd_batch_encode_impl(mtmd_batch * batch) {
if (batch->entries.empty()) {
LOG_ERR("%s: batch is empty\n", __func__);
return 1;
}
for (const auto * chunk : batch->entries) {
if (chunk->is_placeholder()) {
LOG_ERR("%s: chunk is placeholder\n", __func__);
return 1;
}
}
// represent the whole batch as one single chunk
mtmd::input_chunk_ptr batch_chunk(mtmd_input_chunk_copy(batch->entries[0]));
if (batch_chunk->tokens_image) {
auto & b0_f32 = batch_chunk->tokens_image->batch_f32;
// copy all entries from other chunks into the first chunk's batch_f32
// note: skip first entry because it's already in batch_chunk
for (size_t ic = 1; ic < batch->entries.size(); ic++) {
auto & chunk = batch->entries[ic];
GGML_ASSERT(chunk->tokens_image);
auto b1_f32 = chunk->tokens_image->batch_f32.clone();
for (size_t i = 0; i < b1_f32.entries.size(); i++) {
b0_f32.entries.push_back(std::move(b1_f32.entries[i]));
}
}
} else if (batch_chunk->tokens_audio) {
auto & b0_f32 = batch_chunk->tokens_audio->batch_f32;
// copy all entries from other chunks into the first chunk's batch_f32
// note: skip first entry because it's already in batch_chunk
for (size_t ic = 1; ic < batch->entries.size(); ic++) {
auto & chunk = batch->entries[ic];
GGML_ASSERT(chunk->tokens_audio);
auto b1_f32 = chunk->tokens_audio->batch_f32.clone();
for (size_t i = 0; i < b1_f32.entries.size(); i++) {
b0_f32.entries.push_back(std::move(b1_f32.entries[i]));
}
}
} else {
LOG_ERR("%s: unsupported chunk type\n", __func__);
return 1;
}
LOG_DBG("%s: encoding batch with %zu entries and total %zu tokens\n",
__func__, batch->entries.size(), mtmd_input_chunk_get_n_tokens(batch_chunk.get()));
int32_t res = mtmd_encode_chunk_impl(
batch->ctx,
batch_chunk.get(),
batch->output_embd);
return res;
}
int32_t mtmd_batch_encode(mtmd_batch * batch) {
try {
return mtmd_batch_encode_impl(batch);
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
return 1;
}
}
float * mtmd_batch_get_output_embd(mtmd_batch * batch, const mtmd_input_chunk * chunk) {
if (batch->output_embd.empty()) {
LOG_ERR("%s: batch has not been encoded yet\n", __func__);
return nullptr;
}
size_t offset = 0;
const size_t n_embd = batch->ctx->n_embd_out();
for (const auto * c : batch->entries) {
size_t offset_prev = offset;
size_t n_tokens = mtmd_input_chunk_get_n_tokens(c);
offset += n_tokens * n_embd;
GGML_ASSERT(offset_prev < batch->output_embd.size());
GGML_ASSERT(offset <= batch->output_embd.size());
if (c == chunk) {
return &batch->output_embd.data()[offset_prev];
}
}
return nullptr; // not found
}
bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk) {
@ -1813,7 +2043,7 @@ static void mtmd_debug_encode_impl(mtmd_context * ctx, clip_ctx * ctx_clip, clip
ctx_clip,
ctx->n_threads,
&image,
embd_output.data());
embd_output);
if (!ok) {
LOG_ERR("%s: failed to encode image\n", __func__);
}

40
tools/mtmd/mtmd.h
View file

@ -63,6 +63,7 @@ struct mtmd_bitmap;
struct mtmd_image_tokens;
struct mtmd_input_chunk;
struct mtmd_input_chunks;
struct mtmd_batch;
struct mtmd_input_text {
const char * text;
@ -80,6 +81,7 @@ typedef struct mtmd_image_tokens mtmd_image_tokens;
typedef struct mtmd_input_chunk mtmd_input_chunk;
typedef struct mtmd_input_chunks mtmd_input_chunks;
typedef struct mtmd_input_text mtmd_input_text;
typedef struct mtmd_batch mtmd_batch;
struct mtmd_context_params {
bool use_gpu;
@ -97,6 +99,11 @@ struct mtmd_context_params {
// callback function passed over to mtmd proper
ggml_backend_sched_eval_callback cb_eval;
void * cb_eval_user_data;
// batching params
int32_t batch_max_tokens; // maximum number of output tokens in a batch
// (note: this is not a hard-limit, the first image will always be added even if it exceeds this limit)
// (default: 1024)
};
MTMD_API const char * mtmd_default_marker(void);
@ -265,12 +272,12 @@ MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx,
const mtmd_bitmap ** bitmaps,
size_t n_bitmaps);
// returns 0 on success
// TODO: deprecate
MTMD_API int32_t mtmd_encode(mtmd_context * ctx,
const mtmd_image_tokens * image_tokens);
DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens),
"use mtmd_encode_chunk() instead");
// text chunk will be ignored silently, only media chunk will be encoded
// returns 0 on success
// returns 1 on generic error
MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx,
const mtmd_input_chunk * chunk);
@ -279,6 +286,26 @@ MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx,
// llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float)
MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx);
// batch encoding API
// chunks are not owned by the batch, they will not be freed by mtmd_batch_free()
// batch is valid for a given context, cannot be shared across contexts
MTMD_API mtmd_batch * mtmd_batch_init(mtmd_context * ctx);
MTMD_API void mtmd_batch_free(mtmd_batch * batch);
// only media chunks are allowed, text chunks will be rejected
// returns 0 on success
// returns 1 on generic error
// returns 2 if the batch is too large (chunk won't be added)
// returns 3 if it cannot be batched with the existing chunks in the batch
MTMD_API int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk);
// returns 0 on success
// returns 1 on generic error
MTMD_API int32_t mtmd_batch_encode(mtmd_batch * batch);
MTMD_API float * mtmd_batch_get_output_embd(mtmd_batch * batch, const mtmd_input_chunk * chunk);
// Set callback for all future logging events.
// If this is not called, or NULL is supplied, everything is output on stderr.
MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);
@ -339,6 +366,11 @@ struct mtmd_input_chunk_deleter {
};
using input_chunk_ptr = std::unique_ptr<mtmd_input_chunk, mtmd_input_chunk_deleter>;
struct mtmd_batch_deleter {
void operator()(mtmd_batch * val) { mtmd_batch_free(val); }
};
using batch_ptr = std::unique_ptr<mtmd_batch, mtmd_batch_deleter>;
struct bitmap {
bitmap_ptr ptr;
bitmap() : ptr(nullptr) {}

14
tools/server/server-common.cpp
View file

@ -344,6 +344,14 @@ const mtmd::input_chunk_ptr & server_tokens::find_chunk(size_t idx) const {
throw std::runtime_error("Chunk not found");
}
std::pair<const mtmd::input_chunk_ptr *, size_t> server_tokens::find_next_media_chunk(size_t idx) const {
auto it = map_idx_to_media.upper_bound(idx);
if (it != map_idx_to_media.end()) {
return { &it->second, it->first };
}
return { nullptr, 0 };
}
void server_tokens::push_back(llama_token tok) {
if (tok == LLAMA_TOKEN_NULL) {
throw std::runtime_error("Invalid token");
@ -1126,9 +1134,9 @@ json oaicompat_chat_params_parse(
// Reasoning budget: pass parameters through to sampling layer
{
int reasoning_budget = opt.reasoning_budget;
if (reasoning_budget == -1 && body.contains("thinking_budget_tokens")) {
reasoning_budget = json_value(body, "thinking_budget_tokens", -1);
int reasoning_budget = json_value(body, "thinking_budget_tokens", -1);
if (reasoning_budget == -1) {
reasoning_budget = opt.reasoning_budget;
}
if (!chat_params.thinking_end_tag.empty()) {

4
tools/server/server-common.h
View file

@ -180,6 +180,10 @@ public:
const mtmd::input_chunk_ptr & find_chunk(size_t idx) const;
// find next media chunk after idx
// returns a pair of pointer to the chunk (nullptr if not found) and its start index in tokens
std::pair<const mtmd::input_chunk_ptr *, size_t> find_next_media_chunk(size_t idx) const;
void push_back(llama_token tok);
// will create a copy of the chunk if it contains non-text data

128
tools/server/server-context.cpp
View file

@ -80,6 +80,8 @@ struct server_slot {
// multimodal
mtmd_context * mctx = nullptr;
mtmd::batch_ptr mbatch = nullptr;
std::array<llama_context *, 2> mtgt = {nullptr, nullptr}; // [0] for main context, [1] for optional draft context
// speculative decoding
common_speculative * spec;
@ -239,6 +241,18 @@ struct server_slot {
// clear alora start
alora_invocation_start = -1;
// clear multimodal state
mbatch.reset();
mtgt[0] = ctx_tgt;
mtgt[1] = nullptr;
if (ctx_dft && llama_get_ctx_other(ctx_dft) != ctx_tgt) {
// TODO: in the future, figure out how to infuse target embeddings to the images
// for now, we re-decode the same chunk in both ctx_tgt and ctx_dft
// maybe we simply need to call `common_speculative_process()` ?
// [TAG_MTMD_DRAFT_PROCESSING]
mtgt[1] = ctx_dft;
}
}
void init_sampler() const {
@ -578,6 +592,87 @@ struct server_slot {
other.prompt = prompt.clone();
other.init_sampler();
}
// returns 0 on success
// caller need to update prompt.tokens after a successful call to keep track of the processing progress
int process_mtmd_chunk(size_t idx, size_t & n_tokens_out) {
GGML_ASSERT(mctx);
const auto & input_tokens = task->tokens;
auto & chunk = input_tokens.find_chunk(idx);
int32_t res = 0;
auto try_decode = [&]() -> int32_t {
if (mbatch) {
float * embd = mtmd_batch_get_output_embd(mbatch.get(), chunk.get());
if (embd) {
for (auto * lctx : mtgt) {
if (lctx == nullptr) {
continue;
}
llama_pos new_n_past; // unused for now
res = mtmd_helper_decode_image_chunk(
mctx,
lctx,
chunk.get(),
embd,
prompt.tokens.pos_next(),
id,
llama_n_batch(lctx),
&new_n_past
);
if (res != 0) {
SLT_ERR(*this, "failed to decode mtmd chunk, idx = %zu, res = %d\n", idx, res);
return -1;
}
}
n_tokens_out = mtmd_input_chunk_get_n_tokens(chunk.get());
return 0; // success
}
}
return 1; // (non-error) need to create & encode batch
};
// if the batch is already exist, try searching & encode
res = try_decode();
if (res == 0) {
return 0;
} else if (res < 0) {
// fatal error
return res;
}
// otherwise, the batch is either uninitialized or is used up
// we need to create & encode a new batch
mbatch.reset(mtmd_batch_init(mctx));
res = mtmd_batch_add_chunk(mbatch.get(), chunk.get());
GGML_ASSERT(res == 0); // we should never have an empty batch
// try batching as much as possible
int n_added = 1;
size_t idx_cur = idx;
while (res == 0) {
auto [next_chunk, next_idx] = input_tokens.find_next_media_chunk(idx_cur);
if (next_chunk == nullptr) {
break;
}
res = mtmd_batch_add_chunk(mbatch.get(), next_chunk->get());
n_added += (res == 0 ? 1 : 0);
idx_cur = next_idx;
SLT_DBG(*this, "try adding media chunk idx = %zu to batch, res = %d\n", next_idx, res);
// if res != 0, batch is full or chunk is not compatible -> this loop breaks
}
// TODO @ngxson : move this log line to debug when it become more stable
SLT_INF(*this, "encoding mtmd batch from idx = %zu, n_chunks = %d\n", idx, n_added);
res = mtmd_batch_encode(mbatch.get());
if (res != 0) {
SLT_ERR(*this, "failed to encode mtmd batch for chunk idx = %zu, res = %d\n", idx, res);
return -1;
}
return try_decode();
}
};
@ -781,6 +876,7 @@ private:
mparams.warmup = params_base.warmup;
mparams.image_min_tokens = params_base.image_min_tokens;
mparams.image_max_tokens = params_base.image_max_tokens;
mparams.batch_max_tokens = params_base.mtmd_batch_max_tokens;
mparams.media_marker = get_media_marker();
}
@ -866,10 +962,7 @@ private:
}
for (size_t j = 0; j < devs.size(); ++j) {
const size_t bytes =
(measure_model_bytes ? dmd[j].mb.model : 0) +
dmd[j].mb.context +
dmd[j].mb.compute;
const size_t bytes = (measure_model_bytes ? dmd[j].model : 0) + dmd[j].context + dmd[j].compute;
total += bytes;
for (size_t i = 0; i < tgt_devices.size(); i++) {
if (tgt_devices[i] == devs[j]) {
@ -2928,7 +3021,7 @@ private:
send_partial_response(slot, {}, false, true);
}
}
}
} // end of SLOT_STATE_STARTED
if (!slot.can_split()) {
// cannot fit the prompt in the current batch - will try next iter
@ -2983,10 +3076,18 @@ private:
bool has_mtmd = false;
// check if we should process the image
while (slot.prompt.n_tokens() < slot.task->n_tokens() && input_tokens[slot.prompt.n_tokens()] == LLAMA_TOKEN_NULL) {
while (true) {
auto cur_token_idx = slot.prompt.n_tokens();
if (
cur_token_idx >= slot.task->n_tokens() ||
input_tokens[cur_token_idx] != LLAMA_TOKEN_NULL // encountered a text token
) {
break;
}
// process the image
size_t n_tokens_out = 0;
int32_t res = input_tokens.process_chunk(ctx_tgt, mctx, slot.prompt.n_tokens(), slot.prompt.tokens.pos_next(), slot.id, n_tokens_out);
int32_t res = slot.process_mtmd_chunk(cur_token_idx, n_tokens_out);
if (res != 0) {
SLT_ERR(slot, "failed to process image, res = %d\n", res);
send_error(slot, "failed to process image", ERROR_TYPE_SERVER);
@ -2994,22 +3095,11 @@ private:
continue;
}
if (ctx_dft && llama_get_ctx_other(ctx_dft.get()) != ctx_tgt) {
// TODO: in the future, figure out how to infuse target embeddings to the images
// for now, we skip this for simplicity
// maybe we simply need to call `common_speculative_process()` on the mtmd batches in the `process_chunk` above?
// [TAG_MTMD_DRAFT_PROCESSING]
res = input_tokens.process_chunk(ctx_dft.get(), mctx, slot.prompt.n_tokens(), slot.prompt.tokens.pos_next(), slot.id, n_tokens_out);
if (res != 0) {
GGML_ABORT("failed to process multi-modal data on draft context\n");
}
}
slot.n_prompt_tokens_processed += n_tokens_out;
// add the image chunk to cache
{
const auto & chunk = input_tokens.find_chunk(slot.prompt.n_tokens());
const auto & chunk = input_tokens.find_chunk(cur_token_idx);
slot.prompt.tokens.push_back(chunk.get()); // copy
}

99
tools/server/server-http.cpp
View file

@ -113,7 +113,7 @@ bool server_http_context::init(const common_params & params) {
#endif
srv->set_default_headers({{"Server", "llama.cpp"}});
srv->set_logger(log_server_request);
// srv->set_logger(log_server_request); // TODO @ngxson : this is too spamy, no very useful; improve it in the future
srv->set_exception_handler([](const httplib::Request &, httplib::Response & res, const std::exception_ptr & ep) {
// this is fail-safe; exceptions should already handled by `ex_wrapper`
@ -173,25 +173,29 @@ bool server_http_context::init(const common_params & params) {
// Middlewares
//
auto middleware_validate_api_key = [api_keys = params.api_keys](const httplib::Request & req, httplib::Response & res) {
static const std::unordered_set<std::string> public_endpoints = {
// Public endpoints - API routes plus all embedded UI assets
static const std::unordered_set<std::string> get_public_endpoints = []() {
std::unordered_set<std::string> endpoints {
"/health",
"/v1/health",
"/models",
"/v1/models",
"/",
"/index.html",
"/bundle.js",
"/bundle.css",
};
for (const llama_ui_asset & a : llama_ui_get_assets()) {
endpoints.insert("/" + a.name);
}
return endpoints;
}();
auto middleware_validate_api_key = [api_keys = params.api_keys](const httplib::Request & req, httplib::Response & res) {
// If API key is not set, skip validation
if (api_keys.empty()) {
return true;
}
// If path is public or static file, skip validation
if (public_endpoints.find(req.path) != public_endpoints.end()) {
// If path is public or a UI asset, skip validation
if (get_public_endpoints.count(req.path)) {
return true;
}
@ -315,33 +319,84 @@ bool server_http_context::init(const common_params & params) {
}
} else {
#if defined(LLAMA_UI_HAS_ASSETS)
auto serve_asset = [](const std::string & name, const char * mime, bool with_isolation_headers) {
return [name, mime, with_isolation_headers](const httplib::Request & req, httplib::Response & res) {
const llama_ui_asset * a = llama_ui_find_asset(name.c_str());
if (!a) {
res.status = 404;
return false;
static auto handle_gzip_header = [](const httplib::Request & req, httplib::Response & res) {
if (!llama_ui_use_gzip()) {
// no gzip build, skip
return true;
}
if (req.get_header_value("Accept-Encoding").find("gzip") == std::string::npos) {
res.status = 415; // unsupported media type
res.set_content("Error: gzip is not supported by this browser", "text/plain");
return false;
} else {
res.set_header("Content-Encoding", "gzip");
}
return true;
};
auto serve_asset_cached = [](const std::string & name, bool isolation) {
return [name, isolation](const httplib::Request & req, httplib::Response & res) {
if (!handle_gzip_header(req, res)) {
return true; // returns error message
}
const llama_ui_asset * a = llama_ui_find_asset(name);
if (!a) { res.status = 404; return false; }
res.set_header("ETag", a->etag);
// Check If-None-Match for conditional GET (304 Not Modified)
if (const std::string & inm = req.get_header_value("If-None-Match");
!inm.empty() && (inm == a->etag || inm == std::string("W/") + a->etag)) {
res.status = 304;
return false;
}
if (with_isolation_headers) {
// COEP and COOP headers, required by pyodide (python interpreter)
if (isolation) {
res.set_header("Cross-Origin-Embedder-Policy", "require-corp");
res.set_header("Cross-Origin-Opener-Policy", "same-origin");
res.set_header("Cross-Origin-Opener-Policy", "same-origin");
}
res.set_content(reinterpret_cast<const char*>(a->data), a->size, mime);
res.set_header("Cache-Control", "public, max-age=31536000, immutable");
res.set_content(reinterpret_cast<const char*>(a->data), a->size, a->type.c_str());
return false;
};
};
srv->Get(params.api_prefix + "/", serve_asset("index.html", "text/html; charset=utf-8", true));
srv->Get(params.api_prefix + "/bundle.js", serve_asset("bundle.js", "application/javascript; charset=utf-8", false));
srv->Get(params.api_prefix + "/bundle.css", serve_asset("bundle.css", "text/css; charset=utf-8", false));
auto serve_asset_nocache = [](const std::string & name) {
return [name](const httplib::Request & req, httplib::Response & res) {
if (!handle_gzip_header(req, res)) {
return true; // returns error message
}
const llama_ui_asset * a = llama_ui_find_asset(name);
if (!a) {
res.status = 404;
return false;
}
res.set_header("Cache-Control", "no-cache");
res.set_content(reinterpret_cast<const char*>(a->data), a->size, a->type.c_str());
return false;
};
};
// main index file
srv->Get(params.api_prefix + "/", serve_asset_cached("index.html", true));
srv->Get(params.api_prefix + "/index.html", serve_asset_cached("index.html", true));
// All remaining assets registered directly from the embedded asset table.
// PWA revalidation files (sw.js, manifest, version.json) use no-cache;
// everything else is immutable.
static const std::unordered_set<std::string> no_cache_names = {
"sw.js",
"manifest.webmanifest",
"_app/version.json",
"build.json"
};
for (const auto & a : llama_ui_get_assets()) {
if (a.name == "index.html") continue; // served at "/" and "/index.html" above
if (no_cache_names.count(a.name)) {
SRV_DBG("serve nocache for %s\n", a.name.c_str());
srv->Get(params.api_prefix + "/" + a.name, serve_asset_nocache(a.name));
} else {
srv->Get(params.api_prefix + "/" + a.name, serve_asset_cached(a.name, false));
}
}
#endif
}
}

2
tools/server/tests/unit/test_security.py
View file

@ -26,7 +26,7 @@ def test_access_static_assets_without_api_key():
"""Static web UI assets should not require API key authentication (issue #21229)"""
global server
server.start()
for path in ["/", "/bundle.js", "/bundle.css"]:
for path in ["/", "/sw.js", "/manifest.webmanifest", "/_app/version.json"]:
res = server.make_request("GET", path)
assert res.status_code == 200, f"Expected 200 for {path}, got {res.status_code}"

11
tools/ui/.gitignore vendored
View file

@ -8,6 +8,8 @@ node_modules
.wrangler
/.svelte-kit
/build
dev-dist
dist
# OS
.DS_Store
@ -23,6 +25,15 @@ Thumbs.db
vite.config.js.timestamp-*
vite.config.ts.timestamp-*
# PWA Artifacts
apple-splash-*.png
apple-touch-icon-*.png
favicon.ico
favicon-dark.ico
maskable-icon-*.png
pwa-*.png
# Storybook
*storybook.log
storybook-static
*.code-workspace

5
tools/ui/CMakeLists.txt
View file

@ -1,6 +1,7 @@
set(TARGET llama-ui)
set(LLAMA_UI_HF_BUCKET "llama-ui" CACHE STRING "Hugging Face bucket name for prebuilt UI assets")
set(LLAMA_UI_HF_BUCKET "ggml-org/llama-ui" CACHE STRING "Hugging Face bucket name for prebuilt UI assets")
set(LLAMA_UI_GZIP ON CACHE BOOL "Apply gzip compress to assets to save bandwidth")
# Backward compat: forward old var to new one
if(DEFINED LLAMA_BUILD_WEBUI)
@ -77,11 +78,13 @@ add_custom_target(llama-ui-assets ALL
"-DUI_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}"
"-DUI_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}"
"-DLLAMA_SOURCE_DIR=${PROJECT_SOURCE_DIR}"
"-DLLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER}"
"-DHF_BUCKET=${LLAMA_UI_HF_BUCKET}"
"-DHF_VERSION=${HF_UI_VERSION}"
"-DHF_ENABLED=${LLAMA_USE_PREBUILT_UI}"
"-DBUILD_UI=${LLAMA_BUILD_UI}"
"-DLLAMA_UI_EMBED=${LLAMA_UI_EMBED_EXE}"
"-DLLAMA_UI_GZIP=${LLAMA_UI_GZIP}"
-P "${PROJECT_SOURCE_DIR}/scripts/ui-assets.cmake"
COMMENT "Provisioning UI assets"
VERBATIM

200
tools/ui/embed.cpp
View file

@ -1,16 +1,44 @@
// llama-ui-embed: generate ui.cpp / ui.h that embed UI assets as C arrays.
//
// Usage:
// llama-ui-embed <out_cpp> <out_h> [<asset_name> <asset_path>]...
// llama-ui-embed <out_cpp> <out_h> [<asset_dir>]
//
// Recursively embeds every regular file under <asset_dir>.
// Asset names are relative paths from <asset_dir> (e.g. "_app/immutable/bundle.HASH.js").
// Without <asset_dir>, emits an empty asset table.
#include <inttypes.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <algorithm>
#include <filesystem>
#include <fstream>
#include <functional>
#include <string>
#include <vector>
#include <cinttypes>
#include <cstdint>
static const char * mime_from_ext(const std::string & name) {
auto ext = name.rfind('.');
if (ext == std::string::npos) return "application/octet-stream";
std::string e = name.substr(ext + 1);
if (e == "html") return "text/html; charset=utf-8";
if (e == "css") return "text/css";
if (e == "js") return "application/javascript";
if (e == "json") return "application/json";
if (e == "webmanifest") return "application/manifest+json";
if (e == "svg") return "image/svg+xml";
if (e == "png") return "image/png";
if (e == "jpg" ||
e == "jpeg") return "image/jpeg";
if (e == "ico") return "image/x-icon";
if (e == "woff") return "font/woff";
if (e == "woff2") return "font/woff2";
return "application/octet-stream";
}
// Computes FNV-1a hash of the data
static uint64_t fnv_hash(const uint8_t * data, size_t len) {
@ -24,10 +52,10 @@ static uint64_t fnv_hash(const uint8_t * data, size_t len) {
return hash;
}
static bool read_file(const std::string & path, std::vector<unsigned char> & out) {
static bool read_file(const std::filesystem::path & path, std::vector<unsigned char> & out) {
std::ifstream f(path, std::ios::binary | std::ios::ate);
if (!f) {
fprintf(stderr, "embed: cannot open %s\n", path.c_str());
fprintf(stderr, "embed: cannot open %s\n", path.string().c_str());
return false;
}
const auto sz = f.tellg();
@ -77,7 +105,24 @@ static bool write_if_different(const std::string & path, const std::string & con
if (!content.empty()) {
out.write(content.data(), static_cast<std::streamsize>(content.size()));
}
return out.good();
bool ok = out.good();
if (ok) {
printf("embed: write output file %s\n", path.c_str());
}
return ok;
}
static std::string path_basename(const std::string & name) {
const size_t p = name.rfind('/');
return p == std::string::npos ? name : name.substr(p + 1);
}
static bool str_starts_with(const std::string & s, const char * prefix) {
const size_t n = strlen(prefix);
return s.size() >= n && s.compare(0, n, prefix) == 0;
}
static bool str_ends_with(const std::string & s, const char * suffix) {
const size_t n = strlen(suffix);
return s.size() >= n && s.compare(s.size() - n, n, suffix) == 0;
}
static std::string fmt(const char * pattern, ...) {
@ -89,72 +134,171 @@ static std::string fmt(const char * pattern, ...) {
return (n > 0) ? std::string(tmp, static_cast<size_t>(n)) : std::string();
}
struct asset_entry {
std::string name;
std::filesystem::path path;
};
int main(int argc, char ** argv) {
if (argc < 3 || ((argc - 3) % 2) != 0) {
fprintf(stderr, "usage: %s <out_cpp> <out_h> [<name> <path>]...\n", argv[0]);
if (argc < 3 || argc > 4) {
fprintf(stderr, "usage: %s <out_cpp> <out_h> [<asset_dir>]\n", argv[0]);
return 1;
}
const std::string out_cpp = argv[1];
const std::string out_h = argv[2];
const int n_assets = (argc - 3) / 2;
const std::string out_cpp = argv[1];
const std::string out_h = argv[2];
const std::string asset_dir = (argc >= 4) ? argv[3] : std::string();
const bool use_gzip = !asset_dir.empty() && std::filesystem::exists(asset_dir + "/_gzip");
const std::string in_dir = use_gzip ? (asset_dir + "/_gzip") : asset_dir;
std::vector<asset_entry> assets;
if (!in_dir.empty()) {
const std::filesystem::path dir = in_dir;
std::error_code ec;
std::filesystem::recursive_directory_iterator it(dir, ec);
if (ec) {
fprintf(stderr, "embed: cannot iterate %s: %s\n", argv[3], ec.message().c_str());
return 1;
}
for (const auto & entry : it) {
if (!entry.is_regular_file()) {
continue;
}
// name is the relative path from dir, with forward slashes
const std::string name = entry.path().lexically_relative(dir).generic_string();
assets.push_back({ name, entry.path() });
}
// directory iteration order is unspecified; sort for reproducible output
std::sort(assets.begin(), assets.end(),
[](const asset_entry & a, const asset_entry & b) { return a.name < b.name; });
}
const int n_assets = static_cast<int>(assets.size());
if (n_assets > 0) {
using match_fn = std::function<bool(const std::string &)>;
auto exact = [](const char * name) -> match_fn {
return [name](const std::string & base) { return base == name; };
};
struct required_check { const char * label; match_fn match; bool found; };
required_check checks[] = {
{ "index.html", exact("index.html"), false },
{ "loading.html", exact("loading.html"), false },
{ "manifest.webmanifest", exact("manifest.webmanifest"), false },
{ "sw.js", exact("sw.js"), false },
{ "build.json", exact("build.json"), false },
{ "version.json", exact("version.json"), false },
{ "bundle[hash].js", [](const std::string & b) {
return str_starts_with(b, "bundle") && str_ends_with(b, ".js");
}, false },
{ "bundle[hash].css", [](const std::string & b) {
return str_starts_with(b, "bundle") && str_ends_with(b, ".css");
}, false },
{ "workbox[hash].js", [](const std::string & b) {
return str_starts_with(b, "workbox") && str_ends_with(b, ".js");
}, false },
};
for (const auto & a : assets) {
const std::string base = path_basename(a.name);
for (auto & c : checks) {
if (!c.found) { c.found = c.match(base); }
}
}
std::vector<const char *> missing;
for (const auto & c : checks) {
if (!c.found) { missing.push_back(c.label); }
}
if (!missing.empty()) {
fprintf(stderr, "\ncurrent asset files:\n");
for (const auto & a : assets) {
fprintf(stderr, " %s\n", a.name.c_str());
}
fprintf(stderr, "missing required asset(s):\n");
for (const char * m : missing) {
fprintf(stderr, " %s\n", m);
}
fprintf(stderr, "hint: try cleaning your build directory: %s\n", in_dir.c_str());
return 1;
}
}
std::string h;
h += "#pragma once\n\n#include <stddef.h>\n\n";
h += "#pragma once\n\n#include <array>\n#include <string>\n\n";
if (n_assets > 0) {
h += "#define LLAMA_UI_HAS_ASSETS 1\n\n";
}
h +=
"struct llama_ui_asset {\n"
" const char * name;\n"
" std::string name;\n"
" const unsigned char * data;\n"
" size_t size;\n"
" const char * etag;\n"
" std::size_t size;\n"
" std::string etag;\n"
" std::string type;\n"
"};\n\n"
"const llama_ui_asset * llama_ui_find_asset(const char * name);\n";
"const llama_ui_asset * llama_ui_find_asset(const std::string & name);\n"
"bool llama_ui_use_gzip();\n";
h += fmt("const std::array<llama_ui_asset, %d> & llama_ui_get_assets();\n", n_assets);
std::string cpp;
cpp += "#include \"ui.h\"\n\n#include <string.h>\n\n";
cpp += "#include \"ui.h\"\n\n";
if (n_assets > 0) {
for (int i = 0; i < n_assets; i++) {
const char * path = argv[3 + i * 2 + 1];
std::vector<unsigned char> bytes;
if (!read_file(path, bytes)) {
if (!read_file(assets[i].path, bytes)) {
return 1;
}
if (bytes.empty()) {
fprintf(stderr, "embed: empty file: %s\n", assets[i].path.generic_string().c_str());
return 1;
}
cpp += fmt("static const unsigned char asset_%d_data[] = {", i);
append_bytes_hex(cpp, bytes);
const auto hash = fnv_hash(bytes.data(), bytes.size());
cpp += fmt("};\nstatic const size_t asset_%d_size = %zu;\n",
cpp += fmt("};\nstatic const std::size_t asset_%d_size = %zu;\n",
i, bytes.size());
cpp += fmt("static const char asset_%d_etag[] = \"\\\"0x%016" PRIx64 "\\\"\";\n\n",
cpp += fmt("static const char asset_%d_etag[] = \"\\\"0x%016" PRIx64 "\\\"\";\n\n",
i, hash);
}
cpp += "static const llama_ui_asset g_assets[] = {\n";
cpp += fmt("static const std::array<llama_ui_asset, %d> g_assets = {{\n", n_assets);
for (int i = 0; i < n_assets; i++) {
cpp += fmt(" { \"%s\", asset_%d_data, asset_%d_size, asset_%d_etag },\n",
argv[3 + i * 2], i, i, i);
const std::string & name = assets[i].name;
cpp += fmt(" { \"%s\", asset_%d_data, asset_%d_size, asset_%d_etag, \"%s\" },\n",
name.c_str(), i, i, i, mime_from_ext(name));
}
cpp += "};\n\n";
cpp += "}};\n\n";
cpp +=
"const llama_ui_asset * llama_ui_find_asset(const char * name) {\n"
"const llama_ui_asset * llama_ui_find_asset(const std::string & name) {\n"
" for (const auto & a : g_assets) {\n"
" if (strcmp(a.name, name) == 0) {\n"
" if (a.name == name) {\n"
" return &a;\n"
" }\n"
" }\n"
" return nullptr;\n"
"}\n";
cpp += fmt("const std::array<llama_ui_asset, %d> & llama_ui_get_assets() {\n", n_assets);
cpp += " return g_assets;\n"
"}\n";
} else {
cpp +=
"const llama_ui_asset * llama_ui_find_asset(const char *) {\n"
"const llama_ui_asset * llama_ui_find_asset(const std::string &) {\n"
" return nullptr;\n"
"}\n"
"const std::array<llama_ui_asset, 0> & llama_ui_get_assets() {\n"
" static const std::array<llama_ui_asset, 0> empty{};\n"
" return empty;\n"
"}\n";
}
cpp += fmt("bool llama_ui_use_gzip() { return %s; }\n", use_gzip ? "true" : "false");
bool ok = true;
ok = write_if_different(out_h, h) && ok;

8113
tools/ui/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

56
tools/ui/package.json
View file

@ -4,8 +4,9 @@
"version": "1.0.0",
"type": "module",
"scripts": {
"build": "npm run build-pwa-assets && vite build",
"build-pwa-assets": "npx @vite-pwa/assets-generator --root . --config pwa-assets.config.ts && npx @vite-pwa/assets-generator --root . --config pwa-assets-dark.config.ts && node scripts/make-icons-circular.js",
"dev": "bash scripts/dev.sh",
"build": "vite build",
"preview": "vite preview",
"prepare": "svelte-kit sync || echo ''",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
@ -15,12 +16,15 @@
"lint": "prettier --check . && eslint .",
"test": "npm run test:ui -- --run && npm run test:client -- --run && npm run test:unit -- --run && npm run test:e2e",
"test:e2e": "playwright test",
"test:e2e:pwa": "playwright test tests/e2e/pwa.e2e.ts",
"test:client": "vitest --project=client",
"test:unit": "vitest --project=unit",
"test:unit:pwa": "vitest --project=unit --run tests/unit/pwa.spec.ts",
"test:pwa": "npm run test:unit:pwa && npm run test:e2e:pwa",
"test:ui": "vitest --project=ui",
"storybook": "storybook dev -p 6006",
"build-storybook": "storybook build",
"cleanup": "rm -rf .svelte-kit build node_modules test-results"
"cleanup": "rm -rf .svelte-kit build node_modules test-results dist dev-dist debug-storybook.log static/pwa-*.png static/maskable-icon-*.png static/apple-touch-icon-*.png static/apple-splash-*.png static/favicon*.ico"
},
"devDependencies": {
"@chromatic-com/storybook": "5.0.0",
@ -41,29 +45,31 @@
"@tailwindcss/forms": "0.5.10",
"@tailwindcss/typography": "0.5.16",
"@tailwindcss/vite": "4.1.11",
"@types/node": "^24",
"@types/node": "24.13.0",
"@vite-pwa/assets-generator": "1.0.2",
"@vite-pwa/sveltekit": "1.1.0",
"@vitest/browser": "4.1.8",
"@vitest/browser-playwright": "4.1.8",
"@vitest/coverage-v8": "4.1.8",
"bits-ui": "2.18.1",
"clsx": "2.1.1",
"dexie": "4.0.11",
"eslint": "9.39.2",
"dexie": "4.4.3",
"eslint": "9.39.4",
"eslint-config-prettier": "10.1.8",
"eslint-plugin-storybook": "10.2.4",
"eslint-plugin-svelte": "3.15.0",
"globals": "16.3.0",
"eslint-plugin-storybook": "10.4.2",
"eslint-plugin-svelte": "3.19.0",
"globals": "16.5.0",
"highlight.js": "11.11.1",
"http-server": "14.1.1",
"mdast": "3.0.0",
"mdsvex": "0.12.6",
"mdsvex": "0.12.7",
"mermaid": "11.15.0",
"mode-watcher": "1.1.0",
"pdfjs-dist": "5.4.54",
"playwright": "1.56.1",
"prettier": "3.6.2",
"prettier-plugin-svelte": "3.4.0",
"prettier-plugin-tailwindcss": "0.6.14",
"prettier": "3.8.3",
"prettier-plugin-svelte": "4.1.0",
"prettier-plugin-tailwindcss": "0.8.0",
"rehype-highlight": "7.0.2",
"rehype-katex": "7.0.1",
"rehype-stringify": "10.0.1",
@ -73,25 +79,25 @@
"remark-html": "16.0.1",
"remark-math": "6.0.0",
"remark-rehype": "11.1.2",
"sass": "1.93.3",
"storybook": "10.3.3",
"svelte": "5.55.7",
"svelte-check": "4.3.0",
"svelte-sonner": "1.0.5",
"tailwind-merge": "3.3.1",
"sass": "1.100.0",
"storybook": "10.4.2",
"svelte": "5.56.1",
"svelte-check": "4.6.0",
"svelte-sonner": "1.1.1",
"tailwind-merge": "3.6.0",
"tailwind-variants": "3.2.2",
"tailwindcss": "4.1.11",
"tw-animate-css": "1.3.5",
"typescript": "5.8.3",
"typescript-eslint": "8.56.0",
"tailwindcss": "4.3.0",
"tw-animate-css": "1.4.0",
"typescript": "5.9.3",
"typescript-eslint": "8.60.1",
"unified": "11.0.5",
"unist-util-visit": "5.0.0",
"unist-util-visit": "5.1.0",
"uuid": "13.0.2",
"vite": "7.3.2",
"vite": "7.3.5",
"vite-plugin-devtools-json": "0.2.1",
"vitest": "4.1.8",
"vitest-browser-svelte": "2.1.1",
"zod": "4.2.1"
"workbox-window": "7.4.1"
},
"overrides": {
"cookie": "1.1.1"

28
tools/ui/playwright.config.ts
View file

@ -1,11 +1,31 @@
import { defineConfig } from '@playwright/test';
import { defineConfig, devices } from '@playwright/test';
export default defineConfig({
testDir: 'tests/e2e',
testMatch: ['**/*.e2e.ts'],
timeout: 30000,
expect: {
timeout: 5000
},
fullyParallel: true,
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 2 : 0,
workers: process.env.CI ? 1 : undefined,
reporter: 'line',
use: {
baseURL: 'http://localhost:8181',
trace: 'on-first-retry'
},
projects: [
{
name: 'chromium',
use: { ...devices['Desktop Chrome'] }
}
],
webServer: {
command: 'npm run build && npx http-server ./dist -p 8181',
port: 8181,
timeout: 120000,
reuseExistingServer: false
},
testDir: 'tests/e2e'
reuseExistingServer: !process.env.CI
}
});

20
tools/ui/pwa-assets-dark.config.ts Normal file
View file

@ -0,0 +1,20 @@
import { defineConfig } from '@vite-pwa/assets-generator/config';
export default defineConfig({
headLinkOptions: {
preset: '2023'
},
preset: {
transparent: {
sizes: [],
favicons: [[48, 'favicon-dark.ico']]
},
maskable: {
sizes: []
},
apple: {
sizes: []
}
},
images: ['static/favicon-dark.svg']
});

51
tools/ui/pwa-assets.config.ts Normal file
View file

@ -0,0 +1,51 @@
import {
combinePresetAndAppleSplashScreens,
defineConfig,
minimal2023Preset
} from '@vite-pwa/assets-generator/config';
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { THEME_COLORS, PWA_GENERATOR_DEVICES, PWA_ASSET_GENERATOR } from './src/lib/constants/pwa';
import { SplashOrientation } from './src/lib/enums/splash.enums';
export default defineConfig({
headLinkOptions: {
preset: PWA_ASSET_GENERATOR.LINK_PRESET
},
preset: combinePresetAndAppleSplashScreens(
minimal2023Preset,
{
padding: PWA_ASSET_GENERATOR.SPLASH_PADDING,
resizeOptions: {
background: THEME_COLORS.BACKGROUND_LIGHT,
fit: PWA_ASSET_GENERATOR.FIT_MODE
},
darkResizeOptions: {
background: THEME_COLORS.BACKGROUND_DARK,
fit: PWA_ASSET_GENERATOR.FIT_MODE
},
darkImageResolver: async (imageName: string) => {
if (imageName.endsWith('favicon.svg')) {
return readFileSync(resolve('static/favicon-dark.svg'));
}
},
linkMediaOptions: {
log: true,
addMediaScreen: PWA_ASSET_GENERATOR.ADD_MEDIA_SCREEN,
basePath: PWA_ASSET_GENERATOR.BASE_PATH,
xhtml: PWA_ASSET_GENERATOR.XHTML
},
png: {
compressionLevel: PWA_ASSET_GENERATOR.PNG_COMPRESSION_LEVEL,
quality: PWA_ASSET_GENERATOR.PNG_QUALITY
},
name: (landscape, size, dark) => {
const orientation = landscape ? SplashOrientation.LANDSCAPE : SplashOrientation.PORTRAIT;
const darkPrefix = dark ? PWA_ASSET_GENERATOR.DARK_PREFIX : '';
return `apple-splash-${orientation}-${darkPrefix}${size.width}x${size.height}.png`;
}
},
PWA_GENERATOR_DEVICES
),
images: ['static/favicon.svg']
});

137
tools/ui/scripts/make-icons-circular.js Normal file
View file

@ -0,0 +1,137 @@
#!/usr/bin/env node
/**
* Apply circular mask to pwa-*.png icons.
* Uses the maskable icon as source (white bg, full logo) to avoid
* the small-colormap pwa icons looking bad when cropped to a circle.
*
* Usage: node scripts/make-icons-circular.js [--padding-pct <0-50>] [--scale-pct <50-100>]
*
* - padding-pct: percentage of icon size kept as padding around the circle (default: 25)
* - scale-pct: scale down the source image before cropping (default: 85)
*
* maskable-icon and apple-touch-icon are left untouched.
*/
import sharp from 'sharp';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const STATIC_DIR = path.resolve(__dirname, '..', 'static');
const paddingPct = process.argv.reduce((acc, arg, i, args) => {
if (arg === '--padding-pct' && args[i + 1]) return parseFloat(args[i + 1]);
return acc;
}, 0);
// Scale down the source image before cropping to circle
const scalePct = process.argv.reduce((acc, arg, i, args) => {
if (arg === '--scale-pct' && args[i + 1]) return parseFloat(args[i + 1]);
return acc;
}, 85); // default 85% - icon fills 85% of the circular area
// Source for circular icons: the maskable icon (white bg, full logo)
const sourceIcon = 'maskable-icon-512x512.png';
const targetIcons = ['pwa-64x64.png', 'pwa-192x192.png', 'pwa-512x512.png'];
// maskable-icon and apple-touch-icon stay square
const untouchedIcons = ['maskable-icon-512x512.png', 'apple-touch-icon-180x180.png'];
async function makeCircle(targetFilename) {
const targetPath = path.join(STATIC_DIR, targetFilename);
const sourcePath = path.join(STATIC_DIR, sourceIcon);
if (!fs.existsSync(sourcePath)) {
console.log(`⏭️ ${sourceIcon} not found, skipping`);
return;
}
if (!fs.existsSync(targetPath)) {
console.log(`⏭️ ${targetFilename} not found, skipping`);
return;
}
const metadata = await sharp(targetPath).metadata();
const size = Math.max(metadata.width, metadata.height);
const radius = Math.floor((size * (1 - paddingPct / 100)) / 2);
const center = Math.floor(size / 2);
// Build circular mask as RGBA buffer: white opaque circle on transparent bg
const maskBuf = Buffer.alloc(size * size * 4, 0);
for (let y = 0; y < size; y++) {
for (let x = 0; x < size; x++) {
const dx = x - center;
const dy = y - center;
const dist = Math.sqrt(dx * dx + dy * dy);
if (dist < radius) {
const i = (y * size + x) * 4;
maskBuf[i] = 255;
maskBuf[i + 1] = 255;
maskBuf[i + 2] = 255;
maskBuf[i + 3] = 255;
}
}
}
const tmpMask = path.join(STATIC_DIR, '.mask-tmp.png');
await sharp(maskBuf, {
raw: { width: size, height: size, channels: 4 }
})
.png()
.toFile(tmpMask);
// Step 1: Scale source relative to circle diameter (not full icon), composite centered onto white canvas of full size
const circleDiameter = Math.floor(size * (1 - paddingPct / 100));
const scaledSize = Math.floor((circleDiameter * scalePct) / 100);
const offset = Math.floor((size - scaledSize) / 2);
const scaledBuf = await sharp(sourcePath)
.resize(scaledSize, scaledSize, {
fit: 'cover',
background: { r: 255, g: 255, b: 255, alpha: 1 }
})
.ensureAlpha()
.png()
.toBuffer();
// Step 2: Composite scaled image onto white background, then apply circular mask
const output = await sharp({
create: {
width: size,
height: size,
channels: 4,
background: { r: 255, g: 255, b: 255, alpha: 1 }
}
})
.composite([
{ input: scaledBuf, top: offset, left: offset },
{ input: tmpMask, top: 0, left: 0, blend: 'dest-in' }
])
.png()
.toBuffer();
fs.writeFileSync(targetPath, output);
fs.unlinkSync(tmpMask);
console.log(
`✓ ${targetFilename} → circle from ${sourceIcon}, ${paddingPct}% padding (size=${size}, r=${radius}, scale=${scalePct}%, circleDiameter=${circleDiameter})`
);
}
async function main() {
console.log(`Circular mask: ${paddingPct}% padding, ${scalePct}% scale, source=${sourceIcon}\n`);
for (const icon of targetIcons) {
await makeCircle(icon);
}
console.log('\nUnchanged:');
for (const icon of untouchedIcons) {
const fp = path.join(STATIC_DIR, icon);
console.log(` ${icon} (${fs.existsSync(fp) ? fs.statSync(fp).size + ' bytes' : 'missing'})`);
}
}
main();

41
tools/ui/scripts/vite-plugin-build-info.ts Normal file
View file

@ -0,0 +1,41 @@
import { writeFileSync, existsSync } from 'node:fs';
import { resolve } from 'path';
import type { Plugin } from 'vite';
import { BUILD_CONFIG } from '../src/lib/constants/pwa';
let processed = false;
const OUTPUT_DIR = process.env.LLAMA_UI_OUT_DIR ?? BUILD_CONFIG.OUTPUT_DIR;
/**
* Write build.json with the llama.cpp release build number.
*
* LLAMA_BUILD_NUMBER is passed from CMake -> npm -> vite via env var.
* Used for display of the current llama-server release (e.g. "b1234").
*/
export function buildInfoPlugin(): Plugin {
return {
name: 'llamacpp:build-info',
apply: 'build',
closeBundle() {
setTimeout(() => {
try {
if (processed) return;
processed = true;
const buildNumber = process.env.LLAMA_BUILD_NUMBER || 'b0000';
const outDir = resolve(OUTPUT_DIR);
const indexPath = resolve(outDir, 'index.html');
if (!existsSync(indexPath)) return;
const buildJsonPath = resolve(outDir, 'build.json');
writeFileSync(buildJsonPath, JSON.stringify({ version: buildNumber }), 'utf-8');
console.log(`Created build.json (version: ${buildNumber})`);
} catch (error) {
console.error('Failed to write build.json:', error);
}
}, 100);
}
};
}

105
tools/ui/scripts/vite-plugin-llama-cpp-build.ts
View file

@ -1,105 +0,0 @@
import {
readFileSync,
writeFileSync,
existsSync,
readdirSync,
copyFileSync,
rmSync,
unlinkSync
} from 'fs';
import { resolve } from 'path';
import type { Plugin } from 'vite';
const GUIDE_FOR_FRONTEND = `
<!--
This is a static build of the frontend.
It is automatically generated by the build process.
Do not edit this file directly.
To make changes, refer to the "Web UI" section in the README.
-->
`.trim();
const OUTPUT_DIR = process.env.LLAMA_UI_OUT_DIR ?? './dist';
export function llamaCppBuildPlugin(): Plugin {
return {
name: 'llamacpp:build',
apply: 'build',
closeBundle() {
setTimeout(() => {
try {
const outDir = resolve(OUTPUT_DIR);
const indexPath = resolve(outDir, 'index.html');
if (!existsSync(indexPath)) return;
let content = readFileSync(indexPath, 'utf-8');
// Inline favicon as base64 data URL
const faviconPath = resolve('static/favicon.svg');
if (existsSync(faviconPath)) {
const faviconContent = readFileSync(faviconPath, 'utf-8');
const faviconBase64 = Buffer.from(faviconContent).toString('base64');
const faviconDataUrl = `data:image/svg+xml;base64,${faviconBase64}`;
content = content.replace(/href="[^"]*favicon\.svg"/g, `href="${faviconDataUrl}"`);
console.log('✓ Inlined favicon.svg as base64 data URL');
}
content = content.replace(/\r/g, '');
content = GUIDE_FOR_FRONTEND + '\n' + content;
// Keep the Vite hash as a query string so each build busts the browser cache
content = content.replace(/\/_app\/immutable\/bundle\.([^".]+)\.js/g, './bundle.js?$1');
content = content.replace(
/\/_app\/immutable\/assets\/bundle\.([^".]+)\.css/g,
'./bundle.css?$1'
);
content = content.replace(/__sveltekit_[a-z0-9]+/g, '__sveltekit__');
writeFileSync(indexPath, content, 'utf-8');
console.log('✓ Updated index.html');
// Copy bundle.*.js -> bundle.js at output root
const immutableDir = resolve(outDir, '_app/immutable');
const bundleDir = resolve(outDir, '_app/immutable/assets');
if (existsSync(immutableDir)) {
const jsFiles = readdirSync(immutableDir).filter((f) => f.match(/^bundle\..+\.js$/));
if (jsFiles.length > 0) {
copyFileSync(resolve(immutableDir, jsFiles[0]), resolve(outDir, 'bundle.js'));
// Normalize __sveltekit_<hash> to __sveltekit__ in bundle.js
const bundleJsPath = resolve(outDir, 'bundle.js');
let bundleJs = readFileSync(bundleJsPath, 'utf-8');
bundleJs = bundleJs.replace(/__sveltekit_[a-z0-9]+/g, '__sveltekit__');
writeFileSync(bundleJsPath, bundleJs, 'utf-8');
console.log(`✓ Copied ${jsFiles[0]} -> bundle.js`);
}
}
// Copy bundle.*.css -> bundle.css at output root
if (existsSync(bundleDir)) {
const cssFiles = readdirSync(bundleDir).filter((f) => f.match(/^bundle\..+\.css$/));
if (cssFiles.length > 0) {
copyFileSync(resolve(bundleDir, cssFiles[0]), resolve(outDir, 'bundle.css'));
console.log(`✓ Copied ${cssFiles[0]} -> bundle.css`);
}
}
// Cleanup: remove _app directory, favicon.svg, and legacy index.html.gz
const appDir = resolve(outDir, '_app');
if (existsSync(appDir)) {
rmSync(appDir, { recursive: true, force: true });
console.log('✓ Removed _app directory');
}
const faviconOut = resolve(outDir, 'favicon.svg');
if (existsSync(faviconOut)) {
unlinkSync(faviconOut);
console.log('✓ Removed favicon.svg');
}
} catch (error) {
console.error('Failed to process build output:', error);
}
}, 100);
}
};
}

61
tools/ui/scripts/vite-plugin-relativize-base.ts Normal file
View file

@ -0,0 +1,61 @@
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
import { resolve } from 'path';
import type { Plugin } from 'vite';
import { BUILD_CONFIG } from '../src/lib/constants/pwa';
let processed = false;
const OUTPUT_DIR = process.env.LLAMA_UI_OUT_DIR ?? BUILD_CONFIG.OUTPUT_DIR;
function rewrite(path: string, pairs: [string, string][]): void {
if (!existsSync(path)) {
return;
}
const text = readFileSync(path, 'utf-8');
let out = text;
for (const [from, to] of pairs) {
out = out.split(from).join(to);
}
if (out !== text) {
writeFileSync(path, out, 'utf-8');
}
}
/**
* Relativize SvelteKit absolute base refs so the build is relocatable under any subpath.
*
* SvelteKit bakes root absolute /_app/ paths into the SPA fallback because paths.relative
* does not apply to a depth agnostic fallback page. Rewriting to ./_app/ lets a plain
* recursive copy of the output into /any/subdir/ resolve assets against the document URL.
* Runs after adapter-static writes index.html and the PWA plugin writes sw.js, deferred the
* same way as buildInfoPlugin so the emitted files exist.
*/
export function relativizeBasePlugin(): Plugin {
return {
name: 'llamacpp:relativize-base',
apply: 'build',
closeBundle() {
setTimeout(() => {
try {
if (processed) return;
processed = true;
const outDir = resolve(OUTPUT_DIR);
// index.html: modulepreload, stylesheet and bootstrap import reference "/_app/
rewrite(resolve(outDir, 'index.html'), [['"/_app/', '"./_app/']]);
// sw.js: the only absolute entries are the navigate fallback precache key and handler
rewrite(resolve(outDir, 'sw.js'), [
['{url:"/"', '{url:"./"'],
['createHandlerBoundToURL("/"', 'createHandlerBoundToURL("./"']
]);
console.log('Relativized base refs in index.html and sw.js');
} catch (error) {
console.error('Failed to relativize base refs:', error);
}
}, 100);
}
};
}

115
tools/ui/scripts/vite-plugin-splash-screen.ts Normal file
View file

@ -0,0 +1,115 @@
import { readdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs';
import { resolve } from 'path';
import type { Plugin } from 'vite';
import { TAB, NEWLINE } from '../src/lib/constants/code';
import { APPLE_DEVICES, BUILD_CONFIG, REGEX_PATTERNS, SPLASH_LINK } from '../src/lib/constants/pwa';
import type { SplashDimensions } from '../src/lib/types';
import { SplashOrientation } from '../src/lib/enums/splash.enums';
let processed = false;
const OUTPUT_DIR = process.env.LLAMA_UI_OUT_DIR ?? BUILD_CONFIG.OUTPUT_DIR;
/**
* Generate iOS splash screen <link> tags from generated apple-splash-*.png files.
* Returns an array of HTML link strings to be injected into the page head.
*/
export function generateSplashScreenLinks(outDir: string): string[] {
const files = readdirSync(outDir).filter((f) => f.match(REGEX_PATTERNS.SPLASH_FILE));
if (files.length === 0) return [];
const dimMap = new Map<string, SplashDimensions>();
for (const [dims, spec] of Object.entries(APPLE_DEVICES)) {
const [w, h] = dims.split('x').map(Number);
// logical-point dimensions
dimMap.set(`${w}x${h}`, { deviceW: spec.width, deviceH: spec.height, dpr: spec.dpr });
dimMap.set(`${h}x${w}`, { deviceW: spec.width, deviceH: spec.height, dpr: spec.dpr });
// pixel dimensions (used by actual generated splash files)
dimMap.set(`${w * spec.dpr}x${h * spec.dpr}`, {
deviceW: spec.width,
deviceH: spec.height,
dpr: spec.dpr
});
dimMap.set(`${h * spec.dpr}x${w * spec.dpr}`, {
deviceW: spec.width,
deviceH: spec.height,
dpr: spec.dpr
});
}
const lightLinks: string[] = [];
const darkLinks: string[] = [];
for (const file of files) {
const match = file.match(REGEX_PATTERNS.SPLASH_FILE);
if (!match) continue;
const orientation = match[1] as SplashOrientation;
const isDark = !!match[2];
const pixelW = parseInt(match[3]);
const pixelH = parseInt(match[4]);
const key = `${pixelW}x${pixelH}`;
const spec = dimMap.get(key);
if (!spec) {
console.warn(`Unknown splash screen dimensions: ${key} (${file})`);
continue;
}
const { deviceW, deviceH, dpr } = spec;
const media = `screen and (device-width: ${deviceW}px) and (device-height: ${deviceH}px) and (-webkit-device-pixel-ratio: ${dpr}) and (orientation: ${orientation})`;
const href = `./${file}`;
if (isDark) {
darkLinks.push(
`${SPLASH_LINK.HTML} media="${media}${SPLASH_LINK.DARK_MEDIA_SUFFIX}" href="${href}">`
);
} else {
lightLinks.push(`${SPLASH_LINK.HTML} media="${media}" href="${href}">`);
}
}
return [...lightLinks, ...darkLinks];
}
export function splashScreenPlugin(): Plugin {
return {
name: 'llamacpp:splash-screen',
apply: 'build',
closeBundle() {
setTimeout(() => {
try {
if (processed) return;
processed = true;
const outDir = resolve(OUTPUT_DIR);
const indexPath = resolve(outDir, 'index.html');
if (!existsSync(indexPath)) return;
let content = readFileSync(indexPath, 'utf-8');
// Inject iOS splash screen <link> tags into <head>.
// The @vite-pwa/assets-generator generates apple-splash-*.png files;
// this scans them and creates the <link> tags SvelteKit needs.
const splashLinks = generateSplashScreenLinks(outDir);
if (splashLinks.length > 0) {
console.log(`Generated ${splashLinks.length} apple-splash link tags`);
const splashHtml = splashLinks.map((l) => TAB + TAB + l).join(NEWLINE);
content = content.replace(
REGEX_PATTERNS.HEAD_CLOSE,
splashHtml + NEWLINE + TAB + TAB + '</head>'
);
}
// Remove trailing \r from Windows line endings
content = content.replace(/\r/g, '');
content = BUILD_CONFIG.GUIDE_COMMENT + NEWLINE + content;
writeFileSync(indexPath, content, 'utf-8');
console.log('Updated index.html');
} catch (error) {
console.error('Failed to process build output:', error);
}
}, 100);
}
};
}

3
tools/ui/src/app.d.ts vendored
View file

@ -1,6 +1,9 @@
// See https://svelte.dev/docs/kit/types#app.d.ts
// for information about these interfaces
import 'vite-plugin-pwa/pwa-assets';
import 'vite-plugin-pwa/svelte';
// Import chat types from dedicated module
import type {

9
tools/ui/src/app.html
View file

@ -2,10 +2,17 @@
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="icon" href="%sveltekit.assets%/favicon.svg" />
<link rel="icon" href="favicon.ico" sizes="48x48" />
<link rel="icon" href="favicon.svg" sizes="any" type="image/svg+xml" />
<link rel="apple-touch-icon" href="apple-touch-icon-180x180.png" />
<link rel="manifest" href="./manifest.webmanifest" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
%sveltekit.head%
</head>
<body data-sveltekit-preload-data="hover">
<div style="display: contents">%sveltekit.body%</div>
</body>

11
tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte
View file

@ -20,6 +20,8 @@
import { ColorMode } from '$lib/enums/ui.enums';
import { fade } from 'svelte/transition';
import { goto } from '$app/navigation';
import { Button } from '$lib/components/ui/button';
import { RefreshCw } from '@lucide/svelte';
import { page } from '$app/state';
import { setChatSettingsConfigContext } from '$lib/contexts';
import { settingsReferrer } from '$lib/stores/settings-referrer.svelte';
@ -164,6 +166,15 @@
onConfigChange={handleConfigChange}
onThemeChange={handleThemeChange}
/>
{#if currentSection.title === SETTINGS_SECTION_TITLES.GENERAL}
<div class="flex justify-end">
<Button variant="outline" onclick={() => window.location.reload()}>
<RefreshCw class="h-3 w-3" />
Reload app
</Button>
</div>
{/if}
</div>
{/if}
</div>

23
tools/ui/src/lib/components/pwa/PwaMetaTags.svelte Normal file
View file

@ -0,0 +1,23 @@
<script lang="ts">
import { APPLE_META_TAGS, MEDIA_QUERIES, THEME_COLORS } from '$lib/constants/pwa';
import { APP_NAME } from '$lib/constants';
let { appName = APP_NAME } = $props();
</script>
<svelte:head>
<!-- Theme color for light/dark modes -->
<meta name="theme-color" content={THEME_COLORS.LIGHT} media={MEDIA_QUERIES.PREFERS_LIGHT} />
<meta name="theme-color" content={THEME_COLORS.DARK} media={MEDIA_QUERIES.PREFERS_DARK} />
<!-- Apple mobile web app meta tags -->
<meta
name={APPLE_META_TAGS.MOBILE_WEB_APP_CAPABLE.name}
content={APPLE_META_TAGS.MOBILE_WEB_APP_CAPABLE.content}
/>
<meta
name={APPLE_META_TAGS.STATUS_BAR_STYLE.name}
content={APPLE_META_TAGS.STATUS_BAR_STYLE.content}
/>
<meta name={APPLE_META_TAGS.MOBILE_WEB_APP_TITLE.name} content={appName} />
</svelte:head>

35
tools/ui/src/lib/components/pwa/PwaRefreshAlert.svelte Normal file
View file

@ -0,0 +1,35 @@
<script lang="ts">
import * as Card from '$lib/components/ui/card';
import { Button } from '$lib/components/ui/button';
let { needRefresh: needRefreshProp, updateServiceWorker, forceReload } = $props();
let needRefresh = $derived(needRefreshProp ?? false);
</script>
{#if needRefresh}
<Card.Root class="overflow-hidden gap-1 py-5">
<Card.Header class="px-5">
<Card.Title class="text-sm font-medium">Update available</Card.Title>
</Card.Header>
<Card.Content class="gap-6 grid px-5">
<p class="text-xs text-muted-foreground">A new version is available. Reload to update.</p>
<Button
class="justify-self-end-safe"
size="sm"
onclick={() => {
updateServiceWorker();
if (forceReload) {
window.location.reload();
}
needRefresh = false;
}}
>
Reload
</Button>
</Card.Content>
</Card.Root>
{/if}

2
tools/ui/src/lib/components/pwa/index.ts Normal file
View file

@ -0,0 +1,2 @@
export { default as PwaMetaTags } from './PwaMetaTags.svelte';
export { default as PwaRefreshAlert } from './PwaRefreshAlert.svelte';

1
tools/ui/src/lib/constants/app.ts Normal file
View file

@ -0,0 +1 @@
export const APP_NAME = import.meta.env?.VITE_PUBLIC_APP_NAME || 'llama-ui';

1
tools/ui/src/lib/constants/code.ts
View file

@ -1,4 +1,5 @@
export const NEWLINE = '\n';
export const TAB = '\t';
export const DEFAULT_LANGUAGE = 'text';
export const LANG_PATTERN = /^(\w*)\n?/;
export const AMPERSAND_REGEX = /&/g;

2
tools/ui/src/lib/constants/index.ts
View file

@ -3,6 +3,7 @@
export * from './agentic';
export * from './api-endpoints';
export * from './app';
export * from './attachment-labels';
export * from './database';
export * from './reasoning-effort';
@ -36,6 +37,7 @@ export * from './message-export';
export * from './model-id';
export * from './precision';
export * from './processing-info';
export * from './pwa';
export * from './routes';
export * from './sandbox';
export * from './settings-keys';

30
tools/ui/src/lib/constants/jpeg-exif.ts Normal file
View file

@ -0,0 +1,30 @@
/**
* JPEG and EXIF binary format constants for orientation parsing.
*/
/** Bytes of file prefix to scan, the APP1 EXIF segment sits near the start */
export const EXIF_SCAN_BYTE_LIMIT = 128 * 1024;
/** JPEG start of image marker */
export const JPEG_SOI_MARKER = 0xffd8;
/** APP1 segment marker byte, carries the EXIF payload */
export const APP1_MARKER = 0xe1;
/** Start of scan marker byte, compressed data begins and no EXIF follows */
export const SOS_MARKER = 0xda;
/** "Exif" signature opening the APP1 payload, big endian uint32 */
export const EXIF_SIGNATURE = 0x45786966;
/** TIFF byte order mark for little endian ("II") */
export const TIFF_LITTLE_ENDIAN = 0x4949;
/** TIFF magic number following the byte order mark */
export const TIFF_MAGIC = 42;
/** EXIF tag id holding the orientation value */
export const EXIF_ORIENTATION_TAG = 0x0112;
/** Size in bytes of one IFD directory entry */
export const IFD_ENTRY_SIZE = 12;

352
tools/ui/src/lib/constants/pwa.ts Normal file
View file

@ -0,0 +1,352 @@
/**
* Centralized PWA constants to avoid magic strings, regexes, and duplicated
* definitions across the codebase.
*/
import { APP_NAME } from './app';
export const MEDIA_QUERIES = {
PREFERS_DARK: '(prefers-color-scheme: dark)',
PREFERS_LIGHT: '(prefers-color-scheme: light)'
} as const;
export const THEME_COLORS = {
LIGHT: '#ffffff',
DARK: '#0d0d0d',
ACCENT_BLUE: '#2563eb',
ACCENT_BLUE_HOVER: '#1d4ed8',
BACKGROUND_LIGHT: 'white',
BACKGROUND_DARK: '#111111',
TITLE_UPDATE_ALERT: {
BORDER_LIGHT: 'zinc-200',
BORDER_DARK: 'zinc-700',
BG_LIGHT: 'white',
BG_DARK: 'zinc-800',
TEXT_LIGHT: 'zinc-500',
TEXT_DARK: 'zinc-400'
}
} as const;
export const FAVICON_PATHS = {
ICO_LIGHT: 'favicon.ico',
ICO_DARK: 'favicon-dark.ico',
SVG_LIGHT: 'favicon.svg',
SVG_DARK: 'favicon-dark.svg'
} as const;
export const FAVICON_SELECTORS = {
ICO_48X48: 'link[rel="icon"][sizes="48x48"]',
SVG_ANY: 'link[rel="icon"][type="image/svg+xml"]'
} as const;
export const APPLE_ASSETS = {
TOUCH_ICON: 'apple-touch-icon-180x180.png'
} as const;
export const PWA_MANIFEST = {
name: APP_NAME,
short_name: APP_NAME,
description: 'Local AI chat interface powered by llama.cpp',
start_url: './',
display: 'standalone' as const,
background_color: THEME_COLORS.BACKGROUND_LIGHT,
theme_color: THEME_COLORS.BACKGROUND_LIGHT,
icons: [
{ src: 'pwa-64x64.png', sizes: '64x64', type: 'image/png' },
{ src: 'pwa-192x192.png', sizes: '192x192', type: 'image/png' },
{ src: 'pwa-512x512.png', sizes: '512x512', type: 'image/png', purpose: 'any' as const },
{
src: 'maskable-icon-512x512.png',
sizes: '512x512',
type: 'image/png',
purpose: 'maskable' as const
}
]
};
export const PWA_ICON_PATHS = {
PWA_64: '/pwa-64x64.png',
PWA_192: '/pwa-192x192.png',
PWA_512: '/pwa-512x512.png',
MASKABLE_512: '/maskable-icon-512x512.png'
} as const;
/** Apple device dimensions (logical points) and DPR, from Apple HIG. */
export const APPLE_DEVICES = {
// iPhones (DPR 3)
'1170x2532': { width: 390, height: 844, dpr: 3 }, // iPhone 13, 15
'1179x2556': { width: 393, height: 852, dpr: 3 }, // iPhone 14, 15 Pro, 16
'1206x2622': { width: 402, height: 874, dpr: 3 }, // iPhone 16 Plus, 16e
'1284x2778': { width: 428, height: 926, dpr: 3 }, // iPhone 15 Plus
'1290x2796': { width: 430, height: 932, dpr: 3 }, // iPhone 15 Pro Max, 16 Pro
'1320x2868': { width: 440, height: 956, dpr: 3 }, // iPhone 16 Pro Max
'750x1334': { width: 375, height: 667, dpr: 2 }, // iPhone 6/7/8, 14
'640x1136': { width: 320, height: 568, dpr: 2 }, // iPhone 6/7/8 Plus
// iPads (DPR 2)
'1668x2388': { width: 834, height: 1194, dpr: 2 }, // iPad Air 11", iPad 11"
'2048x2732': { width: 1024, height: 1366, dpr: 2 }, // iPad Pro 12.9"
'1640x2360': { width: 820, height: 1180, dpr: 2 }, // iPad Air 10.9"
'1032x1376': { width: 1032, height: 1376, dpr: 2 }, // iPad Air 13"
'744x1133': { width: 376, height: 573, dpr: 2 } // iPad mini 8.3"
} as const;
export type AppleDeviceKey = keyof typeof APPLE_DEVICES;
export const PWA_FILE_PATHS = {
MANIFEST: '/manifest.webmanifest',
SERVICE_WORKER: '/sw.js',
VERSION: '/version.json',
WORKBOX: '/workbox-<hash>.js'
} as const;
// Used by the server middleware to skip API key validation.
// Keep in sync with tools/server/server-http.cpp public_endpoints list.
export const PUBLIC_ENDPOINTS = [
'/health',
'/v1/health',
'/models',
'/v1/models',
'/props',
'/metrics',
'/',
'/index.html',
'/favicon.ico',
'/favicon-dark.ico',
'/favicon.svg',
'/favicon-dark.svg',
'/pwa-64x64.png',
'/pwa-192x192.png',
'/pwa-512x512.png',
'/maskable-icon-512x512.png',
'/apple-touch-icon-180x180.png',
'/apple-splash-portrait-640x1136.png',
'/apple-splash-landscape-640x1136.png',
'/apple-splash-portrait-750x1334.png',
'/apple-splash-landscape-750x1334.png',
'/apple-splash-portrait-1170x2532.png',
'/apple-splash-landscape-1170x2532.png',
'/apple-splash-portrait-1179x2556.png',
'/apple-splash-landscape-1179x2556.png',
'/apple-splash-portrait-1206x2622.png',
'/apple-splash-landscape-1206x2622.png',
'/apple-splash-portrait-1284x2778.png',
'/apple-splash-landscape-1284x2778.png',
'/apple-splash-portrait-1290x2796.png',
'/apple-splash-landscape-1290x2796.png',
'/apple-splash-portrait-1320x2868.png',
'/apple-splash-landscape-1320x2868.png',
'/apple-splash-portrait-1488x2266.png',
'/apple-splash-landscape-1488x2266.png',
'/apple-splash-portrait-1640x2360.png',
'/apple-splash-landscape-1640x2360.png',
'/apple-splash-portrait-1668x2388.png',
'/apple-splash-landscape-1668x2388.png',
'/apple-splash-portrait-2048x2732.png',
'/apple-splash-landscape-2048x2732.png',
'/apple-splash-portrait-dark-640x1136.png',
'/apple-splash-landscape-dark-640x1136.png',
'/apple-splash-portrait-dark-750x1334.png',
'/apple-splash-landscape-dark-750x1334.png',
'/apple-splash-portrait-dark-1170x2532.png',
'/apple-splash-landscape-dark-1170x2532.png',
'/apple-splash-portrait-dark-1179x2556.png',
'/apple-splash-landscape-dark-1179x2556.png',
'/apple-splash-portrait-dark-1206x2622.png',
'/apple-splash-landscape-dark-1206x2622.png',
'/apple-splash-portrait-dark-1284x2778.png',
'/apple-splash-landscape-dark-1284x2778.png',
'/apple-splash-portrait-dark-1290x2796.png',
'/apple-splash-landscape-dark-1290x2796.png',
'/apple-splash-portrait-dark-1320x2868.png',
'/apple-splash-landscape-dark-1320x2868.png',
'/apple-splash-portrait-dark-1488x2266.png',
'/apple-splash-landscape-dark-1488x2266.png',
'/apple-splash-portrait-dark-1640x2360.png',
'/apple-splash-landscape-dark-1640x2360.png',
'/apple-splash-portrait-dark-1668x2388.png',
'/apple-splash-landscape-dark-1668x2388.png',
'/apple-splash-portrait-dark-2048x2732.png',
'/apple-splash-landscape-dark-2048x2732.png',
'/manifest.webmanifest',
'/sw.js',
'/version.json',
'/workbox-<hash>.js'
] as const;
export const BUILD_CONFIG = {
OUTPUT_DIR: './dist',
GUIDE_COMMENT: `
<!--
This is a static build of the frontend.
It is automatically generated by the build process.
Do not edit this file directly.
To make changes, refer to the "Web UI" section in the README.
-->
`.trim()
} as const;
export const REGEX_PATTERNS = {
SPLASH_FILE: /^apple-splash-(portrait|landscape)-(dark-)?(\d+)x(\d+)\.png$/,
HEAD_CLOSE: /\t*<\/head>/
} as const;
// Device names used by @vite-pwa/assets-generator for splash screen generation.
// Keep in sync with pwa-assets.config.ts.
export const PWA_GENERATOR_DEVICES = [
'iPhone 13',
'iPhone 13 Pro',
'iPhone 13 Pro Max',
'iPhone 14',
'iPhone 14 Plus',
'iPhone 14 Pro',
'iPhone 14 Pro Max',
'iPhone 15',
'iPhone 15 Plus',
'iPhone 15 Pro',
'iPhone 15 Pro Max',
'iPhone 16',
'iPhone 16 Plus',
'iPhone 16 Pro',
'iPhone 16 Pro Max',
'iPhone 16e',
'iPhone SE 4"',
'iPhone SE 4.7"',
'iPad 11"',
'iPad Air 10.9"',
'iPad Air 11"',
'iPad Air 13"',
'iPad Pro 11"',
'iPad Pro 12.9"',
'iPad mini 8.3"'
] as const;
// PWA assets generator configuration — used by pwa-assets.config.ts
export const PWA_ASSET_GENERATOR = {
LINK_PRESET: '2023',
SPLASH_PADDING: 0.75,
FIT_MODE: 'contain',
ADD_MEDIA_SCREEN: true,
BASE_PATH: './',
XHTML: false,
PNG_COMPRESSION_LEVEL: 9,
PNG_QUALITY: 60,
DARK_PREFIX: 'dark-'
} as const;
export const CACHE_SETTINGS = {
IMMUTABLE_MAX_AGE_SECONDS: 31536000,
API_CACHE_MAX_AGE_SECONDS: 60 * 60 * 24,
API_CACHE_MAX_ENTRIES: 50,
MAX_FILE_SIZE_BYTES: 10 * 1024 * 1024
} as const;
export const GLOB_PATTERNS: string[] = [
'**/*.{js,css,html,ico,svg,png,webp,woff,woff2,json,webmanifest}'
];
// loading.html is the model loading page served by llama-server itself.
// The SvelteKit PWA manifest transform strips the html extension from every
// precache entry to match clean URLs, but loading.html is a plain static asset
// with no clean URL, so static servers answer 404 and the SW install fails.
export const GLOB_IGNORES: string[] = ['**/loading.html'];
export const SW_CONFIG = {
CHECK_INTERVAL_MS: 60000,
UPDATE_FETCH_OPTIONS: {
CACHE: 'no-store',
HEADERS: {
CACHE: 'no-store',
CACHE_CONTROL: 'no-cache'
}
}
} as const;
// Runtime caching configuration for Workbox
export const RUNTIME_CACHING = {
HANDLER: 'NetworkFirst',
CACHE_NAME: 'api-cache'
} as const;
// Workbox runtime caching patterns
export const API_CACHING_PATTERNS = {
V1_API: /^\/v1\/.*/,
STATIC_API: /^\/(health|props|models|tools|slots|cors-proxy).*/
} as const;
// SvelteKit PWA plugin options
export const PWA_KIT_OPTIONS = {
NAVIGATE_FALLBACK: './'
} as const;
export const APPLE_META_TAGS = {
MOBILE_WEB_APP_CAPABLE: { name: 'apple-mobile-web-app-capable', content: 'yes' },
STATUS_BAR_STYLE: { name: 'apple-mobile-web-app-status-bar-style', content: 'black-translucent' },
MOBILE_WEB_APP_TITLE: { name: 'apple-mobile-web-app-title' }
} as const;
// Splash screen HTML link tag prefix used by generateSplashScreenLinks
export const SPLASH_LINK = {
HTML: '<link rel="apple-touch-startup-image"',
DARK_MEDIA_SUFFIX: ' and (prefers-color-scheme: dark)'
} as const;
// SvelteKit PWA plugin configuration — used by @vite.config.ts
import type { SvelteKitPWAOptions } from '@vite-pwa/sveltekit';
export const SVELTEKIT_PWA_OPTIONS: SvelteKitPWAOptions = {
// Strategy: generateSW - the plugin generates a service worker automatically
// using Workbox. For a custom SW, use 'injectManifest' instead.
// Manifest configuration
manifest: PWA_MANIFEST,
// Workbox configuration for generateSW strategy
workbox: {
// Match all static assets in the build output.
// Uses '**/' because SvelteKit outputs files under _app/immutable/
// subdirectories.
globPatterns: GLOB_PATTERNS,
globIgnores: GLOB_IGNORES,
maximumFileSizeToCacheInBytes: CACHE_SETTINGS.MAX_FILE_SIZE_BYTES,
// Runtime caching for API calls - use NetworkFirst so APIs are always fresh
runtimeCaching: [
{
urlPattern: API_CACHING_PATTERNS.V1_API,
handler: RUNTIME_CACHING.HANDLER,
options: {
cacheName: RUNTIME_CACHING.CACHE_NAME,
expiration: {
maxEntries: CACHE_SETTINGS.API_CACHE_MAX_ENTRIES,
maxAgeSeconds: CACHE_SETTINGS.API_CACHE_MAX_AGE_SECONDS
}
}
},
{
urlPattern: API_CACHING_PATTERNS.STATIC_API,
handler: RUNTIME_CACHING.HANDLER,
options: {
cacheName: RUNTIME_CACHING.CACHE_NAME,
expiration: {
maxEntries: CACHE_SETTINGS.API_CACHE_MAX_ENTRIES,
maxAgeSeconds: CACHE_SETTINGS.API_CACHE_MAX_AGE_SECONDS
}
}
}
]
},
devOptions: {
enabled: true,
suppressWarnings: true,
// Use PWA_KIT_OPTIONS.NAVIGATE_FALLBACK to match production SW behaviour
// (navigateFallback defaults to the configured base path, which is '/' for this SPA).
navigateFallback: PWA_KIT_OPTIONS.NAVIGATE_FALLBACK
},
// SvelteKit-specific options
kit: {
// Include version file for proper cache invalidation
includeVersionFile: true
}
};

1
tools/ui/src/lib/constants/settings-keys.ts
View file

@ -31,6 +31,7 @@ export const SETTINGS_KEYS = {
SHOW_RAW_MODEL_NAMES: 'showRawModelNames',
SHOW_MODEL_QUANTIZATION: 'showModelQuantization',
SHOW_MODEL_TAGS: 'showModelTags',
SHOW_BUILD_VERSION: 'showBuildVersion',
SHOW_SYSTEM_MESSAGE: 'showSystemMessage',
// Sampling
TEMPERATURE: 'temperature',

8
tools/ui/src/lib/constants/settings-registry.ts
View file

@ -365,6 +365,14 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
serverKey: SETTINGS_KEYS.ALWAYS_SHOW_AGENTIC_TURNS,
paramType: SyncableParameterType.BOOLEAN
}
},
{
key: SETTINGS_KEYS.SHOW_BUILD_VERSION,
label: 'Show build version information',
help: 'Display the current build version in the bottom-right corner of the interface.',
defaultValue: false,
type: SettingsFieldType.CHECKBOX,
section: SETTINGS_SECTION_SLUGS.DISPLAY
}
]
},

3
tools/ui/src/lib/constants/storage.ts
View file

@ -40,6 +40,9 @@ export const DEPRECATED_MCP_DEFAULT_ENABLED_LOCALSTORAGE_KEY = `${STORAGE_APP_NA
/** @deprecated Use {@link USER_OVERRIDES_LOCALSTORAGE_KEY} instead */
export const DEPRECATED_USER_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME_DEPRECATED}.userOverrides`;
/** Build version stored in localStorage for non-PWA update detection */
export const BUILD_VERSION_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.buildVersion`;
/** Maps new keys to their deprecated fallback keys */
export const NEW_TO_DEPRECATED_MAP: Record<string, string> = {
[ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY]: DEPRECATED_ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY,

1
tools/ui/src/lib/constants/ui.ts
View file

@ -5,7 +5,6 @@ import { ROUTES } from './routes';
export const FORK_TREE_DEPTH_PADDING = 8;
export const SYSTEM_MESSAGE_PLACEHOLDER = 'System message';
export const APP_NAME = import.meta.env.VITE_PUBLIC_APP_NAME || 'llama-ui';
export const ICON_STRIP_TRANSITION_DURATION = 150;
export const ICON_STRIP_TRANSITION_DELAY_MULTIPLIER = 50;

2
tools/ui/src/lib/enums/index.ts
View file

@ -63,3 +63,5 @@ export { ColorMode, HtmlInputType, McpPromptVariant, TooltipSide, UrlProtocol }
export { KeyboardKey } from './keyboard.enums';
export { ToolSource, ToolPermissionDecision, ToolResponseField } from './tools.enums';
export { SplashOrientation } from './splash.enums';

7
tools/ui/src/lib/enums/splash.enums.ts Normal file
View file

@ -0,0 +1,7 @@
/**
* Splash screen orientation for iOS apple-touch-startup-image
*/
export enum SplashOrientation {
PORTRAIT = 'portrait',
LANDSCAPE = 'landscape'
}

80
tools/ui/src/lib/hooks/use-pwa.svelte.ts Normal file
View file

@ -0,0 +1,80 @@
import { browser } from '$app/environment';
import { useRegisterSW } from 'virtual:pwa-register/svelte';
import { versionStore } from '$lib/stores/version.svelte';
import { BUILD_VERSION_LOCALSTORAGE_KEY } from '$lib/constants/storage';
import { SW_CONFIG } from '$lib/constants/pwa';
/**
* Hook for PWA service worker registration, update polling, and build version mismatch detection.
*
* Combines two concerns that always belong together:
* 1. SW registration with periodic polling for updates
* 2. localStorage-based version tracking for non-PWA users
*/
export function usePwa() {
let swCheckInterval: ReturnType<typeof setInterval> | null = null;
let needRefreshByStorage = $state(false);
const {
// offlineReady, // to do - add installation banners for iOS
needRefresh: pwaNeedRefresh,
updateServiceWorker
} = useRegisterSW({
onRegisteredSW(swUrl: string, r: ServiceWorkerRegistration | undefined) {
if (swCheckInterval) {
clearInterval(swCheckInterval);
}
swCheckInterval = setInterval(async () => {
if (!r || r.installing || !navigator?.onLine) return;
try {
const resp = await fetch(swUrl, {
cache: SW_CONFIG.UPDATE_FETCH_OPTIONS.CACHE,
headers: {
cache: SW_CONFIG.UPDATE_FETCH_OPTIONS.HEADERS.CACHE,
'cache-control': SW_CONFIG.UPDATE_FETCH_OPTIONS.HEADERS.CACHE_CONTROL
}
});
if (resp?.status === 200) {
await r.update();
}
} catch (e) {
console.error(e);
}
}, SW_CONFIG.CHECK_INTERVAL_MS);
},
onRegisterError(error: unknown) {
console.error('[PWA] SW registration error:', error);
}
});
// Detect version mismatch via localStorage.
// _app/version.json is SvelteKit's native version file for PWA cache invalidation.
// This comparison detects server upgrades for non-PWA users.
$effect(() => {
if (!browser) return;
const currentVersion = versionStore.value;
if (!currentVersion) return;
try {
const storedVersion = localStorage.getItem(BUILD_VERSION_LOCALSTORAGE_KEY);
needRefreshByStorage = !!storedVersion && storedVersion !== currentVersion;
localStorage.setItem(BUILD_VERSION_LOCALSTORAGE_KEY, currentVersion);
} catch {
needRefreshByStorage = false;
}
});
return {
/** Writable that is true when a PWA service worker update is available */
get needRefresh() {
return pwaNeedRefresh;
},
updateServiceWorker,
/** Version mismatch detected via localStorage (non-PWA users) */
get needRefreshByStorage() {
return needRefreshByStorage;
}
};
}

10
tools/ui/src/lib/services/chat.service.ts
View file

@ -34,7 +34,6 @@ import type {
import { modelsStore } from '$lib/stores/models.svelte';
import { settingsStore } from '../stores/settings.svelte';
import { capImageDataURLSize } from '../utils/cap-img-size';
import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size';
function getAudioInputFormat(mimeType: string): AudioInputFormat {
const normalizedMimeType = mimeType.trim().toLowerCase();
@ -965,10 +964,11 @@ export class ChatService {
for (const image of imageFiles) {
const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION);
let base64Url = image.base64Url;
if (maxImageResolution > 1 / MEGAPIXELS_TO_PIXELS) {
base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution);
}
// Caps the resolution and bakes the jpeg exif orientation in one pass,
// untouched images pass through as is
const base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution);
contentParts.push({
type: ContentPartType.IMAGE_URL,
image_url: { url: base64Url }

42
tools/ui/src/lib/stores/build-info.svelte.ts Normal file
View file

@ -0,0 +1,42 @@
/**
* buildInfoStore - llama.cpp build information
*
* Reads the build version from `build.json` — embedded at llama.cpp build time
* with the llama.cpp build number (LLAMA_BUILD_NUMBER). Shown in the UI when
* `showBuildVersion` is enabled.
*
* In dev mode (via `npm run dev`), falls back to `import.meta.env.DEV`'s truthy
* value since the artifact is not produced.
*/
import { browser } from '$app/environment';
import { base } from '$app/paths';
let build = $state<string>('');
async function loadBuild() {
if (!browser) return;
if (import.meta.env.DEV) {
build = 'dev';
return;
}
try {
const res = await fetch(`${base}/build.json`, { cache: 'no-store' });
if (res.ok) {
const data = await res.json();
build = data.version ?? '';
}
} catch {
// build.json missing or unreachable - leave as empty string
}
}
loadBuild();
export const buildInfoStore = {
get value(): string {
return build;
}
};

2
tools/ui/src/lib/stores/mcp.svelte.ts
View file

@ -489,7 +489,7 @@ class MCPStore {
if (!rootDomain) return null;
const origin = `${url.protocol}//${rootDomain}`;
const candidates = ['favicon.ico', 'favicon.svg', 'favicon.png'];
const candidates = ['favicon.ico', 'favicon.png'];
for (const path of candidates) {
const faviconUrl = `${origin}/${path}`;

14
tools/ui/src/lib/stores/theme.svelte.ts Normal file
View file

@ -0,0 +1,14 @@
import { browser } from '$app/environment';
import { MEDIA_QUERIES } from '$lib/constants';
export const theme = $state({
isSystemDark: browser && window.matchMedia(MEDIA_QUERIES.PREFERS_DARK).matches
});
if (browser) {
const mql = window.matchMedia(MEDIA_QUERIES.PREFERS_DARK);
mql.addEventListener('change', (e) => {
theme.isSystemDark = e.matches;
});
}

41
tools/ui/src/lib/stores/version.svelte.ts Normal file
View file

@ -0,0 +1,41 @@
/**
* versionStore - Frontend build version
*
* Reads from SvelteKit's `_app/version.json` — generated by the @vite-pwa/sveltekit
* plugin. The version string changes on every build, so comparing it against
* localStorage reliably detects server upgrades.
*
* In dev mode, falls back to `'dev'`.
*/
import { browser } from '$app/environment';
import { base } from '$app/paths';
let version = $state<string>('');
async function loadVersion() {
if (!browser) return;
if (import.meta.env.DEV) {
version = 'dev';
return;
}
try {
const res = await fetch(`${base}/_app/version.json`, { cache: 'no-store' });
if (res.ok) {
const data = await res.json();
version = data.version ?? '';
}
} catch {
// _app/version.json missing or unreachable - leave as empty string
}
}
loadVersion();
export const versionStore = {
get value(): string {
return version;
}
};

3
tools/ui/src/lib/types/index.ts
View file

@ -165,3 +165,6 @@ export type { ToolEntry, ToolGroup } from './tools';
// Reasoning
export type { ReasoningEffortLevel } from './reasoning';
// Splash
export type { SplashDimensions } from './splash';

Some files were not shown because too many files have changed in this diff Show more