vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-27 01:04:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge commit '873e5d8e39' into concedo_experimental

Browse source 
# Conflicts:
#	.github/workflows/build-cmake-pkg.yml
#	.github/workflows/build-cpu.yml
#	.github/workflows/make-release.yml
#	.github/workflows/release.yml
#	.pi/gg/SYSTEM.md
#	CMakeLists.txt
#	cmake/arm64-windows-llvm.cmake
#	docs/backend/ET.md
#	docs/build.md
#	docs/development/HOWTO-add-model.md
#	ggml/CMakeLists.txt
#	ggml/src/CMakeLists.txt
#	ggml/src/ggml-cpu/CMakeLists.txt
#	ggml/src/ggml-cpu/kleidiai/kernels.cpp
#	ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
#	ggml/src/ggml-hexagon/ggml-hexagon.cpp
#	ggml/src/ggml-hexagon/htp/rope-ops.c
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl
#	ggml/src/ggml-opencl/kernels/rope.cl
#	ggml/src/ggml-sycl/dmmv.cpp
#	ggml/src/ggml-sycl/dpct/helper.hpp
#	ggml/src/ggml-sycl/element_wise.cpp
#	ggml/src/ggml-sycl/esimd.hpp
#	ggml/src/ggml-sycl/fattn-mkl.cpp
#	ggml/src/ggml-sycl/fattn-onednn.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-sycl/im2col.cpp
#	ggml/src/ggml-sycl/norm.cpp
#	ggml/src/ggml-sycl/rope.cpp
#	ggml/src/ggml-sycl/set_rows.cpp
#	ggml/src/ggml-vulkan/CMakeLists.txt
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl
#	ggml/src/ggml-zendnn/CMakeLists.txt
#	scripts/make-release-desc.sh
#	scripts/sync-ggml.last
#	tests/test-backend-ops.cpp
#	tests/test-json-schema-to-grammar.cpp
#	tools/cli/README.md
#	tools/server/README.md
#	tools/ui/src/lib/constants/settings.constants.ts
#	tools/ui/src/lib/services/chat.service.ts
This commit is contained in:
Concedo 2026-08-24 21:15:18 +08:00
commit bfd6500450
172 changed files with 12496 additions and 13725 deletions
Download patch file Download diff file Expand all files Collapse all files

22
.github/actions/ccache-clear/action.yml vendored
View file

@ -1,22 +0,0 @@
name: "ccache-clear"
description: "Delete all GitHub Actions caches matching a key prefix"
inputs:
key:
description: "Cache key prefix to match and delete"
required: true
runs:
using: "composite"
steps:
- name: Clear caches
shell: bash
run: |
CACHES=$(gh cache list --key "ccache-${{ inputs.key }}" --json id,key --jq '.[] | "\(.id) \(.key)"' 2>/dev/null)
if [ -z "$CACHES" ]; then
echo "No caches found with key prefix: ${{ inputs.key }}"
exit 0
fi
while read -r id key; do
echo "Deleting cache: $id ($key)"
gh cache delete "$id"
done <<< "$CACHES"

20
common/arg.cpp
View file

@ -2596,6 +2596,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.mmproj_use_gpu = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_OFFLOAD"));
add_opt(common_arg(
// note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet
{"-mmdev", "--mmproj-device"}, "DEVICE",
"device to use for multimodal projector (none = don't offload, default: auto)\n"
"use --list-devices to see a list of available devices",
[](common_params & params, const std::string & value) {
if (value == "none") {
params.mmproj_use_gpu = false;
params.mmproj_device = nullptr;
return;
}
auto devices = parse_device_list(value);
// parse_device_list pushes nullptr at back so devices is length 2 for single device.
if (devices.size() > 2) {
throw std::invalid_argument("only one device may be specified for mmproj");
}
params.mmproj_use_gpu = true;
params.mmproj_device = devices.front();
}
).set_examples(mmproj_examples).set_env("MTMD_BACKEND_DEVICE")); // no LLAMA_ARG_ prefix for backward compatibility reason
add_opt(common_arg(
{"--image", "--audio", "--video"}, "FILE",
"path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files\n",

7
common/common.h
View file

@ -582,9 +582,10 @@ struct common_params {
// multimodal models (see tools/mtmd)
struct common_params_model mmproj;
bool mmproj_use_gpu = true; // use GPU for multimodal model
bool no_mmproj = false; // explicitly disable multimodal model
std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
bool mmproj_use_gpu = true; // use GPU for multimodal model
ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model
bool no_mmproj = false; // explicitly disable multimodal model
std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
int image_min_tokens = -1;
int image_max_tokens = -1;
int mtmd_batch_max_tokens = 1024;

142
common/json-schema-to-grammar.cpp
View file

@ -278,7 +278,9 @@ static std::unordered_map<char, std::string> GRAMMAR_LITERAL_ESCAPES = {
{'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}, {'\\', "\\\\"}
};
static std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'};
static const int MAX_PATTERN_DEPTH = 100;
static std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?', '^', '$'};
static std::unordered_set<char> ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'};
static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function<std::string(const std::smatch &)> & replacement) {
@ -309,6 +311,32 @@ static std::string format_literal(const std::string & literal) {
std::string gbnf_format_literal(const std::string & literal) { return format_literal(literal); }
static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
if (pos + 1 >= pattern.length() || pattern[pos] != '\\') {
return 0;
}
size_t n_hex = 0;
switch (pattern[pos + 1]) {
case 'x': n_hex = 2; break;
case 'u': n_hex = 4; break;
case 'U': n_hex = 8; break;
case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']':
return 2;
default:
return 0;
}
if (pos + 2 + n_hex > pattern.length()) {
return 0;
}
for (size_t i = pos + 2; i < pos + 2 + n_hex; i++) {
char h = pattern[i];
if (!((h >= '0' && h <= '9') || (h >= 'a' && h <= 'f') || (h >= 'A' && h <= 'F'))) {
return 0;
}
}
return 2 + n_hex;
}
class common_schema_converter {
private:
friend class common_schema_info;
@ -345,16 +373,42 @@ private:
return string_join(rules, " | ");
}
// thrown when the pattern is a valid regex with no grammar equivalent
struct unsupported_pattern : public std::runtime_error {
using std::runtime_error::runtime_error;
};
// thrown when the pattern is not a valid regex
struct invalid_pattern : public std::runtime_error {
using std::runtime_error::runtime_error;
};
std::string _visit_pattern(const std::string & pattern, const std::string & name) {
if (!(pattern.front() == '^' && pattern.back() == '$')) {
_errors.push_back("Pattern must start with '^' and end with '$'");
auto rules_snapshot = _rules;
try {
return _pattern_to_rule(pattern, name);
} catch (const unsupported_pattern & err) {
// revert rules
_rules = std::move(rules_snapshot);
_warnings.push_back("pattern " + pattern + " is not supported (" + err.what() + "), accepting any string");
return _add_rule(name, _add_primitive("string", PRIMITIVE_RULES.at("string")));
} catch (const invalid_pattern & err) {
_rules = std::move(rules_snapshot);
_errors.push_back("Invalid pattern " + pattern + ": " + err.what());
return "";
}
}
std::string _pattern_to_rule(const std::string & pattern, const std::string & name) {
if (pattern.length() < 2 || pattern.front() != '^' || pattern.back() != '$') {
throw unsupported_pattern("not anchored with '^' and '$'");
}
std::string sub_pattern = pattern.substr(1, pattern.length() - 2);
std::unordered_map<std::string, std::string> sub_rule_ids;
size_t i = 0;
size_t length = sub_pattern.length();
int paren_depth = 0;
using literal_or_rule = std::pair<std::string, bool>;
auto to_rule = [&](const literal_or_rule & ls) {
@ -363,7 +417,6 @@ private:
return is_literal ? "\"" + s + "\"" : s;
};
std::function<literal_or_rule()> transform = [&]() -> literal_or_rule {
size_t start = i;
std::vector<literal_or_rule> seq;
auto get_dot = [&]() {
@ -420,43 +473,42 @@ private:
if (i + 1 < length && sub_pattern[i + 1] == ':') {
i += 2; // skip "?:" for non-capturing group, treat as regular group
} else {
// lookahead/lookbehind (?=, ?!, ?<=, ?<!) - not supported
_warnings.push_back("Unsupported pattern syntax");
// skip to matching ')' to avoid UB on empty seq
int depth = 1;
while (i < length && depth > 0) {
if (sub_pattern[i] == '\\' && i + 1 < length) {
i += 2; // skip escaped character
} else {
if (sub_pattern[i] == '(') depth++;
else if (sub_pattern[i] == ')') depth--;
i++;
}
}
continue;
// lookaround, named group, inline flags, ...
throw unsupported_pattern("unsupported group syntax");
}
}
paren_depth++;
if (paren_depth > MAX_PATTERN_DEPTH) {
throw unsupported_pattern("pattern nesting too deep");
}
seq.emplace_back("(" + to_rule(transform()) + ")", false);
} else if (c == ')') {
i++;
if (start > 0 && sub_pattern[start - 1] != '(' && (start < 2 || sub_pattern[start - 2] != '?' || sub_pattern[start - 1] != ':')) {
_errors.push_back("Unbalanced parentheses");
if (paren_depth == 0) {
throw invalid_pattern("unbalanced parentheses");
}
paren_depth--;
return join_seq();
} else if (c == '^' || c == '$') {
throw unsupported_pattern("anchor inside the pattern");
} else if (c == '[') {
std::string square_brackets = std::string(1, c);
i++;
while (i < length && sub_pattern[i] != ']') {
if (sub_pattern[i] == '\\') {
square_brackets += sub_pattern.substr(i, 2);
i += 2;
auto escape_length = gbnf_escape_length(sub_pattern, i);
if (escape_length == 0) {
throw unsupported_pattern("unsupported escape in character class: " + sub_pattern.substr(i, 2));
}
square_brackets += sub_pattern.substr(i, escape_length);
i += escape_length;
} else {
square_brackets += sub_pattern[i];
i++;
}
}
if (i >= length) {
_errors.push_back("Unbalanced square brackets");
throw invalid_pattern("unterminated character class");
}
square_brackets += ']';
i++;
@ -465,6 +517,9 @@ private:
seq.emplace_back("|", false);
i++;
} else if (c == '*' || c == '+' || c == '?') {
if (seq.empty()) {
throw invalid_pattern("nothing to repeat");
}
seq.back() = std::make_pair(to_rule(seq.back()) + c, false);
i++;
} else if (c == '{') {
@ -475,18 +530,19 @@ private:
i++;
}
if (i >= length) {
_errors.push_back("Unbalanced curly brackets");
throw unsupported_pattern("unterminated curly brackets");
}
curly_brackets += '}';
i++;
auto nums = string_split(curly_brackets.substr(1, curly_brackets.length() - 2), ",");
int min_times = 0;
int max_times = std::numeric_limits<int>::max();
if (nums.size() != 1 && nums.size() != 2) {
throw unsupported_pattern("wrong number of values in curly brackets");
}
try {
if (nums.size() == 1) {
min_times = max_times = std::stoi(nums[0]);
} else if (nums.size() != 2) {
_errors.push_back("Wrong number of values in curly brackets");
} else {
if (!nums[0].empty()) {
min_times = std::stoi(nums[0]);
@ -495,9 +551,11 @@ private:
max_times = std::stoi(nums[1]);
}
}
} catch (const std::invalid_argument & e) {
_errors.push_back("Invalid number in curly brackets");
return std::make_pair("", false);
} catch (const std::logic_error &) {
throw unsupported_pattern("invalid number in curly brackets");
}
if (seq.empty()) {
throw invalid_pattern("nothing to repeat");
}
auto &last = seq.back();
auto &sub = last.first;
@ -523,15 +581,22 @@ private:
return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end();
};
while (i < length) {
if (sub_pattern[i] == '\\' && i < length - 1) {
if (sub_pattern[i] == '\\') {
if (i == length - 1) {
throw invalid_pattern("trailing backslash");
}
char next = sub_pattern[i + 1];
if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) {
i++;
literal += sub_pattern[i];
i++;
} else {
literal += sub_pattern.substr(i, 2);
i += 2;
auto escape_length = gbnf_escape_length(sub_pattern, i);
if (escape_length == 0) {
throw unsupported_pattern("unsupported escape: " + sub_pattern.substr(i, 2));
}
literal += sub_pattern.substr(i, escape_length);
i += escape_length;
}
} else if (sub_pattern[i] == '"') {
literal += "\\\"";
@ -544,14 +609,21 @@ private:
break;
}
}
if (!literal.empty()) {
seq.emplace_back(literal, true);
if (literal.empty()) { // nothing was consumed, ex. a stray ']' or '}'
throw unsupported_pattern(std::string("unsupported character: ") + c);
}
seq.emplace_back(literal, true);
}
}
return join_seq();
};
return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\"");
auto rule = to_rule(transform());
if (paren_depth != 0) {
throw invalid_pattern("unbalanced parentheses");
}
return _add_rule(name, "\"\\\"\" (" + rule + ") \"\\\"\"");
}
/*

4
common/speculative.cpp
View file

@ -2649,6 +2649,10 @@ void common_speculative_draft(common_speculative * spec) {
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) dparams.size(); ++seq_id) {
auto & dp = dparams[seq_id];
if (!dp.drafting) {
continue;
}
auto & result = *dp.result;
// a new draft has been sampled

1
conversion/__init__.py
View file

@ -57,6 +57,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Qwen3DSparkModel": "qwen",
"DSparkDraftModel": "qwen",
"DSparkSpeculator": "qwen",
"Lfm2DSparkDraftModel": "qwen",
"DeepseekV4ForCausalLM": "deepseek",
"DeepseekV4DSparkModel": "deepseek",
"DistilBertForMaskedLM": "bert",

9
conversion/nemotron.py
View file

@ -207,7 +207,9 @@ class NemotronHModel(GraniteHybridModel):
# calling the parent __init__. This is because the parent constructor
# uses self.model_arch to build the tensor name map, and all MoE-specific
# mappings would be missed if it were called with the default non-MoE arch.
hparams = ModelBase.load_hparams(args[0], self.is_mistral_format)
hparams = kwargs.pop("hparams", None)
if hparams is None:
hparams = ModelBase.load_hparams(args[0], self.is_mistral_format)
has_moe_params = (
"num_experts_per_tok" in hparams
or (isinstance(hparams.get("llm_config"), dict) and "num_experts_per_tok" in hparams["llm_config"])
@ -215,8 +217,11 @@ class NemotronHModel(GraniteHybridModel):
if has_moe_params:
self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE
self.is_moe = True
layers_block_type = hparams.get("layers_block_type")
if layers_block_type is not None:
hparams["num_hidden_layers"] = len(layers_block_type)
super().__init__(*args, **kwargs)
super().__init__(*args, hparams=hparams, **kwargs)
# Save the top-level head_dim for later
self.head_dim = self.hparams.get("head_dim", self.hparams.get("attention_head_dim"))

15
conversion/qwen.py
View file

@ -709,7 +709,7 @@ class DFlashModel(Qwen3Model):
yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator")
@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator", "Lfm2DSparkDraftModel")
@ModelBase.example("satgeze/Qwen3.6-27B-DSpark")
class DSparkModel(DFlashModel):
# DSpark = DFlash + a semi-autoregressive Markov head.
@ -759,6 +759,13 @@ class DSparkModel(DFlashModel):
return None
return super().filter_tensors(item)
_ROPE_PERMUTE_SUFFIXES = (
"self_attn.q_proj.weight",
"self_attn.k_proj.weight",
"self_attn.q_norm.weight",
"self_attn.k_norm.weight",
)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name == "model.d2t":
self._d2t = data_torch
@ -767,6 +774,12 @@ class DSparkModel(DFlashModel):
if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")):
return
# interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd
if not self.hparams.get("rope_is_neox_style", True) and name.endswith(self._ROPE_PERMUTE_SUFFIXES):
head_dim = self.hparams["head_dim"]
shape = data_torch.shape
data_torch = data_torch.reshape(-1, head_dim // 2, 2, *shape[1:]).transpose(1, 2).reshape(shape)
yield from super().modify_tensors(data_torch, name, bid)
def prepare_tensors(self):

1
ggml/src/ggml-backend-impl.h
View file

@ -83,7 +83,6 @@ extern "C" {
GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
GGML_API void ggml_backend_meta_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
//
// Backend (meta)

20
ggml/src/ggml-backend-meta.cpp
View file

@ -1118,6 +1118,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
}
static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync) {
GGML_ASSERT(ggml_backend_buffer_is_meta(tensor->buffer));
ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context;
return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync);
}
@ -1177,15 +1178,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m
t_ij->flags = tensor->flags;
memcpy(t_ij->op_params, tensor->op_params, sizeof(tensor->op_params));
ggml_set_name(t_ij, tensor->name);
t_ij->buffer = simple_buf;
if (simple_buf) {
// the backend that owns the buffer will set .extra
ggml_backend_buffer_init_tensor(simple_buf, t_ij);
} else {
t_ij->extra = tensor->extra;
}
t_ij->view_src = tensor->view_src;
t_ij->view_offs = tensor->view_offs;
if (t_ij->view_src != nullptr && ggml_backend_buffer_is_meta(t_ij->view_src->buffer)) {
@ -1216,6 +1209,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m
t_ij->data = (char *) ggml_backend_buffer_get_base(simple_buf)
+ size_t(tensor->data) - size_t(ggml_backend_buffer_get_base(tensor->buffer));
}
t_ij->extra = tensor->extra;
for (int i = 0; i < GGML_MAX_SRC; i++) {
t_ij->src[i] = tensor->src[i];
if (tensor->src[i] == tensor) {
@ -1508,16 +1502,6 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) {
return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer;
}
void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {
GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));
ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;
for (size_t i = 0; i < buf_ctx->bufs.size(); i++) {
if (buf_ctx->bufs[i]) {
ggml_backend_buffer_set_usage(buf_ctx->bufs[i].get(), usage);
}
}
}
static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);

24
ggml/src/ggml-backend.cpp
View file

@ -182,8 +182,6 @@ void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backe
// FIXME: add a generic callback to the buffer interface
if (ggml_backend_buffer_is_multi_buffer(buffer)) {
ggml_backend_multi_buffer_set_usage(buffer, usage);
} else if (ggml_backend_buffer_is_meta(buffer)) {
ggml_backend_meta_buffer_set_usage(buffer, usage);
}
}
@ -1608,11 +1606,23 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
std::vector<int32_t> ids;
std::vector<ggml_bitset_t> used_ids;
int prev_backend_id = -1;
for (int split_id = 0; split_id < sched->n_splits; split_id++) {
struct ggml_backend_sched_split * split = &splits[split_id];
int split_backend_id = split->backend_id;
ggml_backend_t split_backend = sched->backends[split_backend_id];
// ensure the previous split's async work has completed before we start
// this split, the allocator may have reused buffer regions across splits
if (split->n_inputs == 0 && prev_backend_id >= 0 && prev_backend_id != split_backend_id) {
if (sched->events[prev_backend_id][sched->cur_copy] != NULL) {
ggml_backend_event_synchronize(sched->events[prev_backend_id][sched->cur_copy]);
} else {
ggml_backend_synchronize(sched->backends[prev_backend_id]);
}
}
// copy the input tensors to the split backend
for (int input_id = 0; input_id < split->n_inputs; input_id++) {
ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]);
@ -1775,12 +1785,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
}
}
// record the event of this copy
if (split->n_inputs > 0) {
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend);
}
// record the event of this split
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend);
}
prev_backend_id = split_backend_id;
}
return GGML_STATUS_SUCCESS;

29
ggml/src/ggml-cuda/common.cuh
View file

@ -1425,7 +1425,9 @@ struct ggml_backend_cuda_context {
cudaEvent_t copy_event = nullptr;
cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } };
cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};
cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr};
void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr};
size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {0};
int curr_stream_no = 0;
@ -1502,17 +1504,22 @@ struct ggml_backend_cuda_context {
ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; }
cublasHandle_t cublas_handle(int device) {
if (cublas_handles[device] == nullptr) {
ggml_cuda_set_device(device);
CUBLAS_CHECK(cublasCreate(&cublas_handles[device]));
CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH));
}
return cublas_handles[device];
}
cublasHandle_t cublas_handle() {
return cublas_handle(device);
if (cublas_handles[device][curr_stream_no] == nullptr) {
ggml_cuda_set_device(device);
CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no]));
CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH));
CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream()));
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2))
if (cublas_workspace_sizes[device] == 0) {
const int cc = ggml_cuda_info().devices[device].cc;
cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024;
}
CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device]));
CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device]));
#endif
}
return cublas_handles[device][curr_stream_no];
}
// pool

19
ggml/src/ggml-cuda/ggml-cuda.cu
View file

@ -711,9 +711,12 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() {
if (streams[i][j] != nullptr) {
CUDA_CHECK(cudaStreamDestroy(streams[i][j]));
}
}
if (cublas_handles[i] != nullptr) {
CUBLAS_CHECK(cublasDestroy(cublas_handles[i]));
if (cublas_handles[i][j] != nullptr) {
CUBLAS_CHECK(cublasDestroy(cublas_handles[i][j]));
}
if (cublas_workspaces[i][j] != nullptr) {
CUDA_CHECK(cudaFree(cublas_workspaces[i][j]));
}
}
}
}
@ -1416,7 +1419,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const
const int64_t ne_dst = ggml_nelements(dst);
cudaStream_t main_stream = ctx.stream();
CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream));
cublasHandle_t cublas_h = ctx.cublas_handle();
const size_t src0_ts = ggml_type_size(src0->type);
GGML_ASSERT(nb00 == src0_ts);
@ -1539,14 +1542,14 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const
// probably because the internal kernel selection logic is suboptimal.
if (compute_type == GGML_TYPE_F32 && ne12 == 1 && ne13 == 1) {
CUBLAS_CHECK(
cublasSgemm(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,
cublasSgemm(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N,
ne01, ne11, ne10,
(const float *) alpha, (const float *) src0_ptr, s01,
(const float *) src1_ptr, s11,
(const float *) beta, (float *) dst_ptr, ne0));
} else if (ne12 == 1 && ne13 == 1) {
CUBLAS_CHECK(
cublasGemmEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,
cublasGemmEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N,
ne01, ne11, ne10,
alpha, src0_ptr, cu_data_type_a, s01,
src1_ptr, cu_data_type_b, s11,
@ -1561,7 +1564,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const
// there is no broadcast and src0, src1 are contiguous across dims 2, 3
// use cublasGemmStridedBatchedEx
CUBLAS_CHECK(
cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,
cublasGemmStridedBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N,
ne01, ne11, ne10,
alpha, src0_ptr, cu_data_type_a, s01, sma, // strideA
src1_ptr, cu_data_type_b, s11, smb, // strideB
@ -1599,7 +1602,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const
CUDA_CHECK(cudaGetLastError());
CUBLAS_CHECK(
cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,
cublasGemmBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N,
ne01, ne11, ne10,
alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, s01,
(const void **) (ptrs_src.get() + 1*ne23), cu_data_type_b, s11,

36
ggml/src/ggml-cuda/mmvq.cu
View file

@ -290,6 +290,42 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
if (!ggml_is_quantized(type)) {
return false;
}
// k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner.
// Only list quant-types MMQ supports, others would fall back to cuBLAS.
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) {
switch (type) { // tuned on RTX 4090
case GGML_TYPE_Q2_K:
return ne11 <= 4;
case GGML_TYPE_Q3_K:
return ne11 <= 6;
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q5_K:
return ne11 <= 7;
default:
return ne11 <= MMVQ_MAX_BATCH_SIZE;
}
}
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) {
switch (type) { // tuned on RTX 5090
case GGML_TYPE_Q2_K:
case GGML_TYPE_Q3_K:
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q5_K:
return ne11 <= 5;
case GGML_TYPE_Q6_K:
return ne11 <= 7;
default:
return ne11 <= MMVQ_MAX_BATCH_SIZE;
}
}
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
switch (type) { // tuned on DGX Spark GB10
case GGML_TYPE_Q2_K:
return ne11 <= 6;
default:
return ne11 <= MMVQ_MAX_BATCH_SIZE;
}
}
if (GGML_CUDA_CC_IS_CDNA(cc)) {
if (GGML_CUDA_CC_IS_CDNA1(cc)) {
switch (type) {

2
ggml/src/ggml-cuda/out-prod.cu
View file

@ -54,8 +54,6 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const float alpha = 1.0f;
const float beta = 0.0f;
CUBLAS_CHECK(cublasSetStream(handle, stream));
const int64_t lda = nb01 / sizeof(float);
const int64_t ldc = nb1 / sizeof(float);

8
ggml/src/ggml-cuda/solve_tri.cu
View file

@ -65,15 +65,13 @@ static void solve_tri_f32_cublas(ggml_backend_cuda_context & ctx,
get_batch_pointers<<<(total_batches + 255) / 256, 256, 0, stream>>>(A, X, A_ptrs_dev, X_ptrs_dev, ne02,
total_batches, s02, s03, s2, s3);
CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));
// Yes, this is necessary, without this we get RMSE errors
CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_DEFAULT_MATH));
CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(id), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N,
CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_DEFAULT_MATH));
CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N,
CUBLAS_DIAG_NON_UNIT, k, n, &alpha, A_ptrs_dev, n, X_ptrs_dev, k, total_batches));
// revert to standard mode from common.cuh
CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_TF32_TENSOR_OP_MATH));
CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_TF32_TENSOR_OP_MATH));
GGML_UNUSED_VARS(s12, s13);
}

1
ggml/src/ggml-cuda/ssm-scan.cu
View file

@ -632,7 +632,6 @@ static void ssm_scan_ssd_f32_cuda(
// Step 3: chunked SSD loop
// Per chunk: pre_matmul (incl. M) + 4 cuBLAS (CB, Y, S@C, state update) + scale_state
cublasHandle_t handle = ctx.cublas_handle();
CUBLAS_CHECK(cublasSetStream(handle, stream));
const float alpha_one = 1.0f;
const float beta_zero = 0.0f;
const float beta_one = 1.0f;

37
ggml/src/ggml-metal/ggml-metal-device.cpp
View file

@ -1409,6 +1409,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_p
return res;
}
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(
ggml_metal_library_t lib,
const ggml_tensor * op) {
assert(op->op == GGML_OP_FLASH_ATTN_EXT);
char base[256];
snprintf(base, 256, "kernel_flash_attn_ext_kv_%s_f16", ggml_type_name(op->src[1]->type));
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, base);
if (!res.pipeline) {
res = ggml_metal_library_compile_pipeline(lib, base, base, nullptr);
}
return res;
}
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk(
ggml_metal_library_t lib,
const struct ggml_tensor * op,
@ -1460,7 +1477,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(
bool has_bias,
bool has_scap,
bool has_kvpad,
int32_t nsg) {
int32_t nsg,
bool use_kv_f16,
int32_t ns10,
int32_t ns20) {
assert(op->op == GGML_OP_FLASH_ATTN_EXT);
char base[256];
@ -1469,15 +1489,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(
const int32_t dk = (int32_t) op->src[1]->ne[0];
const int32_t dv = (int32_t) op->src[2]->ne[0];
const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0];
const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0];
const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type);
// do bounds checks for the mask?
const bool bc_mask = op->src[3] && (op->src[3]->ne[1] % 8 != 0);
snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d",
"flash_attn_ext",
ggml_type_name(op->src[1]->type),
type,
dk,
dv);
@ -1526,7 +1545,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
bool has_scap,
bool has_kvpad,
int32_t nsg,
int32_t nwg) {
int32_t nwg,
bool use_kv_f16,
int32_t ns10,
int32_t ns20) {
assert(op->op == GGML_OP_FLASH_ATTN_EXT);
char base[256];
@ -1535,12 +1557,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
const int32_t dk = (int32_t) op->src[1]->ne[0];
const int32_t dv = (int32_t) op->src[2]->ne[0];
const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0];
const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0];
const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type);
snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d",
"flash_attn_ext_vec",
ggml_type_name(op->src[1]->type),
type,
dk,
dv);

14
ggml/src/ggml-metal/ggml-metal-device.h
View file

@ -176,6 +176,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att
bool has_mask,
int32_t ncpsg);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(
ggml_metal_library_t lib,
const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk(
ggml_metal_library_t lib,
const struct ggml_tensor * op,
@ -190,7 +194,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att
bool has_bias,
bool has_scap,
bool has_kvpad,
int32_t nsg);
int32_t nsg,
bool use_kv_f16,
int32_t ns10,
int32_t ns20);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec(
ggml_metal_library_t lib,
@ -201,7 +208,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att
bool has_scap,
bool has_kvpad,
int32_t nsg,
int32_t nwg);
int32_t nwg,
bool use_kv_f16,
int32_t ns10,
int32_t ns20);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce(
ggml_metal_library_t lib,

12
ggml/src/ggml-metal/ggml-metal-impl.h
View file

@ -345,6 +345,18 @@ typedef struct {
bool inplace;
} ggml_metal_kargs_rope;
typedef struct {
int32_t ne0;
int32_t ne1;
int32_t ne2;
int32_t ne3;
uint64_t nb0;
uint64_t nb1;
uint64_t nb2;
uint64_t nb3;
int32_t nblocks;
} ggml_metal_kargs_flash_attn_ext_kv_f16;
typedef struct {
int32_t ne11;
int32_t ne_12_2; // assume K and V are same shape

277
ggml/src/ggml-metal/ggml-metal-ops.cpp
View file

@ -2801,6 +2801,51 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) {
return (ne01 < 20) && (ne00 % 32 == 0);
}
// ref: https://github.com/ggml-org/llama.cpp/pull/27390
// dequantize the quantized KV cache to F16 before running the F16 flash attention kernels
static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) {
assert(op->op == GGML_OP_FLASH_ATTN_EXT);
// depending on compute/bandwidth ratio, dequant to f16 kv is not always beneficial
// ref: https://github.com/ggml-org/llama.cpp/pull/27390#issuecomment-5355152767
// TODO: tune per device
if (op->src[0]->ne[1] < 32) {
return false;
}
switch (op->src[1]->type) {
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
return true;
default:
return false;
}
}
// in some models (e.g. MLA-based), V is a view of K (the first ne20 elements of each K row);
// the dequantized V is then a view of the dequantized K and does not need its own dequant or scratch
// - ref: https://github.com/ggml-org/llama.cpp/pull/13435
static bool ggml_metal_op_flash_attn_ext_v_is_view_of_k(const ggml_tensor * op) {
assert(op->op == GGML_OP_FLASH_ATTN_EXT);
const ggml_tensor * K = op->src[1];
const ggml_tensor * V = op->src[2];
return V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs));
}
// size of the F16 dequantized K tensor; the dequantized V tensor follows it in the same scratch buffer
static size_t ggml_metal_op_flash_attn_ext_kv_f16_k_size(const ggml_tensor * op) {
assert(op->op == GGML_OP_FLASH_ATTN_EXT);
GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);
return GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*ne11*ne12*ne13, 16);
}
size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) {
assert(op->op == GGML_OP_FLASH_ATTN_EXT);
@ -2816,6 +2861,18 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) {
size_t res = 0;
const bool has_mask = op->src[3] != nullptr;
const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op);
// when the KV is dequantized to F16, the pad kernel copies the tail chunk from the F16 scratch buffer
// note: when V is a view of K, the dequantized V is read from the dequantized K with K's row stride
const bool v_is_view_of_k = use_kv_f16 && ggml_metal_op_flash_attn_ext_v_is_view_of_k(op);
uint64_t nb11_pad = nb11;
uint64_t nb21_pad = nb21;
if (use_kv_f16) {
nb11_pad = sizeof(ggml_fp16_t)*ne10;
nb21_pad = sizeof(ggml_fp16_t)*(v_is_view_of_k ? ne10 : ne20);
}
// note: the non-vec kernel requires more extra memory, so always reserve for it
GGML_ASSERT(OP_FLASH_ATTN_EXT_NCPSG >= OP_FLASH_ATTN_EXT_VEC_NCPSG);
@ -2828,8 +2885,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) {
if (has_kvpad) {
res += OP_FLASH_ATTN_EXT_VEC_NCPSG*(
nb11*ne12*ne13 +
nb21*ne22*ne23 +
nb11_pad*ne12*ne13 +
nb21_pad*ne22*ne23 +
(has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0));
}
} else {
@ -2838,8 +2895,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) {
if (has_kvpad) {
res += OP_FLASH_ATTN_EXT_NCPSG*(
nb11*ne12*ne13 +
nb21*ne22*ne23 +
nb11_pad*ne12*ne13 +
nb21_pad*ne22*ne23 +
(has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0));
}
}
@ -2915,6 +2972,29 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) {
return res;
}
size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) {
assert(op->op == GGML_OP_FLASH_ATTN_EXT);
// note: always reserve the temp buffer to avoid graph reallocations
//if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {
// return 0;
//}
GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);
const size_t k_size = ggml_metal_op_flash_attn_ext_kv_f16_k_size(op);
// when V is a view of K, the dequantized V is a view of the dequantized K
const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op);
if (v_is_view_of_k) {
return k_size;
}
const size_t v_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne20*ne21*ne22*ne23, 16);
return k_size + v_size;
}
int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
ggml_tensor * op = ctx->node(idx);
@ -2989,6 +3069,111 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
ggml_metal_buffer_id bid_tmp = bid_blk;
bid_tmp.offs += ggml_metal_op_flash_attn_ext_extra_blk(op);
ggml_metal_buffer_id bid_kv_f16 = bid_tmp;
bid_kv_f16.offs += ggml_metal_op_flash_attn_ext_extra_tmp(op);
const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op);
ggml_metal_buffer_id bid_k = bid_src1;
ggml_metal_buffer_id bid_v = bid_src2;
uint64_t nb10_attn = nb10;
uint64_t nb11_attn = nb11;
uint64_t nb12_attn = nb12;
uint64_t nb13_attn = nb13;
uint64_t nb20_attn = nb20;
uint64_t nb21_attn = nb21;
uint64_t nb22_attn = nb22;
uint64_t nb23_attn = nb23;
if (use_kv_f16) {
assert(ggml_metal_op_flash_attn_ext_extra_kv_f16(op) != 0);
const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op);
const int64_t nblocks1_64 = (ne10/ggml_blck_size(op->src[1]->type))*(int64_t) ne11*ne12*ne13;
GGML_ASSERT(nblocks1_64 <= INT32_MAX);
const int32_t nblocks1 = nblocks1_64;
ggml_metal_buffer_id bid_v_f16 = bid_kv_f16;
bid_v_f16.offs += ggml_metal_op_flash_attn_ext_kv_f16_k_size(op);
auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(lib, op);
const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 256);
// K
ggml_metal_kargs_flash_attn_ext_kv_f16 args_k = {
/*.ne0 =*/ ne10,
/*.ne1 =*/ ne11,
/*.ne2 =*/ ne12,
/*.ne3 =*/ ne13,
/*.nb0 =*/ nb10,
/*.nb1 =*/ nb11,
/*.nb2 =*/ nb12,
/*.nb3 =*/ nb13,
/*.nblocks =*/ nblocks1,
};
ggml_metal_encoder_set_pipeline(enc, pipeline0);
ggml_metal_encoder_set_bytes (enc, &args_k, sizeof(args_k), 0);
ggml_metal_encoder_set_buffer (enc, bid_src1, 1);
ggml_metal_encoder_set_buffer (enc, bid_kv_f16, 2);
ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks1 + nth - 1)/nth, 1, 1, nth, 1, 1);
// V (skip when V is a view of K: the dequantized V is a view of the dequantized K)
if (!v_is_view_of_k) {
const int64_t nblocks2_64 = (ne20/ggml_blck_size(op->src[2]->type))*(int64_t) ne21*ne22*ne23;
GGML_ASSERT(nblocks2_64 <= INT32_MAX);
const int32_t nblocks2 = nblocks2_64;
ggml_metal_kargs_flash_attn_ext_kv_f16 args_v = {
/*.ne0 =*/ ne20,
/*.ne1 =*/ ne21,
/*.ne2 =*/ ne22,
/*.ne3 =*/ ne23,
/*.nb0 =*/ nb20,
/*.nb1 =*/ nb21,
/*.nb2 =*/ nb22,
/*.nb3 =*/ nb23,
/*.nblocks =*/ nblocks2,
};
ggml_metal_encoder_set_pipeline(enc, pipeline0);
ggml_metal_encoder_set_bytes (enc, &args_v, sizeof(args_v), 0);
ggml_metal_encoder_set_buffer (enc, bid_src2, 1);
ggml_metal_encoder_set_buffer (enc, bid_v_f16, 2);
ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks2 + nth - 1)/nth, 1, 1, nth, 1, 1);
}
// the pad and attention kernels read the dequantized KV
ggml_metal_op_concurrency_reset(ctx);
bid_k = bid_kv_f16;
bid_v = v_is_view_of_k ? bid_k : bid_v_f16;
// contiguous F16 layout of the dequantized K
nb10_attn = sizeof(ggml_fp16_t);
nb11_attn = nb10_attn*ne10;
nb12_attn = nb11_attn*ne11;
nb13_attn = nb12_attn*ne12;
// if V is a view of K, the dequantized V is read from the dequantized K with K's strides
if (v_is_view_of_k) {
nb20_attn = nb10_attn;
nb21_attn = nb11_attn;
nb22_attn = nb12_attn;
nb23_attn = nb13_attn;
} else {
// contiguous F16 layout of the dequantized V
nb20_attn = sizeof(ggml_fp16_t);
nb21_attn = nb20_attn*ne20;
nb22_attn = nb21_attn*ne21;
nb23_attn = nb22_attn*ne22;
}
}
if (!ggml_metal_op_flash_attn_ext_use_vec(op)) {
// half8x8 kernel
const int nqptg = OP_FLASH_ATTN_EXT_NQPSG; // queries per threadgroup
@ -3009,12 +3194,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
/*.ne11 =*/ne11,
/*.ne_12_2 =*/ne12,
/*.ne_12_3 =*/ne13,
/*.nb11 =*/nb11,
/*.nb12 =*/nb12,
/*.nb13 =*/nb13,
/*.nb21 =*/nb21,
/*.nb22 =*/nb22,
/*.nb23 =*/nb23,
/*.nb11 =*/nb11_attn,
/*.nb12 =*/nb12_attn,
/*.nb13 =*/nb13_attn,
/*.nb21 =*/nb21_attn,
/*.nb22 =*/nb22_attn,
/*.nb23 =*/nb23_attn,
/*.ne31 =*/ne31,
/*.ne32 =*/ne32,
/*.ne33 =*/ne33,
@ -3027,8 +3212,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
ggml_metal_encoder_set_pipeline(enc, pipeline0);
ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);
ggml_metal_encoder_set_buffer (enc, bid_src1, 1);
ggml_metal_encoder_set_buffer (enc, bid_src2, 2);
ggml_metal_encoder_set_buffer (enc, bid_k, 1);
ggml_metal_encoder_set_buffer (enc, bid_v, 2);
ggml_metal_encoder_set_buffer (enc, bid_src3, 3);
ggml_metal_encoder_set_buffer (enc, bid_pad, 4);
@ -3073,7 +3258,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
ggml_metal_op_concurrency_reset(ctx);
}
const int is_q = ggml_is_quantized(op->src[1]->type) ? 1 : 0;
const int is_q = !use_kv_f16 && ggml_is_quantized(op->src[1]->type) ? 1 : 0;
// 2*(2*ncpsg)
// ncpsg soft_max values + ncpsg mask values
@ -3104,6 +3289,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
const size_t smem = FATTN_SMEM(nsg);
const int32_t ns10 = nb11_attn/nb10_attn;
const int32_t ns20 = nb21_attn/nb20_attn;
ggml_metal_kargs_flash_attn_ext args = {
/*.ne01 =*/ ne01,
/*.ne02 =*/ ne02,
@ -3114,14 +3302,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
/*.ne11 =*/ ne11,
/*.ne_12_2 =*/ ne12,
/*.ne_12_3 =*/ ne13,
/*.ns10 =*/ int32_t(nb11/nb10),
/*.nb11 =*/ nb11,
/*.nb12 =*/ nb12,
/*.nb13 =*/ nb13,
/*.ns20 =*/ int32_t(nb21/nb20),
/*.nb21 =*/ nb21,
/*.nb22 =*/ nb22,
/*.nb23 =*/ nb23,
/*.ns10 =*/ ns10,
/*.nb11 =*/ nb11_attn,
/*.nb12 =*/ nb12_attn,
/*.nb13 =*/ nb13_attn,
/*.ns20 =*/ ns20,
/*.nb21 =*/ nb21_attn,
/*.nb22 =*/ nb22_attn,
/*.nb23 =*/ nb23_attn,
/*.ne31 =*/ ne31,
/*.ne32 =*/ ne32,
/*.ne33 =*/ ne33,
@ -3139,13 +3327,13 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
/*.logit_softcap =*/ logit_softcap,
};
auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg);
auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, use_kv_f16, ns10, ns20);
ggml_metal_encoder_set_pipeline(enc, pipeline);
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
ggml_metal_encoder_set_buffer (enc, bid_src0, 1);
ggml_metal_encoder_set_buffer (enc, bid_src1, 2);
ggml_metal_encoder_set_buffer (enc, bid_src2, 3);
ggml_metal_encoder_set_buffer (enc, bid_k, 2);
ggml_metal_encoder_set_buffer (enc, bid_v, 3);
ggml_metal_encoder_set_buffer (enc, bid_src3, 4);
ggml_metal_encoder_set_buffer (enc, bid_src4, 5);
ggml_metal_encoder_set_buffer (enc, bid_pad, 6);
@ -3177,12 +3365,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
/*.ne11 =*/ne11,
/*.ne_12_2 =*/ne12,
/*.ne_12_3 =*/ne13,
/*.nb11 =*/nb11,
/*.nb12 =*/nb12,
/*.nb13 =*/nb13,
/*.nb21 =*/nb21,
/*.nb22 =*/nb22,
/*.nb23 =*/nb23,
/*.nb11 =*/nb11_attn,
/*.nb12 =*/nb12_attn,
/*.nb13 =*/nb13_attn,
/*.nb21 =*/nb21_attn,
/*.nb22 =*/nb22_attn,
/*.nb23 =*/nb23_attn,
/*.ne31 =*/ne31,
/*.ne32 =*/ne32,
/*.ne33 =*/ne33,
@ -3195,8 +3383,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
ggml_metal_encoder_set_pipeline(enc, pipeline0);
ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);
ggml_metal_encoder_set_buffer (enc, bid_src1, 1);
ggml_metal_encoder_set_buffer (enc, bid_src2, 2);
ggml_metal_encoder_set_buffer (enc, bid_k, 1);
ggml_metal_encoder_set_buffer (enc, bid_v, 2);
ggml_metal_encoder_set_buffer (enc, bid_src3, 3);
ggml_metal_encoder_set_buffer (enc, bid_pad, 4);
@ -3242,6 +3430,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
}
}
const int32_t ns10 = nb11_attn/nb10_attn;
const int32_t ns20 = nb21_attn/nb20_attn;
ggml_metal_kargs_flash_attn_ext_vec args = {
/*.ne01 =*/ ne01,
/*.ne02 =*/ ne02,
@ -3252,14 +3443,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
/*.ne11 =*/ ne11,
/*.ne_12_2 =*/ ne12,
/*.ne_12_3 =*/ ne13,
/*.ns10 =*/ int32_t(nb11/nb10),
/*.nb11 =*/ nb11,
/*.nb12 =*/ nb12,
/*.nb13 =*/ nb13,
/*.ns20 =*/ int32_t(nb21/nb20),
/*.nb21 =*/ nb21,
/*.nb22 =*/ nb22,
/*.nb23 =*/ nb23,
/*.ns10 =*/ ns10,
/*.nb11 =*/ nb11_attn,
/*.nb12 =*/ nb12_attn,
/*.nb13 =*/ nb13_attn,
/*.ns20 =*/ ns20,
/*.nb21 =*/ nb21_attn,
/*.nb22 =*/ nb22_attn,
/*.nb23 =*/ nb23_attn,
/*.ne31 =*/ ne31,
/*.ne32 =*/ ne32,
/*.ne33 =*/ ne33,
@ -3277,15 +3468,15 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
/*.logit_softcap =*/ logit_softcap,
};
auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg);
auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg, use_kv_f16, ns10, ns20);
GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
ggml_metal_encoder_set_pipeline(enc, pipeline);
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
ggml_metal_encoder_set_buffer (enc, bid_src0, 1);
ggml_metal_encoder_set_buffer (enc, bid_src1, 2);
ggml_metal_encoder_set_buffer (enc, bid_src2, 3);
ggml_metal_encoder_set_buffer (enc, bid_k, 2);
ggml_metal_encoder_set_buffer (enc, bid_v, 3);
ggml_metal_encoder_set_buffer (enc, bid_src3, 4);
ggml_metal_encoder_set_buffer (enc, bid_src4, 5);

1
ggml/src/ggml-metal/ggml-metal-ops.h
View file

@ -42,6 +42,7 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op);
size_t ggml_metal_op_flash_attn_ext_extra_pad(const struct ggml_tensor * op);
size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op);
size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op);
size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op);
int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);

1
ggml/src/ggml-metal/ggml-metal.cpp
View file

@ -225,6 +225,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
res += ggml_metal_op_flash_attn_ext_extra_pad(tensor);
res += ggml_metal_op_flash_attn_ext_extra_blk(tensor);
res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor);
res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor);
} break;
case GGML_OP_CUMSUM:
case GGML_OP_ARGSORT:

62
ggml/src/ggml-metal/ggml-metal.metal
View file

@ -6318,6 +6318,53 @@ template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_t kernel_fwht_f
template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_t kernel_fwht_f32<256>;
template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_t kernel_fwht_f32<512>;
// dequantize a quantized KV cache tensor to contiguous F16 before running the F16 flash attention kernels
// - one thread per block; dispatched separately for K and V
// - ref: https://github.com/ggml-org/llama.cpp/pull/27390
template <
typename block_t,
short QK,
void (*deq_t4x4)(device const block_t *, short, thread float4x4 &)>
kernel void kernel_flash_attn_ext_kv_f16(
constant ggml_metal_kargs_flash_attn_ext_kv_f16 & args,
device const char * x,
device half * x_dst,
uint gid [[thread_position_in_grid]]) {
if (gid >= (uint) args.nblocks) {
return;
}
const uint nb = args.ne0/QK;
const uint i0 = gid%nb;
uint ib = gid/nb;
const uint i1 = ib%args.ne1;
ib /= args.ne1;
const uint i2 = ib%args.ne2;
const uint i3 = ib/args.ne2;
const uint64_t offs = i0*args.nb0 + i1*args.nb1 + i2*args.nb2 + i3*args.nb3;
device const block_t * src = (device const block_t *) (x + offs);
device half4 * dst = (device half4 *) x_dst + (QK/4)*gid;
for (short i = 0; i < QK/16; ++i) {
float4x4 reg;
deq_t4x4(src, i, reg);
dst[4*i + 0] = (half4) reg[0];
dst[4*i + 1] = (half4) reg[1];
dst[4*i + 2] = (half4) reg[2];
dst[4*i + 3] = (half4) reg[3];
}
}
typedef decltype(kernel_flash_attn_ext_kv_f16<block_q8_0, 32, dequantize_q8_0>) kernel_flash_attn_ext_kv_f16_t;
template [[host_name("kernel_flash_attn_ext_kv_q4_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q4_0, 32, dequantize_q4_0>;
template [[host_name("kernel_flash_attn_ext_kv_q4_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q4_1, 32, dequantize_q4_1>;
template [[host_name("kernel_flash_attn_ext_kv_q5_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q5_0, 32, dequantize_q5_0>;
template [[host_name("kernel_flash_attn_ext_kv_q5_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q5_1, 32, dequantize_q5_1>;
template [[host_name("kernel_flash_attn_ext_kv_q8_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q8_0, 32, dequantize_q8_0>;
constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]];
constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]];
@ -10318,9 +10365,12 @@ kernel void kernel_mul_mm(
auto tB = tensor(ptrB, dextents<int32_t, 2>(K, N), array<int, 2>({1, strideB}));
// Configure matmul operation
// note: K is dynamic_extent (clamped to the valid range in PHASE 2), since a static
// N_MM_NK_TOTAL K tile would read src1 out of bounds when K % N_MM_NK_TOTAL != 0
// ref: https://github.com/ggml-org/llama.cpp/pull/27064
mpp::tensor_ops::matmul2d<
mpp::tensor_ops::matmul2d_descriptor(
NRB, NRA, N_MM_NK_TOTAL, false, true, true,
NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,
mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;
@ -10372,10 +10422,14 @@ kernel void kernel_mul_mm(
threadgroup_barrier(mem_flags::mem_threadgroup);
// === PHASE 2: Tensor matmul ===
auto mA = tA.slice(0, 0);
auto mB = tB.slice(loop_k, rb);
// Clamp the K extent of both operand tensors to the remaining valid K range so
// the dynamic-K op never reads past the K extent of src1 (or the staged A tile).
const int kExt = min(N_MM_NK_TOTAL, K - loop_k);
mm.run(mB, mA, cT);
auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, N_MM_NK_TOTAL}));
auto tBv = tensor(ptrB + loop_k + rb * strideB, dextents<int32_t, 2>(kExt, N - rb), array<int, 2>({1, strideB}));
mm.run(tBv, tAv, cT);
threadgroup_barrier(mem_flags::mem_threadgroup);
}

18
ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp
View file

@ -121,13 +121,13 @@ void main() {
const uint buf_ib = r * qf_stride + d / 8;
const uint buf_iqs = d % 8;
FLOAT_TYPEV4 vals = is_in_bounds ? FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale) : FLOAT_TYPEV4(0.0f);
const FLOAT_TYPEV4 abs_vals = abs(vals);
vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale : vec4(0.0f);
const vec4 abs_vals = abs(vals);
const FLOAT_TYPE thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w));
const FLOAT_TYPE amax = subgroupClusteredMax(thread_max, 8);
const FLOAT_TYPE qd = amax / FLOAT_TYPE(127.0);
const FLOAT_TYPE qd_inv = qd != FLOAT_TYPE(0.0) ? FLOAT_TYPE(1.0) / qd : FLOAT_TYPE(0.0);
const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w));
const float amax = subgroupClusteredMax(thread_max, 8);
const float qd = amax / 127.0f;
const float qd_inv = qd != 0.0f ? 1.0f / qd : 0.0f;
vals = round(vals * qd_inv);
Qf[buf_ib].qs[buf_iqs] = pack32(i8vec4(vals));
@ -136,11 +136,11 @@ void main() {
// the row-sum scaled by qd, used in k_dot_correction.
if (FaTypeK == FA_TYPE_Q8_0) {
if (buf_iqs == 0) {
Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0);
Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f);
}
} else {
const FLOAT_TYPE thread_sum = vals.x + vals.y + vals.z + vals.w;
const FLOAT_TYPE sum = subgroupClusteredAdd(thread_sum, 8);
const float thread_sum = vals.x + vals.y + vals.z + vals.w;
const float sum = subgroupClusteredAdd(thread_sum, 8);
if (buf_iqs == 0) {
Qf[buf_ib].ds = FLOAT_TYPEV2(qd, sum * qd);

4
src/llama-arch.cpp
View file

@ -1032,6 +1032,8 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_LFM2:
case LLM_ARCH_LFM2MOE:
return true;
default:
return false;
@ -1060,8 +1062,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_GRANITE_HYBRID:
case LLM_ARCH_LFM2:
case LLM_ARCH_LFM2MOE:
case LLM_ARCH_MINIMAX_01:
case LLM_ARCH_MINIMAX_M2:
case LLM_ARCH_MINIMAX_M3:

4
src/llama-graph.cpp
View file

@ -3100,8 +3100,6 @@ ggml_tensor * llm_graph_context::build_attn(
int il) const {
const bool is_swa = hparams.is_swa(il);
GGML_UNUSED(v_cur);
auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot;
if (k_rot) {
@ -3134,7 +3132,7 @@ ggml_tensor * llm_graph_context::build_attn(
// MLA-style attention: the cached K is used as V
ggml_tensor * q = q_cur;
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
ggml_tensor * v = k;
ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
cb(cur, "kqv_out", il);

4
src/llama-model.cpp
View file

@ -637,6 +637,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");
}
if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {
if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) {
// the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, "");
}
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");
}
if (std::regex_match(tensor_name, pattern_ssm_conv1d)) {

40
src/models/deepseek2.cpp
View file

@ -524,17 +524,9 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
cb(q, "q", il);
}
// split into {n_embd_head_qk_nope, n_head, n_tokens}
ggml_tensor * q_nope =
ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),
ggml_row_size(q->type, n_embd_head_k) * n_head, 0);
cb(q_nope, "q_nope", il);
// and {n_embd_head_qk_rope, n_head, n_tokens}
ggml_tensor * q_pe = ggml_view_3d(
ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),
ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope));
cb(q_pe, "q_pe", il);
// {n_embd_head_k, n_head, n_tokens}
q = ggml_reshape_3d(ctx0, q, n_embd_head_k, n_head, n_tokens);
cb(q, "q", il);
ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
cb(kv_cmpr_pe, "kv_cmpr_pe", il);
@ -552,10 +544,6 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
cb(k_pe, "k_pe", il);
q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(q_pe, "q_pe", il);
k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(k_pe, "k_pe", il);
@ -564,6 +552,20 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
cb(kv_cmpr, "kv_cmpr", il);
if (is_mla) {
// split into {n_embd_head_qk_nope, n_head, n_tokens}
ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
q->nb[1], q->nb[2], 0);
cb(q_nope, "q_nope", il);
// and {n_embd_head_qk_rope, n_head, n_tokens}
ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
q->nb[1], q->nb[2], ggml_row_size(q->type, n_embd_head_qk_nope));
cb(q_pe, "q_pe", il);
q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(q_pe, "q_pe", il);
// {n_embd_head_qk_nope, n_tokens, n_head}
q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
cb(q_nope, "q_nope_perm", il);
@ -623,10 +625,14 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p
Vcur = ggml_cont(ctx0, Vcur);
cb(Vcur, "Vcur_cont", il);
ggml_tensor * Qcur = ggml_concat(ctx0, q_nope, q_pe, 0);
// RoPE is applied to the trailing dims only
ggml_tensor * Qcur = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
Qcur = ggml_rope_set_offset(Qcur, n_embd_head_qk_nope);
cb(Qcur, "Qcur", il);
ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
ggml_tensor * Kcur = ggml_concat(ctx0, k_nope,
ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0);
cb(Kcur, "Kcur", il);
if (inp_attn_scale) {

85
src/models/deepseek4.cpp
View file

@ -501,21 +501,10 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_compressed_kv_from_state(
comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il);
cb(comp, name, il);
ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks,
ggml_row_size(comp->type, n_embd_head),
ggml_row_size(comp->type, n_embd_head),
0);
ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks,
ggml_row_size(comp->type, n_embd_head),
ggml_row_size(comp->type, n_embd_head),
ggml_row_size(comp->type, n_embd_head_nope));
comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
comp = ggml_rope_ext(ctx0, comp, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
hparams.dsv4_compress_rope_base, freq_scale, ext_factor,
dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow);
cb(comp_pe, name, il);
comp = ggml_concat(ctx0, comp_nope, comp_pe, 0);
comp = ggml_rope_set_offset(comp, n_embd_head_nope);
cb(comp, name, il);
return comp;
@ -585,21 +574,10 @@ ggml_tensor * llama_model_deepseek4::graph::build_overlap_compressed_kv_from_sta
comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il);
cb(comp, name, il);
ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks,
ggml_row_size(comp->type, n_embd_head),
ggml_row_size(comp->type, n_embd_head),
0);
ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks,
ggml_row_size(comp->type, n_embd_head),
ggml_row_size(comp->type, n_embd_head),
ggml_row_size(comp->type, n_embd_head_nope));
comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
comp = ggml_rope_ext(ctx0, comp, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig,
hparams.dsv4_compress_rope_base, freq_scale, ext_factor,
dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow);
cb(comp_pe, name, il);
comp = ggml_concat(ctx0, comp_nope, comp_pe, 0);
comp = ggml_rope_set_offset(comp, n_embd_head_nope);
cb(comp, name, il);
return comp;
@ -628,21 +606,12 @@ ggml_tensor * llama_model_deepseek4::graph::build_lid_top_k(
indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, nt);
cb(indexer_q, "lid_q", il);
ggml_tensor * indexer_q_nope = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_nope, n_indexer_head, nt,
ggml_row_size(indexer_q->type, n_embd_indexer_head),
ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head,
0);
ggml_tensor * indexer_q_pe = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_rope, n_indexer_head, nt,
ggml_row_size(indexer_q->type, n_embd_indexer_head),
ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head,
ggml_row_size(indexer_q->type, n_embd_indexer_head_nope));
indexer_q_pe = ggml_rope_ext(ctx0, indexer_q_pe, inp_pos, nullptr, n_embd_indexer_head_rope,
indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_embd_indexer_head_rope,
rope_type, n_ctx_orig, hparams.dsv4_compress_rope_base, freq_scale,
ext_factor, dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow);
cb(indexer_q_pe, "lid_q_pe", il);
indexer_q = ggml_rope_set_offset(indexer_q, n_embd_indexer_head_nope);
cb(indexer_q, "lid_q_rope", il);
indexer_q = ggml_concat(ctx0, indexer_q_nope, indexer_q_pe, 0);
indexer_q = llama_mul_mat_hadamard(ctx0, indexer_q, inp_lid.k_rot);
cb(indexer_q, "lid_q_rot", il);
@ -945,18 +914,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
q = ggml_rms_norm(ctx0, q, norm_rms_eps);
cb(q, "q_norm", il);
ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_nope, n_head, nt,
ggml_row_size(q->type, n_embd_head),
ggml_row_size(q->type, n_embd_head)*n_head,
0);
ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_rope, n_head, nt,
ggml_row_size(q->type, n_embd_head),
ggml_row_size(q->type, n_embd_head)*n_head,
ggml_row_size(q->type, n_embd_head_nope));
q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
cb(q_pe, "q_pe", il);
q = ggml_concat(ctx0, q_nope, q_pe, 0);
q = ggml_rope_set_offset(q, n_embd_head_nope);
cb(q, "q", il);
ggml_tensor * kv = build_lora_mm(layer.wkv, cur);
@ -964,18 +924,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, nt);
cb(kv, "kv_norm", il);
ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, nt,
ggml_row_size(kv->type, n_embd_head),
ggml_row_size(kv->type, n_embd_head),
0);
ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, nt,
ggml_row_size(kv->type, n_embd_head),
ggml_row_size(kv->type, n_embd_head),
ggml_row_size(kv->type, n_embd_head_nope));
kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
kv = ggml_rope_ext(ctx0, kv, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
cb(kv_pe, "kv_pe", il);
kv = ggml_concat(ctx0, kv_nope, kv_pe, 0);
kv = ggml_rope_set_offset(kv, n_embd_head_nope);
cb(kv, "kv", il);
const int64_t ratio = hparams.dsv4_compress_ratios[il];
@ -1225,7 +1176,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
if (inp_mtp) {
out = build_attn(inp_mtp,
nullptr, nullptr, nullptr,
q, kv, nullptr,
q, kv, kv,
nullptr, layer.attn_sinks, nullptr,
1.0f/sqrtf(float(n_embd_head)), il);
cb(out, "attn_raw", il);
@ -1245,17 +1196,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl(
}
out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, nt);
ggml_tensor * out_nope = ggml_view_3d(ctx0, out, n_embd_head_nope, n_head, nt,
ggml_row_size(out->type, n_embd_head),
ggml_row_size(out->type, n_embd_head)*n_head,
0);
ggml_tensor * out_pe = ggml_view_3d(ctx0, out, n_embd_head_rope, n_head, nt,
ggml_row_size(out->type, n_embd_head),
ggml_row_size(out->type, n_embd_head)*n_head,
ggml_row_size(out->type, n_embd_head_nope));
out_pe = ggml_rope_ext_back(ctx0, out_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
out = ggml_concat(ctx0, out_nope, out_pe, 0);
out = ggml_rope_set_offset(out, n_embd_head_nope);
cb(out, "attn_derope", il);
out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, nt);

12
src/models/dflash.cpp
View file

@ -591,17 +591,9 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, n_tokens,
ggml_row_size(kv->type, n_embd_head),
ggml_row_size(kv->type, n_embd_head),
0);
ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, n_tokens,
ggml_row_size(kv->type, n_embd_head),
ggml_row_size(kv->type, n_embd_head),
ggml_row_size(kv->type, n_embd_head_nope));
kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
kv = ggml_rope_ext(ctx0, kv, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
kv = ggml_concat(ctx0, kv_nope, kv_pe, 0);
kv = ggml_rope_set_offset(kv, n_embd_head_nope);
cb(kv, "kv_injected", il);
if (inp_attn->self_k_rot_swa) {

25
src/models/lfm2.cpp
View file

@ -2,6 +2,8 @@
#include "../llama-memory-hybrid-iswa.h"
#include "../llama-memory-hybrid.h"
#include <algorithm>
void llama_model_lfm2::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
@ -202,15 +204,20 @@ llama_model_lfm2::graph<iswa>::graph(const llama_model & model, const llm_graph_
}
GGML_ASSERT(bx->ne[0] > conv->ne[0]);
// last d_conv columns is a new conv state
auto * new_conv = ggml_view_3d(ctx0, bx, conv->ne[0], bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2],
(bx->ne[0] - conv->ne[0]) * ggml_element_size(bx));
GGML_ASSERT(ggml_are_same_shape(conv, new_conv));
// write conv states: slot 0 = the final state, slot s = the state s tokens back (partial rollback)
const int64_t K = hparams.causal_attn && cparams.n_rs_seq > 0 ? (int64_t) cparams.n_rs_seq + 1 : 1;
const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);
const auto mem_size = mctx_cur->get_size();
const size_t row_size = ggml_row_size(conv_state->type, (int64_t) d_conv * n_embd);
// write new conv conv state
ggml_build_forward_expand(gf, ggml_cpy(ctx0, new_conv,
ggml_view_1d(ctx0, conv_state, ggml_nelements(new_conv),
kv_head * d_conv * n_embd * ggml_element_size(new_conv))));
for (int64_t slot = 0; slot < n_written; ++slot) {
auto * conv_snap = ggml_view_3d(ctx0, bx, d_conv, bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2],
(bx->ne[0] - d_conv - slot) * ggml_element_size(bx));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap,
ggml_view_2d(ctx0, conv_state, (int64_t) d_conv * n_embd, n_seqs,
conv_state->nb[1],
((size_t) slot * mem_size + kv_head) * row_size)));
}
auto * conv_kernel = model.layers[il].shortconv.conv;
auto * conv_out = ggml_ssm_conv(ctx0, bx, conv_kernel);
@ -242,6 +249,8 @@ llama_model_lfm2::graph<iswa>::graph(const llama_model & model, const llm_graph_
ggml_tensor * inp_out_ids = build_inp_out_ids();
for (int il = 0; il < n_layer; ++il) {
res->t_layer_inp[il] = cur;
const bool is_moe_layer = il >= static_cast<int>(hparams.n_layer_dense_lead);
auto * prev_cur = cur;

28
src/models/minicpm3.cpp
View file

@ -115,19 +115,9 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q);
cb(q, "q", il);
// split into {n_head * n_embd_head_qk_nope, n_tokens}
ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
ggml_row_size(q->type, hparams.n_embd_head_k()),
ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
0);
cb(q_nope, "q_nope", il);
// and {n_head * n_embd_head_qk_rope, n_tokens}
ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
ggml_row_size(q->type, hparams.n_embd_head_k()),
ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
ggml_row_size(q->type, n_embd_head_qk_nope));
cb(q_pe, "q_pe", il);
// {n_embd_head_k, n_head, n_tokens}, RoPE is applied to the trailing dims only
q = ggml_reshape_3d(ctx0, q, hparams.n_embd_head_k(), n_head, n_tokens);
cb(q, "q", il);
// {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}
ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
@ -172,12 +162,13 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
v_states = ggml_cont(ctx0, v_states);
cb(v_states, "v_states", il);
q_pe = ggml_rope_ext(
ctx0, q_pe, inp_pos, rope_factors,
q = ggml_rope_ext(
ctx0, q, inp_pos, rope_factors,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow
);
cb(q_pe, "q_pe", il);
q = ggml_rope_set_offset(q, n_embd_head_qk_nope);
cb(q, "q_rope", il);
// shared RoPE key
k_pe = ggml_rope_ext(
@ -187,10 +178,11 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
);
cb(k_pe, "k_pe", il);
ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);
ggml_tensor * q_states = q;
cb(q_states, "q_states", il);
ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
ggml_tensor * k_states = ggml_concat(ctx0, k_nope,
ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0);
cb(k_states, "k_states", il);
cur = build_attn(inp_attn,

28
src/models/plm.cpp
View file

@ -81,19 +81,9 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
cb(q, "q", il);
// split into {n_head * n_embd_head_qk_nope, n_tokens}
ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
ggml_row_size(q->type, hparams.n_embd_head_k()),
ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
0);
cb(q_nope, "q_nope", il);
// and {n_head * n_embd_head_qk_rope, n_tokens}
ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
ggml_row_size(q->type, hparams.n_embd_head_k()),
ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),
ggml_row_size(q->type, n_embd_head_qk_nope));
cb(q_pe, "q_pe", il);
// {n_embd_head_k, n_head, n_tokens}, RoPE is applied to the trailing dims only
q = ggml_reshape_3d(ctx0, q, hparams.n_embd_head_k(), n_head, n_tokens);
cb(q, "q", il);
// {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}
ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
@ -143,12 +133,13 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
0);
cb(v_states, "v_states", il);
q_pe = ggml_rope_ext(
ctx0, q_pe, inp_pos, nullptr,
q = ggml_rope_ext(
ctx0, q, inp_pos, nullptr,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow
);
cb(q_pe, "q_pe", il);
q = ggml_rope_set_offset(q, n_embd_head_qk_nope);
cb(q, "q_rope", il);
// shared RoPE key
k_pe = ggml_rope_ext(
@ -158,10 +149,11 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
);
cb(k_pe, "k_pe", il);
ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);
ggml_tensor * q_states = q;
cb(q_states, "q_states", il);
ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
ggml_tensor * k_states = ggml_concat(ctx0, k_nope,
ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0);
cb(k_states, "k_states", il);
cur = build_attn(inp_attn,

11
tools/mtmd/clip.cpp
View file

@ -243,14 +243,13 @@ struct clip_ctx {
throw std::runtime_error("failed to initialize CPU backend");
}
if (ctx_params.use_gpu) {
auto * backend_name = std::getenv("MTMD_BACKEND_DEVICE");
if (backend_name != nullptr) {
backend = ggml_backend_init_by_name(backend_name, nullptr);
if (ctx_params.device != nullptr) {
backend = ggml_backend_dev_init(ctx_params.device, nullptr);
if (!backend) {
LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name);
throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n",
__func__, ggml_backend_dev_name(ctx_params.device)));
}
}
if (!backend) {
} else {
backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr);
backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr);
}

1
tools/mtmd/clip.h
View file

@ -48,6 +48,7 @@ enum clip_flash_attn_type {
struct clip_context_params {
bool use_gpu;
ggml_backend_dev_t device;
enum clip_flash_attn_type flash_attn_type;
int image_min_tokens;
int image_max_tokens;

1
tools/mtmd/debug/mtmd-debug.cpp
View file

@ -84,6 +84,7 @@ int main(int argc, char ** argv) {
const char * clip_path = params.mmproj.path.c_str();
mtmd_context_params mparams = mtmd_context_params_default();
mparams.use_gpu = params.mmproj_use_gpu;
mparams.device = params.mmproj_device;
mparams.print_timings = true;
mparams.n_threads = params.cpuparams.n_threads;
mparams.flash_attn_type = params.flash_attn_type;

1
tools/mtmd/mtmd-cli.cpp
View file

@ -154,6 +154,7 @@ struct mtmd_cli_context {
const char * clip_path = params.mmproj.path.c_str();
mtmd_context_params mparams = mtmd_context_params_default();
mparams.use_gpu = params.mmproj_use_gpu;
mparams.device = params.mmproj_device;
mparams.print_timings = true;
mparams.n_threads = params.cpuparams.n_threads;
mparams.flash_attn_type = params.flash_attn_type;

2
tools/mtmd/mtmd.cpp
View file

@ -456,6 +456,7 @@ static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_
mtmd_context_params mtmd_context_params_default() {
mtmd_context_params params {
/* use_gpu */ true,
/* device */ nullptr,
/* print_timings */ true,
/* n_threads */ 4,
/* image_marker */ nullptr,
@ -564,6 +565,7 @@ struct mtmd_context {
clip_context_params ctx_clip_params {
/* use_gpu */ ctx_params.use_gpu,
/* device */ ctx_params.device,
/* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type),
/* image_min_tokens */ ctx_params.image_min_tokens,
/* image_max_tokens */ ctx_params.image_max_tokens,

1
tools/mtmd/mtmd.h
View file

@ -89,6 +89,7 @@ typedef bool (*mtmd_progress_callback)(float progress, void * user_data);
struct mtmd_context_params {
bool use_gpu;
ggml_backend_dev_t device;
bool print_timings;
int n_threads;
const char * image_marker; // deprecated, use media_marker instead

1
tools/server/server-context.cpp
View file

@ -998,6 +998,7 @@ private:
mtmd_context_params mparams = mtmd_context_params_default();
if (has_mmproj) {
mparams.use_gpu = params_base.mmproj_use_gpu;
mparams.device = params_base.mmproj_device;
mparams.print_timings = false;
mparams.n_threads = params_base.cpuparams.n_threads;
mparams.flash_attn_type = params_base.flash_attn_type;

70
tools/server/server-models.cpp
View file

@ -672,24 +672,26 @@ void server_models::load_models() {
apply_hidden();
log_available_models();
std::vector<std::string> models_to_load;
for (const auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
models_to_load.push_back(name);
// skipped on reload, see startup_models
if (startup_models.has_value()) {
std::vector<std::string> models_to_load;
for (const auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
models_to_load.push_back(name);
}
}
}
if ((int)models_to_load.size() > base_params.models_max) {
throw std::runtime_error(string_format(
"number of models to load on startup (%zu) exceeds models_max (%d)",
models_to_load.size(), base_params.models_max));
if ((int)models_to_load.size() > base_params.models_max) {
throw std::runtime_error(string_format(
"number of models to load on startup (%zu) exceeds models_max (%d)",
models_to_load.size(), base_params.models_max));
}
// to be lazy-loaded after main() setup phase is completed
startup_models = std::move(models_to_load);
}
lk.unlock();
for (const auto & name : models_to_load) {
SRV_INF("(startup) loading model %s\n", name.c_str());
load(name);
}
} else {
// RELOAD: diff the new preset list against the current mapping and reconcile
is_reloading = true;
@ -819,8 +821,8 @@ void server_models::load_models() {
inst.meta.update_caps();
}
// add models that are new in this reload
std::vector<std::string> newly_added;
// add models that are new in this reload, load-on-startup is not honored here since a
// reload never spawns an instance
for (const auto & [name, preset] : final_presets) {
if (mapping.find(name) == mapping.end()) {
server_model_meta meta{
@ -841,42 +843,40 @@ void server_models::load_models() {
// /* need_download */ false,
};
add_model(std::move(meta));
newly_added.push_back(name);
}
}
apply_stop_timeout();
apply_hidden();
// clear reload flag before unlocking for autoload - load() blocks on !is_reloading,
// so clearing it here (while still locked) prevents a deadlock in the autoload calls below
// clear reload flag under the lock, this releases the load() calls waiting on !is_reloading
is_reloading = false;
cv.notify_all();
log_available_models();
// collect autoload candidates while still under the lock
std::vector<std::string> to_autoload;
for (const auto & name : newly_added) {
auto it = mapping.find(name);
if (it != mapping.end()) {
std::string val;
if (it->second.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
to_autoload.push_back(name);
}
}
}
lk.unlock();
for (const auto & name : to_autoload) {
SRV_INF("(reload) loading new model %s\n", name.c_str());
load(name);
}
notify_sse("models_reload", "*");
}
}
void server_models::load_startup_models() {
std::vector<std::string> to_load;
{
std::lock_guard<std::mutex> lk(mutex);
if (!startup_models.has_value()) {
return; // already drained
}
to_load = std::move(*startup_models);
startup_models.reset();
}
for (const auto & name : to_load) {
SRV_INF("(startup) loading model %s\n", name.c_str());
load(name);
}
}
void server_models::update_meta(const std::string & name, const server_model_meta & meta) {
std::lock_guard<std::mutex> lk(mutex);
auto it = mapping.find(name);

7
tools/server/server-models.h
View file

@ -136,6 +136,10 @@ private:
// if true, the next get_meta() will trigger a reload of model list
bool need_reload = false;
// models marked with load-on-startup, unset once load_startup_models() drains it
// no value means the startup phase is over, so a reload must not queue anything
std::optional<std::vector<std::string>> startup_models{std::in_place};
// conv_id -> model name that currently serves its stream session, lets the resumable stream
// routes go straight to the owning child instead of polling every one. populated when
// proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just
@ -231,6 +235,9 @@ public:
// - if a model is not running, it will be added or updated according to the source
void load_models();
// lazy-load startup_models, to be called after main() setup phase
void load_startup_models();
// check if a model instance exists (thread-safe)
bool has_model(const std::string & name);

15
tools/server/server.cpp
View file

@ -133,7 +133,8 @@ int llama_server(common_params & params, int argc, char ** argv) {
// router server never loads a model and must not touch the GPU
const bool is_router_server = params.model.path.empty()
&& params.model.hf_repo.empty();
&& params.model.hf_repo.empty()
&& params.model.docker_repo.empty();
// skip device enumeration so the CUDA primary context stays uncreated
common_params_print_info(params, !is_router_server);
@ -423,6 +424,18 @@ int llama_server(common_params & params, int argc, char ** argv) {
ctx_http.stop();
};
try {
models_routes->models.load_startup_models();
} catch (const std::exception & e) {
SRV_ERR("failed to load models on startup: %s\n", e.what());
ctx_http.stop();
if (ctx_http.thread.joinable()) {
ctx_http.thread.join();
}
clean_up();
return 1;
}
} else {
// setup clean up function, to be called before exit
clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() {

1
tools/tts/tts.cpp
View file

@ -86,6 +86,7 @@ int main(int argc, char ** argv) {
mtmd_context_params mtmd_params = mtmd_context_params_default();
mtmd_params.use_gpu = params.mmproj_use_gpu;
mtmd_params.device = params.mmproj_device;
mtmd::context_ptr mctx(mtmd_init_from_file(params.mmproj.path.c_str(), model, mtmd_params));
if (!mctx) {
LOG_ERR("failed to load mmproj %s\n", params.mmproj.path.c_str());

166
tools/ui/README.md
View file

@ -239,31 +239,44 @@ Routes → Components → Hooks → Stores → Services → Storage/API
### High-Level Architecture
See: [`docs/architecture/high-level-architecture-simplified.md`](docs/architecture/high-level-architecture-simplified.md)
```mermaid
flowchart TB
subgraph Routes["📍 Routes"]
R1["/ (Welcome)"]
R2["/chat/[id]"]
R3["/mcp-servers"]
R4["/search"]
R5["/settings"]
RL["+layout.svelte"]
end
subgraph Components["🧩 Components"]
C_Sidebar["ChatSidebar"]
C_Screen["ChatScreen"]
C_Form["ChatForm"]
C_Messages["ChatMessages"]
C_ModelsSelector["ModelsSelector"]
C_Sidebar["ChatSidebar"]
C_Models["ModelsSelector"]
C_Settings["ChatSettings"]
C_Mcp["McpServers"]
end
subgraph Hooks["🔌 Hooks"]
H1["use-chat-screen-active-model"]
H2["use-processing-state"]
H3["use-context-gauge"]
H4["use-models-selector"]
H5["use-tools-panel"]
end
subgraph Stores["🗄️ Stores"]
S1["chatStore"]
S2["conversationsStore"]
S3["modelsStore"]
S4["serverStore"]
S5["settingsStore"]
S4["mcpStore"]
S5["agenticStore"]
S6["serverStore"]
S7["settingsStore"]
S8["toolsStore"]
end
subgraph Services["⚙️ Services"]
@ -271,6 +284,9 @@ flowchart TB
SV2["ModelsService"]
SV3["PropsService"]
SV4["DatabaseService"]
SV5["MCPService"]
SV6["ToolsService"]
SV7["SandboxService"]
end
subgraph Storage["💾 Storage"]
@ -282,19 +298,28 @@ flowchart TB
API1["/v1/chat/completions"]
API2["/props"]
API3["/models/*"]
API4["/tools"]
end
R1 & R2 --> C_Screen
RL --> C_Sidebar
C_Screen --> C_Form & C_Messages & C_Settings
C_Screen --> S1 & S2
C_ModelsSelector --> S3 & S4
C_Screen --> H1 & H2 & H3
C_Models --> H4
C_Mcp --> S4
C_Screen --> S1 & S2 & S3
C_Models --> S3
H1 --> S3
S1 --> SV1 & SV4
S2 --> SV4
S3 --> SV2 & SV3
S4 --> SV5
S5 --> SV1 & SV5 & SV6 & SV7
SV4 --> ST1
SV1 --> API1
SV2 --> API3
SV3 --> API2
SV6 --> API4
```
### Layer Breakdown
@ -303,6 +328,9 @@ flowchart TB
- **`/`** - Welcome screen, creates new conversation
- **`/chat/[id]`** - Active chat interface
- **`/mcp-servers`** - MCP server management
- **`/search`** - Conversation search
- **`/settings`** - Settings (optional `[[section]]`)
- **`+layout.svelte`** - Sidebar, navigation, global initialization
#### Components (`src/lib/components/`)
@ -348,28 +376,68 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel
#### Hooks (`src/lib/hooks/`)
- **`useModelChangeValidation`** - Validates model switch against conversation modalities
- **`useProcessingState`** - Tracks streaming progress and token generation
Hooks are the thin view-layer between components and stores: they own UI concerns (scroll, drag-and-drop, keyboard shortcuts, pickers, selection) and translate store state into view state.
| Hook | Responsibility |
| ------------------------------- | -------------------------------------------------------------- |
| `use-chat-screen-active-model` | Active model resolution + modality capability detection |
| `use-processing-state` | View over `chatStore.processing` for streaming progress/tokens |
| `use-context-gauge` | View over `contextStatsStore` for the context usage gauge |
| `use-models-selector` | Model selector dropdown state (loaded/available groups) |
| `use-tools-panel` | Tools panel state |
| `use-reasoning-menu` | Reasoning-effort menu state |
| `use-attachment-menu` | Attachment menu + modality flags |
| `use-draft-messages` | Per-chat draft message/files persistence |
| `use-chat-form-pickers` | Chat form pickers (commands, mentions) |
| `use-debounced-search` | Shared debounced async search for pickers |
| `use-picker-navigation` | Picker keyboard navigation |
| `use-chat-message-edit-context` | Message edit context (content + extras) |
| `use-chat-screen-drag-and-drop` | Drag-and-drop state machine |
| `use-chat-screen-file-upload` | File upload queue + capability validation |
| `use-chat-screen-scroll` | Scroll container binding + navigation guard |
| `use-auto-scroll` | Auto-scroll controller for streaming |
| `use-marquee-selection` | Shift+click / marquee range selection |
| `use-keyboard-shortcuts` | Global keyboard shortcuts |
| `use-settings-navigation` | Settings section navigation |
| `use-pwa` | PWA install/update + version mismatch detection |
#### Stores (`src/lib/stores/`)
| Store | Responsibility |
| -------------------- | --------------------------------------------------------- |
| `chatStore` | Message sending, streaming, abort control, error handling |
| `conversationsStore` | CRUD for conversations, message branching, navigation |
| `modelsStore` | Model list, selection, loading/unloading (ROUTER) |
| `serverStore` | Server properties, role detection, modalities |
| `settingsStore` | User preferences, parameter sync with server defaults |
Stores own reactive application state as Svelte 5 runes. Larger stores are split into directories and compose focused sub-stores behind a narrow host interface (see Architectural Patterns).
| Store | Responsibility |
| -------------------- | --------------------------------------------------------------------------------------------------------------- |
| `chatStore` | Chat lifecycle, streaming, abort control, error handling; composes `processing`, `activity`, `streams`, `flows` |
| `conversationsStore` | Conversation CRUD, message branching, navigation, import/export; composes `preferences` |
| `modelsStore` | Model list, selection, loading/unloading (ROUTER); composes `props`, `status` |
| `mcpStore` | MCP host role: multi-server lifecycle, tool routing; composes `health`, `resources` |
| `agenticStore` | Multi-turn agentic loop orchestration, tool execution; composes `gates` |
| `serverStore` | Server connection state, `/props`, role detection, modalities |
| `settingsStore` | User preferences, theme, parameter sync with server defaults |
| `toolsStore` | Tool registry: server + MCP tools, enabled set for the LLM |
| `permissionsStore` | Persisted tool permission grants |
| `contextStatsStore` | Context window usage for the active conversation |
| `draftMessagesStore` | Per-chat draft message/files |
| `deviceStore` | Browser environment signals (mobile, OS, theme) |
| `versionStore` | Build version information |
#### Services (`src/lib/services/`)
| Service | Responsibility |
| ---------------------- | ----------------------------------------------- |
| `ChatService` | API calls to`/v1/chat/completions`, SSE parsing |
| `ModelsService` | `/models`, `/models/load`, `/models/unload` |
| `PropsService` | `/props`, `/props?model=` |
| `DatabaseService` | IndexedDB operations via Dexie |
| `ParameterSyncService` | Syncs settings with server defaults |
Services are a stateless protocol layer: static methods, pure I/O, no reactive state. Stores consume them for all API and storage access.
| Service | Responsibility |
| ----------------------------- | ------------------------------------------------------------------------- |
| `ChatService` | `/v1/chat/completions` streaming + SSE parsing, message format conversion |
| `ModelsService` | `/models`, `/models/load`, `/models/unload` |
| `PropsService` | `/props`, `/props?model=` |
| `DatabaseService` | IndexedDB operations via Dexie |
| `MCPService` | MCP protocol: transports, connect, list/execute tools, prompts, resources |
| `ToolsService` | Server tool list/execute/stream (`/tools`) |
| `SandboxService` | Browser JS execution in a sandboxed worker |
| `ParameterSyncService` | Syncs settings with server defaults |
| `ConversationTransferService` | Conversation import/export JSONL + ZIP format |
| `MigrationService` | Non-destructive localStorage/IndexedDB migrations |
| `RouterService` | Dynamic route URL construction |
---
@ -377,8 +445,6 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel
### MODEL Mode (Single Model)
See: [`docs/flows/data-flow-simplified-model-mode.md`](docs/flows/data-flow-simplified-model-mode.md)
```mermaid
sequenceDiagram
participant User
@ -388,8 +454,9 @@ sequenceDiagram
participant API as llama-server
Note over User,API: Initialization
UI->>Stores: initialize()
Stores->>DB: load conversations
UI->>Stores: initStores() (awaited by route loads)
Stores->>Stores: run migrations
Stores->>DB: load conversations (background)
Stores->>API: GET /props
API-->>Stores: server config
Stores->>API: GET /v1/models
@ -408,8 +475,6 @@ sequenceDiagram
### ROUTER Mode (Multi-Model)
See: [`docs/flows/data-flow-simplified-router-mode.md`](docs/flows/data-flow-simplified-router-mode.md)
```mermaid
sequenceDiagram
participant User
@ -441,17 +506,6 @@ sequenceDiagram
end
```
### Detailed Flow Diagrams
| Flow | Description | File |
| ------------- | ------------------------------------------ | ----------------------------------------------------------- |
| Chat | Message lifecycle, streaming, regeneration | [`chat-flow.md`](docs/flows/chat-flow.md) |
| Models | Loading, unloading, modality caching | [`models-flow.md`](docs/flows/models-flow.md) |
| Server | Props fetching, role detection | [`server-flow.md`](docs/flows/server-flow.md) |
| Conversations | CRUD, branching, import/export | [`conversations-flow.md`](docs/flows/conversations-flow.md) |
| Database | IndexedDB schema, operations | [`database-flow.md`](docs/flows/database-flow.md) |
| Settings | Parameter sync, user overrides | [`settings-flow.md`](docs/flows/settings-flow.md) |
---
## Architectural Patterns
@ -505,13 +559,14 @@ Components dispatch actions to stores, stores coordinate with services for I/O,
### 3. Per-Conversation State
Enables concurrent streaming across multiple conversations:
Enables concurrent streaming across multiple conversations. Loading is tracked
per conversation by the activity ledger (`chatStore.activity`), while streaming
state and abort controllers live in per-conversation maps:
```typescript
class ChatStore {
chatLoadingStates = new Map<string, boolean>();
chatStreamingStates = new Map<string, { response: string; messageId: string }>();
abortControllers = new Map<string, AbortController>();
chatStreamingStates = new SvelteMap<string, { response: string; messageId: string }>();
abortControllers = new SvelteMap<string, AbortController>();
}
```
@ -567,20 +622,14 @@ get isRouterMode() {
### 7. Modality Validation
Prevents sending attachments to incompatible models:
Prevents sending attachments to incompatible models. The
`use-chat-screen-active-model` hook derives the active model's capabilities
from `modelsStore.props`:
```typescript
// useModelChangeValidation hook
const validate = (modelId: string) => {
const modelModalities = modelsStore.getModelModalities(modelId);
const conversationModalities = conversationsStore.usedModalities;
// Check if model supports all used modalities
if (conversationModalities.hasImages && !modelModalities.vision) {
return { valid: false, reason: 'Model does not support images' };
}
// ...
};
// use-chat-screen-active-model hook
const hasVisionModality = $derived.by(() => modelsStore.props.modelSupportsVision(activeModelId));
const hasAudioModality = $derived.by(() => modelsStore.props.modelSupportsAudio(activeModelId));
```
### 8. Persistent Storage Strategy
@ -673,9 +722,6 @@ tools/ui/
│ └── styles/ # Global styles
├── static/ # Static assets
├── tests/ # Test files
├── docs/ # Architecture diagrams
│ ├── architecture/ # High-level architecture
│ └── flows/ # Feature-specific flows
└── .storybook/ # Storybook configuration
```

145
tools/ui/docs/architecture/high-level-architecture-simplified.md
View file

@ -1,145 +0,0 @@
```mermaid
flowchart TB
subgraph Routes["📍 Routes"]
R1["/ (Welcome)"]
R2["/chat/[id]"]
RL["+layout.svelte"]
end
subgraph Components["🧩 Components"]
C_Sidebar["ChatSidebar"]
C_Screen["ChatScreen"]
C_Form["ChatForm"]
C_Messages["ChatMessages"]
C_Message["ChatMessage"]
C_ChatMessageAgenticContent["ChatMessageAgenticContent"]
C_MessageEditForm["ChatMessageEditForm"]
C_ModelsSelector["ModelsSelector"]
C_Settings["ChatSettings"]
C_McpSettings["McpServersSettings"]
C_McpResourceBrowser["McpResourceBrowser"]
C_McpServersSelector["McpServersSelector"]
end
subgraph Hooks["🪝 Hooks"]
H1["useModelChangeValidation"]
H2["useProcessingState"]
end
subgraph Stores["🗄️ Stores"]
S1["chatStore<br/><i>Chat interactions & streaming</i>"]
SA["agenticStore<br/><i>Multi-turn agentic loop orchestration</i>"]
S2["conversationsStore<br/><i>Conversation data, messages & MCP overrides</i>"]
S3["modelsStore<br/><i>Model selection & loading</i>"]
S4["serverStore<br/><i>Server props & role detection</i>"]
S5["settingsStore<br/><i>User configuration incl. MCP</i>"]
S6["mcpStore<br/><i>MCP servers, tools, prompts</i>"]
S7["mcpResourceStore<br/><i>MCP resources & attachments</i>"]
end
subgraph Services["⚙️ Services"]
SV1["ChatService"]
SV2["ModelsService"]
SV3["PropsService"]
SV4["DatabaseService"]
SV5["ParameterSyncService"]
SV6["MCPService<br/><i>protocol operations</i>"]
end
subgraph Storage["💾 Storage"]
ST1["IndexedDB<br/><i>conversations, messages</i>"]
ST2["LocalStorage<br/><i>config, userOverrides, mcpServers</i>"]
end
subgraph APIs["🌐 llama-server API"]
API1["/v1/chat/completions"]
API2["/props"]
API3["/models/*"]
API4["/v1/models"]
end
subgraph ExternalMCP["🔌 External MCP Servers"]
EXT1["MCP Server 1<br/><i>WebSocket/HTTP/SSE</i>"]
EXT2["MCP Server N"]
end
%% Routes → Components
R1 & R2 --> C_Screen
RL --> C_Sidebar
%% Layout runs MCP health checks
RL --> S6
%% Component hierarchy
C_Screen --> C_Form & C_Messages & C_Settings
C_Messages --> C_Message
C_Message --> C_ChatMessageAgenticContent
C_Message --> C_MessageEditForm
C_Form & C_MessageEditForm --> C_ModelsSelector
C_Form --> C_McpServersSelector
C_Settings --> C_McpSettings
C_McpSettings --> C_McpResourceBrowser
%% Components → Hooks → Stores
C_Form & C_Messages --> H1 & H2
H1 --> S3 & S4
H2 --> S1 & S5
%% Components → Stores
C_Screen --> S1 & S2
C_Sidebar --> S2
C_ModelsSelector --> S3 & S4
C_Settings --> S5
C_McpSettings --> S6
C_McpResourceBrowser --> S6 & S7
C_McpServersSelector --> S6
C_Form --> S6
%% chatStore → agenticStore → mcpStore (agentic loop)
S1 --> SA
SA --> SV1
SA --> S6
%% Stores → Services
S1 --> SV1 & SV4
S2 --> SV4
S3 --> SV2 & SV3
S4 --> SV3
S5 --> SV5
S6 --> SV6
S7 --> SV6
%% Services → Storage
SV4 --> ST1
SV5 --> ST2
%% Services → APIs
SV1 --> API1
SV2 --> API3 & API4
SV3 --> API2
%% MCP → External Servers
SV6 --> EXT1 & EXT2
%% Styling
classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px
classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px
classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px
classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px
classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px
classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px
classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px
classDef mcpStyle fill:#e0f2f1,stroke:#00695c,stroke-width:2px
classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px
classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5
class R1,R2,RL routeStyle
class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_ChatMessageAgenticContent,C_MessageEditForm,C_ModelsSelector,C_Settings componentStyle
class C_McpSettings,C_McpResourceBrowser,C_McpServersSelector componentStyle
class H1,H2 hookStyle
class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle
class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle
class ST1,ST2 storageStyle
class API1,API2,API3,API4 apiStyle
class EXT1,EXT2 externalStyle
```

373
tools/ui/docs/architecture/high-level-architecture.md
View file

@ -1,373 +0,0 @@
```mermaid
flowchart TB
subgraph Routes["📍 Routes"]
R1["/ (+page.svelte)"]
R2["/chat/[id]"]
RL["+layout.svelte"]
end
subgraph Components["🧩 Components"]
direction TB
subgraph LayoutComponents["Layout"]
C_Sidebar["ChatSidebar"]
C_Screen["ChatScreen"]
end
subgraph ChatUIComponents["Chat UI"]
C_Form["ChatForm"]
C_Messages["ChatMessages"]
C_Message["ChatMessage"]
C_MessageUser["ChatMessageUser"]
C_MessageEditForm["ChatMessageEditForm"]
C_Attach["ChatAttachments"]
C_ModelsSelector["ModelsSelector"]
C_Settings["ChatSettings"]
end
subgraph MCPComponents["MCP UI"]
C_McpSettings["McpServersSettings"]
C_McpServerCard["McpServerCard"]
C_McpResourceBrowser["McpResourceBrowser"]
C_McpResourcePreview["McpResourcePreview"]
C_McpServersSelector["McpServersSelector"]
end
end
subgraph Hooks["🪝 Hooks"]
H1["useModelChangeValidation"]
H2["useProcessingState"]
H3["isMobile"]
end
subgraph Stores["🗄️ Stores"]
direction TB
subgraph S1["chatStore"]
S1State["<b>State:</b><br/>isLoading, currentResponse<br/>errorDialogState<br/>activeProcessingState<br/>chatLoadingStates<br/>chatStreamingStates<br/>abortControllers<br/>processingStates<br/>activeConversationId<br/>isStreamingActive"]
S1LoadState["<b>Loading State:</b><br/>setChatLoading()<br/>isChatLoading()<br/>syncLoadingStateForChat()<br/>clearUIState()<br/>isChatLoadingPublic()<br/>getAllLoadingChats()<br/>getAllStreamingChats()"]
S1ProcState["<b>Processing State:</b><br/>setActiveProcessingConversation()<br/>getProcessingState()<br/>clearProcessingState()<br/>getActiveProcessingState()<br/>updateProcessingStateFromTimings()<br/>getCurrentProcessingStateSync()<br/>restoreProcessingStateFromMessages()"]
S1Stream["<b>Streaming:</b><br/>streamChatCompletion()<br/>startStreaming()<br/>stopStreaming()<br/>stopGeneration()<br/>isStreaming()"]
S1Error["<b>Error Handling:</b><br/>showErrorDialog()<br/>dismissErrorDialog()<br/>isAbortError()"]
S1Msg["<b>Message Operations:</b><br/>addMessage()<br/>sendMessage()<br/>updateMessage()<br/>deleteMessage()<br/>getDeletionInfo()"]
S1Regen["<b>Regeneration:</b><br/>regenerateMessage()<br/>regenerateMessageWithBranching()<br/>continueAssistantMessage()"]
S1Edit["<b>Editing:</b><br/>editAssistantMessage()<br/>editUserMessagePreserveResponses()<br/>editMessageWithBranching()<br/>clearEditMode()<br/>isEditModeActive()<br/>getAddFilesHandler()<br/>setEditModeActive()"]
S1Utils["<b>Utilities:</b><br/>getApiOptions()<br/>parseTimingData()<br/>getOrCreateAbortController()<br/>getConversationModel()"]
end
subgraph SA["agenticStore"]
SAState["<b>State:</b><br/>sessions (Map)<br/>isAnyRunning"]
SASession["<b>Session Management:</b><br/>getSession()<br/>updateSession()<br/>clearSession()<br/>getActiveSessions()<br/>isRunning()<br/>currentTurn()<br/>totalToolCalls()<br/>lastError()<br/>streamingToolCall()"]
SAConfig["<b>Configuration:</b><br/>getConfig()<br/>maxTurns, maxToolPreviewLines"]
SAFlow["<b>Agentic Loop:</b><br/>runAgenticFlow()<br/>executeAgenticLoop()<br/>normalizeToolCalls()<br/>emitToolCallResult()<br/>extractBase64Attachments()"]
end
subgraph S2["conversationsStore"]
S2State["<b>State:</b><br/>conversations<br/>activeConversation<br/>activeMessages<br/>isInitialized<br/>pendingMcpServerOverrides<br/>titleUpdateConfirmationCallback"]
S2Lifecycle["<b>Lifecycle:</b><br/>initialize()<br/>loadConversations()<br/>clearActiveConversation()"]
S2ConvCRUD["<b>Conversation CRUD:</b><br/>createConversation()<br/>loadConversation()<br/>deleteConversation()<br/>deleteAll()<br/>updateConversationName()<br/>updateConversationTitleWithConfirmation()"]
S2MsgMgmt["<b>Message Management:</b><br/>refreshActiveMessages()<br/>addMessageToActive()<br/>updateMessageAtIndex()<br/>findMessageIndex()<br/>sliceActiveMessages()<br/>removeMessageAtIndex()<br/>getConversationMessages()"]
S2Nav["<b>Navigation:</b><br/>navigateToSibling()<br/>updateCurrentNode()<br/>updateConversationTimestamp()"]
S2McpOverrides["<b>MCP Per-Chat Overrides:</b><br/>getMcpServerOverride()<br/>getAllMcpServerOverrides()<br/>setMcpServerOverride()<br/>toggleMcpServerForChat()<br/>removeMcpServerOverride()<br/>isMcpServerEnabledForChat()<br/>clearPendingMcpServerOverrides()"]
S2Export["<b>Import/Export:</b><br/>downloadConversation()<br/>exportAllConversations()<br/>importConversations()<br/>importConversationsData()<br/>triggerDownload()"]
S2Utils["<b>Utilities:</b><br/>setTitleUpdateConfirmationCallback()"]
end
subgraph S3["modelsStore"]
S3State["<b>State:</b><br/>models, routerModels<br/>selectedModelId<br/>selectedModelName<br/>loading, updating, error<br/>modelLoadingStates<br/>modelPropsCache<br/>modelPropsFetching<br/>propsCacheVersion"]
S3Getters["<b>Computed Getters:</b><br/>selectedModel<br/>loadedModelIds<br/>loadingModelIds<br/>singleModelName"]
S3Modal["<b>Modalities:</b><br/>getModelModalities()<br/>modelSupportsVision()<br/>modelSupportsAudio()<br/>getModelModalitiesArray()<br/>getModelProps()<br/>updateModelModalities()"]
S3Status["<b>Status Queries:</b><br/>isModelLoaded()<br/>isModelOperationInProgress()<br/>getModelStatus()<br/>isModelPropsFetching()"]
S3Fetch["<b>Data Fetching:</b><br/>fetch()<br/>fetchRouterModels()<br/>fetchModelProps()<br/>fetchModalitiesForLoadedModels()"]
S3Select["<b>Model Selection:</b><br/>selectModelById()<br/>selectModelByName()<br/>clearSelection()<br/>findModelByName()<br/>findModelById()<br/>hasModel()"]
S3LoadUnload["<b>Loading/Unloading Models:</b><br/>loadModel()<br/>unloadModel()<br/>ensureModelLoaded()<br/>waitForModelStatus()<br/>pollForModelStatus()"]
S3Utils["<b>Utilities:</b><br/>toDisplayName()<br/>clear()"]
end
subgraph S4["serverStore"]
S4State["<b>State:</b><br/>props<br/>loading, error<br/>role<br/>fetchPromise"]
S4Getters["<b>Getters:</b><br/>defaultParams<br/>contextSize<br/>isRouterMode<br/>isModelMode"]
S4Data["<b>Data Handling:</b><br/>fetch()<br/>getErrorMessage()<br/>clear()"]
S4Utils["<b>Utilities:</b><br/>detectRole()"]
end
subgraph S5["settingsStore"]
S5State["<b>State:</b><br/>config<br/>theme<br/>isInitialized<br/>userOverrides"]
S5Lifecycle["<b>Lifecycle:</b><br/>initialize()<br/>loadConfig()<br/>saveConfig()<br/>loadTheme()<br/>saveTheme()"]
S5Update["<b>Config Updates:</b><br/>updateConfig()<br/>updateMultipleConfig()<br/>updateTheme()"]
S5Reset["<b>Reset:</b><br/>resetConfig()<br/>resetTheme()<br/>resetAll()<br/>resetParameterToServerDefault()"]
S5Sync["<b>Server Sync:</b><br/>syncWithServerDefaults()<br/>forceSyncWithServerDefaults()"]
S5Utils["<b>Utilities:</b><br/>getConfig()<br/>getAllConfig()<br/>getParameterInfo()<br/>getParameterDiff()<br/>getServerDefaults()<br/>clearAllUserOverrides()"]
end
subgraph S6["mcpStore"]
S6State["<b>State:</b><br/>isInitializing, error<br/>toolCount, connectedServers<br/>healthChecks (Map)<br/>connections (Map)<br/>toolsIndex (Map)"]
S6Lifecycle["<b>Lifecycle:</b><br/>ensureInitialized()<br/>initialize()<br/>shutdown()<br/>acquireConnection()<br/>releaseConnection()"]
S6Health["<b>Health Checks:</b><br/>runHealthCheck()<br/>runHealthChecksForServers()<br/>updateHealthCheck()<br/>getHealthCheckState()<br/>clearHealthCheck()"]
S6Servers["<b>Server Management:</b><br/>getServers()<br/>addServer()<br/>updateServer()<br/>removeServer()<br/>getServerById()<br/>getServerDisplayName()"]
S6Tools["<b>Tool Operations:</b><br/>getToolDefinitionsForLLM()<br/>getToolNames()<br/>hasTool()<br/>getToolServer()<br/>executeTool()<br/>executeToolByName()"]
S6Prompts["<b>Prompt Operations:</b><br/>getAllPrompts()<br/>getPrompt()<br/>hasPromptsCapability()<br/>getPromptCompletions()"]
end
subgraph S7["mcpResourceStore"]
S7State["<b>State:</b><br/>serverResources (Map)<br/>cachedResources (Map)<br/>subscriptions (Map)<br/>attachments[]<br/>isLoading"]
S7Resources["<b>Resource Discovery:</b><br/>setServerResources()<br/>getServerResources()<br/>getAllResourceInfos()<br/>getAllTemplateInfos()<br/>clearServerResources()"]
S7Cache["<b>Caching:</b><br/>cacheResourceContent()<br/>getCachedContent()<br/>invalidateCache()<br/>clearCache()"]
S7Subs["<b>Subscriptions:</b><br/>addSubscription()<br/>removeSubscription()<br/>isSubscribed()<br/>handleResourceUpdate()"]
S7Attach["<b>Attachments:</b><br/>addAttachment()<br/>updateAttachmentContent()<br/>removeAttachment()<br/>clearAttachments()<br/>toMessageExtras()"]
end
subgraph ReactiveExports["⚡ Reactive Exports"]
direction LR
subgraph ChatExports["chatStore"]
RE1["isLoading()"]
RE2["currentResponse()"]
RE3["errorDialog()"]
RE4["activeProcessingState()"]
RE5["isChatStreaming()"]
RE6["isChatLoading()"]
RE7["getChatStreaming()"]
RE8["getAllLoadingChats()"]
RE9["getAllStreamingChats()"]
RE9a["isEditModeActive()"]
RE9b["getAddFilesHandler()"]
RE9c["setEditModeActive()"]
RE9d["clearEditMode()"]
end
subgraph AgenticExports["agenticStore"]
REA1["agenticIsRunning()"]
REA2["agenticCurrentTurn()"]
REA3["agenticTotalToolCalls()"]
REA4["agenticLastError()"]
REA5["agenticStreamingToolCall()"]
REA6["agenticIsAnyRunning()"]
end
subgraph ConvExports["conversationsStore"]
RE10["conversations()"]
RE11["activeConversation()"]
RE12["activeMessages()"]
RE13["isConversationsInitialized()"]
end
subgraph ModelsExports["modelsStore"]
RE15["modelOptions()"]
RE16["routerModels()"]
RE17["modelsLoading()"]
RE18["modelsUpdating()"]
RE19["modelsError()"]
RE20["selectedModelId()"]
RE21["selectedModelName()"]
RE22["selectedModelOption()"]
RE23["loadedModelIds()"]
RE24["loadingModelIds()"]
RE25["propsCacheVersion()"]
RE26["singleModelName()"]
end
subgraph ServerExports["serverStore"]
RE27["serverProps()"]
RE28["serverLoading()"]
RE29["serverError()"]
RE30["serverRole()"]
RE31["defaultParams()"]
RE32["contextSize()"]
RE33["isRouterMode()"]
RE34["isModelMode()"]
end
subgraph SettingsExports["settingsStore"]
RE35["config()"]
RE36["theme()"]
RE37["isInitialized()"]
end
subgraph MCPExports["mcpStore / mcpResourceStore"]
RE38["mcpResources()"]
RE39["mcpResourceAttachments()"]
RE40["mcpHasResourceAttachments()"]
RE41["mcpTotalResourceCount()"]
RE42["mcpResourcesLoading()"]
end
end
end
subgraph Services["⚙️ Services"]
direction TB
subgraph SV1["ChatService"]
SV1Msg["<b>Messaging:</b><br/>sendMessage()"]
SV1Stream["<b>Streaming:</b><br/>handleStreamResponse()<br/>handleNonStreamResponse()"]
SV1Convert["<b>Conversion:</b><br/>convertDbMessageToApiChatMessageData()<br/>mergeToolCallDeltas()"]
SV1Utils["<b>Utilities:</b><br/>stripReasoningContent()<br/>extractModelName()<br/>parseErrorResponse()"]
end
subgraph SV2["ModelsService"]
SV2List["<b>Listing:</b><br/>list()<br/>listRouter()"]
SV2LoadUnload["<b>Load/Unload:</b><br/>load()<br/>unload()"]
SV2Status["<b>Status:</b><br/>isModelLoaded()<br/>isModelLoading()"]
end
subgraph SV3["PropsService"]
SV3Fetch["<b>Fetching:</b><br/>fetch()<br/>fetchForModel()"]
end
subgraph SV4["DatabaseService"]
SV4Conv["<b>Conversations:</b><br/>createConversation()<br/>getConversation()<br/>getAllConversations()<br/>updateConversation()<br/>deleteConversation()"]
SV4Msg["<b>Messages:</b><br/>createMessageBranch()<br/>createRootMessage()<br/>createSystemMessage()<br/>getConversationMessages()<br/>updateMessage()<br/>deleteMessage()<br/>deleteMessageCascading()"]
SV4Node["<b>Navigation:</b><br/>updateCurrentNode()"]
SV4Import["<b>Import:</b><br/>importConversations()"]
end
subgraph SV5["ParameterSyncService"]
SV5Extract["<b>Extraction:</b><br/>extractServerDefaults()"]
SV5Merge["<b>Merging:</b><br/>mergeWithServerDefaults()"]
SV5Info["<b>Info:</b><br/>getParameterInfo()<br/>canSyncParameter()<br/>getSyncableParameterKeys()<br/>validateServerParameter()"]
SV5Diff["<b>Diff:</b><br/>createParameterDiff()"]
end
subgraph SV6["MCPService"]
SV6Transport["<b>Transport:</b><br/>createTransport()<br/>WebSocket / StreamableHTTP / SSE"]
SV6Conn["<b>Connection:</b><br/>connect()<br/>disconnect()"]
SV6Tools["<b>Tools:</b><br/>listTools()<br/>callTool()"]
SV6Prompts["<b>Prompts:</b><br/>listPrompts()<br/>getPrompt()"]
SV6Resources["<b>Resources:</b><br/>listResources()<br/>listResourceTemplates()<br/>readResource()<br/>subscribeResource()<br/>unsubscribeResource()"]
SV6Complete["<b>Completions:</b><br/>complete()"]
end
end
subgraph ExternalMCP["🔌 External MCP Servers"]
EXT1["MCP Server 1<br/>(WebSocket/StreamableHTTP/SSE)"]
EXT2["MCP Server N"]
end
subgraph Storage["💾 Storage"]
ST1["IndexedDB"]
ST2["conversations"]
ST3["messages"]
ST5["LocalStorage"]
ST6["config"]
ST7["userOverrides"]
ST8["mcpServers"]
end
subgraph APIs["🌐 llama-server API"]
API1["/v1/chat/completions"]
API2["/props<br/>/props?model="]
API3["/models<br/>/models/load<br/>/models/unload"]
API4["/v1/models"]
end
%% Routes render Components
R1 --> C_Screen
R2 --> C_Screen
RL --> C_Sidebar
%% Layout runs MCP health checks on startup
RL --> S6
%% Component hierarchy
C_Screen --> C_Form & C_Messages & C_Settings
C_Messages --> C_Message
C_Message --> C_MessageUser
C_MessageUser --> C_MessageEditForm
C_MessageEditForm --> C_ModelsSelector
C_MessageEditForm --> C_Attach
C_Form --> C_ModelsSelector
C_Form --> C_Attach
C_Form --> C_McpServersSelector
C_Message --> C_Attach
%% MCP Components hierarchy
C_Settings --> C_McpSettings
C_McpSettings --> C_McpServerCard
C_McpServerCard --> C_McpResourceBrowser
C_McpResourceBrowser --> C_McpResourcePreview
%% Components use Hooks
C_Form --> H1
C_Message --> H1 & H2
C_MessageEditForm --> H1
C_Screen --> H2
%% Hooks use Stores
H1 --> S3 & S4
H2 --> S1 & S5
%% Components use Stores
C_Screen --> S1 & S2
C_Messages --> S2
C_Message --> S1 & S2 & S3
C_Form --> S1 & S3 & S6
C_Sidebar --> S2
C_ModelsSelector --> S3 & S4
C_Settings --> S5
C_McpSettings --> S6
C_McpServerCard --> S6
C_McpResourceBrowser --> S6 & S7
C_McpServersSelector --> S6
%% Stores export Reactive State
S1 -. exports .-> ChatExports
SA -. exports .-> AgenticExports
S2 -. exports .-> ConvExports
S3 -. exports .-> ModelsExports
S4 -. exports .-> ServerExports
S5 -. exports .-> SettingsExports
S6 -. exports .-> MCPExports
S7 -. exports .-> MCPExports
%% chatStore → agenticStore (agentic loop orchestration)
S1 --> SA
SA --> SV1
SA --> S6
%% Stores use Services
S1 --> SV1 & SV4
S2 --> SV4
S3 --> SV2 & SV3
S4 --> SV3
S5 --> SV5
S6 --> SV6
S7 --> SV6
%% Services to Storage
SV4 --> ST1
ST1 --> ST2 & ST3
SV5 --> ST5
ST5 --> ST6 & ST7 & ST8
%% Services to APIs
SV1 --> API1
SV2 --> API3 & API4
SV3 --> API2
%% MCP → External Servers
SV6 --> EXT1 & EXT2
%% Styling
classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px
classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px
classDef componentGroupStyle fill:#e1bee7,stroke:#7b1fa2,stroke-width:1px
classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px
classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px
classDef stateStyle fill:#ffe0b2,stroke:#e65100,stroke-width:1px
classDef methodStyle fill:#ffecb3,stroke:#e65100,stroke-width:1px
classDef reactiveStyle fill:#fffde7,stroke:#f9a825,stroke-width:1px
classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px
classDef serviceMStyle fill:#c8e6c9,stroke:#2e7d32,stroke-width:1px
classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5
classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px
classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px
class R1,R2,RL routeStyle
class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_MessageUser,C_MessageEditForm componentStyle
class C_ModelsSelector,C_Settings componentStyle
class C_Attach componentStyle
class C_McpSettings,C_McpServerCard,C_McpResourceBrowser,C_McpResourcePreview,C_McpServersSelector componentStyle
class H1,H2,H3 hookStyle
class LayoutComponents,ChatUIComponents,MCPComponents componentGroupStyle
class Hooks hookStyle
classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px
classDef agenticMethodStyle fill:#c5cae9,stroke:#283593,stroke-width:1px
class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle
class S1State,S2State,S3State,S4State,S5State,SAState,S6State,S7State stateStyle
class S1Msg,S1Regen,S1Edit,S1Stream,S1LoadState,S1ProcState,S1Error,S1Utils methodStyle
class SASession,SAConfig,SAFlow methodStyle
class S2Lifecycle,S2ConvCRUD,S2MsgMgmt,S2Nav,S2McpOverrides,S2Export,S2Utils methodStyle
class S3Getters,S3Modal,S3Status,S3Fetch,S3Select,S3LoadUnload,S3Utils methodStyle
class S4Getters,S4Data,S4Utils methodStyle
class S5Lifecycle,S5Update,S5Reset,S5Sync,S5Utils methodStyle
class S6Lifecycle,S6Health,S6Servers,S6Tools,S6Prompts methodStyle
class S7Resources,S7Cache,S7Subs,S7Attach methodStyle
class ChatExports,AgenticExports,ConvExports,ModelsExports,ServerExports,SettingsExports,MCPExports reactiveStyle
class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle
class SV6Transport,SV6Conn,SV6Tools,SV6Prompts,SV6Resources,SV6Complete serviceMStyle
class EXT1,EXT2 externalStyle
class SV1Msg,SV1Stream,SV1Convert,SV1Utils serviceMStyle
class SV2List,SV2LoadUnload,SV2Status serviceMStyle
class SV3Fetch serviceMStyle
class SV4Conv,SV4Msg,SV4Node,SV4Import serviceMStyle
class SV5Extract,SV5Merge,SV5Info,SV5Diff serviceMStyle
class ST1,ST2,ST3,ST5,ST6,ST7,ST8 storageStyle
class API1,API2,API3,API4 apiStyle
```

228
tools/ui/docs/flows/chat-flow.md
View file

@ -1,228 +0,0 @@
```mermaid
sequenceDiagram
participant UI as 🧩 ChatForm / ChatMessage
participant chatStore as 🗄️ chatStore
participant agenticStore as 🗄️ agenticStore
participant convStore as 🗄️ conversationsStore
participant settingsStore as 🗄️ settingsStore
participant mcpStore as 🗄️ mcpStore
participant ChatSvc as ⚙️ ChatService
participant DbSvc as ⚙️ DatabaseService
participant API as 🌐 /v1/chat/completions
Note over chatStore: State:<br/>isLoading, currentResponse<br/>errorDialogState, activeProcessingState<br/>chatLoadingStates (Map)<br/>chatStreamingStates (Map)<br/>abortControllers (Map)<br/>processingStates (Map)
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 💬 SEND MESSAGE
%% ═══════════════════════════════════════════════════════════════════════════
UI->>chatStore: sendMessage(content, extras)
activate chatStore
chatStore->>chatStore: setChatLoading(convId, true)
chatStore->>chatStore: clearChatStreaming(convId)
alt no active conversation
chatStore->>convStore: createConversation()
Note over convStore: → see conversations-flow.mmd
end
chatStore->>mcpStore: consumeResourceAttachmentsAsExtras()
Note right of mcpStore: Converts pending MCP resource<br/>attachments into message extras
chatStore->>chatStore: addMessage("user", content, extras)
chatStore->>DbSvc: createMessageBranch(userMsg, parentId)
chatStore->>convStore: addMessageToActive(userMsg)
chatStore->>convStore: updateCurrentNode(userMsg.id)
chatStore->>chatStore: createAssistantMessage(userMsg.id)
chatStore->>DbSvc: createMessageBranch(assistantMsg, userMsg.id)
chatStore->>convStore: addMessageToActive(assistantMsg)
chatStore->>chatStore: streamChatCompletion(messages, assistantMsg)
deactivate chatStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 🌊 STREAMING (with agentic flow detection)
%% ═══════════════════════════════════════════════════════════════════════════
activate chatStore
chatStore->>chatStore: startStreaming()
Note right of chatStore: isStreamingActive = true
chatStore->>chatStore: setActiveProcessingConversation(convId)
chatStore->>chatStore: getOrCreateAbortController(convId)
Note right of chatStore: abortControllers.set(convId, new AbortController())
chatStore->>chatStore: getApiOptions()
Note right of chatStore: Merge from settingsStore.config:<br/>temperature, max_tokens, top_p, etc.
alt agenticConfig.enabled && mcpStore has connected servers
chatStore->>agenticStore: runAgenticFlow(convId, messages, assistantMsg, options, signal)
Note over agenticStore: Multi-turn agentic loop:<br/>1. Call ChatService.sendMessage()<br/>2. If response has tool_calls → execute via mcpStore<br/>3. Append tool results as messages<br/>4. Loop until no more tool_calls or maxTurns<br/>→ see agentic flow details below
agenticStore-->>chatStore: final response with timings
else standard (non-agentic) flow
chatStore->>ChatSvc: sendMessage(messages, options, signal)
end
activate ChatSvc
ChatSvc->>ChatSvc: convertDbMessageToApiChatMessageData(messages)
Note right of ChatSvc: DatabaseMessage[] → ApiChatMessageData[]<br/>Process attachments (images, PDFs, audio)
ChatSvc->>API: POST /v1/chat/completions
Note right of API: {messages, model?, stream: true, ...params}
loop SSE chunks
API-->>ChatSvc: data: {"choices":[{"delta":{...}}]}
ChatSvc->>ChatSvc: handleStreamResponse(response)
alt content chunk
ChatSvc-->>chatStore: onChunk(content)
chatStore->>chatStore: setChatStreaming(convId, response, msgId)
Note right of chatStore: currentResponse = $state(accumulated)
chatStore->>convStore: updateMessageAtIndex(idx, {content})
end
alt reasoning chunk
ChatSvc-->>chatStore: onReasoningChunk(reasoning)
chatStore->>convStore: updateMessageAtIndex(idx, {thinking})
end
alt tool_calls chunk
ChatSvc-->>chatStore: onToolCallChunk(toolCalls)
chatStore->>convStore: updateMessageAtIndex(idx, {toolCalls})
end
alt model info
ChatSvc-->>chatStore: onModel(modelName)
chatStore->>chatStore: recordModel(modelName)
chatStore->>DbSvc: updateMessage(msgId, {model})
end
alt timings (during stream)
ChatSvc-->>chatStore: onTimings(timings, promptProgress)
chatStore->>chatStore: updateProcessingStateFromTimings()
end
chatStore-->>UI: reactive $state update
end
API-->>ChatSvc: data: [DONE]
ChatSvc-->>chatStore: onComplete(content, reasoning, timings, toolCalls)
deactivate ChatSvc
chatStore->>chatStore: stopStreaming()
chatStore->>DbSvc: updateMessage(msgId, {content, timings, model})
chatStore->>convStore: updateCurrentNode(msgId)
chatStore->>chatStore: setChatLoading(convId, false)
chatStore->>chatStore: clearChatStreaming(convId)
chatStore->>chatStore: clearProcessingState(convId)
deactivate chatStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: ⏹️ STOP GENERATION
%% ═══════════════════════════════════════════════════════════════════════════
UI->>chatStore: stopGeneration()
activate chatStore
chatStore->>chatStore: savePartialResponseIfNeeded(convId)
Note right of chatStore: Save currentResponse to DB if non-empty
chatStore->>chatStore: abortControllers.get(convId).abort()
Note right of chatStore: fetch throws AbortError → caught by isAbortError()
chatStore->>chatStore: stopStreaming()
chatStore->>chatStore: setChatLoading(convId, false)
chatStore->>chatStore: clearChatStreaming(convId)
chatStore->>chatStore: clearProcessingState(convId)
deactivate chatStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 🔁 REGENERATE
%% ═══════════════════════════════════════════════════════════════════════════
UI->>chatStore: regenerateMessageWithBranching(msgId, model?)
activate chatStore
chatStore->>convStore: findMessageIndex(msgId)
chatStore->>chatStore: Get parent of target message
chatStore->>chatStore: createAssistantMessage(parentId)
chatStore->>DbSvc: createMessageBranch(newAssistantMsg, parentId)
chatStore->>convStore: refreshActiveMessages()
Note right of chatStore: Same streaming flow
chatStore->>chatStore: streamChatCompletion(...)
deactivate chatStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: ➡️ CONTINUE
%% ═══════════════════════════════════════════════════════════════════════════
UI->>chatStore: continueAssistantMessage(msgId)
activate chatStore
chatStore->>chatStore: Get existing content from message
chatStore->>chatStore: streamChatCompletion(..., existingContent)
Note right of chatStore: Appends to existing message content
deactivate chatStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: ✏️ EDIT USER MESSAGE
%% ═══════════════════════════════════════════════════════════════════════════
UI->>chatStore: editMessageWithBranching(msgId, newContent, extras)
activate chatStore
chatStore->>chatStore: Get parent of target message
chatStore->>DbSvc: createMessageBranch(editedMsg, parentId)
chatStore->>convStore: refreshActiveMessages()
Note right of chatStore: Creates new branch, original preserved
chatStore->>chatStore: createAssistantMessage(editedMsg.id)
chatStore->>chatStore: streamChatCompletion(...)
Note right of chatStore: Automatically regenerates response
deactivate chatStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: ❌ ERROR HANDLING
%% ═══════════════════════════════════════════════════════════════════════════
Note over chatStore: On stream error (non-abort):
chatStore->>chatStore: showErrorDialog(type, message)
Note right of chatStore: errorDialogState = {type: 'timeout'|'server', message}
chatStore->>convStore: removeMessageAtIndex(failedMsgIdx)
chatStore->>DbSvc: deleteMessage(failedMsgId)
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 🤖 AGENTIC LOOP (when agenticConfig.enabled)
%% ═══════════════════════════════════════════════════════════════════════════
Note over agenticStore: agenticStore.runAgenticFlow(convId, messages, assistantMsg, options, signal)
activate agenticStore
agenticStore->>agenticStore: getSession(convId) or create new
agenticStore->>agenticStore: updateSession(turn: 0, running: true)
loop executeAgenticLoop (until no tool_calls or maxTurns)
agenticStore->>agenticStore: turn++
agenticStore->>ChatSvc: sendMessage(messages, options, signal)
ChatSvc->>API: POST /v1/chat/completions
API-->>ChatSvc: response with potential tool_calls
ChatSvc-->>agenticStore: onComplete(content, reasoning, timings, toolCalls)
alt response has tool_calls
agenticStore->>agenticStore: normalizeToolCalls(toolCalls)
loop for each tool_call
agenticStore->>agenticStore: updateSession(streamingToolCall)
agenticStore->>mcpStore: executeTool(mcpCall, signal)
mcpStore-->>agenticStore: tool result
agenticStore->>agenticStore: extractBase64Attachments(result)
agenticStore->>agenticStore: emitToolCallResult(convId, ...)
agenticStore->>convStore: addMessageToActive(toolResultMsg)
agenticStore->>DbSvc: createMessageBranch(toolResultMsg)
end
agenticStore->>agenticStore: Create new assistantMsg for next turn
Note right of agenticStore: Continue loop with updated messages
else no tool_calls (final response)
agenticStore->>agenticStore: buildFinalTimings(allTurns)
Note right of agenticStore: Break loop, return final response
end
end
agenticStore->>agenticStore: updateSession(running: false)
agenticStore-->>chatStore: final content, timings, model
deactivate agenticStore
```

183
tools/ui/docs/flows/conversations-flow.md
View file

@ -1,183 +0,0 @@
```mermaid
sequenceDiagram
participant UI as 🧩 ChatSidebar / ChatScreen
participant convStore as 🗄️ conversationsStore
participant chatStore as 🗄️ chatStore
participant DbSvc as ⚙️ DatabaseService
participant IDB as 💾 IndexedDB
Note over convStore: State:<br/>conversations: DatabaseConversation[]<br/>activeConversation: DatabaseConversation | null<br/>activeMessages: DatabaseMessage[]<br/>isInitialized: boolean<br/>pendingMcpServerOverrides: Map&lt;string, McpServerOverride&gt;
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,IDB: 🚀 INITIALIZATION
%% ═══════════════════════════════════════════════════════════════════════════
Note over convStore: Auto-initialized in constructor (browser only)
convStore->>convStore: initialize()
activate convStore
convStore->>convStore: loadConversations()
convStore->>DbSvc: getAllConversations()
DbSvc->>IDB: SELECT * FROM conversations ORDER BY lastModified DESC
IDB-->>DbSvc: Conversation[]
DbSvc-->>convStore: conversations
convStore->>convStore: conversations = $state(data)
convStore->>convStore: isInitialized = true
deactivate convStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,IDB: ➕ CREATE CONVERSATION
%% ═══════════════════════════════════════════════════════════════════════════
UI->>convStore: createConversation(name?)
activate convStore
convStore->>DbSvc: createConversation(name || "New Chat")
DbSvc->>IDB: INSERT INTO conversations
IDB-->>DbSvc: conversation {id, name, lastModified, currNode: ""}
DbSvc-->>convStore: conversation
convStore->>convStore: conversations.unshift(conversation)
convStore->>convStore: activeConversation = $state(conversation)
convStore->>convStore: activeMessages = $state([])
alt pendingMcpServerOverrides has entries
loop each pending override
convStore->>DbSvc: Store MCP server override for new conversation
end
convStore->>convStore: clearPendingMcpServerOverrides()
end
deactivate convStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,IDB: 📂 LOAD CONVERSATION
%% ═══════════════════════════════════════════════════════════════════════════
UI->>convStore: loadConversation(convId)
activate convStore
convStore->>DbSvc: getConversation(convId)
DbSvc->>IDB: SELECT * FROM conversations WHERE id = ?
IDB-->>DbSvc: conversation
convStore->>convStore: activeConversation = $state(conversation)
convStore->>convStore: refreshActiveMessages()
convStore->>DbSvc: getConversationMessages(convId)
DbSvc->>IDB: SELECT * FROM messages WHERE convId = ?
IDB-->>DbSvc: allMessages[]
convStore->>convStore: filterByLeafNodeId(allMessages, currNode)
Note right of convStore: Filter to show only current branch path
convStore->>convStore: activeMessages = $state(filtered)
Note right of convStore: Route (+page.svelte) then calls:<br/>chatStore.syncLoadingStateForChat(convId)
deactivate convStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,IDB: 🌳 MESSAGE BRANCHING MODEL
%% ═══════════════════════════════════════════════════════════════════════════
Note over IDB: Message Tree Structure:<br/>- Each message has parent (null for root)<br/>- Each message has children[] array<br/>- Conversation.currNode points to active leaf<br/>- filterByLeafNodeId() traverses from root to currNode
rect rgb(240, 240, 255)
Note over convStore: Example Branch Structure:
Note over convStore: root → user1 → assistant1 → user2 → assistant2a (currNode)<br/> ↘ assistant2b (alt branch)
end
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,IDB: ↔️ BRANCH NAVIGATION
%% ═══════════════════════════════════════════════════════════════════════════
UI->>convStore: navigateToSibling(msgId, direction)
activate convStore
convStore->>convStore: Find message in activeMessages
convStore->>convStore: Get parent message
convStore->>convStore: Find sibling in parent.children[]
convStore->>convStore: findLeafNode(siblingId, allMessages)
Note right of convStore: Navigate to leaf of sibling branch
convStore->>convStore: updateCurrentNode(leafId)
convStore->>DbSvc: updateCurrentNode(convId, leafId)
DbSvc->>IDB: UPDATE conversations SET currNode = ?
convStore->>convStore: refreshActiveMessages()
deactivate convStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,IDB: 📝 UPDATE CONVERSATION
%% ═══════════════════════════════════════════════════════════════════════════
UI->>convStore: updateConversationName(convId, newName)
activate convStore
convStore->>DbSvc: updateConversation(convId, {name: newName})
DbSvc->>IDB: UPDATE conversations SET name = ?
convStore->>convStore: Update in conversations array
deactivate convStore
Note over convStore: Auto-title update (after first response):
convStore->>convStore: updateConversationTitleWithConfirmation()
convStore->>convStore: titleUpdateConfirmationCallback?()
Note right of convStore: Shows dialog if title would change
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,IDB: 🗑️ DELETE CONVERSATION
%% ═══════════════════════════════════════════════════════════════════════════
UI->>convStore: deleteConversation(convId)
activate convStore
convStore->>DbSvc: deleteConversation(convId)
DbSvc->>IDB: DELETE FROM conversations WHERE id = ?
DbSvc->>IDB: DELETE FROM messages WHERE convId = ?
convStore->>convStore: conversations.filter(c => c.id !== convId)
alt deleted active conversation
convStore->>convStore: clearActiveConversation()
end
deactivate convStore
UI->>convStore: deleteAll()
activate convStore
convStore->>DbSvc: Delete all conversations and messages
convStore->>convStore: conversations = []
convStore->>convStore: clearActiveConversation()
deactivate convStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,IDB: <20> MCP SERVER PER-CHAT OVERRIDES
%% ═══════════════════════════════════════════════════════════════════════════
Note over convStore: Conversations can override which MCP servers are enabled.
Note over convStore: Uses pendingMcpServerOverrides before conversation<br/>is created, then persists to conversation metadata.
UI->>convStore: setMcpServerOverride(convId, serverName, override)
Note right of convStore: override = {enabled: boolean}
UI->>convStore: toggleMcpServerForChat(convId, serverName, enabled)
activate convStore
convStore->>convStore: setMcpServerOverride(convId, serverName, {enabled})
deactivate convStore
UI->>convStore: isMcpServerEnabledForChat(convId, serverName)
Note right of convStore: Check override → fall back to global MCP config
UI->>convStore: getAllMcpServerOverrides(convId)
Note right of convStore: Returns all overrides for a conversation
UI->>convStore: removeMcpServerOverride(convId, serverName)
UI->>convStore: getMcpServerOverride(convId, serverName)
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,IDB: 📤 EXPORT / 📥 IMPORT
%% ═══════════════════════════════════════════════════════════════════════════
UI->>convStore: exportAllConversations()
activate convStore
convStore->>DbSvc: getAllConversations()
loop each conversation
convStore->>DbSvc: getConversationMessages(convId)
end
convStore->>convStore: triggerDownload(JSON blob)
deactivate convStore
UI->>convStore: importConversations(file)
activate convStore
convStore->>convStore: Parse JSON file
convStore->>convStore: importConversationsData(parsed)
convStore->>DbSvc: importConversations(parsed)
Note right of DbSvc: Skips duplicate conversations<br/>(checks existing by ID)
DbSvc->>IDB: INSERT conversations + messages (skip existing)
convStore->>convStore: loadConversations()
deactivate convStore
```

45
tools/ui/docs/flows/data-flow-simplified-model-mode.md
View file

@ -1,45 +0,0 @@
```mermaid
%% MODEL Mode Data Flow (single model)
%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd
sequenceDiagram
participant User as 👤 User
participant UI as 🧩 UI
participant Stores as 🗄️ Stores
participant DB as 💾 IndexedDB
participant API as 🌐 llama-server
Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd)
UI->>Stores: initialize()
Stores->>DB: load conversations
Stores->>API: GET /props
API-->>Stores: server config + modalities
Stores->>API: GET /v1/models
API-->>Stores: single model (auto-selected)
Note over User,API: 💬 Chat Flow (see: chat-flow.mmd)
User->>UI: send message
UI->>Stores: sendMessage()
Stores->>DB: save user message
Stores->>API: POST /v1/chat/completions (stream)
loop streaming
API-->>Stores: SSE chunks
Stores-->>UI: reactive update
end
API-->>Stores: done + timings
Stores->>DB: save assistant message
Note over User,API: 🔁 Regenerate
User->>UI: regenerate
Stores->>DB: create message branch
Note right of Stores: same streaming flow
Note over User,API: ⏹️ Stop
User->>UI: stop
Stores->>Stores: abort stream
Stores->>DB: save partial response
```

77
tools/ui/docs/flows/data-flow-simplified-router-mode.md
View file

@ -1,77 +0,0 @@
```mermaid
%% ROUTER Mode Data Flow (multi-model)
%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd
sequenceDiagram
participant User as 👤 User
participant UI as 🧩 UI
participant Stores as 🗄️ Stores
participant DB as 💾 IndexedDB
participant API as 🌐 llama-server
Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd)
UI->>Stores: initialize()
Stores->>DB: load conversations
Stores->>API: GET /props
API-->>Stores: {role: "router"}
Stores->>API: GET /v1/models
API-->>Stores: models[] with status (loaded/available)
loop each loaded model
Stores->>API: GET /props?model=X
API-->>Stores: modalities (vision/audio)
end
Note over User,API: 🔄 Model Selection (see: models-flow.mmd)
User->>UI: select model
alt model not loaded
Stores->>API: POST /models/load
loop poll status
Stores->>API: GET /v1/models
API-->>Stores: check if loaded
end
Stores->>API: GET /props?model=X
API-->>Stores: cache modalities
end
Stores->>Stores: validate modalities vs conversation
alt valid
Stores->>Stores: select model
else invalid
Stores->>API: POST /models/unload
UI->>User: show error toast
end
Note over User,API: 💬 Chat Flow (see: chat-flow.mmd)
User->>UI: send message
UI->>Stores: sendMessage()
Stores->>DB: save user message
Stores->>API: POST /v1/chat/completions {model: X}
Note right of API: router forwards to model
loop streaming
API-->>Stores: SSE chunks + model info
Stores-->>UI: reactive update
end
API-->>Stores: done + timings
Stores->>DB: save assistant message + model used
Note over User,API: 🔁 Regenerate (optional: different model)
User->>UI: regenerate
Stores->>Stores: validate modalities up to this message
Stores->>DB: create message branch
Note right of Stores: same streaming flow
Note over User,API: ⏹️ Stop
User->>UI: stop
Stores->>Stores: abort stream
Stores->>DB: save partial response
Note over User,API: 🗑️ LRU Unloading
Note right of API: Server auto-unloads LRU models<br/>when cache full
User->>UI: select unloaded model
Note right of Stores: triggers load flow again
```

174
tools/ui/docs/flows/database-flow.md
View file

@ -1,174 +0,0 @@
```mermaid
sequenceDiagram
participant Store as 🗄️ Stores
participant DbSvc as ⚙️ DatabaseService
participant Dexie as 📦 Dexie ORM
participant IDB as 💾 IndexedDB
Note over DbSvc: Stateless service - all methods static<br/>Database: "LlamacppWebui"
%% ═══════════════════════════════════════════════════════════════════════════
Note over Store,IDB: 📊 SCHEMA
%% ═══════════════════════════════════════════════════════════════════════════
rect rgb(240, 248, 255)
Note over IDB: conversations table:<br/>id (PK), lastModified, currNode, name
end
rect rgb(255, 248, 240)
Note over IDB: messages table:<br/>id (PK), convId (FK), type, role, timestamp,<br/>parent, children[], content, thinking,<br/>toolCalls, extra[], model, timings
end
%% ═══════════════════════════════════════════════════════════════════════════
Note over Store,IDB: 💬 CONVERSATIONS CRUD
%% ═══════════════════════════════════════════════════════════════════════════
Store->>DbSvc: createConversation(name)
activate DbSvc
DbSvc->>DbSvc: Generate UUID
DbSvc->>Dexie: db.conversations.add({id, name, lastModified, currNode: ""})
Dexie->>IDB: INSERT
IDB-->>Dexie: success
DbSvc-->>Store: DatabaseConversation
deactivate DbSvc
Store->>DbSvc: getConversation(convId)
DbSvc->>Dexie: db.conversations.get(convId)
Dexie->>IDB: SELECT WHERE id = ?
IDB-->>DbSvc: DatabaseConversation
Store->>DbSvc: getAllConversations()
DbSvc->>Dexie: db.conversations.orderBy('lastModified').reverse().toArray()
Dexie->>IDB: SELECT ORDER BY lastModified DESC
IDB-->>DbSvc: DatabaseConversation[]
Store->>DbSvc: updateConversation(convId, updates)
DbSvc->>Dexie: db.conversations.update(convId, {...updates, lastModified})
Dexie->>IDB: UPDATE
Store->>DbSvc: deleteConversation(convId)
activate DbSvc
DbSvc->>Dexie: db.conversations.delete(convId)
Dexie->>IDB: DELETE FROM conversations
DbSvc->>Dexie: db.messages.where('convId').equals(convId).delete()
Dexie->>IDB: DELETE FROM messages WHERE convId = ?
deactivate DbSvc
%% ═══════════════════════════════════════════════════════════════════════════
Note over Store,IDB: 📝 MESSAGES CRUD
%% ═══════════════════════════════════════════════════════════════════════════
Store->>DbSvc: createRootMessage(convId)
activate DbSvc
DbSvc->>DbSvc: Create root message {type: "root", parent: null}
DbSvc->>Dexie: db.messages.add(rootMsg)
Dexie->>IDB: INSERT
DbSvc-->>Store: rootMessageId
deactivate DbSvc
Store->>DbSvc: createSystemMessage(convId, content, parentId)
activate DbSvc
DbSvc->>DbSvc: Create message {role: "system", parent: parentId}
DbSvc->>Dexie: db.messages.add(systemMsg)
Dexie->>IDB: INSERT
DbSvc-->>Store: DatabaseMessage
deactivate DbSvc
Store->>DbSvc: createMessageBranch(message, parentId)
activate DbSvc
DbSvc->>DbSvc: Generate UUID for new message
DbSvc->>Dexie: db.messages.add({...message, id, parent: parentId})
Dexie->>IDB: INSERT message
alt parentId exists
DbSvc->>Dexie: db.messages.get(parentId)
Dexie->>IDB: SELECT parent
DbSvc->>DbSvc: parent.children.push(newId)
DbSvc->>Dexie: db.messages.update(parentId, {children})
Dexie->>IDB: UPDATE parent.children
end
DbSvc->>Dexie: db.conversations.update(convId, {currNode: newId})
Dexie->>IDB: UPDATE conversation.currNode
DbSvc-->>Store: DatabaseMessage
deactivate DbSvc
Store->>DbSvc: getConversationMessages(convId)
DbSvc->>Dexie: db.messages.where('convId').equals(convId).toArray()
Dexie->>IDB: SELECT WHERE convId = ?
IDB-->>DbSvc: DatabaseMessage[]
Store->>DbSvc: updateMessage(msgId, updates)
DbSvc->>Dexie: db.messages.update(msgId, updates)
Dexie->>IDB: UPDATE
Store->>DbSvc: deleteMessage(msgId)
DbSvc->>Dexie: db.messages.delete(msgId)
Dexie->>IDB: DELETE
%% ═══════════════════════════════════════════════════════════════════════════
Note over Store,IDB: 🌳 BRANCHING OPERATIONS
%% ═══════════════════════════════════════════════════════════════════════════
Store->>DbSvc: updateCurrentNode(convId, nodeId)
DbSvc->>Dexie: db.conversations.update(convId, {currNode: nodeId, lastModified})
Dexie->>IDB: UPDATE
Store->>DbSvc: deleteMessageCascading(msgId)
activate DbSvc
DbSvc->>DbSvc: findDescendantMessages(msgId, allMessages)
Note right of DbSvc: Recursively find all children
loop each descendant
DbSvc->>Dexie: db.messages.delete(descendantId)
Dexie->>IDB: DELETE
end
DbSvc->>Dexie: db.messages.delete(msgId)
Dexie->>IDB: DELETE target message
alt target message has a parent
DbSvc->>Dexie: db.messages.get(parentId)
DbSvc->>DbSvc: parent.children.filter(id !== msgId)
DbSvc->>Dexie: db.messages.update(parentId, {children})
Note right of DbSvc: Remove deleted message from parent's children[]
end
deactivate DbSvc
%% ═══════════════════════════════════════════════════════════════════════════
Note over Store,IDB: 📥 IMPORT
%% ═══════════════════════════════════════════════════════════════════════════
Store->>DbSvc: importConversations(data)
activate DbSvc
loop each conversation in data
DbSvc->>Dexie: db.conversations.get(conv.id)
alt conversation already exists
Note right of DbSvc: Skip duplicate (keep existing)
else conversation is new
DbSvc->>Dexie: db.conversations.add(conversation)
Dexie->>IDB: INSERT conversation
loop each message
DbSvc->>Dexie: db.messages.add(message)
Dexie->>IDB: INSERT message
end
end
end
deactivate DbSvc
%% ═══════════════════════════════════════════════════════════════════════════
Note over Store,IDB: 🔗 MESSAGE TREE UTILITIES
%% ═══════════════════════════════════════════════════════════════════════════
Note over DbSvc: Used by stores (imported from utils):
rect rgb(240, 255, 240)
Note over DbSvc: filterByLeafNodeId(messages, leafId)<br/>→ Returns path from root to leaf<br/>→ Used to display current branch
end
rect rgb(240, 255, 240)
Note over DbSvc: findLeafNode(startId, messages)<br/>→ Traverse to deepest child<br/>→ Used for branch navigation
end
rect rgb(240, 255, 240)
Note over DbSvc: findDescendantMessages(msgId, messages)<br/>→ Find all children recursively<br/>→ Used for cascading deletes
end
```

226
tools/ui/docs/flows/mcp-flow.md
View file

@ -1,226 +0,0 @@
```mermaid
sequenceDiagram
participant UI as 🧩 McpServersSettings / ChatForm
participant chatStore as 🗄️ chatStore
participant mcpStore as 🗄️ mcpStore
participant mcpResStore as 🗄️ mcpResourceStore
participant convStore as 🗄️ conversationsStore
participant MCPSvc as ⚙️ MCPService
participant LS as 💾 LocalStorage
participant ExtMCP as 🔌 External MCP Server
Note over mcpStore: State:<br/>isInitializing, error<br/>toolCount, connectedServers<br/>healthChecks (Map)<br/>connections (Map)<br/>toolsIndex (Map)<br/>serverConfigs (Map)
Note over mcpResStore: State:<br/>serverResources (Map)<br/>cachedResources (Map)<br/>subscriptions (Map)<br/>attachments[]
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,ExtMCP: 🚀 INITIALIZATION (App Startup)
%% ═══════════════════════════════════════════════════════════════════════════
UI->>mcpStore: ensureInitialized()
activate mcpStore
mcpStore->>LS: get(MCP_SERVERS_LOCALSTORAGE_KEY)
LS-->>mcpStore: MCPServerSettingsEntry[]
mcpStore->>mcpStore: parseServerSettings(servers)
Note right of mcpStore: Filter enabled servers<br/>Build MCPServerConfig objects<br/>Per-chat overrides checked via convStore
loop For each enabled server
mcpStore->>mcpStore: runHealthCheck(serverId)
mcpStore->>mcpStore: updateHealthCheck(id, CONNECTING)
mcpStore->>MCPSvc: connect(serverName, config, clientInfo, capabilities, onPhase)
activate MCPSvc
MCPSvc->>MCPSvc: createTransport(config)
Note right of MCPSvc: WebSocket / StreamableHTTP / SSE<br/>with optional CORS proxy
MCPSvc->>ExtMCP: Transport handshake
ExtMCP-->>MCPSvc: Connection established
MCPSvc->>ExtMCP: Initialize request
Note right of ExtMCP: Exchange capabilities<br/>Server info, protocol version
ExtMCP-->>MCPSvc: InitializeResult (serverInfo, capabilities)
MCPSvc->>ExtMCP: listTools()
ExtMCP-->>MCPSvc: Tool[]
MCPSvc-->>mcpStore: MCPConnection
deactivate MCPSvc
mcpStore->>mcpStore: connections.set(serverName, connection)
mcpStore->>mcpStore: indexTools(connection.tools, serverName)
Note right of mcpStore: toolsIndex.set(toolName, serverName)<br/>Handle name conflicts with prefixes
mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS)
mcpStore->>mcpStore: _connectedServers.push(serverName)
alt Server supports resources
mcpStore->>MCPSvc: listAllResources(connection)
MCPSvc->>ExtMCP: listResources()
ExtMCP-->>MCPSvc: MCPResource[]
MCPSvc-->>mcpStore: resources
mcpStore->>MCPSvc: listAllResourceTemplates(connection)
MCPSvc->>ExtMCP: listResourceTemplates()
ExtMCP-->>MCPSvc: MCPResourceTemplate[]
MCPSvc-->>mcpStore: templates
mcpStore->>mcpResStore: setServerResources(serverName, resources, templates)
end
end
mcpStore->>mcpStore: _isInitializing = false
deactivate mcpStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,ExtMCP: 🔧 TOOL EXECUTION (Chat with Tools)
%% ═══════════════════════════════════════════════════════════════════════════
UI->>mcpStore: executeTool(mcpCall: MCPToolCall, signal?)
activate mcpStore
mcpStore->>mcpStore: toolsIndex.get(mcpCall.function.name)
Note right of mcpStore: Resolve serverName from toolsIndex<br/>MCPToolCall = {id, type, function: {name, arguments}}
mcpStore->>mcpStore: acquireConnection()
Note right of mcpStore: activeFlowCount++<br/>Prevent shutdown during execution
mcpStore->>mcpStore: connection = connections.get(serverName)
mcpStore->>MCPSvc: callTool(connection, {name, arguments}, signal)
activate MCPSvc
MCPSvc->>MCPSvc: throwIfAborted(signal)
MCPSvc->>ExtMCP: callTool(name, arguments)
alt Tool execution success
ExtMCP-->>MCPSvc: ToolCallResult (content, isError)
MCPSvc->>MCPSvc: formatToolResult(result)
Note right of MCPSvc: Handle text, image (base64),<br/>embedded resource content
MCPSvc-->>mcpStore: ToolExecutionResult
else Tool execution error
ExtMCP-->>MCPSvc: Error
MCPSvc-->>mcpStore: throw Error
else Aborted
MCPSvc-->>mcpStore: throw AbortError
end
deactivate MCPSvc
mcpStore->>mcpStore: releaseConnection()
Note right of mcpStore: activeFlowCount--
mcpStore-->>UI: ToolExecutionResult
deactivate mcpStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,ExtMCP: <20> RESOURCE ATTACHMENT CONSUMPTION
%% ═══════════════════════════════════════════════════════════════════════════
chatStore->>mcpStore: consumeResourceAttachmentsAsExtras()
activate mcpStore
mcpStore->>mcpResStore: getAttachments()
mcpResStore-->>mcpStore: MCPResourceAttachment[]
mcpStore->>mcpStore: Convert attachments to message extras
mcpStore->>mcpResStore: clearAttachments()
mcpStore-->>chatStore: MessageExtra[] (for user message)
deactivate mcpStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,ExtMCP: <20>📝 PROMPT OPERATIONS
%% ═══════════════════════════════════════════════════════════════════════════
UI->>mcpStore: getAllPrompts()
activate mcpStore
loop For each connected server with prompts capability
mcpStore->>MCPSvc: listPrompts(connection)
MCPSvc->>ExtMCP: listPrompts()
ExtMCP-->>MCPSvc: Prompt[]
MCPSvc-->>mcpStore: prompts
end
mcpStore-->>UI: MCPPromptInfo[] (with serverName)
deactivate mcpStore
UI->>mcpStore: getPrompt(serverName, promptName, args?)
activate mcpStore
mcpStore->>MCPSvc: getPrompt(connection, name, args)
MCPSvc->>ExtMCP: getPrompt({name, arguments})
ExtMCP-->>MCPSvc: GetPromptResult (messages)
MCPSvc-->>mcpStore: GetPromptResult
mcpStore-->>UI: GetPromptResult
deactivate mcpStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,ExtMCP: 📁 RESOURCE OPERATIONS
%% ═══════════════════════════════════════════════════════════════════════════
UI->>mcpResStore: addAttachment(resourceInfo)
activate mcpResStore
mcpResStore->>mcpResStore: Create MCPResourceAttachment (loading: true)
mcpResStore-->>UI: attachment
UI->>mcpStore: readResource(serverName, uri)
activate mcpStore
mcpStore->>MCPSvc: readResource(connection, uri)
MCPSvc->>ExtMCP: readResource({uri})
ExtMCP-->>MCPSvc: MCPReadResourceResult (contents)
MCPSvc-->>mcpStore: contents
mcpStore-->>UI: MCPResourceContent[]
deactivate mcpStore
UI->>mcpResStore: updateAttachmentContent(attachmentId, content)
mcpResStore->>mcpResStore: cacheResourceContent(resource, content)
deactivate mcpResStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,ExtMCP: 🔄 AUTO-RECONNECTION
%% ═══════════════════════════════════════════════════════════════════════════
Note over mcpStore: On WebSocket close or connection error:
mcpStore->>mcpStore: autoReconnect(serverName, attempt)
activate mcpStore
mcpStore->>mcpStore: Calculate backoff delay
Note right of mcpStore: delay = min(30s, 1s * 2^attempt)
mcpStore->>mcpStore: Wait for delay
mcpStore->>mcpStore: reconnectServer(serverName)
alt Reconnection success
mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS)
else Max attempts reached
mcpStore->>mcpStore: updateHealthCheck(id, ERROR)
end
deactivate mcpStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,ExtMCP: 🛑 SHUTDOWN
%% ═══════════════════════════════════════════════════════════════════════════
UI->>mcpStore: shutdown()
activate mcpStore
mcpStore->>mcpStore: Wait for activeFlowCount == 0
loop For each connection
mcpStore->>MCPSvc: disconnect(connection)
MCPSvc->>MCPSvc: transport.onclose = undefined
MCPSvc->>ExtMCP: close()
end
mcpStore->>mcpStore: connections.clear()
mcpStore->>mcpStore: toolsIndex.clear()
mcpStore->>mcpStore: _connectedServers = []
mcpStore->>mcpResStore: clear()
deactivate mcpStore
```

181
tools/ui/docs/flows/models-flow.md
View file

@ -1,181 +0,0 @@
```mermaid
sequenceDiagram
participant UI as 🧩 ModelsSelector
participant Hooks as 🪝 useModelChangeValidation
participant modelsStore as 🗄️ modelsStore
participant serverStore as 🗄️ serverStore
participant convStore as 🗄️ conversationsStore
participant ModelsSvc as ⚙️ ModelsService
participant PropsSvc as ⚙️ PropsService
participant API as 🌐 llama-server
Note over modelsStore: State:<br/>models: ModelOption[]<br/>routerModels: ApiModelDataEntry[]<br/>selectedModelId, selectedModelName<br/>loading, updating, error<br/>modelLoadingStates (Map)<br/>modelPropsCache (Map)<br/>propsCacheVersion
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 🚀 INITIALIZATION (MODEL mode)
%% ═══════════════════════════════════════════════════════════════════════════
UI->>modelsStore: fetch()
activate modelsStore
modelsStore->>modelsStore: loading = true
alt serverStore.props not loaded
modelsStore->>serverStore: fetch()
Note over serverStore: → see server-flow.mmd
end
modelsStore->>ModelsSvc: list()
ModelsSvc->>API: GET /v1/models
API-->>ModelsSvc: ApiModelListResponse {data: [model]}
modelsStore->>modelsStore: models = $state(mapped)
Note right of modelsStore: Map to ModelOption[]:<br/>{id, name, model, description, capabilities}
Note over modelsStore: MODEL mode: Get modalities from serverStore.props
modelsStore->>modelsStore: modelPropsCache.set(model.id, serverStore.props)
modelsStore->>modelsStore: models[0].modalities = props.modalities
modelsStore->>modelsStore: Auto-select single model
Note right of modelsStore: selectedModelId = models[0].id
modelsStore->>modelsStore: loading = false
deactivate modelsStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 🚀 INITIALIZATION (ROUTER mode)
%% ═══════════════════════════════════════════════════════════════════════════
UI->>modelsStore: fetch()
activate modelsStore
modelsStore->>ModelsSvc: list()
ModelsSvc->>API: GET /v1/models
API-->>ModelsSvc: ApiModelListResponse
modelsStore->>modelsStore: models = $state(mapped)
deactivate modelsStore
Note over UI: After models loaded, layout triggers:
UI->>modelsStore: fetchRouterModels()
activate modelsStore
modelsStore->>ModelsSvc: listRouter()
ModelsSvc->>API: GET /v1/models
API-->>ModelsSvc: ApiRouterModelsListResponse
Note right of API: {data: [{id, status, path, in_cache}]}
modelsStore->>modelsStore: routerModels = $state(data)
modelsStore->>modelsStore: fetchModalitiesForLoadedModels()
loop each model where status === "loaded"
modelsStore->>PropsSvc: fetchForModel(modelId)
PropsSvc->>API: GET /props?model={modelId}
API-->>PropsSvc: ApiLlamaCppServerProps
modelsStore->>modelsStore: modelPropsCache.set(modelId, props)
end
modelsStore->>modelsStore: propsCacheVersion++
deactivate modelsStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 🔄 MODEL SELECTION (ROUTER mode)
%% ═══════════════════════════════════════════════════════════════════════════
UI->>Hooks: useModelChangeValidation({getRequiredModalities, onSuccess?, onValidationFailure?})
Note over Hooks: Hook configured per-component:<br/>ChatForm: getRequiredModalities = usedModalities<br/>ChatMessage: getRequiredModalities = getModalitiesUpToMessage(msgId)
UI->>Hooks: handleModelChange(modelId, modelName)
activate Hooks
Hooks->>Hooks: previousSelectedModelId = modelsStore.selectedModelId
Hooks->>modelsStore: isModelLoaded(modelName)?
alt model NOT loaded
Hooks->>modelsStore: loadModel(modelName)
Note over modelsStore: → see LOAD MODEL section below
end
Note over Hooks: Always fetch props (from cache or API)
Hooks->>modelsStore: fetchModelProps(modelName)
modelsStore-->>Hooks: props
Hooks->>convStore: getRequiredModalities()
convStore-->>Hooks: {vision, audio}
Hooks->>Hooks: Validate: model.modalities ⊇ required?
alt validation PASSED
Hooks->>modelsStore: selectModelById(modelId)
Hooks-->>UI: return true
else validation FAILED
Hooks->>UI: toast.error("Model doesn't support required modalities")
alt model was just loaded
Hooks->>modelsStore: unloadModel(modelName)
end
alt onValidationFailure provided
Hooks->>modelsStore: selectModelById(previousSelectedModelId)
end
Hooks-->>UI: return false
end
deactivate Hooks
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: ⬆️ LOAD MODEL (ROUTER mode)
%% ═══════════════════════════════════════════════════════════════════════════
modelsStore->>modelsStore: loadModel(modelId)
activate modelsStore
alt already loaded
modelsStore-->>modelsStore: return (no-op)
end
modelsStore->>modelsStore: modelLoadingStates.set(modelId, true)
modelsStore->>ModelsSvc: load(modelId)
ModelsSvc->>API: POST /models/load {model: modelId}
API-->>ModelsSvc: {status: "loading"}
modelsStore->>modelsStore: pollForModelStatus(modelId, LOADED)
loop poll every 500ms (max 60 attempts)
modelsStore->>modelsStore: fetchRouterModels()
modelsStore->>ModelsSvc: listRouter()
ModelsSvc->>API: GET /v1/models
API-->>ModelsSvc: models[]
modelsStore->>modelsStore: getModelStatus(modelId)
alt status === LOADED
Note right of modelsStore: break loop
else status === LOADING
Note right of modelsStore: wait 500ms, continue
end
end
modelsStore->>modelsStore: updateModelModalities(modelId)
modelsStore->>PropsSvc: fetchForModel(modelId)
PropsSvc->>API: GET /props?model={modelId}
API-->>PropsSvc: props with modalities
modelsStore->>modelsStore: modelPropsCache.set(modelId, props)
modelsStore->>modelsStore: propsCacheVersion++
modelsStore->>modelsStore: modelLoadingStates.set(modelId, false)
deactivate modelsStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: ⬇️ UNLOAD MODEL (ROUTER mode)
%% ═══════════════════════════════════════════════════════════════════════════
modelsStore->>modelsStore: unloadModel(modelId)
activate modelsStore
modelsStore->>modelsStore: modelLoadingStates.set(modelId, true)
modelsStore->>ModelsSvc: unload(modelId)
ModelsSvc->>API: POST /models/unload {model: modelId}
modelsStore->>modelsStore: pollForModelStatus(modelId, UNLOADED)
loop poll until unloaded
modelsStore->>ModelsSvc: listRouter()
ModelsSvc->>API: GET /v1/models
end
modelsStore->>modelsStore: modelLoadingStates.set(modelId, false)
deactivate modelsStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 📊 COMPUTED GETTERS
%% ═══════════════════════════════════════════════════════════════════════════
Note over modelsStore: Getters:<br/>- selectedModel: ModelOption | null<br/>- loadedModelIds: string[] (from routerModels)<br/>- loadingModelIds: string[] (from modelLoadingStates)<br/>- singleModelName: string | null (MODEL mode only)
Note over modelsStore: Modality helpers:<br/>- getModelModalities(modelId): {vision, audio}<br/>- modelSupportsVision(modelId): boolean<br/>- modelSupportsAudio(modelId): boolean
```

76
tools/ui/docs/flows/server-flow.md
View file

@ -1,76 +0,0 @@
```mermaid
sequenceDiagram
participant UI as 🧩 +layout.svelte
participant serverStore as 🗄️ serverStore
participant PropsSvc as ⚙️ PropsService
participant API as 🌐 llama-server
Note over serverStore: State:<br/>props: ApiLlamaCppServerProps | null<br/>loading, error<br/>role: ServerRole | null (MODEL | ROUTER)<br/>fetchPromise (deduplication)
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 🚀 INITIALIZATION
%% ═══════════════════════════════════════════════════════════════════════════
UI->>serverStore: fetch()
activate serverStore
alt fetchPromise exists (already fetching)
serverStore-->>UI: return fetchPromise
Note right of serverStore: Deduplicate concurrent calls
end
serverStore->>serverStore: loading = true
serverStore->>serverStore: fetchPromise = new Promise()
serverStore->>PropsSvc: fetch()
PropsSvc->>API: GET /props
API-->>PropsSvc: ApiLlamaCppServerProps
Note right of API: {role, model_path, model_alias,<br/>modalities, default_generation_settings, ...}
PropsSvc-->>serverStore: props
serverStore->>serverStore: props = $state(data)
serverStore->>serverStore: detectRole(props)
Note right of serverStore: role = props.role === "router"<br/> ? ServerRole.ROUTER<br/> : ServerRole.MODEL
serverStore->>serverStore: loading = false
serverStore->>serverStore: fetchPromise = null
deactivate serverStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 📊 COMPUTED GETTERS
%% ═══════════════════════════════════════════════════════════════════════════
Note over serverStore: Getters from props:
rect rgb(240, 255, 240)
Note over serverStore: defaultParams<br/>→ props.default_generation_settings.params<br/>(temperature, top_p, top_k, etc.)
end
rect rgb(240, 255, 240)
Note over serverStore: contextSize<br/>→ props.default_generation_settings.n_ctx
end
rect rgb(255, 240, 240)
Note over serverStore: isRouterMode<br/>→ role === ServerRole.ROUTER
end
rect rgb(255, 240, 240)
Note over serverStore: isModelMode<br/>→ role === ServerRole.MODEL
end
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: 🔗 RELATIONSHIPS
%% ═══════════════════════════════════════════════════════════════════════════
Note over serverStore: Used by:
Note right of serverStore: - modelsStore: role detection, MODEL mode modalities<br/>- settingsStore: syncWithServerDefaults (defaultParams)<br/>- chatStore: contextSize for processing state<br/>- UI components: isRouterMode for conditional rendering
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,API: ❌ ERROR HANDLING
%% ═══════════════════════════════════════════════════════════════════════════
Note over serverStore: getErrorMessage(): string | null<br/>Returns formatted error for UI display
Note over serverStore: clear(): void<br/>Resets all state (props, error, loading, role)
```

156
tools/ui/docs/flows/settings-flow.md
View file

@ -1,156 +0,0 @@
```mermaid
sequenceDiagram
participant UI as 🧩 ChatSettings
participant settingsStore as 🗄️ settingsStore
participant serverStore as 🗄️ serverStore
participant ParamSvc as ⚙️ ParameterSyncService
participant LS as 💾 LocalStorage
Note over settingsStore: State:<br/>config: SettingsConfigType<br/>theme: string ("auto" | "light" | "dark")<br/>isInitialized: boolean<br/>userOverrides: Set&lt;string&gt;
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,LS: 🚀 INITIALIZATION
%% ═══════════════════════════════════════════════════════════════════════════
Note over settingsStore: Auto-initialized in constructor (browser only)
settingsStore->>settingsStore: initialize()
activate settingsStore
settingsStore->>settingsStore: loadConfig()
settingsStore->>LS: get("llama-config")
LS-->>settingsStore: StoredConfig | null
alt config exists
settingsStore->>settingsStore: Merge with SETTING_CONFIG_DEFAULT
Note right of settingsStore: Fill missing keys with defaults
else no config
settingsStore->>settingsStore: config = SETTING_CONFIG_DEFAULT
end
settingsStore->>LS: get("llama-userOverrides")
LS-->>settingsStore: string[] | null
settingsStore->>settingsStore: userOverrides = new Set(data)
settingsStore->>settingsStore: loadTheme()
settingsStore->>LS: get("llama-theme")
LS-->>settingsStore: theme | "auto"
settingsStore->>settingsStore: isInitialized = true
deactivate settingsStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,LS: 🔄 SYNC WITH SERVER DEFAULTS
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI: Triggered from +layout.svelte when serverStore.props loaded
UI->>settingsStore: syncWithServerDefaults()
activate settingsStore
settingsStore->>serverStore: defaultParams
serverStore-->>settingsStore: {temperature, top_p, top_k, ...}
loop each SYNCABLE_PARAMETER
alt key NOT in userOverrides
settingsStore->>settingsStore: config[key] = serverDefault[key]
Note right of settingsStore: Non-overridden params adopt server default
else key in userOverrides
Note right of settingsStore: Keep user value, skip server default
end
end
alt serverStore.props has uiSettings
settingsStore->>settingsStore: Apply uiSettings from server
Note right of settingsStore: Server-provided UI settings<br/>(e.g. showRawOutputSwitch)
end
settingsStore->>settingsStore: saveConfig()
deactivate settingsStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,LS: ⚙️ UPDATE CONFIG
%% ═══════════════════════════════════════════════════════════════════════════
UI->>settingsStore: updateConfig(key, value)
activate settingsStore
settingsStore->>settingsStore: config[key] = value
alt value matches server default for key
settingsStore->>settingsStore: userOverrides.delete(key)
Note right of settingsStore: Matches server default, remove override
else value differs from server default
settingsStore->>settingsStore: userOverrides.add(key)
Note right of settingsStore: Mark as user-modified (won't be overwritten)
end
settingsStore->>settingsStore: saveConfig()
settingsStore->>LS: set(CONFIG_LOCALSTORAGE_KEY, config)
settingsStore->>LS: set(USER_OVERRIDES_LOCALSTORAGE_KEY, [...userOverrides])
deactivate settingsStore
UI->>settingsStore: updateMultipleConfig({key1: val1, key2: val2})
activate settingsStore
Note right of settingsStore: Batch update, single save
settingsStore->>settingsStore: For each key: config[key] = value
settingsStore->>settingsStore: For each key: userOverrides.add(key)
settingsStore->>settingsStore: saveConfig()
deactivate settingsStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,LS: 🔄 RESET
%% ═══════════════════════════════════════════════════════════════════════════
UI->>settingsStore: resetConfig()
activate settingsStore
settingsStore->>settingsStore: config = {...SETTING_CONFIG_DEFAULT}
settingsStore->>settingsStore: userOverrides.clear()
Note right of settingsStore: All params reset to defaults<br/>Next syncWithServerDefaults will adopt server values
settingsStore->>settingsStore: saveConfig()
deactivate settingsStore
UI->>settingsStore: resetParameterToServerDefault(key)
activate settingsStore
settingsStore->>settingsStore: userOverrides.delete(key)
settingsStore->>serverStore: defaultParams[key]
settingsStore->>settingsStore: config[key] = serverDefault
settingsStore->>settingsStore: saveConfig()
deactivate settingsStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,LS: 🎨 THEME
%% ═══════════════════════════════════════════════════════════════════════════
UI->>settingsStore: updateTheme(newTheme)
activate settingsStore
settingsStore->>settingsStore: theme = newTheme
settingsStore->>settingsStore: saveTheme()
settingsStore->>LS: set("llama-theme", theme)
deactivate settingsStore
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,LS: 📊 PARAMETER INFO
%% ═══════════════════════════════════════════════════════════════════════════
UI->>settingsStore: getParameterInfo(key)
settingsStore->>ParamSvc: getParameterInfo(key, config, serverDefaults, userOverrides)
ParamSvc-->>settingsStore: ParameterInfo
Note right of ParamSvc: {<br/> currentValue,<br/> serverDefault,<br/> isUserOverride: boolean,<br/> canSync: boolean,<br/> isDifferentFromServer: boolean<br/>}
UI->>settingsStore: getParameterDiff()
settingsStore->>ParamSvc: createParameterDiff(config, serverDefaults, userOverrides)
ParamSvc-->>settingsStore: ParameterDiff[]
Note right of ParamSvc: Array of parameters where user != server
%% ═══════════════════════════════════════════════════════════════════════════
Note over UI,LS: 📋 CONFIG CATEGORIES
%% ═══════════════════════════════════════════════════════════════════════════
Note over settingsStore: Syncable with server (from /props):
rect rgb(240, 255, 240)
Note over settingsStore: temperature, top_p, top_k, min_p<br/>repeat_penalty, presence_penalty, frequency_penalty<br/>dynatemp_range, dynatemp_exponent<br/>typ_p, xtc_probability, xtc_threshold<br/>dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n
end
Note over settingsStore: UI-only (not synced):
rect rgb(255, 240, 240)
Note over settingsStore: systemMessage, custom (JSON)<br/>showStatistics, enableContinueGeneration<br/>autoMicOnEmpty, disableAutoScroll<br/>apiKey, pdfAsImage, disableReasoningParsing, showRawOutputSwitch
end
```

84
tools/ui/eslint.config.js
View file

@ -12,6 +12,49 @@ import { fileURLToPath } from 'node:url';
import ts from 'typescript-eslint';
const gitignorePath = fileURLToPath(new URL('./.gitignore', import.meta.url));
// Require a blank line between consecutive class accessors (get/set). The core
// `padding-line-between-statements` rule only handles statements, not class
// members, so this is enforced with a small custom rule.
const blankLineBetweenAccessors = {
create(context) {
return {
MethodDefinition(node) {
if (node.kind !== 'get' && node.kind !== 'set') return;
const body = node.parent;
if (!body || body.type !== 'ClassBody') return;
const index = body.body.indexOf(node);
if (index <= 0) return;
const prev = body.body[index - 1];
if (prev.type !== 'MethodDefinition' || (prev.kind !== 'get' && prev.kind !== 'set'))
return;
if (node.loc.start.line - prev.loc.end.line <= 1) {
context.report({
fix(fixer) {
// Insert after the previous accessor's closing brace so the blank
// line keeps the current accessor's indentation.
return fixer.insertTextAfter(prev, '\n');
},
message: 'Expected a blank line between class accessors (get/set).',
node
});
}
}
};
},
meta: {
docs: { description: 'Require a blank line between consecutive class accessors (get/set).' },
fixable: 'whitespace',
schema: [],
type: 'layout'
}
};
export default ts.config(
includeIgnoreFile(gitignorePath),
@ -22,7 +65,11 @@ export default ts.config(
...svelte.configs.prettier,
{
languageOptions: { globals: { ...globals.browser, ...globals.node } },
plugins: { perfectionist, 'simple-import-sort': simpleImportSort },
plugins: {
local: { rules: { 'blank-line-between-accessors': blankLineBetweenAccessors } },
perfectionist,
'simple-import-sort': simpleImportSort
},
rules: {
// Snippet bodies often ignore one or more of the parent's params
// (e.g. `{#snippet children(_meta, ctx)}` when only ctx is read).
@ -30,8 +77,11 @@ export default ts.config(
'error',
{ argsIgnorePattern: '^_', varsIgnorePattern: '^_' }
],
// Enforce empty line at end of file
'eol-last': 'error',
// Enforce a blank line between consecutive get/set accessors
'local/blank-line-between-accessors': 'error',
// typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects.
// see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
'no-undef': 'off',
@ -61,6 +111,38 @@ export default ts.config(
{ blankLine: 'always', next: ['return', 'throw', 'break', 'continue'], prev: '*' }
],
// Class member order: public fields -> private fields -> constructor -> getters
// -> setters -> public methods -> private methods, alphabetical within each.
// Svelte $derived fields must stay in dependency order (forward references are
// rejected), so the two stores that rely on that are exempted below.
'perfectionist/sort-classes': [
'error',
{
customGroups: [
{ groupName: 'public-field', modifiers: ['public'], selector: 'property' },
{ groupName: 'private-field', modifiers: ['private'], selector: 'property' },
{ groupName: 'get-method', selector: 'get-method' },
{ groupName: 'set-method', selector: 'set-method' },
{ groupName: 'public-method', modifiers: ['public'], selector: 'method' },
{ groupName: 'private-method', modifiers: ['private'], selector: 'method' }
],
groups: [
'public-field',
'private-field',
'constructor',
'get-method',
'set-method',
'public-method',
'private-method',
'unknown'
],
type: 'natural',
// Keep members in dependency order (Svelte rejects forward references in
// $derived fields), while still sorting the rest alphabetically.
useExperimentalDependencyDetection: true
}
],
// Alphabetical order for enum members
'perfectionist/sort-enums': ['error', { type: 'natural' }],

2
tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte
View file

@ -139,7 +139,7 @@
let fileSize = $derived(currentItem?.size ? formatFileSize(currentItem.size) : '');
let hasVisionModality = $derived(
currentItem && activeModelId ? modelsStore.modelSupportsVision(activeModelId) : false
currentItem && activeModelId ? modelsStore.props.modelSupportsVision(activeModelId) : false
);
let audioSrc = $derived(

12
tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte
View file

@ -28,7 +28,6 @@
import {
chatStore,
conversationsStore,
mcpResourceStore,
mcpStore,
modelsStore,
serverStore,
@ -140,7 +139,9 @@
// float above the box.
let mentionAnchor: HTMLDivElement | null = $state(null);
let cwd = $derived(conversationsStore.activeConversation?.cwd ?? conversationsStore.pendingCwd);
let cwd = $derived(
conversationsStore.activeConversation?.cwd ?? conversationsStore.preferences.pendingCwd
);
const pickers = useChatFormPickers({
focusInput: refocusInput,
@ -151,7 +152,8 @@
getShowModelSelector: () => showModelSelector,
getValue: () => value,
hasCwdTools: () => toolsStore.hasEnabledCwdTools,
hasPrompts: () => mcpStore.hasPromptsCapability(conversationsStore.getAllMcpServerOverrides()),
hasPrompts: () =>
mcpStore.hasPromptsCapability(conversationsStore.preferences.getAllMcpServerOverrides()),
openModelSelector: () => chatFormActionsRef?.openModelSelector(),
setCaretOffset: (offset) => inputRef?.setCaretOffset(offset),
setValue: (v) => {
@ -170,7 +172,7 @@
onValueChange?.('');
}
await conversationsStore.setCwd(newDir);
await conversationsStore.preferences.setCwd(newDir);
if (conversationsStore.activeConversation) {
await chatStore.recordCwdChange(newDir?.trim() || null);
@ -595,7 +597,7 @@
{useRichInput}
/>
{#if mcpResourceStore.hasAttachments}
{#if mcpStore.resources.hasAttachments}
<ChatFormMcpResourcesList
class="mb-3"
onResourceClick={(uri) => {

4
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte
View file

@ -38,11 +38,11 @@
}
function isServerEnabledForChat(serverId: string): boolean {
return conversationsStore.isMcpServerEnabledForChat(serverId);
return conversationsStore.preferences.isMcpServerEnabledForChat(serverId);
}
async function toggleServerForChat(serverId: string) {
await conversationsStore.toggleMcpServerForChat(serverId);
await conversationsStore.preferences.toggleMcpServerForChat(serverId);
}
function handleMcpSubMenuOpen(open: boolean) {

10
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte
View file

@ -218,12 +218,15 @@
{@const hasError = healthState.status === HealthCheckStatus.ERROR}
{@const displayName = mcpStore.getServerLabel(server)}
{@const faviconUrl = mcpStore.getServerFavicon(server.id)}
{@const isEnabled = conversationsStore.isMcpServerEnabledForChat(server.id)}
{@const isEnabled = conversationsStore.preferences.isMcpServerEnabledForChat(
server.id
)}
<button
type="button"
class={sheetItemRowClass}
onclick={() => !hasError && conversationsStore.toggleMcpServerForChat(server.id)}
onclick={() =>
!hasError && conversationsStore.preferences.toggleMcpServerForChat(server.id)}
disabled={hasError}
>
<div class="flex min-w-0 flex-1 items-center gap-2">
@ -250,7 +253,8 @@
{:else}
<Switch
checked={isEnabled}
onCheckedChange={() => conversationsStore.toggleMcpServerForChat(server.id)}
onCheckedChange={() =>
conversationsStore.preferences.toggleMcpServerForChat(server.id)}
/>
{/if}
</button>

12
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte
View file

@ -81,10 +81,10 @@
$effect(() => {
if (activeModelId) {
const cached = modelsStore.getModelProps(activeModelId);
const cached = modelsStore.props.getModelProps(activeModelId);
if (!cached) {
modelsStore.fetchModelProps(activeModelId).then(() => {
modelsStore.props.fetchModelProps(activeModelId).then(() => {
modelPropsVersion++;
});
}
@ -94,19 +94,21 @@
$effect(() => {
void modelPropsVersion;
hasAudioModality = activeModelId ? modelsStore.modelSupportsAudio(activeModelId) : false;
hasAudioModality = activeModelId ? modelsStore.props.modelSupportsAudio(activeModelId) : false;
});
$effect(() => {
void modelPropsVersion;
hasVideoModality = activeModelId ? modelsStore.modelSupportsVideo(activeModelId) : false;
hasVideoModality = activeModelId ? modelsStore.props.modelSupportsVideo(activeModelId) : false;
});
$effect(() => {
void modelPropsVersion;
hasVisionModality = activeModelId ? modelsStore.modelSupportsVision(activeModelId) : false;
hasVisionModality = activeModelId
? modelsStore.props.modelSupportsVision(activeModelId)
: false;
});
$effect(() => {

6
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte
View file

@ -58,13 +58,13 @@
let currentConfig = $derived(settingsStore.config);
let hasMcpPromptsSupport = $derived.by(() => {
const perChatOverrides = conversationsStore.getAllMcpServerOverrides();
const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides();
return mcpStore.hasPromptsCapability(perChatOverrides);
});
let hasMcpResourcesSupport = $derived.by(() => {
const perChatOverrides = conversationsStore.getAllMcpServerOverrides();
const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides();
return mcpStore.hasResourcesCapability(perChatOverrides);
});
@ -121,7 +121,7 @@
if (!chatStore.isLoading && !chatStore.isStreaming()) return false;
const processingState = chatStore.activeProcessingState;
const processingState = chatStore.processing.activeState;
if (!processingState) return false;

6
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte
View file

@ -16,7 +16,7 @@
$effect(() => {
const conv = conversationsStore.activeConversation;
untrack(() => chatStore.setActiveProcessingConversation(conv?.id ?? null));
untrack(() => chatStore.processing.setActiveConversation(conv?.id ?? null));
});
$effect(() => {
@ -28,12 +28,12 @@
if (chatStore.isLoading || chatStore.isStreaming()) return;
if (messages.length === 0) {
untrack(() => chatStore.clearProcessingState(conv.id));
untrack(() => chatStore.processing.setState(conv.id, null));
return;
}
untrack(() => chatStore.restoreProcessingStateFromMessages(messages, conv.id));
untrack(() => chatStore.processing.restoreFromMessages(messages, conv.id));
});
$effect(() => {

6
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte
View file

@ -3,7 +3,7 @@
ChatAttachmentsListItemMcpResource,
HorizontalScrollCarousel
} from '$lib/components/app';
import { mcpResourceStore, mcpStore } from '$lib/stores';
import { mcpStore } from '$lib/stores';
interface Props {
class?: string;
@ -12,8 +12,8 @@
let { class: className, onResourceClick }: Props = $props();
const attachments = $derived(mcpResourceStore.attachments);
const hasAttachments = $derived(mcpResourceStore.hasAttachments);
const attachments = $derived(mcpStore.resources.attachments);
const hasAttachments = $derived(mcpStore.resources.hasAttachments);
function handleRemove(attachmentId: string) {
mcpStore.removeResourceAttachment(attachmentId);

2
tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte
View file

@ -87,7 +87,7 @@
isLoading = true;
try {
const perChatOverrides = conversationsStore.getAllMcpServerOverrides();
const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides();
const initialized = await mcpStore.ensureInitialized(perChatOverrides);
if (!initialized) {

2
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte
View file

@ -59,7 +59,7 @@
message.model ?? chatStore.getResumeModel(message.convId) ?? modelsStore.selectedModelName
);
let modelLoadProgress = $derived(
isRouter && loadTargetModel ? modelsStore.getLoadProgress(loadTargetModel) : null
isRouter && loadTargetModel ? modelsStore.status.getLoadProgress(loadTargetModel) : null
);
let modelLoadingText = $derived(modelLoadProgressText(modelLoadProgress));

2
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte
View file

@ -31,7 +31,7 @@
pendingModel = modelId;
try {
await modelsStore.loadModel(modelId);
await modelsStore.status.load(modelId);
} finally {
pendingModel = null;
}

8
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte
View file

@ -43,14 +43,14 @@
);
const hasReasoningError = $derived(
isLastAssistantMessage ? !!agenticStore.lastError(message.convId) : false
isLastAssistantMessage ? !!agenticStore.getLastError(message.convId) : false
);
let permissionDismissed = $state(false);
const pendingPermission = $derived(
isStreaming && isLastAssistantMessage
? agenticStore.pendingPermissionRequest(message.convId)
? agenticStore.getPendingPermissionRequest(message.convId)
: null
);
@ -74,7 +74,7 @@
const pendingContinue = $derived(
isStreaming && isLastAssistantMessage
? agenticStore.pendingContinueRequest(message.convId)
? agenticStore.getPendingContinueRequest(message.convId)
: false
);
@ -97,7 +97,7 @@
const sections = $derived(deriveAgenticSections(message, toolMessages, [], isStreaming));
const currentlyExecutingToolCallId = $derived(
isStreaming ? agenticStore.executingToolCallId(message.convId) : null
isStreaming ? agenticStore.getExecutingToolCallId(message.convId) : null
);
type TurnGroup = {

12
tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte
View file

@ -238,30 +238,30 @@
/>
{/each}
{#if conversationsStore.activeConversation && agenticStore.pendingSteeringMessageContent(conversationsStore.activeConversation!.id)}
{#if conversationsStore.activeConversation && agenticStore.getPendingSteeringMessageContent(conversationsStore.activeConversation!.id)}
{@const convId = conversationsStore.activeConversation!.id}
{@const pendingContent = agenticStore.pendingSteeringMessageContent(convId)}
{@const pendingContent = agenticStore.getPendingSteeringMessageContent(convId)}
{#if pendingContent}
<ChatMessageUserPending
class="mx-auto mt-12 w-full max-w-[48rem]"
content={pendingContent}
extras={agenticStore.pendingSteeringMessageExtras(convId)}
extras={agenticStore.getPendingSteeringMessageExtras(convId)}
onSendImmediately={() => chatStore.abortCurrentFlow(convId)}
onEdit={(newContent, extras) =>
agenticStore.injectSteeringMessage(convId, newContent, extras)}
onDelete={() => agenticStore.clearSteeringMessage(convId)}
/>
{/if}
{:else if conversationsStore.activeConversation && chatStore.pendingMessageContent(conversationsStore.activeConversation!.id)}
{:else if conversationsStore.activeConversation && chatStore.getPendingMessageContent(conversationsStore.activeConversation!.id)}
{@const convId = conversationsStore.activeConversation!.id}
{@const pendingContent = chatStore.pendingMessageContent(convId)}
{@const pendingContent = chatStore.getPendingMessageContent(convId)}
{#if pendingContent}
<ChatMessageUserPending
class="mx-auto mt-12 w-full max-w-[48rem]"
content={pendingContent}
extras={chatStore.pendingMessageExtras(convId)}
extras={chatStore.getPendingMessageExtras(convId)}
onSendImmediately={() => chatStore.abortCurrentFlow(convId)}
onEdit={(newContent, extras) => chatStore.injectPendingMessage(convId, newContent, extras)}
onDelete={() => chatStore.clearPendingMessage(convId)}

18
tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte
View file

@ -8,7 +8,7 @@
import { Button } from '$lib/components/ui/button';
import * as Dialog from '$lib/components/ui/dialog';
import { ICON_CLASS_DEFAULT } from '$lib/constants';
import { conversationsStore, mcpResourceStore, mcpStore } from '$lib/stores';
import { conversationsStore, mcpStore } from '$lib/stores';
import type { MCPResourceContent, MCPResourceInfo, MCPResourceTemplateInfo } from '$lib/types';
import { getResourceDisplayName } from '$lib/utils';
import { SvelteSet } from 'svelte/reactivity';
@ -33,7 +33,7 @@
let templatePreviewLoading = $state(false);
let templatePreviewError = $state<string | null>(null);
const totalCount = $derived(mcpResourceStore.totalResourceCount);
const totalCount = $derived(mcpStore.resources.totalResourceCount);
$effect(() => {
if (open) {
@ -48,7 +48,7 @@
});
async function loadResources() {
const perChatOverrides = conversationsStore.getAllMcpServerOverrides();
const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides();
const initialized = await mcpStore.ensureInitialized(perChatOverrides);
if (initialized) {
@ -126,16 +126,16 @@
isAttaching = true;
try {
const knownResource = mcpResourceStore.findResourceByUri(templatePreviewUri);
const knownResource = mcpStore.resources.findResourceByUri(templatePreviewUri);
if (knownResource) {
if (!mcpResourceStore.isAttached(knownResource.uri)) {
if (!mcpStore.resources.isAttached(knownResource.uri)) {
await mcpStore.attachResource(knownResource.uri);
}
toast.success(`Resource attached: ${knownResource.title || knownResource.name}`);
} else {
if (mcpResourceStore.isAttached(templatePreviewUri)) {
if (mcpStore.resources.isAttached(templatePreviewUri)) {
toast.info('Resource already attached');
handleOpenChange(false);
@ -147,9 +147,9 @@
serverName: selectedTemplate.serverName,
uri: templatePreviewUri
};
const attachment = mcpResourceStore.addAttachment(resourceInfo);
const attachment = mcpStore.resources.addAttachment(resourceInfo);
mcpResourceStore.updateAttachmentContent(attachment.id, templatePreviewContent);
mcpStore.resources.updateAttachmentContent(attachment.id, templatePreviewContent);
toast.success(`Resource attached: ${resourceInfo.name}`);
}
@ -199,7 +199,7 @@
function getAllResourcesFlatInTreeOrder(): MCPResourceInfo[] {
const allResources: MCPResourceInfo[] = [];
const resourcesMap = mcpResourceStore.serverResources;
const resourcesMap = mcpStore.resources.serverResources;
for (const [serverName, serverRes] of resourcesMap.entries()) {
for (const resource of serverRes.resources) {

2
tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte
View file

@ -234,7 +234,7 @@
useProxy: newServerUseProxy
});
conversationsStore.setMcpServerOverride(newServerId, true);
conversationsStore.preferences.setMcpServerOverride(newServerId, true);
handleOpenChange(false);
}

4
tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte
View file

@ -42,7 +42,7 @@
let modalities = $derived.by(() => {
if (!firstModel?.id) return [];
return modelsStore.getModelModalitiesArray(firstModel.id);
return modelsStore.props.getModelModalitiesArray(firstModel.id);
});
// Ensure models are fetched when dialog opens
@ -56,7 +56,7 @@
$effect(() => {
if (open && isRouter && modelId) {
isLoadingRouterProps = true;
modelsStore
modelsStore.props
.fetchModelProps(modelId)
.then((props) => {
routerModelProps = props;

4
tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte
View file

@ -14,7 +14,9 @@
let mcpServers = $derived(mcpStore.getServers().filter((s) => s.enabled));
let enabledMcpServersForChat = $derived(
mcpServers.filter((s) => conversationsStore.isMcpServerEnabledForChat(s.id) && s.url.trim())
mcpServers.filter(
(s) => conversationsStore.preferences.isMcpServerEnabledForChat(s.id) && s.url.trim()
)
);
let healthyEnabledMcpServers = $derived(
enabledMcpServersForChat.filter((s) => {

6
tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte
View file

@ -2,7 +2,7 @@
import McpResourcesBrowserEmptyState from './McpResourcesBrowserEmptyState.svelte';
import McpResourcesBrowserHeader from './McpResourcesBrowserHeader.svelte';
import McpResourcesBrowserServerItem from './McpResourcesBrowserServerItem.svelte';
import { mcpResourceStore, mcpStore } from '$lib/stores';
import { mcpStore } from '$lib/stores';
import type { MCPResourceInfo, MCPResourceTemplateInfo, MCPServerResources } from '$lib/types';
import { parseResourcePath } from '$lib/utils';
import { SvelteMap, SvelteSet } from 'svelte/reactivity';
@ -31,8 +31,8 @@
let expandedFolders = new SvelteSet<string>();
let searchQuery = $state('');
const resources = $derived(mcpResourceStore.serverResources);
const isLoading = $derived(mcpResourceStore.isLoading);
const resources = $derived(mcpStore.resources.serverResources);
const isLoading = $derived(mcpStore.resources.isLoading);
const filteredResources = $derived.by(() => {
if (!searchQuery.trim()) {

6
tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte
View file

@ -116,7 +116,7 @@
if (status === ServerModelStatus.LOADING) return;
await modelsStore.unloadModel(modelId);
await modelsStore.status.unload(modelId);
}
export function open() {
@ -174,9 +174,9 @@
{@const triggerLoading =
!!triggerModel &&
(triggerStatus === ServerModelStatus.LOADING ||
modelsStore.isModelOperationInProgress(triggerModel))}
modelsStore.status.isOperationInProgress(triggerModel))}
{@const triggerLoadPercent = triggerLoading
? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100)
? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100)
: 0}
{#if ms.isRouter}

12
tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte
View file

@ -47,7 +47,7 @@
return (model?.status?.value as ServerModelStatus) ?? null;
});
let isOperationInProgress = $derived(modelsStore.isModelOperationInProgress(option.model));
let isOperationInProgress = $derived(modelsStore.status.isOperationInProgress(option.model));
let isFailed = $derived(serverStatus === ServerModelStatus.FAILED);
let isSleeping = $derived(serverStatus === ServerModelStatus.SLEEPING);
let isLoaded = $derived(
@ -55,7 +55,7 @@
);
let isLoading = $derived(serverStatus === ServerModelStatus.LOADING || isOperationInProgress);
let loadProgress = $derived(isLoading ? modelsStore.getLoadProgress(option.model) : null);
let loadProgress = $derived(isLoading ? modelsStore.status.getLoadProgress(option.model) : null);
let loadPercent = $derived(Math.round(modelLoadFraction(loadProgress) * 100));
let loadTitle = $derived(modelLoadProgressText(loadProgress));
</script>
@ -138,7 +138,7 @@
icon={RotateCw}
tooltip="Retry loading model"
class="h-3 w-3 text-red-500 hover:text-foreground"
onclick={() => modelsStore.loadModel(option.model)}
onclick={() => modelsStore.status.load(option.model)}
stopPropagationOnClick
/>
</div>
@ -157,7 +157,7 @@
class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-amber-500 [@media(pointer:coarse)]:hover:text-amber-600"
onclick={(e) => {
e?.stopPropagation();
modelsStore.unloadModel(option.model);
modelsStore.status.unload(option.model);
}}
/>
</div>
@ -174,7 +174,7 @@
icon={PowerOff}
tooltip="Unload model"
class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-green-500 [@media(pointer:coarse)]:hover:text-green-600"
onclick={() => modelsStore.unloadModel(option.model)}
onclick={() => modelsStore.status.unload(option.model)}
stopPropagationOnClick
/>
</div>
@ -191,7 +191,7 @@
icon={Power}
tooltip="Load model"
class="h-3 w-3 [@media(pointer:coarse)]:text-muted-foreground"
onclick={() => modelsStore.loadModel(option.model)}
onclick={() => modelsStore.status.load(option.model)}
stopPropagationOnClick
/>
</div>

4
tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte
View file

@ -72,9 +72,9 @@
{@const triggerLoading =
!!triggerModel &&
(triggerStatus === ServerModelStatus.LOADING ||
modelsStore.isModelOperationInProgress(triggerModel))}
modelsStore.status.isOperationInProgress(triggerModel))}
{@const triggerLoadPercent = triggerLoading
? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100)
? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100)
: 0}
{#if ms.isRouter}

12
tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte
View file

@ -15,7 +15,7 @@
NUMERIC_FIELDS,
POSITIVE_INTEGER_FIELDS,
SETTINGS_CHAT_SECTIONS,
SETTINGS_SECTION_TITLES
SETTINGS_SECTION_SLUGS
} from '$lib/constants';
import { ColorMode } from '$lib/enums/ui.enums';
import { RouterService } from '$lib/services/router.service';
@ -46,13 +46,13 @@
let fetchInitiated = false;
$effect(() => {
if (serverStore.isRouterMode && currentSection.fields && !fetchInitiated) {
if (serverStore.isRouterMode && currentSection.fields?.length && !fetchInitiated) {
fetchInitiated = true;
void modelsStore
.fetch()
.then(() => modelsStore.fetchRouterModels())
.then(() => modelsStore.fetchModalitiesForLoadedModels())
.then(() => modelsStore.props.fetchModalitiesForLoadedModels())
.then(() => modelsStore.ensureFirstModelSelected());
}
});
@ -148,9 +148,9 @@
<h3 class="text-lg font-semibold">{currentSection.title}</h3>
</div>
{#if currentSection.title === SETTINGS_SECTION_TITLES.TOOLS}
{#if currentSection.slug === SETTINGS_SECTION_SLUGS.TOOLS}
<SettingsChatToolsTab />
{:else if currentSection.title === SETTINGS_SECTION_TITLES.IMPORT_EXPORT}
{:else if currentSection.slug === SETTINGS_SECTION_SLUGS.IMPORT_EXPORT}
<SettingsChatImportExportTab />
{:else if currentSection.fields}
<div class="space-y-6">
@ -161,7 +161,7 @@
onThemeChange={handleThemeChange}
/>
{#if currentSection.title === SETTINGS_SECTION_TITLES.GENERAL}
{#if currentSection.slug === SETTINGS_SECTION_SLUGS.GENERAL}
<div class="flex justify-end">
<Button variant="outline" onclick={() => window.location.reload()}>
<RefreshCw class="h-3 w-3" />

4
tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte
View file

@ -23,13 +23,13 @@
let { fields, localConfig, onConfigChange, onThemeChange }: Props = $props();
let currentModelParams = $derived.by(() => {
void modelsStore.propsCacheVersion;
void modelsStore.props.cacheVersion;
if (serverStore.isRouterMode) {
const currentModelName = modelsStore.selectedModelName;
if (currentModelName) {
const currentModelProps = modelsStore.getModelProps(currentModelName);
const currentModelProps = modelsStore.props.getModelProps(currentModelName);
return (currentModelProps?.default_generation_settings?.params ?? {}) as Record<
string,

8
tools/ui/src/lib/components/app/settings/SettingsMcpServers.svelte
View file

@ -121,11 +121,13 @@
{:else}
<McpServerCard
{server}
enabled={conversationsStore.isMcpServerEnabledForChat(server.id)}
enabled={conversationsStore.preferences.isMcpServerEnabledForChat(server.id)}
onToggle={async () => {
const wasEnabled = conversationsStore.isMcpServerEnabledForChat(server.id);
const wasEnabled = conversationsStore.preferences.isMcpServerEnabledForChat(
server.id
);
await conversationsStore.toggleMcpServerForChat(server.id);
await conversationsStore.preferences.toggleMcpServerForChat(server.id);
if (!wasEnabled) {
// Promote the connection so tools/prompts/resources become

2
tools/ui/src/lib/constants/attachment-menu.constants.ts
View file

@ -74,7 +74,7 @@ export const ATTACHMENT_PROMPT_ITEMS: AttachmentMenuItem[] = [
enabledWhen: AttachmentItemEnabledWhen.ALWAYS,
icon: Zap,
id: AttachmentMenuItemId.MCP_PROMPT,
label: 'MCP Prompt',
label: 'MCP Prompts',
visibleWhen: AttachmentItemVisibleWhen.HAS_MCP_PROMPTS_SUPPORT
}
];

10
tools/ui/src/lib/constants/cache.constants.ts
View file

@ -32,13 +32,3 @@ export const MCP_RESOURCE_CACHE = {
/** TTL for MCP resource cache entries in milliseconds (5 minutes) */
TTL_MS: 5 * 60 * 1000
} as const;
/**
* Limits for pruning inactive conversation states held in memory.
*/
export const INACTIVE_CONVERSATION = {
/** Maximum age (in ms) for inactive conversation states before cleanup (30 minutes) */
MAX_AGE_MS: 30 * 60 * 1000,
/** Maximum number of inactive conversation states to keep in memory */
MAX_STATES: 10
} as const;

2
tools/ui/src/lib/constants/index.ts
View file

@ -48,7 +48,7 @@ export * from './pwa.constants';
export * from './routes.constants';
export * from './sandbox.constants';
export * from './settings-keys.constants';
export * from './settings-registry.constants';
export * from './settings.constants';
export * from './special-characters.constants';
export * from './stream.constants';
export * from './supported-file-types.constants';

14
tools/ui/src/lib/constants/routes.constants.ts
View file

@ -10,18 +10,6 @@ export const URL_PARAMS = {
QUERY: 'q'
} as const;
/** Settings section slugs — used for routes and navigation. */
export const SETTINGS_SECTION_SLUGS = {
AGENTIC: 'agentic',
DEVELOPER: 'developer',
DISPLAY: 'display',
GENERAL: 'general',
IMPORT_EXPORT: 'import-export',
PENALTIES: 'penalties',
SAMPLING: 'sampling',
TOOLS: 'tools'
} as const;
export const ROUTES = {
/** Chat base — for dynamic chat URLs use RouterService. */
CHAT: '#/chat',
@ -33,6 +21,8 @@ export const ROUTES = {
SEARCH: '#/search',
/** Settings base — for dynamic settings URLs use RouterService. */
SETTINGS: '#/settings',
/** Exit destination for the settings view (fallback when no referrer). */
SETTINGS_EXIT: '#/',
/** Root — start of the app. */
START: '#/'
} as const;

1005
tools/ui/src/lib/constants/settings-registry.constants.ts → tools/ui/src/lib/constants/settings.constants.ts
View file

File diff suppressed because it is too large Load diff

6
tools/ui/src/lib/constants/url.constants.ts
View file

@ -1,3 +1,5 @@
import { UrlProtocol } from '$lib/enums';
const STD = ['com', 'net', 'org', 'gov', 'edu'] as const;
const STD_MIL = [...STD, 'mil'] as const;
const ccTLD_PREFIXES: Record<string, readonly string[]> = {
@ -184,3 +186,7 @@ export const WILDCARD_PUBLIC_SUFFIXES = buildSuffixSet(WILDCARD_BASES);
// Matches one or more trailing "/" characters at the end of a URL/path.
export const TRAILING_SLASHES_REGEX = /\/+$/;
// Protocols that apiFetch treats as absolute and passes through untouched.
// Add a protocol here when a caller needs to fetch an absolute URL with it.
export const API_ABSOLUTE_URL_PROTOCOLS = [UrlProtocol.HTTP, UrlProtocol.HTTPS] as const;

178
tools/ui/src/lib/hooks/use-auto-scroll.svelte.ts
View file

@ -14,18 +14,14 @@ export interface AutoScrollOptions {
*/
export class AutoScrollController {
private _autoScrollEnabled = $state(true);
private _userScrolledUp = $state(false);
private _lastScrollTop = $state(0);
private _scrollInterval: ReturnType<typeof setInterval> | undefined;
private _container: HTMLElement | undefined;
private _disabled: boolean;
private _lastScrollTop = $state(0);
private _mutationObserver: MutationObserver | null = null;
private _rafPending = false;
private _observerEnabled = false;
constructor(options: AutoScrollOptions = {}) {
this._disabled = options.disabled ?? false;
}
private _rafPending = false;
private _scrollInterval: ReturnType<typeof setInterval> | undefined;
private _userScrolledUp = $state(false);
get autoScrollEnabled(): boolean {
return this._autoScrollEnabled;
}
@ -34,6 +30,71 @@ export class AutoScrollController {
return this._userScrolledUp;
}
constructor(options: AutoScrollOptions = {}) {
this._disabled = options.disabled ?? false;
}
/**
* Cleans up resources. Call this in onDestroy or when the component unmounts.
*/
destroy(): void {
this.stopInterval();
this._doStopObserving();
}
/**
* Enables auto-scroll (e.g., when user sends a message).
*/
enable(): void {
if (this._disabled) return;
this._userScrolledUp = false;
this._autoScrollEnabled = true;
}
/**
* Handles scroll events to detect user scroll direction and toggle auto-scroll.
*/
handleScroll(): void {
if (this._disabled || !this._container) return;
const { clientHeight, scrollHeight, scrollTop } = this._container;
const distanceFromBottom = scrollHeight - clientHeight - scrollTop;
const isScrollingUp = scrollTop < this._lastScrollTop;
const isAtBottom = distanceFromBottom < AUTO_SCROLL_AT_BOTTOM_THRESHOLD;
if (isScrollingUp && !isAtBottom) {
this._userScrolledUp = true;
this._autoScrollEnabled = false;
} else if (isAtBottom && this._userScrolledUp) {
this._userScrolledUp = false;
this._autoScrollEnabled = true;
}
this._lastScrollTop = scrollTop;
}
/**
* Resets scroll state when switching conversations.
*/
resetScrollState(): void {
this._userScrolledUp = false;
this._autoScrollEnabled = !this._disabled;
if (this._container) {
this._lastScrollTop = this._container.scrollTop;
}
}
/**
* Scrolls the container to the bottom instantly.
*/
scrollToBottom(): void {
if (this._disabled || !this._container) return;
this._container.scrollTop = this._container.scrollHeight;
}
/**
* Binds the controller to a scrollable container element.
*/
@ -63,59 +124,6 @@ export class AutoScrollController {
}
}
/**
* Handles scroll events to detect user scroll direction and toggle auto-scroll.
*/
handleScroll(): void {
if (this._disabled || !this._container) return;
const { clientHeight, scrollHeight, scrollTop } = this._container;
const distanceFromBottom = scrollHeight - clientHeight - scrollTop;
const isScrollingUp = scrollTop < this._lastScrollTop;
const isAtBottom = distanceFromBottom < AUTO_SCROLL_AT_BOTTOM_THRESHOLD;
if (isScrollingUp && !isAtBottom) {
this._userScrolledUp = true;
this._autoScrollEnabled = false;
} else if (isAtBottom && this._userScrolledUp) {
this._userScrolledUp = false;
this._autoScrollEnabled = true;
}
this._lastScrollTop = scrollTop;
}
/**
* Scrolls the container to the bottom instantly.
*/
scrollToBottom(): void {
if (this._disabled || !this._container) return;
this._container.scrollTop = this._container.scrollHeight;
}
/**
* Enables auto-scroll (e.g., when user sends a message).
*/
enable(): void {
if (this._disabled) return;
this._userScrolledUp = false;
this._autoScrollEnabled = true;
}
/**
* Resets scroll state when switching conversations.
*/
resetScrollState(): void {
this._userScrolledUp = false;
this._autoScrollEnabled = !this._disabled;
if (this._container) {
this._lastScrollTop = this._container.scrollTop;
}
}
/**
* Starts the auto-scroll interval for continuous scrolling during streaming.
*/
@ -127,6 +135,18 @@ export class AutoScrollController {
}, AUTO_SCROLL_INTERVAL);
}
/**
* Starts a MutationObserver on the container that auto-scrolls to bottom
* on content changes. More responsive than interval-based polling.
*/
startObserving(): void {
this._observerEnabled = true;
if (this._container && !this._disabled && !this._mutationObserver) {
this._doStartObserving();
}
}
/**
* Stops the auto-scroll interval.
*/
@ -137,6 +157,14 @@ export class AutoScrollController {
}
}
/**
* Stops the MutationObserver.
*/
stopObserving(): void {
this._observerEnabled = false;
this._doStopObserving();
}
/**
* Updates the auto-scroll interval based on streaming state.
* Call this in a $effect to automatically manage the interval.
@ -157,34 +185,6 @@ export class AutoScrollController {
}
}
/**
* Cleans up resources. Call this in onDestroy or when the component unmounts.
*/
destroy(): void {
this.stopInterval();
this._doStopObserving();
}
/**
* Starts a MutationObserver on the container that auto-scrolls to bottom
* on content changes. More responsive than interval-based polling.
*/
startObserving(): void {
this._observerEnabled = true;
if (this._container && !this._disabled && !this._mutationObserver) {
this._doStartObserving();
}
}
/**
* Stops the MutationObserver.
*/
stopObserving(): void {
this._observerEnabled = false;
this._doStopObserving();
}
private _doStartObserving(): void {
if (!this._container || this._mutationObserver) return;

10
tools/ui/src/lib/hooks/use-chat-screen-active-model.svelte.ts
View file

@ -22,10 +22,10 @@ export function useChatScreenActiveModel() {
$effect(() => {
if (activeModelId) {
const cached = modelsStore.getModelProps(activeModelId);
const cached = modelsStore.props.getModelProps(activeModelId);
if (!cached) {
modelsStore.fetchModelProps(activeModelId).then(() => {
modelsStore.props.fetchModelProps(activeModelId).then(() => {
modelPropsVersion++;
});
}
@ -36,7 +36,7 @@ export function useChatScreenActiveModel() {
if (activeModelId) {
void modelPropsVersion;
return modelsStore.modelSupportsAudio(activeModelId);
return modelsStore.props.modelSupportsAudio(activeModelId);
}
return false;
@ -45,7 +45,7 @@ export function useChatScreenActiveModel() {
if (activeModelId) {
void modelPropsVersion;
return modelsStore.modelSupportsVideo(activeModelId);
return modelsStore.props.modelSupportsVideo(activeModelId);
}
return false;
@ -54,7 +54,7 @@ export function useChatScreenActiveModel() {
if (activeModelId) {
void modelPropsVersion;
return modelsStore.modelSupportsVision(activeModelId);
return modelsStore.props.modelSupportsVision(activeModelId);
}
return false;

8
tools/ui/src/lib/hooks/use-context-gauge.svelte.ts
View file

@ -54,10 +54,10 @@ export function useContextGauge(): UseContextGaugeReturn {
const modelId = contextStatsStore.activeModelId;
if (modelId && contextStatsStore.isActiveModelLoaded) {
const cached = modelsStore.getModelProps(modelId);
const cached = modelsStore.props.getModelProps(modelId);
if (!cached) {
void modelsStore.fetchModelProps(modelId);
void modelsStore.props.fetchModelProps(modelId);
}
}
});
@ -80,9 +80,9 @@ export function useContextGauge(): UseContextGaugeReturn {
if (!modelId || contextStatsStore.isActiveModelLoading) return;
try {
await modelsStore.loadModel(modelId);
await modelsStore.status.load(modelId);
} catch {
// toast already surfaced by modelsStore.loadModel
// toast already surfaced by modelsStore.status.load
}
}

8
tools/ui/src/lib/hooks/use-models-selector.svelte.ts
View file

@ -47,7 +47,7 @@ export interface UseModelsSelectorReturn {
export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSelectorReturn {
const options = $derived(
modelsStore.models.filter((option) => {
const modelProps = modelsStore.getModelProps(option.model);
const modelProps = modelsStore.props.getModelProps(option.model);
return modelProps?.ui !== false;
})
@ -102,7 +102,7 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
if (open) {
modelsStore.fetchRouterModels().then(() => {
modelsStore.fetchModalitiesForLoadedModels();
modelsStore.props.fetchModalitiesForLoadedModels();
});
}
@ -142,8 +142,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
if (!onModelChange && isRouter && !modelsStore.isModelLoaded(option.model)) {
isLoadingModel = true;
modelsStore
.loadModel(option.model)
modelsStore.status
.load(option.model)
.catch((error) => console.error('Failed to load model:', error))
.finally(() => (isLoadingModel = false));
}

2
tools/ui/src/lib/hooks/use-processing-state.svelte.ts
View file

@ -43,7 +43,7 @@ export function useProcessingState(): UseProcessingStateReturn {
}
// Read directly from the reactive state
return chatStore.activeProcessingState;
return chatStore.processing.activeState;
});
$effect(() => {

11
tools/ui/src/lib/hooks/use-reasoning-menu.svelte.ts
View file

@ -42,19 +42,20 @@ export function useReasoningMenu(): UseReasoningMenuReturn {
});
const modelSupportsThinking = $derived.by(() => {
void modelsStore.loadedModelIds;
void modelsStore.propsCacheVersion;
void modelsStore.props.cacheVersion;
if (serverStore.isRouterMode) {
const modelId = modelsStore.selectedModelName || conversationModel;
return (
modelsStore.checkModelSupportsThinking(modelId ?? '') || modelSupportsThinkingFromMessages
modelsStore.props.checkModelSupportsThinking(modelId ?? '') ||
modelSupportsThinkingFromMessages
);
}
return modelsStore.supportsThinking || modelSupportsThinkingFromMessages;
return modelsStore.props.supportsThinking || modelSupportsThinkingFromMessages;
});
const currentEffort = $derived(conversationsStore.getReasoningEffort());
const currentEffort = $derived(conversationsStore.preferences.getReasoningEffort());
const thinkingEnabled = $derived(
currentEffort !== ReasoningEffort.OFF && currentEffort !== ReasoningEffort.DEFAULT
);
@ -76,7 +77,7 @@ export function useReasoningMenu(): UseReasoningMenuReturn {
return modelSupportsThinking;
},
select(level: ReasoningEffortLevel): void {
conversationsStore.setReasoningEffort(level.value as ReasoningEffort);
conversationsStore.preferences.setReasoningEffort(level.value as ReasoningEffort);
},
get thinkingEnabled() {
return thinkingEnabled;

4
tools/ui/src/lib/hooks/use-tools-panel.svelte.ts
View file

@ -35,7 +35,7 @@ export function useToolsPanel(): UseToolsPanelReturn {
(g) =>
g.source !== ToolSource.MCP ||
!g.serverId ||
conversationsStore.isMcpServerEnabledForChat(g.serverId)
conversationsStore.preferences.isMcpServerEnabledForChat(g.serverId)
)
);
const totalToolCount = $derived(activeGroups.reduce((n, g) => n + g.tools.length, 0));
@ -72,7 +72,7 @@ export function useToolsPanel(): UseToolsPanelReturn {
return (
group.source === ToolSource.MCP &&
!!group.serverId &&
!conversationsStore.isMcpServerEnabledForChat(group.serverId)
!conversationsStore.preferences.isMcpServerEnabledForChat(group.serverId)
);
}

1981
tools/ui/src/lib/services/chat.service.ts
View file

File diff suppressed because it is too large Load diff

346
tools/ui/src/lib/services/conversation-transfer.service.ts
View file

@ -16,187 +16,6 @@ import {
import { strFromU8, strToU8, unzipSync, zipSync } from 'fflate';
export class ConversationTransferService {
/**
*
*
* JSONL Session Format
*
*
*/
/**
* Serializes a session (a conversation with its messages) as JSONL.
* The first line is the session header (a `SessionRecordType.SESSION` record
* carrying the conversation properties); each subsequent line is a single message.
* @param data - The exported conversation payload
* @returns The JSONL string (one record per line)
*/
static serializeSessionToJsonl(data: ExportedConversation): string {
const { conv, messages } = data;
const sessionLine = JSON.stringify({
harness: EXPORT_CONV.HARNESS,
type: SessionRecordType.SESSION,
...conv
});
const messageLines = messages.map((message: DatabaseMessage) => {
// `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it.
const { toolCalls, ...rest } = message;
const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest;
return JSON.stringify({ message: normalized, type: SessionRecordType.MESSAGE });
});
return [sessionLine, ...messageLines].join(NEWLINE);
}
/**
* Parses the JSONL session format produced by {@link serializeSessionToJsonl}.
* A `SessionRecordType.SESSION` line starts a new session; following
* `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple
* sessions in a single file.
* @param text - The JSONL file contents
* @returns The parsed conversations with their messages
*/
static parseSessionsJsonl(text: string): ExportedConversation[] {
const sessions: ExportedConversation[] = [];
let current: ExportedConversation | null = null;
for (const line of text.split(NEWLINE)) {
const trimmed = line.trim();
if (!trimmed) continue;
const record = JSON.parse(trimmed);
if (record.type === SessionRecordType.SESSION) {
// Drop the discriminator and harness marker; the rest is the conversation.
const conv = { ...record };
delete conv.type;
delete conv.harness;
current = { conv: conv as DatabaseConversation, messages: [] };
sessions.push(current);
} else if (record.type === SessionRecordType.MESSAGE) {
if (!current) {
throw new Error('Invalid JSONL: message record before any session record');
}
const message = record.message as DatabaseMessage;
// `toolCalls` is parsed to an array on export; the DB stores it as a string.
if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') {
message.toolCalls = JSON.stringify(message.toolCalls);
}
current.messages.push(message);
}
// Ignore unknown record types for forward compatibility.
}
return sessions;
}
/**
* Reports whether the text is the JSONL session format, whose first non-empty
* line is a `SessionRecordType.SESSION` record. A legacy JSON export starts
* with an array or an object that has no such discriminator.
* @param text - The file contents
*/
private static isSessionsJsonl(text: string): boolean {
const trimmed = text.trimStart();
const lineEnd = trimmed.indexOf(NEWLINE);
const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd);
try {
return JSON.parse(firstLine).type === SessionRecordType.SESSION;
} catch {
// Not a standalone JSON record, so not the JSONL format.
return false;
}
}
/**
* Parses an import file into conversations, accepting the current JSONL and
* ZIP formats as well as the legacy JSON format. The format comes from the
* contents, so an import works whatever the file is named.
* @param file - The user-selected file
* @returns The parsed conversations with their messages
*/
static async parseImportFile(file: File): Promise<ExportedConversation[]> {
const bytes = new Uint8Array(await file.arrayBuffer());
if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) {
const entries = unzipSync(bytes);
const sessions: ExportedConversation[] = [];
for (const [entryName, entryBytes] of Object.entries(entries)) {
if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue;
sessions.push(...ConversationTransferService.parseSessionsJsonl(strFromU8(entryBytes)));
}
return sessions;
}
const text = strFromU8(bytes);
if (ConversationTransferService.isSessionsJsonl(text)) {
return ConversationTransferService.parseSessionsJsonl(text);
}
// Legacy JSON format: an array of conversations or a single conversation object.
const parsed = JSON.parse(text);
if (Array.isArray(parsed)) {
return parsed;
}
if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) {
return [parsed];
}
throw new Error(
'Invalid file format: expected array of conversations or single conversation object'
);
}
/**
*
*
* Downloads
*
*
*/
/**
* Generates a sanitized filename for a conversation export
* @param conversation - The conversation metadata
* @param msgs - Optional array of messages belonging to the conversation
* @returns The generated filename string
*/
static generateConversationFilename(
conversation: { id?: string; name?: string },
msgs?: DatabaseMessage[]
): string {
const conversationName = (conversation.name ?? '').trim().toLowerCase();
const sanitizedName = conversationName
.replace(EXPORT_CONV.NON_ALPHANUMERIC_REGEX, EXPORT_CONV.NONALNUM_REPLACEMENT)
.replace(EXPORT_CONV.MULTIPLE_UNDERSCORE_REGEX, '_')
.substring(0, EXPORT_CONV.NAME_SUFFIX_MAX_LENGTH);
// If we have messages, use the timestamp of the newest message
const referenceDate = msgs?.length
? new Date(Math.max(...msgs.map((m) => m.timestamp)))
: new Date();
const iso = referenceDate.toISOString().slice(0, EXPORT_CONV.ISO_TIMESTAMP_SLICE);
const formattedDate = iso
.replace(EXPORT_CONV.ISO_DATE_TIME_SEPARATOR, EXPORT_CONV.ISO_DATE_TIME_SEPARATOR_REPLACEMENT)
.replaceAll(EXPORT_CONV.ISO_TIME_SEPARATOR, EXPORT_CONV.ISO_TIME_SEPARATOR_REPLACEMENT);
const trimmedConvId = conversation.id?.slice(0, EXPORT_CONV.ID_TRIM_LENGTH) ?? '';
return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}${FileExtensionText.JSONL}`;
}
/**
* Triggers a browser download of the provided exported conversation data
* @param data - The exported conversation payload (a single conversation with its messages)
@ -262,6 +81,171 @@ export class ConversationTransferService {
ConversationTransferService.triggerDownload(blob, archiveName);
}
/**
* Generates a sanitized filename for a conversation export
* @param conversation - The conversation metadata
* @param msgs - Optional array of messages belonging to the conversation
* @returns The generated filename string
*/
static generateConversationFilename(
conversation: { id?: string; name?: string },
msgs?: DatabaseMessage[]
): string {
const conversationName = (conversation.name ?? '').trim().toLowerCase();
const sanitizedName = conversationName
.replace(EXPORT_CONV.NON_ALPHANUMERIC_REGEX, EXPORT_CONV.NONALNUM_REPLACEMENT)
.replace(EXPORT_CONV.MULTIPLE_UNDERSCORE_REGEX, '_')
.substring(0, EXPORT_CONV.NAME_SUFFIX_MAX_LENGTH);
// If we have messages, use the timestamp of the newest message
const referenceDate = msgs?.length
? new Date(Math.max(...msgs.map((m) => m.timestamp)))
: new Date();
const iso = referenceDate.toISOString().slice(0, EXPORT_CONV.ISO_TIMESTAMP_SLICE);
const formattedDate = iso
.replace(EXPORT_CONV.ISO_DATE_TIME_SEPARATOR, EXPORT_CONV.ISO_DATE_TIME_SEPARATOR_REPLACEMENT)
.replaceAll(EXPORT_CONV.ISO_TIME_SEPARATOR, EXPORT_CONV.ISO_TIME_SEPARATOR_REPLACEMENT);
const trimmedConvId = conversation.id?.slice(0, EXPORT_CONV.ID_TRIM_LENGTH) ?? '';
return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}${FileExtensionText.JSONL}`;
}
/**
* Parses an import file into conversations, accepting the current JSONL and
* ZIP formats as well as the legacy JSON format. The format comes from the
* contents, so an import works whatever the file is named.
* @param file - The user-selected file
* @returns The parsed conversations with their messages
*/
static async parseImportFile(file: File): Promise<ExportedConversation[]> {
const bytes = new Uint8Array(await file.arrayBuffer());
if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) {
const entries = unzipSync(bytes);
const sessions: ExportedConversation[] = [];
for (const [entryName, entryBytes] of Object.entries(entries)) {
if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue;
sessions.push(...ConversationTransferService.parseSessionsJsonl(strFromU8(entryBytes)));
}
return sessions;
}
const text = strFromU8(bytes);
if (ConversationTransferService.isSessionsJsonl(text)) {
return ConversationTransferService.parseSessionsJsonl(text);
}
// Legacy JSON format: an array of conversations or a single conversation object.
const parsed = JSON.parse(text);
if (Array.isArray(parsed)) {
return parsed;
}
if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) {
return [parsed];
}
throw new Error(
'Invalid file format: expected array of conversations or single conversation object'
);
}
/**
* Parses the JSONL session format produced by {@link serializeSessionToJsonl}.
* A `SessionRecordType.SESSION` line starts a new session; following
* `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple
* sessions in a single file.
* @param text - The JSONL file contents
* @returns The parsed conversations with their messages
*/
static parseSessionsJsonl(text: string): ExportedConversation[] {
const sessions: ExportedConversation[] = [];
let current: ExportedConversation | null = null;
for (const line of text.split(NEWLINE)) {
const trimmed = line.trim();
if (!trimmed) continue;
const record = JSON.parse(trimmed);
if (record.type === SessionRecordType.SESSION) {
// Drop the discriminator and harness marker; the rest is the conversation.
const conv = { ...record };
delete conv.type;
delete conv.harness;
current = { conv: conv as DatabaseConversation, messages: [] };
sessions.push(current);
} else if (record.type === SessionRecordType.MESSAGE) {
if (!current) {
throw new Error('Invalid JSONL: message record before any session record');
}
const message = record.message as DatabaseMessage;
// `toolCalls` is parsed to an array on export; the DB stores it as a string.
if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') {
message.toolCalls = JSON.stringify(message.toolCalls);
}
current.messages.push(message);
}
// Ignore unknown record types for forward compatibility.
}
return sessions;
}
/**
* Serializes a session (a conversation with its messages) as JSONL.
* The first line is the session header (a `SessionRecordType.SESSION` record
* carrying the conversation properties); each subsequent line is a single message.
* @param data - The exported conversation payload
* @returns The JSONL string (one record per line)
*/
static serializeSessionToJsonl(data: ExportedConversation): string {
const { conv, messages } = data;
const sessionLine = JSON.stringify({
harness: EXPORT_CONV.HARNESS,
type: SessionRecordType.SESSION,
...conv
});
const messageLines = messages.map((message: DatabaseMessage) => {
// `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it.
const { toolCalls, ...rest } = message;
const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest;
return JSON.stringify({ message: normalized, type: SessionRecordType.MESSAGE });
});
return [sessionLine, ...messageLines].join(NEWLINE);
}
/**
* Reports whether the text is the JSONL session format, whose first non-empty
* line is a `SessionRecordType.SESSION` record. A legacy JSON export starts
* with an array or an object that has no such discriminator.
* @param text - The file contents
*/
private static isSessionsJsonl(text: string): boolean {
const trimmed = text.trimStart();
const lineEnd = trimmed.indexOf(NEWLINE);
const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd);
try {
return JSON.parse(firstLine).type === SessionRecordType.SESSION;
} catch {
// Not a standalone JSON record, so not the JSONL format.
return false;
}
}
/**
* Triggers a browser download of a blob under the given filename.
*/

765
tools/ui/src/lib/services/database.service.ts
View file

@ -1,3 +1,11 @@
/**
* DatabaseService - IndexedDB persistence for conversations and messages
*
* Thin Dexie layer over the conversations/messages tables: CRUD, tree
* navigation (descendants, reparenting) and cascading deletes. No reactive
* state; consumed by conversationsStore and the chat flows.
*/
import { IDXDB_STORES, IDXDB_TABLES, STORAGE_APP_NAME } from '$lib/constants';
import { MessageRole } from '$lib/enums';
import type { McpServerOverride } from '$lib/types/database';
@ -20,12 +28,99 @@ const db = new LlamaUiDatabase();
export class DatabaseService {
/**
* Deletes multiple conversations in a single transaction. Each deleted
* conversation has its direct children reparented to the nearest surviving
* ancestor (or promoted to top-level). Children also in `ids` are dropped
* entirely rather than reparented.
*
*
* Conversations
*
*
* @param ids - Conversation IDs to delete
*/
static async bulkDeleteConversations(ids: string[]): Promise<void> {
const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0);
if (cleanIds.length === 0) return;
const idSet = new Set(cleanIds);
await db.transaction(
'rw',
[db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]],
async () => {
// Pre-load each to-delete conversation so the per-id reparent
// walk-up doesn't ping-pong the same ancestry chain.
const prefetched = new Map<string, DatabaseConversation>();
let frontier = [...cleanIds];
const requested = new Set<string>(frontier);
while (frontier.length > 0) {
const fetched = await db[IDXDB_TABLES.conversations].bulkGet(frontier);
frontier = [];
for (let i = 0; i < fetched.length; i++) {
const conv = fetched[i];
if (!conv || !conv.id) continue;
prefetched.set(conv.id, conv);
const ancestor = conv.forkedFromConversationId;
if (ancestor && !prefetched.has(ancestor) && !requested.has(ancestor)) {
frontier.push(ancestor);
requested.add(ancestor);
}
}
}
for (const id of cleanIds) {
await this.reparentDirectChildren(id, idSet, prefetched);
}
await db[IDXDB_TABLES.conversations].bulkDelete(cleanIds);
await db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).delete();
}
);
}
/**
* Toggles the pinned status of each conversation in `ids` inside a single
* transaction. Treats `pinned === undefined` as `false`, matching the
* semantics of {@link toggleConversationPin} where `!undefined` evaluates
* to `true`. Returns the resulting pinned state for every id that was
* updated; missing ids are omitted from the map.
*
* @param ids - Conversation IDs to toggle
* @returns Map of id -> new pinned state
*/
static async bulkToggleConversationPins(ids: string[]): Promise<Map<string, boolean>> {
const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0);
const result = new Map<string, boolean>();
if (cleanIds.length === 0) return result;
await db.transaction('rw', db[IDXDB_TABLES.conversations], async () => {
const convs = await db[IDXDB_TABLES.conversations].bulkGet(cleanIds);
const updates: DatabaseConversation[] = [];
for (let i = 0; i < cleanIds.length; i++) {
const conv = convs[i];
if (!conv) continue;
const newPinned = !conv.pinned;
updates.push({ ...conv, pinned: newPinned });
result.set(cleanIds[i], newPinned);
}
if (updates.length === 0) return;
await db[IDXDB_TABLES.conversations].bulkPut(updates);
});
return result;
}
/**
* Creates a new conversation.
@ -51,14 +146,6 @@ export class DatabaseService {
return conversation;
}
/**
*
*
* Messages
*
*
*/
/**
* Creates a new message branch by adding a message and updating parent/child relationships.
* Also updates the conversation's currNode to point to the new message.
@ -96,13 +183,7 @@ export class DatabaseService {
// Update parent's children array if parent exists
if (parentId !== null) {
const parentMessage = await db[IDXDB_TABLES.messages].get(parentId);
if (parentMessage) {
await db[IDXDB_TABLES.messages].update(parentId, {
children: [...parentMessage.children, newMessage.id]
});
}
await this.addChildToParent(parentId, newMessage.id);
}
await this.updateConversation(message.convId, {
@ -178,9 +259,7 @@ export class DatabaseService {
};
await db[IDXDB_TABLES.messages].add(systemMessage);
await db[IDXDB_TABLES.messages].update(parentId, {
children: [...parentMessage.children, systemMessage.id]
});
await this.addChildToParent(parentId, systemMessage.id);
return systemMessage;
});
@ -230,121 +309,6 @@ export class DatabaseService {
);
}
/**
* Reparents direct children of `parentId` to the nearest surviving
* ancestor (or promotes them to top-level when the immediate parent was
* top-level). Walking skips any ancestor listed in `excludeIds`, since
* those will be deleted in the same batch — leaving a grandchild pointing
* at an `excludeIds` entry would orphan it. Children whose own id is in
* `excludeIds` are dropped from the updates (the bulk-delete pass will
* remove them). `prefetched` may carry a pre-fetched ancestor map to
* avoid repeat reads inside a bulk transaction.
*/
private static async reparentDirectChildren(
parentId: string,
excludeIds: ReadonlySet<string> = new Set(),
prefetched?: ReadonlyMap<string, DatabaseConversation>
): Promise<void> {
const conv = prefetched?.get(parentId) ?? (await db[IDXDB_TABLES.conversations].get(parentId));
if (!conv) return;
let newParent = conv.forkedFromConversationId;
const visited = new Set<string>([parentId]);
while (newParent && excludeIds.has(newParent)) {
if (visited.has(newParent)) {
newParent = undefined;
break;
}
visited.add(newParent);
const next =
prefetched?.get(newParent) ?? (await db[IDXDB_TABLES.conversations].get(newParent));
if (!next) {
newParent = undefined;
break;
}
newParent = next.forkedFromConversationId;
}
const directChildren = await db[IDXDB_TABLES.conversations]
.filter((c) => c.forkedFromConversationId === parentId)
.toArray();
const updates: DatabaseConversation[] = [];
for (const child of directChildren) {
if (excludeIds.has(child.id)) continue;
updates.push({ ...child, forkedFromConversationId: newParent });
}
if (updates.length === 0) return;
await db[IDXDB_TABLES.conversations].bulkPut(updates);
}
/**
* Deletes multiple conversations in a single transaction. Each deleted
* conversation has its direct children reparented to the nearest surviving
* ancestor (or promoted to top-level). Children also in `ids` are dropped
* entirely rather than reparented.
*
* @param ids - Conversation IDs to delete
*/
static async bulkDeleteConversations(ids: string[]): Promise<void> {
const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0);
if (cleanIds.length === 0) return;
const idSet = new Set(cleanIds);
await db.transaction(
'rw',
[db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]],
async () => {
// Pre-load each to-delete conversation so the per-id reparent
// walk-up doesn't ping-pong the same ancestry chain.
const prefetched = new Map<string, DatabaseConversation>();
let frontier = [...cleanIds];
const requested = new Set<string>(frontier);
while (frontier.length > 0) {
const fetched = await db[IDXDB_TABLES.conversations].bulkGet(frontier);
frontier = [];
for (let i = 0; i < fetched.length; i++) {
const conv = fetched[i];
if (!conv || !conv.id) continue;
prefetched.set(conv.id, conv);
const ancestor = conv.forkedFromConversationId;
if (ancestor && !prefetched.has(ancestor) && !requested.has(ancestor)) {
frontier.push(ancestor);
requested.add(ancestor);
}
}
}
for (const id of cleanIds) {
await this.reparentDirectChildren(id, idSet, prefetched);
}
await db[IDXDB_TABLES.conversations].bulkDelete(cleanIds);
await db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).delete();
}
);
}
/**
* Deletes a message and removes it from its parent's children array.
*
@ -356,17 +320,8 @@ export class DatabaseService {
if (!message) return;
// Remove this message from its parent's children array
if (message.parent) {
const parent = await db[IDXDB_TABLES.messages].get(message.parent);
await this.removeChildFromParent(messageId);
if (parent) {
parent.children = parent.children.filter((childId: string) => childId !== messageId);
await db[IDXDB_TABLES.messages].put(parent);
}
}
// Delete the message
await db[IDXDB_TABLES.messages].delete(messageId);
});
}
@ -389,20 +344,10 @@ export class DatabaseService {
.where('convId')
.equals(conversationId)
.toArray();
// Find all descendant messages
const descendants = findDescendantMessages(allMessages, messageId);
const allToDelete = [messageId, ...descendants];
// Get the message to delete for parent cleanup
const message = await db[IDXDB_TABLES.messages].get(messageId);
if (message && message.parent) {
const parent = await db[IDXDB_TABLES.messages].get(message.parent);
if (parent) {
parent.children = parent.children.filter((childId: string) => childId !== messageId);
await db[IDXDB_TABLES.messages].put(parent);
}
}
await this.removeChildFromParent(messageId);
// Delete all messages in the branch
await db[IDXDB_TABLES.messages].bulkDelete(allToDelete);
@ -411,243 +356,6 @@ export class DatabaseService {
});
}
/**
* Gets all conversations, sorted by last modified time (newest first).
*
* @returns Array of conversations
*/
static async getAllConversations(): Promise<DatabaseConversation[]> {
return await db[IDXDB_TABLES.conversations].orderBy('lastModified').reverse().toArray();
}
/**
* Gets a conversation by ID.
*
* @param id - Conversation ID
* @returns The conversation if found, otherwise undefined
*/
static async getConversation(id: string): Promise<DatabaseConversation | undefined> {
return await db[IDXDB_TABLES.conversations].get(id);
}
/**
* Gets all messages in a conversation, sorted by timestamp (oldest first).
*
* @param convId - Conversation ID
* @returns Array of messages in the conversation
*/
static async getConversationMessages(convId: string): Promise<DatabaseMessage[]> {
return await db[IDXDB_TABLES.messages].where('convId').equals(convId).sortBy('timestamp');
}
/**
* Loads multiple conversations with all of their messages in two bulk
* reads. Missing conversations are silently omitted from the result.
*
* @param convIds - Conversation IDs to load
* @returns Map of id -> { conv, messages }. Messages are sorted ascending by timestamp.
*/
static async getConversationsWithMessages(
convIds: string[]
): Promise<Map<string, ExportedConversation>> {
const result = new Map<string, ExportedConversation>();
const cleanIds = convIds.filter((id): id is string => typeof id === 'string' && id.length > 0);
if (cleanIds.length === 0) return result;
const [convs, allMessages] = await Promise.all([
db[IDXDB_TABLES.conversations].bulkGet(cleanIds),
db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).toArray()
]);
const messagesByConv = new Map<string, DatabaseMessage[]>();
for (const msg of allMessages) {
const bucket = messagesByConv.get(msg.convId);
if (bucket) bucket.push(msg);
else messagesByConv.set(msg.convId, [msg]);
}
for (let i = 0; i < cleanIds.length; i++) {
const conv = convs[i];
if (!conv) continue;
const messages = (messagesByConv.get(conv.id) ?? []).sort(
(a, b) => a.timestamp - b.timestamp
);
result.set(conv.id, { conv, messages });
}
return result;
}
/**
* Updates a conversation. `lastModified` is never stamped implicitly;
* pass it in `updates` to bump the conversation in recency ordering.
*
* @param id - Conversation ID
* @param updates - Partial updates to apply
* @returns Promise that resolves when the conversation is updated
*/
static async updateConversation(
id: string,
updates: Partial<Omit<DatabaseConversation, 'id'>>
): Promise<void> {
await db[IDXDB_TABLES.conversations].update(id, updates);
}
/**
*
*
* Navigation
*
*
*/
/**
* Toggles the pinned status of a conversation.
*
* @param id - Conversation ID
* @returns The new pinned status
*/
static async toggleConversationPin(id: string): Promise<boolean> {
const conversation = await db[IDXDB_TABLES.conversations].get(id);
if (!conversation) {
throw new Error(`Conversation ${id} not found`);
}
const newPinnedState = !conversation.pinned;
await this.updateConversation(id, { pinned: newPinnedState });
return newPinnedState;
}
/**
* Toggles the pinned status of each conversation in `ids` inside a single
* transaction. Treats `pinned === undefined` as `false`, matching the
* semantics of {@link toggleConversationPin} where `!undefined` evaluates
* to `true`. Returns the resulting pinned state for every id that was
* updated; missing ids are omitted from the map.
*
* @param ids - Conversation IDs to toggle
* @returns Map of id -> new pinned state
*/
static async bulkToggleConversationPins(ids: string[]): Promise<Map<string, boolean>> {
const cleanIds = ids.filter((id): id is string => typeof id === 'string' && id.length > 0);
const result = new Map<string, boolean>();
if (cleanIds.length === 0) return result;
await db.transaction('rw', db[IDXDB_TABLES.conversations], async () => {
const convs = await db[IDXDB_TABLES.conversations].bulkGet(cleanIds);
const updates: DatabaseConversation[] = [];
for (let i = 0; i < cleanIds.length; i++) {
const conv = convs[i];
if (!conv) continue;
const newPinned = !conv.pinned;
updates.push({ ...conv, pinned: newPinned });
result.set(cleanIds[i], newPinned);
}
if (updates.length === 0) return;
await db[IDXDB_TABLES.conversations].bulkPut(updates);
});
return result;
}
/**
* Updates the conversation's current node (active branch).
* This determines which conversation path is currently being viewed.
*
* @param convId - Conversation ID
* @param nodeId - Message ID to set as current node
*/
static async updateCurrentNode(convId: string, nodeId: string): Promise<void> {
await this.updateConversation(convId, {
currNode: nodeId
});
}
/**
* Updates a message.
*
* @param id - Message ID
* @param updates - Partial updates to apply
* @returns Promise that resolves when the message is updated
*/
static async updateMessage(
id: string,
updates: Partial<Omit<DatabaseMessage, 'id'>>
): Promise<void> {
await db[IDXDB_TABLES.messages].update(id, updates);
}
/**
*
*
* Import
*
*
*/
/**
* Imports multiple conversations and their messages.
* Skips conversations that already exist.
*
* @param data - Array of { conv, messages } objects
* @returns The conversations written to the database and the ones skipped
*/
static async importConversations(
data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[]
): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {
const imported: DatabaseConversation[] = [];
const skipped: DatabaseConversation[] = [];
return await db.transaction(
'rw',
[db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]],
async () => {
for (const item of data) {
const { conv, messages } = item;
const existing = await db[IDXDB_TABLES.conversations].get(conv.id);
if (existing) {
skipped.push(conv);
continue;
}
await db[IDXDB_TABLES.conversations].add(conv);
for (const msg of messages) {
await db[IDXDB_TABLES.messages].put(msg);
}
imported.push(conv);
}
return { imported, skipped };
}
);
}
/**
*
*
* Forking
*
*
*/
/**
* Forks a conversation at a specific message, creating a new conversation
* containing all messages from the root up to (and including) the target message.
@ -726,13 +434,272 @@ export class DatabaseService {
};
await db[IDXDB_TABLES.conversations].add(newConv);
for (const msg of clonedMessages) {
await db[IDXDB_TABLES.messages].add(msg);
}
await db[IDXDB_TABLES.messages].bulkAdd(clonedMessages);
return newConv;
}
);
}
/**
* Gets all conversations, sorted by last modified time (newest first).
*
* @returns Array of conversations
*/
static async getAllConversations(): Promise<DatabaseConversation[]> {
return await db[IDXDB_TABLES.conversations].orderBy('lastModified').reverse().toArray();
}
/**
* Gets a conversation by ID.
*
* @param id - Conversation ID
* @returns The conversation if found, otherwise undefined
*/
static async getConversation(id: string): Promise<DatabaseConversation | undefined> {
return await db[IDXDB_TABLES.conversations].get(id);
}
/**
* Gets all messages in a conversation, sorted by timestamp (oldest first).
*
* @param convId - Conversation ID
* @returns Array of messages in the conversation
*/
static async getConversationMessages(convId: string): Promise<DatabaseMessage[]> {
return await db[IDXDB_TABLES.messages].where('convId').equals(convId).sortBy('timestamp');
}
/**
* Loads multiple conversations with all of their messages in two bulk
* reads. Missing conversations are silently omitted from the result.
*
* @param convIds - Conversation IDs to load
* @returns Map of id -> { conv, messages }. Messages are sorted ascending by timestamp.
*/
static async getConversationsWithMessages(
convIds: string[]
): Promise<Map<string, ExportedConversation>> {
const result = new Map<string, ExportedConversation>();
const cleanIds = convIds.filter((id): id is string => typeof id === 'string' && id.length > 0);
if (cleanIds.length === 0) return result;
const [convs, allMessages] = await Promise.all([
db[IDXDB_TABLES.conversations].bulkGet(cleanIds),
db[IDXDB_TABLES.messages].where('convId').anyOf(cleanIds).toArray()
]);
const messagesByConv = new Map<string, DatabaseMessage[]>();
for (const msg of allMessages) {
const bucket = messagesByConv.get(msg.convId);
if (bucket) bucket.push(msg);
else messagesByConv.set(msg.convId, [msg]);
}
for (let i = 0; i < cleanIds.length; i++) {
const conv = convs[i];
if (!conv) continue;
const messages = (messagesByConv.get(conv.id) ?? []).sort(
(a, b) => a.timestamp - b.timestamp
);
result.set(conv.id, { conv, messages });
}
return result;
}
/**
* Imports multiple conversations and their messages.
* Skips conversations that already exist.
*
* @param data - Array of { conv, messages } objects
* @returns The conversations written to the database and the ones skipped
*/
static async importConversations(
data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[]
): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {
const imported: DatabaseConversation[] = [];
const skipped: DatabaseConversation[] = [];
return await db.transaction(
'rw',
[db[IDXDB_TABLES.conversations], db[IDXDB_TABLES.messages]],
async () => {
for (const item of data) {
const { conv, messages } = item;
const existing = await db[IDXDB_TABLES.conversations].get(conv.id);
if (existing) {
skipped.push(conv);
continue;
}
await db[IDXDB_TABLES.conversations].add(conv);
for (const msg of messages) {
await db[IDXDB_TABLES.messages].put(msg);
}
imported.push(conv);
}
return { imported, skipped };
}
);
}
/**
* Toggles the pinned status of a conversation.
*
* @param id - Conversation ID
* @returns The new pinned status
*/
static async toggleConversationPin(id: string): Promise<boolean> {
const conversation = await db[IDXDB_TABLES.conversations].get(id);
if (!conversation) {
throw new Error(`Conversation ${id} not found`);
}
const newPinnedState = !conversation.pinned;
await this.updateConversation(id, { pinned: newPinnedState });
return newPinnedState;
}
/**
* Updates a conversation. `lastModified` is never stamped implicitly;
* pass it in `updates` to bump the conversation in recency ordering.
*
* @param id - Conversation ID
* @param updates - Partial updates to apply
* @returns Promise that resolves when the conversation is updated
*/
static async updateConversation(
id: string,
updates: Partial<Omit<DatabaseConversation, 'id'>>
): Promise<void> {
await db[IDXDB_TABLES.conversations].update(id, updates);
}
/**
* Updates the conversation's current node (active branch).
* This determines which conversation path is currently being viewed.
*
* @param convId - Conversation ID
* @param nodeId - Message ID to set as current node
*/
static async updateCurrentNode(convId: string, nodeId: string): Promise<void> {
await this.updateConversation(convId, {
currNode: nodeId
});
}
/**
* Updates a message.
*
* @param id - Message ID
* @param updates - Partial updates to apply
* @returns Promise that resolves when the message is updated
*/
static async updateMessage(
id: string,
updates: Partial<Omit<DatabaseMessage, 'id'>>
): Promise<void> {
await db[IDXDB_TABLES.messages].update(id, updates);
}
/**
* Appends a child id to a parent message's children array.
*/
private static async addChildToParent(parentId: string, childId: string): Promise<void> {
const parent = await db[IDXDB_TABLES.messages].get(parentId);
if (!parent) return;
await db[IDXDB_TABLES.messages].update(parentId, {
children: [...parent.children, childId]
});
}
/**
* Removes a child id from its parent message's children array.
*/
private static async removeChildFromParent(messageId: string): Promise<void> {
const message = await db[IDXDB_TABLES.messages].get(messageId);
if (!message?.parent) return;
const parent = await db[IDXDB_TABLES.messages].get(message.parent);
if (!parent) return;
parent.children = parent.children.filter((childId: string) => childId !== messageId);
await db[IDXDB_TABLES.messages].put(parent);
}
/**
* Reparents direct children of `parentId` to the nearest surviving
* ancestor (or promotes them to top-level when the immediate parent was
* top-level). Walking skips any ancestor listed in `excludeIds`, since
* those will be deleted in the same batch — leaving a grandchild pointing
* at an `excludeIds` entry would orphan it. Children whose own id is in
* `excludeIds` are dropped from the updates (the bulk-delete pass will
* remove them). `prefetched` may carry a pre-fetched ancestor map to
* avoid repeat reads inside a bulk transaction.
*/
private static async reparentDirectChildren(
parentId: string,
excludeIds: ReadonlySet<string> = new Set(),
prefetched?: ReadonlyMap<string, DatabaseConversation>
): Promise<void> {
const conv = prefetched?.get(parentId) ?? (await db[IDXDB_TABLES.conversations].get(parentId));
if (!conv) return;
let newParent = conv.forkedFromConversationId;
const visited = new Set<string>([parentId]);
while (newParent && excludeIds.has(newParent)) {
if (visited.has(newParent)) {
newParent = undefined;
break;
}
visited.add(newParent);
const next =
prefetched?.get(newParent) ?? (await db[IDXDB_TABLES.conversations].get(newParent));
if (!next) {
newParent = undefined;
break;
}
newParent = next.forkedFromConversationId;
}
const directChildren = await db[IDXDB_TABLES.conversations]
.filter((c) => c.forkedFromConversationId === parentId)
.toArray();
const updates: DatabaseConversation[] = [];
for (const child of directChildren) {
if (excludeIds.has(child.id)) continue;
updates.push({ ...child, forkedFromConversationId: newParent });
}
if (updates.length === 0) return;
await db[IDXDB_TABLES.conversations].bulkPut(updates);
}
}

38
tools/ui/src/lib/services/index.ts
View file

@ -53,9 +53,9 @@
* - Reasoning content stripping from prompt history to avoid KV cache pollution
* - Error translation (network, timeout, server errors → user-friendly messages)
*
* @see chatStore in stores/chat.svelte.ts — primary consumer for chat state management
* @see agenticStore in stores/agentic.svelte.ts — uses ChatService for agentic loop streaming
* @see conversationsStore in stores/conversations.svelte.ts — provides message context
* @see chatStore in stores/chat/index.svelte.ts — primary consumer for chat state management
* @see agenticStore in stores/agentic/index.svelte.ts — uses ChatService for agentic loop streaming
* @see conversationsStore in stores/conversations/index.svelte.ts — provides message context
*/
export { ChatService } from './chat.service';
@ -98,8 +98,8 @@ export { ChatService } from './chat.service';
* enabling conversation branching and alternative response paths. The conversation's
* `currNode` tracks the currently active branch endpoint.
*
* @see conversationsStore in stores/conversations.svelte.ts — reactive layer on top of DatabaseService
* @see chatStore in stores/chat.svelte.ts — uses DatabaseService directly for message CRUD during streaming
* @see conversationsStore in stores/conversations/index.svelte.ts — reactive layer on top of DatabaseService
* @see chatStore in stores/chat/index.svelte.ts — uses DatabaseService directly for message CRUD during streaming
*/
export { DatabaseService } from './database.service';
@ -143,7 +143,7 @@ export { ConversationTransferService } from './conversation-transfer.service';
* - `POST /models/load` — Load a model (ROUTER mode only)
* - `POST /models/unload` — Unload a model (ROUTER mode only)
*
* @see modelsStore in stores/models.svelte.ts — primary consumer for reactive model state
* @see modelsStore in stores/models/index.svelte.ts — primary consumer for reactive model state
*/
export { ModelsService } from './models.service';
@ -174,8 +174,8 @@ export { ModelsService } from './models.service';
* - `&autoload=false` → Prevents model auto-loading when querying props
*
* @see serverStore in stores/server.svelte.ts — consumes global server props
* @see modelsStore in stores/models.svelte.ts — consumes per-model props for modalities
* @see settingsStore in stores/settings.svelte.ts — syncs default generation params from props
* @see modelsStore in stores/models/index.svelte.ts — consumes per-model props for modalities
* @see settingsStore in stores/settings/index.svelte.ts — syncs default generation params from props
*/
export { PropsService } from './props.service';
@ -217,7 +217,7 @@ export { PropsService } from './props.service';
* - `ParameterSyncService` class — static methods for sync logic
* - `SYNCABLE_PARAMETERS` — mapping of UI setting keys to server parameter keys
*
* @see settingsStore in stores/settings.svelte.ts — primary consumer for settings sync
* @see settingsStore in stores/settings/index.svelte.ts — primary consumer for settings sync
* @see SettingsChatParameterSourceIndicator — displays parameter source badges in UI
*/
export { ParameterSyncService } from './parameter-sync.service';
@ -241,7 +241,7 @@ export { ParameterSyncService } from './parameter-sync.service';
* - Manages connection lifecycle, health checks, reconnection
* - Handles tool name conflict resolution and server coordination
*
* - **mcpResourceStore**: Reactive resource state
* - **mcpResourceStore** (composed as mcpStore.resources): Reactive resource state
* - Receives resource data fetched via MCPService
* - Manages resource caching, subscriptions, and attachments
*
@ -263,9 +263,9 @@ export { ParameterSyncService } from './parameter-sync.service';
* 2. **StreamableHTTP** — modern HTTP-based, supports CORS proxy
* 3. **SSE** — legacy fallback, supports CORS proxy
*
* @see mcpStore in stores/mcp.svelte.ts — reactive business logic facade on top of MCPService
* @see mcpResourceStore in stores/mcp-resources.svelte.ts — reactive resource state management
* @see agenticStore in stores/agentic.svelte.ts — uses MCPService (via mcpStore) for tool execution
* @see mcpStore in stores/mcp/index.svelte.ts — reactive business logic facade on top of MCPService
* @see mcpStore.resources in stores/mcp/resources.svelte.ts — reactive resource state management
* @see agenticStore in stores/agentic/index.svelte.ts — uses MCPService (via mcpStore) for tool execution
* @see MCP Protocol Specification: https://modelcontextprotocol.io/specification/2025-06-18
*/
export { MCPService } from './mcp.service';
@ -286,7 +286,7 @@ export { MCPService } from './mcp.service';
* - **agenticStore**: Dispatches ToolSource.BROWSER calls here
*
* @see buildSandboxToolDefinition in utils/sandbox-tool - tool schema sent to the LLM
* @see agenticStore in stores/agentic.svelte.ts - tool dispatch
* @see agenticStore in stores/agentic/index.svelte.ts - tool dispatch
*/
export { SandboxService } from './sandbox.service';
@ -340,3 +340,13 @@ export { RouterService } from './router.service';
* @see migration.service.ts — full implementation (non-destructive)
*/
export { MigrationService } from './migration.service';
/**
* **SettingsService** - localStorage persistence layer for settings
*
* Stateless read/write of the settings config and user-override keys. Business
* logic (default merging, mobile defaults, theme migration) stays in the store.
*
* @see settingsStore in stores/settings/index.svelte.ts - reactive state + business logic
*/
export { SettingsService } from './settings.service';

1426
tools/ui/src/lib/services/mcp.service.ts
View file

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more