diff --git a/.github/actions/ccache-clear/action.yml b/.github/actions/ccache-clear/action.yml deleted file mode 100644 index d38587efa..000000000 --- a/.github/actions/ccache-clear/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "ccache-clear" -description: "Delete all GitHub Actions caches matching a key prefix" -inputs: - key: - description: "Cache key prefix to match and delete" - required: true - -runs: - using: "composite" - steps: - - name: Clear caches - shell: bash - run: | - CACHES=$(gh cache list --key "ccache-${{ inputs.key }}" --json id,key --jq '.[] | "\(.id) \(.key)"' 2>/dev/null) - if [ -z "$CACHES" ]; then - echo "No caches found with key prefix: ${{ inputs.key }}" - exit 0 - fi - while read -r id key; do - echo "Deleting cache: $id ($key)" - gh cache delete "$id" - done <<< "$CACHES" diff --git a/common/arg.cpp b/common/arg.cpp index 4235e303e..b82bbd188 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2596,6 +2596,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.mmproj_use_gpu = value; } ).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_OFFLOAD")); + add_opt(common_arg( + // note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet + {"-mmdev", "--mmproj-device"}, "DEVICE", + "device to use for multimodal projector (none = don't offload, default: auto)\n" + "use --list-devices to see a list of available devices", + [](common_params & params, const std::string & value) { + if (value == "none") { + params.mmproj_use_gpu = false; + params.mmproj_device = nullptr; + return; + } + auto devices = parse_device_list(value); + // parse_device_list pushes nullptr at back so devices is length 2 for single device. + if (devices.size() > 2) { + throw std::invalid_argument("only one device may be specified for mmproj"); + } + params.mmproj_use_gpu = true; + params.mmproj_device = devices.front(); + } + ).set_examples(mmproj_examples).set_env("MTMD_BACKEND_DEVICE")); // no LLAMA_ARG_ prefix for backward compatibility reason add_opt(common_arg( {"--image", "--audio", "--video"}, "FILE", "path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files\n", diff --git a/common/common.h b/common/common.h index ebdc23d18..95d8e332a 100644 --- a/common/common.h +++ b/common/common.h @@ -582,9 +582,10 @@ struct common_params { // multimodal models (see tools/mtmd) struct common_params_model mmproj; - bool mmproj_use_gpu = true; // use GPU for multimodal model - bool no_mmproj = false; // explicitly disable multimodal model - std::vector image; // path to image file(s) ; TODO: change the name to "media" + bool mmproj_use_gpu = true; // use GPU for multimodal model + ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model + bool no_mmproj = false; // explicitly disable multimodal model + std::vector image; // path to image file(s) ; TODO: change the name to "media" int image_min_tokens = -1; int image_max_tokens = -1; int mtmd_batch_max_tokens = 1024; diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index b18607cd6..955b4e014 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -278,7 +278,9 @@ static std::unordered_map GRAMMAR_LITERAL_ESCAPES = { {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}, {'\\', "\\\\"} }; -static std::unordered_set NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'}; +static const int MAX_PATTERN_DEPTH = 100; + +static std::unordered_set NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?', '^', '$'}; static std::unordered_set ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'}; static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function & replacement) { @@ -309,6 +311,32 @@ static std::string format_literal(const std::string & literal) { std::string gbnf_format_literal(const std::string & literal) { return format_literal(literal); } +static size_t gbnf_escape_length(const std::string & pattern, size_t pos) { + if (pos + 1 >= pattern.length() || pattern[pos] != '\\') { + return 0; + } + size_t n_hex = 0; + switch (pattern[pos + 1]) { + case 'x': n_hex = 2; break; + case 'u': n_hex = 4; break; + case 'U': n_hex = 8; break; + case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']': + return 2; + default: + return 0; + } + if (pos + 2 + n_hex > pattern.length()) { + return 0; + } + for (size_t i = pos + 2; i < pos + 2 + n_hex; i++) { + char h = pattern[i]; + if (!((h >= '0' && h <= '9') || (h >= 'a' && h <= 'f') || (h >= 'A' && h <= 'F'))) { + return 0; + } + } + return 2 + n_hex; +} + class common_schema_converter { private: friend class common_schema_info; @@ -345,16 +373,42 @@ private: return string_join(rules, " | "); } + // thrown when the pattern is a valid regex with no grammar equivalent + struct unsupported_pattern : public std::runtime_error { + using std::runtime_error::runtime_error; + }; + + // thrown when the pattern is not a valid regex + struct invalid_pattern : public std::runtime_error { + using std::runtime_error::runtime_error; + }; + std::string _visit_pattern(const std::string & pattern, const std::string & name) { - if (!(pattern.front() == '^' && pattern.back() == '$')) { - _errors.push_back("Pattern must start with '^' and end with '$'"); + auto rules_snapshot = _rules; + try { + return _pattern_to_rule(pattern, name); + } catch (const unsupported_pattern & err) { + // revert rules + _rules = std::move(rules_snapshot); + _warnings.push_back("pattern " + pattern + " is not supported (" + err.what() + "), accepting any string"); + return _add_rule(name, _add_primitive("string", PRIMITIVE_RULES.at("string"))); + } catch (const invalid_pattern & err) { + _rules = std::move(rules_snapshot); + _errors.push_back("Invalid pattern " + pattern + ": " + err.what()); return ""; } + } + + std::string _pattern_to_rule(const std::string & pattern, const std::string & name) { + if (pattern.length() < 2 || pattern.front() != '^' || pattern.back() != '$') { + throw unsupported_pattern("not anchored with '^' and '$'"); + } std::string sub_pattern = pattern.substr(1, pattern.length() - 2); std::unordered_map sub_rule_ids; size_t i = 0; size_t length = sub_pattern.length(); + int paren_depth = 0; using literal_or_rule = std::pair; auto to_rule = [&](const literal_or_rule & ls) { @@ -363,7 +417,6 @@ private: return is_literal ? "\"" + s + "\"" : s; }; std::function transform = [&]() -> literal_or_rule { - size_t start = i; std::vector seq; auto get_dot = [&]() { @@ -420,43 +473,42 @@ private: if (i + 1 < length && sub_pattern[i + 1] == ':') { i += 2; // skip "?:" for non-capturing group, treat as regular group } else { - // lookahead/lookbehind (?=, ?!, ?<=, ? 0) { - if (sub_pattern[i] == '\\' && i + 1 < length) { - i += 2; // skip escaped character - } else { - if (sub_pattern[i] == '(') depth++; - else if (sub_pattern[i] == ')') depth--; - i++; - } - } - continue; + // lookaround, named group, inline flags, ... + throw unsupported_pattern("unsupported group syntax"); } } + paren_depth++; + if (paren_depth > MAX_PATTERN_DEPTH) { + throw unsupported_pattern("pattern nesting too deep"); + } seq.emplace_back("(" + to_rule(transform()) + ")", false); } else if (c == ')') { i++; - if (start > 0 && sub_pattern[start - 1] != '(' && (start < 2 || sub_pattern[start - 2] != '?' || sub_pattern[start - 1] != ':')) { - _errors.push_back("Unbalanced parentheses"); + if (paren_depth == 0) { + throw invalid_pattern("unbalanced parentheses"); } + paren_depth--; return join_seq(); + } else if (c == '^' || c == '$') { + throw unsupported_pattern("anchor inside the pattern"); } else if (c == '[') { std::string square_brackets = std::string(1, c); i++; while (i < length && sub_pattern[i] != ']') { if (sub_pattern[i] == '\\') { - square_brackets += sub_pattern.substr(i, 2); - i += 2; + auto escape_length = gbnf_escape_length(sub_pattern, i); + if (escape_length == 0) { + throw unsupported_pattern("unsupported escape in character class: " + sub_pattern.substr(i, 2)); + } + square_brackets += sub_pattern.substr(i, escape_length); + i += escape_length; } else { square_brackets += sub_pattern[i]; i++; } } if (i >= length) { - _errors.push_back("Unbalanced square brackets"); + throw invalid_pattern("unterminated character class"); } square_brackets += ']'; i++; @@ -465,6 +517,9 @@ private: seq.emplace_back("|", false); i++; } else if (c == '*' || c == '+' || c == '?') { + if (seq.empty()) { + throw invalid_pattern("nothing to repeat"); + } seq.back() = std::make_pair(to_rule(seq.back()) + c, false); i++; } else if (c == '{') { @@ -475,18 +530,19 @@ private: i++; } if (i >= length) { - _errors.push_back("Unbalanced curly brackets"); + throw unsupported_pattern("unterminated curly brackets"); } curly_brackets += '}'; i++; auto nums = string_split(curly_brackets.substr(1, curly_brackets.length() - 2), ","); int min_times = 0; int max_times = std::numeric_limits::max(); + if (nums.size() != 1 && nums.size() != 2) { + throw unsupported_pattern("wrong number of values in curly brackets"); + } try { if (nums.size() == 1) { min_times = max_times = std::stoi(nums[0]); - } else if (nums.size() != 2) { - _errors.push_back("Wrong number of values in curly brackets"); } else { if (!nums[0].empty()) { min_times = std::stoi(nums[0]); @@ -495,9 +551,11 @@ private: max_times = std::stoi(nums[1]); } } - } catch (const std::invalid_argument & e) { - _errors.push_back("Invalid number in curly brackets"); - return std::make_pair("", false); + } catch (const std::logic_error &) { + throw unsupported_pattern("invalid number in curly brackets"); + } + if (seq.empty()) { + throw invalid_pattern("nothing to repeat"); } auto &last = seq.back(); auto &sub = last.first; @@ -523,15 +581,22 @@ private: return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end(); }; while (i < length) { - if (sub_pattern[i] == '\\' && i < length - 1) { + if (sub_pattern[i] == '\\') { + if (i == length - 1) { + throw invalid_pattern("trailing backslash"); + } char next = sub_pattern[i + 1]; if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) { i++; literal += sub_pattern[i]; i++; } else { - literal += sub_pattern.substr(i, 2); - i += 2; + auto escape_length = gbnf_escape_length(sub_pattern, i); + if (escape_length == 0) { + throw unsupported_pattern("unsupported escape: " + sub_pattern.substr(i, 2)); + } + literal += sub_pattern.substr(i, escape_length); + i += escape_length; } } else if (sub_pattern[i] == '"') { literal += "\\\""; @@ -544,14 +609,21 @@ private: break; } } - if (!literal.empty()) { - seq.emplace_back(literal, true); + if (literal.empty()) { // nothing was consumed, ex. a stray ']' or '}' + throw unsupported_pattern(std::string("unsupported character: ") + c); } + seq.emplace_back(literal, true); } } return join_seq(); }; - return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\""); + + auto rule = to_rule(transform()); + if (paren_depth != 0) { + throw invalid_pattern("unbalanced parentheses"); + } + + return _add_rule(name, "\"\\\"\" (" + rule + ") \"\\\"\""); } /* diff --git a/common/speculative.cpp b/common/speculative.cpp index 08ac810cd..75cfd1b54 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2649,6 +2649,10 @@ void common_speculative_draft(common_speculative * spec) { for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) dparams.size(); ++seq_id) { auto & dp = dparams[seq_id]; + if (!dp.drafting) { + continue; + } + auto & result = *dp.result; // a new draft has been sampled diff --git a/conversion/__init__.py b/conversion/__init__.py index 5ae6ad819..4b8817ead 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -57,6 +57,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3DSparkModel": "qwen", "DSparkDraftModel": "qwen", "DSparkSpeculator": "qwen", + "Lfm2DSparkDraftModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", "DeepseekV4DSparkModel": "deepseek", "DistilBertForMaskedLM": "bert", diff --git a/conversion/nemotron.py b/conversion/nemotron.py index 3e37c7b46..e5d167185 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -207,7 +207,9 @@ class NemotronHModel(GraniteHybridModel): # calling the parent __init__. This is because the parent constructor # uses self.model_arch to build the tensor name map, and all MoE-specific # mappings would be missed if it were called with the default non-MoE arch. - hparams = ModelBase.load_hparams(args[0], self.is_mistral_format) + hparams = kwargs.pop("hparams", None) + if hparams is None: + hparams = ModelBase.load_hparams(args[0], self.is_mistral_format) has_moe_params = ( "num_experts_per_tok" in hparams or (isinstance(hparams.get("llm_config"), dict) and "num_experts_per_tok" in hparams["llm_config"]) @@ -215,8 +217,11 @@ class NemotronHModel(GraniteHybridModel): if has_moe_params: self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE self.is_moe = True + layers_block_type = hparams.get("layers_block_type") + if layers_block_type is not None: + hparams["num_hidden_layers"] = len(layers_block_type) - super().__init__(*args, **kwargs) + super().__init__(*args, hparams=hparams, **kwargs) # Save the top-level head_dim for later self.head_dim = self.hparams.get("head_dim", self.hparams.get("attention_head_dim")) diff --git a/conversion/qwen.py b/conversion/qwen.py index 26b10452b..355365763 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -709,7 +709,7 @@ class DFlashModel(Qwen3Model): yield from super().modify_tensors(data_torch, name, bid) -@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator") +@ModelBase.register("Qwen3DSparkModel", "DSparkDraftModel", "DSparkSpeculator", "Lfm2DSparkDraftModel") @ModelBase.example("satgeze/Qwen3.6-27B-DSpark") class DSparkModel(DFlashModel): # DSpark = DFlash + a semi-autoregressive Markov head. @@ -759,6 +759,13 @@ class DSparkModel(DFlashModel): return None return super().filter_tensors(item) + _ROPE_PERMUTE_SUFFIXES = ( + "self_attn.q_proj.weight", + "self_attn.k_proj.weight", + "self_attn.q_norm.weight", + "self_attn.k_norm.weight", + ) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name == "model.d2t": self._d2t = data_torch @@ -767,6 +774,12 @@ class DSparkModel(DFlashModel): if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")): return + # interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd + if not self.hparams.get("rope_is_neox_style", True) and name.endswith(self._ROPE_PERMUTE_SUFFIXES): + head_dim = self.hparams["head_dim"] + shape = data_torch.shape + data_torch = data_torch.reshape(-1, head_dim // 2, 2, *shape[1:]).transpose(1, 2).reshape(shape) + yield from super().modify_tensors(data_torch, name, bid) def prepare_tensors(self): diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index 40cea024c..9c56ec30c 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -83,7 +83,6 @@ extern "C" { GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers); GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer); GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); - GGML_API void ggml_backend_meta_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); // // Backend (meta) diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 775ae9926..7654ea1f3 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -1118,6 +1118,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( } static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync) { + GGML_ASSERT(ggml_backend_buffer_is_meta(tensor->buffer)); ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context; return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync); } @@ -1177,15 +1178,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m t_ij->flags = tensor->flags; memcpy(t_ij->op_params, tensor->op_params, sizeof(tensor->op_params)); ggml_set_name(t_ij, tensor->name); - t_ij->buffer = simple_buf; - if (simple_buf) { - // the backend that owns the buffer will set .extra - ggml_backend_buffer_init_tensor(simple_buf, t_ij); - } else { - t_ij->extra = tensor->extra; - } - t_ij->view_src = tensor->view_src; t_ij->view_offs = tensor->view_offs; if (t_ij->view_src != nullptr && ggml_backend_buffer_is_meta(t_ij->view_src->buffer)) { @@ -1216,6 +1209,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m t_ij->data = (char *) ggml_backend_buffer_get_base(simple_buf) + size_t(tensor->data) - size_t(ggml_backend_buffer_get_base(tensor->buffer)); } + t_ij->extra = tensor->extra; for (int i = 0; i < GGML_MAX_SRC; i++) { t_ij->src[i] = tensor->src[i]; if (tensor->src[i] == tensor) { @@ -1508,16 +1502,6 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) { return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer; } -void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) { - GGML_ASSERT(ggml_backend_buffer_is_meta(buffer)); - ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context; - for (size_t i = 0; i < buf_ctx->bufs.size(); i++) { - if (buf_ctx->bufs[i]) { - ggml_backend_buffer_set_usage(buf_ctx->bufs[i].get(), usage); - } - } -} - static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft); diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 1f35c5f86..3b9316c9c 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -182,8 +182,6 @@ void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backe // FIXME: add a generic callback to the buffer interface if (ggml_backend_buffer_is_multi_buffer(buffer)) { ggml_backend_multi_buffer_set_usage(buffer, usage); - } else if (ggml_backend_buffer_is_meta(buffer)) { - ggml_backend_meta_buffer_set_usage(buffer, usage); } } @@ -1608,11 +1606,23 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s std::vector ids; std::vector used_ids; + int prev_backend_id = -1; + for (int split_id = 0; split_id < sched->n_splits; split_id++) { struct ggml_backend_sched_split * split = &splits[split_id]; int split_backend_id = split->backend_id; ggml_backend_t split_backend = sched->backends[split_backend_id]; + // ensure the previous split's async work has completed before we start + // this split, the allocator may have reused buffer regions across splits + if (split->n_inputs == 0 && prev_backend_id >= 0 && prev_backend_id != split_backend_id) { + if (sched->events[prev_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[prev_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(sched->backends[prev_backend_id]); + } + } + // copy the input tensors to the split backend for (int input_id = 0; input_id < split->n_inputs; input_id++) { ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); @@ -1775,12 +1785,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } } - // record the event of this copy - if (split->n_inputs > 0) { - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend); - } + // record the event of this split + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend); } + + prev_backend_id = split_backend_id; } return GGML_STATUS_SUCCESS; diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index f1ce30e6c..268f3c8ef 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1425,7 +1425,9 @@ struct ggml_backend_cuda_context { cudaEvent_t copy_event = nullptr; cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } }; - cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr}; + cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; + void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {nullptr}; + size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {0}; int curr_stream_no = 0; @@ -1502,17 +1504,22 @@ struct ggml_backend_cuda_context { ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; } - cublasHandle_t cublas_handle(int device) { - if (cublas_handles[device] == nullptr) { - ggml_cuda_set_device(device); - CUBLAS_CHECK(cublasCreate(&cublas_handles[device])); - CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH)); - } - return cublas_handles[device]; - } - cublasHandle_t cublas_handle() { - return cublas_handle(device); + if (cublas_handles[device][curr_stream_no] == nullptr) { + ggml_cuda_set_device(device); + CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no])); + CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream())); +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2)) + if (cublas_workspace_sizes[device] == 0) { + const int cc = ggml_cuda_info().devices[device].cc; + cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024; + } + CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); + CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); +#endif + } + return cublas_handles[device][curr_stream_no]; } // pool diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index a68544f66..e3fa8304a 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -711,9 +711,12 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { if (streams[i][j] != nullptr) { CUDA_CHECK(cudaStreamDestroy(streams[i][j])); } - } - if (cublas_handles[i] != nullptr) { - CUBLAS_CHECK(cublasDestroy(cublas_handles[i])); + if (cublas_handles[i][j] != nullptr) { + CUBLAS_CHECK(cublasDestroy(cublas_handles[i][j])); + } + if (cublas_workspaces[i][j] != nullptr) { + CUDA_CHECK(cudaFree(cublas_workspaces[i][j])); + } } } } @@ -1416,7 +1419,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const const int64_t ne_dst = ggml_nelements(dst); cudaStream_t main_stream = ctx.stream(); - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream)); + cublasHandle_t cublas_h = ctx.cublas_handle(); const size_t src0_ts = ggml_type_size(src0->type); GGML_ASSERT(nb00 == src0_ts); @@ -1539,14 +1542,14 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const // probably because the internal kernel selection logic is suboptimal. if (compute_type == GGML_TYPE_F32 && ne12 == 1 && ne13 == 1) { CUBLAS_CHECK( - cublasSgemm(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasSgemm(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, (const float *) alpha, (const float *) src0_ptr, s01, (const float *) src1_ptr, s11, (const float *) beta, (float *) dst_ptr, ne0)); } else if (ne12 == 1 && ne13 == 1) { CUBLAS_CHECK( - cublasGemmEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, src0_ptr, cu_data_type_a, s01, src1_ptr, cu_data_type_b, s11, @@ -1561,7 +1564,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const // there is no broadcast and src0, src1 are contiguous across dims 2, 3 // use cublasGemmStridedBatchedEx CUBLAS_CHECK( - cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmStridedBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, src0_ptr, cu_data_type_a, s01, sma, // strideA src1_ptr, cu_data_type_b, s11, smb, // strideB @@ -1599,7 +1602,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const CUDA_CHECK(cudaGetLastError()); CUBLAS_CHECK( - cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, s01, (const void **) (ptrs_src.get() + 1*ne23), cu_data_type_b, s11, diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index c99923804..970534809 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -290,6 +290,42 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { if (!ggml_is_quantized(type)) { return false; } + // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner. + // Only list quant-types MMQ supports, others would fall back to cuBLAS. + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) { + switch (type) { // tuned on RTX 4090 + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q3_K: + return ne11 <= 6; + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) { + switch (type) { // tuned on RTX 5090 + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 5; + case GGML_TYPE_Q6_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + switch (type) { // tuned on DGX Spark GB10 + case GGML_TYPE_Q2_K: + return ne11 <= 6; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } if (GGML_CUDA_CC_IS_CDNA(cc)) { if (GGML_CUDA_CC_IS_CDNA1(cc)) { switch (type) { diff --git a/ggml/src/ggml-cuda/out-prod.cu b/ggml/src/ggml-cuda/out-prod.cu index 46b9f3a67..c46e0455d 100644 --- a/ggml/src/ggml-cuda/out-prod.cu +++ b/ggml/src/ggml-cuda/out-prod.cu @@ -54,8 +54,6 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const float alpha = 1.0f; const float beta = 0.0f; - CUBLAS_CHECK(cublasSetStream(handle, stream)); - const int64_t lda = nb01 / sizeof(float); const int64_t ldc = nb1 / sizeof(float); diff --git a/ggml/src/ggml-cuda/solve_tri.cu b/ggml/src/ggml-cuda/solve_tri.cu index 07ca33f51..d96783420 100644 --- a/ggml/src/ggml-cuda/solve_tri.cu +++ b/ggml/src/ggml-cuda/solve_tri.cu @@ -65,15 +65,13 @@ static void solve_tri_f32_cublas(ggml_backend_cuda_context & ctx, get_batch_pointers<<<(total_batches + 255) / 256, 256, 0, stream>>>(A, X, A_ptrs_dev, X_ptrs_dev, ne02, total_batches, s02, s03, s2, s3); - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - // Yes, this is necessary, without this we get RMSE errors - CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_DEFAULT_MATH)); - CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(id), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_DEFAULT_MATH)); + CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT, k, n, &alpha, A_ptrs_dev, n, X_ptrs_dev, k, total_batches)); // revert to standard mode from common.cuh - CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_TF32_TENSOR_OP_MATH)); GGML_UNUSED_VARS(s12, s13); } diff --git a/ggml/src/ggml-cuda/ssm-scan.cu b/ggml/src/ggml-cuda/ssm-scan.cu index ef342f01f..40cb38dee 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cu +++ b/ggml/src/ggml-cuda/ssm-scan.cu @@ -632,7 +632,6 @@ static void ssm_scan_ssd_f32_cuda( // Step 3: chunked SSD loop // Per chunk: pre_matmul (incl. M) + 4 cuBLAS (CB, Y, S@C, state update) + scale_state cublasHandle_t handle = ctx.cublas_handle(); - CUBLAS_CHECK(cublasSetStream(handle, stream)); const float alpha_one = 1.0f; const float beta_zero = 0.0f; const float beta_one = 1.0f; diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 953c75755..52043696e 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -1409,6 +1409,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_p return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16( + ggml_metal_library_t lib, + const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + char base[256]; + + snprintf(base, 256, "kernel_flash_attn_ext_kv_%s_f16", ggml_type_name(op->src[1]->type)); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, base); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, base, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -1460,7 +1477,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( bool has_bias, bool has_scap, bool has_kvpad, - int32_t nsg) { + int32_t nsg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); char base[256]; @@ -1469,15 +1489,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( const int32_t dk = (int32_t) op->src[1]->ne[0]; const int32_t dv = (int32_t) op->src[2]->ne[0]; - const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0]; - const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0]; + const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type); // do bounds checks for the mask? const bool bc_mask = op->src[3] && (op->src[3]->ne[1] % 8 != 0); snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d", "flash_attn_ext", - ggml_type_name(op->src[1]->type), + type, dk, dv); @@ -1526,7 +1545,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v bool has_scap, bool has_kvpad, int32_t nsg, - int32_t nwg) { + int32_t nwg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); char base[256]; @@ -1535,12 +1557,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v const int32_t dk = (int32_t) op->src[1]->ne[0]; const int32_t dv = (int32_t) op->src[2]->ne[0]; - const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0]; - const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0]; + const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type); snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d", "flash_attn_ext_vec", - ggml_type_name(op->src[1]->type), + type, dk, dv); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 7e1deeaa2..b7d466058 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -176,6 +176,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_mask, int32_t ncpsg); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16( + ggml_metal_library_t lib, + const struct ggml_tensor * op); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -190,7 +194,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_bias, bool has_scap, bool has_kvpad, - int32_t nsg); + int32_t nsg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec( ggml_metal_library_t lib, @@ -201,7 +208,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_scap, bool has_kvpad, int32_t nsg, - int32_t nwg); + int32_t nwg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce( ggml_metal_library_t lib, diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 05ea7470e..f0b779979 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -345,6 +345,18 @@ typedef struct { bool inplace; } ggml_metal_kargs_rope; +typedef struct { + int32_t ne0; + int32_t ne1; + int32_t ne2; + int32_t ne3; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; + int32_t nblocks; +} ggml_metal_kargs_flash_attn_ext_kv_f16; + typedef struct { int32_t ne11; int32_t ne_12_2; // assume K and V are same shape diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index d8435e957..8311544b3 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2801,6 +2801,51 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) { return (ne01 < 20) && (ne00 % 32 == 0); } +// ref: https://github.com/ggml-org/llama.cpp/pull/27390 +// dequantize the quantized KV cache to F16 before running the F16 flash attention kernels +static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + // depending on compute/bandwidth ratio, dequant to f16 kv is not always beneficial + // ref: https://github.com/ggml-org/llama.cpp/pull/27390#issuecomment-5355152767 + // TODO: tune per device + if (op->src[0]->ne[1] < 32) { + return false; + } + + switch (op->src[1]->type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return true; + default: + return false; + } +} + +// in some models (e.g. MLA-based), V is a view of K (the first ne20 elements of each K row); +// the dequantized V is then a view of the dequantized K and does not need its own dequant or scratch +// - ref: https://github.com/ggml-org/llama.cpp/pull/13435 +static bool ggml_metal_op_flash_attn_ext_v_is_view_of_k(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + const ggml_tensor * K = op->src[1]; + const ggml_tensor * V = op->src[2]; + + return V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)); +} + +// size of the F16 dequantized K tensor; the dequantized V tensor follows it in the same scratch buffer +static size_t ggml_metal_op_flash_attn_ext_kv_f16_k_size(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + + return GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*ne11*ne12*ne13, 16); +} + size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); @@ -2816,6 +2861,18 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { size_t res = 0; const bool has_mask = op->src[3] != nullptr; + const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op); + + // when the KV is dequantized to F16, the pad kernel copies the tail chunk from the F16 scratch buffer + // note: when V is a view of K, the dequantized V is read from the dequantized K with K's row stride + const bool v_is_view_of_k = use_kv_f16 && ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + uint64_t nb11_pad = nb11; + uint64_t nb21_pad = nb21; + + if (use_kv_f16) { + nb11_pad = sizeof(ggml_fp16_t)*ne10; + nb21_pad = sizeof(ggml_fp16_t)*(v_is_view_of_k ? ne10 : ne20); + } // note: the non-vec kernel requires more extra memory, so always reserve for it GGML_ASSERT(OP_FLASH_ATTN_EXT_NCPSG >= OP_FLASH_ATTN_EXT_VEC_NCPSG); @@ -2828,8 +2885,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { if (has_kvpad) { res += OP_FLASH_ATTN_EXT_VEC_NCPSG*( - nb11*ne12*ne13 + - nb21*ne22*ne23 + + nb11_pad*ne12*ne13 + + nb21_pad*ne22*ne23 + (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0)); } } else { @@ -2838,8 +2895,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { if (has_kvpad) { res += OP_FLASH_ATTN_EXT_NCPSG*( - nb11*ne12*ne13 + - nb21*ne22*ne23 + + nb11_pad*ne12*ne13 + + nb21_pad*ne22*ne23 + (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0)); } } @@ -2915,6 +2972,29 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) { return res; } +size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + // note: always reserve the temp buffer to avoid graph reallocations + //if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { + // return 0; + //} + + GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); + + const size_t k_size = ggml_metal_op_flash_attn_ext_kv_f16_k_size(op); + + // when V is a view of K, the dequantized V is a view of the dequantized K + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + if (v_is_view_of_k) { + return k_size; + } + + const size_t v_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne20*ne21*ne22*ne23, 16); + + return k_size + v_size; +} + int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -2989,6 +3069,111 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_buffer_id bid_tmp = bid_blk; bid_tmp.offs += ggml_metal_op_flash_attn_ext_extra_blk(op); + ggml_metal_buffer_id bid_kv_f16 = bid_tmp; + bid_kv_f16.offs += ggml_metal_op_flash_attn_ext_extra_tmp(op); + + const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op); + + ggml_metal_buffer_id bid_k = bid_src1; + ggml_metal_buffer_id bid_v = bid_src2; + + uint64_t nb10_attn = nb10; + uint64_t nb11_attn = nb11; + uint64_t nb12_attn = nb12; + uint64_t nb13_attn = nb13; + uint64_t nb20_attn = nb20; + uint64_t nb21_attn = nb21; + uint64_t nb22_attn = nb22; + uint64_t nb23_attn = nb23; + + if (use_kv_f16) { + assert(ggml_metal_op_flash_attn_ext_extra_kv_f16(op) != 0); + + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + + const int64_t nblocks1_64 = (ne10/ggml_blck_size(op->src[1]->type))*(int64_t) ne11*ne12*ne13; + GGML_ASSERT(nblocks1_64 <= INT32_MAX); + const int32_t nblocks1 = nblocks1_64; + + ggml_metal_buffer_id bid_v_f16 = bid_kv_f16; + bid_v_f16.offs += ggml_metal_op_flash_attn_ext_kv_f16_k_size(op); + + auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(lib, op); + const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 256); + + // K + ggml_metal_kargs_flash_attn_ext_kv_f16 args_k = { + /*.ne0 =*/ ne10, + /*.ne1 =*/ ne11, + /*.ne2 =*/ ne12, + /*.ne3 =*/ ne13, + /*.nb0 =*/ nb10, + /*.nb1 =*/ nb11, + /*.nb2 =*/ nb12, + /*.nb3 =*/ nb13, + /*.nblocks =*/ nblocks1, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args_k, sizeof(args_k), 0); + ggml_metal_encoder_set_buffer (enc, bid_src1, 1); + ggml_metal_encoder_set_buffer (enc, bid_kv_f16, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks1 + nth - 1)/nth, 1, 1, nth, 1, 1); + + // V (skip when V is a view of K: the dequantized V is a view of the dequantized K) + if (!v_is_view_of_k) { + const int64_t nblocks2_64 = (ne20/ggml_blck_size(op->src[2]->type))*(int64_t) ne21*ne22*ne23; + GGML_ASSERT(nblocks2_64 <= INT32_MAX); + const int32_t nblocks2 = nblocks2_64; + + ggml_metal_kargs_flash_attn_ext_kv_f16 args_v = { + /*.ne0 =*/ ne20, + /*.ne1 =*/ ne21, + /*.ne2 =*/ ne22, + /*.ne3 =*/ ne23, + /*.nb0 =*/ nb20, + /*.nb1 =*/ nb21, + /*.nb2 =*/ nb22, + /*.nb3 =*/ nb23, + /*.nblocks =*/ nblocks2, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args_v, sizeof(args_v), 0); + ggml_metal_encoder_set_buffer (enc, bid_src2, 1); + ggml_metal_encoder_set_buffer (enc, bid_v_f16, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks2 + nth - 1)/nth, 1, 1, nth, 1, 1); + } + + // the pad and attention kernels read the dequantized KV + ggml_metal_op_concurrency_reset(ctx); + + bid_k = bid_kv_f16; + bid_v = v_is_view_of_k ? bid_k : bid_v_f16; + + // contiguous F16 layout of the dequantized K + nb10_attn = sizeof(ggml_fp16_t); + nb11_attn = nb10_attn*ne10; + nb12_attn = nb11_attn*ne11; + nb13_attn = nb12_attn*ne12; + + // if V is a view of K, the dequantized V is read from the dequantized K with K's strides + if (v_is_view_of_k) { + nb20_attn = nb10_attn; + nb21_attn = nb11_attn; + nb22_attn = nb12_attn; + nb23_attn = nb13_attn; + } else { + // contiguous F16 layout of the dequantized V + nb20_attn = sizeof(ggml_fp16_t); + nb21_attn = nb20_attn*ne20; + nb22_attn = nb21_attn*ne21; + nb23_attn = nb22_attn*ne22; + } + } + if (!ggml_metal_op_flash_attn_ext_use_vec(op)) { // half8x8 kernel const int nqptg = OP_FLASH_ATTN_EXT_NQPSG; // queries per threadgroup @@ -3009,12 +3194,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ne11, /*.ne_12_2 =*/ne12, /*.ne_12_3 =*/ne13, - /*.nb11 =*/nb11, - /*.nb12 =*/nb12, - /*.nb13 =*/nb13, - /*.nb21 =*/nb21, - /*.nb22 =*/nb22, - /*.nb23 =*/nb23, + /*.nb11 =*/nb11_attn, + /*.nb12 =*/nb12_attn, + /*.nb13 =*/nb13_attn, + /*.nb21 =*/nb21_attn, + /*.nb22 =*/nb22_attn, + /*.nb23 =*/nb23_attn, /*.ne31 =*/ne31, /*.ne32 =*/ne32, /*.ne33 =*/ne33, @@ -3027,8 +3212,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_pipeline(enc, pipeline0); ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); - ggml_metal_encoder_set_buffer (enc, bid_src1, 1); - ggml_metal_encoder_set_buffer (enc, bid_src2, 2); + ggml_metal_encoder_set_buffer (enc, bid_k, 1); + ggml_metal_encoder_set_buffer (enc, bid_v, 2); ggml_metal_encoder_set_buffer (enc, bid_src3, 3); ggml_metal_encoder_set_buffer (enc, bid_pad, 4); @@ -3073,7 +3258,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_op_concurrency_reset(ctx); } - const int is_q = ggml_is_quantized(op->src[1]->type) ? 1 : 0; + const int is_q = !use_kv_f16 && ggml_is_quantized(op->src[1]->type) ? 1 : 0; // 2*(2*ncpsg) // ncpsg soft_max values + ncpsg mask values @@ -3104,6 +3289,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { const size_t smem = FATTN_SMEM(nsg); + const int32_t ns10 = nb11_attn/nb10_attn; + const int32_t ns20 = nb21_attn/nb20_attn; + ggml_metal_kargs_flash_attn_ext args = { /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, @@ -3114,14 +3302,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ ne11, /*.ne_12_2 =*/ ne12, /*.ne_12_3 =*/ ne13, - /*.ns10 =*/ int32_t(nb11/nb10), - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, - /*.ns20 =*/ int32_t(nb21/nb20), - /*.nb21 =*/ nb21, - /*.nb22 =*/ nb22, - /*.nb23 =*/ nb23, + /*.ns10 =*/ ns10, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb13 =*/ nb13_attn, + /*.ns20 =*/ ns20, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + /*.nb23 =*/ nb23_attn, /*.ne31 =*/ ne31, /*.ne32 =*/ ne32, /*.ne33 =*/ ne33, @@ -3139,13 +3327,13 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.logit_softcap =*/ logit_softcap, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, use_kv_f16, ns10, ns20); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, bid_src0, 1); - ggml_metal_encoder_set_buffer (enc, bid_src1, 2); - ggml_metal_encoder_set_buffer (enc, bid_src2, 3); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_v, 3); ggml_metal_encoder_set_buffer (enc, bid_src3, 4); ggml_metal_encoder_set_buffer (enc, bid_src4, 5); ggml_metal_encoder_set_buffer (enc, bid_pad, 6); @@ -3177,12 +3365,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ne11, /*.ne_12_2 =*/ne12, /*.ne_12_3 =*/ne13, - /*.nb11 =*/nb11, - /*.nb12 =*/nb12, - /*.nb13 =*/nb13, - /*.nb21 =*/nb21, - /*.nb22 =*/nb22, - /*.nb23 =*/nb23, + /*.nb11 =*/nb11_attn, + /*.nb12 =*/nb12_attn, + /*.nb13 =*/nb13_attn, + /*.nb21 =*/nb21_attn, + /*.nb22 =*/nb22_attn, + /*.nb23 =*/nb23_attn, /*.ne31 =*/ne31, /*.ne32 =*/ne32, /*.ne33 =*/ne33, @@ -3195,8 +3383,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_pipeline(enc, pipeline0); ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); - ggml_metal_encoder_set_buffer (enc, bid_src1, 1); - ggml_metal_encoder_set_buffer (enc, bid_src2, 2); + ggml_metal_encoder_set_buffer (enc, bid_k, 1); + ggml_metal_encoder_set_buffer (enc, bid_v, 2); ggml_metal_encoder_set_buffer (enc, bid_src3, 3); ggml_metal_encoder_set_buffer (enc, bid_pad, 4); @@ -3242,6 +3430,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { } } + const int32_t ns10 = nb11_attn/nb10_attn; + const int32_t ns20 = nb21_attn/nb20_attn; + ggml_metal_kargs_flash_attn_ext_vec args = { /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, @@ -3252,14 +3443,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ ne11, /*.ne_12_2 =*/ ne12, /*.ne_12_3 =*/ ne13, - /*.ns10 =*/ int32_t(nb11/nb10), - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, - /*.ns20 =*/ int32_t(nb21/nb20), - /*.nb21 =*/ nb21, - /*.nb22 =*/ nb22, - /*.nb23 =*/ nb23, + /*.ns10 =*/ ns10, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb13 =*/ nb13_attn, + /*.ns20 =*/ ns20, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + /*.nb23 =*/ nb23_attn, /*.ne31 =*/ ne31, /*.ne32 =*/ ne32, /*.ne33 =*/ ne33, @@ -3277,15 +3468,15 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.logit_softcap =*/ logit_softcap, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg, use_kv_f16, ns10, ns20); GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, bid_src0, 1); - ggml_metal_encoder_set_buffer (enc, bid_src1, 2); - ggml_metal_encoder_set_buffer (enc, bid_src2, 3); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_v, 3); ggml_metal_encoder_set_buffer (enc, bid_src3, 4); ggml_metal_encoder_set_buffer (enc, bid_src4, 5); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index b03b59e0b..159a628d0 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -42,6 +42,7 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_pad(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op); +size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op); int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx); int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index ef3c92f27..0e8d409e0 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -225,6 +225,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_ res += ggml_metal_op_flash_attn_ext_extra_pad(tensor); res += ggml_metal_op_flash_attn_ext_extra_blk(tensor); res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor); + res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor); } break; case GGML_OP_CUMSUM: case GGML_OP_ARGSORT: diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 0537fa4cf..27f97b5e0 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -6318,6 +6318,53 @@ template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_t kernel_fwht_f template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_t kernel_fwht_f32<256>; template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_t kernel_fwht_f32<512>; +// dequantize a quantized KV cache tensor to contiguous F16 before running the F16 flash attention kernels +// - one thread per block; dispatched separately for K and V +// - ref: https://github.com/ggml-org/llama.cpp/pull/27390 +template < + typename block_t, + short QK, + void (*deq_t4x4)(device const block_t *, short, thread float4x4 &)> +kernel void kernel_flash_attn_ext_kv_f16( + constant ggml_metal_kargs_flash_attn_ext_kv_f16 & args, + device const char * x, + device half * x_dst, + uint gid [[thread_position_in_grid]]) { + if (gid >= (uint) args.nblocks) { + return; + } + + const uint nb = args.ne0/QK; + const uint i0 = gid%nb; + uint ib = gid/nb; + const uint i1 = ib%args.ne1; + ib /= args.ne1; + const uint i2 = ib%args.ne2; + const uint i3 = ib/args.ne2; + + const uint64_t offs = i0*args.nb0 + i1*args.nb1 + i2*args.nb2 + i3*args.nb3; + + device const block_t * src = (device const block_t *) (x + offs); + device half4 * dst = (device half4 *) x_dst + (QK/4)*gid; + + for (short i = 0; i < QK/16; ++i) { + float4x4 reg; + deq_t4x4(src, i, reg); + dst[4*i + 0] = (half4) reg[0]; + dst[4*i + 1] = (half4) reg[1]; + dst[4*i + 2] = (half4) reg[2]; + dst[4*i + 3] = (half4) reg[3]; + } +} + +typedef decltype(kernel_flash_attn_ext_kv_f16) kernel_flash_attn_ext_kv_f16_t; + +template [[host_name("kernel_flash_attn_ext_kv_q4_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q4_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q5_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q5_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q8_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; + constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]]; constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]]; @@ -10318,9 +10365,12 @@ kernel void kernel_mul_mm( auto tB = tensor(ptrB, dextents(K, N), array({1, strideB})); // Configure matmul operation + // note: K is dynamic_extent (clamped to the valid range in PHASE 2), since a static + // N_MM_NK_TOTAL K tile would read src1 out of bounds when K % N_MM_NK_TOTAL != 0 + // ref: https://github.com/ggml-org/llama.cpp/pull/27064 mpp::tensor_ops::matmul2d< mpp::tensor_ops::matmul2d_descriptor( - NRB, NRA, N_MM_NK_TOTAL, false, true, true, + NRB, NRA, static_cast(dynamic_extent), false, true, true, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), execution_simdgroups> mm; @@ -10372,10 +10422,14 @@ kernel void kernel_mul_mm( threadgroup_barrier(mem_flags::mem_threadgroup); // === PHASE 2: Tensor matmul === - auto mA = tA.slice(0, 0); - auto mB = tB.slice(loop_k, rb); + // Clamp the K extent of both operand tensors to the remaining valid K range so + // the dynamic-K op never reads past the K extent of src1 (or the staged A tile). + const int kExt = min(N_MM_NK_TOTAL, K - loop_k); - mm.run(mB, mA, cT); + auto tAv = tensor(sa, dextents(kExt, NRA), array({1, N_MM_NK_TOTAL})); + auto tBv = tensor(ptrB + loop_k + rb * strideB, dextents(kExt, N - rb), array({1, strideB})); + + mm.run(tBv, tAv, cT); threadgroup_barrier(mem_flags::mem_threadgroup); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp index 6c264c786..0c1b6d067 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp @@ -121,13 +121,13 @@ void main() { const uint buf_ib = r * qf_stride + d / 8; const uint buf_iqs = d % 8; - FLOAT_TYPEV4 vals = is_in_bounds ? FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale) : FLOAT_TYPEV4(0.0f); - const FLOAT_TYPEV4 abs_vals = abs(vals); + vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale : vec4(0.0f); + const vec4 abs_vals = abs(vals); - const FLOAT_TYPE thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w)); - const FLOAT_TYPE amax = subgroupClusteredMax(thread_max, 8); - const FLOAT_TYPE qd = amax / FLOAT_TYPE(127.0); - const FLOAT_TYPE qd_inv = qd != FLOAT_TYPE(0.0) ? FLOAT_TYPE(1.0) / qd : FLOAT_TYPE(0.0); + const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w)); + const float amax = subgroupClusteredMax(thread_max, 8); + const float qd = amax / 127.0f; + const float qd_inv = qd != 0.0f ? 1.0f / qd : 0.0f; vals = round(vals * qd_inv); Qf[buf_ib].qs[buf_iqs] = pack32(i8vec4(vals)); @@ -136,11 +136,11 @@ void main() { // the row-sum scaled by qd, used in k_dot_correction. if (FaTypeK == FA_TYPE_Q8_0) { if (buf_iqs == 0) { - Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0); + Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f); } } else { - const FLOAT_TYPE thread_sum = vals.x + vals.y + vals.z + vals.w; - const FLOAT_TYPE sum = subgroupClusteredAdd(thread_sum, 8); + const float thread_sum = vals.x + vals.y + vals.z + vals.w; + const float sum = subgroupClusteredAdd(thread_sum, 8); if (buf_iqs == 0) { Qf[buf_ib].ds = FLOAT_TYPEV2(qd, sum * qd); diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 955c2d796..c9b504c33 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1032,6 +1032,8 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) { case LLM_ARCH_DEEPSEEK4: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: + case LLM_ARCH_LFM2: + case LLM_ARCH_LFM2MOE: return true; default: return false; @@ -1060,8 +1062,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: case LLM_ARCH_GRANITE_HYBRID: - case LLM_ARCH_LFM2: - case LLM_ARCH_LFM2MOE: case LLM_ARCH_MINIMAX_01: case LLM_ARCH_MINIMAX_M2: case LLM_ARCH_MINIMAX_M3: diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index ba10f0aa4..48d27b60a 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -3100,8 +3100,6 @@ ggml_tensor * llm_graph_context::build_attn( int il) const { const bool is_swa = hparams.is_swa(il); - GGML_UNUSED(v_cur); - auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot; if (k_rot) { @@ -3134,7 +3132,7 @@ ggml_tensor * llm_graph_context::build_attn( // MLA-style attention: the cached K is used as V ggml_tensor * q = q_cur; ggml_tensor * k = mctx_cur->get_k(ctx0, il); - ggml_tensor * v = k; + ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0); ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il); cb(cur, "kqv_out", il); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index e38c89d03..356730df0 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -637,6 +637,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight"); } if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) { + if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) { + // the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, ""); + } return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight"); } if (std::regex_match(tensor_name, pattern_ssm_conv1d)) { diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp index ba90c0d07..e0e537e00 100644 --- a/src/models/deepseek2.cpp +++ b/src/models/deepseek2.cpp @@ -524,17 +524,9 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p q = ggml_mul_mat(ctx0, model.layers[il].wq, cur); cb(q, "q", il); } - // split into {n_embd_head_qk_nope, n_head, n_tokens} - ggml_tensor * q_nope = - ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), - ggml_row_size(q->type, n_embd_head_k) * n_head, 0); - cb(q_nope, "q_nope", il); - - // and {n_embd_head_qk_rope, n_head, n_tokens} - ggml_tensor * q_pe = ggml_view_3d( - ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), - ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope)); - cb(q_pe, "q_pe", il); + // {n_embd_head_k, n_head, n_tokens} + q = ggml_reshape_3d(ctx0, q, n_embd_head_k, n_head, n_tokens); + cb(q, "q", il); ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); cb(kv_cmpr_pe, "kv_cmpr_pe", il); @@ -552,10 +544,6 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(k_pe, "k_pe", il); @@ -564,6 +552,20 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p cb(kv_cmpr, "kv_cmpr", il); if (is_mla) { + // split into {n_embd_head_qk_nope, n_head, n_tokens} + ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, + q->nb[1], q->nb[2], 0); + cb(q_nope, "q_nope", il); + + // and {n_embd_head_qk_rope, n_head, n_tokens} + ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, + q->nb[1], q->nb[2], ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "q_pe", il); + + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "q_pe", il); + // {n_embd_head_qk_nope, n_tokens, n_head} q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); cb(q_nope, "q_nope_perm", il); @@ -623,10 +625,14 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p Vcur = ggml_cont(ctx0, Vcur); cb(Vcur, "Vcur_cont", il); - ggml_tensor * Qcur = ggml_concat(ctx0, q_nope, q_pe, 0); + // RoPE is applied to the trailing dims only + ggml_tensor * Qcur = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = ggml_rope_set_offset(Qcur, n_embd_head_qk_nope); cb(Qcur, "Qcur", il); - ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0); + ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, + ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0); cb(Kcur, "Kcur", il); if (inp_attn_scale) { diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 89cd46176..1c278e435 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -501,21 +501,10 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_compressed_kv_from_state( comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il); cb(comp, name, il); - ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks, - ggml_row_size(comp->type, n_embd_head), - ggml_row_size(comp->type, n_embd_head), - 0); - ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks, - ggml_row_size(comp->type, n_embd_head), - ggml_row_size(comp->type, n_embd_head), - ggml_row_size(comp->type, n_embd_head_nope)); - - comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig, + comp = ggml_rope_ext(ctx0, comp, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig, hparams.dsv4_compress_rope_base, freq_scale, ext_factor, dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow); - cb(comp_pe, name, il); - - comp = ggml_concat(ctx0, comp_nope, comp_pe, 0); + comp = ggml_rope_set_offset(comp, n_embd_head_nope); cb(comp, name, il); return comp; @@ -585,21 +574,10 @@ ggml_tensor * llama_model_deepseek4::graph::build_overlap_compressed_kv_from_sta comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il); cb(comp, name, il); - ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks, - ggml_row_size(comp->type, n_embd_head), - ggml_row_size(comp->type, n_embd_head), - 0); - ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks, - ggml_row_size(comp->type, n_embd_head), - ggml_row_size(comp->type, n_embd_head), - ggml_row_size(comp->type, n_embd_head_nope)); - - comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig, + comp = ggml_rope_ext(ctx0, comp, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig, hparams.dsv4_compress_rope_base, freq_scale, ext_factor, dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow); - cb(comp_pe, name, il); - - comp = ggml_concat(ctx0, comp_nope, comp_pe, 0); + comp = ggml_rope_set_offset(comp, n_embd_head_nope); cb(comp, name, il); return comp; @@ -628,21 +606,12 @@ ggml_tensor * llama_model_deepseek4::graph::build_lid_top_k( indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, nt); cb(indexer_q, "lid_q", il); - ggml_tensor * indexer_q_nope = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_nope, n_indexer_head, nt, - ggml_row_size(indexer_q->type, n_embd_indexer_head), - ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head, - 0); - ggml_tensor * indexer_q_pe = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_rope, n_indexer_head, nt, - ggml_row_size(indexer_q->type, n_embd_indexer_head), - ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head, - ggml_row_size(indexer_q->type, n_embd_indexer_head_nope)); - - indexer_q_pe = ggml_rope_ext(ctx0, indexer_q_pe, inp_pos, nullptr, n_embd_indexer_head_rope, + indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_embd_indexer_head_rope, rope_type, n_ctx_orig, hparams.dsv4_compress_rope_base, freq_scale, ext_factor, dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow); - cb(indexer_q_pe, "lid_q_pe", il); + indexer_q = ggml_rope_set_offset(indexer_q, n_embd_indexer_head_nope); + cb(indexer_q, "lid_q_rope", il); - indexer_q = ggml_concat(ctx0, indexer_q_nope, indexer_q_pe, 0); indexer_q = llama_mul_mat_hadamard(ctx0, indexer_q, inp_lid.k_rot); cb(indexer_q, "lid_q_rot", il); @@ -945,18 +914,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( q = ggml_rms_norm(ctx0, q, norm_rms_eps); cb(q, "q_norm", il); - ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_nope, n_head, nt, - ggml_row_size(q->type, n_embd_head), - ggml_row_size(q->type, n_embd_head)*n_head, - 0); - ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_rope, n_head, nt, - ggml_row_size(q->type, n_embd_head), - ggml_row_size(q->type, n_embd_head)*n_head, - ggml_row_size(q->type, n_embd_head_nope)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, + q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); - cb(q_pe, "q_pe", il); - q = ggml_concat(ctx0, q_nope, q_pe, 0); + q = ggml_rope_set_offset(q, n_embd_head_nope); cb(q, "q", il); ggml_tensor * kv = build_lora_mm(layer.wkv, cur); @@ -964,18 +924,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, nt); cb(kv, "kv_norm", il); - ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, nt, - ggml_row_size(kv->type, n_embd_head), - ggml_row_size(kv->type, n_embd_head), - 0); - ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, nt, - ggml_row_size(kv->type, n_embd_head), - ggml_row_size(kv->type, n_embd_head), - ggml_row_size(kv->type, n_embd_head_nope)); - kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, + kv = ggml_rope_ext(ctx0, kv, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); - cb(kv_pe, "kv_pe", il); - kv = ggml_concat(ctx0, kv_nope, kv_pe, 0); + kv = ggml_rope_set_offset(kv, n_embd_head_nope); cb(kv, "kv", il); const int64_t ratio = hparams.dsv4_compress_ratios[il]; @@ -1225,7 +1176,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( if (inp_mtp) { out = build_attn(inp_mtp, nullptr, nullptr, nullptr, - q, kv, nullptr, + q, kv, kv, nullptr, layer.attn_sinks, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); cb(out, "attn_raw", il); @@ -1245,17 +1196,9 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( } out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, nt); - ggml_tensor * out_nope = ggml_view_3d(ctx0, out, n_embd_head_nope, n_head, nt, - ggml_row_size(out->type, n_embd_head), - ggml_row_size(out->type, n_embd_head)*n_head, - 0); - ggml_tensor * out_pe = ggml_view_3d(ctx0, out, n_embd_head_rope, n_head, nt, - ggml_row_size(out->type, n_embd_head), - ggml_row_size(out->type, n_embd_head)*n_head, - ggml_row_size(out->type, n_embd_head_nope)); - out_pe = ggml_rope_ext_back(ctx0, out_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, + out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); - out = ggml_concat(ctx0, out_nope, out_pe, 0); + out = ggml_rope_set_offset(out, n_embd_head_nope); cb(out, "attn_derope", il); out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, nt); diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 5b70a5179..d3c919b35 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -591,17 +591,9 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il); kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens); - ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, n_tokens, - ggml_row_size(kv->type, n_embd_head), - ggml_row_size(kv->type, n_embd_head), - 0); - ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, n_tokens, - ggml_row_size(kv->type, n_embd_head), - ggml_row_size(kv->type, n_embd_head), - ggml_row_size(kv->type, n_embd_head_nope)); - kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, 0, + kv = ggml_rope_ext(ctx0, kv, inp_pos, nullptr, n_embd_head_rope, rope_type, 0, freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); - kv = ggml_concat(ctx0, kv_nope, kv_pe, 0); + kv = ggml_rope_set_offset(kv, n_embd_head_nope); cb(kv, "kv_injected", il); if (inp_attn->self_k_rot_swa) { diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp index 70e837d6e..9a4295557 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp @@ -2,6 +2,8 @@ #include "../llama-memory-hybrid-iswa.h" #include "../llama-memory-hybrid.h" +#include + void llama_model_lfm2::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -202,15 +204,20 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ } GGML_ASSERT(bx->ne[0] > conv->ne[0]); - // last d_conv columns is a new conv state - auto * new_conv = ggml_view_3d(ctx0, bx, conv->ne[0], bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2], - (bx->ne[0] - conv->ne[0]) * ggml_element_size(bx)); - GGML_ASSERT(ggml_are_same_shape(conv, new_conv)); + // write conv states: slot 0 = the final state, slot s = the state s tokens back (partial rollback) + const int64_t K = hparams.causal_attn && cparams.n_rs_seq > 0 ? (int64_t) cparams.n_rs_seq + 1 : 1; + const int64_t n_written = std::min(n_seq_tokens, K); + const auto mem_size = mctx_cur->get_size(); + const size_t row_size = ggml_row_size(conv_state->type, (int64_t) d_conv * n_embd); - // write new conv conv state - ggml_build_forward_expand(gf, ggml_cpy(ctx0, new_conv, - ggml_view_1d(ctx0, conv_state, ggml_nelements(new_conv), - kv_head * d_conv * n_embd * ggml_element_size(new_conv)))); + for (int64_t slot = 0; slot < n_written; ++slot) { + auto * conv_snap = ggml_view_3d(ctx0, bx, d_conv, bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2], + (bx->ne[0] - d_conv - slot) * ggml_element_size(bx)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap, + ggml_view_2d(ctx0, conv_state, (int64_t) d_conv * n_embd, n_seqs, + conv_state->nb[1], + ((size_t) slot * mem_size + kv_head) * row_size))); + } auto * conv_kernel = model.layers[il].shortconv.conv; auto * conv_out = ggml_ssm_conv(ctx0, bx, conv_kernel); @@ -242,6 +249,8 @@ llama_model_lfm2::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = cur; + const bool is_moe_layer = il >= static_cast(hparams.n_layer_dense_lead); auto * prev_cur = cur; diff --git a/src/models/minicpm3.cpp b/src/models/minicpm3.cpp index e011b1ff0..7820d5224 100644 --- a/src/models/minicpm3.cpp +++ b/src/models/minicpm3.cpp @@ -115,19 +115,9 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q); cb(q, "q", il); - // split into {n_head * n_embd_head_qk_nope, n_tokens} - ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, - ggml_row_size(q->type, hparams.n_embd_head_k()), - ggml_row_size(q->type, hparams.n_embd_head_k() * n_head), - 0); - cb(q_nope, "q_nope", il); - - // and {n_head * n_embd_head_qk_rope, n_tokens} - ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, - ggml_row_size(q->type, hparams.n_embd_head_k()), - ggml_row_size(q->type, hparams.n_embd_head_k() * n_head), - ggml_row_size(q->type, n_embd_head_qk_nope)); - cb(q_pe, "q_pe", il); + // {n_embd_head_k, n_head, n_tokens}, RoPE is applied to the trailing dims only + q = ggml_reshape_3d(ctx0, q, hparams.n_embd_head_k(), n_head, n_tokens); + cb(q, "q", il); // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens} ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); @@ -172,12 +162,13 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa v_states = ggml_cont(ctx0, v_states); cb(v_states, "v_states", il); - q_pe = ggml_rope_ext( - ctx0, q_pe, inp_pos, rope_factors, + q = ggml_rope_ext( + ctx0, q, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(q_pe, "q_pe", il); + q = ggml_rope_set_offset(q, n_embd_head_qk_nope); + cb(q, "q_rope", il); // shared RoPE key k_pe = ggml_rope_ext( @@ -187,10 +178,11 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa ); cb(k_pe, "k_pe", il); - ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0); + ggml_tensor * q_states = q; cb(q_states, "q_states", il); - ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0); + ggml_tensor * k_states = ggml_concat(ctx0, k_nope, + ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0); cb(k_states, "k_states", il); cur = build_attn(inp_attn, diff --git a/src/models/plm.cpp b/src/models/plm.cpp index 8ca325f5e..5abefd53b 100644 --- a/src/models/plm.cpp +++ b/src/models/plm.cpp @@ -81,19 +81,9 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params q = ggml_mul_mat(ctx0, model.layers[il].wq, cur); cb(q, "q", il); - // split into {n_head * n_embd_head_qk_nope, n_tokens} - ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, - ggml_row_size(q->type, hparams.n_embd_head_k()), - ggml_row_size(q->type, hparams.n_embd_head_k() * n_head), - 0); - cb(q_nope, "q_nope", il); - - // and {n_head * n_embd_head_qk_rope, n_tokens} - ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, - ggml_row_size(q->type, hparams.n_embd_head_k()), - ggml_row_size(q->type, hparams.n_embd_head_k() * n_head), - ggml_row_size(q->type, n_embd_head_qk_nope)); - cb(q_pe, "q_pe", il); + // {n_embd_head_k, n_head, n_tokens}, RoPE is applied to the trailing dims only + q = ggml_reshape_3d(ctx0, q, hparams.n_embd_head_k(), n_head, n_tokens); + cb(q, "q", il); // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens} ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); @@ -143,12 +133,13 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params 0); cb(v_states, "v_states", il); - q_pe = ggml_rope_ext( - ctx0, q_pe, inp_pos, nullptr, + q = ggml_rope_ext( + ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - cb(q_pe, "q_pe", il); + q = ggml_rope_set_offset(q, n_embd_head_qk_nope); + cb(q, "q_rope", il); // shared RoPE key k_pe = ggml_rope_ext( @@ -158,10 +149,11 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params ); cb(k_pe, "k_pe", il); - ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0); + ggml_tensor * q_states = q; cb(q_states, "q_states", il); - ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0); + ggml_tensor * k_states = ggml_concat(ctx0, k_nope, + ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0); cb(k_states, "k_states", il); cur = build_attn(inp_attn, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index c876d3342..9d634e338 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -243,14 +243,13 @@ struct clip_ctx { throw std::runtime_error("failed to initialize CPU backend"); } if (ctx_params.use_gpu) { - auto * backend_name = std::getenv("MTMD_BACKEND_DEVICE"); - if (backend_name != nullptr) { - backend = ggml_backend_init_by_name(backend_name, nullptr); + if (ctx_params.device != nullptr) { + backend = ggml_backend_dev_init(ctx_params.device, nullptr); if (!backend) { - LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name); + throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n", + __func__, ggml_backend_dev_name(ctx_params.device))); } - } - if (!backend) { + } else { backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr); } diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index a5b713775..e07f25815 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -48,6 +48,7 @@ enum clip_flash_attn_type { struct clip_context_params { bool use_gpu; + ggml_backend_dev_t device; enum clip_flash_attn_type flash_attn_type; int image_min_tokens; int image_max_tokens; diff --git a/tools/mtmd/debug/mtmd-debug.cpp b/tools/mtmd/debug/mtmd-debug.cpp index b88a16f0f..2719dae9b 100644 --- a/tools/mtmd/debug/mtmd-debug.cpp +++ b/tools/mtmd/debug/mtmd-debug.cpp @@ -84,6 +84,7 @@ int main(int argc, char ** argv) { const char * clip_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; + mparams.device = params.mmproj_device; mparams.print_timings = true; mparams.n_threads = params.cpuparams.n_threads; mparams.flash_attn_type = params.flash_attn_type; diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 07b45b644..f6c787fdb 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -154,6 +154,7 @@ struct mtmd_cli_context { const char * clip_path = params.mmproj.path.c_str(); mtmd_context_params mparams = mtmd_context_params_default(); mparams.use_gpu = params.mmproj_use_gpu; + mparams.device = params.mmproj_device; mparams.print_timings = true; mparams.n_threads = params.cpuparams.n_threads; mparams.flash_attn_type = params.flash_attn_type; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 0f5cb8c7a..95f17f7af 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -456,6 +456,7 @@ static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_ mtmd_context_params mtmd_context_params_default() { mtmd_context_params params { /* use_gpu */ true, + /* device */ nullptr, /* print_timings */ true, /* n_threads */ 4, /* image_marker */ nullptr, @@ -564,6 +565,7 @@ struct mtmd_context { clip_context_params ctx_clip_params { /* use_gpu */ ctx_params.use_gpu, + /* device */ ctx_params.device, /* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type), /* image_min_tokens */ ctx_params.image_min_tokens, /* image_max_tokens */ ctx_params.image_max_tokens, diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index ef4f99c0b..ef88efd31 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -89,6 +89,7 @@ typedef bool (*mtmd_progress_callback)(float progress, void * user_data); struct mtmd_context_params { bool use_gpu; + ggml_backend_dev_t device; bool print_timings; int n_threads; const char * image_marker; // deprecated, use media_marker instead diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 21ff78394..1293c8640 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -998,6 +998,7 @@ private: mtmd_context_params mparams = mtmd_context_params_default(); if (has_mmproj) { mparams.use_gpu = params_base.mmproj_use_gpu; + mparams.device = params_base.mmproj_device; mparams.print_timings = false; mparams.n_threads = params_base.cpuparams.n_threads; mparams.flash_attn_type = params_base.flash_attn_type; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 35b935570..d60545194 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -672,24 +672,26 @@ void server_models::load_models() { apply_hidden(); log_available_models(); - std::vector models_to_load; - for (const auto & [name, inst] : mapping) { - std::string val; - if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) { - models_to_load.push_back(name); + // skipped on reload, see startup_models + if (startup_models.has_value()) { + std::vector models_to_load; + for (const auto & [name, inst] : mapping) { + std::string val; + if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) { + models_to_load.push_back(name); + } } - } - if ((int)models_to_load.size() > base_params.models_max) { - throw std::runtime_error(string_format( - "number of models to load on startup (%zu) exceeds models_max (%d)", - models_to_load.size(), base_params.models_max)); + if ((int)models_to_load.size() > base_params.models_max) { + throw std::runtime_error(string_format( + "number of models to load on startup (%zu) exceeds models_max (%d)", + models_to_load.size(), base_params.models_max)); + } + + // to be lazy-loaded after main() setup phase is completed + startup_models = std::move(models_to_load); } lk.unlock(); - for (const auto & name : models_to_load) { - SRV_INF("(startup) loading model %s\n", name.c_str()); - load(name); - } } else { // RELOAD: diff the new preset list against the current mapping and reconcile is_reloading = true; @@ -819,8 +821,8 @@ void server_models::load_models() { inst.meta.update_caps(); } - // add models that are new in this reload - std::vector newly_added; + // add models that are new in this reload, load-on-startup is not honored here since a + // reload never spawns an instance for (const auto & [name, preset] : final_presets) { if (mapping.find(name) == mapping.end()) { server_model_meta meta{ @@ -841,42 +843,40 @@ void server_models::load_models() { // /* need_download */ false, }; add_model(std::move(meta)); - newly_added.push_back(name); } } apply_stop_timeout(); apply_hidden(); - // clear reload flag before unlocking for autoload - load() blocks on !is_reloading, - // so clearing it here (while still locked) prevents a deadlock in the autoload calls below + // clear reload flag under the lock, this releases the load() calls waiting on !is_reloading is_reloading = false; cv.notify_all(); log_available_models(); - // collect autoload candidates while still under the lock - std::vector to_autoload; - for (const auto & name : newly_added) { - auto it = mapping.find(name); - if (it != mapping.end()) { - std::string val; - if (it->second.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) { - to_autoload.push_back(name); - } - } - } - lk.unlock(); - for (const auto & name : to_autoload) { - SRV_INF("(reload) loading new model %s\n", name.c_str()); - load(name); - } notify_sse("models_reload", "*"); } } +void server_models::load_startup_models() { + std::vector to_load; + { + std::lock_guard lk(mutex); + if (!startup_models.has_value()) { + return; // already drained + } + to_load = std::move(*startup_models); + startup_models.reset(); + } + for (const auto & name : to_load) { + SRV_INF("(startup) loading model %s\n", name.c_str()); + load(name); + } +} + void server_models::update_meta(const std::string & name, const server_model_meta & meta) { std::lock_guard lk(mutex); auto it = mapping.find(name); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 79b231cba..5cbb6a801 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -136,6 +136,10 @@ private: // if true, the next get_meta() will trigger a reload of model list bool need_reload = false; + // models marked with load-on-startup, unset once load_startup_models() drains it + // no value means the startup phase is over, so a reload must not queue anything + std::optional> startup_models{std::in_place}; + // conv_id -> model name that currently serves its stream session, lets the resumable stream // routes go straight to the owning child instead of polling every one. populated when // proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just @@ -231,6 +235,9 @@ public: // - if a model is not running, it will be added or updated according to the source void load_models(); + // lazy-load startup_models, to be called after main() setup phase + void load_startup_models(); + // check if a model instance exists (thread-safe) bool has_model(const std::string & name); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 01cc6633a..5fe2729ba 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -133,7 +133,8 @@ int llama_server(common_params & params, int argc, char ** argv) { // router server never loads a model and must not touch the GPU const bool is_router_server = params.model.path.empty() - && params.model.hf_repo.empty(); + && params.model.hf_repo.empty() + && params.model.docker_repo.empty(); // skip device enumeration so the CUDA primary context stays uncreated common_params_print_info(params, !is_router_server); @@ -423,6 +424,18 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.stop(); }; + try { + models_routes->models.load_startup_models(); + } catch (const std::exception & e) { + SRV_ERR("failed to load models on startup: %s\n", e.what()); + ctx_http.stop(); + if (ctx_http.thread.joinable()) { + ctx_http.thread.join(); + } + clean_up(); + return 1; + } + } else { // setup clean up function, to be called before exit clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() { diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index fd7522f8d..368123baf 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -86,6 +86,7 @@ int main(int argc, char ** argv) { mtmd_context_params mtmd_params = mtmd_context_params_default(); mtmd_params.use_gpu = params.mmproj_use_gpu; + mtmd_params.device = params.mmproj_device; mtmd::context_ptr mctx(mtmd_init_from_file(params.mmproj.path.c_str(), model, mtmd_params)); if (!mctx) { LOG_ERR("failed to load mmproj %s\n", params.mmproj.path.c_str()); diff --git a/tools/ui/README.md b/tools/ui/README.md index 53b5925e2..99abfaa41 100644 --- a/tools/ui/README.md +++ b/tools/ui/README.md @@ -239,31 +239,44 @@ Routes → Components → Hooks → Stores → Services → Storage/API ### High-Level Architecture -See: [`docs/architecture/high-level-architecture-simplified.md`](docs/architecture/high-level-architecture-simplified.md) - ```mermaid flowchart TB subgraph Routes["📍 Routes"] R1["/ (Welcome)"] R2["/chat/[id]"] + R3["/mcp-servers"] + R4["/search"] + R5["/settings"] RL["+layout.svelte"] end subgraph Components["🧩 Components"] - C_Sidebar["ChatSidebar"] C_Screen["ChatScreen"] C_Form["ChatForm"] C_Messages["ChatMessages"] - C_ModelsSelector["ModelsSelector"] + C_Sidebar["ChatSidebar"] + C_Models["ModelsSelector"] C_Settings["ChatSettings"] + C_Mcp["McpServers"] + end + + subgraph Hooks["🔌 Hooks"] + H1["use-chat-screen-active-model"] + H2["use-processing-state"] + H3["use-context-gauge"] + H4["use-models-selector"] + H5["use-tools-panel"] end subgraph Stores["🗄️ Stores"] S1["chatStore"] S2["conversationsStore"] S3["modelsStore"] - S4["serverStore"] - S5["settingsStore"] + S4["mcpStore"] + S5["agenticStore"] + S6["serverStore"] + S7["settingsStore"] + S8["toolsStore"] end subgraph Services["⚙️ Services"] @@ -271,6 +284,9 @@ flowchart TB SV2["ModelsService"] SV3["PropsService"] SV4["DatabaseService"] + SV5["MCPService"] + SV6["ToolsService"] + SV7["SandboxService"] end subgraph Storage["💾 Storage"] @@ -282,19 +298,28 @@ flowchart TB API1["/v1/chat/completions"] API2["/props"] API3["/models/*"] + API4["/tools"] end R1 & R2 --> C_Screen RL --> C_Sidebar C_Screen --> C_Form & C_Messages & C_Settings - C_Screen --> S1 & S2 - C_ModelsSelector --> S3 & S4 + C_Screen --> H1 & H2 & H3 + C_Models --> H4 + C_Mcp --> S4 + C_Screen --> S1 & S2 & S3 + C_Models --> S3 + H1 --> S3 S1 --> SV1 & SV4 + S2 --> SV4 S3 --> SV2 & SV3 + S4 --> SV5 + S5 --> SV1 & SV5 & SV6 & SV7 SV4 --> ST1 SV1 --> API1 SV2 --> API3 SV3 --> API2 + SV6 --> API4 ``` ### Layer Breakdown @@ -303,6 +328,9 @@ flowchart TB - **`/`** - Welcome screen, creates new conversation - **`/chat/[id]`** - Active chat interface +- **`/mcp-servers`** - MCP server management +- **`/search`** - Conversation search +- **`/settings`** - Settings (optional `[[section]]`) - **`+layout.svelte`** - Sidebar, navigation, global initialization #### Components (`src/lib/components/`) @@ -348,28 +376,68 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel #### Hooks (`src/lib/hooks/`) -- **`useModelChangeValidation`** - Validates model switch against conversation modalities -- **`useProcessingState`** - Tracks streaming progress and token generation +Hooks are the thin view-layer between components and stores: they own UI concerns (scroll, drag-and-drop, keyboard shortcuts, pickers, selection) and translate store state into view state. + +| Hook | Responsibility | +| ------------------------------- | -------------------------------------------------------------- | +| `use-chat-screen-active-model` | Active model resolution + modality capability detection | +| `use-processing-state` | View over `chatStore.processing` for streaming progress/tokens | +| `use-context-gauge` | View over `contextStatsStore` for the context usage gauge | +| `use-models-selector` | Model selector dropdown state (loaded/available groups) | +| `use-tools-panel` | Tools panel state | +| `use-reasoning-menu` | Reasoning-effort menu state | +| `use-attachment-menu` | Attachment menu + modality flags | +| `use-draft-messages` | Per-chat draft message/files persistence | +| `use-chat-form-pickers` | Chat form pickers (commands, mentions) | +| `use-debounced-search` | Shared debounced async search for pickers | +| `use-picker-navigation` | Picker keyboard navigation | +| `use-chat-message-edit-context` | Message edit context (content + extras) | +| `use-chat-screen-drag-and-drop` | Drag-and-drop state machine | +| `use-chat-screen-file-upload` | File upload queue + capability validation | +| `use-chat-screen-scroll` | Scroll container binding + navigation guard | +| `use-auto-scroll` | Auto-scroll controller for streaming | +| `use-marquee-selection` | Shift+click / marquee range selection | +| `use-keyboard-shortcuts` | Global keyboard shortcuts | +| `use-settings-navigation` | Settings section navigation | +| `use-pwa` | PWA install/update + version mismatch detection | #### Stores (`src/lib/stores/`) -| Store | Responsibility | -| -------------------- | --------------------------------------------------------- | -| `chatStore` | Message sending, streaming, abort control, error handling | -| `conversationsStore` | CRUD for conversations, message branching, navigation | -| `modelsStore` | Model list, selection, loading/unloading (ROUTER) | -| `serverStore` | Server properties, role detection, modalities | -| `settingsStore` | User preferences, parameter sync with server defaults | +Stores own reactive application state as Svelte 5 runes. Larger stores are split into directories and compose focused sub-stores behind a narrow host interface (see Architectural Patterns). + +| Store | Responsibility | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | +| `chatStore` | Chat lifecycle, streaming, abort control, error handling; composes `processing`, `activity`, `streams`, `flows` | +| `conversationsStore` | Conversation CRUD, message branching, navigation, import/export; composes `preferences` | +| `modelsStore` | Model list, selection, loading/unloading (ROUTER); composes `props`, `status` | +| `mcpStore` | MCP host role: multi-server lifecycle, tool routing; composes `health`, `resources` | +| `agenticStore` | Multi-turn agentic loop orchestration, tool execution; composes `gates` | +| `serverStore` | Server connection state, `/props`, role detection, modalities | +| `settingsStore` | User preferences, theme, parameter sync with server defaults | +| `toolsStore` | Tool registry: server + MCP tools, enabled set for the LLM | +| `permissionsStore` | Persisted tool permission grants | +| `contextStatsStore` | Context window usage for the active conversation | +| `draftMessagesStore` | Per-chat draft message/files | +| `deviceStore` | Browser environment signals (mobile, OS, theme) | +| `versionStore` | Build version information | #### Services (`src/lib/services/`) -| Service | Responsibility | -| ---------------------- | ----------------------------------------------- | -| `ChatService` | API calls to`/v1/chat/completions`, SSE parsing | -| `ModelsService` | `/models`, `/models/load`, `/models/unload` | -| `PropsService` | `/props`, `/props?model=` | -| `DatabaseService` | IndexedDB operations via Dexie | -| `ParameterSyncService` | Syncs settings with server defaults | +Services are a stateless protocol layer: static methods, pure I/O, no reactive state. Stores consume them for all API and storage access. + +| Service | Responsibility | +| ----------------------------- | ------------------------------------------------------------------------- | +| `ChatService` | `/v1/chat/completions` streaming + SSE parsing, message format conversion | +| `ModelsService` | `/models`, `/models/load`, `/models/unload` | +| `PropsService` | `/props`, `/props?model=` | +| `DatabaseService` | IndexedDB operations via Dexie | +| `MCPService` | MCP protocol: transports, connect, list/execute tools, prompts, resources | +| `ToolsService` | Server tool list/execute/stream (`/tools`) | +| `SandboxService` | Browser JS execution in a sandboxed worker | +| `ParameterSyncService` | Syncs settings with server defaults | +| `ConversationTransferService` | Conversation import/export JSONL + ZIP format | +| `MigrationService` | Non-destructive localStorage/IndexedDB migrations | +| `RouterService` | Dynamic route URL construction | --- @@ -377,8 +445,6 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel ### MODEL Mode (Single Model) -See: [`docs/flows/data-flow-simplified-model-mode.md`](docs/flows/data-flow-simplified-model-mode.md) - ```mermaid sequenceDiagram participant User @@ -388,8 +454,9 @@ sequenceDiagram participant API as llama-server Note over User,API: Initialization - UI->>Stores: initialize() - Stores->>DB: load conversations + UI->>Stores: initStores() (awaited by route loads) + Stores->>Stores: run migrations + Stores->>DB: load conversations (background) Stores->>API: GET /props API-->>Stores: server config Stores->>API: GET /v1/models @@ -408,8 +475,6 @@ sequenceDiagram ### ROUTER Mode (Multi-Model) -See: [`docs/flows/data-flow-simplified-router-mode.md`](docs/flows/data-flow-simplified-router-mode.md) - ```mermaid sequenceDiagram participant User @@ -441,17 +506,6 @@ sequenceDiagram end ``` -### Detailed Flow Diagrams - -| Flow | Description | File | -| ------------- | ------------------------------------------ | ----------------------------------------------------------- | -| Chat | Message lifecycle, streaming, regeneration | [`chat-flow.md`](docs/flows/chat-flow.md) | -| Models | Loading, unloading, modality caching | [`models-flow.md`](docs/flows/models-flow.md) | -| Server | Props fetching, role detection | [`server-flow.md`](docs/flows/server-flow.md) | -| Conversations | CRUD, branching, import/export | [`conversations-flow.md`](docs/flows/conversations-flow.md) | -| Database | IndexedDB schema, operations | [`database-flow.md`](docs/flows/database-flow.md) | -| Settings | Parameter sync, user overrides | [`settings-flow.md`](docs/flows/settings-flow.md) | - --- ## Architectural Patterns @@ -505,13 +559,14 @@ Components dispatch actions to stores, stores coordinate with services for I/O, ### 3. Per-Conversation State -Enables concurrent streaming across multiple conversations: +Enables concurrent streaming across multiple conversations. Loading is tracked +per conversation by the activity ledger (`chatStore.activity`), while streaming +state and abort controllers live in per-conversation maps: ```typescript class ChatStore { - chatLoadingStates = new Map(); - chatStreamingStates = new Map(); - abortControllers = new Map(); + chatStreamingStates = new SvelteMap(); + abortControllers = new SvelteMap(); } ``` @@ -567,20 +622,14 @@ get isRouterMode() { ### 7. Modality Validation -Prevents sending attachments to incompatible models: +Prevents sending attachments to incompatible models. The +`use-chat-screen-active-model` hook derives the active model's capabilities +from `modelsStore.props`: ```typescript -// useModelChangeValidation hook -const validate = (modelId: string) => { - const modelModalities = modelsStore.getModelModalities(modelId); - const conversationModalities = conversationsStore.usedModalities; - - // Check if model supports all used modalities - if (conversationModalities.hasImages && !modelModalities.vision) { - return { valid: false, reason: 'Model does not support images' }; - } - // ... -}; +// use-chat-screen-active-model hook +const hasVisionModality = $derived.by(() => modelsStore.props.modelSupportsVision(activeModelId)); +const hasAudioModality = $derived.by(() => modelsStore.props.modelSupportsAudio(activeModelId)); ``` ### 8. Persistent Storage Strategy @@ -673,9 +722,6 @@ tools/ui/ │ └── styles/ # Global styles ├── static/ # Static assets ├── tests/ # Test files -├── docs/ # Architecture diagrams -│ ├── architecture/ # High-level architecture -│ └── flows/ # Feature-specific flows └── .storybook/ # Storybook configuration ``` diff --git a/tools/ui/docs/architecture/high-level-architecture-simplified.md b/tools/ui/docs/architecture/high-level-architecture-simplified.md deleted file mode 100644 index 500f477c9..000000000 --- a/tools/ui/docs/architecture/high-level-architecture-simplified.md +++ /dev/null @@ -1,145 +0,0 @@ -```mermaid -flowchart TB - subgraph Routes["📍 Routes"] - R1["/ (Welcome)"] - R2["/chat/[id]"] - RL["+layout.svelte"] - end - - subgraph Components["🧩 Components"] - C_Sidebar["ChatSidebar"] - C_Screen["ChatScreen"] - C_Form["ChatForm"] - C_Messages["ChatMessages"] - C_Message["ChatMessage"] - C_ChatMessageAgenticContent["ChatMessageAgenticContent"] - C_MessageEditForm["ChatMessageEditForm"] - C_ModelsSelector["ModelsSelector"] - C_Settings["ChatSettings"] - C_McpSettings["McpServersSettings"] - C_McpResourceBrowser["McpResourceBrowser"] - C_McpServersSelector["McpServersSelector"] - end - - subgraph Hooks["🪝 Hooks"] - H1["useModelChangeValidation"] - H2["useProcessingState"] - end - - subgraph Stores["🗄️ Stores"] - S1["chatStore
Chat interactions & streaming"] - SA["agenticStore
Multi-turn agentic loop orchestration"] - S2["conversationsStore
Conversation data, messages & MCP overrides"] - S3["modelsStore
Model selection & loading"] - S4["serverStore
Server props & role detection"] - S5["settingsStore
User configuration incl. MCP"] - S6["mcpStore
MCP servers, tools, prompts"] - S7["mcpResourceStore
MCP resources & attachments"] - end - - subgraph Services["⚙️ Services"] - SV1["ChatService"] - SV2["ModelsService"] - SV3["PropsService"] - SV4["DatabaseService"] - SV5["ParameterSyncService"] - SV6["MCPService
protocol operations"] - end - - subgraph Storage["💾 Storage"] - ST1["IndexedDB
conversations, messages"] - ST2["LocalStorage
config, userOverrides, mcpServers"] - end - - subgraph APIs["🌐 llama-server API"] - API1["/v1/chat/completions"] - API2["/props"] - API3["/models/*"] - API4["/v1/models"] - end - - subgraph ExternalMCP["🔌 External MCP Servers"] - EXT1["MCP Server 1
WebSocket/HTTP/SSE"] - EXT2["MCP Server N"] - end - - %% Routes → Components - R1 & R2 --> C_Screen - RL --> C_Sidebar - - %% Layout runs MCP health checks - RL --> S6 - - %% Component hierarchy - C_Screen --> C_Form & C_Messages & C_Settings - C_Messages --> C_Message - C_Message --> C_ChatMessageAgenticContent - C_Message --> C_MessageEditForm - C_Form & C_MessageEditForm --> C_ModelsSelector - C_Form --> C_McpServersSelector - C_Settings --> C_McpSettings - C_McpSettings --> C_McpResourceBrowser - - %% Components → Hooks → Stores - C_Form & C_Messages --> H1 & H2 - H1 --> S3 & S4 - H2 --> S1 & S5 - - %% Components → Stores - C_Screen --> S1 & S2 - C_Sidebar --> S2 - C_ModelsSelector --> S3 & S4 - C_Settings --> S5 - C_McpSettings --> S6 - C_McpResourceBrowser --> S6 & S7 - C_McpServersSelector --> S6 - C_Form --> S6 - - %% chatStore → agenticStore → mcpStore (agentic loop) - S1 --> SA - SA --> SV1 - SA --> S6 - - %% Stores → Services - S1 --> SV1 & SV4 - S2 --> SV4 - S3 --> SV2 & SV3 - S4 --> SV3 - S5 --> SV5 - S6 --> SV6 - S7 --> SV6 - - %% Services → Storage - SV4 --> ST1 - SV5 --> ST2 - - %% Services → APIs - SV1 --> API1 - SV2 --> API3 & API4 - SV3 --> API2 - - %% MCP → External Servers - SV6 --> EXT1 & EXT2 - - %% Styling - classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px - classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px - classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px - classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px - classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px - classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px - classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px - classDef mcpStyle fill:#e0f2f1,stroke:#00695c,stroke-width:2px - classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px - classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5 - - class R1,R2,RL routeStyle - class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_ChatMessageAgenticContent,C_MessageEditForm,C_ModelsSelector,C_Settings componentStyle - class C_McpSettings,C_McpResourceBrowser,C_McpServersSelector componentStyle - class H1,H2 hookStyle - class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle - class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle - class ST1,ST2 storageStyle - class API1,API2,API3,API4 apiStyle - class EXT1,EXT2 externalStyle -``` diff --git a/tools/ui/docs/architecture/high-level-architecture.md b/tools/ui/docs/architecture/high-level-architecture.md deleted file mode 100644 index 42ddb3f4f..000000000 --- a/tools/ui/docs/architecture/high-level-architecture.md +++ /dev/null @@ -1,373 +0,0 @@ -```mermaid -flowchart TB -subgraph Routes["📍 Routes"] -R1["/ (+page.svelte)"] -R2["/chat/[id]"] -RL["+layout.svelte"] -end - - subgraph Components["🧩 Components"] - direction TB - subgraph LayoutComponents["Layout"] - C_Sidebar["ChatSidebar"] - C_Screen["ChatScreen"] - end - subgraph ChatUIComponents["Chat UI"] - C_Form["ChatForm"] - C_Messages["ChatMessages"] - C_Message["ChatMessage"] - C_MessageUser["ChatMessageUser"] - C_MessageEditForm["ChatMessageEditForm"] - C_Attach["ChatAttachments"] - C_ModelsSelector["ModelsSelector"] - C_Settings["ChatSettings"] - end - subgraph MCPComponents["MCP UI"] - C_McpSettings["McpServersSettings"] - C_McpServerCard["McpServerCard"] - C_McpResourceBrowser["McpResourceBrowser"] - C_McpResourcePreview["McpResourcePreview"] - C_McpServersSelector["McpServersSelector"] - end - end - - subgraph Hooks["🪝 Hooks"] - H1["useModelChangeValidation"] - H2["useProcessingState"] - H3["isMobile"] - end - - subgraph Stores["🗄️ Stores"] - direction TB - subgraph S1["chatStore"] - S1State["State:
isLoading, currentResponse
errorDialogState
activeProcessingState
chatLoadingStates
chatStreamingStates
abortControllers
processingStates
activeConversationId
isStreamingActive"] - S1LoadState["Loading State:
setChatLoading()
isChatLoading()
syncLoadingStateForChat()
clearUIState()
isChatLoadingPublic()
getAllLoadingChats()
getAllStreamingChats()"] - S1ProcState["Processing State:
setActiveProcessingConversation()
getProcessingState()
clearProcessingState()
getActiveProcessingState()
updateProcessingStateFromTimings()
getCurrentProcessingStateSync()
restoreProcessingStateFromMessages()"] - S1Stream["Streaming:
streamChatCompletion()
startStreaming()
stopStreaming()
stopGeneration()
isStreaming()"] - S1Error["Error Handling:
showErrorDialog()
dismissErrorDialog()
isAbortError()"] - S1Msg["Message Operations:
addMessage()
sendMessage()
updateMessage()
deleteMessage()
getDeletionInfo()"] - S1Regen["Regeneration:
regenerateMessage()
regenerateMessageWithBranching()
continueAssistantMessage()"] - S1Edit["Editing:
editAssistantMessage()
editUserMessagePreserveResponses()
editMessageWithBranching()
clearEditMode()
isEditModeActive()
getAddFilesHandler()
setEditModeActive()"] - S1Utils["Utilities:
getApiOptions()
parseTimingData()
getOrCreateAbortController()
getConversationModel()"] - end - subgraph SA["agenticStore"] - SAState["State:
sessions (Map)
isAnyRunning"] - SASession["Session Management:
getSession()
updateSession()
clearSession()
getActiveSessions()
isRunning()
currentTurn()
totalToolCalls()
lastError()
streamingToolCall()"] - SAConfig["Configuration:
getConfig()
maxTurns, maxToolPreviewLines"] - SAFlow["Agentic Loop:
runAgenticFlow()
executeAgenticLoop()
normalizeToolCalls()
emitToolCallResult()
extractBase64Attachments()"] - end - subgraph S2["conversationsStore"] - S2State["State:
conversations
activeConversation
activeMessages
isInitialized
pendingMcpServerOverrides
titleUpdateConfirmationCallback"] - S2Lifecycle["Lifecycle:
initialize()
loadConversations()
clearActiveConversation()"] - S2ConvCRUD["Conversation CRUD:
createConversation()
loadConversation()
deleteConversation()
deleteAll()
updateConversationName()
updateConversationTitleWithConfirmation()"] - S2MsgMgmt["Message Management:
refreshActiveMessages()
addMessageToActive()
updateMessageAtIndex()
findMessageIndex()
sliceActiveMessages()
removeMessageAtIndex()
getConversationMessages()"] - S2Nav["Navigation:
navigateToSibling()
updateCurrentNode()
updateConversationTimestamp()"] - S2McpOverrides["MCP Per-Chat Overrides:
getMcpServerOverride()
getAllMcpServerOverrides()
setMcpServerOverride()
toggleMcpServerForChat()
removeMcpServerOverride()
isMcpServerEnabledForChat()
clearPendingMcpServerOverrides()"] - S2Export["Import/Export:
downloadConversation()
exportAllConversations()
importConversations()
importConversationsData()
triggerDownload()"] - S2Utils["Utilities:
setTitleUpdateConfirmationCallback()"] - end - subgraph S3["modelsStore"] - S3State["State:
models, routerModels
selectedModelId
selectedModelName
loading, updating, error
modelLoadingStates
modelPropsCache
modelPropsFetching
propsCacheVersion"] - S3Getters["Computed Getters:
selectedModel
loadedModelIds
loadingModelIds
singleModelName"] - S3Modal["Modalities:
getModelModalities()
modelSupportsVision()
modelSupportsAudio()
getModelModalitiesArray()
getModelProps()
updateModelModalities()"] - S3Status["Status Queries:
isModelLoaded()
isModelOperationInProgress()
getModelStatus()
isModelPropsFetching()"] - S3Fetch["Data Fetching:
fetch()
fetchRouterModels()
fetchModelProps()
fetchModalitiesForLoadedModels()"] - S3Select["Model Selection:
selectModelById()
selectModelByName()
clearSelection()
findModelByName()
findModelById()
hasModel()"] - S3LoadUnload["Loading/Unloading Models:
loadModel()
unloadModel()
ensureModelLoaded()
waitForModelStatus()
pollForModelStatus()"] - S3Utils["Utilities:
toDisplayName()
clear()"] - end - subgraph S4["serverStore"] - S4State["State:
props
loading, error
role
fetchPromise"] - S4Getters["Getters:
defaultParams
contextSize
isRouterMode
isModelMode"] - S4Data["Data Handling:
fetch()
getErrorMessage()
clear()"] - S4Utils["Utilities:
detectRole()"] - end - subgraph S5["settingsStore"] - S5State["State:
config
theme
isInitialized
userOverrides"] - S5Lifecycle["Lifecycle:
initialize()
loadConfig()
saveConfig()
loadTheme()
saveTheme()"] - S5Update["Config Updates:
updateConfig()
updateMultipleConfig()
updateTheme()"] - S5Reset["Reset:
resetConfig()
resetTheme()
resetAll()
resetParameterToServerDefault()"] - S5Sync["Server Sync:
syncWithServerDefaults()
forceSyncWithServerDefaults()"] - S5Utils["Utilities:
getConfig()
getAllConfig()
getParameterInfo()
getParameterDiff()
getServerDefaults()
clearAllUserOverrides()"] - end - subgraph S6["mcpStore"] - S6State["State:
isInitializing, error
toolCount, connectedServers
healthChecks (Map)
connections (Map)
toolsIndex (Map)"] - S6Lifecycle["Lifecycle:
ensureInitialized()
initialize()
shutdown()
acquireConnection()
releaseConnection()"] - S6Health["Health Checks:
runHealthCheck()
runHealthChecksForServers()
updateHealthCheck()
getHealthCheckState()
clearHealthCheck()"] - S6Servers["Server Management:
getServers()
addServer()
updateServer()
removeServer()
getServerById()
getServerDisplayName()"] - S6Tools["Tool Operations:
getToolDefinitionsForLLM()
getToolNames()
hasTool()
getToolServer()
executeTool()
executeToolByName()"] - S6Prompts["Prompt Operations:
getAllPrompts()
getPrompt()
hasPromptsCapability()
getPromptCompletions()"] - end - subgraph S7["mcpResourceStore"] - S7State["State:
serverResources (Map)
cachedResources (Map)
subscriptions (Map)
attachments[]
isLoading"] - S7Resources["Resource Discovery:
setServerResources()
getServerResources()
getAllResourceInfos()
getAllTemplateInfos()
clearServerResources()"] - S7Cache["Caching:
cacheResourceContent()
getCachedContent()
invalidateCache()
clearCache()"] - S7Subs["Subscriptions:
addSubscription()
removeSubscription()
isSubscribed()
handleResourceUpdate()"] - S7Attach["Attachments:
addAttachment()
updateAttachmentContent()
removeAttachment()
clearAttachments()
toMessageExtras()"] - end - - subgraph ReactiveExports["⚡ Reactive Exports"] - direction LR - subgraph ChatExports["chatStore"] - RE1["isLoading()"] - RE2["currentResponse()"] - RE3["errorDialog()"] - RE4["activeProcessingState()"] - RE5["isChatStreaming()"] - RE6["isChatLoading()"] - RE7["getChatStreaming()"] - RE8["getAllLoadingChats()"] - RE9["getAllStreamingChats()"] - RE9a["isEditModeActive()"] - RE9b["getAddFilesHandler()"] - RE9c["setEditModeActive()"] - RE9d["clearEditMode()"] - end - subgraph AgenticExports["agenticStore"] - REA1["agenticIsRunning()"] - REA2["agenticCurrentTurn()"] - REA3["agenticTotalToolCalls()"] - REA4["agenticLastError()"] - REA5["agenticStreamingToolCall()"] - REA6["agenticIsAnyRunning()"] - end - subgraph ConvExports["conversationsStore"] - RE10["conversations()"] - RE11["activeConversation()"] - RE12["activeMessages()"] - RE13["isConversationsInitialized()"] - end - subgraph ModelsExports["modelsStore"] - RE15["modelOptions()"] - RE16["routerModels()"] - RE17["modelsLoading()"] - RE18["modelsUpdating()"] - RE19["modelsError()"] - RE20["selectedModelId()"] - RE21["selectedModelName()"] - RE22["selectedModelOption()"] - RE23["loadedModelIds()"] - RE24["loadingModelIds()"] - RE25["propsCacheVersion()"] - RE26["singleModelName()"] - end - subgraph ServerExports["serverStore"] - RE27["serverProps()"] - RE28["serverLoading()"] - RE29["serverError()"] - RE30["serverRole()"] - RE31["defaultParams()"] - RE32["contextSize()"] - RE33["isRouterMode()"] - RE34["isModelMode()"] - end - subgraph SettingsExports["settingsStore"] - RE35["config()"] - RE36["theme()"] - RE37["isInitialized()"] - end - subgraph MCPExports["mcpStore / mcpResourceStore"] - RE38["mcpResources()"] - RE39["mcpResourceAttachments()"] - RE40["mcpHasResourceAttachments()"] - RE41["mcpTotalResourceCount()"] - RE42["mcpResourcesLoading()"] - end - end - end - - subgraph Services["⚙️ Services"] - direction TB - subgraph SV1["ChatService"] - SV1Msg["Messaging:
sendMessage()"] - SV1Stream["Streaming:
handleStreamResponse()
handleNonStreamResponse()"] - SV1Convert["Conversion:
convertDbMessageToApiChatMessageData()
mergeToolCallDeltas()"] - SV1Utils["Utilities:
stripReasoningContent()
extractModelName()
parseErrorResponse()"] - end - subgraph SV2["ModelsService"] - SV2List["Listing:
list()
listRouter()"] - SV2LoadUnload["Load/Unload:
load()
unload()"] - SV2Status["Status:
isModelLoaded()
isModelLoading()"] - end - subgraph SV3["PropsService"] - SV3Fetch["Fetching:
fetch()
fetchForModel()"] - end - subgraph SV4["DatabaseService"] - SV4Conv["Conversations:
createConversation()
getConversation()
getAllConversations()
updateConversation()
deleteConversation()"] - SV4Msg["Messages:
createMessageBranch()
createRootMessage()
createSystemMessage()
getConversationMessages()
updateMessage()
deleteMessage()
deleteMessageCascading()"] - SV4Node["Navigation:
updateCurrentNode()"] - SV4Import["Import:
importConversations()"] - end - subgraph SV5["ParameterSyncService"] - SV5Extract["Extraction:
extractServerDefaults()"] - SV5Merge["Merging:
mergeWithServerDefaults()"] - SV5Info["Info:
getParameterInfo()
canSyncParameter()
getSyncableParameterKeys()
validateServerParameter()"] - SV5Diff["Diff:
createParameterDiff()"] - end - subgraph SV6["MCPService"] - SV6Transport["Transport:
createTransport()
WebSocket / StreamableHTTP / SSE"] - SV6Conn["Connection:
connect()
disconnect()"] - SV6Tools["Tools:
listTools()
callTool()"] - SV6Prompts["Prompts:
listPrompts()
getPrompt()"] - SV6Resources["Resources:
listResources()
listResourceTemplates()
readResource()
subscribeResource()
unsubscribeResource()"] - SV6Complete["Completions:
complete()"] - end - end - - subgraph ExternalMCP["🔌 External MCP Servers"] - EXT1["MCP Server 1
(WebSocket/StreamableHTTP/SSE)"] - EXT2["MCP Server N"] - end - - subgraph Storage["💾 Storage"] - ST1["IndexedDB"] - ST2["conversations"] - ST3["messages"] - ST5["LocalStorage"] - ST6["config"] - ST7["userOverrides"] - ST8["mcpServers"] - end - - subgraph APIs["🌐 llama-server API"] - API1["/v1/chat/completions"] - API2["/props
/props?model="] - API3["/models
/models/load
/models/unload"] - API4["/v1/models"] - end - - %% Routes render Components - R1 --> C_Screen - R2 --> C_Screen - RL --> C_Sidebar - - %% Layout runs MCP health checks on startup - RL --> S6 - - %% Component hierarchy - C_Screen --> C_Form & C_Messages & C_Settings - C_Messages --> C_Message - C_Message --> C_MessageUser - C_MessageUser --> C_MessageEditForm - C_MessageEditForm --> C_ModelsSelector - C_MessageEditForm --> C_Attach - C_Form --> C_ModelsSelector - C_Form --> C_Attach - C_Form --> C_McpServersSelector - C_Message --> C_Attach - - %% MCP Components hierarchy - C_Settings --> C_McpSettings - C_McpSettings --> C_McpServerCard - C_McpServerCard --> C_McpResourceBrowser - C_McpResourceBrowser --> C_McpResourcePreview - - %% Components use Hooks - C_Form --> H1 - C_Message --> H1 & H2 - C_MessageEditForm --> H1 - C_Screen --> H2 - - %% Hooks use Stores - H1 --> S3 & S4 - H2 --> S1 & S5 - - %% Components use Stores - C_Screen --> S1 & S2 - C_Messages --> S2 - C_Message --> S1 & S2 & S3 - C_Form --> S1 & S3 & S6 - C_Sidebar --> S2 - C_ModelsSelector --> S3 & S4 - C_Settings --> S5 - C_McpSettings --> S6 - C_McpServerCard --> S6 - C_McpResourceBrowser --> S6 & S7 - C_McpServersSelector --> S6 - - %% Stores export Reactive State - S1 -. exports .-> ChatExports - SA -. exports .-> AgenticExports - S2 -. exports .-> ConvExports - S3 -. exports .-> ModelsExports - S4 -. exports .-> ServerExports - S5 -. exports .-> SettingsExports - S6 -. exports .-> MCPExports - S7 -. exports .-> MCPExports - - %% chatStore → agenticStore (agentic loop orchestration) - S1 --> SA - SA --> SV1 - SA --> S6 - - %% Stores use Services - S1 --> SV1 & SV4 - S2 --> SV4 - S3 --> SV2 & SV3 - S4 --> SV3 - S5 --> SV5 - S6 --> SV6 - S7 --> SV6 - - %% Services to Storage - SV4 --> ST1 - ST1 --> ST2 & ST3 - SV5 --> ST5 - ST5 --> ST6 & ST7 & ST8 - - %% Services to APIs - SV1 --> API1 - SV2 --> API3 & API4 - SV3 --> API2 - - %% MCP → External Servers - SV6 --> EXT1 & EXT2 - - %% Styling - classDef routeStyle fill:#e1f5fe,stroke:#01579b,stroke-width:2px - classDef componentStyle fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px - classDef componentGroupStyle fill:#e1bee7,stroke:#7b1fa2,stroke-width:1px - classDef hookStyle fill:#fff8e1,stroke:#ff8f00,stroke-width:2px - classDef storeStyle fill:#fff3e0,stroke:#e65100,stroke-width:2px - classDef stateStyle fill:#ffe0b2,stroke:#e65100,stroke-width:1px - classDef methodStyle fill:#ffecb3,stroke:#e65100,stroke-width:1px - classDef reactiveStyle fill:#fffde7,stroke:#f9a825,stroke-width:1px - classDef serviceStyle fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px - classDef serviceMStyle fill:#c8e6c9,stroke:#2e7d32,stroke-width:1px - classDef externalStyle fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,stroke-dasharray: 5 5 - classDef storageStyle fill:#fce4ec,stroke:#c2185b,stroke-width:2px - classDef apiStyle fill:#e3f2fd,stroke:#1565c0,stroke-width:2px - - class R1,R2,RL routeStyle - class C_Sidebar,C_Screen,C_Form,C_Messages,C_Message,C_MessageUser,C_MessageEditForm componentStyle - class C_ModelsSelector,C_Settings componentStyle - class C_Attach componentStyle - class C_McpSettings,C_McpServerCard,C_McpResourceBrowser,C_McpResourcePreview,C_McpServersSelector componentStyle - class H1,H2,H3 hookStyle - class LayoutComponents,ChatUIComponents,MCPComponents componentGroupStyle - class Hooks hookStyle - classDef agenticStyle fill:#e8eaf6,stroke:#283593,stroke-width:2px - classDef agenticMethodStyle fill:#c5cae9,stroke:#283593,stroke-width:1px - - class S1,S2,S3,S4,S5,SA,S6,S7 storeStyle - class S1State,S2State,S3State,S4State,S5State,SAState,S6State,S7State stateStyle - class S1Msg,S1Regen,S1Edit,S1Stream,S1LoadState,S1ProcState,S1Error,S1Utils methodStyle - class SASession,SAConfig,SAFlow methodStyle - class S2Lifecycle,S2ConvCRUD,S2MsgMgmt,S2Nav,S2McpOverrides,S2Export,S2Utils methodStyle - class S3Getters,S3Modal,S3Status,S3Fetch,S3Select,S3LoadUnload,S3Utils methodStyle - class S4Getters,S4Data,S4Utils methodStyle - class S5Lifecycle,S5Update,S5Reset,S5Sync,S5Utils methodStyle - class S6Lifecycle,S6Health,S6Servers,S6Tools,S6Prompts methodStyle - class S7Resources,S7Cache,S7Subs,S7Attach methodStyle - class ChatExports,AgenticExports,ConvExports,ModelsExports,ServerExports,SettingsExports,MCPExports reactiveStyle - class SV1,SV2,SV3,SV4,SV5,SV6 serviceStyle - class SV6Transport,SV6Conn,SV6Tools,SV6Prompts,SV6Resources,SV6Complete serviceMStyle - class EXT1,EXT2 externalStyle - class SV1Msg,SV1Stream,SV1Convert,SV1Utils serviceMStyle - class SV2List,SV2LoadUnload,SV2Status serviceMStyle - class SV3Fetch serviceMStyle - class SV4Conv,SV4Msg,SV4Node,SV4Import serviceMStyle - class SV5Extract,SV5Merge,SV5Info,SV5Diff serviceMStyle - class ST1,ST2,ST3,ST5,ST6,ST7,ST8 storageStyle - class API1,API2,API3,API4 apiStyle -``` diff --git a/tools/ui/docs/flows/chat-flow.md b/tools/ui/docs/flows/chat-flow.md deleted file mode 100644 index 296693c6a..000000000 --- a/tools/ui/docs/flows/chat-flow.md +++ /dev/null @@ -1,228 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ChatForm / ChatMessage - participant chatStore as 🗄️ chatStore - participant agenticStore as 🗄️ agenticStore - participant convStore as 🗄️ conversationsStore - participant settingsStore as 🗄️ settingsStore - participant mcpStore as 🗄️ mcpStore - participant ChatSvc as ⚙️ ChatService - participant DbSvc as ⚙️ DatabaseService - participant API as 🌐 /v1/chat/completions - - Note over chatStore: State:
isLoading, currentResponse
errorDialogState, activeProcessingState
chatLoadingStates (Map)
chatStreamingStates (Map)
abortControllers (Map)
processingStates (Map) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 💬 SEND MESSAGE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: sendMessage(content, extras) - activate chatStore - - chatStore->>chatStore: setChatLoading(convId, true) - chatStore->>chatStore: clearChatStreaming(convId) - - alt no active conversation - chatStore->>convStore: createConversation() - Note over convStore: → see conversations-flow.mmd - end - - chatStore->>mcpStore: consumeResourceAttachmentsAsExtras() - Note right of mcpStore: Converts pending MCP resource
attachments into message extras - - chatStore->>chatStore: addMessage("user", content, extras) - chatStore->>DbSvc: createMessageBranch(userMsg, parentId) - chatStore->>convStore: addMessageToActive(userMsg) - chatStore->>convStore: updateCurrentNode(userMsg.id) - - chatStore->>chatStore: createAssistantMessage(userMsg.id) - chatStore->>DbSvc: createMessageBranch(assistantMsg, userMsg.id) - chatStore->>convStore: addMessageToActive(assistantMsg) - - chatStore->>chatStore: streamChatCompletion(messages, assistantMsg) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🌊 STREAMING (with agentic flow detection) - %% ═══════════════════════════════════════════════════════════════════════════ - - activate chatStore - chatStore->>chatStore: startStreaming() - Note right of chatStore: isStreamingActive = true - - chatStore->>chatStore: setActiveProcessingConversation(convId) - chatStore->>chatStore: getOrCreateAbortController(convId) - Note right of chatStore: abortControllers.set(convId, new AbortController()) - - chatStore->>chatStore: getApiOptions() - Note right of chatStore: Merge from settingsStore.config:
temperature, max_tokens, top_p, etc. - - alt agenticConfig.enabled && mcpStore has connected servers - chatStore->>agenticStore: runAgenticFlow(convId, messages, assistantMsg, options, signal) - Note over agenticStore: Multi-turn agentic loop:
1. Call ChatService.sendMessage()
2. If response has tool_calls → execute via mcpStore
3. Append tool results as messages
4. Loop until no more tool_calls or maxTurns
→ see agentic flow details below - agenticStore-->>chatStore: final response with timings - else standard (non-agentic) flow - chatStore->>ChatSvc: sendMessage(messages, options, signal) - end - - activate ChatSvc - - ChatSvc->>ChatSvc: convertDbMessageToApiChatMessageData(messages) - Note right of ChatSvc: DatabaseMessage[] → ApiChatMessageData[]
Process attachments (images, PDFs, audio) - - ChatSvc->>API: POST /v1/chat/completions - Note right of API: {messages, model?, stream: true, ...params} - - loop SSE chunks - API-->>ChatSvc: data: {"choices":[{"delta":{...}}]} - ChatSvc->>ChatSvc: handleStreamResponse(response) - - alt content chunk - ChatSvc-->>chatStore: onChunk(content) - chatStore->>chatStore: setChatStreaming(convId, response, msgId) - Note right of chatStore: currentResponse = $state(accumulated) - chatStore->>convStore: updateMessageAtIndex(idx, {content}) - end - - alt reasoning chunk - ChatSvc-->>chatStore: onReasoningChunk(reasoning) - chatStore->>convStore: updateMessageAtIndex(idx, {thinking}) - end - - alt tool_calls chunk - ChatSvc-->>chatStore: onToolCallChunk(toolCalls) - chatStore->>convStore: updateMessageAtIndex(idx, {toolCalls}) - end - - alt model info - ChatSvc-->>chatStore: onModel(modelName) - chatStore->>chatStore: recordModel(modelName) - chatStore->>DbSvc: updateMessage(msgId, {model}) - end - - alt timings (during stream) - ChatSvc-->>chatStore: onTimings(timings, promptProgress) - chatStore->>chatStore: updateProcessingStateFromTimings() - end - - chatStore-->>UI: reactive $state update - end - - API-->>ChatSvc: data: [DONE] - ChatSvc-->>chatStore: onComplete(content, reasoning, timings, toolCalls) - deactivate ChatSvc - - chatStore->>chatStore: stopStreaming() - chatStore->>DbSvc: updateMessage(msgId, {content, timings, model}) - chatStore->>convStore: updateCurrentNode(msgId) - chatStore->>chatStore: setChatLoading(convId, false) - chatStore->>chatStore: clearChatStreaming(convId) - chatStore->>chatStore: clearProcessingState(convId) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ⏹️ STOP GENERATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: stopGeneration() - activate chatStore - chatStore->>chatStore: savePartialResponseIfNeeded(convId) - Note right of chatStore: Save currentResponse to DB if non-empty - chatStore->>chatStore: abortControllers.get(convId).abort() - Note right of chatStore: fetch throws AbortError → caught by isAbortError() - chatStore->>chatStore: stopStreaming() - chatStore->>chatStore: setChatLoading(convId, false) - chatStore->>chatStore: clearChatStreaming(convId) - chatStore->>chatStore: clearProcessingState(convId) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🔁 REGENERATE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: regenerateMessageWithBranching(msgId, model?) - activate chatStore - chatStore->>convStore: findMessageIndex(msgId) - chatStore->>chatStore: Get parent of target message - chatStore->>chatStore: createAssistantMessage(parentId) - chatStore->>DbSvc: createMessageBranch(newAssistantMsg, parentId) - chatStore->>convStore: refreshActiveMessages() - Note right of chatStore: Same streaming flow - chatStore->>chatStore: streamChatCompletion(...) - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ➡️ CONTINUE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: continueAssistantMessage(msgId) - activate chatStore - chatStore->>chatStore: Get existing content from message - chatStore->>chatStore: streamChatCompletion(..., existingContent) - Note right of chatStore: Appends to existing message content - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ✏️ EDIT USER MESSAGE - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>chatStore: editMessageWithBranching(msgId, newContent, extras) - activate chatStore - chatStore->>chatStore: Get parent of target message - chatStore->>DbSvc: createMessageBranch(editedMsg, parentId) - chatStore->>convStore: refreshActiveMessages() - Note right of chatStore: Creates new branch, original preserved - chatStore->>chatStore: createAssistantMessage(editedMsg.id) - chatStore->>chatStore: streamChatCompletion(...) - Note right of chatStore: Automatically regenerates response - deactivate chatStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ❌ ERROR HANDLING - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over chatStore: On stream error (non-abort): - chatStore->>chatStore: showErrorDialog(type, message) - Note right of chatStore: errorDialogState = {type: 'timeout'|'server', message} - chatStore->>convStore: removeMessageAtIndex(failedMsgIdx) - chatStore->>DbSvc: deleteMessage(failedMsgId) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🤖 AGENTIC LOOP (when agenticConfig.enabled) - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over agenticStore: agenticStore.runAgenticFlow(convId, messages, assistantMsg, options, signal) - activate agenticStore - agenticStore->>agenticStore: getSession(convId) or create new - agenticStore->>agenticStore: updateSession(turn: 0, running: true) - - loop executeAgenticLoop (until no tool_calls or maxTurns) - agenticStore->>agenticStore: turn++ - agenticStore->>ChatSvc: sendMessage(messages, options, signal) - ChatSvc->>API: POST /v1/chat/completions - API-->>ChatSvc: response with potential tool_calls - ChatSvc-->>agenticStore: onComplete(content, reasoning, timings, toolCalls) - - alt response has tool_calls - agenticStore->>agenticStore: normalizeToolCalls(toolCalls) - loop for each tool_call - agenticStore->>agenticStore: updateSession(streamingToolCall) - agenticStore->>mcpStore: executeTool(mcpCall, signal) - mcpStore-->>agenticStore: tool result - agenticStore->>agenticStore: extractBase64Attachments(result) - agenticStore->>agenticStore: emitToolCallResult(convId, ...) - agenticStore->>convStore: addMessageToActive(toolResultMsg) - agenticStore->>DbSvc: createMessageBranch(toolResultMsg) - end - agenticStore->>agenticStore: Create new assistantMsg for next turn - Note right of agenticStore: Continue loop with updated messages - else no tool_calls (final response) - agenticStore->>agenticStore: buildFinalTimings(allTurns) - Note right of agenticStore: Break loop, return final response - end - end - - agenticStore->>agenticStore: updateSession(running: false) - agenticStore-->>chatStore: final content, timings, model - deactivate agenticStore -``` diff --git a/tools/ui/docs/flows/conversations-flow.md b/tools/ui/docs/flows/conversations-flow.md deleted file mode 100644 index bd2309bc0..000000000 --- a/tools/ui/docs/flows/conversations-flow.md +++ /dev/null @@ -1,183 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ChatSidebar / ChatScreen - participant convStore as 🗄️ conversationsStore - participant chatStore as 🗄️ chatStore - participant DbSvc as ⚙️ DatabaseService - participant IDB as 💾 IndexedDB - - Note over convStore: State:
conversations: DatabaseConversation[]
activeConversation: DatabaseConversation | null
activeMessages: DatabaseMessage[]
isInitialized: boolean
pendingMcpServerOverrides: Map<string, McpServerOverride> - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 🚀 INITIALIZATION - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over convStore: Auto-initialized in constructor (browser only) - convStore->>convStore: initialize() - activate convStore - convStore->>convStore: loadConversations() - convStore->>DbSvc: getAllConversations() - DbSvc->>IDB: SELECT * FROM conversations ORDER BY lastModified DESC - IDB-->>DbSvc: Conversation[] - DbSvc-->>convStore: conversations - convStore->>convStore: conversations = $state(data) - convStore->>convStore: isInitialized = true - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: ➕ CREATE CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: createConversation(name?) - activate convStore - convStore->>DbSvc: createConversation(name || "New Chat") - DbSvc->>IDB: INSERT INTO conversations - IDB-->>DbSvc: conversation {id, name, lastModified, currNode: ""} - DbSvc-->>convStore: conversation - convStore->>convStore: conversations.unshift(conversation) - convStore->>convStore: activeConversation = $state(conversation) - convStore->>convStore: activeMessages = $state([]) - - alt pendingMcpServerOverrides has entries - loop each pending override - convStore->>DbSvc: Store MCP server override for new conversation - end - convStore->>convStore: clearPendingMcpServerOverrides() - end - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 📂 LOAD CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: loadConversation(convId) - activate convStore - convStore->>DbSvc: getConversation(convId) - DbSvc->>IDB: SELECT * FROM conversations WHERE id = ? - IDB-->>DbSvc: conversation - convStore->>convStore: activeConversation = $state(conversation) - - convStore->>convStore: refreshActiveMessages() - convStore->>DbSvc: getConversationMessages(convId) - DbSvc->>IDB: SELECT * FROM messages WHERE convId = ? - IDB-->>DbSvc: allMessages[] - convStore->>convStore: filterByLeafNodeId(allMessages, currNode) - Note right of convStore: Filter to show only current branch path - convStore->>convStore: activeMessages = $state(filtered) - - Note right of convStore: Route (+page.svelte) then calls:
chatStore.syncLoadingStateForChat(convId) - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 🌳 MESSAGE BRANCHING MODEL - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over IDB: Message Tree Structure:
- Each message has parent (null for root)
- Each message has children[] array
- Conversation.currNode points to active leaf
- filterByLeafNodeId() traverses from root to currNode - - rect rgb(240, 240, 255) - Note over convStore: Example Branch Structure: - Note over convStore: root → user1 → assistant1 → user2 → assistant2a (currNode)
↘ assistant2b (alt branch) - end - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: ↔️ BRANCH NAVIGATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: navigateToSibling(msgId, direction) - activate convStore - convStore->>convStore: Find message in activeMessages - convStore->>convStore: Get parent message - convStore->>convStore: Find sibling in parent.children[] - convStore->>convStore: findLeafNode(siblingId, allMessages) - Note right of convStore: Navigate to leaf of sibling branch - convStore->>convStore: updateCurrentNode(leafId) - convStore->>DbSvc: updateCurrentNode(convId, leafId) - DbSvc->>IDB: UPDATE conversations SET currNode = ? - convStore->>convStore: refreshActiveMessages() - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 📝 UPDATE CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: updateConversationName(convId, newName) - activate convStore - convStore->>DbSvc: updateConversation(convId, {name: newName}) - DbSvc->>IDB: UPDATE conversations SET name = ? - convStore->>convStore: Update in conversations array - deactivate convStore - - Note over convStore: Auto-title update (after first response): - convStore->>convStore: updateConversationTitleWithConfirmation() - convStore->>convStore: titleUpdateConfirmationCallback?() - Note right of convStore: Shows dialog if title would change - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 🗑️ DELETE CONVERSATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: deleteConversation(convId) - activate convStore - convStore->>DbSvc: deleteConversation(convId) - DbSvc->>IDB: DELETE FROM conversations WHERE id = ? - DbSvc->>IDB: DELETE FROM messages WHERE convId = ? - convStore->>convStore: conversations.filter(c => c.id !== convId) - alt deleted active conversation - convStore->>convStore: clearActiveConversation() - end - deactivate convStore - - UI->>convStore: deleteAll() - activate convStore - convStore->>DbSvc: Delete all conversations and messages - convStore->>convStore: conversations = [] - convStore->>convStore: clearActiveConversation() - deactivate convStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: � MCP SERVER PER-CHAT OVERRIDES - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over convStore: Conversations can override which MCP servers are enabled. - Note over convStore: Uses pendingMcpServerOverrides before conversation
is created, then persists to conversation metadata. - - UI->>convStore: setMcpServerOverride(convId, serverName, override) - Note right of convStore: override = {enabled: boolean} - - UI->>convStore: toggleMcpServerForChat(convId, serverName, enabled) - activate convStore - convStore->>convStore: setMcpServerOverride(convId, serverName, {enabled}) - deactivate convStore - - UI->>convStore: isMcpServerEnabledForChat(convId, serverName) - Note right of convStore: Check override → fall back to global MCP config - - UI->>convStore: getAllMcpServerOverrides(convId) - Note right of convStore: Returns all overrides for a conversation - - UI->>convStore: removeMcpServerOverride(convId, serverName) - UI->>convStore: getMcpServerOverride(convId, serverName) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,IDB: 📤 EXPORT / 📥 IMPORT - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>convStore: exportAllConversations() - activate convStore - convStore->>DbSvc: getAllConversations() - loop each conversation - convStore->>DbSvc: getConversationMessages(convId) - end - convStore->>convStore: triggerDownload(JSON blob) - deactivate convStore - - UI->>convStore: importConversations(file) - activate convStore - convStore->>convStore: Parse JSON file - convStore->>convStore: importConversationsData(parsed) - convStore->>DbSvc: importConversations(parsed) - Note right of DbSvc: Skips duplicate conversations
(checks existing by ID) - DbSvc->>IDB: INSERT conversations + messages (skip existing) - convStore->>convStore: loadConversations() - deactivate convStore -``` diff --git a/tools/ui/docs/flows/data-flow-simplified-model-mode.md b/tools/ui/docs/flows/data-flow-simplified-model-mode.md deleted file mode 100644 index 07b362147..000000000 --- a/tools/ui/docs/flows/data-flow-simplified-model-mode.md +++ /dev/null @@ -1,45 +0,0 @@ -```mermaid -%% MODEL Mode Data Flow (single model) -%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd - -sequenceDiagram - participant User as 👤 User - participant UI as 🧩 UI - participant Stores as 🗄️ Stores - participant DB as 💾 IndexedDB - participant API as 🌐 llama-server - - Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd) - - UI->>Stores: initialize() - Stores->>DB: load conversations - Stores->>API: GET /props - API-->>Stores: server config + modalities - Stores->>API: GET /v1/models - API-->>Stores: single model (auto-selected) - - Note over User,API: 💬 Chat Flow (see: chat-flow.mmd) - - User->>UI: send message - UI->>Stores: sendMessage() - Stores->>DB: save user message - Stores->>API: POST /v1/chat/completions (stream) - loop streaming - API-->>Stores: SSE chunks - Stores-->>UI: reactive update - end - API-->>Stores: done + timings - Stores->>DB: save assistant message - - Note over User,API: 🔁 Regenerate - - User->>UI: regenerate - Stores->>DB: create message branch - Note right of Stores: same streaming flow - - Note over User,API: ⏹️ Stop - - User->>UI: stop - Stores->>Stores: abort stream - Stores->>DB: save partial response -``` diff --git a/tools/ui/docs/flows/data-flow-simplified-router-mode.md b/tools/ui/docs/flows/data-flow-simplified-router-mode.md deleted file mode 100644 index bccacf568..000000000 --- a/tools/ui/docs/flows/data-flow-simplified-router-mode.md +++ /dev/null @@ -1,77 +0,0 @@ -```mermaid -%% ROUTER Mode Data Flow (multi-model) -%% Detailed flows: ./flows/server-flow.mmd, ./flows/models-flow.mmd, ./flows/chat-flow.mmd - -sequenceDiagram - participant User as 👤 User - participant UI as 🧩 UI - participant Stores as 🗄️ Stores - participant DB as 💾 IndexedDB - participant API as 🌐 llama-server - - Note over User,API: 🚀 Initialization (see: server-flow.mmd, models-flow.mmd) - - UI->>Stores: initialize() - Stores->>DB: load conversations - Stores->>API: GET /props - API-->>Stores: {role: "router"} - Stores->>API: GET /v1/models - API-->>Stores: models[] with status (loaded/available) - loop each loaded model - Stores->>API: GET /props?model=X - API-->>Stores: modalities (vision/audio) - end - - Note over User,API: 🔄 Model Selection (see: models-flow.mmd) - - User->>UI: select model - alt model not loaded - Stores->>API: POST /models/load - loop poll status - Stores->>API: GET /v1/models - API-->>Stores: check if loaded - end - Stores->>API: GET /props?model=X - API-->>Stores: cache modalities - end - Stores->>Stores: validate modalities vs conversation - alt valid - Stores->>Stores: select model - else invalid - Stores->>API: POST /models/unload - UI->>User: show error toast - end - - Note over User,API: 💬 Chat Flow (see: chat-flow.mmd) - - User->>UI: send message - UI->>Stores: sendMessage() - Stores->>DB: save user message - Stores->>API: POST /v1/chat/completions {model: X} - Note right of API: router forwards to model - loop streaming - API-->>Stores: SSE chunks + model info - Stores-->>UI: reactive update - end - API-->>Stores: done + timings - Stores->>DB: save assistant message + model used - - Note over User,API: 🔁 Regenerate (optional: different model) - - User->>UI: regenerate - Stores->>Stores: validate modalities up to this message - Stores->>DB: create message branch - Note right of Stores: same streaming flow - - Note over User,API: ⏹️ Stop - - User->>UI: stop - Stores->>Stores: abort stream - Stores->>DB: save partial response - - Note over User,API: 🗑️ LRU Unloading - - Note right of API: Server auto-unloads LRU models
when cache full - User->>UI: select unloaded model - Note right of Stores: triggers load flow again -``` diff --git a/tools/ui/docs/flows/database-flow.md b/tools/ui/docs/flows/database-flow.md deleted file mode 100644 index 38cd6941c..000000000 --- a/tools/ui/docs/flows/database-flow.md +++ /dev/null @@ -1,174 +0,0 @@ -```mermaid -sequenceDiagram - participant Store as 🗄️ Stores - participant DbSvc as ⚙️ DatabaseService - participant Dexie as 📦 Dexie ORM - participant IDB as 💾 IndexedDB - - Note over DbSvc: Stateless service - all methods static
Database: "LlamacppWebui" - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 📊 SCHEMA - %% ═══════════════════════════════════════════════════════════════════════════ - - rect rgb(240, 248, 255) - Note over IDB: conversations table:
id (PK), lastModified, currNode, name - end - - rect rgb(255, 248, 240) - Note over IDB: messages table:
id (PK), convId (FK), type, role, timestamp,
parent, children[], content, thinking,
toolCalls, extra[], model, timings - end - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 💬 CONVERSATIONS CRUD - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: createConversation(name) - activate DbSvc - DbSvc->>DbSvc: Generate UUID - DbSvc->>Dexie: db.conversations.add({id, name, lastModified, currNode: ""}) - Dexie->>IDB: INSERT - IDB-->>Dexie: success - DbSvc-->>Store: DatabaseConversation - deactivate DbSvc - - Store->>DbSvc: getConversation(convId) - DbSvc->>Dexie: db.conversations.get(convId) - Dexie->>IDB: SELECT WHERE id = ? - IDB-->>DbSvc: DatabaseConversation - - Store->>DbSvc: getAllConversations() - DbSvc->>Dexie: db.conversations.orderBy('lastModified').reverse().toArray() - Dexie->>IDB: SELECT ORDER BY lastModified DESC - IDB-->>DbSvc: DatabaseConversation[] - - Store->>DbSvc: updateConversation(convId, updates) - DbSvc->>Dexie: db.conversations.update(convId, {...updates, lastModified}) - Dexie->>IDB: UPDATE - - Store->>DbSvc: deleteConversation(convId) - activate DbSvc - DbSvc->>Dexie: db.conversations.delete(convId) - Dexie->>IDB: DELETE FROM conversations - DbSvc->>Dexie: db.messages.where('convId').equals(convId).delete() - Dexie->>IDB: DELETE FROM messages WHERE convId = ? - deactivate DbSvc - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 📝 MESSAGES CRUD - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: createRootMessage(convId) - activate DbSvc - DbSvc->>DbSvc: Create root message {type: "root", parent: null} - DbSvc->>Dexie: db.messages.add(rootMsg) - Dexie->>IDB: INSERT - DbSvc-->>Store: rootMessageId - deactivate DbSvc - - Store->>DbSvc: createSystemMessage(convId, content, parentId) - activate DbSvc - DbSvc->>DbSvc: Create message {role: "system", parent: parentId} - DbSvc->>Dexie: db.messages.add(systemMsg) - Dexie->>IDB: INSERT - DbSvc-->>Store: DatabaseMessage - deactivate DbSvc - - Store->>DbSvc: createMessageBranch(message, parentId) - activate DbSvc - DbSvc->>DbSvc: Generate UUID for new message - DbSvc->>Dexie: db.messages.add({...message, id, parent: parentId}) - Dexie->>IDB: INSERT message - - alt parentId exists - DbSvc->>Dexie: db.messages.get(parentId) - Dexie->>IDB: SELECT parent - DbSvc->>DbSvc: parent.children.push(newId) - DbSvc->>Dexie: db.messages.update(parentId, {children}) - Dexie->>IDB: UPDATE parent.children - end - - DbSvc->>Dexie: db.conversations.update(convId, {currNode: newId}) - Dexie->>IDB: UPDATE conversation.currNode - DbSvc-->>Store: DatabaseMessage - deactivate DbSvc - - Store->>DbSvc: getConversationMessages(convId) - DbSvc->>Dexie: db.messages.where('convId').equals(convId).toArray() - Dexie->>IDB: SELECT WHERE convId = ? - IDB-->>DbSvc: DatabaseMessage[] - - Store->>DbSvc: updateMessage(msgId, updates) - DbSvc->>Dexie: db.messages.update(msgId, updates) - Dexie->>IDB: UPDATE - - Store->>DbSvc: deleteMessage(msgId) - DbSvc->>Dexie: db.messages.delete(msgId) - Dexie->>IDB: DELETE - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 🌳 BRANCHING OPERATIONS - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: updateCurrentNode(convId, nodeId) - DbSvc->>Dexie: db.conversations.update(convId, {currNode: nodeId, lastModified}) - Dexie->>IDB: UPDATE - - Store->>DbSvc: deleteMessageCascading(msgId) - activate DbSvc - DbSvc->>DbSvc: findDescendantMessages(msgId, allMessages) - Note right of DbSvc: Recursively find all children - loop each descendant - DbSvc->>Dexie: db.messages.delete(descendantId) - Dexie->>IDB: DELETE - end - DbSvc->>Dexie: db.messages.delete(msgId) - Dexie->>IDB: DELETE target message - - alt target message has a parent - DbSvc->>Dexie: db.messages.get(parentId) - DbSvc->>DbSvc: parent.children.filter(id !== msgId) - DbSvc->>Dexie: db.messages.update(parentId, {children}) - Note right of DbSvc: Remove deleted message from parent's children[] - end - deactivate DbSvc - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 📥 IMPORT - %% ═══════════════════════════════════════════════════════════════════════════ - - Store->>DbSvc: importConversations(data) - activate DbSvc - loop each conversation in data - DbSvc->>Dexie: db.conversations.get(conv.id) - alt conversation already exists - Note right of DbSvc: Skip duplicate (keep existing) - else conversation is new - DbSvc->>Dexie: db.conversations.add(conversation) - Dexie->>IDB: INSERT conversation - loop each message - DbSvc->>Dexie: db.messages.add(message) - Dexie->>IDB: INSERT message - end - end - end - deactivate DbSvc - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over Store,IDB: 🔗 MESSAGE TREE UTILITIES - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over DbSvc: Used by stores (imported from utils): - - rect rgb(240, 255, 240) - Note over DbSvc: filterByLeafNodeId(messages, leafId)
→ Returns path from root to leaf
→ Used to display current branch - end - - rect rgb(240, 255, 240) - Note over DbSvc: findLeafNode(startId, messages)
→ Traverse to deepest child
→ Used for branch navigation - end - - rect rgb(240, 255, 240) - Note over DbSvc: findDescendantMessages(msgId, messages)
→ Find all children recursively
→ Used for cascading deletes - end -``` diff --git a/tools/ui/docs/flows/mcp-flow.md b/tools/ui/docs/flows/mcp-flow.md deleted file mode 100644 index c8aa66659..000000000 --- a/tools/ui/docs/flows/mcp-flow.md +++ /dev/null @@ -1,226 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 McpServersSettings / ChatForm - participant chatStore as 🗄️ chatStore - participant mcpStore as 🗄️ mcpStore - participant mcpResStore as 🗄️ mcpResourceStore - participant convStore as 🗄️ conversationsStore - participant MCPSvc as ⚙️ MCPService - participant LS as 💾 LocalStorage - participant ExtMCP as 🔌 External MCP Server - - Note over mcpStore: State:
isInitializing, error
toolCount, connectedServers
healthChecks (Map)
connections (Map)
toolsIndex (Map)
serverConfigs (Map) - - Note over mcpResStore: State:
serverResources (Map)
cachedResources (Map)
subscriptions (Map)
attachments[] - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🚀 INITIALIZATION (App Startup) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: ensureInitialized() - activate mcpStore - - mcpStore->>LS: get(MCP_SERVERS_LOCALSTORAGE_KEY) - LS-->>mcpStore: MCPServerSettingsEntry[] - - mcpStore->>mcpStore: parseServerSettings(servers) - Note right of mcpStore: Filter enabled servers
Build MCPServerConfig objects
Per-chat overrides checked via convStore - - loop For each enabled server - mcpStore->>mcpStore: runHealthCheck(serverId) - mcpStore->>mcpStore: updateHealthCheck(id, CONNECTING) - - mcpStore->>MCPSvc: connect(serverName, config, clientInfo, capabilities, onPhase) - activate MCPSvc - - MCPSvc->>MCPSvc: createTransport(config) - Note right of MCPSvc: WebSocket / StreamableHTTP / SSE
with optional CORS proxy - - MCPSvc->>ExtMCP: Transport handshake - ExtMCP-->>MCPSvc: Connection established - - MCPSvc->>ExtMCP: Initialize request - Note right of ExtMCP: Exchange capabilities
Server info, protocol version - - ExtMCP-->>MCPSvc: InitializeResult (serverInfo, capabilities) - - MCPSvc->>ExtMCP: listTools() - ExtMCP-->>MCPSvc: Tool[] - - MCPSvc-->>mcpStore: MCPConnection - deactivate MCPSvc - - mcpStore->>mcpStore: connections.set(serverName, connection) - mcpStore->>mcpStore: indexTools(connection.tools, serverName) - Note right of mcpStore: toolsIndex.set(toolName, serverName)
Handle name conflicts with prefixes - - mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS) - mcpStore->>mcpStore: _connectedServers.push(serverName) - - alt Server supports resources - mcpStore->>MCPSvc: listAllResources(connection) - MCPSvc->>ExtMCP: listResources() - ExtMCP-->>MCPSvc: MCPResource[] - MCPSvc-->>mcpStore: resources - - mcpStore->>MCPSvc: listAllResourceTemplates(connection) - MCPSvc->>ExtMCP: listResourceTemplates() - ExtMCP-->>MCPSvc: MCPResourceTemplate[] - MCPSvc-->>mcpStore: templates - - mcpStore->>mcpResStore: setServerResources(serverName, resources, templates) - end - end - - mcpStore->>mcpStore: _isInitializing = false - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🔧 TOOL EXECUTION (Chat with Tools) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: executeTool(mcpCall: MCPToolCall, signal?) - activate mcpStore - - mcpStore->>mcpStore: toolsIndex.get(mcpCall.function.name) - Note right of mcpStore: Resolve serverName from toolsIndex
MCPToolCall = {id, type, function: {name, arguments}} - - mcpStore->>mcpStore: acquireConnection() - Note right of mcpStore: activeFlowCount++
Prevent shutdown during execution - - mcpStore->>mcpStore: connection = connections.get(serverName) - - mcpStore->>MCPSvc: callTool(connection, {name, arguments}, signal) - activate MCPSvc - - MCPSvc->>MCPSvc: throwIfAborted(signal) - MCPSvc->>ExtMCP: callTool(name, arguments) - - alt Tool execution success - ExtMCP-->>MCPSvc: ToolCallResult (content, isError) - MCPSvc->>MCPSvc: formatToolResult(result) - Note right of MCPSvc: Handle text, image (base64),
embedded resource content - MCPSvc-->>mcpStore: ToolExecutionResult - else Tool execution error - ExtMCP-->>MCPSvc: Error - MCPSvc-->>mcpStore: throw Error - else Aborted - MCPSvc-->>mcpStore: throw AbortError - end - - deactivate MCPSvc - - mcpStore->>mcpStore: releaseConnection() - Note right of mcpStore: activeFlowCount-- - - mcpStore-->>UI: ToolExecutionResult - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: � RESOURCE ATTACHMENT CONSUMPTION - %% ═══════════════════════════════════════════════════════════════════════════ - - chatStore->>mcpStore: consumeResourceAttachmentsAsExtras() - activate mcpStore - mcpStore->>mcpResStore: getAttachments() - mcpResStore-->>mcpStore: MCPResourceAttachment[] - mcpStore->>mcpStore: Convert attachments to message extras - mcpStore->>mcpResStore: clearAttachments() - mcpStore-->>chatStore: MessageExtra[] (for user message) - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: �📝 PROMPT OPERATIONS - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: getAllPrompts() - activate mcpStore - - loop For each connected server with prompts capability - mcpStore->>MCPSvc: listPrompts(connection) - MCPSvc->>ExtMCP: listPrompts() - ExtMCP-->>MCPSvc: Prompt[] - MCPSvc-->>mcpStore: prompts - end - - mcpStore-->>UI: MCPPromptInfo[] (with serverName) - deactivate mcpStore - - UI->>mcpStore: getPrompt(serverName, promptName, args?) - activate mcpStore - - mcpStore->>MCPSvc: getPrompt(connection, name, args) - MCPSvc->>ExtMCP: getPrompt({name, arguments}) - ExtMCP-->>MCPSvc: GetPromptResult (messages) - MCPSvc-->>mcpStore: GetPromptResult - - mcpStore-->>UI: GetPromptResult - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 📁 RESOURCE OPERATIONS - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpResStore: addAttachment(resourceInfo) - activate mcpResStore - mcpResStore->>mcpResStore: Create MCPResourceAttachment (loading: true) - mcpResStore-->>UI: attachment - - UI->>mcpStore: readResource(serverName, uri) - activate mcpStore - - mcpStore->>MCPSvc: readResource(connection, uri) - MCPSvc->>ExtMCP: readResource({uri}) - ExtMCP-->>MCPSvc: MCPReadResourceResult (contents) - MCPSvc-->>mcpStore: contents - - mcpStore-->>UI: MCPResourceContent[] - deactivate mcpStore - - UI->>mcpResStore: updateAttachmentContent(attachmentId, content) - mcpResStore->>mcpResStore: cacheResourceContent(resource, content) - deactivate mcpResStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🔄 AUTO-RECONNECTION - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over mcpStore: On WebSocket close or connection error: - mcpStore->>mcpStore: autoReconnect(serverName, attempt) - activate mcpStore - - mcpStore->>mcpStore: Calculate backoff delay - Note right of mcpStore: delay = min(30s, 1s * 2^attempt) - - mcpStore->>mcpStore: Wait for delay - mcpStore->>mcpStore: reconnectServer(serverName) - - alt Reconnection success - mcpStore->>mcpStore: updateHealthCheck(id, SUCCESS) - else Max attempts reached - mcpStore->>mcpStore: updateHealthCheck(id, ERROR) - end - deactivate mcpStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,ExtMCP: 🛑 SHUTDOWN - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>mcpStore: shutdown() - activate mcpStore - - mcpStore->>mcpStore: Wait for activeFlowCount == 0 - - loop For each connection - mcpStore->>MCPSvc: disconnect(connection) - MCPSvc->>MCPSvc: transport.onclose = undefined - MCPSvc->>ExtMCP: close() - end - - mcpStore->>mcpStore: connections.clear() - mcpStore->>mcpStore: toolsIndex.clear() - mcpStore->>mcpStore: _connectedServers = [] - - mcpStore->>mcpResStore: clear() - deactivate mcpStore -``` diff --git a/tools/ui/docs/flows/models-flow.md b/tools/ui/docs/flows/models-flow.md deleted file mode 100644 index c3031b729..000000000 --- a/tools/ui/docs/flows/models-flow.md +++ /dev/null @@ -1,181 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ModelsSelector - participant Hooks as 🪝 useModelChangeValidation - participant modelsStore as 🗄️ modelsStore - participant serverStore as 🗄️ serverStore - participant convStore as 🗄️ conversationsStore - participant ModelsSvc as ⚙️ ModelsService - participant PropsSvc as ⚙️ PropsService - participant API as 🌐 llama-server - - Note over modelsStore: State:
models: ModelOption[]
routerModels: ApiModelDataEntry[]
selectedModelId, selectedModelName
loading, updating, error
modelLoadingStates (Map)
modelPropsCache (Map)
propsCacheVersion - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🚀 INITIALIZATION (MODEL mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>modelsStore: fetch() - activate modelsStore - modelsStore->>modelsStore: loading = true - - alt serverStore.props not loaded - modelsStore->>serverStore: fetch() - Note over serverStore: → see server-flow.mmd - end - - modelsStore->>ModelsSvc: list() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: ApiModelListResponse {data: [model]} - - modelsStore->>modelsStore: models = $state(mapped) - Note right of modelsStore: Map to ModelOption[]:
{id, name, model, description, capabilities} - - Note over modelsStore: MODEL mode: Get modalities from serverStore.props - modelsStore->>modelsStore: modelPropsCache.set(model.id, serverStore.props) - modelsStore->>modelsStore: models[0].modalities = props.modalities - - modelsStore->>modelsStore: Auto-select single model - Note right of modelsStore: selectedModelId = models[0].id - modelsStore->>modelsStore: loading = false - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🚀 INITIALIZATION (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>modelsStore: fetch() - activate modelsStore - modelsStore->>ModelsSvc: list() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: ApiModelListResponse - modelsStore->>modelsStore: models = $state(mapped) - deactivate modelsStore - - Note over UI: After models loaded, layout triggers: - UI->>modelsStore: fetchRouterModels() - activate modelsStore - modelsStore->>ModelsSvc: listRouter() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: ApiRouterModelsListResponse - Note right of API: {data: [{id, status, path, in_cache}]} - modelsStore->>modelsStore: routerModels = $state(data) - - modelsStore->>modelsStore: fetchModalitiesForLoadedModels() - loop each model where status === "loaded" - modelsStore->>PropsSvc: fetchForModel(modelId) - PropsSvc->>API: GET /props?model={modelId} - API-->>PropsSvc: ApiLlamaCppServerProps - modelsStore->>modelsStore: modelPropsCache.set(modelId, props) - end - modelsStore->>modelsStore: propsCacheVersion++ - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🔄 MODEL SELECTION (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>Hooks: useModelChangeValidation({getRequiredModalities, onSuccess?, onValidationFailure?}) - Note over Hooks: Hook configured per-component:
ChatForm: getRequiredModalities = usedModalities
ChatMessage: getRequiredModalities = getModalitiesUpToMessage(msgId) - - UI->>Hooks: handleModelChange(modelId, modelName) - activate Hooks - Hooks->>Hooks: previousSelectedModelId = modelsStore.selectedModelId - Hooks->>modelsStore: isModelLoaded(modelName)? - - alt model NOT loaded - Hooks->>modelsStore: loadModel(modelName) - Note over modelsStore: → see LOAD MODEL section below - end - - Note over Hooks: Always fetch props (from cache or API) - Hooks->>modelsStore: fetchModelProps(modelName) - modelsStore-->>Hooks: props - - Hooks->>convStore: getRequiredModalities() - convStore-->>Hooks: {vision, audio} - - Hooks->>Hooks: Validate: model.modalities ⊇ required? - - alt validation PASSED - Hooks->>modelsStore: selectModelById(modelId) - Hooks-->>UI: return true - else validation FAILED - Hooks->>UI: toast.error("Model doesn't support required modalities") - alt model was just loaded - Hooks->>modelsStore: unloadModel(modelName) - end - alt onValidationFailure provided - Hooks->>modelsStore: selectModelById(previousSelectedModelId) - end - Hooks-->>UI: return false - end - deactivate Hooks - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ⬆️ LOAD MODEL (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - modelsStore->>modelsStore: loadModel(modelId) - activate modelsStore - - alt already loaded - modelsStore-->>modelsStore: return (no-op) - end - - modelsStore->>modelsStore: modelLoadingStates.set(modelId, true) - modelsStore->>ModelsSvc: load(modelId) - ModelsSvc->>API: POST /models/load {model: modelId} - API-->>ModelsSvc: {status: "loading"} - - modelsStore->>modelsStore: pollForModelStatus(modelId, LOADED) - loop poll every 500ms (max 60 attempts) - modelsStore->>modelsStore: fetchRouterModels() - modelsStore->>ModelsSvc: listRouter() - ModelsSvc->>API: GET /v1/models - API-->>ModelsSvc: models[] - modelsStore->>modelsStore: getModelStatus(modelId) - alt status === LOADED - Note right of modelsStore: break loop - else status === LOADING - Note right of modelsStore: wait 500ms, continue - end - end - - modelsStore->>modelsStore: updateModelModalities(modelId) - modelsStore->>PropsSvc: fetchForModel(modelId) - PropsSvc->>API: GET /props?model={modelId} - API-->>PropsSvc: props with modalities - modelsStore->>modelsStore: modelPropsCache.set(modelId, props) - modelsStore->>modelsStore: propsCacheVersion++ - - modelsStore->>modelsStore: modelLoadingStates.set(modelId, false) - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ⬇️ UNLOAD MODEL (ROUTER mode) - %% ═══════════════════════════════════════════════════════════════════════════ - - modelsStore->>modelsStore: unloadModel(modelId) - activate modelsStore - modelsStore->>modelsStore: modelLoadingStates.set(modelId, true) - modelsStore->>ModelsSvc: unload(modelId) - ModelsSvc->>API: POST /models/unload {model: modelId} - - modelsStore->>modelsStore: pollForModelStatus(modelId, UNLOADED) - loop poll until unloaded - modelsStore->>ModelsSvc: listRouter() - ModelsSvc->>API: GET /v1/models - end - - modelsStore->>modelsStore: modelLoadingStates.set(modelId, false) - deactivate modelsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 📊 COMPUTED GETTERS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over modelsStore: Getters:
- selectedModel: ModelOption | null
- loadedModelIds: string[] (from routerModels)
- loadingModelIds: string[] (from modelLoadingStates)
- singleModelName: string | null (MODEL mode only) - - Note over modelsStore: Modality helpers:
- getModelModalities(modelId): {vision, audio}
- modelSupportsVision(modelId): boolean
- modelSupportsAudio(modelId): boolean -``` diff --git a/tools/ui/docs/flows/server-flow.md b/tools/ui/docs/flows/server-flow.md deleted file mode 100644 index d6a1611f6..000000000 --- a/tools/ui/docs/flows/server-flow.md +++ /dev/null @@ -1,76 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 +layout.svelte - participant serverStore as 🗄️ serverStore - participant PropsSvc as ⚙️ PropsService - participant API as 🌐 llama-server - - Note over serverStore: State:
props: ApiLlamaCppServerProps | null
loading, error
role: ServerRole | null (MODEL | ROUTER)
fetchPromise (deduplication) - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🚀 INITIALIZATION - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>serverStore: fetch() - activate serverStore - - alt fetchPromise exists (already fetching) - serverStore-->>UI: return fetchPromise - Note right of serverStore: Deduplicate concurrent calls - end - - serverStore->>serverStore: loading = true - serverStore->>serverStore: fetchPromise = new Promise() - - serverStore->>PropsSvc: fetch() - PropsSvc->>API: GET /props - API-->>PropsSvc: ApiLlamaCppServerProps - Note right of API: {role, model_path, model_alias,
modalities, default_generation_settings, ...} - - PropsSvc-->>serverStore: props - serverStore->>serverStore: props = $state(data) - - serverStore->>serverStore: detectRole(props) - Note right of serverStore: role = props.role === "router"
? ServerRole.ROUTER
: ServerRole.MODEL - - serverStore->>serverStore: loading = false - serverStore->>serverStore: fetchPromise = null - deactivate serverStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 📊 COMPUTED GETTERS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over serverStore: Getters from props: - - rect rgb(240, 255, 240) - Note over serverStore: defaultParams
→ props.default_generation_settings.params
(temperature, top_p, top_k, etc.) - end - - rect rgb(240, 255, 240) - Note over serverStore: contextSize
→ props.default_generation_settings.n_ctx - end - - rect rgb(255, 240, 240) - Note over serverStore: isRouterMode
→ role === ServerRole.ROUTER - end - - rect rgb(255, 240, 240) - Note over serverStore: isModelMode
→ role === ServerRole.MODEL - end - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: 🔗 RELATIONSHIPS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over serverStore: Used by: - Note right of serverStore: - modelsStore: role detection, MODEL mode modalities
- settingsStore: syncWithServerDefaults (defaultParams)
- chatStore: contextSize for processing state
- UI components: isRouterMode for conditional rendering - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,API: ❌ ERROR HANDLING - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over serverStore: getErrorMessage(): string | null
Returns formatted error for UI display - - Note over serverStore: clear(): void
Resets all state (props, error, loading, role) -``` diff --git a/tools/ui/docs/flows/settings-flow.md b/tools/ui/docs/flows/settings-flow.md deleted file mode 100644 index 260713a17..000000000 --- a/tools/ui/docs/flows/settings-flow.md +++ /dev/null @@ -1,156 +0,0 @@ -```mermaid -sequenceDiagram - participant UI as 🧩 ChatSettings - participant settingsStore as 🗄️ settingsStore - participant serverStore as 🗄️ serverStore - participant ParamSvc as ⚙️ ParameterSyncService - participant LS as 💾 LocalStorage - - Note over settingsStore: State:
config: SettingsConfigType
theme: string ("auto" | "light" | "dark")
isInitialized: boolean
userOverrides: Set<string> - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🚀 INITIALIZATION - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over settingsStore: Auto-initialized in constructor (browser only) - settingsStore->>settingsStore: initialize() - activate settingsStore - - settingsStore->>settingsStore: loadConfig() - settingsStore->>LS: get("llama-config") - LS-->>settingsStore: StoredConfig | null - - alt config exists - settingsStore->>settingsStore: Merge with SETTING_CONFIG_DEFAULT - Note right of settingsStore: Fill missing keys with defaults - else no config - settingsStore->>settingsStore: config = SETTING_CONFIG_DEFAULT - end - - settingsStore->>LS: get("llama-userOverrides") - LS-->>settingsStore: string[] | null - settingsStore->>settingsStore: userOverrides = new Set(data) - - settingsStore->>settingsStore: loadTheme() - settingsStore->>LS: get("llama-theme") - LS-->>settingsStore: theme | "auto" - - settingsStore->>settingsStore: isInitialized = true - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🔄 SYNC WITH SERVER DEFAULTS - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over UI: Triggered from +layout.svelte when serverStore.props loaded - UI->>settingsStore: syncWithServerDefaults() - activate settingsStore - - settingsStore->>serverStore: defaultParams - serverStore-->>settingsStore: {temperature, top_p, top_k, ...} - - loop each SYNCABLE_PARAMETER - alt key NOT in userOverrides - settingsStore->>settingsStore: config[key] = serverDefault[key] - Note right of settingsStore: Non-overridden params adopt server default - else key in userOverrides - Note right of settingsStore: Keep user value, skip server default - end - end - - alt serverStore.props has uiSettings - settingsStore->>settingsStore: Apply uiSettings from server - Note right of settingsStore: Server-provided UI settings
(e.g. showRawOutputSwitch) - end - - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: ⚙️ UPDATE CONFIG - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: updateConfig(key, value) - activate settingsStore - settingsStore->>settingsStore: config[key] = value - - alt value matches server default for key - settingsStore->>settingsStore: userOverrides.delete(key) - Note right of settingsStore: Matches server default, remove override - else value differs from server default - settingsStore->>settingsStore: userOverrides.add(key) - Note right of settingsStore: Mark as user-modified (won't be overwritten) - end - - settingsStore->>settingsStore: saveConfig() - settingsStore->>LS: set(CONFIG_LOCALSTORAGE_KEY, config) - settingsStore->>LS: set(USER_OVERRIDES_LOCALSTORAGE_KEY, [...userOverrides]) - deactivate settingsStore - - UI->>settingsStore: updateMultipleConfig({key1: val1, key2: val2}) - activate settingsStore - Note right of settingsStore: Batch update, single save - settingsStore->>settingsStore: For each key: config[key] = value - settingsStore->>settingsStore: For each key: userOverrides.add(key) - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🔄 RESET - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: resetConfig() - activate settingsStore - settingsStore->>settingsStore: config = {...SETTING_CONFIG_DEFAULT} - settingsStore->>settingsStore: userOverrides.clear() - Note right of settingsStore: All params reset to defaults
Next syncWithServerDefaults will adopt server values - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - UI->>settingsStore: resetParameterToServerDefault(key) - activate settingsStore - settingsStore->>settingsStore: userOverrides.delete(key) - settingsStore->>serverStore: defaultParams[key] - settingsStore->>settingsStore: config[key] = serverDefault - settingsStore->>settingsStore: saveConfig() - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 🎨 THEME - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: updateTheme(newTheme) - activate settingsStore - settingsStore->>settingsStore: theme = newTheme - settingsStore->>settingsStore: saveTheme() - settingsStore->>LS: set("llama-theme", theme) - deactivate settingsStore - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 📊 PARAMETER INFO - %% ═══════════════════════════════════════════════════════════════════════════ - - UI->>settingsStore: getParameterInfo(key) - settingsStore->>ParamSvc: getParameterInfo(key, config, serverDefaults, userOverrides) - ParamSvc-->>settingsStore: ParameterInfo - Note right of ParamSvc: {
currentValue,
serverDefault,
isUserOverride: boolean,
canSync: boolean,
isDifferentFromServer: boolean
} - - UI->>settingsStore: getParameterDiff() - settingsStore->>ParamSvc: createParameterDiff(config, serverDefaults, userOverrides) - ParamSvc-->>settingsStore: ParameterDiff[] - Note right of ParamSvc: Array of parameters where user != server - - %% ═══════════════════════════════════════════════════════════════════════════ - Note over UI,LS: 📋 CONFIG CATEGORIES - %% ═══════════════════════════════════════════════════════════════════════════ - - Note over settingsStore: Syncable with server (from /props): - rect rgb(240, 255, 240) - Note over settingsStore: temperature, top_p, top_k, min_p
repeat_penalty, presence_penalty, frequency_penalty
dynatemp_range, dynatemp_exponent
typ_p, xtc_probability, xtc_threshold
dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n - end - - Note over settingsStore: UI-only (not synced): - rect rgb(255, 240, 240) - Note over settingsStore: systemMessage, custom (JSON)
showStatistics, enableContinueGeneration
autoMicOnEmpty, disableAutoScroll
apiKey, pdfAsImage, disableReasoningParsing, showRawOutputSwitch - end -``` diff --git a/tools/ui/eslint.config.js b/tools/ui/eslint.config.js index b8bdb216e..6ad065f5a 100644 --- a/tools/ui/eslint.config.js +++ b/tools/ui/eslint.config.js @@ -12,6 +12,49 @@ import { fileURLToPath } from 'node:url'; import ts from 'typescript-eslint'; const gitignorePath = fileURLToPath(new URL('./.gitignore', import.meta.url)); +// Require a blank line between consecutive class accessors (get/set). The core +// `padding-line-between-statements` rule only handles statements, not class +// members, so this is enforced with a small custom rule. +const blankLineBetweenAccessors = { + create(context) { + return { + MethodDefinition(node) { + if (node.kind !== 'get' && node.kind !== 'set') return; + + const body = node.parent; + + if (!body || body.type !== 'ClassBody') return; + + const index = body.body.indexOf(node); + + if (index <= 0) return; + + const prev = body.body[index - 1]; + + if (prev.type !== 'MethodDefinition' || (prev.kind !== 'get' && prev.kind !== 'set')) + return; + + if (node.loc.start.line - prev.loc.end.line <= 1) { + context.report({ + fix(fixer) { + // Insert after the previous accessor's closing brace so the blank + // line keeps the current accessor's indentation. + return fixer.insertTextAfter(prev, '\n'); + }, + message: 'Expected a blank line between class accessors (get/set).', + node + }); + } + } + }; + }, + meta: { + docs: { description: 'Require a blank line between consecutive class accessors (get/set).' }, + fixable: 'whitespace', + schema: [], + type: 'layout' + } +}; export default ts.config( includeIgnoreFile(gitignorePath), @@ -22,7 +65,11 @@ export default ts.config( ...svelte.configs.prettier, { languageOptions: { globals: { ...globals.browser, ...globals.node } }, - plugins: { perfectionist, 'simple-import-sort': simpleImportSort }, + plugins: { + local: { rules: { 'blank-line-between-accessors': blankLineBetweenAccessors } }, + perfectionist, + 'simple-import-sort': simpleImportSort + }, rules: { // Snippet bodies often ignore one or more of the parent's params // (e.g. `{#snippet children(_meta, ctx)}` when only ctx is read). @@ -30,8 +77,11 @@ export default ts.config( 'error', { argsIgnorePattern: '^_', varsIgnorePattern: '^_' } ], + // Enforce empty line at end of file 'eol-last': 'error', + // Enforce a blank line between consecutive get/set accessors + 'local/blank-line-between-accessors': 'error', // typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects. // see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors 'no-undef': 'off', @@ -61,6 +111,38 @@ export default ts.config( { blankLine: 'always', next: ['return', 'throw', 'break', 'continue'], prev: '*' } ], + // Class member order: public fields -> private fields -> constructor -> getters + // -> setters -> public methods -> private methods, alphabetical within each. + // Svelte $derived fields must stay in dependency order (forward references are + // rejected), so the two stores that rely on that are exempted below. + 'perfectionist/sort-classes': [ + 'error', + { + customGroups: [ + { groupName: 'public-field', modifiers: ['public'], selector: 'property' }, + { groupName: 'private-field', modifiers: ['private'], selector: 'property' }, + { groupName: 'get-method', selector: 'get-method' }, + { groupName: 'set-method', selector: 'set-method' }, + { groupName: 'public-method', modifiers: ['public'], selector: 'method' }, + { groupName: 'private-method', modifiers: ['private'], selector: 'method' } + ], + groups: [ + 'public-field', + 'private-field', + 'constructor', + 'get-method', + 'set-method', + 'public-method', + 'private-method', + 'unknown' + ], + type: 'natural', + // Keep members in dependency order (Svelte rejects forward references in + // $derived fields), while still sorting the rest alphabetically. + useExperimentalDependencyDetection: true + } + ], + // Alphabetical order for enum members 'perfectionist/sort-enums': ['error', { type: 'natural' }], diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte index 8e8949172..304e5a600 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreview.svelte @@ -139,7 +139,7 @@ let fileSize = $derived(currentItem?.size ? formatFileSize(currentItem.size) : ''); let hasVisionModality = $derived( - currentItem && activeModelId ? modelsStore.modelSupportsVision(activeModelId) : false + currentItem && activeModelId ? modelsStore.props.modelSupportsVision(activeModelId) : false ); let audioSrc = $derived( diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index e937d2730..18061728a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -28,7 +28,6 @@ import { chatStore, conversationsStore, - mcpResourceStore, mcpStore, modelsStore, serverStore, @@ -140,7 +139,9 @@ // float above the box. let mentionAnchor: HTMLDivElement | null = $state(null); - let cwd = $derived(conversationsStore.activeConversation?.cwd ?? conversationsStore.pendingCwd); + let cwd = $derived( + conversationsStore.activeConversation?.cwd ?? conversationsStore.preferences.pendingCwd + ); const pickers = useChatFormPickers({ focusInput: refocusInput, @@ -151,7 +152,8 @@ getShowModelSelector: () => showModelSelector, getValue: () => value, hasCwdTools: () => toolsStore.hasEnabledCwdTools, - hasPrompts: () => mcpStore.hasPromptsCapability(conversationsStore.getAllMcpServerOverrides()), + hasPrompts: () => + mcpStore.hasPromptsCapability(conversationsStore.preferences.getAllMcpServerOverrides()), openModelSelector: () => chatFormActionsRef?.openModelSelector(), setCaretOffset: (offset) => inputRef?.setCaretOffset(offset), setValue: (v) => { @@ -170,7 +172,7 @@ onValueChange?.(''); } - await conversationsStore.setCwd(newDir); + await conversationsStore.preferences.setCwd(newDir); if (conversationsStore.activeConversation) { await chatStore.recordCwdChange(newDir?.trim() || null); @@ -595,7 +597,7 @@ {useRichInput} /> - {#if mcpResourceStore.hasAttachments} + {#if mcpStore.resources.hasAttachments} { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte index 3d04d14cb..92f5b9349 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte @@ -38,11 +38,11 @@ } function isServerEnabledForChat(serverId: string): boolean { - return conversationsStore.isMcpServerEnabledForChat(serverId); + return conversationsStore.preferences.isMcpServerEnabledForChat(serverId); } async function toggleServerForChat(serverId: string) { - await conversationsStore.toggleMcpServerForChat(serverId); + await conversationsStore.preferences.toggleMcpServerForChat(serverId); } function handleMcpSubMenuOpen(open: boolean) { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte index 63a8c267d..2e61bb07d 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte @@ -218,12 +218,15 @@ {@const hasError = healthState.status === HealthCheckStatus.ERROR} {@const displayName = mcpStore.getServerLabel(server)} {@const faviconUrl = mcpStore.getServerFavicon(server.id)} - {@const isEnabled = conversationsStore.isMcpServerEnabledForChat(server.id)} + {@const isEnabled = conversationsStore.preferences.isMcpServerEnabledForChat( + server.id + )} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte index 690e13dfa..f2f902881 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte @@ -81,10 +81,10 @@ $effect(() => { if (activeModelId) { - const cached = modelsStore.getModelProps(activeModelId); + const cached = modelsStore.props.getModelProps(activeModelId); if (!cached) { - modelsStore.fetchModelProps(activeModelId).then(() => { + modelsStore.props.fetchModelProps(activeModelId).then(() => { modelPropsVersion++; }); } @@ -94,19 +94,21 @@ $effect(() => { void modelPropsVersion; - hasAudioModality = activeModelId ? modelsStore.modelSupportsAudio(activeModelId) : false; + hasAudioModality = activeModelId ? modelsStore.props.modelSupportsAudio(activeModelId) : false; }); $effect(() => { void modelPropsVersion; - hasVideoModality = activeModelId ? modelsStore.modelSupportsVideo(activeModelId) : false; + hasVideoModality = activeModelId ? modelsStore.props.modelSupportsVideo(activeModelId) : false; }); $effect(() => { void modelPropsVersion; - hasVisionModality = activeModelId ? modelsStore.modelSupportsVision(activeModelId) : false; + hasVisionModality = activeModelId + ? modelsStore.props.modelSupportsVision(activeModelId) + : false; }); $effect(() => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte index d8fad772d..118e54a0a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte @@ -58,13 +58,13 @@ let currentConfig = $derived(settingsStore.config); let hasMcpPromptsSupport = $derived.by(() => { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); return mcpStore.hasPromptsCapability(perChatOverrides); }); let hasMcpResourcesSupport = $derived.by(() => { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); return mcpStore.hasResourcesCapability(perChatOverrides); }); @@ -121,7 +121,7 @@ if (!chatStore.isLoading && !chatStore.isStreaming()) return false; - const processingState = chatStore.activeProcessingState; + const processingState = chatStore.processing.activeState; if (!processingState) return false; diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte index 7b071d99e..d6bad98dc 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte @@ -16,7 +16,7 @@ $effect(() => { const conv = conversationsStore.activeConversation; - untrack(() => chatStore.setActiveProcessingConversation(conv?.id ?? null)); + untrack(() => chatStore.processing.setActiveConversation(conv?.id ?? null)); }); $effect(() => { @@ -28,12 +28,12 @@ if (chatStore.isLoading || chatStore.isStreaming()) return; if (messages.length === 0) { - untrack(() => chatStore.clearProcessingState(conv.id)); + untrack(() => chatStore.processing.setState(conv.id, null)); return; } - untrack(() => chatStore.restoreProcessingStateFromMessages(messages, conv.id)); + untrack(() => chatStore.processing.restoreFromMessages(messages, conv.id)); }); $effect(() => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte index 3f178da18..452fd7a68 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte @@ -3,7 +3,7 @@ ChatAttachmentsListItemMcpResource, HorizontalScrollCarousel } from '$lib/components/app'; - import { mcpResourceStore, mcpStore } from '$lib/stores'; + import { mcpStore } from '$lib/stores'; interface Props { class?: string; @@ -12,8 +12,8 @@ let { class: className, onResourceClick }: Props = $props(); - const attachments = $derived(mcpResourceStore.attachments); - const hasAttachments = $derived(mcpResourceStore.hasAttachments); + const attachments = $derived(mcpStore.resources.attachments); + const hasAttachments = $derived(mcpStore.resources.hasAttachments); function handleRemove(attachmentId: string) { mcpStore.removeResourceAttachment(attachmentId); diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte index 9b5a57b9b..9a5c3e747 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte @@ -87,7 +87,7 @@ isLoading = true; try { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); const initialized = await mcpStore.ensureInitialized(perChatOverrides); if (!initialized) { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte index b92be9fbd..c92af719a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte @@ -59,7 +59,7 @@ message.model ?? chatStore.getResumeModel(message.convId) ?? modelsStore.selectedModelName ); let modelLoadProgress = $derived( - isRouter && loadTargetModel ? modelsStore.getLoadProgress(loadTargetModel) : null + isRouter && loadTargetModel ? modelsStore.status.getLoadProgress(loadTargetModel) : null ); let modelLoadingText = $derived(modelLoadProgressText(modelLoadProgress)); diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte index d3fb33a00..c5b80f156 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantModel.svelte @@ -31,7 +31,7 @@ pendingModel = modelId; try { - await modelsStore.loadModel(modelId); + await modelsStore.status.load(modelId); } finally { pendingModel = null; } diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte index 584979979..7a21c6766 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte @@ -43,14 +43,14 @@ ); const hasReasoningError = $derived( - isLastAssistantMessage ? !!agenticStore.lastError(message.convId) : false + isLastAssistantMessage ? !!agenticStore.getLastError(message.convId) : false ); let permissionDismissed = $state(false); const pendingPermission = $derived( isStreaming && isLastAssistantMessage - ? agenticStore.pendingPermissionRequest(message.convId) + ? agenticStore.getPendingPermissionRequest(message.convId) : null ); @@ -74,7 +74,7 @@ const pendingContinue = $derived( isStreaming && isLastAssistantMessage - ? agenticStore.pendingContinueRequest(message.convId) + ? agenticStore.getPendingContinueRequest(message.convId) : false ); @@ -97,7 +97,7 @@ const sections = $derived(deriveAgenticSections(message, toolMessages, [], isStreaming)); const currentlyExecutingToolCallId = $derived( - isStreaming ? agenticStore.executingToolCallId(message.convId) : null + isStreaming ? agenticStore.getExecutingToolCallId(message.convId) : null ); type TurnGroup = { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte index 2a8f45ba5..c32c66d91 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte @@ -238,30 +238,30 @@ /> {/each} - {#if conversationsStore.activeConversation && agenticStore.pendingSteeringMessageContent(conversationsStore.activeConversation!.id)} + {#if conversationsStore.activeConversation && agenticStore.getPendingSteeringMessageContent(conversationsStore.activeConversation!.id)} {@const convId = conversationsStore.activeConversation!.id} - {@const pendingContent = agenticStore.pendingSteeringMessageContent(convId)} + {@const pendingContent = agenticStore.getPendingSteeringMessageContent(convId)} {#if pendingContent} chatStore.abortCurrentFlow(convId)} onEdit={(newContent, extras) => agenticStore.injectSteeringMessage(convId, newContent, extras)} onDelete={() => agenticStore.clearSteeringMessage(convId)} /> {/if} - {:else if conversationsStore.activeConversation && chatStore.pendingMessageContent(conversationsStore.activeConversation!.id)} + {:else if conversationsStore.activeConversation && chatStore.getPendingMessageContent(conversationsStore.activeConversation!.id)} {@const convId = conversationsStore.activeConversation!.id} - {@const pendingContent = chatStore.pendingMessageContent(convId)} + {@const pendingContent = chatStore.getPendingMessageContent(convId)} {#if pendingContent} chatStore.abortCurrentFlow(convId)} onEdit={(newContent, extras) => chatStore.injectPendingMessage(convId, newContent, extras)} onDelete={() => chatStore.clearPendingMessage(convId)} diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte index 1ddad694b..c48dcb38c 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpResourcesBrowser.svelte @@ -8,7 +8,7 @@ import { Button } from '$lib/components/ui/button'; import * as Dialog from '$lib/components/ui/dialog'; import { ICON_CLASS_DEFAULT } from '$lib/constants'; - import { conversationsStore, mcpResourceStore, mcpStore } from '$lib/stores'; + import { conversationsStore, mcpStore } from '$lib/stores'; import type { MCPResourceContent, MCPResourceInfo, MCPResourceTemplateInfo } from '$lib/types'; import { getResourceDisplayName } from '$lib/utils'; import { SvelteSet } from 'svelte/reactivity'; @@ -33,7 +33,7 @@ let templatePreviewLoading = $state(false); let templatePreviewError = $state(null); - const totalCount = $derived(mcpResourceStore.totalResourceCount); + const totalCount = $derived(mcpStore.resources.totalResourceCount); $effect(() => { if (open) { @@ -48,7 +48,7 @@ }); async function loadResources() { - const perChatOverrides = conversationsStore.getAllMcpServerOverrides(); + const perChatOverrides = conversationsStore.preferences.getAllMcpServerOverrides(); const initialized = await mcpStore.ensureInitialized(perChatOverrides); if (initialized) { @@ -126,16 +126,16 @@ isAttaching = true; try { - const knownResource = mcpResourceStore.findResourceByUri(templatePreviewUri); + const knownResource = mcpStore.resources.findResourceByUri(templatePreviewUri); if (knownResource) { - if (!mcpResourceStore.isAttached(knownResource.uri)) { + if (!mcpStore.resources.isAttached(knownResource.uri)) { await mcpStore.attachResource(knownResource.uri); } toast.success(`Resource attached: ${knownResource.title || knownResource.name}`); } else { - if (mcpResourceStore.isAttached(templatePreviewUri)) { + if (mcpStore.resources.isAttached(templatePreviewUri)) { toast.info('Resource already attached'); handleOpenChange(false); @@ -147,9 +147,9 @@ serverName: selectedTemplate.serverName, uri: templatePreviewUri }; - const attachment = mcpResourceStore.addAttachment(resourceInfo); + const attachment = mcpStore.resources.addAttachment(resourceInfo); - mcpResourceStore.updateAttachmentContent(attachment.id, templatePreviewContent); + mcpStore.resources.updateAttachmentContent(attachment.id, templatePreviewContent); toast.success(`Resource attached: ${resourceInfo.name}`); } @@ -199,7 +199,7 @@ function getAllResourcesFlatInTreeOrder(): MCPResourceInfo[] { const allResources: MCPResourceInfo[] = []; - const resourcesMap = mcpResourceStore.serverResources; + const resourcesMap = mcpStore.resources.serverResources; for (const [serverName, serverRes] of resourcesMap.entries()) { for (const resource of serverRes.resources) { diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte index 9123dcef9..a339c6a42 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte @@ -234,7 +234,7 @@ useProxy: newServerUseProxy }); - conversationsStore.setMcpServerOverride(newServerId, true); + conversationsStore.preferences.setMcpServerOverride(newServerId, true); handleOpenChange(false); } diff --git a/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte b/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte index 61155fceb..fb7498870 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogModelInformation.svelte @@ -42,7 +42,7 @@ let modalities = $derived.by(() => { if (!firstModel?.id) return []; - return modelsStore.getModelModalitiesArray(firstModel.id); + return modelsStore.props.getModelModalitiesArray(firstModel.id); }); // Ensure models are fetched when dialog opens @@ -56,7 +56,7 @@ $effect(() => { if (open && isRouter && modelId) { isLoadingRouterProps = true; - modelsStore + modelsStore.props .fetchModelProps(modelId) .then((props) => { routerModelProps = props; diff --git a/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte b/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte index 301c39699..a8772aa1c 100644 --- a/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpActiveServersAvatars.svelte @@ -14,7 +14,9 @@ let mcpServers = $derived(mcpStore.getServers().filter((s) => s.enabled)); let enabledMcpServersForChat = $derived( - mcpServers.filter((s) => conversationsStore.isMcpServerEnabledForChat(s.id) && s.url.trim()) + mcpServers.filter( + (s) => conversationsStore.preferences.isMcpServerEnabledForChat(s.id) && s.url.trim() + ) ); let healthyEnabledMcpServers = $derived( enabledMcpServersForChat.filter((s) => { diff --git a/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte b/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte index 18e974653..c8cf8bbbc 100644 --- a/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpResourcesBrowser/McpResourcesBrowser.svelte @@ -2,7 +2,7 @@ import McpResourcesBrowserEmptyState from './McpResourcesBrowserEmptyState.svelte'; import McpResourcesBrowserHeader from './McpResourcesBrowserHeader.svelte'; import McpResourcesBrowserServerItem from './McpResourcesBrowserServerItem.svelte'; - import { mcpResourceStore, mcpStore } from '$lib/stores'; + import { mcpStore } from '$lib/stores'; import type { MCPResourceInfo, MCPResourceTemplateInfo, MCPServerResources } from '$lib/types'; import { parseResourcePath } from '$lib/utils'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -31,8 +31,8 @@ let expandedFolders = new SvelteSet(); let searchQuery = $state(''); - const resources = $derived(mcpResourceStore.serverResources); - const isLoading = $derived(mcpResourceStore.isLoading); + const resources = $derived(mcpStore.resources.serverResources); + const isLoading = $derived(mcpStore.resources.isLoading); const filteredResources = $derived.by(() => { if (!searchQuery.trim()) { diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte index c23bca1ae..1cd56c124 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte @@ -116,7 +116,7 @@ if (status === ServerModelStatus.LOADING) return; - await modelsStore.unloadModel(modelId); + await modelsStore.status.unload(modelId); } export function open() { @@ -174,9 +174,9 @@ {@const triggerLoading = !!triggerModel && (triggerStatus === ServerModelStatus.LOADING || - modelsStore.isModelOperationInProgress(triggerModel))} + modelsStore.status.isOperationInProgress(triggerModel))} {@const triggerLoadPercent = triggerLoading - ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100) + ? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100) : 0} {#if ms.isRouter} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte index 18c885a62..acdb36bca 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte @@ -47,7 +47,7 @@ return (model?.status?.value as ServerModelStatus) ?? null; }); - let isOperationInProgress = $derived(modelsStore.isModelOperationInProgress(option.model)); + let isOperationInProgress = $derived(modelsStore.status.isOperationInProgress(option.model)); let isFailed = $derived(serverStatus === ServerModelStatus.FAILED); let isSleeping = $derived(serverStatus === ServerModelStatus.SLEEPING); let isLoaded = $derived( @@ -55,7 +55,7 @@ ); let isLoading = $derived(serverStatus === ServerModelStatus.LOADING || isOperationInProgress); - let loadProgress = $derived(isLoading ? modelsStore.getLoadProgress(option.model) : null); + let loadProgress = $derived(isLoading ? modelsStore.status.getLoadProgress(option.model) : null); let loadPercent = $derived(Math.round(modelLoadFraction(loadProgress) * 100)); let loadTitle = $derived(modelLoadProgressText(loadProgress)); @@ -138,7 +138,7 @@ icon={RotateCw} tooltip="Retry loading model" class="h-3 w-3 text-red-500 hover:text-foreground" - onclick={() => modelsStore.loadModel(option.model)} + onclick={() => modelsStore.status.load(option.model)} stopPropagationOnClick /> @@ -157,7 +157,7 @@ class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-amber-500 [@media(pointer:coarse)]:hover:text-amber-600" onclick={(e) => { e?.stopPropagation(); - modelsStore.unloadModel(option.model); + modelsStore.status.unload(option.model); }} /> @@ -174,7 +174,7 @@ icon={PowerOff} tooltip="Unload model" class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-green-500 [@media(pointer:coarse)]:hover:text-green-600" - onclick={() => modelsStore.unloadModel(option.model)} + onclick={() => modelsStore.status.unload(option.model)} stopPropagationOnClick /> @@ -191,7 +191,7 @@ icon={Power} tooltip="Load model" class="h-3 w-3 [@media(pointer:coarse)]:text-muted-foreground" - onclick={() => modelsStore.loadModel(option.model)} + onclick={() => modelsStore.status.load(option.model)} stopPropagationOnClick /> diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte index 7228a2e74..0d10dd106 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte @@ -72,9 +72,9 @@ {@const triggerLoading = !!triggerModel && (triggerStatus === ServerModelStatus.LOADING || - modelsStore.isModelOperationInProgress(triggerModel))} + modelsStore.status.isOperationInProgress(triggerModel))} {@const triggerLoadPercent = triggerLoading - ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100) + ? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100) : 0} {#if ms.isRouter} diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index c8b2c814c..97ff30ba7 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -15,7 +15,7 @@ NUMERIC_FIELDS, POSITIVE_INTEGER_FIELDS, SETTINGS_CHAT_SECTIONS, - SETTINGS_SECTION_TITLES + SETTINGS_SECTION_SLUGS } from '$lib/constants'; import { ColorMode } from '$lib/enums/ui.enums'; import { RouterService } from '$lib/services/router.service'; @@ -46,13 +46,13 @@ let fetchInitiated = false; $effect(() => { - if (serverStore.isRouterMode && currentSection.fields && !fetchInitiated) { + if (serverStore.isRouterMode && currentSection.fields?.length && !fetchInitiated) { fetchInitiated = true; void modelsStore .fetch() .then(() => modelsStore.fetchRouterModels()) - .then(() => modelsStore.fetchModalitiesForLoadedModels()) + .then(() => modelsStore.props.fetchModalitiesForLoadedModels()) .then(() => modelsStore.ensureFirstModelSelected()); } }); @@ -148,9 +148,9 @@

{currentSection.title}

- {#if currentSection.title === SETTINGS_SECTION_TITLES.TOOLS} + {#if currentSection.slug === SETTINGS_SECTION_SLUGS.TOOLS} - {:else if currentSection.title === SETTINGS_SECTION_TITLES.IMPORT_EXPORT} + {:else if currentSection.slug === SETTINGS_SECTION_SLUGS.IMPORT_EXPORT} {:else if currentSection.fields}
@@ -161,7 +161,7 @@ onThemeChange={handleThemeChange} /> - {#if currentSection.title === SETTINGS_SECTION_TITLES.GENERAL} + {#if currentSection.slug === SETTINGS_SECTION_SLUGS.GENERAL}