diff --git a/common/chat.cpp b/common/chat.cpp index 21c82ce30..28fba2ff7 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -2391,6 +2391,23 @@ static void func_args_not_string(json & messages) { } } +// Trim leading/trailing whitespace from message contents before rendering. This +// has to run on the messages (not on the rendered JSON) because templates with +// string-only content caps concatenate typed content parts into a single string +// during rendering, after which the per-part whitespace can no longer be reached. +// Both the plain string content and the text of typed content parts are trimmed. +static void trim_all_content(std::vector & messages) { + for (auto & message : messages) { + message.content = trim_whitespace(message.content); + message.reasoning_content = trim_whitespace(message.reasoning_content); + for (auto & part : message.content_parts) { + if (part.type == "text") { + part.text = trim_whitespace(part.text); + } + } + } +} + } // MiniCPM5 format: @@ -2647,7 +2664,16 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ params.tools.is_array() && tmpls->template_tool_use ? *tmpls->template_tool_use : *tmpls->template_default; const auto & src = tmpl.source(); const auto & caps = tmpl.original_caps(); - params.messages = render_message_to_json(inputs.messages, tmpl.original_caps()); + std::vector trimmed_messages; + const std::vector * messages_to_render = &inputs.messages; + if (src.find("You have access to the following functions in JSONSchema format") != std::string::npos) { + // StepFun: trim message contents (including typed content parts) before rendering, + // otherwise leftover whitespace drives the model into reasoning loops (issue #24181) + trimmed_messages = inputs.messages; + workaround::trim_all_content(trimmed_messages); + messages_to_render = &trimmed_messages; + } + params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps()); params.tool_choice = inputs.tool_choice; params.reasoning_format = inputs.reasoning_format; params.enable_thinking = inputs.enable_thinking; diff --git a/common/speculative.cpp b/common/speculative.cpp index f81660682..4e0439216 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -955,10 +955,11 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n); // DFlash input is [id_last, * (block_size-1)], so it can draft at most block_size-1 tokens per step - if (this->params.n_max > block_size - 1) { - LOG_WRN("%s: requested draft size %d exceeds the trained DFlash block size %d -- clamping to %d draft tokens per step\n", - __func__, this->params.n_max, block_size - 1, block_size - 1); - this->params.n_max = block_size - 1; + if (this->params.n_max > block_size - 1 || this->params.n_min > block_size - 1) { + LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained DFlash block size %d -- clamping to %d\n", + __func__, this->params.n_max, this->params.n_min, block_size, block_size - 1); + this->params.n_max = std::min(this->params.n_max, block_size - 1); + this->params.n_min = std::min(this->params.n_min, block_size - 1); } batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq); @@ -968,7 +969,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { for (auto & s : smpls) { common_params_sampling sparams; sparams.no_perf = false; - sparams.top_k = 1; + sparams.top_k = 10; sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K }; s.reset(common_sampler_init(model_dft, sparams)); } @@ -1173,10 +1174,18 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { const llama_token id = cur_p->data[0].id; + if (cur_p->data[0].p < params.p_min) { + break; + } + common_sampler_accept(smpl, id, true); result.push_back(id); } + + if (result.size() < (size_t) params.n_min) { + result.clear(); + } } } diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 2c921f424..168f1a89d 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -1913,7 +1913,11 @@ static void ggml_compute_forward_concat_any( GGML_ASSERT(dim >= 0 && dim < 4); int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; + if (dim == 0) { + o[dim] = src0->ne[dim]/ggml_blck_size(src0->type); + } else { + o[dim] = src0->ne[dim]; + } const char * x; @@ -1921,8 +1925,8 @@ static void ggml_compute_forward_concat_any( for (int i3 = 0; i3 < ne3; i3++) { for (int i2 = ith; i2 < ne2; i2 += nth) { for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { + for (int i0 = 0; i0 < ne0/ggml_blck_size(dst->type); i0++) { + if (i0 < ne00/ggml_blck_size(src0->type) && i1 < ne01 && i2 < ne02 && i3 < ne03) { x = (const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03; } else { x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13; @@ -2071,6 +2075,14 @@ void ggml_compute_forward_concat( ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + if (ggml_is_quantized(src0->type)) { + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); + GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0); + GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0); + } switch (src0->type) { case GGML_TYPE_F16: diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index a547360eb..1b431a724 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -10,6 +10,7 @@ gated_delta_net_cuda(const float * q, const float * beta, const float * curr_state, float * dst, + float * state, int64_t H, int64_t n_tokens, int64_t n_seqs, @@ -25,6 +26,7 @@ gated_delta_net_cuda(const float * q, const uint3 neqk1_magic, const uint3 rq3_magic, float scale, + int64_t state_slot_stride, int K) { const uint32_t h_idx = blockIdx.x; const uint32_t sequence = blockIdx.y; @@ -35,9 +37,7 @@ gated_delta_net_cuda(const float * q, const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic); const uint32_t iq3 = fastdiv(sequence, rq3_magic); - const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs; float * attn_data = dst; - float * state = dst + attn_score_elems; // input state holds s0 only: [S_v, S_v, H, n_seqs] — seq stride is D = H * S_v * S_v. // output state layout (per-slot D * n_seqs) — same per-(seq,head) offset as before. @@ -145,10 +145,9 @@ gated_delta_net_cuda(const float * q, if constexpr (keep_rs_t) { // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. // When n_tokens < K only slots 0..n_tokens-1 are written; older slots are caller-owned. - const int64_t state_size_per_token = S_v * S_v * H * n_seqs; // per-slot stride in output const int target_slot = (int) n_tokens - 1 - t; if (target_slot >= 0 && target_slot < K) { - float * curr_state = (dst + attn_score_elems) + target_slot * state_size_per_token + state_out_offset; + float * curr_state = state + target_slot * state_slot_stride; #pragma unroll for (int r = 0; r < rows_per_lane; r++) { const int i = r * warp_size + lane; @@ -171,13 +170,13 @@ template static void launch_gated_delta_net( const float * q_d, const float * k_d, const float * v_d, const float * g_d, const float * b_d, const float * s_d, - float * dst_d, + float * dst_d, float * state_d, int64_t S_v, int64_t H, int64_t n_tokens, int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, int64_t sv2, int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, int64_t rq3, - float scale, int K, cudaStream_t stream) { + float scale, int64_t state_slot_stride, int K, cudaStream_t stream) { //TODO: Add chunked kernel for even faster pre-fill const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; const int num_warps = 4; @@ -187,34 +186,32 @@ static void launch_gated_delta_net( const uint3 neqk1_magic = init_fastdiv_values(neqk1); const uint3 rq3_magic = init_fastdiv_values(rq3); - int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); switch (S_v) { case 16: ggml_cuda_kernel_launch(gated_delta_net_cuda<16, KDA, keep_rs_t>, launch_params, - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, K); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; case 32: ggml_cuda_kernel_launch(gated_delta_net_cuda<32, KDA, keep_rs_t>, launch_params, - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, K); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; case 64: { ggml_cuda_kernel_launch(gated_delta_net_cuda<64, KDA, keep_rs_t>, launch_params, - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, K); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; } case 128: { ggml_cuda_kernel_launch(gated_delta_net_cuda<128, KDA, keep_rs_t>, launch_params, - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, K); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; } default: @@ -223,7 +220,8 @@ static void launch_gated_delta_net( } } -void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { +static void ggml_cuda_op_gated_delta_net_impl( + ggml_backend_cuda_context & ctx, ggml_tensor * dst, const ggml_cuda_gated_delta_net_fused_cache * cache) { ggml_tensor * src_q = dst->src[0]; ggml_tensor * src_k = dst->src[1]; ggml_tensor * src_v = dst->src[2]; @@ -288,25 +286,42 @@ void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * const int K = ggml_get_op_params_i32(dst, 0); const bool keep_rs = K > 1; + // recurrent state -> gdn_out tail (after attention scores), or the cache when fusing + float * state_d = dst_d + S_v * H * n_tokens * n_seqs; + int64_t state_slot_stride = S_v * S_v * H * n_seqs; + if (cache != nullptr) { + state_d = cache->data; + state_slot_stride = cache->slot_stride; + } + if (kda) { if (keep_rs) { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); } else { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); } } else { if (keep_rs) { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); } else { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); } } } + +void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + ggml_cuda_op_gated_delta_net_impl(ctx, dst, nullptr); +} + +void ggml_cuda_op_gated_delta_net_fused_cache( + ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_cuda_gated_delta_net_fused_cache cache) { + ggml_cuda_op_gated_delta_net_impl(ctx, dst, &cache); +} diff --git a/ggml/src/ggml-cuda/gated_delta_net.cuh b/ggml/src/ggml-cuda/gated_delta_net.cuh index 7375e81c0..f9bf43706 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cuh +++ b/ggml/src/ggml-cuda/gated_delta_net.cuh @@ -1,4 +1,14 @@ #include "common.cuh" #include "ggml.h" +// fused-kernel recurrent-state output; strides in elements (per-seq stride is always D, set in-kernel) +struct ggml_cuda_gated_delta_net_fused_cache { + float * data; // rollback slot 0 + int64_t slot_stride; // between rollback slots (0 when K==1) +}; + void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// same op, but writes the snapshot(s) into the cache instead of dst (see ggml_cuda_try_gdn_cache_fusion) +void ggml_cuda_op_gated_delta_net_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + ggml_cuda_gated_delta_net_fused_cache cache); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 42a7e1c52..c185ad5da 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3263,6 +3263,11 @@ static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { GGML_UNUSED(backend); } +static bool ggml_cuda_is_view_or_noop(const ggml_tensor * t) { + return ggml_is_empty(t) || t->op == GGML_OP_RESHAPE || t->op == GGML_OP_TRANSPOSE || + t->op == GGML_OP_VIEW || t->op == GGML_OP_PERMUTE || t->op == GGML_OP_NONE; +} + #ifdef USE_CUDA_GRAPH static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { @@ -3272,7 +3277,7 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_cuda_is_view_or_noop(node)) { continue; } @@ -3419,6 +3424,70 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope, return true; } +// match gated_delta_net + the strided cpy that scatters its state snapshots into the cache +// (slot i -> rollback group i, slot 0 newest), so the kernel can write them and skip the cpy. +static int ggml_cuda_try_gdn_cache_fusion( + const ggml_cgraph * cgraph, int node_idx, ggml_cuda_gated_delta_net_fused_cache & fused_state_cpy) { + const ggml_tensor * gdn = cgraph->nodes[node_idx]; + // the kernel skips the snapshot tail, so the gdn output must not be a graph output + if (gdn->op != GGML_OP_GATED_DELTA_NET || gdn->type != GGML_TYPE_F32 || + (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + + const ggml_tensor * src_v = gdn->src[2]; + const int64_t S_v = src_v->ne[0]; + const int64_t H = src_v->ne[1]; + const int64_t n_tokens = src_v->ne[2]; + const int64_t n_seqs = src_v->ne[3]; + const int64_t D = S_v * S_v * H; + const int64_t K = ggml_get_op_params_i32(gdn, 0); // snapshot slot count + const int64_t n_written = std::min(n_tokens, K); // newest n_written slots are written + + // snapshot tail starts right after the attention scores + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + // snapshot cpy is the first real node after the gdn (skip views/no-ops) + const ggml_tensor * cpy = nullptr; + int skip = 0; + for (int j = node_idx + 1; j < cgraph->n_nodes && cpy == nullptr; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_cuda_is_view_or_noop(n)) { + continue; + } + if (n->op != GGML_OP_CPY || (n->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + cpy = n; + skip = j - node_idx; + } + if (cpy == nullptr) { + return 0; + } + + const ggml_tensor * src = cpy->src[0]; // view of the gdn snapshot tail + const ggml_tensor * dst = cpy->src[1]; // cache view the kernel writes to + + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != gdn || src->view_offs != tail_off || + !ggml_is_contiguous(src)) { + return 0; + } + + // dst is the [D, n_seqs, n_written] cache view; require nb[1] == D (the per-seq stride the kernel + // assumes). ggml_cpy pins src to the same element count. + const std::array expected_ne = { D, n_seqs, n_written, 1 }; + if (dst->op != GGML_OP_VIEW || dst->type != GGML_TYPE_F32 || dst->data == nullptr || + !std::equal(expected_ne.begin(), expected_ne.end(), dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || dst->nb[1] != (size_t) ggml_row_size(GGML_TYPE_F32, D)) { + return 0; + } + + fused_state_cpy.data = (float *) dst->data; // rollback group 0 (newest) + fused_state_cpy.slot_stride = K > 1 ? (int64_t) (dst->nb[2] / sizeof(float)) : 0; + return skip; +} + static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) { args.sigmoid = false; args.softmax = false; @@ -3860,6 +3929,20 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache + if (node->op == GGML_OP_GATED_DELTA_NET) { + ggml_cuda_gated_delta_net_fused_cache fused_state_cpy; + const int nodes_to_skip = ggml_cuda_try_gdn_cache_fusion(cgraph, i, fused_state_cpy); + if (nodes_to_skip > 0) { +#ifdef GGML_CUDA_DEBUG + GGML_LOG_INFO("%s: fused gated_delta_net snapshot copies for %s (skipped %d nodes)\n", + __func__, node->name, nodes_to_skip); +#endif + ggml_cuda_op_gated_delta_net_fused_cache(*cuda_ctx, node, fused_state_cpy); + return nodes_to_skip; + } + } + //topk-moe if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || cgraph->nodes[i]->op == GGML_OP_ARGSORT) { @@ -4388,7 +4471,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud #endif prev_i = i; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_cuda_is_view_or_noop(node)) { continue; } diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu index c4253bfa4..c80394e31 100644 --- a/ggml/src/ggml-cuda/topk-moe.cu +++ b/ggml/src/ggml-cuda/topk-moe.cu @@ -312,6 +312,10 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx, ggml_cuda_kernel_launch(topk_moe_cuda<256, has_bias>, launch_params, logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; + case 288: // StepFun 3.7 + ggml_cuda_kernel_launch(topk_moe_cuda<288, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); + break; case 512: ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params, logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); @@ -377,8 +381,10 @@ bool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op, const ggml_tensor * weights, const ggml_tensor * logits, const ggml_tensor * ids) { + // must match an instantiation of launch_topk_moe_cuda: a power of 2 up to 512, + // or one of the non-power-of-2 expert counts of supported models const int n_expert = ids->nb[1] / ids->nb[0]; - if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 576) { + if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 288 && n_expert != 576) { return false; } diff --git a/include/llama.h b/include/llama.h index fccfbc7dc..0df6fdf3f 100644 --- a/include/llama.h +++ b/include/llama.h @@ -162,6 +162,9 @@ extern "C" { LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; + // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" + LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); + enum llama_rope_scaling_type { LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, LLAMA_ROPE_SCALING_TYPE_NONE = 0, @@ -609,6 +612,9 @@ extern "C" { // Get a string describing the model type LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size); + // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 + LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); + // Returns the total size of all the tensors in the model in bytes LLAMA_API uint64_t llama_model_size(const struct llama_model * model); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index d789210ea..598ff28eb 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -494,11 +494,11 @@ void llm_graph_input_attn_kv::set_input(const llama_ubatch * ubatch) { mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn); } - if (self_k_rot) { + if (self_k_rot && self_k_rot->buffer) { mctx->set_input_k_rot(self_k_rot); } - if (self_v_rot) { + if (self_v_rot && self_v_rot->buffer) { mctx->set_input_v_rot(self_v_rot); } } @@ -592,19 +592,19 @@ void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn); } - if (self_k_rot) { + if (self_k_rot && self_k_rot->buffer) { mctx->get_base()->set_input_k_rot(self_k_rot); } - if (self_v_rot) { + if (self_v_rot && self_v_rot->buffer) { mctx->get_base()->set_input_v_rot(self_v_rot); } - if (self_k_rot_swa) { + if (self_k_rot_swa && self_k_rot_swa->buffer) { mctx->get_swa()->set_input_k_rot(self_k_rot_swa); } - if (self_v_rot_swa) { + if (self_v_rot_swa && self_v_rot_swa->buffer) { mctx->get_swa()->set_input_v_rot(self_v_rot_swa); } } diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index a12719415..d05f8def0 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -27,52 +27,54 @@ const char * llama_file_version_name(llama_fver version) { return "unknown"; } -static std::string llama_model_ftype_name(llama_ftype ftype) { - if (ftype & LLAMA_FTYPE_GUESSED) { - return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)"; - } +#define LLAMA_FTYPE_PREFIX "(guessed) " - switch (ftype) { - case LLAMA_FTYPE_ALL_F32: return "all F32"; - case LLAMA_FTYPE_MOSTLY_F16: return "F16"; - case LLAMA_FTYPE_MOSTLY_BF16: return "BF16"; - case LLAMA_FTYPE_MOSTLY_Q1_0: return "Q1_0"; - case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0"; - case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1"; - case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0"; - case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1"; - case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0"; - case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return "MXFP4 MoE"; - case LLAMA_FTYPE_MOSTLY_NVFP4: return "NVFP4"; - case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large"; - case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K"; - case LLAMA_FTYPE_MOSTLY_TQ1_0: return "TQ1_0 - 1.69 bpw ternary"; - case LLAMA_FTYPE_MOSTLY_TQ2_0: return "TQ2_0 - 2.06 bpw ternary"; - case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw"; - - default: return "unknown, may not work"; +const char * llama_ftype_name(llama_ftype ftype) { + static constexpr size_t guessed_prefix_len = sizeof(LLAMA_FTYPE_PREFIX) - 1; + const char * name; + switch ((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) { + case LLAMA_FTYPE_ALL_F32: name = LLAMA_FTYPE_PREFIX "all F32"; break; + case LLAMA_FTYPE_MOSTLY_F16: name = LLAMA_FTYPE_PREFIX "F16"; break; + case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break; + case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break; + case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break; + case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break; + case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break; + case LLAMA_FTYPE_MOSTLY_Q5_1: name = LLAMA_FTYPE_PREFIX "Q5_1"; break; + case LLAMA_FTYPE_MOSTLY_Q8_0: name = LLAMA_FTYPE_PREFIX "Q8_0"; break; + case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: name = LLAMA_FTYPE_PREFIX "MXFP4 MoE"; break; + case LLAMA_FTYPE_MOSTLY_NVFP4: name = LLAMA_FTYPE_PREFIX "NVFP4"; break; + case LLAMA_FTYPE_MOSTLY_Q2_K: name = LLAMA_FTYPE_PREFIX "Q2_K - Medium"; break; + case LLAMA_FTYPE_MOSTLY_Q2_K_S: name = LLAMA_FTYPE_PREFIX "Q2_K - Small"; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_S: name = LLAMA_FTYPE_PREFIX "Q3_K - Small"; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_M: name = LLAMA_FTYPE_PREFIX "Q3_K - Medium"; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_L: name = LLAMA_FTYPE_PREFIX "Q3_K - Large"; break; + case LLAMA_FTYPE_MOSTLY_Q4_K_S: name = LLAMA_FTYPE_PREFIX "Q4_K - Small"; break; + case LLAMA_FTYPE_MOSTLY_Q4_K_M: name = LLAMA_FTYPE_PREFIX "Q4_K - Medium"; break; + case LLAMA_FTYPE_MOSTLY_Q5_K_S: name = LLAMA_FTYPE_PREFIX "Q5_K - Small"; break; + case LLAMA_FTYPE_MOSTLY_Q5_K_M: name = LLAMA_FTYPE_PREFIX "Q5_K - Medium"; break; + case LLAMA_FTYPE_MOSTLY_Q6_K: name = LLAMA_FTYPE_PREFIX "Q6_K"; break; + case LLAMA_FTYPE_MOSTLY_TQ1_0: name = LLAMA_FTYPE_PREFIX "TQ1_0 - 1.69 bpw ternary"; break; + case LLAMA_FTYPE_MOSTLY_TQ2_0: name = LLAMA_FTYPE_PREFIX "TQ2_0 - 2.06 bpw ternary"; break; + case LLAMA_FTYPE_MOSTLY_IQ2_XXS: name = LLAMA_FTYPE_PREFIX "IQ2_XXS - 2.0625 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ2_XS: name = LLAMA_FTYPE_PREFIX "IQ2_XS - 2.3125 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ2_S: name = LLAMA_FTYPE_PREFIX "IQ2_S - 2.5 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ2_M: name = LLAMA_FTYPE_PREFIX "IQ2_M - 2.7 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ3_XS: name = LLAMA_FTYPE_PREFIX "IQ3_XS - 3.3 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ3_XXS: name = LLAMA_FTYPE_PREFIX "IQ3_XXS - 3.0625 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ1_S: name = LLAMA_FTYPE_PREFIX "IQ1_S - 1.5625 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ1_M: name = LLAMA_FTYPE_PREFIX "IQ1_M - 1.75 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ4_NL: name = LLAMA_FTYPE_PREFIX "IQ4_NL - 4.5 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ4_XS: name = LLAMA_FTYPE_PREFIX "IQ4_XS - 4.25 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ3_S: name = LLAMA_FTYPE_PREFIX "IQ3_S - 3.4375 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ3_M: name = LLAMA_FTYPE_PREFIX "IQ3_S mix - 3.66 bpw"; break; + default: name = LLAMA_FTYPE_PREFIX "unknown, may not work"; break; } + return (ftype & LLAMA_FTYPE_GUESSED) ? name : name + guessed_prefix_len; } +#undef LLAMA_FTYPE_PREFIX + // return a list of splits for a given path // for example, given "-00002-of-00004.gguf", returns list of all 4 splits static std::vector llama_get_list_splits(const std::string & path, const int idx, const int n_split) { @@ -1711,17 +1713,12 @@ bool llama_model_loader::load_all_data( } std::string llama_model_loader::ftype_name() const { - return llama_model_ftype_name(ftype); + return llama_ftype_name(ftype); } void llama_model_loader::print_info() const { LLAMA_LOG_INFO("%s: file format = %s\n", __func__, llama_file_version_name(fver)); - //LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_model_ftype_name(ftype).c_str()); - if (n_elements <= 0) { - LLAMA_LOG_INFO("%s: file size = %.2f MiB\n", __func__, n_bytes/1024.0/1024.0); - LLAMA_LOG_INFO("%s: ERROR: n_elements is invalid, cannot compute BPW\n", __func__); - return; - } + LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_ftype_name(ftype)); if (n_bytes < GiB) { LLAMA_LOG_INFO("%s: file size = %.2f MiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0, n_bytes*8.0/n_elements); } else { diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 92429b23d..74f909f96 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1125,6 +1125,8 @@ struct llama_model::impl { std::string desc_str; + llama_ftype ftype = LLAMA_FTYPE_ALL_F32; + // model memory mapped files llama_mmaps mappings; @@ -1338,6 +1340,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { pimpl->desc_str = arch_name() + " " + type_name() + " " + ml.ftype_name(); + pimpl->ftype = ml.ftype; + if (hparams.f_max_alibi_bias > 0.0f) { hparams.use_alibi = true; } @@ -1784,6 +1788,10 @@ std::string llama_model::desc() const { return pimpl->desc_str; } +llama_ftype llama_model::ftype() const { + return pimpl->ftype; +} + size_t llama_model::size() const { return pimpl->n_bytes; } @@ -2754,6 +2762,10 @@ int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size) return snprintf(buf, buf_size, "%s", model->desc().c_str()); } +llama_ftype llama_model_ftype(const llama_model * model) { + return model->ftype(); +} + uint64_t llama_model_size(const llama_model * model) { return model->size(); } diff --git a/src/llama-model.h b/src/llama-model.h index 4800d2928..45b054ced 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -637,6 +637,8 @@ struct llama_model { std::string desc() const; + llama_ftype ftype() const; + size_t size() const; // file size size_t n_tensors() const; size_t n_devices() const; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 39aa20b32..bb3b91ab5 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3989,6 +3989,8 @@ server_context_meta server_context::get_meta() const { auto bos_token_str = bos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, bos_id, true) : ""; auto eos_token_str = eos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, eos_id, true) : ""; + const char * ftype_name = llama_ftype_name(llama_model_ftype(impl->model_tgt)); + return server_context_meta { /* build_info */ std::string(llama_build_info()), /* model_name */ impl->model_name, @@ -4023,6 +4025,7 @@ server_context_meta server_context::get_meta() const { /* model_n_embd_inp */ llama_model_n_embd(impl->model_tgt), /* model_n_params */ llama_model_n_params(impl->model_tgt), /* model_size */ llama_model_size(impl->model_tgt), + /* model_ftype */ ftype_name, }; } @@ -4086,6 +4089,8 @@ std::unique_ptr server_routes::handle_completions_impl( auto & rd = res->rd; auto & params = this->params; + int32_t sse_ping_interval = params.sse_ping_interval; + try { std::vector tasks; @@ -4136,6 +4141,7 @@ std::unique_ptr server_routes::handle_completions_impl( task.params.message_spans = task.tokens.find_message_spans(delimiters); task.id_slot = json_value(data, "id_slot", -1); + sse_ping_interval = task.params.sse_ping_interval; // OAI-compat task.params.res_type = res_type; @@ -4225,7 +4231,7 @@ std::unique_ptr server_routes::handle_completions_impl( } res->status = 200; res->content_type = "text/event-stream"; - res->next = [res_this = res.get(), res_type, &req, ¶ms](std::string & output) -> bool { + res->next = [res_this = res.get(), res_type, sse_ping_interval, &req](std::string & output) -> bool { static auto format_error = [](task_response_type res_type, const json & res_json) { if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { return format_anthropic_sse({ @@ -4274,10 +4280,10 @@ std::unique_ptr server_routes::handle_completions_impl( // receive subsequent results bool timeout = false; int64_t start_time = ggml_time_ms(); - auto result = rd.next([&timeout, &start_time, ¶ms, &effective_should_stop]() { + auto result = rd.next([&timeout, &start_time, sse_ping_interval, &effective_should_stop]() { if (effective_should_stop()) { return true; // should_stop condition met - } else if (params.sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)params.sse_ping_interval * 1000) { + } else if (sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)sse_ping_interval * 1000) { timeout = true; return true; // timeout } @@ -5118,6 +5124,7 @@ json server_routes::get_model_info() const { {"n_embd", meta->model_n_embd_inp}, {"n_params", meta->model_n_params}, {"size", meta->model_size}, + {"ftype", meta->model_ftype}, }}, }; } diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 952f825f7..f9ab1132b 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -50,6 +50,7 @@ struct server_context_meta { int32_t model_n_embd_inp; uint64_t model_n_params; uint64_t model_size; + std::string model_ftype; }; enum server_state { diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 07a842bd6..5713cc831 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -37,6 +37,10 @@ std::vector> make_llama_cmpl_schema(const common_params & add((new field_bool("return_progress", params.return_progress)) ->set_desc("Include prompt processing progress events in stream mode")); + add((new field_num("sse_ping_interval", params.sse_ping_interval)) + ->set_hard_limits(-1, INT32_MAX) + ->set_desc("Interval in seconds between SSE comment pings emitted while the stream stays silent, -1 disables pings")); + add((new field_num("n_predict", params.n_predict)) ->set_hard_limits(-1, INT32_MAX) ->add_alias("max_completion_tokens") @@ -504,6 +508,7 @@ task_params eval_llama_cmpl_schema( params.n_cache_reuse = params_base.n_cache_reuse; params.cache_prompt = params_base.cache_prompt; params.antiprompt = params_base.antiprompt; + params.sse_ping_interval = params_base.sse_ping_interval; // enabling this will output extra debug information in the HTTP responses from the server params.verbose = params_base.verbosity > 9; diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 293bdf053..49f62d386 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -54,6 +54,8 @@ struct task_params { bool return_tokens = false; bool return_progress = false; + int32_t sse_ping_interval = 30; // seconds between SSE comment pings while the stream stays silent, -1 disables + int32_t n_keep = 0; // number of tokens to keep from initial prompt int32_t n_discard = 0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half int32_t n_predict = -1; // new tokens to predict diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte index dd357d6cd..a75f45f37 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte @@ -18,7 +18,7 @@ let mcpSearchQuery = $state(''); let allMcpServers = $derived(mcpStore.getServersSorted()); - let mcpServers = $derived(allMcpServers.filter((s) => s.enabled)); + let mcpServers = $derived(mcpStore.visibleMcpServers); let hasMcpServers = $derived(mcpServers.length > 0); // let hasAnyMcpServers = $derived(allMcpServers.length > 0); let filteredMcpServers = $derived.by(() => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte index 2b708aae5..b67fb267b 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte @@ -74,9 +74,7 @@ const sheetItemRowClass = 'flex w-full items-center justify-between gap-2 rounded-md px-3 py-2 text-left text-sm transition-colors hover:bg-accent'; - function getEnabledMcpServers() { - return mcpStore.getServersSorted().filter((s) => s.enabled); - } + let visibleMcpServers = $derived(mcpStore.visibleMcpServers);
@@ -153,13 +151,13 @@ MCP Servers - {getEnabledMcpServers().length} server{getEnabledMcpServers().length !== 1 ? 's' : ''} + {visibleMcpServers.length} server{visibleMcpServers.length !== 1 ? 's' : ''}
- {#each getEnabledMcpServers() as server (server.id)} + {#each visibleMcpServers as server (server.id)} {@const healthState = mcpStore.getHealthCheckState(server.id)} {@const hasError = healthState.status === HealthCheckStatus.ERROR} {@const displayName = mcpStore.getServerLabel(server)} @@ -202,7 +200,7 @@ {/each} - {#if getEnabledMcpServers().length === 0} + {#if visibleMcpServers.length === 0}
No MCP servers configured
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index 7189ce1c7..dadcae0c4 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -43,7 +43,7 @@ assistantMessages: number; messageTypes: string[]; } | null>(null); - let editedContent = $state(message.content); + let editedContent = $derived(message.content); let rawEditContent = $derived.by(() => { if (message.role !== MessageRole.ASSISTANT) return undefined; diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionCard/ChatMessageActionCardPermissionRequest.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionCard/ChatMessageActionCardPermissionRequest.svelte index e466c84ee..4337bb6a1 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionCard/ChatMessageActionCardPermissionRequest.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionCard/ChatMessageActionCardPermissionRequest.svelte @@ -1,8 +1,9 @@ @@ -60,29 +69,27 @@ Add New Server -
- (newServerUrl = v)} - onHeadersChange={(v) => (newServerHeaders = v)} - urlError={newServerUrl ? newServerUrlError : null} - id="new-server" - /> -
+
+
+ (newServerUrl = v)} + onHeadersChange={(v) => (newServerHeaders = v)} + urlError={newServerUrl ? newServerUrlError : null} + id="new-server" + /> +
- - + + - - + + +
diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte new file mode 100644 index 000000000..9b4489b82 --- /dev/null +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte @@ -0,0 +1,180 @@ + + + + + + Do more with MCP + + Power-up your experience by adding tools, resources and more capabilities provided by MCP + servers. + + + +
+

Quickly get started with

+ + {#each RECOMMENDED_MCP_SERVERS as server (server.id)} + (selected[server.id] = enabled)} + /> + {/each} + + {#if addedServers.length > 0} + {#each addedServers as server (server.id)} + + {/each} + {/if} + + {#if showAddForm} + + (newServerUrl = v)} + onHeadersChange={(v) => (newServerHeaders = v)} + urlError={newServerUrl ? newServerUrlError : null} + id="recommendation-new-server" + /> + +
+ + + +
+ + {:else} + + + + {/if} +
+ + + + + + + + diff --git a/tools/ui/src/lib/components/app/dialogs/index.ts b/tools/ui/src/lib/components/app/dialogs/index.ts index 29136308c..73f22c565 100644 --- a/tools/ui/src/lib/components/app/dialogs/index.ts +++ b/tools/ui/src/lib/components/app/dialogs/index.ts @@ -18,6 +18,15 @@ */ export { default as DialogMcpServerAddNew } from './DialogMcpServerAddNew.svelte'; +/** + * **DialogMcpServerRecommendations** - Suggested MCP servers opt-in dialog + * + * Prompts the user to enable pre-defined recommended MCP servers on first launch. + * Shows one switch per suggested server and persists the choice as a per-chat + * override so the selected servers become available in conversations. + */ +export { default as DialogMcpServerRecommendations } from './DialogMcpServerRecommendations.svelte'; + /** * **DialogExportSettings** - Settings export dialog with sensitive data warning * diff --git a/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte b/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte index e0bd8d98e..fd6e59a5b 100644 --- a/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte +++ b/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte @@ -1,4 +1,5 @@
@@ -103,6 +123,7 @@ {#each pairs as pair, index (index)}
-
+
{#if showSkeleton} {:else if protocolVersion} diff --git a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardCompact.svelte b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardCompact.svelte new file mode 100644 index 000000000..6cb3e18b6 --- /dev/null +++ b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardCompact.svelte @@ -0,0 +1,156 @@ + + + +
+
+ {#if showSkeleton} + + + + + {:else} + + {/if} +
+ + +
+ + {#if isError && errorMessage} +

{errorMessage}

+ {/if} + + {#if showSkeleton} +
+ +
+ +
+ + + + +
+ {:else} + {#if description} + {#if description.lines === 2} +

+ {description.text} +

+ {:else} +

+ {description.text} +

+ {/if} + {/if} + + {#if tools.length > 0} +
+ {#each visibleTools as tool (tool.name)} + + + + {tool.name} + + + + +

+ {tool.description ?? 'No description'} +

+ + + {/each} + + {#if hiddenToolCount > 0} + + + + + {hiddenToolCount} more tools + + + + +

+ {hiddenTools.map((tool) => tool.name).join(', ')} +

+ + + {/if} +
+ {/if} + {/if} + diff --git a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte index 6727a9000..8ed4ee8b8 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte @@ -1,6 +1,7 @@ -
-

Configure Server

+
+
+

Configure Server

- (editUrl = v)} - onHeadersChange={(v) => (editHeaders = v)} - onUseProxyChange={(v) => (editUseProxy = v)} - urlError={editUrl ? urlError : null} - id={serverId} - /> + (editUrl = v)} + onHeadersChange={(v) => (editHeaders = v)} + onUseProxyChange={(v) => (editUseProxy = v)} + urlError={editUrl ? urlError : null} + id={serverId} + /> -
- +
+ - + +
-
+
diff --git a/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte b/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte index 79738e30d..7f05d5fef 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte @@ -38,14 +38,87 @@ let headerPairs = $derived(parseHeadersToArray(headers)); + const AUTHORIZATION_HEADER = 'Authorization'; + const BEARER_PREFIX = 'Bearer '; + + // Heuristic: this dedicated UI only owns Authorization headers that already + // carry a Bearer scheme. Anything else (e.g. Basic, raw tokens) stays in the + // KV section so the user can still edit those values verbatim. + const matchesAuthorizationKey = (key: string): boolean => + key.trim().toLowerCase() === AUTHORIZATION_HEADER.toLowerCase(); + + const isBearerScheme = (value: string): boolean => + value.trim().toLowerCase().startsWith(BEARER_PREFIX.toLowerCase()); + + const ownedByBearerUi = (p: KeyValuePair): boolean => + matchesAuthorizationKey(p.key) && isBearerScheme(p.value); + + let hasAuthorization = $derived(headerPairs.some(ownedByBearerUi)); + + let wantsAuthorization = $state(false); + + let showAuthorization = $derived(hasAuthorization || wantsAuthorization); + + let urlInput: HTMLInputElement | null = $state(null); + let bearerInput: HTMLInputElement | null = $state(null); + + $effect(() => { + urlInput?.focus(); + }); + + $effect(() => { + if (wantsAuthorization && bearerInput) { + bearerInput.focus(); + } + }); + + let bearerToken = $derived.by(() => { + const auth = headerPairs.find(ownedByBearerUi); + if (!auth) return ''; + return auth.value.trim().slice(BEARER_PREFIX.length).trim(); + }); + + $effect(() => { + if (!headers.trim()) { + wantsAuthorization = false; + } + }); + function updateHeaderPairs(newPairs: KeyValuePair[]) { headerPairs = newPairs; onHeadersChange(serializeHeaders(newPairs)); } + + // The dedicated UI owns the Authorization slot end-to-end when the user + // engages it: any prior Authorization row (Bearer or otherwise) is replaced + // by exactly one { Authorization: "Bearer " } entry. JSON's last-key + // behavior would otherwise pick one arbitrarily, so we strip first. + function updateBearerToken(token: string) { + const filtered = headerPairs.filter((p) => !matchesAuthorizationKey(p.key)); + + const trimmed = token.trim(); + + if (trimmed) { + filtered.push({ key: AUTHORIZATION_HEADER, value: `${BEARER_PREFIX}${trimmed}` }); + } + + updateHeaderPairs(filtered); + } + + function setUseAuthorization(checked: boolean) { + wantsAuthorization = checked; + + if (!checked) { + // Only drop the entry this UI owns; a non-Bearer Authorization row + // authored in the KV section must survive a toggle off untouched. + const filtered = headerPairs.filter((p) => !ownedByBearerUi(p)); + updateHeaderPairs(filtered); + } + } -
-
+
+
@@ -57,50 +130,52 @@ value={url} oninput={(e) => onUrlChange(e.currentTarget.value)} class={urlError ? 'border-destructive' : ''} + bind:ref={urlInput} /> {#if urlError}

{urlError}

{/if} - - {#if !isWebSocket && onUseProxyChange} - - {/if}
+ + + {#if showAuthorization} +
+ updateBearerToken(e.currentTarget.value)} + class="pl-16" + bind:ref={bearerInput} + /> + + + Bearer + +
+ {/if} + !ownedByBearerUi(p))} + onPairsChange={(pairs) => { + const auth = headerPairs.find(ownedByBearerUi); + updateHeaderPairs(auth ? [...pairs, auth] : pairs); + }} keyPlaceholder="Header name" valuePlaceholder="Value" addButtonLabel="Add" @@ -108,4 +183,37 @@ sectionLabel="Custom Headers" sectionLabelOptional /> + + {#if !isWebSocket && onUseProxyChange} + + {/if}
diff --git a/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte b/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte index feafc5d81..3f128e02c 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte @@ -1,6 +1,7 @@ + + + {}} + onHeadersChange={(value) => { + headersState = value; + }} + id="mcp-server-form-test" +/> + + diff --git a/tools/ui/tests/client/mcp-server-form.svelte.test.ts b/tools/ui/tests/client/mcp-server-form.svelte.test.ts new file mode 100644 index 000000000..b4bd89234 --- /dev/null +++ b/tools/ui/tests/client/mcp-server-form.svelte.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from 'vitest'; +import { render } from 'vitest-browser-svelte'; +import McpServerFormWrapper from './components/McpServerFormWrapper.svelte'; + +const AUTHORIZATION_HEADER = 'Authorization'; +const BEARER_PREFIX = 'Bearer '; +const BEARER_PLACEHOLDER = 'Paste token here'; + +/** + * Client-side tests for the McpServerForm bearer UI. + * + * The dedicated UI only "owns" Authorization headers that already carry a + * Bearer scheme (heuristic check on the value). Other Authorization values + * stay in the KV section so the user can still edit them verbatim. Storage + * always goes through the same custom-headers slot, so a round-trip via this + * UI produces exactly one `Authorization: Bearer ` entry. + * + * Equivalent parser coverage lives in `tests/unit/headers.test.ts`. + */ +describe('McpServerForm - Authorization / bearer UI', () => { + function bearerInput(screen: Awaited>) { + return screen.locator.getByPlaceholder(BEARER_PLACEHOLDER); + } + + function capturedHeaders(screen: Awaited>) { + return screen.getByTestId('captured-headers'); + } + + it('mounts with the bearer input hidden when no auth header is present', async () => { + const screen = await render(McpServerFormWrapper, { headers: '' }); + + await expect.element(screen.getByRole('textbox', { name: /server url/i })).toBeVisible(); + + await expect.element(bearerInput(screen)).not.toBeInTheDocument(); + }); + + it('toggling Authorization shows the bearer input', async () => { + const screen = await render(McpServerFormWrapper, { headers: '' }); + + await screen.getByRole('switch', { name: /authorization/i }).click(); + + await expect.element(bearerInput(screen)).toBeVisible(); + }); + + it('typing a token writes the Authorization row with the Bearer prefix prepended', async () => { + const screen = await render(McpServerFormWrapper, { headers: '' }); + + await screen.getByRole('switch', { name: /authorization/i }).click(); + + const token = 'super-secret'; + await bearerInput(screen).fill(token); + + const expected = JSON.stringify({ [AUTHORIZATION_HEADER]: `${BEARER_PREFIX}${token}` }); + await expect + .element(capturedHeaders(screen)) + .toHaveAttribute('data-captured-headers', expected); + }); + + it('pre-existing Bearer header pre-fills the bearer input with the token stripped', async () => { + const existing = JSON.stringify({ + 'X-Trace-Id': 'abc', + [AUTHORIZATION_HEADER]: `${BEARER_PREFIX}preexisting` + }); + + const screen = await render(McpServerFormWrapper, { headers: existing }); + + await expect.element(bearerInput(screen)).toBeVisible(); + await expect.element(bearerInput(screen)).toHaveValue('preexisting'); + }); + + it('non-Bearer Authorization is ignored by the dedicated UI and stays in the KV section', async () => { + const existing = JSON.stringify({ [AUTHORIZATION_HEADER]: 'Basic czNjcjpwYXNz' }); + + const screen = await render(McpServerFormWrapper, { headers: existing }); + + await expect.element(bearerInput(screen)).not.toBeInTheDocument(); + + const headerKeyInput = screen.getByPlaceholder('Header name'); + await expect.element(headerKeyInput).toBeVisible(); + }); + + it('engaging the token UI replaces a non-Bearer Authorization with the Bearer scheme', async () => { + const existing = JSON.stringify({ [AUTHORIZATION_HEADER]: 'Basic old' }); + + const screen = await render(McpServerFormWrapper, { headers: existing }); + + await screen.getByRole('switch', { name: /authorization/i }).click(); + await bearerInput(screen).fill('new'); + + const expected = JSON.stringify({ [AUTHORIZATION_HEADER]: `${BEARER_PREFIX}new` }); + await expect + .element(capturedHeaders(screen)) + .toHaveAttribute('data-captured-headers', expected); + }); + + it('toggling Authorization off with no token drops the Bearer row but keeps non-Bearer schemes', async () => { + const existing = JSON.stringify({ [AUTHORIZATION_HEADER]: `${BEARER_PREFIX}xyz` }); + const screen = await render(McpServerFormWrapper, { headers: existing }); + + await screen.getByRole('switch', { name: /authorization/i }).click(); + + await expect.element(capturedHeaders(screen)).toHaveAttribute('data-captured-headers', ''); + }); + + it('toggling Authorization off when no Bearer row is present leaves headers untouched', async () => { + const existing = JSON.stringify({ [AUTHORIZATION_HEADER]: 'Basic czNjcjpwYXNz' }); + const screen = await render(McpServerFormWrapper, { headers: existing }); + + await screen.getByRole('switch', { name: /authorization/i }).click(); + await screen.getByRole('switch', { name: /authorization/i }).click(); + + await expect + .element(capturedHeaders(screen)) + .toHaveAttribute('data-captured-headers', existing); + }); + + it('clearing the bearer input drops the Authorization row', async () => { + const existing = JSON.stringify({ [AUTHORIZATION_HEADER]: `${BEARER_PREFIX}xyz` }); + const screen = await render(McpServerFormWrapper, { headers: existing }); + + await bearerInput(screen).fill(''); + + await expect.element(capturedHeaders(screen)).toHaveAttribute('data-captured-headers', ''); + }); + + it('does not surface Bearer Authorization in the KV section even when pre-existing', async () => { + const existing = JSON.stringify({ [AUTHORIZATION_HEADER]: `${BEARER_PREFIX}xyz` }); + const screen = await render(McpServerFormWrapper, { headers: existing }); + + const headerKeyInput = screen.getByPlaceholder('Header name'); + await expect.element(headerKeyInput).not.toBeInTheDocument(); + }); +}); diff --git a/tools/ui/tests/unit/headers.test.ts b/tools/ui/tests/unit/headers.test.ts new file mode 100644 index 000000000..33619547c --- /dev/null +++ b/tools/ui/tests/unit/headers.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from 'vitest'; +import { parseHeadersToArray, serializeHeaders } from '$lib/utils/headers'; + +/** + * Tests for the header serialization helpers used by the MCP server form + * (custom header rows) and the new Authorization/Bearer-token flow. + */ +describe('parseHeadersToArray', () => { + it('returns an empty array for empty or whitespace-only input', () => { + expect(parseHeadersToArray('')).toEqual([]); + expect(parseHeadersToArray(' ')).toEqual([]); + expect(parseHeadersToArray(undefined as unknown as string)).toEqual([]); + }); + + it('returns an empty array for invalid JSON input', () => { + expect(parseHeadersToArray('{not-json')).toEqual([]); + expect(parseHeadersToArray('[]')).toEqual([]); + expect(parseHeadersToArray('"plain-string"')).toEqual([]); + }); + + it('converts an object into ordered key/value pairs', () => { + expect(parseHeadersToArray('{"X-Foo":"bar","Authorization":"Bearer abc"}')).toEqual([ + { key: 'X-Foo', value: 'bar' }, + { key: 'Authorization', value: 'Bearer abc' } + ]); + }); + + it('stringifies non-string values', () => { + expect(parseHeadersToArray('{"count":"42","flag":"true"}')).toEqual([ + { key: 'count', value: '42' }, + { key: 'flag', value: 'true' } + ]); + }); +}); + +describe('serializeHeaders', () => { + it('returns an empty string when there are no valid pairs', () => { + expect(serializeHeaders([])).toBe(''); + expect(serializeHeaders([{ key: '', value: 'value' }])).toBe(''); + expect(serializeHeaders([{ key: ' ', value: 'value' }])).toBe(''); + }); + + it('returns an empty string when every pair has a blank key', () => { + expect( + serializeHeaders([ + { key: '', value: 'drop-me' }, + { key: ' ', value: 'drop-me-too' }, + { key: '\t', value: 'tab-key' } + ]) + ).toBe(''); + }); + + it('drops pairs with empty keys but keeps the rest', () => { + expect( + serializeHeaders([ + { key: '', value: 'drop-me' }, + { key: 'X-Keep', value: 'ok' } + ]) + ).toBe('{"X-Keep":"ok"}'); + }); + + it('trims keys before serializing', () => { + expect(serializeHeaders([{ key: ' X-Space ', value: 'ok' }])).toBe('{"X-Space":"ok"}'); + }); + + it('preserves the input order of surviving pairs', () => { + const serialized = serializeHeaders([ + { key: 'X-C', value: '3' }, + { key: 'X-A', value: '1' }, + { key: 'X-B', value: '2' } + ]); + + // Object key order follows insertion order in modern JS engines, so + // the serialized JSON writes keys in our input order. + expect(JSON.parse(serialized)).toEqual({ 'X-C': '3', 'X-A': '1', 'X-B': '2' }); + }); +}); + +describe('parseHeadersToArray / serializeHeaders roundtrip', () => { + it('serializes back to an equal header object after a parse', () => { + const original = JSON.stringify({ + 'Content-Type': 'application/json', + 'X-Trace-Id': 'abc-123' + }); + + const roundtrip = serializeHeaders(parseHeadersToArray(original)); + + expect(JSON.parse(roundtrip)).toEqual(JSON.parse(original)); + }); + + it('drops rows whose keys are blank after trimming during serialization', () => { + const pairs = parseHeadersToArray('{"X-Keep":"ok","":"drop-me"}'); + + // parseHeadersToArray keeps raw key strings (the consumer is expected to + // filter blanks, not the parser); serialization must strip them. + expect(pairs).toEqual([ + { key: 'X-Keep', value: 'ok' }, + { key: '', value: 'drop-me' } + ]); + expect(serializeHeaders(pairs)).toBe('{"X-Keep":"ok"}'); + }); + + it('preserves upstream keys untouched (does not lowercase them)', () => { + const upperCased = '{"Authorization":"Bearer xyz"}'; + + const parsed = parseHeadersToArray(upperCased); + + expect(parsed).toEqual([{ key: 'Authorization', value: 'Bearer xyz' }]); + }); + + it('bearer-token write survives a re-parse when paired with regular custom headers', () => { + // The McpServerForm bearer UI writes {Authorization: `Bearer `} + // into the same headers string as the custom KV section. The round + // trip below mirrors the exact shape the form produces so a future + // refactor of either code path cannot silently change the on-disk key. + const pairs = [ + { key: 'X-Trace-Id', value: 'abc-123' }, + { key: 'Authorization', value: 'Bearer super-secret' } + ]; + + const serialized = serializeHeaders(pairs); + + expect(serialized).toBe('{"X-Trace-Id":"abc-123","Authorization":"Bearer super-secret"}'); + expect(parseHeadersToArray(serialized)).toEqual(pairs); + }); +}); diff --git a/tools/ui/tests/unit/parse-mcp-server-settings.test.ts b/tools/ui/tests/unit/parse-mcp-server-settings.test.ts new file mode 100644 index 000000000..956c677d5 --- /dev/null +++ b/tools/ui/tests/unit/parse-mcp-server-settings.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it, vi } from 'vitest'; +import { parseMcpServerSettings } from '$lib/utils/mcp'; +import { DEFAULT_MCP_CONFIG, MCP_SERVER_ID_PREFIX } from '$lib/constants/mcp'; + +/** + * Tests for the mcpServers settings parser. + * + * The branch seeds the MCP servers setting with a default value of + * `JSON.stringify(RECOMMENDED_MCP_SERVERS)`, so the parser has to be + * resilient to anything that may live in the user's localStorage: malformed + * JSON, wrong shapes, missing fields, falsy-but-not-zero numbers, and entry + * arrays that have been mutated by the user via the settings form. + */ +describe('parseMcpServerSettings', () => { + it('returns an empty array for falsy or whitespace-only input', () => { + expect(parseMcpServerSettings(null)).toEqual([]); + expect(parseMcpServerSettings(undefined)).toEqual([]); + expect(parseMcpServerSettings('')).toEqual([]); + expect(parseMcpServerSettings(' ')).toEqual([]); + }); + + it('returns an empty array and logs a warning for invalid JSON strings', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + expect(parseMcpServerSettings('{not-json')).toEqual([]); + expect(warn).toHaveBeenCalled(); + + warn.mockRestore(); + }); + + it('returns an empty array for valid JSON that is not an array', () => { + expect(parseMcpServerSettings('"plain-string"')).toEqual([]); + expect(parseMcpServerSettings('{"id":"foo"}')).toEqual([]); + expect(parseMcpServerSettings('42')).toEqual([]); + expect(parseMcpServerSettings('null')).toEqual([]); + }); + + it('drops entries with no parseable id and substitutes a stable fallback', () => { + const parsed = parseMcpServerSettings( + JSON.stringify([{ url: 'https://a.test', enabled: true }, { url: 'https://b.test' }]) + ); + + expect(parsed).toHaveLength(2); + expect(parsed[0]?.id).toBe(`${MCP_SERVER_ID_PREFIX}-1`); + expect(parsed[1]?.id).toBe(`${MCP_SERVER_ID_PREFIX}-2`); + }); + + it('reuses the first id when it is present and falls back only for missing ones', () => { + const parsed = parseMcpServerSettings( + JSON.stringify([ + { id: 'custom-1', url: 'https://a.test' }, + { url: 'https://b.test' }, + { id: 'custom-3', url: 'https://c.test' } + ]) + ); + + expect(parsed[0]?.id).toBe('custom-1'); + expect(parsed[1]?.id).toBe(`${MCP_SERVER_ID_PREFIX}-2`); + expect(parsed[2]?.id).toBe('custom-3'); + }); + + it('falls back to the configured default requestTimeoutSeconds only for nullish values', () => { + const fallback = DEFAULT_MCP_CONFIG.requestTimeoutSeconds; + + const parsed = parseMcpServerSettings( + JSON.stringify([ + { id: 'a', url: 'https://a.test' }, + { id: 'b', url: 'https://b.test', requestTimeoutSeconds: undefined }, + { id: 'c', url: 'https://c.test', requestTimeoutSeconds: 0 }, + { id: 'd', url: 'https://d.test', requestTimeoutSeconds: 45 } + ]) + ); + + // The parser uses ?? for timeout fallback, which only triggers on + // null/undefined. Explicit 0 is preserved at face value. + expect(parsed[0]?.requestTimeoutSeconds).toBe(fallback); + expect(parsed[1]?.requestTimeoutSeconds).toBe(fallback); + expect(parsed[2]?.requestTimeoutSeconds).toBe(0); + expect(parsed[3]?.requestTimeoutSeconds).toBe(45); + }); + + it('treats whitespace-only headers strings as undefined', () => { + const parsed = parseMcpServerSettings( + JSON.stringify([ + { id: 'a', url: 'https://a.test', headers: ' ' }, + { id: 'b', url: 'https://b.test', headers: '{"X-Foo":"bar"}' } + ]) + ); + + // The parser trims headers and coerces empty/whitespace to undefined. + expect(parsed[0]?.headers).toBeUndefined(); + expect(parsed[1]?.headers).toBe('{"X-Foo":"bar"}'); + }); + + it('defaults coercion for booleans (undefined -> false, true -> true)', () => { + const parsed = parseMcpServerSettings( + JSON.stringify([ + { id: 'a', url: 'https://a.test' }, + { id: 'b', url: 'https://b.test', enabled: true }, + { id: 'c', url: 'https://c.test', enabled: false }, + { id: 'd', url: 'https://d.test', useProxy: true } + ]) + ); + + expect(parsed[0]?.enabled).toBe(false); + expect(parsed[1]?.enabled).toBe(true); + expect(parsed[2]?.enabled).toBe(false); + expect(parsed[0]?.useProxy).toBe(false); + expect(parsed[3]?.useProxy).toBe(true); + }); + + it('preserves input order when mapping entries', () => { + const source = [ + { id: 'gamma', url: 'https://c.test' }, + { id: 'alpha', url: 'https://a.test' }, + { id: 'beta', url: 'https://b.test' } + ]; + + const parsed = parseMcpServerSettings(JSON.stringify(source)); + + expect(parsed.map((entry) => entry.id)).toEqual(['gamma', 'alpha', 'beta']); + }); + + it('passes non-string raw input through the JSON-equality path', () => { + const parsed = parseMcpServerSettings([ + { id: 'a', url: 'https://a.test' }, + { id: 'b', url: 'https://b.test', enabled: true } + ]); + + expect(parsed).toHaveLength(2); + expect(parsed[0]?.id).toBe('a'); + expect(parsed[1]?.enabled).toBe(true); + }); + + it('coerces non-string url values to an empty string rather than throwing', () => { + const parsed = parseMcpServerSettings( + JSON.stringify([{ id: 'a', url: 42 }, { id: 'b' }, { id: 'c', url: 'https://c.test' }]) + ); + + expect(parsed[0]?.url).toBe(''); + expect(parsed[1]?.url).toBe(''); + expect(parsed[2]?.url).toBe('https://c.test'); + }); +}); diff --git a/tools/ui/tests/unit/recommended-mcp-servers.test.ts b/tools/ui/tests/unit/recommended-mcp-servers.test.ts new file mode 100644 index 000000000..3f6fd8f11 --- /dev/null +++ b/tools/ui/tests/unit/recommended-mcp-servers.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest'; +import { + RECOMMENDED_MCP_SERVER_IDS, + RECOMMENDED_MCP_SERVERS +} from '$lib/constants/recommended-mcp-servers'; +import { parseMcpServerSettings } from '$lib/utils/mcp'; +import { DEFAULT_MCP_CONFIG, MCP_SERVER_ID_PREFIX } from '$lib/constants/mcp'; + +/** + * Tests for the predefined recommended MCP servers. + * + * These are surfaced to first-time users via + * DialogMcpServerRecommendations and used as the default value of the MCP + * servers setting, so a regression that breaks the round-trip through the + * settings parser would silently break onboarding for new users. + */ +describe('RECOMMENDED_MCP_SERVERS', () => { + it('lists at least one entry and uses stable, unique ids', () => { + expect(RECOMMENDED_MCP_SERVERS.length).toBeGreaterThan(0); + + const ids = RECOMMENDED_MCP_SERVERS.map((server) => server.id); + expect(new Set(ids).size).toBe(ids.length); + + for (const id of ids) { + expect(id).toMatch(/^[a-z0-9-]+$/); + expect(id.toLowerCase()).not.toContain(MCP_SERVER_ID_PREFIX.toLowerCase()); + } + }); + + it('requires a name, description and url for every entry', () => { + for (const server of RECOMMENDED_MCP_SERVERS) { + expect(server.name?.trim().length ?? 0).toBeGreaterThan(0); + expect(server.description.trim().length).toBeGreaterThan(0); + expect(server.url.trim().length).toBeGreaterThan(0); + expect(() => new URL(server.url)).not.toThrow(); + } + }); +}); + +describe('RECOMMENDED_MCP_SERVER_IDS', () => { + it('matches the ids declared in RECOMMENDED_MCP_SERVERS', () => { + expect(RECOMMENDED_MCP_SERVER_IDS.size).toBe(RECOMMENDED_MCP_SERVERS.length); + + for (const server of RECOMMENDED_MCP_SERVERS) { + expect(RECOMMENDED_MCP_SERVER_IDS.has(server.id)).toBe(true); + } + }); +}); + +describe('recommended-mcp-servers default value', () => { + it('round-trips cleanly through parseMcpServerSettings', () => { + const serialized = JSON.stringify(RECOMMENDED_MCP_SERVERS); + const parsed = parseMcpServerSettings(serialized); + + expect(parsed).toHaveLength(RECOMMENDED_MCP_SERVERS.length); + + for (let index = 0; index < RECOMMENDED_MCP_SERVERS.length; index++) { + const source = RECOMMENDED_MCP_SERVERS[index]; + const entry = parsed[index]; + + expect(entry).toBeDefined(); + expect(entry?.id).toBe(source.id); + expect(entry?.url).toBe(source.url); + expect(entry?.enabled).toBe(source.enabled); + expect(entry?.requestTimeoutSeconds).toBe(source.requestTimeoutSeconds); + expect(entry?.name).toBe(source.name); + + // Headers and useProxy are not set on recommended servers; the + // parser must fall back to the inactive defaults rather than + // surfacing undefined-boundary states. + expect(entry?.headers).toBeUndefined(); + expect(entry?.useProxy).toBe(false); + } + }); + + it('uses the global default timeout when one is not specified on an entry', () => { + const sourceOnlyRequired = { + id: 'roundtrip-only', + name: 'Only required fields', + url: 'https://example.test/mcp', + description: 'Smoke entry for parser roundtrip with default timeout.', + enabled: true + }; + + const parsed = parseMcpServerSettings(JSON.stringify([sourceOnlyRequired])); + const entry = parsed[0]; + + expect(entry?.requestTimeoutSeconds).toBe(DEFAULT_MCP_CONFIG.requestTimeoutSeconds); + }); +}); diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp index 1ac4fa4ba..d65b7921b 100644 --- a/vendor/cpp-httplib/httplib.cpp +++ b/vendor/cpp-httplib/httplib.cpp @@ -478,7 +478,7 @@ bool set_socket_opt_time(socket_t sock, int level, int optname, } bool is_hex(char c, int &v) { - if (isdigit(static_cast(c))) { + if (is_ascii_digit(c)) { v = c - '0'; return true; } else if ('A' <= c && c <= 'F') { @@ -695,7 +695,11 @@ std::string base64_encode(const std::string &in) { std::string out; out.reserve(in.size()); - auto val = 0; + // Unsigned: the accumulator is never masked, so with a signed int the + // `val << 8` below overflows once enough bytes are folded in (undefined + // behaviour before C++20). Only the low bits are ever emitted, so the + // wrap-around of an unsigned accumulator does not affect the output. + uint32_t val = 0; auto valb = -6; for (auto c : in) { @@ -3887,8 +3891,7 @@ bool parse_range_header(const std::string &s, Ranges &ranges) { bool parse_range_header(const std::string &s, Ranges &ranges) try { #endif auto is_valid = [](const std::string &str) { - return std::all_of(str.cbegin(), str.cend(), - [](unsigned char c) { return std::isdigit(c); }); + return std::all_of(str.cbegin(), str.cend(), is_ascii_digit); }; if (s.size() > 7 && s.compare(0, 6, "bytes=") == 0) { @@ -4336,7 +4339,7 @@ bool is_multipart_boundary_chars_valid(const std::string &boundary) { auto valid = true; for (size_t i = 0; i < boundary.size(); i++) { auto c = boundary[i]; - if (!std::isalnum(static_cast(c)) && c != '-' && c != '_') { + if (!is_ascii_alnum(c) && c != '-' && c != '_') { valid = false; break; } @@ -4344,18 +4347,47 @@ bool is_multipart_boundary_chars_valid(const std::string &boundary) { return valid; } +// Escape a multipart field name/filename following the WHATWG HTML standard +// ("escape a multipart form-data name"), which is what browsers send: +// '"' -> %22, CR -> %0D, LF -> %0A +// With escape_quote = false, only CR and LF are escaped; this is for header +// values outside a quoted-string (e.g. Content-Type), where '"' is legal. +std::string escape_multipart_field(const std::string &s, + bool escape_quote = true) { + std::string result; + result.reserve(s.size()); + for (auto c : s) { + switch (c) { + case '"': + if (escape_quote) { + result += "%22"; + } else { + result += c; + } + break; + case '\r': result += "%0D"; break; + case '\n': result += "%0A"; break; + default: result += c; break; + } + } + return result; +} + template std::string serialize_multipart_formdata_item_begin(const T &item, const std::string &boundary) { std::string body = "--" + boundary + "\r\n"; - body += "Content-Disposition: form-data; name=\"" + item.name + "\""; + body += "Content-Disposition: form-data; name=\"" + + escape_multipart_field(item.name) + "\""; if (!item.filename.empty()) { - body += "; filename=\"" + item.filename + "\""; + body += "; filename=\"" + escape_multipart_field(item.filename) + "\""; } body += "\r\n"; if (!item.content_type.empty()) { - body += "Content-Type: " + item.content_type + "\r\n"; + body += + "Content-Type: " + escape_multipart_field(item.content_type, false) + + "\r\n"; } body += "\r\n"; @@ -4821,10 +4853,9 @@ private: namespace fields { bool is_token_char(char c) { - return std::isalnum(static_cast(c)) || c == '!' || c == '#' || - c == '$' || c == '%' || c == '&' || c == '\'' || c == '*' || - c == '+' || c == '-' || c == '.' || c == '^' || c == '_' || c == '`' || - c == '|' || c == '~'; + return is_ascii_alnum(c) || c == '!' || c == '#' || c == '$' || c == '%' || + c == '&' || c == '\'' || c == '*' || c == '+' || c == '-' || + c == '.' || c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; } bool is_token(const std::string &s) { @@ -4873,7 +4904,8 @@ bool is_field_value(const std::string &s) { return is_field_content(s); } } // namespace fields bool perform_websocket_handshake(Stream &strm, const std::string &host, - int port, const std::string &path, + int port, bool is_ssl, + const std::string &path, const Headers &headers, std::string &selected_subprotocol) { // Validate path and host @@ -4899,7 +4931,7 @@ bool perform_websocket_handshake(Stream &strm, const std::string &host, // Build upgrade request std::string req_str = "GET " + path + " HTTP/1.1\r\n"; - req_str += "Host: " + host + ":" + std::to_string(port) + "\r\n"; + req_str += "Host: " + make_host_and_port_string(host, port, is_ssl) + "\r\n"; req_str += "Upgrade: websocket\r\n"; req_str += "Connection: Upgrade\r\n"; req_str += "Sec-WebSocket-Key: " + client_key + "\r\n"; @@ -5599,9 +5631,8 @@ std::string encode_uri_component(const std::string &value) { escaped << std::hex; for (auto c : value) { - if (std::isalnum(static_cast(c)) || c == '-' || c == '_' || - c == '.' || c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || - c == ')') { + if (detail::is_ascii_alnum(c) || c == '-' || c == '_' || c == '.' || + c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || c == ')') { escaped << c; } else { escaped << std::uppercase; @@ -5620,10 +5651,10 @@ std::string encode_uri(const std::string &value) { escaped << std::hex; for (auto c : value) { - if (std::isalnum(static_cast(c)) || c == '-' || c == '_' || - c == '.' || c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || - c == ')' || c == ';' || c == '/' || c == '?' || c == ':' || c == '@' || - c == '&' || c == '=' || c == '+' || c == '$' || c == ',' || c == '#') { + if (detail::is_ascii_alnum(c) || c == '-' || c == '_' || c == '.' || + c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || c == ')' || + c == ';' || c == '/' || c == '?' || c == ':' || c == '@' || c == '&' || + c == '=' || c == '+' || c == '$' || c == ',' || c == '#') { escaped << c; } else { escaped << std::uppercase; @@ -5684,7 +5715,8 @@ std::string encode_path_component(const std::string &component) { auto c = static_cast(component[i]); // Unreserved characters per RFC 3986: ALPHA / DIGIT / "-" / "." / "_" / "~" - if (std::isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~') { + if (detail::is_ascii_alnum(static_cast(c)) || c == '-' || c == '.' || + c == '_' || c == '~') { result += static_cast(c); } // Path-safe sub-delimiters: "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / @@ -5757,7 +5789,8 @@ std::string encode_query_component(const std::string &component, auto c = static_cast(component[i]); // Unreserved characters per RFC 3986 - if (std::isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~') { + if (detail::is_ascii_alnum(static_cast(c)) || c == '-' || c == '.' || + c == '_' || c == '~') { result += static_cast(c); } // Space handling @@ -6010,6 +6043,48 @@ size_t MultipartFormData::get_file_count(const std::string &key) const { return static_cast(std::distance(r.first, r.second)); } +// Multipart FormData writer implementation +bool is_valid_multipart_boundary(const std::string &boundary) { + return detail::is_multipart_boundary_chars_valid(boundary); +} + +MultipartFormDataWriter::MultipartFormDataWriter() + : boundary_(detail::make_multipart_data_boundary()) {} + +MultipartFormDataWriter::MultipartFormDataWriter(std::string boundary) + : boundary_(std::move(boundary)) {} + +const std::string &MultipartFormDataWriter::boundary() const { + return boundary_; +} + +std::string MultipartFormDataWriter::content_type() const { + return detail::serialize_multipart_formdata_get_content_type(boundary_); +} + +std::string +MultipartFormDataWriter::serialize(const UploadFormDataItems &items) const { + return detail::serialize_multipart_formdata(items, boundary_); +} + +size_t MultipartFormDataWriter::content_length( + const UploadFormDataItems &items) const { + return detail::get_multipart_content_length(items, boundary_); +} + +std::string +MultipartFormDataWriter::item_begin(const UploadFormData &item) const { + return detail::serialize_multipart_formdata_item_begin(item, boundary_); +} + +std::string MultipartFormDataWriter::item_end() { + return detail::serialize_multipart_formdata_item_end(); +} + +std::string MultipartFormDataWriter::finish() const { + return detail::serialize_multipart_formdata_finish(boundary_); +} + // Response implementation size_t Response::get_header_value_u64(const std::string &key, size_t def, size_t id) const { @@ -6229,8 +6304,10 @@ ssize_t detail::BodyReader::read(char *buf, size_t len) { } // ThreadPool implementation -ThreadPool::ThreadPool(size_t n, size_t max_n, size_t mqr) - : base_thread_count_(n), max_queued_requests_(mqr), idle_thread_count_(0), +ThreadPool::ThreadPool(size_t n, size_t max_n, size_t mqr, + time_t idle_timeout_sec) + : base_thread_count_(n), max_queued_requests_(mqr), + idle_timeout_sec_(idle_timeout_sec), idle_thread_count_(0), shutdown_(false) { #ifndef CPPHTTPLIB_NO_EXCEPTIONS if (max_n != 0 && max_n < n) { @@ -6340,9 +6417,9 @@ void ThreadPool::worker(bool is_dynamic) { idle_thread_count_++; if (is_dynamic) { - auto has_work = cond_.wait_for( - lock, std::chrono::seconds(CPPHTTPLIB_THREAD_POOL_IDLE_TIMEOUT), - [&] { return !jobs_.empty() || shutdown_; }); + auto has_work = + cond_.wait_for(lock, std::chrono::seconds(idle_timeout_sec_), + [&] { return !jobs_.empty() || shutdown_; }); if (!has_work) { // Timed out with no work - exit this dynamic thread idle_thread_count_--; @@ -9687,9 +9764,18 @@ bool ClientImpl::write_request(Stream &strm, Request &req, if (!query_part.empty()) { // Normalize the query string (decode then re-encode) while preserving - // the original parameter order. - auto normalized = detail::normalize_query_string(query_part); - if (!normalized.empty()) { path_with_query += '?' + normalized; } + // the original parameter order. When path encoding is disabled the + // caller has supplied an already-encoded target and expects the exact + // bytes to be sent on the wire, so skip normalization for the query + // too. Normalizing here would decode-then-re-encode the query and + // corrupt pre-encoded binary payloads (e.g. turning `%20` into `+`, + // which a strict RFC 3986 server decodes back as `+`, not a space). + if (path_encode_) { + auto normalized = detail::normalize_query_string(query_part); + if (!normalized.empty()) { path_with_query += '?' + normalized; } + } else { + path_with_query += '?' + query_part; + } // Still populate req.params for handlers/users who read them. detail::parse_query_text(query_part, req.params); @@ -12518,7 +12604,7 @@ bool is_ipv4_address(const std::string &str) { for (char c : str) { if (c == '.') { dots++; - } else if (!isdigit(static_cast(c))) { + } else if (!detail::is_ascii_digit(c)) { return false; } } @@ -12535,7 +12621,7 @@ bool parse_ipv4(const std::string &str, unsigned char *out) { } int val = 0; int digits = 0; - while (*p >= '0' && *p <= '9') { + while (detail::is_ascii_digit(*p)) { val = val * 10 + (*p - '0'); if (val > 255) { return false; } p++; @@ -16487,9 +16573,15 @@ bool WebSocketClient::connect() { return false; } +#ifdef CPPHTTPLIB_SSL_ENABLED + auto is_ssl = is_ssl_; +#else + auto is_ssl = false; +#endif + std::string selected_subprotocol; - if (!detail::perform_websocket_handshake(*strm, host_, port_, path_, headers_, - selected_subprotocol)) { + if (!detail::perform_websocket_handshake(*strm, host_, port_, is_ssl, path_, + headers_, selected_subprotocol)) { shutdown_and_close(); return false; } diff --git a/vendor/cpp-httplib/httplib.h b/vendor/cpp-httplib/httplib.h index bfdbfc1da..e7ef56370 100644 --- a/vendor/cpp-httplib/httplib.h +++ b/vendor/cpp-httplib/httplib.h @@ -8,8 +8,8 @@ #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.48.0" -#define CPPHTTPLIB_VERSION_NUM "0x003000" +#define CPPHTTPLIB_VERSION "0.49.0" +#define CPPHTTPLIB_VERSION_NUM "0x003100" #ifdef _WIN32 #if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00 @@ -309,7 +309,6 @@ using socket_t = int; #include #include #include -#include #include #include #include @@ -540,6 +539,21 @@ make_unique(std::size_t n) { return std::unique_ptr(new RT[n]); } +// Locale-independent ASCII character classification. The +// counterparts (std::isalnum, std::isdigit, ...) consult the global C locale, +// so e.g. std::isalnum(0xC5) can return true once an embedder calls +// setlocale(). HTTP grammars are defined over ASCII, so raw bytes must be +// classified without regard to the locale. +inline bool is_ascii_digit(char c) { return '0' <= c && c <= '9'; } + +inline bool is_ascii_alpha(char c) { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'); +} + +inline bool is_ascii_alnum(char c) { + return is_ascii_digit(c) || is_ascii_alpha(c); +} + namespace case_ignore { inline unsigned char to_lower(int c) { @@ -661,7 +675,7 @@ inline from_chars_result from_chars(const char *first, const char *last, for (; p != last; ++p) { char c = *p; int digit = -1; - if ('0' <= c && c <= '9') { + if (is_ascii_digit(c)) { digit = c - '0'; } else if ('a' <= c && c <= 'z') { digit = c - 'a' + 10; @@ -733,14 +747,14 @@ inline from_chars_result from_chars(const char *first, const char *last, return false; }; - for (; p != last && '0' <= *p && *p <= '9'; ++p) { + for (; p != last && is_ascii_digit(*p); ++p) { seen_digit = true; accumulate(*p); } if (p != last && *p == '.') { ++p; - for (; p != last && '0' <= *p && *p <= '9'; ++p) { + for (; p != last && is_ascii_digit(*p); ++p) { seen_digit = true; if (frac_digits < max_frac_digits && accumulate(*p)) { ++frac_digits; } } @@ -803,8 +817,8 @@ inline bool parse_url(const std::string &url, UrlComponents &uc) { // IPv6 host must be [a-fA-F0-9:]+ only if (uc.host.empty()) { return false; } for (auto c : uc.host) { - if (!((c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || - (c >= '0' && c <= '9') || c == ':')) { + if (!(is_ascii_digit(c) || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F') || c == ':')) { return false; } } @@ -1541,7 +1555,9 @@ public: class ThreadPool final : public TaskQueue { public: - explicit ThreadPool(size_t n, size_t max_n = 0, size_t mqr = 0); + explicit ThreadPool( + size_t n, size_t max_n = 0, size_t mqr = 0, + time_t idle_timeout_sec = CPPHTTPLIB_THREAD_POOL_IDLE_TIMEOUT); ThreadPool(const ThreadPool &) = delete; ~ThreadPool() override = default; @@ -1556,6 +1572,7 @@ private: size_t base_thread_count_; size_t max_thread_count_; size_t max_queued_requests_; + time_t idle_timeout_sec_; size_t idle_thread_count_; bool shutdown_; @@ -1680,6 +1697,35 @@ make_multipart_content_provider(const UploadFormDataItems &items, } // namespace detail +bool is_valid_multipart_boundary(const std::string &boundary); + +// Serializer for multipart/form-data request bodies. The boundary is owned +// by the writer so that per-part framing and the final terminator always +// agree. Field names and filenames are escaped following the WHATWG HTML +// standard ('"' -> %22, CR -> %0D, LF -> %0A); CR and LF are also escaped +// in content types. +class MultipartFormDataWriter { +public: + MultipartFormDataWriter(); + // precondition: is_valid_multipart_boundary(boundary) + explicit MultipartFormDataWriter(std::string boundary); + + const std::string &boundary() const; + std::string content_type() const; + + // In-memory items -> whole body (known length) + std::string serialize(const UploadFormDataItems &items) const; + size_t content_length(const UploadFormDataItems &items) const; + + // Per-part framing for streaming via a content provider + std::string item_begin(const UploadFormData &item) const; + static std::string item_end(); + std::string finish() const; + +private: + std::string boundary_; +}; + class Server { public: using Handler = std::function; @@ -2897,9 +2943,7 @@ template inline constexpr size_t str_len(const char (&)[N]) { } inline bool is_numeric(const std::string &str) { - return !str.empty() && - std::all_of(str.cbegin(), str.cend(), - [](unsigned char c) { return std::isdigit(c); }); + return !str.empty() && std::all_of(str.cbegin(), str.cend(), is_ascii_digit); } inline size_t get_header_value_u64(const Headers &headers,