diff --git a/common/arg.cpp b/common/arg.cpp index cc067a32c..9a9bb438c 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1042,11 +1042,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex // we define here to make sure it's included in llama-gen-docs if (ex == LLAMA_EXAMPLE_COMPLETION) { params.use_jinja = false; // disable jinja by default - } else if (ex == LLAMA_EXAMPLE_MTMD) { params.use_jinja = false; // disable jinja by default params.sampling.temp = 0.2; // lower temp by default for better quality - } else if (ex == LLAMA_EXAMPLE_SERVER) { params.n_parallel = -1; // auto by default } @@ -1067,7 +1065,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex sampler_type_names.pop_back(); // remove last semicolon } - /** * filter options by example * rules: @@ -1081,7 +1078,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } }; - add_opt(common_arg( {"-h", "--help", "--usage"}, "print usage and exit", @@ -4082,7 +4078,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.top_k = 0; params.sampling.min_p = 0.01f; params.use_jinja = true; - //params.default_template_kwargs["reasoning_effort"] = "\"high\""; } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); @@ -4101,7 +4096,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.top_k = 0; params.sampling.min_p = 0.01f; params.use_jinja = true; - //params.default_template_kwargs["reasoning_effort"] = "\"high\""; } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); diff --git a/common/common.cpp b/common/common.cpp index b3c10e8bb..01f360c9f 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1395,8 +1395,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode if (params.warmup) { LOG_INF("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__); - llama_set_warmup(lctx, true); - std::vector tmp; llama_token bos = llama_vocab_bos(vocab); llama_token eos = llama_vocab_eos(vocab); @@ -1427,7 +1425,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode llama_memory_clear(llama_get_memory(lctx), true); llama_synchronize(lctx); llama_perf_context_reset(lctx); - llama_set_warmup(lctx, false); // reset samplers to reset RNG state after warmup to the seeded state res->reset_samplers(); @@ -1569,6 +1566,7 @@ struct llama_context_params common_context_params_to_llama(const common_params & cparams.n_ctx = params.n_ctx; cparams.n_seq_max = params.n_parallel; cparams.n_rs_seq = params.speculative.need_n_rs_seq(); + cparams.n_outputs_max = std::max(params.n_outputs_max, 0); cparams.n_batch = params.n_batch; cparams.n_ubatch = params.n_ubatch; cparams.n_threads = params.cpuparams.n_threads; diff --git a/common/common.h b/common/common.h index 34912128d..f2a70a322 100644 --- a/common/common.h +++ b/common/common.h @@ -278,6 +278,7 @@ struct common_params_sampling { std::vector reasoning_budget_end; // end tag token sequence std::vector reasoning_budget_forced; // forced sequence (message + end tag) std::string reasoning_budget_message; // message injected before end tag when budget exhausted + bool reasoning_control = false; // create the budget sampler on demand so reasoning can be ended at runtime bool backend_sampling = false; @@ -432,6 +433,7 @@ struct common_params { int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited) int32_t n_parallel = 1; // number of parallel sequences to decode int32_t n_sequences = 1; // number of sequences to decode + int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch) int32_t grp_attn_n = 1; // group-attention factor int32_t grp_attn_w = 512; // group-attention width int32_t n_print = -1; // print token count every n tokens (-1 = disabled) diff --git a/common/sampling.cpp b/common/sampling.cpp index bbfa9a9ec..85f8ed50b 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -293,7 +293,7 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st } // reasoning budget sampler (skip when budget is unlimited unless a lazy grammar is active, which needs rbudget for thinking-block suppression) - if (!params.reasoning_budget_start.empty() && !params.reasoning_budget_end.empty() && (params.grammar_lazy || params.reasoning_budget_tokens >= 0)) { + if (!params.reasoning_budget_start.empty() && !params.reasoning_budget_end.empty() && (params.grammar_lazy || params.reasoning_budget_tokens >= 0 || params.reasoning_control)) { rbudget = common_reasoning_budget_init( vocab, params.reasoning_budget_start, diff --git a/common/speculative.cpp b/common/speculative.cpp index dfbe2bdb6..26afebd1a 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1317,6 +1317,40 @@ static uint32_t common_get_enabled_speculative_configs(const std::vectortypes) { + switch (type) { + case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: + case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: + case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: + n_max = std::max(n_max, std::max(0, spec->draft.n_max)); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: + n_max = std::max(n_max, (int32_t) spec->ngram_simple.size_m); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: + n_max = std::max(n_max, (int32_t) spec->ngram_map_k.size_m); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: + n_max = std::max(n_max, (int32_t) spec->ngram_map_k4v.size_m); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_MOD: + n_max = std::max(n_max, std::max(0, spec->ngram_mod.n_max)); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: + n_max = std::max(n_max, (int32_t) 8); + break; + case COMMON_SPECULATIVE_TYPE_NONE: + case COMMON_SPECULATIVE_TYPE_COUNT: + break; + } + } + + return n_max; +} + // initialization of the speculative decoding system // common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) { @@ -1325,8 +1359,6 @@ common_speculative * common_speculative_init(common_params_speculative & params, { uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types); - bool has_draft_model_path = !params.draft.mparams.path.empty(); - bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3 bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; @@ -1359,16 +1391,6 @@ common_speculative * common_speculative_init(common_params_speculative & params, if (has_ngram_cache) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params)); } - if (has_draft_simple) { - if (!has_draft_model_path) { - LOG_WRN("%s: draft model is not specified - cannot use 'draft' type\n", __func__); - has_draft_simple = false; - } - } else if (has_draft_model_path && !has_mtp && !has_draft_eagle3) { - LOG_WRN("%s: draft model is specified but 'draft' speculative type is not explicitly enabled - enabling it\n", __func__); - has_draft_simple = true; - } - if (has_draft_simple) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params)); } diff --git a/common/speculative.h b/common/speculative.h index f24bac79e..deba7dac7 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -20,6 +20,9 @@ enum common_speculative_type common_speculative_type_from_name(const std::string // convert type to string std::string common_speculative_type_to_str(enum common_speculative_type type); +// return the max number of draft tokens based on the speculative parameters +int32_t common_speculative_n_max(const common_params_speculative * spec); + common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq); void common_speculative_free(common_speculative * spec); diff --git a/conversion/__init__.py b/conversion/__init__.py index 222005740..3ceb2d385 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -215,6 +215,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Starcoder2ForCausalLM": "starcoder", "Step3p5ForCausalLM": "step3", "StepVLForConditionalGeneration": "step3", + "Step3p7ForConditionalGeneration": "step3", "T5EncoderModel": "t5", "T5ForConditionalGeneration": "t5", "T5WithLMHeadModel": "t5", @@ -283,6 +284,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Sarashina2VisionForCausalLM": "sarashina2", "SmolVLMForConditionalGeneration": "smolvlm", "StepVLForConditionalGeneration": "step3", + "Step3p7ForConditionalGeneration": "step3", "UltravoxModel": "ultravox", "VoxtralForConditionalGeneration": "ultravox", "YoutuVLForConditionalGeneration": "youtuvl", diff --git a/conversion/base.py b/conversion/base.py index 55682c827..729ddbca4 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -2593,7 +2593,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st # Step3-VL keeps text config under text_config but uses a custom top-level architecture. # For text conversion we route to a dedicated text-only class. # TODO: refactor this later to avoid adding exception here - if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration"): + if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration"): return arch # if "architectures" is found in the sub-config, use that instead diff --git a/conversion/step3.py b/conversion/step3.py index ba867fb83..eeba66c7a 100644 --- a/conversion/step3.py +++ b/conversion/step3.py @@ -15,7 +15,7 @@ from .base import MmprojModel, ModelBase, TextModel, _MISTRAL_COMMON_DATASET_MEA from .qwen import Qwen3Model -@ModelBase.register("StepVLForConditionalGeneration") +@ModelBase.register("StepVLForConditionalGeneration", "Step3p7ForConditionalGeneration") class Step3VLVisionModel(MmprojModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -95,7 +95,7 @@ class Step3VLTextModel(Qwen3Model): model_arch = gguf.MODEL_ARCH.QWEN3 -@ModelBase.register("Step3p5ForCausalLM") +@ModelBase.register("Step3p5ForCausalLM", "Step3p7ForConditionalGeneration") class Step35Model(TextModel): model_arch = gguf.MODEL_ARCH.STEP35 @@ -203,11 +203,23 @@ class Step35Model(TextModel): if isinstance(rope_theta, list): rope_theta = rope_theta[0] base = float(rope_theta) - if (dim := self.hparams.get("head_dim")) is None: - dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] - dim = int(dim) - freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + if (storage_dim := self.hparams.get("head_dim")) is None: + storage_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] + storage_dim = int(storage_dim) + + # Llama 3 factors apply only to the rotary dims used by full_attention layers + # (partial_rotary_factor * head_dim). Remaining slots are padded with 1.0 so + # sliding_attention layers remain unaffected. set_gguf_parameters already + # guarantees at least one full_attention layer. + layer_types = (self.hparams.get("layer_types") or [])[: self.block_count] + partial_rotary_factors = (self.hparams.get("partial_rotary_factors") or [])[: self.block_count] + full_attention_factor = next( + float(f) for lt, f in zip(layer_types, partial_rotary_factors) if lt == "full_attention" + ) + rotary_dim = int(storage_dim * full_attention_factor) + + freqs = 1.0 / (base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float32) / rotary_dim)) factor = float(rope_params.get("factor", 8.0)) low_freq_factor = float(rope_params.get("low_freq_factor", 1.0)) @@ -228,4 +240,8 @@ class Step35Model(TextModel): smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor) rope_factors.append(1.0 / ((1.0 - smooth) / factor + smooth)) + # Pad to head_dim/2 with 1.0 so non-scaled layers remain neutral. + if len(rope_factors) < storage_dim // 2: + rope_factors.extend([1.0] * (storage_dim // 2 - len(rope_factors))) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32)) diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 3c8b6eaaf..ac5abb133 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -568,7 +568,6 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2); - constexpr int stride_tile_Q = DKQ/2 + 4; constexpr int stride_tile_K = nbatch_K2 + 4; constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4; @@ -604,9 +603,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( #pragma unroll for (int k0_start = (DKQ/2-1) - (DKQ/2-1) % nbatch_K2; k0_start >= 0; k0_start -= nbatch_K2) { const int k0_stop = k0_start + nbatch_K2 < DKQ/2 ? k0_start + nbatch_K2 : DKQ/2; - const int k0_diff = k0_stop - k0_start; if constexpr (nstages <= 1) { + const int k0_diff = k0_stop - k0_start; constexpr bool use_cp_async = nstages == 1; flash_attn_ext_f16_load_tile (K_h2 + int64_t(k_VKQ_0)*stride_K + k0_start, tile_K, k0_diff, stride_K, k_VKQ_sup); @@ -640,6 +639,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } } } else { + constexpr int stride_tile_Q = DKQ/2 + 4; #pragma unroll for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) { load_ldmatrix(Q_B[0], tile_Q + (threadIdx.y / np)*(T_B_KQ::I*stride_tile_Q) + k_KQ_0, stride_tile_Q); @@ -954,9 +954,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( for (int i0_start = 0; i0_start < DV; i0_start += 2*nbatch_V2) { static_assert(DV % (2*nbatch_V2) == 0, "bad loop size"); const int i0_stop = i0_start + 2*nbatch_V2; - const int i0_diff = i0_stop - i0_start; if constexpr (nstages <= 1) { + const int i0_diff = i0_stop - i0_start; if (!V_is_K_view || i0_stop > 2*nbatch_K2) { constexpr bool use_cp_async = nstages == 1; flash_attn_ext_f16_load_tile diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index 018d5d37d..7cfda6523 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -43,7 +43,6 @@ gated_delta_net_cuda(const float * q, // output state layout (per-slot D * n_seqs) — same per-(seq,head) offset as before. const int64_t state_in_offset = sequence * K * H * S_v * S_v + h_idx * S_v * S_v; const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v; - const int64_t state_size_per_token = S_v * S_v * H * n_seqs; // per-slot stride in output state += state_out_offset; curr_state += state_in_offset + col * S_v; attn_data += (sequence * n_tokens * H + h_idx) * S_v; @@ -61,10 +60,6 @@ gated_delta_net_cuda(const float * q, s_shard[r] = curr_state[i]; } - // slot mapping: target_slot = t - shift. When n_tokens < K only the last n_tokens slots - // are written; earlier slots are left untouched (caller-owned). - const int shift = (int) n_tokens - K; - for (int t = 0; t < n_tokens; t++) { const float * q_t = q + iq3 * sq3 + t * sq2 + iq1 * sq1; const float * k_t = k + iq3 * sq3 + t * sq2 + iq1 * sq1; @@ -148,6 +143,11 @@ gated_delta_net_cuda(const float * q, attn_data += S_v * H; if constexpr (keep_rs_t) { + // slot mapping: target_slot = t - shift. When n_tokens < K only the last n_tokens slots + // are written; earlier slots are left untouched (caller-owned). + const int shift = (int) n_tokens - K; + + const int64_t state_size_per_token = S_v * S_v * H * n_seqs; // per-slot stride in output const int target_slot = t - shift; if (target_slot >= 0 && target_slot < K) { float * curr_state = (dst + attn_score_elems) + target_slot * state_size_per_token + state_out_offset; diff --git a/ggml/src/ggml-cuda/mmf.cuh b/ggml/src/ggml-cuda/mmf.cuh index c2a8d54c9..d55cc1ec7 100644 --- a/ggml/src/ggml-cuda/mmf.cuh +++ b/ggml/src/ggml-cuda/mmf.cuh @@ -91,7 +91,7 @@ static __global__ void mul_mat_f( const int row0 = blockIdx.x * rows_per_block; int expert_idx = 0; - int col_base = 0; + [[maybe_unused]] int col_base = 0; const int channel_dst = has_ids ? 0 : blockIdx.y; @@ -122,12 +122,12 @@ static __global__ void mul_mat_f( ids += col_offset * stride_row_id; } - const float2 * y2 = (const float2 *) y; + [[maybe_unused]] const float2 * y2 = (const float2 *) y; extern __shared__ char data_mmv[]; char * shmem_base = data_mmv; - int * slot_map = (int *) shmem_base; + [[maybe_unused]] int * slot_map = (int *) shmem_base; char * compute_base = has_ids ? (shmem_base + GGML_PAD(cols_per_block, 16) * sizeof(int)) : shmem_base; tile_C C[ntA][ntB]; diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu index 09d95f309..3d6de64b7 100644 --- a/ggml/src/ggml-cuda/mmvf.cu +++ b/ggml/src/ggml-cuda/mmvf.cu @@ -80,9 +80,8 @@ static __global__ void mul_mat_vec_f( gate_x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row*stride_row; } - const int channel_bias = ids ? channel_x : channel_dst; - if constexpr (has_fusion) { + const int channel_bias = ids ? channel_x : channel_dst; if (use_bias) { x_bias += int64_t(sample_dst)*stride_sample_dst + channel_bias*stride_channel_dst; } @@ -95,7 +94,7 @@ static __global__ void mul_mat_vec_f( extern __shared__ char data_mmv[]; float * buf_iw = (float *) data_mmv; - float * buf_iw_gate = nullptr; + [[maybe_unused]] float * buf_iw_gate = nullptr; if constexpr (has_fusion) { buf_iw_gate = (float *) (data_mmv + warp_size*sizeof(float)); } @@ -123,7 +122,7 @@ static __global__ void mul_mat_vec_f( if constexpr (std::is_same_v) { const float2 * x2 = (const float2 *) x; - const float2 * gate_x2 = nullptr; + [[maybe_unused]] const float2 * gate_x2 = nullptr; if constexpr (has_fusion) { if (use_gate) { gate_x2 = (const float2 *) gate_x; @@ -155,7 +154,7 @@ static __global__ void mul_mat_vec_f( } } else if constexpr (std::is_same_v) { const half2 * x2 = (const half2 *) x; - const half2 * gate_x2 = nullptr; + [[maybe_unused]] const half2 * gate_x2 = nullptr; if constexpr (has_fusion) { if (use_gate) { gate_x2 = (const half2 *) gate_x; @@ -266,7 +265,7 @@ static __global__ void mul_mat_vec_f( } #else const nv_bfloat162 * x2 = (const nv_bfloat162 *) x; - const nv_bfloat162 * gate_x2 = nullptr; + [[maybe_unused]] const nv_bfloat162 * gate_x2 = nullptr; if constexpr (has_fusion) { if (use_gate) { gate_x2 = (const nv_bfloat162 *) gate_x; @@ -274,7 +273,7 @@ static __global__ void mul_mat_vec_f( } for (int col2 = tid; col2 < ncols2; col2 += block_size) { const nv_bfloat162 tmpx = x2[col2]; - nv_bfloat162 tmpx_gate; + [[maybe_unused]] nv_bfloat162 tmpx_gate; if constexpr (has_fusion) { if (use_gate) { tmpx_gate = gate_x2[col2]; diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index ecb6fdeda..86b4a4930 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -515,7 +515,7 @@ static __global__ void mul_mat_vec_q( bool use_gate = false; bool use_bias = false; bool use_gate_bias = false; - const void * vgate = nullptr; + [[maybe_unused]] const void * vgate = nullptr; const float * x_bias = nullptr; const float * gate_bias = nullptr; ggml_glu_op active_glu; @@ -531,8 +531,8 @@ static __global__ void mul_mat_vec_q( } - float x_biases[ncols_dst] = { 0.0f }; - float gate_biases[ncols_dst] = { 0.0f }; + [[maybe_unused]] float x_biases[ncols_dst] = { 0.0f }; + [[maybe_unused]] float gate_biases[ncols_dst] = { 0.0f }; if constexpr (has_fusion) { const uint32_t channel_bias = ids ? channel_x : channel_dst; if (use_bias) { @@ -589,12 +589,7 @@ static __global__ void mul_mat_vec_q( } __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size]; - __shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size]; - if constexpr (!has_fusion) { - (void) tmp_shared_gate; - } else if (!use_gate) { - (void) tmp_shared_gate; - } + [[maybe_unused]] __shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size]; if (threadIdx.y > 0) { #pragma unroll diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu index da20c9aab..c4253bfa4 100644 --- a/ggml/src/ggml-cuda/topk-moe.cu +++ b/ggml/src/ggml-cuda/topk-moe.cu @@ -134,7 +134,7 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * // selection_wt is only needed when bias is present (selection uses wt + bias) // when no bias, we use wt directly for both selection and weight values - float selection_wt[has_bias ? experts_per_thread : 1]; + [[maybe_unused]] float selection_wt[has_bias ? experts_per_thread : 1]; if constexpr (has_bias) { #pragma unroll diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index 14b232240..8a1228ccd 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -56,7 +56,7 @@ struct htp_opnode { } std::vector get_inputs() const { - std::vector inputs; + std::vector inputs(GGML_MAX_SRC, nullptr); std::vector outputs; outputs.push_back(node); for (const auto * f : fused) { @@ -70,20 +70,38 @@ struct htp_opnode { return false; }; + int count = 0; auto add_input = [&](const ggml_tensor * t) { if (t && !contains(outputs, t) && !contains(inputs, t)) { - inputs.push_back(t); + if (count < (int)inputs.size()) { + inputs[count++] = t; + } else { + inputs.push_back(t); + } } }; - for (int i = 0; i < GGML_MAX_SRC && node->src[i]; i++) { - add_input(node->src[i]); - } - for (const auto * f : fused) { - for (int i = 0; i < GGML_MAX_SRC && f->src[i]; i++) { - add_input(f->src[i]); + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (fused.empty()) { + inputs[i] = node->src[i]; + } else { + if (node->src[i]) { + add_input(node->src[i]); + } } } + for (const auto * f : fused) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (f->src[i]) { + add_input(f->src[i]); + } + } + } + + if (!fused.empty()) { + inputs.resize(count); + } + return inputs; } @@ -108,6 +126,9 @@ struct htp_opformat { char names[64 * GGML_MAX_SRC]; int format_tensor_dims(char * str, const struct ggml_tensor * t) { + if (!t) { + return sprintf(str, "NONE"); + } if (t->ne[2] == 1 && t->ne[3] == 1) { return sprintf(str, "%d:%d", (int) t->ne[0], (int) t->ne[1]); } else { @@ -136,6 +157,9 @@ struct htp_opformat { } int format_tensor_strides(char * str, const struct ggml_tensor * t) { + if (!t) { + return sprintf(str, "NONE"); + } const char * c = ggml_is_contiguous(t) ? "" : "!"; if (t->ne[2] == 1 && t->ne[3] == 1) { @@ -170,11 +194,11 @@ struct htp_opformat { auto inputs = node.get_inputs(); if (!inputs.empty()) { - p += sprintf(p, "%s", ggml_type_name(inputs[0]->type)); + p += sprintf(p, "%s", inputs[0] ? ggml_type_name(inputs[0]->type) : "NONE"); for (size_t i = 1; i < inputs.size(); i++) { p += sprintf(p, " x "); - p += sprintf(p, "%s", ggml_type_name(inputs[i]->type)); + p += sprintf(p, "%s", inputs[i] ? ggml_type_name(inputs[i]->type) : "NONE"); } p += sprintf(p, " -> "); @@ -184,7 +208,7 @@ struct htp_opformat { } const char * tensor_buff_name(const struct ggml_tensor * t) { - if (t->buffer) { + if (t && t->buffer) { return ggml_backend_buffer_name(t->buffer); } return "NONE"; @@ -213,11 +237,11 @@ struct htp_opformat { auto inputs = node.get_inputs(); if (!inputs.empty()) { - p += sprintf(p, "%s", inputs[0]->name); + p += sprintf(p, "%s", inputs[0] ? inputs[0]->name : "NONE"); for (size_t i = 1; i < inputs.size(); i++) { p += sprintf(p, " x "); - p += sprintf(p, "%s", inputs[i]->name); + p += sprintf(p, "%s", inputs[i] ? inputs[i]->name : "NONE"); } p += sprintf(p, " -> "); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 9138678b9..bc3d83b67 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1113,7 +1113,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_GLU_OP_SWIGLU_OAI: case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: - return ggml_is_contiguous_1(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; + return ggml_is_contiguous_1(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); default: return false; } diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 4adf4614a..2bd310d94 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -1421,7 +1421,8 @@ template [[host_name("kernel_repeat_f16")]] kernel kernel_repeat_t kernel_repeat template [[host_name("kernel_repeat_i32")]] kernel kernel_repeat_t kernel_repeat; template [[host_name("kernel_repeat_i16")]] kernel kernel_repeat_t kernel_repeat; -kernel void kernel_reglu_f32( +template +kernel void kernel_reglu( constant ggml_metal_kargs_glu & args, device const char * src0, device const char * src1, @@ -1429,19 +1430,25 @@ kernel void kernel_reglu_f32( uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { const float x0 = src0_row[i0]; const float x1 = src1_row[i0]; - dst_row[i0] = x0*x1*(x0 > 0.0f); + dst_row[i0] = (T)(x0*x1*(x0 > 0.0f)); } } -kernel void kernel_geglu_f32( +typedef decltype(kernel_reglu) kernel_reglu_t; + +template [[host_name("kernel_reglu_f32")]] kernel kernel_reglu_t kernel_reglu; +template [[host_name("kernel_reglu_f16")]] kernel kernel_reglu_t kernel_reglu; + +template +kernel void kernel_geglu( constant ggml_metal_kargs_glu & args, device const char * src0, device const char * src1, @@ -1449,9 +1456,9 @@ kernel void kernel_geglu_f32( uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { const float x0 = src0_row[i0]; @@ -1459,11 +1466,17 @@ kernel void kernel_geglu_f32( const float gelu = 0.5f*x0*(1.0f + precise::tanh(SQRT_2_OVER_PI*x0*(1.0f + GELU_COEF_A*x0*x0))); - dst_row[i0] = gelu*x1; + dst_row[i0] = (T)(gelu*x1); } } -kernel void kernel_swiglu_f32( +typedef decltype(kernel_geglu) kernel_geglu_t; + +template [[host_name("kernel_geglu_f32")]] kernel kernel_geglu_t kernel_geglu; +template [[host_name("kernel_geglu_f16")]] kernel kernel_geglu_t kernel_geglu; + +template +kernel void kernel_swiglu( constant ggml_metal_kargs_glu & args, device const char * src0, device const char * src1, @@ -1471,9 +1484,9 @@ kernel void kernel_swiglu_f32( uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { const float x0 = src0_row[i0]; @@ -1481,11 +1494,17 @@ kernel void kernel_swiglu_f32( const float silu = x0 / (1.0f + exp(-x0)); - dst_row[i0] = silu*x1; + dst_row[i0] = (T)(silu*x1); } } -kernel void kernel_swiglu_oai_f32( +typedef decltype(kernel_swiglu) kernel_swiglu_t; + +template [[host_name("kernel_swiglu_f32")]] kernel kernel_swiglu_t kernel_swiglu; +template [[host_name("kernel_swiglu_f16")]] kernel kernel_swiglu_t kernel_swiglu; + +template +kernel void kernel_swiglu_oai( constant ggml_metal_kargs_glu & args, device const char * src0, device const char * src1, @@ -1493,9 +1512,9 @@ kernel void kernel_swiglu_oai_f32( uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { float x0 = src0_row[i0]; @@ -1507,11 +1526,17 @@ kernel void kernel_swiglu_oai_f32( float out_glu = x0 / (1.0f + exp(-x0 * args.alpha)); out_glu = out_glu * (1.0f + x1); - dst_row[i0] = out_glu; + dst_row[i0] = (T)out_glu; } } -kernel void kernel_geglu_erf_f32( +typedef decltype(kernel_swiglu_oai) kernel_swiglu_oai_t; + +template [[host_name("kernel_swiglu_oai_f32")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai; +template [[host_name("kernel_swiglu_oai_f16")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai; + +template +kernel void kernel_geglu_erf( constant ggml_metal_kargs_glu & args, device const char * src0, device const char * src1, @@ -1519,9 +1544,9 @@ kernel void kernel_geglu_erf_f32( uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { const float x0 = src0_row[i0]; @@ -1529,11 +1554,17 @@ kernel void kernel_geglu_erf_f32( const float gelu_erf = 0.5f*x0*(1.0f+erf_approx(x0*SQRT_2_INV)); - dst_row[i0] = gelu_erf*x1; + dst_row[i0] = (T)(gelu_erf*x1); } } -kernel void kernel_geglu_quick_f32( +typedef decltype(kernel_geglu_erf) kernel_geglu_erf_t; + +template [[host_name("kernel_geglu_erf_f32")]] kernel kernel_geglu_erf_t kernel_geglu_erf; +template [[host_name("kernel_geglu_erf_f16")]] kernel kernel_geglu_erf_t kernel_geglu_erf; + +template +kernel void kernel_geglu_quick( constant ggml_metal_kargs_glu & args, device const char * src0, device const char * src1, @@ -1541,9 +1572,9 @@ kernel void kernel_geglu_quick_f32( uint tgpig[[threadgroup_position_in_grid]], uint tpitg[[thread_position_in_threadgroup]], uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { const float x0 = src0_row[i0]; @@ -1551,10 +1582,15 @@ kernel void kernel_geglu_quick_f32( const float gelu_quick = x0*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x0))); - dst_row[i0] = gelu_quick*x1; + dst_row[i0] = (T)(gelu_quick*x1); } } +typedef decltype(kernel_geglu_quick) kernel_geglu_quick_t; + +template [[host_name("kernel_geglu_quick_f32")]] kernel kernel_geglu_quick_t kernel_geglu_quick; +template [[host_name("kernel_geglu_quick_f16")]] kernel kernel_geglu_quick_t kernel_geglu_quick; + kernel void kernel_op_sum_f32( constant ggml_metal_kargs_sum & args, device const float * src0, diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index dabfaccac..53708a486 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -64,8 +64,10 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV { #include #include #include +#include #include #include +#include #include #if defined(_MSC_VER) @@ -164,8 +166,9 @@ struct vk_pipeline_struct { uint32_t align; // true if fields have been set by ggml_vk_create_pipeline bool initialized {}; - // set to true to request the pipeline is compiled - std::atomic needed {}; + // true while a compile is in flight, used to dedupe concurrent claims. + // Protected by device->compile_mutex. + bool compile_pending {}; // set to true when the shader has been compiled std::atomic compiled {}; // number of registers used, extracted from pipeline executable properties @@ -624,6 +627,14 @@ static constexpr std::initializer_list> rms_norm_mul_rope_vie struct vk_device_struct { std::recursive_mutex mutex; + mutable std::shared_mutex pinned_memory_mutex; + + // Guards compile_pending, all_pipelines, and the dynamic pipeline maps + // (flash_attn, fa_mask_opt, solve_tri, conv2d, etc). The actual compile + // runs with no lock held, so different pipelines can compile in parallel. + // Lock order is device->mutex -> compile_mutex, never the reverse. + std::mutex compile_mutex; + std::condition_variable compile_cv; vk::PhysicalDevice physical_device; vk::PhysicalDeviceProperties properties; @@ -1733,7 +1744,7 @@ struct ggml_vk_garbage_collector { }; static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx); -static void ggml_vk_load_shaders(vk_device& device); +static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr); static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx); static bool vk_memory_logger_enabled = false; @@ -2200,11 +2211,6 @@ static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) { ctx->device->device.resetFences({ ctx->fence }); } -// variables to track number of compiles in progress -static uint32_t compile_count = 0; -static std::mutex compile_count_mutex; -static std::condition_variable compile_count_cond; - static constexpr uint32_t kSpvOpCooperativeMatrixLoadTensorNV = 5367; static constexpr uint32_t kSpvCapabilityCooperativeMatrixDecodeVectorNV = 5447; static constexpr uint32_t kSpvTensorAddressingDecodeVectorFuncBit = 0x4; @@ -2499,7 +2505,6 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin std::cerr << "ggml_vulkan: " << e.what() << std::endl; throw e; } - pipeline->compiled = true; if (vk_instance.debug_utils_support) { vk::DebugUtilsObjectNameInfoEXT duoni; @@ -2548,14 +2553,13 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin } } - device->all_pipelines.push_back(pipeline); - { - std::lock_guard guard(compile_count_mutex); - assert(compile_count > 0); - compile_count--; + std::lock_guard guard(device->compile_mutex); + device->all_pipelines.push_back(pipeline); + pipeline->compiled = true; + pipeline->compile_pending = false; } - compile_count_cond.notify_all(); + device->compile_cv.notify_all(); } static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) { @@ -2571,8 +2575,7 @@ static void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, VK_LOG_DEBUG("ggml_pipeline_request_descriptor_sets(" << pipeline->name << ", " << n << ")"); ctx->pipeline_descriptor_set_requirements += n; if (!pipeline->compiled) { - pipeline->needed = true; - ggml_vk_load_shaders(ctx->device); + ggml_vk_load_shaders(ctx->device, pipeline); } ggml_pipeline_allocate_descriptor_sets(ctx); } @@ -3571,10 +3574,26 @@ static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type #endif } -static void ggml_vk_load_shaders(vk_device& device) { +// load_shaders walks the pipeline list under compile_mutex and either claims +// the requested pipeline for compilation or, if another thread is already +// compiling it, drops the lock and waits on compile_cv. Compiles themselves +// run unlocked. +struct CompileTask { + vk_pipeline pipeline; + size_t spv_size; + const void * spv_data; + std::string entrypoint; + uint32_t parameter_count; + std::array wg_denoms; + std::vector specialization_constants; + bool disable_robustness; + bool require_full_subgroups; + uint32_t required_subgroup_size; +}; + +static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")"); - std::lock_guard guard(device->mutex); // some shaders have a minimum subgroup size const uint32_t subgroup_size_8 = std::max(device->subgroup_size, 8u); const uint32_t subgroup_size_16 = std::max(device->subgroup_size, 16u); @@ -3604,6 +3623,15 @@ static void ggml_vk_load_shaders(vk_device& device) { l_mmqid_wg_denoms, m_mmqid_wg_denoms, s_mmqid_wg_denoms; uint32_t l_align, m_align, s_align; + + vk_pipeline wait_pipeline; + CompileTask claimed_task {}; + bool has_claimed_task = false; + + // The rest of the walk reads and writes shared device state, so hold the + // lock until we're done deciding what to compile. + std::unique_lock compile_lock(device->compile_mutex); + if (device->coopmat2) { // spec constants and tile sizes for non-quant matmul/matmul_id l_warptile = { 256, 128, 256, 64, 1 }; @@ -3789,7 +3817,6 @@ static void ggml_vk_load_shaders(vk_device& device) { device->pipeline_matmul_id_bf16 = std::make_shared(); } - std::vector> compiles; auto const &ggml_vk_create_pipeline = [&](vk_device& device, vk_pipeline& base_pipeline, const char *name, size_t spv_size, const void* spv_data, const char *entrypoint, uint32_t parameter_count, uint32_t push_constant_size, std::array wg_denoms, const std::vector& specialization_constants, uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { @@ -3823,23 +3850,33 @@ static void ggml_vk_load_shaders(vk_device& device) { #endif } - if (!pipeline->needed || pipeline->compiled) { + // We only care about the pipeline this call asked for; the rest + // (including the 64-bit indexing variant) are handled by their + // own request_descriptor_sets / load_shaders calls. + if (pipeline.get() != requested.get()) { continue; } - // TODO: We're no longer benefitting from the async compiles (shaders are - // compiled individually, as needed) and this complexity can be removed. - { - // wait until fewer than N compiles are in progress - uint32_t N = std::max(1u, std::thread::hardware_concurrency()); - std::unique_lock guard(compile_count_mutex); - while (compile_count >= N) { - compile_count_cond.wait(guard); - } - compile_count++; + + if (pipeline->compiled) { + continue; } - compiles.push_back(std::async(ggml_vk_create_pipeline_func, std::ref(device), std::ref(pipeline), spv_size, spv_data, entrypoint, - parameter_count, wg_denoms, specialization_constants, disable_robustness, require_full_subgroups, required_subgroup_size)); + wait_pipeline = pipeline; + + if (!pipeline->compile_pending) { + pipeline->compile_pending = true; + claimed_task.pipeline = pipeline; + claimed_task.spv_size = spv_size; + claimed_task.spv_data = spv_data; + claimed_task.entrypoint = entrypoint; + claimed_task.parameter_count = parameter_count; + claimed_task.wg_denoms = wg_denoms; + claimed_task.specialization_constants = specialization_constants; + claimed_task.disable_robustness = disable_robustness; + claimed_task.require_full_subgroups = require_full_subgroups; + claimed_task.required_subgroup_size = required_subgroup_size; + has_claimed_task = true; + } } }; @@ -5336,8 +5373,25 @@ static void ggml_vk_load_shaders(vk_device& device) { } } - for (auto &c : compiles) { - c.wait(); + // Drop compile_mutex so other threads can walk while we compile. + compile_lock.unlock(); + + // Compile what we claimed; create_pipeline_func reacquires compile_mutex + // at the end to flip compile_pending/compiled and notify waiters. + if (has_claimed_task) { + auto & task = claimed_task; + ggml_vk_create_pipeline_func(device, task.pipeline, task.spv_size, task.spv_data, + task.entrypoint, task.parameter_count, task.wg_denoms, + task.specialization_constants, task.disable_robustness, + task.require_full_subgroups, task.required_subgroup_size); + } + + // Another thread may be compiling the pipeline we need; block on it here. + if (wait_pipeline) { + std::unique_lock wait_lock(device->compile_mutex); + device->compile_cv.wait(wait_lock, [&] { + return wait_pipeline->compiled.load(); + }); } } @@ -7030,7 +7084,7 @@ static void * ggml_vk_host_malloc(vk_device& device, size_t size) { return nullptr; } - std::lock_guard guard(device->mutex); + std::lock_guard guard(device->pinned_memory_mutex); device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf)); return buf->ptr; @@ -7041,7 +7095,7 @@ static void ggml_vk_host_free(vk_device& device, void* ptr) { return; } VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")"); - std::lock_guard guard(device->mutex); + std::lock_guard guard(device->pinned_memory_mutex); vk_buffer buf; size_t index; @@ -7065,7 +7119,7 @@ static void ggml_vk_host_free(vk_device& device, void* ptr) { } static void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) { - std::lock_guard guard(device->mutex); + std::shared_lock guard(device->pinned_memory_mutex); buf = nullptr; buf_offset = 0; for (size_t i = 0; i < device->pinned_memory.size(); i++) { @@ -9748,7 +9802,7 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx vk_pipeline pipeline = nullptr; { - std::lock_guard guard(ctx->device->mutex); + std::lock_guard guard(ctx->device->compile_mutex); auto &pipelines = ctx->device->pipeline_flash_attn_f32_f16; auto it = pipelines.find(fa_pipeline_state); if (it != pipelines.end()) { @@ -9812,13 +9866,15 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx vk_pipeline pipeline_fa_mask_opt = nullptr; if (use_mask_opt) { - std::lock_guard guard(ctx->device->mutex); - auto &pipelines = ctx->device->pipeline_fa_mask_opt; - auto it = pipelines.find({Br, Bc}); - if (it != pipelines.end()) { - pipeline_fa_mask_opt = it->second; - } else { - pipelines[{Br, Bc}] = pipeline_fa_mask_opt = std::make_shared(); + { + std::lock_guard guard(ctx->device->compile_mutex); + auto &pipelines = ctx->device->pipeline_fa_mask_opt; + auto it = pipelines.find({Br, Bc}); + if (it != pipelines.end()) { + pipeline_fa_mask_opt = it->second; + } else { + pipelines[{Br, Bc}] = pipeline_fa_mask_opt = std::make_shared(); + } } assert(pipeline_fa_mask_opt); ggml_pipeline_request_descriptor_sets(ctx, pipeline_fa_mask_opt, 1); @@ -10352,7 +10408,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const vk_pipeline pipeline = nullptr; { - std::lock_guard guard(ctx->device->mutex); + std::lock_guard guard(ctx->device->compile_mutex); auto it = ctx->device->pipeline_solve_tri_f32.find(solve_tri_pipeline_state); if (it != ctx->device->pipeline_solve_tri_f32.end()) { pipeline = it->second; @@ -10511,7 +10567,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const vk_pipeline pipeline = nullptr; { - std::lock_guard guard(ctx->device->mutex); + std::lock_guard guard(ctx->device->compile_mutex); auto it = pipelines->find(conv2d_pipeline_state); if (it != pipelines->end()) { pipeline = it->second; diff --git a/include/llama.h b/include/llama.h index e2b6a9bad..667bbd576 100644 --- a/include/llama.h +++ b/include/llama.h @@ -342,6 +342,7 @@ extern "C" { uint32_t n_ubatch; // physical maximum batch size uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) int32_t n_threads; // number of threads to use for generation int32_t n_threads_batch; // number of threads to use for batch processing @@ -978,7 +979,11 @@ extern "C" { // Set whether the model is in warmup mode or not // If true, all model tensors are activated during llama_decode() to load and cache their weights. - LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup); + // + // note: using this can cause extra graph reallocations because it changes the graph topology with MoE models, + // so it is generally not recommended to use in practice. will be removed in the future + DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup), + "user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]"); // Set abort callback LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 0df1e1128..083e56e1a 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -185,6 +185,8 @@ llama_context::llama_context( cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch); + cparams.n_outputs_max = params.n_outputs_max == 0 ? cparams.n_batch : params.n_outputs_max; + cparams.op_offload = params.op_offload; cparams.kv_unified = params.kv_unified; @@ -230,6 +232,7 @@ llama_context::llama_context( LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base); LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale); LLAMA_LOG_INFO("%s: n_rs_seq = %u\n", __func__, cparams.n_rs_seq); + LLAMA_LOG_INFO("%s: n_outputs_max = %u\n", __func__, cparams.n_outputs_max); if (cparams.n_ctx_seq < hparams.n_ctx_train) { LLAMA_LOG_WARN("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n", @@ -540,7 +543,7 @@ void llama_context::sched_reserve() { // note: n_outputs must match n_tokens for embedding models with mean/rank pooling, // because build_pooling creates inp_mean with shape [n_tokens, n_seqs] and multiplies // it with t_embd which is reduced to [n_outputs, ...] via out_ids. if n_outputs != n_tokens, - // the ggml_mul_mat assertion fails. this matches the pp reservation below (line ~553). + // the ggml_mul_mat assertion fails. const uint32_t n_tokens_ch = 16*n_seqs; auto * gf = graph_reserve(n_tokens_ch, n_seqs, n_tokens_ch, mctx.get(), true); if (!gf) { @@ -586,16 +589,18 @@ void llama_context::sched_reserve() { int n_splits_tg = -1; int n_nodes_tg = -1; + const uint32_t n_outputs_pp = std::min(n_tokens, cparams.n_outputs_max); + // reserve pp (prompt processing) graph first so that buffers are only allocated once { - auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get(), + auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr); if (!gf) { if (cparams.pipeline_parallel) { LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__); cparams.pipeline_parallel = false; sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload)); - gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get()); + gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get()); } if (!gf) { throw std::runtime_error("failed to allocate compute pp buffers"); @@ -623,7 +628,7 @@ void llama_context::sched_reserve() { // // auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get()); // - auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get(), model.hparams.no_alloc); + auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc); if (!gf) { throw std::runtime_error("failed to allocate compute pp buffers"); } @@ -784,7 +789,9 @@ bool llama_context::memory_update(bool optimize) { const uint32_t n_seqs = cparams.n_seq_max; const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); - auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get()); + const uint32_t n_outputs_max = std::min(n_tokens, cparams.n_outputs_max); + + auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_max, mctx.get()); if (!gf) { LLAMA_LOG_ERROR("%s: failed to reserve graph after the memory update\n", __func__); } @@ -2150,6 +2157,8 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) { this->n_outputs = 0; + GGML_ASSERT(n_outputs_max <= cparams.n_outputs_max); + return n_outputs_max; } @@ -2236,8 +2245,6 @@ ggml_cgraph * llama_context::graph_reserve( if (n_tokens % n_seqs != 0) { n_tokens = ((n_tokens + (n_seqs - 1)) / n_seqs) * n_seqs; // round to next multiple of n_seqs - n_outputs = std::max(n_outputs, n_tokens); - //LLAMA_LOG_DEBUG("%s: making n_tokens a multiple of n_seqs - n_tokens = %u, n_seqs = %u, n_outputs = %u\n", __func__, n_tokens, n_seqs, n_outputs); } @@ -3347,6 +3354,7 @@ llama_context_params llama_context_default_params() { /*.n_ubatch =*/ 512, /*.n_seq_max =*/ 1, /*.n_rs_seq =*/ 0, + /*.n_outputs_max =*/ 0, /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, /*.ctx_type =*/ LLAMA_CONTEXT_TYPE_DEFAULT, diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 20ec59fe3..52e1c4f54 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -13,6 +13,7 @@ struct llama_cparams { uint32_t n_ubatch; uint32_t n_seq_max; uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback + uint32_t n_outputs_max; // max outputs supported by the context int32_t n_threads; // number of threads to use for generation int32_t n_threads_batch; // number of threads to use for batch processing @@ -38,7 +39,7 @@ struct llama_cparams { bool fused_gdn_ch; // use fused gated delta net (chunked) bool auto_fgdn; bool no_perf; - bool warmup; + bool warmup; // TODO: remove [TAG_LLAMA_GRAPH_NO_WARMUP] bool op_offload; bool kv_unified; bool pipeline_parallel; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index ba4ee708b..015974084 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1885,7 +1885,19 @@ void llama_kv_cache::state_write(llama_io_write_i & io, llama_seq_id seq_id, lla uint32_t cell_range_begin = cells.size(); for (uint32_t i = 0; i < cells.size(); ++i) { - if (!cells.is_empty(i) && (seq_id == -1 || cells.seq_has(i, seq_id))) { + bool add_cell = true; + + add_cell = add_cell && !cells.is_empty(i); + add_cell = add_cell && (seq_id == -1 || cells.seq_has(i, seq_id)); + + // check the cell is not SWA-masked + if (add_cell && seq_id != -1) { + const bool is_masked = llama_hparams::is_masked_swa(n_swa, swa_type, cells.pos_get(i), cells.seq_pos_max(seq_id)); + + add_cell = !is_masked; + } + + if (add_cell) { ++cell_count; if (cell_range_begin == cells.size()) { cell_range_begin = i; @@ -2138,7 +2150,7 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32 sinfo = find_slot(ubatch, false); if (sinfo.empty()) { - LLAMA_LOG_ERROR("%s: failed to find available cells in kv cache\n", __func__); + LLAMA_LOG_ERROR("%s: failed to find %d available cells in kv cache\n", __func__, cell_count); return false; } diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index fb71792fe..4c3f16a0a 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1132,6 +1132,7 @@ json oaicompat_chat_params_parse( llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag; llama_params["reasoning_budget_end_tag"] = chat_params.thinking_end_tag; llama_params["reasoning_budget_message"] = opt.reasoning_budget_message; + llama_params["reasoning_control"] = json_value(body, "reasoning_control", false); } } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index bfe3443c1..fae73f09f 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -37,6 +37,21 @@ using json = nlohmann::ordered_json; constexpr int HTTP_POLLING_SECONDS = 1; +static uint32_t server_n_outputs_max(const common_params & params) { + const uint32_t n_batch = params.n_batch; + + if (params.embedding || + (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) { + return n_batch; + } + + const uint32_t n_outputs_per_seq = 1 + common_speculative_n_max(¶ms.speculative); + + const uint64_t n_outputs = (uint64_t) params.n_parallel * n_outputs_per_seq; + + return std::max(1, std::min(n_batch, n_outputs)); +} + // state diagram: https://github.com/ggml-org/llama.cpp/pull/9283 enum slot_state { SLOT_STATE_IDLE, @@ -753,6 +768,7 @@ private: SRV_INF("loading model '%s'\n", params.model.path.c_str()); params_base = params; + params_base.n_outputs_max = server_n_outputs_max(params_base); std::string & mmproj_path = params_base.mmproj.path; bool has_mmproj = !mmproj_path.empty(); @@ -818,6 +834,8 @@ private: measure_model_bytes = false; } + params_dft.n_outputs_max = params_base.n_parallel; + auto mparams_dft = common_model_params_to_llama(params_dft); auto cparams_dft = common_context_params_to_llama(params_dft); if (spec_mtp) { @@ -941,10 +959,11 @@ private: params_base.model.path.c_str()); auto cparams_mtp = common_context_params_to_llama(params_base); - cparams_mtp.ctx_type = LLAMA_CONTEXT_TYPE_MTP; - cparams_mtp.type_k = params_base.speculative.draft.cache_type_k; - cparams_mtp.type_v = params_base.speculative.draft.cache_type_v; - cparams_mtp.n_rs_seq = 0; + cparams_mtp.ctx_type = LLAMA_CONTEXT_TYPE_MTP; + cparams_mtp.type_k = params_base.speculative.draft.cache_type_k; + cparams_mtp.type_v = params_base.speculative.draft.cache_type_v; + cparams_mtp.n_rs_seq = 0; + cparams_mtp.n_outputs_max = params_base.n_parallel; ctx_dft.reset(llama_init_from_model(model_tgt, cparams_mtp)); if (ctx_dft == nullptr) { @@ -1244,6 +1263,20 @@ private: return nullptr; } + server_slot * get_slot_by_cmpl_id(const std::string & cmpl_id) { + if (cmpl_id.empty()) { + return nullptr; + } + + for (server_slot & slot : slots) { + if (slot.is_processing() && slot.task && slot.task->params.oaicompat_cmpl_id == cmpl_id) { + return &slot; + } + } + + return nullptr; + } + server_slot * get_available_slot(const server_task & task) { server_slot * ret = nullptr; @@ -2095,6 +2128,37 @@ private: } } } break; + case SERVER_TASK_TYPE_CONTROL: + { + auto res = std::make_unique(); + res->id = task.id; + + server_slot * slot = get_slot_by_cmpl_id(task.params.control_cmpl_id); + if (slot == nullptr) { + res->success = false; + res->message = "no active completion for this id"; + queue_results.send(std::move(res)); + break; + } + + if (task.params.control_action == "reasoning_end") { + // the budget sampler only exists when reasoning control was armed + if (!slot->task->params.sampling.reasoning_control) { + res->success = false; + res->message = "reasoning control not enabled for this completion"; + queue_results.send(std::move(res)); + break; + } + // act on the live slot mid generation, never defer + common_sampler_reasoning_budget_force(slot->smpl.get()); + res->success = true; + } else { + res->success = false; + res->message = "unknown control action"; + } + + queue_results.send(std::move(res)); + } break; case SERVER_TASK_TYPE_NEXT_RESPONSE: { // do nothing @@ -4247,6 +4311,43 @@ void server_routes::init_routes() { TASK_RESPONSE_TYPE_OAI_CHAT); }; + this->post_control = [this](const server_http_req & req) { + auto res = create_response(); + const json body = json::parse(req.body); + + const std::string cmpl_id = json_value(body, "id", std::string()); + const std::string action = json_value(body, "action", std::string()); + if (cmpl_id.empty()) { + res->error(format_error_response("missing completion id", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + if (action != "reasoning_end") { + res->error(format_error_response("unknown control action", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + + auto & rd = res->rd; + { + server_task task(SERVER_TASK_TYPE_CONTROL); + task.id = rd.get_new_id(); + task.params.control_cmpl_id = cmpl_id; + task.params.control_action = action; + rd.post_task(std::move(task)); + } + + auto result = rd.next(req.should_stop); + if (!result) { + GGML_ASSERT(req.should_stop()); + return res; + } + if (result->is_error()) { + res->error(result->to_json()); + return res; + } + res->ok(result->to_json()); + return res; + }; + this->post_responses_oai = [this](const server_http_req & req) { auto res = create_response(); std::vector files; diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 65853438c..73caff54a 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -110,6 +110,7 @@ struct server_routes { server_http_context::handler_t post_completions; server_http_context::handler_t post_completions_oai; server_http_context::handler_t post_chat_completions; + server_http_context::handler_t post_control; server_http_context::handler_t post_responses_oai; server_http_context::handler_t post_transcriptions_oai; server_http_context::handler_t post_anthropic_messages; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index ff80be6cc..33de2e4d9 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -499,6 +499,7 @@ task_params server_task::params_from_json_cmpl( const auto end_tag = json_value(data, "reasoning_budget_end_tag", std::string()); const auto message = json_value(data, "reasoning_budget_message", std::string()); params.sampling.reasoning_budget_tokens = budget; + params.sampling.reasoning_control = json_value(data, "reasoning_control", false); if (!start_tag.empty()) { params.sampling.reasoning_budget_start = common_tokenize(vocab, start_tag, false, true); diff --git a/tools/server/server-task.h b/tools/server/server-task.h index d47dc690c..bdadcff76 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -19,6 +19,7 @@ enum server_task_type { SERVER_TASK_TYPE_RERANK, SERVER_TASK_TYPE_INFILL, SERVER_TASK_TYPE_CANCEL, + SERVER_TASK_TYPE_CONTROL, SERVER_TASK_TYPE_NEXT_RESPONSE, SERVER_TASK_TYPE_METRICS, SERVER_TASK_TYPE_SLOT_SAVE, @@ -84,6 +85,10 @@ struct task_params { std::string oaicompat_model; std::string oaicompat_cmpl_id; + // realtime control (SERVER_TASK_TYPE_CONTROL) + std::string control_action; + std::string control_cmpl_id; + // per-request parameters for chat parsing common_chat_parser_params chat_parser_params; @@ -551,6 +556,19 @@ struct server_task_result_slot_erase : server_task_result { virtual json to_json() override; }; +struct server_task_result_control : server_task_result { + bool success = false; + std::string message; // optional detail when success is false + + virtual json to_json() override { + json out = json { { "success", success } }; + if (!message.empty()) { + out["message"] = message; + } + return out; + } +}; + struct server_task_result_get_lora : server_task_result { struct lora { common_adapter_lora_info info; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 4d56d45e8..769e80a80 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -149,6 +149,7 @@ int llama_server(int argc, char ** argv) { routes.post_completions = models_routes->proxy_post; routes.post_completions_oai = models_routes->proxy_post; routes.post_chat_completions = models_routes->proxy_post; + routes.post_control = models_routes->proxy_post; routes.post_responses_oai = models_routes->proxy_post; routes.post_transcriptions_oai = models_routes->proxy_post; routes.post_anthropic_messages = models_routes->proxy_post; @@ -185,6 +186,7 @@ int llama_server(int argc, char ** argv) { ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai)); ctx_http.post("/chat/completions", ex_wrapper(routes.post_chat_completions)); ctx_http.post("/v1/chat/completions", ex_wrapper(routes.post_chat_completions)); + ctx_http.post("/v1/chat/completions/control", ex_wrapper(routes.post_control)); ctx_http.post("/v1/responses", ex_wrapper(routes.post_responses_oai)); ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai)); ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); diff --git a/tools/ui/.env.example b/tools/ui/.env.example index 9a995b746..a1e6c5491 100644 --- a/tools/ui/.env.example +++ b/tools/ui/.env.example @@ -1,2 +1,3 @@ VITE_PUBLIC_APP_NAME='llama-ui' -# VITE_DEBUG='true' +VITE_DEBUG='true' +VITE_PUBLIC_SERVER_ORIGIN='http://localhost:8033' diff --git a/tools/ui/.prettierignore b/tools/ui/.prettierignore index 14b01e961..7bbdcf6a0 100644 --- a/tools/ui/.prettierignore +++ b/tools/ui/.prettierignore @@ -7,6 +7,9 @@ bun.lockb # Miscellaneous /static/ +dist/ +.svelte-kit/ +build/ # Build output /dist/ diff --git a/tools/ui/README.md b/tools/ui/README.md index 4334f9681..1b99ebbfe 100644 --- a/tools/ui/README.md +++ b/tools/ui/README.md @@ -115,16 +115,18 @@ This starts: - **Vite dev server** at `http://localhost:5173` - The main UI frontend app - **Storybook** at `http://localhost:6006` - Component documentation -The Vite dev server proxies API requests to `http://localhost:8080` (default llama-server port): +The Vite dev server proxies API requests to `SERVER_ORIGIN` (with fallback to default llama-server `8080` port): ```typescript // vite.config.ts proxy configuration proxy: { - '/v1': 'http://localhost:8080', - '/props': 'http://localhost:8080', - '/slots': 'http://localhost:8080', - '/models': 'http://localhost:8080' -} + '/v1': SERVER_ORIGIN, + '/props': SERVER_ORIGIN, + '/models': SERVER_ORIGIN, + '/tools': SERVER_ORIGIN, + '/slots': SERVER_ORIGIN, + '/cors-proxy': SERVER_ORIGIN +}, ``` ### Development Workflow diff --git a/tools/ui/playwright.config.ts b/tools/ui/playwright.config.ts index 178fd7ba8..460379238 100644 --- a/tools/ui/playwright.config.ts +++ b/tools/ui/playwright.config.ts @@ -2,7 +2,7 @@ import { defineConfig } from '@playwright/test'; export default defineConfig({ webServer: { - command: 'npm run build && http-server ../../build/tools/ui/dist -p 8181', + command: 'npm run build && npx http-server ./dist -p 8181', port: 8181, timeout: 120000, reuseExistingServer: false diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 46ac82334..ed26f9ea5 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -111,6 +111,7 @@ let preSelectedResourceUri = $state(undefined); let currentConfig = $derived(config()); + let pasteLongTextToFileLength = $derived.by(() => { const n = Number(currentConfig.pasteLongTextToFileLen); return Number.isNaN(n) ? Number(SETTING_CONFIG_DEFAULT.pasteLongTextToFileLen) : n; @@ -541,6 +542,7 @@ canSend={canSubmit} {disabled} {isLoading} + isReasoning={chatStore.isReasoning} {isRecording} {showAddButton} {showModelSelector} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte index 0a8260a29..479540321 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte @@ -1,22 +1,18 @@
{@render trigger({ disabled, onclick: () => (sheetOpen = true) })} - @@ -80,110 +92,206 @@
- {#each ATTACHMENT_FILE_ITEMS as item (item.id)} - {@const enabled = attachmentMenu.isItemEnabled(item.enabledWhen)} - {#if enabled} - - {:else if item.disabledTooltip} - - - - + Add files + - -

{item.disabledTooltip}

- - - {/if} - {/each} + +
+ {#each ATTACHMENT_FILE_ITEMS as item (item.id)} + {@const enabled = attachmentMenu.isItemEnabled(item.enabledWhen)} + {#if enabled} + + {:else if item.disabledTooltip} + + + + + + +

{item.disabledTooltip}

+ + + {/if} + {/each} +
+ + + + (mcpExpanded = open)}> + + {#if mcpExpanded} + + {:else} + + {/if} + + + + MCP Servers + + + {getEnabledMcpServers().length} server{getEnabledMcpServers().length !== 1 ? 's' : ''} + + + + +
+ {#each getEnabledMcpServers() as server (server.id)} + {@const healthState = mcpStore.getHealthCheckState(server.id)} + {@const hasError = healthState.status === HealthCheckStatus.ERROR} + {@const displayName = mcpStore.getServerLabel(server)} + {@const faviconUrl = mcpStore.getServerFavicon(server.id)} + {@const isEnabled = conversationsStore.isMcpServerEnabledForChat(server.id)} - {#if !attachmentMenu.isItemEnabled('hasVisionModality')} - {@const pdfItem = ATTACHMENT_FILE_ITEMS.find((i) => i.id === AttachmentMenuItemId.PDF)} - {#if pdfItem} - - - + {/each} - -

PDFs will be converted to text. Image-based PDFs may not work properly.

- - - {/if} + {#if getEnabledMcpServers().length === 0} +
+ No MCP servers configured +
+ {/if} +
+ + + + {#if toolsPanel.totalToolCount > 0} + (toolsExpanded = open)}> + + {#if toolsExpanded} + + {:else} + + {/if} + + + + Tools + + + {toolsPanel.totalToolCount} tool{toolsPanel.totalToolCount !== 1 ? 's' : ''} + + + + +
+ {#each toolsPanel.activeGroups as group (group.label)} + {@const { checked, indeterminate } = toolsPanel.getGroupCheckedState(group)} + {@const enabledCount = toolsPanel.getEnabledToolCount(group)} + {@const favicon = toolsPanel.getFavicon(group)} + + + {/each} +
+ + {/if} - {#each ATTACHMENT_EXTRA_ITEMS as item (item.id)} - {#if item.id === AttachmentMenuItemId.SYSTEM_MESSAGE} - - - - + System Message + - -

{attachmentMenu.getSystemMessageTooltip()}

- - - {/if} - {/each} + {#if hasMcpPromptsSupport} + + {/if} - - + {#if hasMcpResourcesSupport} + - {/if} - {/each} + MCP Resources + + {/if}
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte index b11467da8..813227fbc 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte @@ -107,7 +107,7 @@ toolsStore.toggleGroup(group)} + onCheckedChange={() => toolsPanel.toggleGroupByLabel(group.label)} class="mr-2 h-4 w-4 shrink-0" /> diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte index fd866b243..712326cba 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte @@ -1,6 +1,11 @@
{#if showAddButton} -
+
{/if} - {#if showModelSelector} - +
+ + + {#if showModelSelector} + + {/if} +
+ + {#if isReasoning} + {/if} {#if isLoading && !canSubmit} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormReasoningEffortSubmenu.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormReasoningEffortSubmenu.svelte new file mode 100644 index 000000000..936e03773 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormReasoningEffortSubmenu.svelte @@ -0,0 +1,132 @@ + + +{#if modelSupportsThinking} + + + {#if thinkingEnabled} + + {:else} + + {/if} + + Thinking + + {#if thinkingEnabled} + {currentEffort} + {:else} + off + {/if} + + + + {#each REASONING_EFFORT_LEVELS as level (level.value)} + + {/each} + + +{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormReasoningToggle.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormReasoningToggle.svelte new file mode 100644 index 000000000..f6bcbcb09 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormReasoningToggle.svelte @@ -0,0 +1,145 @@ + + +{#if modelSupportsThinking} + + + + + {#if thinkingEnabled} + + {:else} + + {/if} + + + + +

{tooltipText}

+ + + + +
Reasoning effort
+ + {#each REASONING_EFFORT_LEVELS as level (level.value)} + + {/each} + + +{/if} diff --git a/tools/ui/src/lib/components/app/chat/index.ts b/tools/ui/src/lib/components/app/chat/index.ts index be5535960..8ed3cc65e 100644 --- a/tools/ui/src/lib/components/app/chat/index.ts +++ b/tools/ui/src/lib/components/app/chat/index.ts @@ -240,6 +240,15 @@ export { default as ChatFormActionAddToolsSubmenu } from './ChatForm/ChatFormAct */ export { default as ChatFormActionAddMcpServersSubmenu } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte'; +/** + * **ChatFormReasoningToggle** - Thinking toggle button with effort dropdown + * + * A toggle button with lightbulb icon that indicates thinking status. + * Shows the reasoning effort dropdown when clicked. + * Only visible when the current model supports thinking. + */ +export { default as ChatFormReasoningToggle } from './ChatForm/ChatFormActions/ChatFormReasoningToggle.svelte'; + /** * Hidden file input element for programmatic file selection. */ diff --git a/tools/ui/src/lib/components/app/models/ModelId.svelte b/tools/ui/src/lib/components/app/models/ModelId.svelte index 2fe952fee..f566b55ee 100644 --- a/tools/ui/src/lib/components/app/models/ModelId.svelte +++ b/tools/ui/src/lib/components/app/models/ModelId.svelte @@ -8,6 +8,7 @@ hideOrgName?: boolean; showRaw?: boolean; hideQuantization?: boolean; + hideTags?: boolean; aliases?: string[]; tags?: string[]; class?: string; @@ -17,7 +18,8 @@ modelId, hideOrgName = false, showRaw = undefined, - hideQuantization = false, + hideQuantization, + hideTags, aliases, tags, class: className = '', @@ -31,6 +33,8 @@ let parsed = $derived(ModelsService.parseModelId(modelId)); let resolvedShowRaw = $derived(showRaw ?? (config().showRawModelNames as boolean) ?? false); + let resolvedHideQuantization = $derived(hideQuantization ?? !config().showModelQuantization); + let resolvedHideTags = $derived(hideTags ?? !config().showModelTags); let uniqueAliases = $derived([...new Set(aliases ?? [])]); let uniqueTags = $derived([...new Set([...(parsed.tags ?? []), ...(tags ?? [])])]); @@ -53,7 +57,7 @@ {/if} - {#if parsed.quantization && !hideQuantization} + {#if parsed.quantization && !resolvedHideQuantization} {parsed.quantization} @@ -69,7 +73,7 @@ {/each} {/if} - {#if uniqueTags.length > 0} + {#if uniqueTags.length > 0 && !resolvedHideTags} {#each uniqueTags as tag (tag)} {tag} {/each} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte index 998a6a0fa..0f1fba880 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte @@ -106,9 +106,7 @@ ]} style="max-width: min(calc(100cqw - 10rem), 20rem)" > - - - + {:else}

No models available.

@@ -120,7 +118,7 @@ - + {#if selectedOption} @@ -144,6 +142,7 @@ modelId={selectedOption.model} class="min-w-0 overflow-hidden" hideOrgName={false} + hideQuantization {...props} /> {/snippet} @@ -158,9 +157,9 @@ {/if} {#if ms.updating || ms.isLoadingModel} - + {:else} - + {/if} @@ -251,7 +250,7 @@ onclick={() => ms.handleOpenChange(true)} disabled={disabled || ms.updating} > - + {#if selectedOption} @@ -262,6 +261,7 @@ modelId={selectedOption.model} class="min-w-0 overflow-hidden" hideOrgName={false} + hideQuantization {...props} /> {/snippet} @@ -274,7 +274,7 @@ {/if} {#if ms.updating} - + {/if} {/if} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte index d38ed8c07..2ddbf2405 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte @@ -6,8 +6,7 @@ DialogModelInformation, ModelId, ModelsSelectorList, - SearchInput, - TruncatedText + SearchInput } from '$lib/components/app'; interface Props { @@ -67,7 +66,7 @@ @@ -168,12 +168,12 @@ onclick={() => ms.handleOpenChange(true)} disabled={disabled || ms.updating} > - + - + {#if ms.updating} - + {/if} {/if} diff --git a/tools/ui/src/lib/components/app/models/index.ts b/tools/ui/src/lib/components/app/models/index.ts index a6ba6817f..3ac6ecb67 100644 --- a/tools/ui/src/lib/components/app/models/index.ts +++ b/tools/ui/src/lib/components/app/models/index.ts @@ -74,8 +74,7 @@ export { default as ModelsSelectorOption } from './ModelsSelectorOption.svelte'; */ export { default as ModelsSelectorSheet } from './ModelsSelectorSheet.svelte'; -/** - * **ModelBadge** - Model name display badge +/** * **ModelBadge** - Model name display badge * * Compact badge showing current model name with package icon. * Only visible in single model mode. Supports tooltip and copy functionality. diff --git a/tools/ui/src/lib/constants/api-endpoints.ts b/tools/ui/src/lib/constants/api-endpoints.ts index f89ebe421..9eb6c74e7 100644 --- a/tools/ui/src/lib/constants/api-endpoints.ts +++ b/tools/ui/src/lib/constants/api-endpoints.ts @@ -4,6 +4,17 @@ export const API_MODELS = { UNLOAD: '/models/unload' }; +// chat completion routes, the control route drives realtime inference (e.g. end reasoning) +export const API_CHAT = { + COMPLETIONS: './v1/chat/completions', + CONTROL: './v1/chat/completions/control' +}; + +// slot introspection, requires the --slots flag on the server +export const API_SLOTS = { + LIST: './slots' +}; + export const API_TOOLS = { LIST: '/tools', EXECUTE: '/tools' diff --git a/tools/ui/src/lib/constants/attachment-menu.ts b/tools/ui/src/lib/constants/attachment-menu.ts index 966f46f41..3d7381812 100644 --- a/tools/ui/src/lib/constants/attachment-menu.ts +++ b/tools/ui/src/lib/constants/attachment-menu.ts @@ -79,7 +79,9 @@ export const ATTACHMENT_FILE_ITEMS: AttachmentMenuItem[] = [ } ]; -export const ATTACHMENT_EXTRA_ITEMS: AttachmentMenuItem[] = [ +export const ATTACHMENT_EXTRA_ITEMS: AttachmentMenuItem[] = []; + +export const ATTACHMENT_PROMPT_ITEMS: AttachmentMenuItem[] = [ { id: AttachmentMenuItemId.SYSTEM_MESSAGE, label: 'System Message', @@ -87,10 +89,7 @@ export const ATTACHMENT_EXTRA_ITEMS: AttachmentMenuItem[] = [ enabledWhen: AttachmentItemEnabledWhen.ALWAYS, hasEnabledTooltip: true, action: AttachmentAction.SYSTEM_PROMPT_CLICK - } -]; - -export const ATTACHMENT_MCP_ITEMS: AttachmentMenuItem[] = [ + }, { id: AttachmentMenuItemId.MCP_PROMPT, label: 'MCP Prompt', @@ -98,7 +97,10 @@ export const ATTACHMENT_MCP_ITEMS: AttachmentMenuItem[] = [ enabledWhen: AttachmentItemEnabledWhen.ALWAYS, action: AttachmentAction.MCP_PROMPT_CLICK, visibleWhen: AttachmentItemVisibleWhen.HAS_MCP_PROMPTS_SUPPORT - }, + } +]; + +export const ATTACHMENT_MCP_ITEMS: AttachmentMenuItem[] = [ { id: AttachmentMenuItemId.MCP_RESOURCES, label: 'MCP Resources', diff --git a/tools/ui/src/lib/constants/control-actions.ts b/tools/ui/src/lib/constants/control-actions.ts new file mode 100644 index 000000000..935ae9542 --- /dev/null +++ b/tools/ui/src/lib/constants/control-actions.ts @@ -0,0 +1,7 @@ +// actions accepted by the realtime inference control endpoint (API_CHAT.CONTROL) +// kept separate from the endpoint paths since these are protocol level verbs +export const CONTROL_ACTION = { + END_REASONING: 'reasoning_end' +} as const; + +export type ControlAction = (typeof CONTROL_ACTION)[keyof typeof CONTROL_ACTION]; diff --git a/tools/ui/src/lib/constants/error.ts b/tools/ui/src/lib/constants/error.ts new file mode 100644 index 000000000..4339bd25d --- /dev/null +++ b/tools/ui/src/lib/constants/error.ts @@ -0,0 +1,23 @@ +export const ERROR_MESSAGES = { + NETWORK: { + GENERIC: 'Failed to connect to server', + NXDOMAIN: 'Server not found - check server address', + REFUSED: 'Connection refused - server may be offline', + TIMEOUT: 'Request timed out', + UNREACHABLE: 'Server is not running or unreachable' + }, + HTTP: { + GENERIC: 'Request failed', + ACCESS_DENIED: 'Access denied', + INTERNAL_ERROR: 'Server error - check server logs', + NOT_FOUND: 'Not found', + TEMPORARILY_UNAVAILABLE: 'Server temporarily unavailable' + } +}; + +export const HTTP_CODE_TO_STRING: Record = { + 401: ERROR_MESSAGES.HTTP.ACCESS_DENIED, + 403: ERROR_MESSAGES.HTTP.ACCESS_DENIED, + 500: ERROR_MESSAGES.HTTP.INTERNAL_ERROR, + 503: ERROR_MESSAGES.HTTP.TEMPORARILY_UNAVAILABLE +}; diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index d3a434802..54f39feee 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -5,6 +5,8 @@ export * from './agentic'; export * from './api-endpoints'; export * from './attachment-labels'; export * from './database'; +export * from './reasoning-effort'; +export * from './reasoning-effort-tokens'; export * from './storage'; export * from './attachment-menu'; export * from './auto-scroll'; @@ -15,6 +17,7 @@ export * from './cli-flags'; export * from './code-blocks'; export * from './code'; export * from './context-keys'; +export * from './control-actions'; export * from './css-classes'; export * from './floating-ui-constraints'; export * from './formatters'; diff --git a/tools/ui/src/lib/constants/reasoning-effort-tokens.ts b/tools/ui/src/lib/constants/reasoning-effort-tokens.ts new file mode 100644 index 000000000..059af71de --- /dev/null +++ b/tools/ui/src/lib/constants/reasoning-effort-tokens.ts @@ -0,0 +1,12 @@ +import { ReasoningEffort } from '$lib/enums'; + +/** + * Reasoning effort to token budget mapping. + * Maps the ReasoningEffort enum values to concrete token counts for the server. + */ +export const REASONING_EFFORT_TOKENS: Record = { + [ReasoningEffort.LOW]: 512, + [ReasoningEffort.MEDIUM]: 2048, + [ReasoningEffort.HIGH]: 8192, + [ReasoningEffort.MAX]: -1 // unlimited +}; diff --git a/tools/ui/src/lib/constants/reasoning-effort.ts b/tools/ui/src/lib/constants/reasoning-effort.ts new file mode 100644 index 000000000..d854e912a --- /dev/null +++ b/tools/ui/src/lib/constants/reasoning-effort.ts @@ -0,0 +1,21 @@ +import { ReasoningEffort } from '$lib/enums'; +import type { ReasoningEffortLevel } from '$lib/types'; + +/** + * Reasoning effort UI labels. + * Keys match the ReasoningEffort enum values for type-safe lookups. + */ +export const REASONING_EFFORT_LABELS: Record = { + [ReasoningEffort.LOW]: 'Low', + [ReasoningEffort.MEDIUM]: 'Medium', + [ReasoningEffort.HIGH]: 'High', + [ReasoningEffort.MAX]: 'Max' +}; + +export const REASONING_EFFORT_LEVELS: ReasoningEffortLevel[] = [ + { value: 'off', label: 'Off', isOff: true }, + { value: ReasoningEffort.LOW, label: 'Low' }, + { value: ReasoningEffort.MEDIUM, label: 'Medium' }, + { value: ReasoningEffort.HIGH, label: 'High' }, + { value: ReasoningEffort.MAX, label: 'Max', hasInfo: true } +]; diff --git a/tools/ui/src/lib/constants/settings-keys.ts b/tools/ui/src/lib/constants/settings-keys.ts index 7cdd2db7c..5fff9f94c 100644 --- a/tools/ui/src/lib/constants/settings-keys.ts +++ b/tools/ui/src/lib/constants/settings-keys.ts @@ -29,6 +29,8 @@ export const SETTINGS_KEYS = { ALWAYS_SHOW_SIDEBAR_ON_DESKTOP: 'alwaysShowSidebarOnDesktop', FULL_HEIGHT_CODE_BLOCKS: 'fullHeightCodeBlocks', SHOW_RAW_MODEL_NAMES: 'showRawModelNames', + SHOW_MODEL_QUANTIZATION: 'showModelQuantization', + SHOW_MODEL_TAGS: 'showModelTags', SHOW_SYSTEM_MESSAGE: 'showSystemMessage', // Sampling TEMPERATURE: 'temperature', @@ -64,6 +66,7 @@ export const SETTINGS_KEYS = { // Developer DISABLE_REASONING_PARSING: 'disableReasoningParsing', EXCLUDE_REASONING_FROM_CONTEXT: 'excludeReasoningFromContext', + ENABLE_THINKING: 'enableThinking', SHOW_RAW_OUTPUT_SWITCH: 'showRawOutputSwitch', // PY_INTERPRETER_ENABLED: 'pyInterpreterEnabled', CUSTOM_JSON: 'customJson', diff --git a/tools/ui/src/lib/constants/settings-registry.ts b/tools/ui/src/lib/constants/settings-registry.ts index 20ac33c85..9246b9703 100644 --- a/tools/ui/src/lib/constants/settings-registry.ts +++ b/tools/ui/src/lib/constants/settings-registry.ts @@ -330,6 +330,30 @@ const SETTINGS_REGISTRY: Record = { paramType: SyncableParameterType.BOOLEAN } }, + { + key: SETTINGS_KEYS.SHOW_MODEL_QUANTIZATION, + label: 'Show model quantization information', + help: 'Display quantization badges (e.g. Q8_0, Q4_K_M) next to model names throughout the interface.', + defaultValue: true, + type: SettingsFieldType.CHECKBOX, + section: SETTINGS_SECTION_SLUGS.DISPLAY, + sync: { + serverKey: SETTINGS_KEYS.SHOW_MODEL_QUANTIZATION, + paramType: SyncableParameterType.BOOLEAN + } + }, + { + key: SETTINGS_KEYS.SHOW_MODEL_TAGS, + label: 'Show model tags', + help: 'Display model tags (e.g. "vision", "reasoning") next to model names throughout the interface.', + defaultValue: true, + type: SettingsFieldType.CHECKBOX, + section: SETTINGS_SECTION_SLUGS.DISPLAY, + sync: { + serverKey: SETTINGS_KEYS.SHOW_MODEL_TAGS, + paramType: SyncableParameterType.BOOLEAN + } + }, { key: SETTINGS_KEYS.ALWAYS_SHOW_AGENTIC_TURNS, label: 'Always show agentic turns in conversation', @@ -646,6 +670,14 @@ const SETTINGS_REGISTRY: Record = { paramType: SyncableParameterType.BOOLEAN } }, + { + key: SETTINGS_KEYS.ENABLE_THINKING, + label: 'Enable thinking', + help: 'Enable model thinking/reasoning for each request. When off, the model will skip the thinking phase and go straight to the response.', + defaultValue: false, + type: SettingsFieldType.CHECKBOX, + section: SETTINGS_SECTION_SLUGS.DEVELOPER + }, { key: SETTINGS_KEYS.SHOW_RAW_OUTPUT_SWITCH, label: 'Enable raw output toggle', diff --git a/tools/ui/src/lib/constants/storage.ts b/tools/ui/src/lib/constants/storage.ts index d03254b9c..5d33e82f3 100644 --- a/tools/ui/src/lib/constants/storage.ts +++ b/tools/ui/src/lib/constants/storage.ts @@ -19,6 +19,8 @@ export const CONFIG_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.config`; export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTools`; export const FAVORITE_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.favoriteModels`; export const MCP_DEFAULT_ENABLED_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.mcpDefaultEnabled`; +export const THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.thinkingEnabledDefault`; +export const REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.reasoningEffortDefault`; export const USER_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.userOverrides`; // Deprecated old key names (kept for backward compat while users migrate) diff --git a/tools/ui/src/lib/enums/index.ts b/tools/ui/src/lib/enums/index.ts index b80b5b61e..7438a7642 100644 --- a/tools/ui/src/lib/enums/index.ts +++ b/tools/ui/src/lib/enums/index.ts @@ -19,6 +19,8 @@ export { ReasoningFormat } from './chat.enums'; +export { ReasoningEffort } from './reasoning-effort.enums'; + export { FileTypeCategory, FileTypeImage, diff --git a/tools/ui/src/lib/enums/reasoning-effort.enums.ts b/tools/ui/src/lib/enums/reasoning-effort.enums.ts new file mode 100644 index 000000000..dadb0c726 --- /dev/null +++ b/tools/ui/src/lib/enums/reasoning-effort.enums.ts @@ -0,0 +1,10 @@ +/** + * Reasoning effort levels for thinking models. + * These values are sent to the server and mapped to token budgets. + */ +export enum ReasoningEffort { + LOW = 'low', + MEDIUM = 'medium', + HIGH = 'high', + MAX = 'max' +} diff --git a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts index 911af322a..9a8acec0f 100644 --- a/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts +++ b/tools/ui/src/lib/hooks/use-tools-panel.svelte.ts @@ -17,6 +17,8 @@ export interface UseToolsPanelReturn { getFavicon(group: { source: ToolSource; label: string }): string | null; isGroupDisabled(group: ToolGroup): boolean; toggleGroupExpanded(label: string): void; + /** Toggle all tools in a group by label (avoids stale group object references). */ + toggleGroupByLabel(label: string): void; handleOpen(): void; } @@ -91,6 +93,13 @@ export function useToolsPanel(): UseToolsPanelReturn { } } + function toggleGroupByLabel(label: string): void { + // Find current group by label to get up-to-date tool references + const group = activeGroups.find((g) => g.label === label); + if (!group) return; + toolsStore.toggleGroup(group); + } + function handleOpen(): void { if (toolsStore.builtinTools.length === 0 && !toolsStore.loading) { toolsStore.fetchBuiltinTools(); @@ -117,6 +126,7 @@ export function useToolsPanel(): UseToolsPanelReturn { getFavicon, isGroupDisabled, toggleGroupExpanded, + toggleGroupByLabel, handleOpen }; } diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index f749ba5f9..2d5879424 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -6,7 +6,11 @@ import { ATTACHMENT_LABEL_MCP_PROMPT, ATTACHMENT_LABEL_MCP_RESOURCE, LEGACY_AGENTIC_REGEX, - SETTINGS_KEYS + REASONING_EFFORT_TOKENS, + SETTINGS_KEYS, + API_CHAT, + API_SLOTS, + CONTROL_ACTION } from '$lib/constants'; import { AttachmentType, @@ -126,6 +130,7 @@ export class ChatService { onReasoningChunk, onToolCallChunk, onModel, + onCompletionId, onTimings, // Tools for function calling tools, @@ -158,6 +163,8 @@ export class ChatService { // Config options disableReasoningParsing, excludeReasoningFromContext, + enableThinking, + reasoningEffort, continueFinalMessage } = options; @@ -239,6 +246,21 @@ export class ChatService { ? ReasoningFormat.NONE : ReasoningFormat.AUTO; + const reasoningBudgetTokens = + enableThinking && reasoningEffort ? (REASONING_EFFORT_TOKENS[reasoningEffort] ?? -1) : -1; + + requestBody.chat_template_kwargs = { + ...(requestBody.chat_template_kwargs ?? {}), + enable_thinking: enableThinking + }; + + if (reasoningBudgetTokens >= 0) { + requestBody.thinking_budget_tokens = reasoningBudgetTokens; + } + + // arms the budget sampler so reasoning can be ended at runtime via the control endpoint + requestBody.reasoning_control = true; + if (continueFinalMessage) { requestBody.continue_final_message = true; requestBody.add_generation_prompt = false; @@ -289,7 +311,7 @@ export class ChatService { } try { - const response = await fetch(`./v1/chat/completions`, { + const response = await fetch(API_CHAT.COMPLETIONS, { method: 'POST', headers: getJsonHeaders(), body: JSON.stringify(requestBody), @@ -315,6 +337,7 @@ export class ChatService { onReasoningChunk, onToolCallChunk, onModel, + onCompletionId, onTimings, conversationId, signal @@ -379,7 +402,7 @@ export class ChatService { */ static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise { try { - const url = model ? `./slots?model=${encodeURIComponent(model)}` : './slots'; + const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST; const res = await fetch(url, { signal }); if (!res.ok) return true; @@ -390,6 +413,50 @@ export class ChatService { } } + /** + * Ends the current reasoning block of a running completion, targeted by its + * chat completion id (streamed back as `id`). Matching the completion rather + * than a slot index avoids a TOCTOU: a finished completion simply matches + * nothing server side. The model is carried so the router forwards to the + * right child, single model ignores it. Returns true on success. + */ + static async stopReasoning(completionId: string, model?: string | null): Promise { + if (!completionId) { + console.error( + 'stopReasoning: no completion id for the active message, cannot target the running completion' + ); + return false; + } + + const body: Record = { + id: completionId, + action: CONTROL_ACTION.END_REASONING + }; + if (model) body.model = model; + + try { + const res = await fetch(API_CHAT.CONTROL, { + method: 'POST', + headers: getJsonHeaders(), + body: JSON.stringify(body) + }); + + const data = await res.json().catch(() => null); + if (!res.ok || data?.success !== true) { + console.error('stopReasoning: control request failed', { + status: res.status, + completionId, + response: data + }); + return false; + } + return true; + } catch (error) { + console.error('stopReasoning: control request threw', { completionId, error }); + return false; + } + } + /** * Sends a fire-and-forget request to pre-encode the conversation in the server's KV cache. * After a response completes, this re-submits the full conversation @@ -457,7 +524,7 @@ export class ChatService { } try { - await fetch(`./v1/chat/completions`, { + await fetch(API_CHAT.COMPLETIONS, { method: 'POST', headers: getJsonHeaders(), body: JSON.stringify(requestBody), @@ -502,6 +569,7 @@ export class ChatService { onReasoningChunk?: (chunk: string) => void, onToolCallChunk?: (chunk: string) => void, onModel?: (model: string) => void, + onCompletionId?: (id: string) => void, onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, conversationId?: string, abortSignal?: AbortSignal @@ -519,6 +587,7 @@ export class ChatService { let lastTimings: ChatMessageTimings | undefined; let streamFinished = false; let modelEmitted = false; + let idEmitted = false; let toolCallIndexOffset = 0; let hasOpenToolCallBatch = false; @@ -607,6 +676,11 @@ export class ChatService { onModel?.(chunkModel); } + if (parsed.id && !idEmitted) { + idEmitted = true; + onCompletionId?.(parsed.id); + } + if (promptProgress) { ChatService.notifyTimings(undefined, promptProgress, onTimings); } diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 4866995b4..947737d7c 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -488,6 +488,7 @@ class AgenticStore { onToolCallsStreaming, onAttachments, onModel, + onCompletionId, onAssistantTurnComplete, createToolResultMessage, createAssistantMessage, @@ -597,6 +598,7 @@ class AgenticStore { } }, onModel, + onCompletionId, onTimings: (timings?: ChatMessageTimings, progress?: ChatMessagePromptProgress) => { onTimings?.(timings, progress); if (timings) { diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index f2f13f25d..b899130e5 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -63,7 +63,10 @@ class ChatStore { currentResponse = $state(''); errorDialogState = $state(null); isLoading = $state(false); + // true while the active conversation streams reasoning content but no visible content yet + isReasoning = $state(false); chatLoadingStates = new SvelteMap(); + chatReasoningStates = new SvelteMap(); chatStreamingStates = new SvelteMap(); private abortControllers = new SvelteMap(); private preEncodeAbortController: AbortController | null = null; @@ -94,6 +97,17 @@ class ChatStore { } else { this.chatLoadingStates.delete(convId); if (convId === conversationsStore.activeConversation?.id) this.isLoading = false; + this.setChatReasoning(convId, false); + } + } + + private setChatReasoning(convId: string, reasoning: boolean): void { + if (reasoning) { + this.chatReasoningStates.set(convId, true); + if (convId === conversationsStore.activeConversation?.id) this.isReasoning = true; + } else { + this.chatReasoningStates.delete(convId); + if (convId === conversationsStore.activeConversation?.id) this.isReasoning = false; } } private setChatStreaming(convId: string, response: string, messageId: string): void { @@ -110,6 +124,7 @@ class ChatStore { } syncLoadingStateForChat(convId: string): void { this.isLoading = this.chatLoadingStates.get(convId) || false; + this.isReasoning = this.chatReasoningStates.get(convId) || false; const s = this.chatStreamingStates.get(convId); this.currentResponse = s?.response || ''; this.isStreamingActive = s !== undefined; @@ -265,6 +280,10 @@ class ChatStore { return this.chatLoadingStates.get(convId) || false; } + isChatReasoningPublic(convId: string): boolean { + return this.chatReasoningStates.get(convId) || false; + } + private isChatLoadingInternal(convId: string): boolean { return this.chatLoadingStates.has(convId) || this.chatStreamingStates.has(convId); } @@ -655,6 +674,17 @@ class ChatStore { } }; + let completionIdRecorded = false; + const recordCompletionId = (id: string): void => { + if (!id || completionIdRecorded) return; + completionIdRecorded = true; + const idx = conversationsStore.findMessageIndex(currentMessageId); + conversationsStore.updateMessageAtIndex(idx, { completionId: id }); + DatabaseService.updateMessage(currentMessageId, { completionId: id }).catch(() => { + completionIdRecorded = false; + }); + }; + const updateStreamingUI = () => { this.setChatStreaming(convId, streamedContent, currentMessageId); const idx = conversationsStore.findMessageIndex(currentMessageId); @@ -676,6 +706,7 @@ class ChatStore { onChunk: (chunk: string) => { streamedContent += chunk; updateStreamingUI(); + this.setChatReasoning(convId, false); }, onReasoningChunk: (chunk: string) => { streamedReasoningContent += chunk; @@ -685,6 +716,7 @@ class ChatStore { conversationsStore.updateMessageAtIndex(idx, { reasoningContent: streamedReasoningContent }); + this.setChatReasoning(convId, true); }, onToolCallsStreaming: (toolCalls) => { const idx = conversationsStore.findMessageIndex(currentMessageId); @@ -702,6 +734,7 @@ class ChatStore { DatabaseService.updateMessage(messageId, { extra: updatedExtras }).catch(console.error); }, onModel: (modelName: string) => recordModel(modelName), + onCompletionId: (id: string) => recordCompletionId(id), onTurnComplete: (intermediateTimings: ChatMessageTimings) => { // Update the first assistant message with cumulative agentic timings const idx = conversationsStore.findMessageIndex(assistantMessage.id); @@ -887,6 +920,7 @@ class ChatStore { onChunk: streamCallbacks.onChunk, onReasoningChunk: streamCallbacks.onReasoningChunk, onModel: streamCallbacks.onModel, + onCompletionId: streamCallbacks.onCompletionId, onTimings: streamCallbacks.onTimings, onComplete: async ( finalContent?: string, @@ -1373,6 +1407,7 @@ class ChatStore { appendedContent += chunk; hasReceivedContent = true; updateStreamingContent(originalContent + appendedContent); + this.setChatReasoning(msg.convId, false); }, onReasoningChunk: (chunk: string) => { appendedReasoning += chunk; @@ -1382,6 +1417,7 @@ class ChatStore { conversationsStore.updateMessageAtIndex(idx, { reasoningContent: originalReasoning + appendedReasoning }); + this.setChatReasoning(msg.convId, true); }, onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { const tokensPerSecond = @@ -1816,6 +1852,9 @@ class ChatStore { if (currentConfig.excludeReasoningFromContext) apiOptions.excludeReasoningFromContext = true; + apiOptions.enableThinking = conversationsStore.getThinkingEnabled(); + apiOptions.reasoningEffort = conversationsStore.getReasoningEffort(); + if (hasValue(currentConfig.temperature)) apiOptions.temperature = Number(currentConfig.temperature); @@ -1924,6 +1963,7 @@ export const isChatLoading = (convId: string) => chatStore.isChatLoadingPublic(c export const isChatStreaming = () => chatStore.isStreaming(); export const isEditing = () => chatStore.isEditing(); export const isLoading = () => chatStore.isLoading; +export const isReasoning = () => chatStore.isReasoning; export const pendingEditMessageId = () => chatStore.pendingEditMessageId; export const chatHasPendingMessage = (convId: string) => chatStore.hasPendingMessage(convId); export const chatPendingMessageContent = (convId: string) => diff --git a/tools/ui/src/lib/stores/conversations.svelte.ts b/tools/ui/src/lib/stores/conversations.svelte.ts index d6589232f..cab6f59fa 100644 --- a/tools/ui/src/lib/stores/conversations.svelte.ts +++ b/tools/ui/src/lib/stores/conversations.svelte.ts @@ -26,7 +26,7 @@ import { MigrationService } from '$lib/services/migration.service'; import { config } from '$lib/stores/settings.svelte'; import { filterByLeafNodeId, findLeafNode, generateConversationTitle } from '$lib/utils'; import type { McpServerOverride } from '$lib/types/database'; -import { MessageRole, HtmlInputType, FileExtensionText } from '$lib/enums'; +import { MessageRole, HtmlInputType, FileExtensionText, ReasoningEffort } from '$lib/enums'; import { ISO_DATE_TIME_SEPARATOR, ISO_DATE_TIME_SEPARATOR_REPLACEMENT, @@ -38,7 +38,9 @@ import { ISO_TIME_SEPARATOR_REPLACEMENT, NON_ALPHANUMERIC_REGEX, MULTIPLE_UNDERSCORE_REGEX, - MCP_DEFAULT_ENABLED_LOCALSTORAGE_KEY + MCP_DEFAULT_ENABLED_LOCALSTORAGE_KEY, + THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY, + REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY } from '$lib/constants'; import { ROUTES } from '$lib/constants/routes'; @@ -74,6 +76,12 @@ class ConversationsStore { /** Pending MCP server overrides for new conversations (before first message) */ pendingMcpServerOverrides = $state(ConversationsStore.loadMcpDefaults()); + /** Global (non-conversation-specific) thinking toggle default */ + pendingThinkingEnabled = $state(ConversationsStore.loadThinkingDefaults()); + + /** Global (non-conversation-specific) reasoning effort default */ + pendingReasoningEffort = $state(ConversationsStore.loadReasoningEffortDefault()); + /** Load MCP default overrides from localStorage */ private static loadMcpDefaults(): McpServerOverride[] { if (typeof globalThis.localStorage === 'undefined') return []; @@ -104,6 +112,45 @@ class ConversationsStore { } } + /** Load thinking-enabled default from localStorage */ + private static loadThinkingDefaults(): boolean { + if (typeof globalThis.localStorage === 'undefined') return false; + try { + const raw = localStorage.getItem(THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY); + if (!raw) return false; + const parsed = raw === 'true'; + return typeof parsed === 'boolean' ? parsed : false; + } catch { + return false; + } + } + + /** Persist thinking-enabled default to localStorage */ + private saveThinkingDefaults(): void { + if (typeof globalThis.localStorage === 'undefined') return; + localStorage.setItem( + THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY, + this.pendingThinkingEnabled ? 'true' : 'false' + ); + } + + /** Load reasoning effort default from localStorage */ + private static loadReasoningEffortDefault(): ReasoningEffort { + if (typeof globalThis.localStorage === 'undefined') return ReasoningEffort.MEDIUM; + try { + const raw = localStorage.getItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY); + return (raw as ReasoningEffort) || ReasoningEffort.MEDIUM; + } catch { + return ReasoningEffort.MEDIUM; + } + } + + /** Persist reasoning effort default to localStorage */ + private saveReasoningEffortDefaults(): void { + if (typeof globalThis.localStorage === 'undefined') return; + localStorage.setItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, this.pendingReasoningEffort); + } + /** Callback for title update confirmation dialog */ titleUpdateConfirmationCallback?: (currentTitle: string, newTitle: string) => Promise; @@ -253,6 +300,12 @@ class ConversationsStore { this.pendingMcpServerOverrides = []; } + // Inherit global thinking default into the new conversation + conversation.thinkingEnabled = this.pendingThinkingEnabled; + await DatabaseService.updateConversation(conversation.id, { + thinkingEnabled: this.pendingThinkingEnabled + }); + this.conversations = [conversation, ...this.conversations]; this.activeConversation = conversation; this.activeMessages = []; @@ -276,6 +329,7 @@ class ConversationsStore { } this.pendingMcpServerOverrides = []; + this.pendingThinkingEnabled = false; this.activeConversation = conversation; if (conversation.currNode) { @@ -304,8 +358,9 @@ class ConversationsStore { clearActiveConversation(): void { this.activeConversation = null; this.activeMessages = []; - // reload MCP defaults so new chats inherit persisted state + // reload defaults so new chats inherit persisted state this.pendingMcpServerOverrides = ConversationsStore.loadMcpDefaults(); + this.pendingThinkingEnabled = ConversationsStore.loadThinkingDefaults(); } /** @@ -703,6 +758,84 @@ class ConversationsStore { this.saveMcpDefaults(); } + /** + * Gets the effective thinking-enabled state for the active conversation. + * Returns the conversation override if set, otherwise the global default. + */ + getThinkingEnabled(): boolean { + if (this.activeConversation) { + return this.activeConversation.thinkingEnabled ?? this.pendingThinkingEnabled; + } + return this.pendingThinkingEnabled; + } + + /** + * Sets the thinking-enabled state for the active conversation. + * If no conversation exists, stores the global default. + * @param enabled - The enabled state + */ + async setThinkingEnabled(enabled: boolean): Promise { + if (!this.activeConversation) { + this.pendingThinkingEnabled = enabled; + this.saveThinkingDefaults(); + return; + } + + this.activeConversation = { + ...this.activeConversation, + thinkingEnabled: enabled + }; + + await DatabaseService.updateConversation(this.activeConversation.id, { + thinkingEnabled: enabled + }); + + const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id); + if (convIndex !== -1) { + this.conversations[convIndex].thinkingEnabled = enabled; + this.conversations = [...this.conversations]; + } + } + + /** + * Gets the effective reasoning effort for the active conversation. + * Returns the conversation override if set, otherwise the global default. + */ + getReasoningEffort(): ReasoningEffort { + if (this.activeConversation) { + return this.activeConversation.reasoningEffort ?? this.pendingReasoningEffort; + } + return this.pendingReasoningEffort; + } + + /** + * Sets the reasoning effort for the active conversation. + * If no conversation exists, stores the global default. + * @param effort - The effort level ('low' | 'medium' | 'high' | 'max') + */ + async setReasoningEffort(effort: ReasoningEffort): Promise { + if (!this.activeConversation) { + this.pendingReasoningEffort = effort; + this.saveReasoningEffortDefaults(); + return; + } + + this.activeConversation = { + ...this.activeConversation, + reasoningEffort: effort + }; + + await DatabaseService.updateConversation(this.activeConversation.id, { + reasoningEffort: effort + }); + + const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id); + if (convIndex !== -1) { + this.conversations[convIndex].reasoningEffort = effort; + this.conversations = [...this.conversations]; + } + } + /** * Forks a conversation at a specific message, creating a new conversation * containing messages from root up to the target message, then navigates to it. diff --git a/tools/ui/src/lib/stores/models.svelte.ts b/tools/ui/src/lib/stores/models.svelte.ts index bc99d7412..1990ba604 100644 --- a/tools/ui/src/lib/stores/models.svelte.ts +++ b/tools/ui/src/lib/stores/models.svelte.ts @@ -4,6 +4,10 @@ import { ServerModelStatus, ModelModality } from '$lib/enums'; import { ModelsService } from '$lib/services/models.service'; import { PropsService } from '$lib/services/props.service'; import { serverStore, isRouterMode } from '$lib/stores/server.svelte'; +import { + detectThinkingSupport, + detectThinkingSupportWithReason +} from '$lib/utils/chat-template-thinking-detector'; import { TTLCache } from '$lib/utils'; import { MODEL_PROPS_CACHE_TTL_MS, @@ -215,6 +219,67 @@ class ModelsStore { return usage !== undefined && usage.size > 0; } + // + // Thinking Support Detection + // + + /** + * Whether the selected model's chat template supports thinking/reasoning. + * Uses heuristic detection on the model's chat_template from /props. + * + * - MODEL mode: uses serverStore.props.chat_template (single loaded model) + * - ROUTER mode: fetches /props?model= for the selected model (cached) + * + * Triggers an async fetch of model props if not yet cached in ROUTER mode. + */ + get supportsThinking(): boolean { + const modelId = this.selectedModelName; + if (!modelId) { + if (!isRouterMode()) { + return detectThinkingSupport(serverStore.props?.chat_template ?? ''); + } + return false; + } + + if (isRouterMode() && !this.modelPropsCache.get(modelId)) { + this.fetchModelProps(modelId); + } + const props = this.getModelProps(modelId); + return detectThinkingSupport(props?.chat_template ?? ''); + } + + /** + * Check if a specific model supports thinking. + * Fetches model props if not cached (in router mode). + */ + checkModelSupportsThinking(modelId: string): boolean { + if (!modelId) return false; + + if (isRouterMode() && !this.modelPropsCache.get(modelId)) { + this.fetchModelProps(modelId); + } + + const props = this.getModelProps(modelId); + return detectThinkingSupport(props?.chat_template ?? ''); + } + + /** + * Detailed thinking support detection result with reason for debugging/UI. + */ + get thinkingSupportDetails(): { supported: boolean; reason: string } { + const modelId = this.selectedModelName; + if (!modelId) { + if (!isRouterMode()) { + return detectThinkingSupportWithReason(serverStore.props?.chat_template ?? ''); + } + return { supported: false, reason: 'No model selected' }; + } + if (isRouterMode() && !this.modelPropsCache.get(modelId)) { + this.fetchModelProps(modelId); + } + const props = this.getModelProps(modelId); + return detectThinkingSupportWithReason(props?.chat_template ?? ''); + } /** * @@ -362,6 +427,7 @@ class ModelsStore { try { const props = await PropsService.fetchForModel(modelId); this.modelPropsCache.set(modelId, props); + this.propsCacheVersion++; return props; } catch (error) { console.warn(`Failed to fetch props for model ${modelId}:`, error); @@ -755,3 +821,7 @@ export const propsCacheVersion = () => modelsStore.propsCacheVersion; export const singleModelName = () => modelsStore.singleModelName; export const selectedModelContextSize = () => modelsStore.selectedModelContextSize; export const favoriteModelIds = () => modelsStore.favoriteModelIds; +export const supportsThinking = () => modelsStore.supportsThinking; +export const checkModelSupportsThinking = (modelId: string) => + modelsStore.checkModelSupportsThinking(modelId); +export const thinkingSupportDetails = () => modelsStore.thinkingSupportDetails; diff --git a/tools/ui/src/lib/stores/server.svelte.ts b/tools/ui/src/lib/stores/server.svelte.ts index dfcb9b2bb..d9a9f855a 100644 --- a/tools/ui/src/lib/stores/server.svelte.ts +++ b/tools/ui/src/lib/stores/server.svelte.ts @@ -82,8 +82,8 @@ class ServerStore { this.props = props; this.error = null; this.detectRole(props); - } catch (error) { - this.error = this.getErrorMessage(error); + } catch (error: unknown) { + this.error = error instanceof Error ? error.message : String(error); console.error('Error fetching server properties:', error); } finally { this.loading = false; @@ -95,32 +95,6 @@ class ServerStore { await fetchPromise; } - private getErrorMessage(error: unknown): string { - if (error instanceof Error) { - const message = error.message || ''; - - if (error.name === 'TypeError' && message.includes('fetch')) { - return 'Server is not running or unreachable'; - } else if (message.includes('ECONNREFUSED')) { - return 'Connection refused - server may be offline'; - } else if (message.includes('ENOTFOUND')) { - return 'Server not found - check server address'; - } else if (message.includes('ETIMEDOUT')) { - return 'Request timed out'; - } else if (message.includes('503')) { - return 'Server temporarily unavailable'; - } else if (message.includes('500')) { - return 'Server error - check server logs'; - } else if (message.includes('404')) { - return 'Server endpoint not found'; - } else if (message.includes('403') || message.includes('401')) { - return 'Access denied'; - } - } - - return 'Failed to connect to server'; - } - clear(): void { this.props = null; this.error = null; diff --git a/tools/ui/src/lib/types/agentic.d.ts b/tools/ui/src/lib/types/agentic.d.ts index b94998384..bcec10c67 100644 --- a/tools/ui/src/lib/types/agentic.d.ts +++ b/tools/ui/src/lib/types/agentic.d.ts @@ -93,6 +93,7 @@ export interface AgenticFlowCallbacks { onAttachments?: (messageId: string, extras: DatabaseMessageExtra[]) => void; /** Model name detected from response */ onModel?: (model: string) => void; + onCompletionId?: (id: string) => void; /** Current assistant turn's streaming is complete - save to DB */ onAssistantTurnComplete?: ( content: string, diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts index c78035180..f620d6735 100644 --- a/tools/ui/src/lib/types/api.d.ts +++ b/tools/ui/src/lib/types/api.d.ts @@ -271,6 +271,7 @@ export interface ApiChatCompletionToolCall extends ApiChatCompletionToolCallDelt } export interface ApiChatCompletionStreamChunk { + id?: string; object?: string; model?: string; choices: Array<{ diff --git a/tools/ui/src/lib/types/chat.d.ts b/tools/ui/src/lib/types/chat.d.ts index acedd0769..b2158807d 100644 --- a/tools/ui/src/lib/types/chat.d.ts +++ b/tools/ui/src/lib/types/chat.d.ts @@ -97,6 +97,7 @@ export interface ChatStreamCallbacks { onToolCallsStreaming?: (toolCalls: ApiChatCompletionToolCall[]) => void; onAttachments?: (messageId: string, extras: DatabaseMessageExtra[]) => void; onModel?: (model: string) => void; + onCompletionId?: (id: string) => void; onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void; onAssistantTurnComplete?: ( content: string, diff --git a/tools/ui/src/lib/types/database.d.ts b/tools/ui/src/lib/types/database.d.ts index 02bdb3ee6..add628476 100644 --- a/tools/ui/src/lib/types/database.d.ts +++ b/tools/ui/src/lib/types/database.d.ts @@ -1,5 +1,5 @@ import type { ChatMessageTimings, ChatRole, ChatMessageType } from '$lib/types/chat'; -import { AttachmentType } from '$lib/enums'; +import { AttachmentType, ReasoningEffort } from '$lib/enums'; export interface McpServerOverride { serverId: string; @@ -12,6 +12,8 @@ export interface DatabaseConversation { lastModified: number; name: string; mcpServerOverrides?: McpServerOverride[]; + thinkingEnabled?: boolean; + reasoningEffort?: ReasoningEffort; forkedFromConversationId?: string; } @@ -112,6 +114,8 @@ export interface DatabaseMessage { reasoningContent?: string; /** Serialized JSON array of tool calls made by assistant messages */ toolCalls?: string; + /** Chat completion id streamed by the server, used to target realtime control (e.g. end reasoning) */ + completionId?: string; /** Tool call ID for tool result messages (role: 'tool') */ toolCallId?: string; children: string[]; diff --git a/tools/ui/src/lib/types/index.ts b/tools/ui/src/lib/types/index.ts index 0eb1e6701..b88de5d66 100644 --- a/tools/ui/src/lib/types/index.ts +++ b/tools/ui/src/lib/types/index.ts @@ -162,3 +162,6 @@ export type { // Tools types export type { ToolEntry, ToolGroup } from './tools'; + +// Reasoning +export type { ReasoningEffortLevel } from './reasoning'; diff --git a/tools/ui/src/lib/types/reasoning.ts b/tools/ui/src/lib/types/reasoning.ts new file mode 100644 index 000000000..b72a3b77f --- /dev/null +++ b/tools/ui/src/lib/types/reasoning.ts @@ -0,0 +1,6 @@ +export interface ReasoningEffortLevel { + value: string; + label: string; + isOff?: boolean; + hasInfo?: boolean; +} diff --git a/tools/ui/src/lib/types/settings.d.ts b/tools/ui/src/lib/types/settings.d.ts index 03818091a..d1cdca957 100644 --- a/tools/ui/src/lib/types/settings.d.ts +++ b/tools/ui/src/lib/types/settings.d.ts @@ -2,7 +2,12 @@ import type { SETTING_CONFIG_DEFAULT, SETTINGS_SECTION_TITLES } from '$lib/const import type { ChatMessagePromptProgress, ChatMessageTimings } from './chat'; import type { OpenAIToolDefinition } from './mcp'; import type { DatabaseMessageExtra } from './database'; -import type { ParameterSource, SyncableParameterType, SettingsFieldType } from '$lib/enums'; +import type { + ParameterSource, + ReasoningEffort, + SyncableParameterType, + SettingsFieldType +} from '$lib/enums'; import type { Icon } from '@lucide/svelte'; import type { Component } from 'svelte'; @@ -65,6 +70,10 @@ export interface SettingsChatServiceOptions { disableReasoningParsing?: boolean; // Strip reasoning content from context before sending excludeReasoningFromContext?: boolean; + // Enable model thinking/reasoning via chat_template_kwargs + enableThinking?: boolean; + // Reasoning effort level (low/medium/high/max) for thinking models + reasoningEffort?: ReasoningEffort; tools?: OpenAIToolDefinition[]; // Generation parameters temperature?: number; @@ -101,6 +110,7 @@ export interface SettingsChatServiceOptions { onToolCallChunk?: (chunk: string) => void; onAttachments?: (extras: DatabaseMessageExtra[]) => void; onModel?: (model: string) => void; + onCompletionId?: (id: string) => void; onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void; onComplete?: ( response: string, diff --git a/tools/ui/src/lib/utils/api-fetch.ts b/tools/ui/src/lib/utils/api-fetch.ts index 80781b98e..82a9383dd 100644 --- a/tools/ui/src/lib/utils/api-fetch.ts +++ b/tools/ui/src/lib/utils/api-fetch.ts @@ -1,6 +1,7 @@ import { base } from '$app/paths'; import { getJsonHeaders, getAuthHeaders } from './api-headers'; import { UrlProtocol } from '$lib/enums'; +import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '$lib/constants/error'; /** * API Fetch Utilities @@ -54,10 +55,15 @@ export async function apiFetch(path: string, options: ApiFetchOptions = {}): ? path : `${base}${path}`; - const response = await fetch(url, { - ...fetchOptions, - headers - }); + let response; + try { + response = await fetch(url, { + ...fetchOptions, + headers + }); + } catch (e) { + throw new Error(beautifyNetworkError(e)); + } if (!response.ok) { const errorMessage = await parseErrorMessage(response); @@ -101,10 +107,15 @@ export async function apiFetchWithParams( const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders(); const headers = { ...baseHeaders, ...customHeaders }; - const response = await fetch(url.toString(), { - ...fetchOptions, - headers - }); + let response; + try { + response = await fetch(url.toString(), { + ...fetchOptions, + headers + }); + } catch (e) { + throw new Error(beautifyNetworkError(e)); + } if (!response.ok) { const errorMessage = await parseErrorMessage(response); @@ -154,5 +165,37 @@ async function parseErrorMessage(response: Response): Promise { // JSON parsing failed, use status text } - return `Request failed: ${response.status} ${response.statusText}`; + const httpErrorStr = HTTP_CODE_TO_STRING[response.status]; + if (httpErrorStr) { + return httpErrorStr; + } + + return `${ERROR_MESSAGES.HTTP.GENERIC}: ${response.status} ${response.statusText}`; +} + +/** + * Converts a network issue into a human-readable message. + * @param throwable - The throwable raised during fetch operation + * @returns Error in an human-readable format + */ +function beautifyNetworkError(throwable: unknown): string { + let message; + if (throwable instanceof Error) { + message = throwable.message; + if (throwable.name === 'TypeError' && message.includes('fetch')) { + return ERROR_MESSAGES.NETWORK.UNREACHABLE; + } + } else { + message = String(throwable); + } + + if (message.includes('ECONNREFUSED')) { + return ERROR_MESSAGES.NETWORK.REFUSED; + } else if (message.includes('ENOTFOUND')) { + return ERROR_MESSAGES.NETWORK.NXDOMAIN; + } else if (message.includes('ETIMEDOUT')) { + return ERROR_MESSAGES.NETWORK.TIMEOUT; + } + + return `${ERROR_MESSAGES.NETWORK.GENERIC} (${message})`; } diff --git a/tools/ui/src/lib/utils/chat-template-thinking-detector.ts b/tools/ui/src/lib/utils/chat-template-thinking-detector.ts new file mode 100644 index 000000000..da6382f5c --- /dev/null +++ b/tools/ui/src/lib/utils/chat-template-thinking-detector.ts @@ -0,0 +1,86 @@ +/** + * Detects whether a model's chat template supports thinking/reasoning control. + * + * The server "/props" endpoint does NOT expose a supports_thinking flag. + * It is computed internally by common_chat_templates_support_enable_thinking + * in common/chat.cpp. A proper server flag would make this unnecessary. + * + * Detection order (most reliable first): + * 1. Thinking-control Jinja2 variables === pass-through via chat_template_kwargs + * 2. Thinking-control Jinja2 conditionals === template-native on/off logic + * 3. Paired thinking-content tag pairs === models that output special tags + */ + +const THINKING_KWARG_VARS = ['enable_thinking', 'reasoning_effort', 'thinking_budget']; + +/** + * Paired thinking-content tag patterns. + * + * Inspected: llama-cpp-deepseek-r1/v3, nim-nemotron-{3,4}-nano, qwen-qwq-32b, + * qwen-3-32b, google-gemma-4-31b-it, kimikimi-k2-thinking, apertus-8b-instruct, + * mistralai-Mistral-Small-3.2-24B, ByteDance-Seed-OSS. + * + * The self-closing entry is Kimi-K2, Gemma4 fixed-length pair, + * where both tags always appear adjacent with no content between. + */ +const THINKING_TAG_PATTERNS: Array<[string, string | null]> = [ + ['', ''], + ['<|channel>thought', '<|channel|>'], + ['<|think|>', ''], + ['', ''], + ['', null] +]; + +const JINJA_THINKING_CONDITIONALS: RegExp[] = [ + // Matches: {% if enable thinking %}, {% if enable_thinking %}, {% if (enable_thinking is defined) %} + // Handles: underscore-separated (enable_thinking), space-separated (enable thinking), + // and optional parens/brackets before enable (if (enable_thinking ) + /\{%-?\s*if\s+\(?\s*\w*enable[\s_]+\w*(thinking|think|reasoning)/i, + /\{%-?\s*if\s+\w*(thinking|reasoning)\s*(is not|==|!=)/i, + /\{%-?\s*if\s+not\s+\w*enable/i, + /\{%-?\s*if\s+ns\.enable_thinking/i +]; + +/** Guards against false positives: + * - Generic thought keyword (tool descriptions say "chain of thought") + * - Qwen vertical-bar token (used for ALL tool calls, not thinking) + */ +export function detectThinkingSupport(t: string): boolean { + if (!t) return false; + for (const kwarg of THINKING_KWARG_VARS) { + const regex = new RegExp( + `(\\{\\{[^{}]*\\b${kwarg}\\b[^{}]*\\}\\}|\\{%[^{}]*\\b${kwarg}\\b[^{}]*%\\})`, + 'i' + ); + if (regex.test(t)) return true; + } + for (const p of JINJA_THINKING_CONDITIONALS) { + if (p.test(t)) return true; + } + for (const [s, e] of THINKING_TAG_PATTERNS) { + if (t.includes(s) && (!e || t.includes(e))) return true; + } + return false; +} + +export function detectThinkingSupportWithReason(t: string): { supported: boolean; reason: string } { + if (!t) return { supported: false, reason: 'No chat template available' }; + for (const kwarg of THINKING_KWARG_VARS) { + const regex = new RegExp( + `(\\{\\{[^{}]*\\b${kwarg}\\b[^{}]*\\}\\}|\\{%[^{}]*\\b${kwarg}\\b[^{}]*%\\})`, + 'i' + ); + if (regex.test(t)) { + return { supported: true, reason: 'Found: ' + kwarg }; + } + } + for (const p of JINJA_THINKING_CONDITIONALS) { + if (p.test(t)) return { supported: true, reason: 'Found: thinking conditional' }; + } + for (const [s, e] of THINKING_TAG_PATTERNS) { + if (t.includes(s) && (!e || t.includes(e))) { + return { supported: true, reason: 'Found: ' + s + (e ? ' .. ' + e : ' (self)') }; + } + } + return { supported: false, reason: 'No thinking patterns found' }; +} diff --git a/tools/ui/tests/e2e/demo.test.ts b/tools/ui/tests/e2e/demo.test.ts index b7b4bac33..99a3e86da 100644 --- a/tools/ui/tests/e2e/demo.test.ts +++ b/tools/ui/tests/e2e/demo.test.ts @@ -1,6 +1,7 @@ import { expect, test } from '@playwright/test'; -test('home page has expected h1', async ({ page }) => { +test('home page loads correctly', async ({ page }) => { await page.goto('/'); - await expect(page.locator('h1').first()).toBeVisible(); + // Wait for the greeting to become visible (stores need time to initialize) + await expect(page.locator('h1', { hasText: /Hello there/ })).toBeVisible(); }); diff --git a/tools/ui/vite.config.ts b/tools/ui/vite.config.ts index f89a689d5..5b57eae3a 100644 --- a/tools/ui/vite.config.ts +++ b/tools/ui/vite.config.ts @@ -10,6 +10,8 @@ import { llamaCppBuildPlugin } from './scripts/vite-plugin-llama-cpp-build'; const __dirname = dirname(fileURLToPath(import.meta.url)); +const SERVER_ORIGIN = import.meta.env?.VITE_PUBLIC_SERVER_ORIGIN || 'http://localhost:8080'; + export default defineConfig({ resolve: { alias: { @@ -75,12 +77,12 @@ export default defineConfig({ server: { proxy: { - '/v1': 'http://localhost:8080', - '/props': 'http://localhost:8080', - '/models': 'http://localhost:8080', - '/tools': 'http://localhost:8080', - '/slots': 'http://localhost:8080', - '/cors-proxy': 'http://localhost:8080' + '/v1': SERVER_ORIGIN, + '/props': SERVER_ORIGIN, + '/models': SERVER_ORIGIN, + '/tools': SERVER_ORIGIN, + '/slots': SERVER_ORIGIN, + '/cors-proxy': SERVER_ORIGIN }, headers: { 'Cross-Origin-Embedder-Policy': 'require-corp', diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp index f3555f2d4..b9d05d929 100644 --- a/vendor/cpp-httplib/httplib.cpp +++ b/vendor/cpp-httplib/httplib.cpp @@ -1832,7 +1832,7 @@ int getaddrinfo_with_timeout(const char *node, const char *service, #ifdef _WIN32 // Windows-specific implementation using GetAddrInfoEx with overlapped I/O - OVERLAPPED overlapped = {0}; + OVERLAPPED overlapped = {}; HANDLE event = CreateEventW(nullptr, TRUE, FALSE, nullptr); if (!event) { return EAI_FAIL; } @@ -1841,7 +1841,7 @@ int getaddrinfo_with_timeout(const char *node, const char *service, PADDRINFOEXW result_addrinfo = nullptr; HANDLE cancel_handle = nullptr; - ADDRINFOEXW hints_ex = {0}; + ADDRINFOEXW hints_ex = {}; if (hints) { hints_ex.ai_flags = hints->ai_flags; hints_ex.ai_family = hints->ai_family; @@ -9912,13 +9912,28 @@ bool ClientImpl::process_request(Stream &strm, Request &req, } #endif - // Handle Expect: 100-continue with timeout - if (expect_100_continue && CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND > 0) { - time_t sec = CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND / 1000; - time_t usec = (CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND % 1000) * 1000; - auto ret = detail::select_read(strm.socket(), sec, usec); - if (ret <= 0) { - // Timeout or error: send body anyway (server didn't respond in time) + // Handle Expect: 100-continue. + // + // Wait for an interim/early response by attempting to read the status line + // under a short timeout, instead of trusting raw socket readability. Over + // TLS, post-handshake records (e.g. session tickets) make the socket + // readable without any HTTP response being available; relying on + // `select_read` there caused the body to be withheld forever and the + // request to fail with `Read` (#2458). If no status line arrives within the + // timeout, send the body anyway (matching curl's behavior). + auto status_line_read = false; + if (expect_100_continue && write_request_success) { + if (CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND > 0) { + time_t sec = CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND / 1000; + time_t usec = (CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND % 1000) * 1000; + strm.set_read_timeout(sec, usec); + status_line_read = read_response_line(strm, req, res, false); + strm.set_read_timeout(read_timeout_sec_, read_timeout_usec_); + } + + if (!status_line_read) { + // No interim response within the timeout: send the body and handle the + // response as usual. if (!write_request_body(strm, req, error)) { return false; } expect_100_continue = false; // Switch to normal response handling } @@ -9926,7 +9941,8 @@ bool ClientImpl::process_request(Stream &strm, Request &req, // Receive response and headers // When using Expect: 100-continue, don't auto-skip `100 Continue` response - if (!read_response_line(strm, req, res, !expect_100_continue) || + if ((!status_line_read && + !read_response_line(strm, req, res, !expect_100_continue)) || !detail::read_headers(strm, res.headers)) { if (write_request_success) { error = Error::Read; } output_error_log(error, &req); diff --git a/vendor/cpp-httplib/httplib.h b/vendor/cpp-httplib/httplib.h index af856dd63..cbb549e71 100644 --- a/vendor/cpp-httplib/httplib.h +++ b/vendor/cpp-httplib/httplib.h @@ -8,8 +8,8 @@ #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.46.0" -#define CPPHTTPLIB_VERSION_NUM "0x002e00" +#define CPPHTTPLIB_VERSION "0.46.1" +#define CPPHTTPLIB_VERSION_NUM "0x002e01" #ifdef _WIN32 #if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00