mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-07-10 01:18:32 +00:00
Merge branch 'upstream' into concedo_experimental
# Conflicts: # .devops/nix/package.nix # .github/workflows/server.yml # docs/development/HOWTO-add-model.md # ggml/src/ggml-hexagon/ggml-hexagon.cpp # ggml/src/ggml-hexagon/htp/CMakeLists.txt # ggml/src/ggml-hexagon/htp/argsort-ops.c # ggml/src/ggml-hexagon/htp/concat-ops.c # ggml/src/ggml-hexagon/htp/flash-attn-ops.c # ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c # ggml/src/ggml-hexagon/htp/hmx-flash-attn-ops.c # ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c # ggml/src/ggml-hexagon/htp/hmx-ops.h # ggml/src/ggml-hexagon/htp/htp-ctx.h # ggml/src/ggml-hexagon/htp/hvx-utils.h # ggml/src/ggml-hexagon/htp/main.c # ggml/src/ggml-hexagon/htp/matmul-ops.c # ggml/src/ggml-hexagon/htp/pad-ops.c # ggml/src/ggml-hexagon/htp/unary-ops.c # ggml/src/ggml-opencl/CMakeLists.txt # ggml/src/ggml-opencl/ggml-opencl.cpp # ggml/src/ggml-opencl/kernels/cvt.cl # ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl # scripts/sync_vendor.py # src/llama-context.cpp # tests/test-backend-sampler.cpp # tools/server/README.md # tools/ui/tests/stories/ChatScreenForm.a11y.stories.svelte
This commit is contained in:
Concedo
commit
13eaa04269
78 changed files with 1801 additions and 477 deletions
|
|
@ -1042,11 +1042,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
// we define here to make sure it's included in llama-gen-docs
|
||||
if (ex == LLAMA_EXAMPLE_COMPLETION) {
|
||||
params.use_jinja = false; // disable jinja by default
|
||||
|
||||
} else if (ex == LLAMA_EXAMPLE_MTMD) {
|
||||
params.use_jinja = false; // disable jinja by default
|
||||
params.sampling.temp = 0.2; // lower temp by default for better quality
|
||||
|
||||
} else if (ex == LLAMA_EXAMPLE_SERVER) {
|
||||
params.n_parallel = -1; // auto by default
|
||||
}
|
||||
|
|
@ -1067,7 +1065,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
sampler_type_names.pop_back(); // remove last semicolon
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* filter options by example
|
||||
* rules:
|
||||
|
|
@ -1081,7 +1078,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
add_opt(common_arg(
|
||||
{"-h", "--help", "--usage"},
|
||||
"print usage and exit",
|
||||
|
|
@ -4082,7 +4078,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
params.sampling.top_k = 0;
|
||||
params.sampling.min_p = 0.01f;
|
||||
params.use_jinja = true;
|
||||
//params.default_template_kwargs["reasoning_effort"] = "\"high\"";
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
|
||||
|
||||
|
|
@ -4101,7 +4096,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
params.sampling.top_k = 0;
|
||||
params.sampling.min_p = 0.01f;
|
||||
params.use_jinja = true;
|
||||
//params.default_template_kwargs["reasoning_effort"] = "\"high\"";
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
|
||||
|
||||
|
|
|
|||
|
|
@ -1395,8 +1395,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
|||
if (params.warmup) {
|
||||
LOG_INF("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__);
|
||||
|
||||
llama_set_warmup(lctx, true);
|
||||
|
||||
std::vector<llama_token> tmp;
|
||||
llama_token bos = llama_vocab_bos(vocab);
|
||||
llama_token eos = llama_vocab_eos(vocab);
|
||||
|
|
@ -1427,7 +1425,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
|||
llama_memory_clear(llama_get_memory(lctx), true);
|
||||
llama_synchronize(lctx);
|
||||
llama_perf_context_reset(lctx);
|
||||
llama_set_warmup(lctx, false);
|
||||
|
||||
// reset samplers to reset RNG state after warmup to the seeded state
|
||||
res->reset_samplers();
|
||||
|
|
@ -1569,6 +1566,7 @@ struct llama_context_params common_context_params_to_llama(const common_params &
|
|||
cparams.n_ctx = params.n_ctx;
|
||||
cparams.n_seq_max = params.n_parallel;
|
||||
cparams.n_rs_seq = params.speculative.need_n_rs_seq();
|
||||
cparams.n_outputs_max = std::max(params.n_outputs_max, 0);
|
||||
cparams.n_batch = params.n_batch;
|
||||
cparams.n_ubatch = params.n_ubatch;
|
||||
cparams.n_threads = params.cpuparams.n_threads;
|
||||
|
|
|
|||
|
|
@ -278,6 +278,7 @@ struct common_params_sampling {
|
|||
std::vector<llama_token> reasoning_budget_end; // end tag token sequence
|
||||
std::vector<llama_token> reasoning_budget_forced; // forced sequence (message + end tag)
|
||||
std::string reasoning_budget_message; // message injected before end tag when budget exhausted
|
||||
bool reasoning_control = false; // create the budget sampler on demand so reasoning can be ended at runtime
|
||||
|
||||
bool backend_sampling = false;
|
||||
|
||||
|
|
@ -432,6 +433,7 @@ struct common_params {
|
|||
int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)
|
||||
int32_t n_parallel = 1; // number of parallel sequences to decode
|
||||
int32_t n_sequences = 1; // number of sequences to decode
|
||||
int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch)
|
||||
int32_t grp_attn_n = 1; // group-attention factor
|
||||
int32_t grp_attn_w = 512; // group-attention width
|
||||
int32_t n_print = -1; // print token count every n tokens (-1 = disabled)
|
||||
|
|
|
|||
|
|
@ -293,7 +293,7 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st
|
|||
}
|
||||
|
||||
// reasoning budget sampler (skip when budget is unlimited unless a lazy grammar is active, which needs rbudget for thinking-block suppression)
|
||||
if (!params.reasoning_budget_start.empty() && !params.reasoning_budget_end.empty() && (params.grammar_lazy || params.reasoning_budget_tokens >= 0)) {
|
||||
if (!params.reasoning_budget_start.empty() && !params.reasoning_budget_end.empty() && (params.grammar_lazy || params.reasoning_budget_tokens >= 0 || params.reasoning_control)) {
|
||||
rbudget = common_reasoning_budget_init(
|
||||
vocab,
|
||||
params.reasoning_budget_start,
|
||||
|
|
|
|||
|
|
@ -1317,6 +1317,40 @@ static uint32_t common_get_enabled_speculative_configs(const std::vector<common_
|
|||
return result;
|
||||
}
|
||||
|
||||
int32_t common_speculative_n_max(const common_params_speculative * spec) {
|
||||
int32_t n_max = 0;
|
||||
|
||||
for (const auto type : spec->types) {
|
||||
switch (type) {
|
||||
case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE:
|
||||
case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:
|
||||
case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:
|
||||
n_max = std::max(n_max, std::max(0, spec->draft.n_max));
|
||||
break;
|
||||
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
|
||||
n_max = std::max(n_max, (int32_t) spec->ngram_simple.size_m);
|
||||
break;
|
||||
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:
|
||||
n_max = std::max(n_max, (int32_t) spec->ngram_map_k.size_m);
|
||||
break;
|
||||
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V:
|
||||
n_max = std::max(n_max, (int32_t) spec->ngram_map_k4v.size_m);
|
||||
break;
|
||||
case COMMON_SPECULATIVE_TYPE_NGRAM_MOD:
|
||||
n_max = std::max(n_max, std::max(0, spec->ngram_mod.n_max));
|
||||
break;
|
||||
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:
|
||||
n_max = std::max(n_max, (int32_t) 8);
|
||||
break;
|
||||
case COMMON_SPECULATIVE_TYPE_NONE:
|
||||
case COMMON_SPECULATIVE_TYPE_COUNT:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return n_max;
|
||||
}
|
||||
|
||||
// initialization of the speculative decoding system
|
||||
//
|
||||
common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) {
|
||||
|
|
@ -1325,8 +1359,6 @@ common_speculative * common_speculative_init(common_params_speculative & params,
|
|||
{
|
||||
uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types);
|
||||
|
||||
bool has_draft_model_path = !params.draft.mparams.path.empty();
|
||||
|
||||
bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE));
|
||||
bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3
|
||||
bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr;
|
||||
|
|
@ -1359,16 +1391,6 @@ common_speculative * common_speculative_init(common_params_speculative & params,
|
|||
if (has_ngram_cache) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params));
|
||||
}
|
||||
if (has_draft_simple) {
|
||||
if (!has_draft_model_path) {
|
||||
LOG_WRN("%s: draft model is not specified - cannot use 'draft' type\n", __func__);
|
||||
has_draft_simple = false;
|
||||
}
|
||||
} else if (has_draft_model_path && !has_mtp && !has_draft_eagle3) {
|
||||
LOG_WRN("%s: draft model is specified but 'draft' speculative type is not explicitly enabled - enabling it\n", __func__);
|
||||
has_draft_simple = true;
|
||||
}
|
||||
|
||||
if (has_draft_simple) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@ enum common_speculative_type common_speculative_type_from_name(const std::string
|
|||
// convert type to string
|
||||
std::string common_speculative_type_to_str(enum common_speculative_type type);
|
||||
|
||||
// return the max number of draft tokens based on the speculative parameters
|
||||
int32_t common_speculative_n_max(const common_params_speculative * spec);
|
||||
|
||||
common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq);
|
||||
|
||||
void common_speculative_free(common_speculative * spec);
|
||||
|
|
|
|||
|
|
@ -215,6 +215,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
|||
"Starcoder2ForCausalLM": "starcoder",
|
||||
"Step3p5ForCausalLM": "step3",
|
||||
"StepVLForConditionalGeneration": "step3",
|
||||
"Step3p7ForConditionalGeneration": "step3",
|
||||
"T5EncoderModel": "t5",
|
||||
"T5ForConditionalGeneration": "t5",
|
||||
"T5WithLMHeadModel": "t5",
|
||||
|
|
@ -283,6 +284,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
|||
"Sarashina2VisionForCausalLM": "sarashina2",
|
||||
"SmolVLMForConditionalGeneration": "smolvlm",
|
||||
"StepVLForConditionalGeneration": "step3",
|
||||
"Step3p7ForConditionalGeneration": "step3",
|
||||
"UltravoxModel": "ultravox",
|
||||
"VoxtralForConditionalGeneration": "ultravox",
|
||||
"YoutuVLForConditionalGeneration": "youtuvl",
|
||||
|
|
|
|||
|
|
@ -2593,7 +2593,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
|
|||
# Step3-VL keeps text config under text_config but uses a custom top-level architecture.
|
||||
# For text conversion we route to a dedicated text-only class.
|
||||
# TODO: refactor this later to avoid adding exception here
|
||||
if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration"):
|
||||
if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration"):
|
||||
return arch
|
||||
|
||||
# if "architectures" is found in the sub-config, use that instead
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from .base import MmprojModel, ModelBase, TextModel, _MISTRAL_COMMON_DATASET_MEA
|
|||
from .qwen import Qwen3Model
|
||||
|
||||
|
||||
@ModelBase.register("StepVLForConditionalGeneration")
|
||||
@ModelBase.register("StepVLForConditionalGeneration", "Step3p7ForConditionalGeneration")
|
||||
class Step3VLVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
|
@ -95,7 +95,7 @@ class Step3VLTextModel(Qwen3Model):
|
|||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
|
||||
@ModelBase.register("Step3p5ForCausalLM")
|
||||
@ModelBase.register("Step3p5ForCausalLM", "Step3p7ForConditionalGeneration")
|
||||
class Step35Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.STEP35
|
||||
|
||||
|
|
@ -203,11 +203,23 @@ class Step35Model(TextModel):
|
|||
if isinstance(rope_theta, list):
|
||||
rope_theta = rope_theta[0]
|
||||
base = float(rope_theta)
|
||||
if (dim := self.hparams.get("head_dim")) is None:
|
||||
dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
|
||||
dim = int(dim)
|
||||
|
||||
freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
|
||||
if (storage_dim := self.hparams.get("head_dim")) is None:
|
||||
storage_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
|
||||
storage_dim = int(storage_dim)
|
||||
|
||||
# Llama 3 factors apply only to the rotary dims used by full_attention layers
|
||||
# (partial_rotary_factor * head_dim). Remaining slots are padded with 1.0 so
|
||||
# sliding_attention layers remain unaffected. set_gguf_parameters already
|
||||
# guarantees at least one full_attention layer.
|
||||
layer_types = (self.hparams.get("layer_types") or [])[: self.block_count]
|
||||
partial_rotary_factors = (self.hparams.get("partial_rotary_factors") or [])[: self.block_count]
|
||||
full_attention_factor = next(
|
||||
float(f) for lt, f in zip(layer_types, partial_rotary_factors) if lt == "full_attention"
|
||||
)
|
||||
rotary_dim = int(storage_dim * full_attention_factor)
|
||||
|
||||
freqs = 1.0 / (base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float32) / rotary_dim))
|
||||
|
||||
factor = float(rope_params.get("factor", 8.0))
|
||||
low_freq_factor = float(rope_params.get("low_freq_factor", 1.0))
|
||||
|
|
@ -228,4 +240,8 @@ class Step35Model(TextModel):
|
|||
smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)
|
||||
rope_factors.append(1.0 / ((1.0 - smooth) / factor + smooth))
|
||||
|
||||
# Pad to head_dim/2 with 1.0 so non-scaled layers remain neutral.
|
||||
if len(rope_factors) < storage_dim // 2:
|
||||
rope_factors.extend([1.0] * (storage_dim // 2 - len(rope_factors)))
|
||||
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32))
|
||||
|
|
|
|||
|
|
@ -568,7 +568,6 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
|||
constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols);
|
||||
constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2);
|
||||
|
||||
constexpr int stride_tile_Q = DKQ/2 + 4;
|
||||
constexpr int stride_tile_K = nbatch_K2 + 4;
|
||||
|
||||
constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4;
|
||||
|
|
@ -604,9 +603,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
|||
#pragma unroll
|
||||
for (int k0_start = (DKQ/2-1) - (DKQ/2-1) % nbatch_K2; k0_start >= 0; k0_start -= nbatch_K2) {
|
||||
const int k0_stop = k0_start + nbatch_K2 < DKQ/2 ? k0_start + nbatch_K2 : DKQ/2;
|
||||
const int k0_diff = k0_stop - k0_start;
|
||||
|
||||
if constexpr (nstages <= 1) {
|
||||
const int k0_diff = k0_stop - k0_start;
|
||||
constexpr bool use_cp_async = nstages == 1;
|
||||
flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
(K_h2 + int64_t(k_VKQ_0)*stride_K + k0_start, tile_K, k0_diff, stride_K, k_VKQ_sup);
|
||||
|
|
@ -640,6 +639,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
|||
}
|
||||
}
|
||||
} else {
|
||||
constexpr int stride_tile_Q = DKQ/2 + 4;
|
||||
#pragma unroll
|
||||
for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) {
|
||||
load_ldmatrix(Q_B[0], tile_Q + (threadIdx.y / np)*(T_B_KQ::I*stride_tile_Q) + k_KQ_0, stride_tile_Q);
|
||||
|
|
@ -954,9 +954,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
|||
for (int i0_start = 0; i0_start < DV; i0_start += 2*nbatch_V2) {
|
||||
static_assert(DV % (2*nbatch_V2) == 0, "bad loop size");
|
||||
const int i0_stop = i0_start + 2*nbatch_V2;
|
||||
const int i0_diff = i0_stop - i0_start;
|
||||
|
||||
if constexpr (nstages <= 1) {
|
||||
const int i0_diff = i0_stop - i0_start;
|
||||
if (!V_is_K_view || i0_stop > 2*nbatch_K2) {
|
||||
constexpr bool use_cp_async = nstages == 1;
|
||||
flash_attn_ext_f16_load_tile<stride_tile_V, nwarps, nbatch_fa, use_cp_async, oob_check>
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ gated_delta_net_cuda(const float * q,
|
|||
// output state layout (per-slot D * n_seqs) — same per-(seq,head) offset as before.
|
||||
const int64_t state_in_offset = sequence * K * H * S_v * S_v + h_idx * S_v * S_v;
|
||||
const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v;
|
||||
const int64_t state_size_per_token = S_v * S_v * H * n_seqs; // per-slot stride in output
|
||||
state += state_out_offset;
|
||||
curr_state += state_in_offset + col * S_v;
|
||||
attn_data += (sequence * n_tokens * H + h_idx) * S_v;
|
||||
|
|
@ -61,10 +60,6 @@ gated_delta_net_cuda(const float * q,
|
|||
s_shard[r] = curr_state[i];
|
||||
}
|
||||
|
||||
// slot mapping: target_slot = t - shift. When n_tokens < K only the last n_tokens slots
|
||||
// are written; earlier slots are left untouched (caller-owned).
|
||||
const int shift = (int) n_tokens - K;
|
||||
|
||||
for (int t = 0; t < n_tokens; t++) {
|
||||
const float * q_t = q + iq3 * sq3 + t * sq2 + iq1 * sq1;
|
||||
const float * k_t = k + iq3 * sq3 + t * sq2 + iq1 * sq1;
|
||||
|
|
@ -148,6 +143,11 @@ gated_delta_net_cuda(const float * q,
|
|||
attn_data += S_v * H;
|
||||
|
||||
if constexpr (keep_rs_t) {
|
||||
// slot mapping: target_slot = t - shift. When n_tokens < K only the last n_tokens slots
|
||||
// are written; earlier slots are left untouched (caller-owned).
|
||||
const int shift = (int) n_tokens - K;
|
||||
|
||||
const int64_t state_size_per_token = S_v * S_v * H * n_seqs; // per-slot stride in output
|
||||
const int target_slot = t - shift;
|
||||
if (target_slot >= 0 && target_slot < K) {
|
||||
float * curr_state = (dst + attn_score_elems) + target_slot * state_size_per_token + state_out_offset;
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ static __global__ void mul_mat_f(
|
|||
const int row0 = blockIdx.x * rows_per_block;
|
||||
|
||||
int expert_idx = 0;
|
||||
int col_base = 0;
|
||||
[[maybe_unused]] int col_base = 0;
|
||||
|
||||
const int channel_dst = has_ids ? 0 : blockIdx.y;
|
||||
|
||||
|
|
@ -122,12 +122,12 @@ static __global__ void mul_mat_f(
|
|||
ids += col_offset * stride_row_id;
|
||||
}
|
||||
|
||||
const float2 * y2 = (const float2 *) y;
|
||||
[[maybe_unused]] const float2 * y2 = (const float2 *) y;
|
||||
|
||||
extern __shared__ char data_mmv[];
|
||||
|
||||
char * shmem_base = data_mmv;
|
||||
int * slot_map = (int *) shmem_base;
|
||||
[[maybe_unused]] int * slot_map = (int *) shmem_base;
|
||||
char * compute_base = has_ids ? (shmem_base + GGML_PAD(cols_per_block, 16) * sizeof(int)) : shmem_base;
|
||||
|
||||
tile_C C[ntA][ntB];
|
||||
|
|
|
|||
|
|
@ -80,9 +80,8 @@ static __global__ void mul_mat_vec_f(
|
|||
gate_x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row*stride_row;
|
||||
}
|
||||
|
||||
const int channel_bias = ids ? channel_x : channel_dst;
|
||||
|
||||
if constexpr (has_fusion) {
|
||||
const int channel_bias = ids ? channel_x : channel_dst;
|
||||
if (use_bias) {
|
||||
x_bias += int64_t(sample_dst)*stride_sample_dst + channel_bias*stride_channel_dst;
|
||||
}
|
||||
|
|
@ -95,7 +94,7 @@ static __global__ void mul_mat_vec_f(
|
|||
|
||||
extern __shared__ char data_mmv[];
|
||||
float * buf_iw = (float *) data_mmv;
|
||||
float * buf_iw_gate = nullptr;
|
||||
[[maybe_unused]] float * buf_iw_gate = nullptr;
|
||||
if constexpr (has_fusion) {
|
||||
buf_iw_gate = (float *) (data_mmv + warp_size*sizeof(float));
|
||||
}
|
||||
|
|
@ -123,7 +122,7 @@ static __global__ void mul_mat_vec_f(
|
|||
|
||||
if constexpr (std::is_same_v<T, float>) {
|
||||
const float2 * x2 = (const float2 *) x;
|
||||
const float2 * gate_x2 = nullptr;
|
||||
[[maybe_unused]] const float2 * gate_x2 = nullptr;
|
||||
if constexpr (has_fusion) {
|
||||
if (use_gate) {
|
||||
gate_x2 = (const float2 *) gate_x;
|
||||
|
|
@ -155,7 +154,7 @@ static __global__ void mul_mat_vec_f(
|
|||
}
|
||||
} else if constexpr (std::is_same_v<T, half>) {
|
||||
const half2 * x2 = (const half2 *) x;
|
||||
const half2 * gate_x2 = nullptr;
|
||||
[[maybe_unused]] const half2 * gate_x2 = nullptr;
|
||||
if constexpr (has_fusion) {
|
||||
if (use_gate) {
|
||||
gate_x2 = (const half2 *) gate_x;
|
||||
|
|
@ -266,7 +265,7 @@ static __global__ void mul_mat_vec_f(
|
|||
}
|
||||
#else
|
||||
const nv_bfloat162 * x2 = (const nv_bfloat162 *) x;
|
||||
const nv_bfloat162 * gate_x2 = nullptr;
|
||||
[[maybe_unused]] const nv_bfloat162 * gate_x2 = nullptr;
|
||||
if constexpr (has_fusion) {
|
||||
if (use_gate) {
|
||||
gate_x2 = (const nv_bfloat162 *) gate_x;
|
||||
|
|
@ -274,7 +273,7 @@ static __global__ void mul_mat_vec_f(
|
|||
}
|
||||
for (int col2 = tid; col2 < ncols2; col2 += block_size) {
|
||||
const nv_bfloat162 tmpx = x2[col2];
|
||||
nv_bfloat162 tmpx_gate;
|
||||
[[maybe_unused]] nv_bfloat162 tmpx_gate;
|
||||
if constexpr (has_fusion) {
|
||||
if (use_gate) {
|
||||
tmpx_gate = gate_x2[col2];
|
||||
|
|
|
|||
|
|
@ -515,7 +515,7 @@ static __global__ void mul_mat_vec_q(
|
|||
bool use_gate = false;
|
||||
bool use_bias = false;
|
||||
bool use_gate_bias = false;
|
||||
const void * vgate = nullptr;
|
||||
[[maybe_unused]] const void * vgate = nullptr;
|
||||
const float * x_bias = nullptr;
|
||||
const float * gate_bias = nullptr;
|
||||
ggml_glu_op active_glu;
|
||||
|
|
@ -531,8 +531,8 @@ static __global__ void mul_mat_vec_q(
|
|||
}
|
||||
|
||||
|
||||
float x_biases[ncols_dst] = { 0.0f };
|
||||
float gate_biases[ncols_dst] = { 0.0f };
|
||||
[[maybe_unused]] float x_biases[ncols_dst] = { 0.0f };
|
||||
[[maybe_unused]] float gate_biases[ncols_dst] = { 0.0f };
|
||||
if constexpr (has_fusion) {
|
||||
const uint32_t channel_bias = ids ? channel_x : channel_dst;
|
||||
if (use_bias) {
|
||||
|
|
@ -589,12 +589,7 @@ static __global__ void mul_mat_vec_q(
|
|||
}
|
||||
|
||||
__shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];
|
||||
__shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];
|
||||
if constexpr (!has_fusion) {
|
||||
(void) tmp_shared_gate;
|
||||
} else if (!use_gate) {
|
||||
(void) tmp_shared_gate;
|
||||
}
|
||||
[[maybe_unused]] __shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];
|
||||
|
||||
if (threadIdx.y > 0) {
|
||||
#pragma unroll
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float *
|
|||
|
||||
// selection_wt is only needed when bias is present (selection uses wt + bias)
|
||||
// when no bias, we use wt directly for both selection and weight values
|
||||
float selection_wt[has_bias ? experts_per_thread : 1];
|
||||
[[maybe_unused]] float selection_wt[has_bias ? experts_per_thread : 1];
|
||||
|
||||
if constexpr (has_bias) {
|
||||
#pragma unroll
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ struct htp_opnode {
|
|||
}
|
||||
|
||||
std::vector<const ggml_tensor *> get_inputs() const {
|
||||
std::vector<const ggml_tensor *> inputs;
|
||||
std::vector<const ggml_tensor *> inputs(GGML_MAX_SRC, nullptr);
|
||||
std::vector<const ggml_tensor *> outputs;
|
||||
outputs.push_back(node);
|
||||
for (const auto * f : fused) {
|
||||
|
|
@ -70,20 +70,38 @@ struct htp_opnode {
|
|||
return false;
|
||||
};
|
||||
|
||||
int count = 0;
|
||||
auto add_input = [&](const ggml_tensor * t) {
|
||||
if (t && !contains(outputs, t) && !contains(inputs, t)) {
|
||||
inputs.push_back(t);
|
||||
if (count < (int)inputs.size()) {
|
||||
inputs[count++] = t;
|
||||
} else {
|
||||
inputs.push_back(t);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (int i = 0; i < GGML_MAX_SRC && node->src[i]; i++) {
|
||||
add_input(node->src[i]);
|
||||
}
|
||||
for (const auto * f : fused) {
|
||||
for (int i = 0; i < GGML_MAX_SRC && f->src[i]; i++) {
|
||||
add_input(f->src[i]);
|
||||
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
||||
if (fused.empty()) {
|
||||
inputs[i] = node->src[i];
|
||||
} else {
|
||||
if (node->src[i]) {
|
||||
add_input(node->src[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const auto * f : fused) {
|
||||
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
||||
if (f->src[i]) {
|
||||
add_input(f->src[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!fused.empty()) {
|
||||
inputs.resize(count);
|
||||
}
|
||||
|
||||
return inputs;
|
||||
}
|
||||
|
||||
|
|
@ -108,6 +126,9 @@ struct htp_opformat {
|
|||
char names[64 * GGML_MAX_SRC];
|
||||
|
||||
int format_tensor_dims(char * str, const struct ggml_tensor * t) {
|
||||
if (!t) {
|
||||
return sprintf(str, "NONE");
|
||||
}
|
||||
if (t->ne[2] == 1 && t->ne[3] == 1) {
|
||||
return sprintf(str, "%d:%d", (int) t->ne[0], (int) t->ne[1]);
|
||||
} else {
|
||||
|
|
@ -136,6 +157,9 @@ struct htp_opformat {
|
|||
}
|
||||
|
||||
int format_tensor_strides(char * str, const struct ggml_tensor * t) {
|
||||
if (!t) {
|
||||
return sprintf(str, "NONE");
|
||||
}
|
||||
const char * c = ggml_is_contiguous(t) ? "" : "!";
|
||||
|
||||
if (t->ne[2] == 1 && t->ne[3] == 1) {
|
||||
|
|
@ -170,11 +194,11 @@ struct htp_opformat {
|
|||
auto inputs = node.get_inputs();
|
||||
|
||||
if (!inputs.empty()) {
|
||||
p += sprintf(p, "%s", ggml_type_name(inputs[0]->type));
|
||||
p += sprintf(p, "%s", inputs[0] ? ggml_type_name(inputs[0]->type) : "NONE");
|
||||
|
||||
for (size_t i = 1; i < inputs.size(); i++) {
|
||||
p += sprintf(p, " x ");
|
||||
p += sprintf(p, "%s", ggml_type_name(inputs[i]->type));
|
||||
p += sprintf(p, "%s", inputs[i] ? ggml_type_name(inputs[i]->type) : "NONE");
|
||||
}
|
||||
|
||||
p += sprintf(p, " -> ");
|
||||
|
|
@ -184,7 +208,7 @@ struct htp_opformat {
|
|||
}
|
||||
|
||||
const char * tensor_buff_name(const struct ggml_tensor * t) {
|
||||
if (t->buffer) {
|
||||
if (t && t->buffer) {
|
||||
return ggml_backend_buffer_name(t->buffer);
|
||||
}
|
||||
return "NONE";
|
||||
|
|
@ -213,11 +237,11 @@ struct htp_opformat {
|
|||
auto inputs = node.get_inputs();
|
||||
|
||||
if (!inputs.empty()) {
|
||||
p += sprintf(p, "%s", inputs[0]->name);
|
||||
p += sprintf(p, "%s", inputs[0] ? inputs[0]->name : "NONE");
|
||||
|
||||
for (size_t i = 1; i < inputs.size(); i++) {
|
||||
p += sprintf(p, " x ");
|
||||
p += sprintf(p, "%s", inputs[i]->name);
|
||||
p += sprintf(p, "%s", inputs[i] ? inputs[i]->name : "NONE");
|
||||
}
|
||||
|
||||
p += sprintf(p, " -> ");
|
||||
|
|
|
|||
|
|
@ -1113,7 +1113,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
|||
case GGML_GLU_OP_SWIGLU_OAI:
|
||||
case GGML_GLU_OP_GEGLU_ERF:
|
||||
case GGML_GLU_OP_GEGLU_QUICK:
|
||||
return ggml_is_contiguous_1(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
|
||||
return ggml_is_contiguous_1(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1421,7 +1421,8 @@ template [[host_name("kernel_repeat_f16")]] kernel kernel_repeat_t kernel_repeat
|
|||
template [[host_name("kernel_repeat_i32")]] kernel kernel_repeat_t kernel_repeat<int>;
|
||||
template [[host_name("kernel_repeat_i16")]] kernel kernel_repeat_t kernel_repeat<short>;
|
||||
|
||||
kernel void kernel_reglu_f32(
|
||||
template<typename T>
|
||||
kernel void kernel_reglu(
|
||||
constant ggml_metal_kargs_glu & args,
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
|
|
@ -1429,19 +1430,25 @@ kernel void kernel_reglu_f32(
|
|||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
const float x1 = src1_row[i0];
|
||||
|
||||
dst_row[i0] = x0*x1*(x0 > 0.0f);
|
||||
dst_row[i0] = (T)(x0*x1*(x0 > 0.0f));
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_geglu_f32(
|
||||
typedef decltype(kernel_reglu<float>) kernel_reglu_t;
|
||||
|
||||
template [[host_name("kernel_reglu_f32")]] kernel kernel_reglu_t kernel_reglu<float>;
|
||||
template [[host_name("kernel_reglu_f16")]] kernel kernel_reglu_t kernel_reglu<half>;
|
||||
|
||||
template<typename T>
|
||||
kernel void kernel_geglu(
|
||||
constant ggml_metal_kargs_glu & args,
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
|
|
@ -1449,9 +1456,9 @@ kernel void kernel_geglu_f32(
|
|||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
|
|
@ -1459,11 +1466,17 @@ kernel void kernel_geglu_f32(
|
|||
|
||||
const float gelu = 0.5f*x0*(1.0f + precise::tanh(SQRT_2_OVER_PI*x0*(1.0f + GELU_COEF_A*x0*x0)));
|
||||
|
||||
dst_row[i0] = gelu*x1;
|
||||
dst_row[i0] = (T)(gelu*x1);
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_swiglu_f32(
|
||||
typedef decltype(kernel_geglu<float>) kernel_geglu_t;
|
||||
|
||||
template [[host_name("kernel_geglu_f32")]] kernel kernel_geglu_t kernel_geglu<float>;
|
||||
template [[host_name("kernel_geglu_f16")]] kernel kernel_geglu_t kernel_geglu<half>;
|
||||
|
||||
template<typename T>
|
||||
kernel void kernel_swiglu(
|
||||
constant ggml_metal_kargs_glu & args,
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
|
|
@ -1471,9 +1484,9 @@ kernel void kernel_swiglu_f32(
|
|||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
|
|
@ -1481,11 +1494,17 @@ kernel void kernel_swiglu_f32(
|
|||
|
||||
const float silu = x0 / (1.0f + exp(-x0));
|
||||
|
||||
dst_row[i0] = silu*x1;
|
||||
dst_row[i0] = (T)(silu*x1);
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_swiglu_oai_f32(
|
||||
typedef decltype(kernel_swiglu<float>) kernel_swiglu_t;
|
||||
|
||||
template [[host_name("kernel_swiglu_f32")]] kernel kernel_swiglu_t kernel_swiglu<float>;
|
||||
template [[host_name("kernel_swiglu_f16")]] kernel kernel_swiglu_t kernel_swiglu<half>;
|
||||
|
||||
template<typename T>
|
||||
kernel void kernel_swiglu_oai(
|
||||
constant ggml_metal_kargs_glu & args,
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
|
|
@ -1493,9 +1512,9 @@ kernel void kernel_swiglu_oai_f32(
|
|||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
float x0 = src0_row[i0];
|
||||
|
|
@ -1507,11 +1526,17 @@ kernel void kernel_swiglu_oai_f32(
|
|||
float out_glu = x0 / (1.0f + exp(-x0 * args.alpha));
|
||||
out_glu = out_glu * (1.0f + x1);
|
||||
|
||||
dst_row[i0] = out_glu;
|
||||
dst_row[i0] = (T)out_glu;
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_geglu_erf_f32(
|
||||
typedef decltype(kernel_swiglu_oai<float>) kernel_swiglu_oai_t;
|
||||
|
||||
template [[host_name("kernel_swiglu_oai_f32")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai<float>;
|
||||
template [[host_name("kernel_swiglu_oai_f16")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai<half>;
|
||||
|
||||
template<typename T>
|
||||
kernel void kernel_geglu_erf(
|
||||
constant ggml_metal_kargs_glu & args,
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
|
|
@ -1519,9 +1544,9 @@ kernel void kernel_geglu_erf_f32(
|
|||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
|
|
@ -1529,11 +1554,17 @@ kernel void kernel_geglu_erf_f32(
|
|||
|
||||
const float gelu_erf = 0.5f*x0*(1.0f+erf_approx<float>(x0*SQRT_2_INV));
|
||||
|
||||
dst_row[i0] = gelu_erf*x1;
|
||||
dst_row[i0] = (T)(gelu_erf*x1);
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_geglu_quick_f32(
|
||||
typedef decltype(kernel_geglu_erf<float>) kernel_geglu_erf_t;
|
||||
|
||||
template [[host_name("kernel_geglu_erf_f32")]] kernel kernel_geglu_erf_t kernel_geglu_erf<float>;
|
||||
template [[host_name("kernel_geglu_erf_f16")]] kernel kernel_geglu_erf_t kernel_geglu_erf<half>;
|
||||
|
||||
template<typename T>
|
||||
kernel void kernel_geglu_quick(
|
||||
constant ggml_metal_kargs_glu & args,
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
|
|
@ -1541,9 +1572,9 @@ kernel void kernel_geglu_quick_f32(
|
|||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1);
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
|
|
@ -1551,10 +1582,15 @@ kernel void kernel_geglu_quick_f32(
|
|||
|
||||
const float gelu_quick = x0*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x0)));
|
||||
|
||||
dst_row[i0] = gelu_quick*x1;
|
||||
dst_row[i0] = (T)(gelu_quick*x1);
|
||||
}
|
||||
}
|
||||
|
||||
typedef decltype(kernel_geglu_quick<float>) kernel_geglu_quick_t;
|
||||
|
||||
template [[host_name("kernel_geglu_quick_f32")]] kernel kernel_geglu_quick_t kernel_geglu_quick<float>;
|
||||
template [[host_name("kernel_geglu_quick_f16")]] kernel kernel_geglu_quick_t kernel_geglu_quick<half>;
|
||||
|
||||
kernel void kernel_op_sum_f32(
|
||||
constant ggml_metal_kargs_sum & args,
|
||||
device const float * src0,
|
||||
|
|
|
|||
|
|
@ -64,8 +64,10 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
|
|||
#include <map>
|
||||
#include <set>
|
||||
#include <unordered_map>
|
||||
#include <shared_mutex>
|
||||
#include <mutex>
|
||||
#include <future>
|
||||
#include <condition_variable>
|
||||
#include <thread>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
|
|
@ -164,8 +166,9 @@ struct vk_pipeline_struct {
|
|||
uint32_t align;
|
||||
// true if fields have been set by ggml_vk_create_pipeline
|
||||
bool initialized {};
|
||||
// set to true to request the pipeline is compiled
|
||||
std::atomic<bool> needed {};
|
||||
// true while a compile is in flight, used to dedupe concurrent claims.
|
||||
// Protected by device->compile_mutex.
|
||||
bool compile_pending {};
|
||||
// set to true when the shader has been compiled
|
||||
std::atomic<bool> compiled {};
|
||||
// number of registers used, extracted from pipeline executable properties
|
||||
|
|
@ -624,6 +627,14 @@ static constexpr std::initializer_list<std::array<int, 3>> rms_norm_mul_rope_vie
|
|||
|
||||
struct vk_device_struct {
|
||||
std::recursive_mutex mutex;
|
||||
mutable std::shared_mutex pinned_memory_mutex;
|
||||
|
||||
// Guards compile_pending, all_pipelines, and the dynamic pipeline maps
|
||||
// (flash_attn, fa_mask_opt, solve_tri, conv2d, etc). The actual compile
|
||||
// runs with no lock held, so different pipelines can compile in parallel.
|
||||
// Lock order is device->mutex -> compile_mutex, never the reverse.
|
||||
std::mutex compile_mutex;
|
||||
std::condition_variable compile_cv;
|
||||
|
||||
vk::PhysicalDevice physical_device;
|
||||
vk::PhysicalDeviceProperties properties;
|
||||
|
|
@ -1733,7 +1744,7 @@ struct ggml_vk_garbage_collector {
|
|||
};
|
||||
|
||||
static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx);
|
||||
static void ggml_vk_load_shaders(vk_device& device);
|
||||
static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr);
|
||||
static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx);
|
||||
|
||||
static bool vk_memory_logger_enabled = false;
|
||||
|
|
@ -2200,11 +2211,6 @@ static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {
|
|||
ctx->device->device.resetFences({ ctx->fence });
|
||||
}
|
||||
|
||||
// variables to track number of compiles in progress
|
||||
static uint32_t compile_count = 0;
|
||||
static std::mutex compile_count_mutex;
|
||||
static std::condition_variable compile_count_cond;
|
||||
|
||||
static constexpr uint32_t kSpvOpCooperativeMatrixLoadTensorNV = 5367;
|
||||
static constexpr uint32_t kSpvCapabilityCooperativeMatrixDecodeVectorNV = 5447;
|
||||
static constexpr uint32_t kSpvTensorAddressingDecodeVectorFuncBit = 0x4;
|
||||
|
|
@ -2499,7 +2505,6 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
|
|||
std::cerr << "ggml_vulkan: " << e.what() << std::endl;
|
||||
throw e;
|
||||
}
|
||||
pipeline->compiled = true;
|
||||
|
||||
if (vk_instance.debug_utils_support) {
|
||||
vk::DebugUtilsObjectNameInfoEXT duoni;
|
||||
|
|
@ -2548,14 +2553,13 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
|
|||
}
|
||||
}
|
||||
|
||||
device->all_pipelines.push_back(pipeline);
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> guard(compile_count_mutex);
|
||||
assert(compile_count > 0);
|
||||
compile_count--;
|
||||
std::lock_guard<std::mutex> guard(device->compile_mutex);
|
||||
device->all_pipelines.push_back(pipeline);
|
||||
pipeline->compiled = true;
|
||||
pipeline->compile_pending = false;
|
||||
}
|
||||
compile_count_cond.notify_all();
|
||||
device->compile_cv.notify_all();
|
||||
}
|
||||
|
||||
static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) {
|
||||
|
|
@ -2571,8 +2575,7 @@ static void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx,
|
|||
VK_LOG_DEBUG("ggml_pipeline_request_descriptor_sets(" << pipeline->name << ", " << n << ")");
|
||||
ctx->pipeline_descriptor_set_requirements += n;
|
||||
if (!pipeline->compiled) {
|
||||
pipeline->needed = true;
|
||||
ggml_vk_load_shaders(ctx->device);
|
||||
ggml_vk_load_shaders(ctx->device, pipeline);
|
||||
}
|
||||
ggml_pipeline_allocate_descriptor_sets(ctx);
|
||||
}
|
||||
|
|
@ -3571,10 +3574,26 @@ static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type
|
|||
#endif
|
||||
}
|
||||
|
||||
static void ggml_vk_load_shaders(vk_device& device) {
|
||||
// load_shaders walks the pipeline list under compile_mutex and either claims
|
||||
// the requested pipeline for compilation or, if another thread is already
|
||||
// compiling it, drops the lock and waits on compile_cv. Compiles themselves
|
||||
// run unlocked.
|
||||
struct CompileTask {
|
||||
vk_pipeline pipeline;
|
||||
size_t spv_size;
|
||||
const void * spv_data;
|
||||
std::string entrypoint;
|
||||
uint32_t parameter_count;
|
||||
std::array<uint32_t, 3> wg_denoms;
|
||||
std::vector<uint32_t> specialization_constants;
|
||||
bool disable_robustness;
|
||||
bool require_full_subgroups;
|
||||
uint32_t required_subgroup_size;
|
||||
};
|
||||
|
||||
static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")");
|
||||
|
||||
std::lock_guard<std::recursive_mutex> guard(device->mutex);
|
||||
// some shaders have a minimum subgroup size
|
||||
const uint32_t subgroup_size_8 = std::max(device->subgroup_size, 8u);
|
||||
const uint32_t subgroup_size_16 = std::max(device->subgroup_size, 16u);
|
||||
|
|
@ -3604,6 +3623,15 @@ static void ggml_vk_load_shaders(vk_device& device) {
|
|||
l_mmqid_wg_denoms, m_mmqid_wg_denoms, s_mmqid_wg_denoms;
|
||||
|
||||
uint32_t l_align, m_align, s_align;
|
||||
|
||||
vk_pipeline wait_pipeline;
|
||||
CompileTask claimed_task {};
|
||||
bool has_claimed_task = false;
|
||||
|
||||
// The rest of the walk reads and writes shared device state, so hold the
|
||||
// lock until we're done deciding what to compile.
|
||||
std::unique_lock<std::mutex> compile_lock(device->compile_mutex);
|
||||
|
||||
if (device->coopmat2) {
|
||||
// spec constants and tile sizes for non-quant matmul/matmul_id
|
||||
l_warptile = { 256, 128, 256, 64, 1 };
|
||||
|
|
@ -3789,7 +3817,6 @@ static void ggml_vk_load_shaders(vk_device& device) {
|
|||
device->pipeline_matmul_id_bf16 = std::make_shared<vk_matmul_pipeline_struct>();
|
||||
}
|
||||
|
||||
std::vector<std::future<void>> compiles;
|
||||
auto const &ggml_vk_create_pipeline = [&](vk_device& device, vk_pipeline& base_pipeline, const char *name, size_t spv_size, const void* spv_data, const char *entrypoint,
|
||||
uint32_t parameter_count, uint32_t push_constant_size, std::array<uint32_t, 3> wg_denoms, const std::vector<uint32_t>& specialization_constants,
|
||||
uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) {
|
||||
|
|
@ -3823,23 +3850,33 @@ static void ggml_vk_load_shaders(vk_device& device) {
|
|||
#endif
|
||||
}
|
||||
|
||||
if (!pipeline->needed || pipeline->compiled) {
|
||||
// We only care about the pipeline this call asked for; the rest
|
||||
// (including the 64-bit indexing variant) are handled by their
|
||||
// own request_descriptor_sets / load_shaders calls.
|
||||
if (pipeline.get() != requested.get()) {
|
||||
continue;
|
||||
}
|
||||
// TODO: We're no longer benefitting from the async compiles (shaders are
|
||||
// compiled individually, as needed) and this complexity can be removed.
|
||||
{
|
||||
// wait until fewer than N compiles are in progress
|
||||
uint32_t N = std::max(1u, std::thread::hardware_concurrency());
|
||||
std::unique_lock<std::mutex> guard(compile_count_mutex);
|
||||
while (compile_count >= N) {
|
||||
compile_count_cond.wait(guard);
|
||||
}
|
||||
compile_count++;
|
||||
|
||||
if (pipeline->compiled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
compiles.push_back(std::async(ggml_vk_create_pipeline_func, std::ref(device), std::ref(pipeline), spv_size, spv_data, entrypoint,
|
||||
parameter_count, wg_denoms, specialization_constants, disable_robustness, require_full_subgroups, required_subgroup_size));
|
||||
wait_pipeline = pipeline;
|
||||
|
||||
if (!pipeline->compile_pending) {
|
||||
pipeline->compile_pending = true;
|
||||
claimed_task.pipeline = pipeline;
|
||||
claimed_task.spv_size = spv_size;
|
||||
claimed_task.spv_data = spv_data;
|
||||
claimed_task.entrypoint = entrypoint;
|
||||
claimed_task.parameter_count = parameter_count;
|
||||
claimed_task.wg_denoms = wg_denoms;
|
||||
claimed_task.specialization_constants = specialization_constants;
|
||||
claimed_task.disable_robustness = disable_robustness;
|
||||
claimed_task.require_full_subgroups = require_full_subgroups;
|
||||
claimed_task.required_subgroup_size = required_subgroup_size;
|
||||
has_claimed_task = true;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -5336,8 +5373,25 @@ static void ggml_vk_load_shaders(vk_device& device) {
|
|||
}
|
||||
}
|
||||
|
||||
for (auto &c : compiles) {
|
||||
c.wait();
|
||||
// Drop compile_mutex so other threads can walk while we compile.
|
||||
compile_lock.unlock();
|
||||
|
||||
// Compile what we claimed; create_pipeline_func reacquires compile_mutex
|
||||
// at the end to flip compile_pending/compiled and notify waiters.
|
||||
if (has_claimed_task) {
|
||||
auto & task = claimed_task;
|
||||
ggml_vk_create_pipeline_func(device, task.pipeline, task.spv_size, task.spv_data,
|
||||
task.entrypoint, task.parameter_count, task.wg_denoms,
|
||||
task.specialization_constants, task.disable_robustness,
|
||||
task.require_full_subgroups, task.required_subgroup_size);
|
||||
}
|
||||
|
||||
// Another thread may be compiling the pipeline we need; block on it here.
|
||||
if (wait_pipeline) {
|
||||
std::unique_lock<std::mutex> wait_lock(device->compile_mutex);
|
||||
device->compile_cv.wait(wait_lock, [&] {
|
||||
return wait_pipeline->compiled.load();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -7030,7 +7084,7 @@ static void * ggml_vk_host_malloc(vk_device& device, size_t size) {
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
std::lock_guard<std::recursive_mutex> guard(device->mutex);
|
||||
std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);
|
||||
device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf));
|
||||
|
||||
return buf->ptr;
|
||||
|
|
@ -7041,7 +7095,7 @@ static void ggml_vk_host_free(vk_device& device, void* ptr) {
|
|||
return;
|
||||
}
|
||||
VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")");
|
||||
std::lock_guard<std::recursive_mutex> guard(device->mutex);
|
||||
std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);
|
||||
|
||||
vk_buffer buf;
|
||||
size_t index;
|
||||
|
|
@ -7065,7 +7119,7 @@ static void ggml_vk_host_free(vk_device& device, void* ptr) {
|
|||
}
|
||||
|
||||
static void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) {
|
||||
std::lock_guard<std::recursive_mutex> guard(device->mutex);
|
||||
std::shared_lock<std::shared_mutex> guard(device->pinned_memory_mutex);
|
||||
buf = nullptr;
|
||||
buf_offset = 0;
|
||||
for (size_t i = 0; i < device->pinned_memory.size(); i++) {
|
||||
|
|
@ -9748,7 +9802,7 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx
|
|||
vk_pipeline pipeline = nullptr;
|
||||
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> guard(ctx->device->mutex);
|
||||
std::lock_guard<std::mutex> guard(ctx->device->compile_mutex);
|
||||
auto &pipelines = ctx->device->pipeline_flash_attn_f32_f16;
|
||||
auto it = pipelines.find(fa_pipeline_state);
|
||||
if (it != pipelines.end()) {
|
||||
|
|
@ -9812,13 +9866,15 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx
|
|||
|
||||
vk_pipeline pipeline_fa_mask_opt = nullptr;
|
||||
if (use_mask_opt) {
|
||||
std::lock_guard<std::recursive_mutex> guard(ctx->device->mutex);
|
||||
auto &pipelines = ctx->device->pipeline_fa_mask_opt;
|
||||
auto it = pipelines.find({Br, Bc});
|
||||
if (it != pipelines.end()) {
|
||||
pipeline_fa_mask_opt = it->second;
|
||||
} else {
|
||||
pipelines[{Br, Bc}] = pipeline_fa_mask_opt = std::make_shared<vk_pipeline_struct>();
|
||||
{
|
||||
std::lock_guard<std::mutex> guard(ctx->device->compile_mutex);
|
||||
auto &pipelines = ctx->device->pipeline_fa_mask_opt;
|
||||
auto it = pipelines.find({Br, Bc});
|
||||
if (it != pipelines.end()) {
|
||||
pipeline_fa_mask_opt = it->second;
|
||||
} else {
|
||||
pipelines[{Br, Bc}] = pipeline_fa_mask_opt = std::make_shared<vk_pipeline_struct>();
|
||||
}
|
||||
}
|
||||
assert(pipeline_fa_mask_opt);
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline_fa_mask_opt, 1);
|
||||
|
|
@ -10352,7 +10408,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
|||
vk_pipeline pipeline = nullptr;
|
||||
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> guard(ctx->device->mutex);
|
||||
std::lock_guard<std::mutex> guard(ctx->device->compile_mutex);
|
||||
auto it = ctx->device->pipeline_solve_tri_f32.find(solve_tri_pipeline_state);
|
||||
if (it != ctx->device->pipeline_solve_tri_f32.end()) {
|
||||
pipeline = it->second;
|
||||
|
|
@ -10511,7 +10567,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
|||
vk_pipeline pipeline = nullptr;
|
||||
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> guard(ctx->device->mutex);
|
||||
std::lock_guard<std::mutex> guard(ctx->device->compile_mutex);
|
||||
auto it = pipelines->find(conv2d_pipeline_state);
|
||||
if (it != pipelines->end()) {
|
||||
pipeline = it->second;
|
||||
|
|
|
|||
|
|
@ -342,6 +342,7 @@ extern "C" {
|
|||
uint32_t n_ubatch; // physical maximum batch size
|
||||
uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)
|
||||
uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
|
||||
uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch)
|
||||
int32_t n_threads; // number of threads to use for generation
|
||||
int32_t n_threads_batch; // number of threads to use for batch processing
|
||||
|
||||
|
|
@ -978,7 +979,11 @@ extern "C" {
|
|||
|
||||
// Set whether the model is in warmup mode or not
|
||||
// If true, all model tensors are activated during llama_decode() to load and cache their weights.
|
||||
LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup);
|
||||
//
|
||||
// note: using this can cause extra graph reallocations because it changes the graph topology with MoE models,
|
||||
// so it is generally not recommended to use in practice. will be removed in the future
|
||||
DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup),
|
||||
"user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]");
|
||||
|
||||
// Set abort callback
|
||||
LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);
|
||||
|
|
|
|||
|
|
@ -185,6 +185,8 @@ llama_context::llama_context(
|
|||
|
||||
cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);
|
||||
|
||||
cparams.n_outputs_max = params.n_outputs_max == 0 ? cparams.n_batch : params.n_outputs_max;
|
||||
|
||||
cparams.op_offload = params.op_offload;
|
||||
cparams.kv_unified = params.kv_unified;
|
||||
|
||||
|
|
@ -230,6 +232,7 @@ llama_context::llama_context(
|
|||
LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);
|
||||
LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);
|
||||
LLAMA_LOG_INFO("%s: n_rs_seq = %u\n", __func__, cparams.n_rs_seq);
|
||||
LLAMA_LOG_INFO("%s: n_outputs_max = %u\n", __func__, cparams.n_outputs_max);
|
||||
|
||||
if (cparams.n_ctx_seq < hparams.n_ctx_train) {
|
||||
LLAMA_LOG_WARN("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n",
|
||||
|
|
@ -540,7 +543,7 @@ void llama_context::sched_reserve() {
|
|||
// note: n_outputs must match n_tokens for embedding models with mean/rank pooling,
|
||||
// because build_pooling creates inp_mean with shape [n_tokens, n_seqs] and multiplies
|
||||
// it with t_embd which is reduced to [n_outputs, ...] via out_ids. if n_outputs != n_tokens,
|
||||
// the ggml_mul_mat assertion fails. this matches the pp reservation below (line ~553).
|
||||
// the ggml_mul_mat assertion fails.
|
||||
const uint32_t n_tokens_ch = 16*n_seqs;
|
||||
auto * gf = graph_reserve(n_tokens_ch, n_seqs, n_tokens_ch, mctx.get(), true);
|
||||
if (!gf) {
|
||||
|
|
@ -586,16 +589,18 @@ void llama_context::sched_reserve() {
|
|||
int n_splits_tg = -1;
|
||||
int n_nodes_tg = -1;
|
||||
|
||||
const uint32_t n_outputs_pp = std::min(n_tokens, cparams.n_outputs_max);
|
||||
|
||||
// reserve pp (prompt processing) graph first so that buffers are only allocated once
|
||||
{
|
||||
auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get(),
|
||||
auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(),
|
||||
model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);
|
||||
if (!gf) {
|
||||
if (cparams.pipeline_parallel) {
|
||||
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
|
||||
cparams.pipeline_parallel = false;
|
||||
sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload));
|
||||
gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get());
|
||||
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get());
|
||||
}
|
||||
if (!gf) {
|
||||
throw std::runtime_error("failed to allocate compute pp buffers");
|
||||
|
|
@ -623,7 +628,7 @@ void llama_context::sched_reserve() {
|
|||
//
|
||||
// auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get());
|
||||
//
|
||||
auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get(), model.hparams.no_alloc);
|
||||
auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
|
||||
if (!gf) {
|
||||
throw std::runtime_error("failed to allocate compute pp buffers");
|
||||
}
|
||||
|
|
@ -784,7 +789,9 @@ bool llama_context::memory_update(bool optimize) {
|
|||
const uint32_t n_seqs = cparams.n_seq_max;
|
||||
const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
|
||||
|
||||
auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get());
|
||||
const uint32_t n_outputs_max = std::min(n_tokens, cparams.n_outputs_max);
|
||||
|
||||
auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_max, mctx.get());
|
||||
if (!gf) {
|
||||
LLAMA_LOG_ERROR("%s: failed to reserve graph after the memory update\n", __func__);
|
||||
}
|
||||
|
|
@ -2150,6 +2157,8 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) {
|
|||
|
||||
this->n_outputs = 0;
|
||||
|
||||
GGML_ASSERT(n_outputs_max <= cparams.n_outputs_max);
|
||||
|
||||
return n_outputs_max;
|
||||
}
|
||||
|
||||
|
|
@ -2236,8 +2245,6 @@ ggml_cgraph * llama_context::graph_reserve(
|
|||
|
||||
if (n_tokens % n_seqs != 0) {
|
||||
n_tokens = ((n_tokens + (n_seqs - 1)) / n_seqs) * n_seqs; // round to next multiple of n_seqs
|
||||
n_outputs = std::max(n_outputs, n_tokens);
|
||||
|
||||
//LLAMA_LOG_DEBUG("%s: making n_tokens a multiple of n_seqs - n_tokens = %u, n_seqs = %u, n_outputs = %u\n", __func__, n_tokens, n_seqs, n_outputs);
|
||||
}
|
||||
|
||||
|
|
@ -3347,6 +3354,7 @@ llama_context_params llama_context_default_params() {
|
|||
/*.n_ubatch =*/ 512,
|
||||
/*.n_seq_max =*/ 1,
|
||||
/*.n_rs_seq =*/ 0,
|
||||
/*.n_outputs_max =*/ 0,
|
||||
/*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default
|
||||
/*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS,
|
||||
/*.ctx_type =*/ LLAMA_CONTEXT_TYPE_DEFAULT,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ struct llama_cparams {
|
|||
uint32_t n_ubatch;
|
||||
uint32_t n_seq_max;
|
||||
uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback
|
||||
uint32_t n_outputs_max; // max outputs supported by the context
|
||||
int32_t n_threads; // number of threads to use for generation
|
||||
int32_t n_threads_batch; // number of threads to use for batch processing
|
||||
|
||||
|
|
@ -38,7 +39,7 @@ struct llama_cparams {
|
|||
bool fused_gdn_ch; // use fused gated delta net (chunked)
|
||||
bool auto_fgdn;
|
||||
bool no_perf;
|
||||
bool warmup;
|
||||
bool warmup; // TODO: remove [TAG_LLAMA_GRAPH_NO_WARMUP]
|
||||
bool op_offload;
|
||||
bool kv_unified;
|
||||
bool pipeline_parallel;
|
||||
|
|
|
|||
|
|
@ -1885,7 +1885,19 @@ void llama_kv_cache::state_write(llama_io_write_i & io, llama_seq_id seq_id, lla
|
|||
uint32_t cell_range_begin = cells.size();
|
||||
|
||||
for (uint32_t i = 0; i < cells.size(); ++i) {
|
||||
if (!cells.is_empty(i) && (seq_id == -1 || cells.seq_has(i, seq_id))) {
|
||||
bool add_cell = true;
|
||||
|
||||
add_cell = add_cell && !cells.is_empty(i);
|
||||
add_cell = add_cell && (seq_id == -1 || cells.seq_has(i, seq_id));
|
||||
|
||||
// check the cell is not SWA-masked
|
||||
if (add_cell && seq_id != -1) {
|
||||
const bool is_masked = llama_hparams::is_masked_swa(n_swa, swa_type, cells.pos_get(i), cells.seq_pos_max(seq_id));
|
||||
|
||||
add_cell = !is_masked;
|
||||
}
|
||||
|
||||
if (add_cell) {
|
||||
++cell_count;
|
||||
if (cell_range_begin == cells.size()) {
|
||||
cell_range_begin = i;
|
||||
|
|
@ -2138,7 +2150,7 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32
|
|||
|
||||
sinfo = find_slot(ubatch, false);
|
||||
if (sinfo.empty()) {
|
||||
LLAMA_LOG_ERROR("%s: failed to find available cells in kv cache\n", __func__);
|
||||
LLAMA_LOG_ERROR("%s: failed to find %d available cells in kv cache\n", __func__, cell_count);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1132,6 +1132,7 @@ json oaicompat_chat_params_parse(
|
|||
llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag;
|
||||
llama_params["reasoning_budget_end_tag"] = chat_params.thinking_end_tag;
|
||||
llama_params["reasoning_budget_message"] = opt.reasoning_budget_message;
|
||||
llama_params["reasoning_control"] = json_value(body, "reasoning_control", false);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,21 @@ using json = nlohmann::ordered_json;
|
|||
|
||||
constexpr int HTTP_POLLING_SECONDS = 1;
|
||||
|
||||
static uint32_t server_n_outputs_max(const common_params & params) {
|
||||
const uint32_t n_batch = params.n_batch;
|
||||
|
||||
if (params.embedding ||
|
||||
(params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) {
|
||||
return n_batch;
|
||||
}
|
||||
|
||||
const uint32_t n_outputs_per_seq = 1 + common_speculative_n_max(¶ms.speculative);
|
||||
|
||||
const uint64_t n_outputs = (uint64_t) params.n_parallel * n_outputs_per_seq;
|
||||
|
||||
return std::max<uint32_t>(1, std::min<uint64_t>(n_batch, n_outputs));
|
||||
}
|
||||
|
||||
// state diagram: https://github.com/ggml-org/llama.cpp/pull/9283
|
||||
enum slot_state {
|
||||
SLOT_STATE_IDLE,
|
||||
|
|
@ -753,6 +768,7 @@ private:
|
|||
SRV_INF("loading model '%s'\n", params.model.path.c_str());
|
||||
|
||||
params_base = params;
|
||||
params_base.n_outputs_max = server_n_outputs_max(params_base);
|
||||
|
||||
std::string & mmproj_path = params_base.mmproj.path;
|
||||
bool has_mmproj = !mmproj_path.empty();
|
||||
|
|
@ -818,6 +834,8 @@ private:
|
|||
measure_model_bytes = false;
|
||||
}
|
||||
|
||||
params_dft.n_outputs_max = params_base.n_parallel;
|
||||
|
||||
auto mparams_dft = common_model_params_to_llama(params_dft);
|
||||
auto cparams_dft = common_context_params_to_llama(params_dft);
|
||||
if (spec_mtp) {
|
||||
|
|
@ -941,10 +959,11 @@ private:
|
|||
params_base.model.path.c_str());
|
||||
|
||||
auto cparams_mtp = common_context_params_to_llama(params_base);
|
||||
cparams_mtp.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
|
||||
cparams_mtp.type_k = params_base.speculative.draft.cache_type_k;
|
||||
cparams_mtp.type_v = params_base.speculative.draft.cache_type_v;
|
||||
cparams_mtp.n_rs_seq = 0;
|
||||
cparams_mtp.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
|
||||
cparams_mtp.type_k = params_base.speculative.draft.cache_type_k;
|
||||
cparams_mtp.type_v = params_base.speculative.draft.cache_type_v;
|
||||
cparams_mtp.n_rs_seq = 0;
|
||||
cparams_mtp.n_outputs_max = params_base.n_parallel;
|
||||
|
||||
ctx_dft.reset(llama_init_from_model(model_tgt, cparams_mtp));
|
||||
if (ctx_dft == nullptr) {
|
||||
|
|
@ -1244,6 +1263,20 @@ private:
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
server_slot * get_slot_by_cmpl_id(const std::string & cmpl_id) {
|
||||
if (cmpl_id.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
for (server_slot & slot : slots) {
|
||||
if (slot.is_processing() && slot.task && slot.task->params.oaicompat_cmpl_id == cmpl_id) {
|
||||
return &slot;
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
server_slot * get_available_slot(const server_task & task) {
|
||||
server_slot * ret = nullptr;
|
||||
|
||||
|
|
@ -2095,6 +2128,37 @@ private:
|
|||
}
|
||||
}
|
||||
} break;
|
||||
case SERVER_TASK_TYPE_CONTROL:
|
||||
{
|
||||
auto res = std::make_unique<server_task_result_control>();
|
||||
res->id = task.id;
|
||||
|
||||
server_slot * slot = get_slot_by_cmpl_id(task.params.control_cmpl_id);
|
||||
if (slot == nullptr) {
|
||||
res->success = false;
|
||||
res->message = "no active completion for this id";
|
||||
queue_results.send(std::move(res));
|
||||
break;
|
||||
}
|
||||
|
||||
if (task.params.control_action == "reasoning_end") {
|
||||
// the budget sampler only exists when reasoning control was armed
|
||||
if (!slot->task->params.sampling.reasoning_control) {
|
||||
res->success = false;
|
||||
res->message = "reasoning control not enabled for this completion";
|
||||
queue_results.send(std::move(res));
|
||||
break;
|
||||
}
|
||||
// act on the live slot mid generation, never defer
|
||||
common_sampler_reasoning_budget_force(slot->smpl.get());
|
||||
res->success = true;
|
||||
} else {
|
||||
res->success = false;
|
||||
res->message = "unknown control action";
|
||||
}
|
||||
|
||||
queue_results.send(std::move(res));
|
||||
} break;
|
||||
case SERVER_TASK_TYPE_NEXT_RESPONSE:
|
||||
{
|
||||
// do nothing
|
||||
|
|
@ -4247,6 +4311,43 @@ void server_routes::init_routes() {
|
|||
TASK_RESPONSE_TYPE_OAI_CHAT);
|
||||
};
|
||||
|
||||
this->post_control = [this](const server_http_req & req) {
|
||||
auto res = create_response();
|
||||
const json body = json::parse(req.body);
|
||||
|
||||
const std::string cmpl_id = json_value(body, "id", std::string());
|
||||
const std::string action = json_value(body, "action", std::string());
|
||||
if (cmpl_id.empty()) {
|
||||
res->error(format_error_response("missing completion id", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
if (action != "reasoning_end") {
|
||||
res->error(format_error_response("unknown control action", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
|
||||
auto & rd = res->rd;
|
||||
{
|
||||
server_task task(SERVER_TASK_TYPE_CONTROL);
|
||||
task.id = rd.get_new_id();
|
||||
task.params.control_cmpl_id = cmpl_id;
|
||||
task.params.control_action = action;
|
||||
rd.post_task(std::move(task));
|
||||
}
|
||||
|
||||
auto result = rd.next(req.should_stop);
|
||||
if (!result) {
|
||||
GGML_ASSERT(req.should_stop());
|
||||
return res;
|
||||
}
|
||||
if (result->is_error()) {
|
||||
res->error(result->to_json());
|
||||
return res;
|
||||
}
|
||||
res->ok(result->to_json());
|
||||
return res;
|
||||
};
|
||||
|
||||
this->post_responses_oai = [this](const server_http_req & req) {
|
||||
auto res = create_response();
|
||||
std::vector<raw_buffer> files;
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ struct server_routes {
|
|||
server_http_context::handler_t post_completions;
|
||||
server_http_context::handler_t post_completions_oai;
|
||||
server_http_context::handler_t post_chat_completions;
|
||||
server_http_context::handler_t post_control;
|
||||
server_http_context::handler_t post_responses_oai;
|
||||
server_http_context::handler_t post_transcriptions_oai;
|
||||
server_http_context::handler_t post_anthropic_messages;
|
||||
|
|
|
|||
|
|
@ -499,6 +499,7 @@ task_params server_task::params_from_json_cmpl(
|
|||
const auto end_tag = json_value(data, "reasoning_budget_end_tag", std::string());
|
||||
const auto message = json_value(data, "reasoning_budget_message", std::string());
|
||||
params.sampling.reasoning_budget_tokens = budget;
|
||||
params.sampling.reasoning_control = json_value(data, "reasoning_control", false);
|
||||
|
||||
if (!start_tag.empty()) {
|
||||
params.sampling.reasoning_budget_start = common_tokenize(vocab, start_tag, false, true);
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ enum server_task_type {
|
|||
SERVER_TASK_TYPE_RERANK,
|
||||
SERVER_TASK_TYPE_INFILL,
|
||||
SERVER_TASK_TYPE_CANCEL,
|
||||
SERVER_TASK_TYPE_CONTROL,
|
||||
SERVER_TASK_TYPE_NEXT_RESPONSE,
|
||||
SERVER_TASK_TYPE_METRICS,
|
||||
SERVER_TASK_TYPE_SLOT_SAVE,
|
||||
|
|
@ -84,6 +85,10 @@ struct task_params {
|
|||
std::string oaicompat_model;
|
||||
std::string oaicompat_cmpl_id;
|
||||
|
||||
// realtime control (SERVER_TASK_TYPE_CONTROL)
|
||||
std::string control_action;
|
||||
std::string control_cmpl_id;
|
||||
|
||||
// per-request parameters for chat parsing
|
||||
common_chat_parser_params chat_parser_params;
|
||||
|
||||
|
|
@ -551,6 +556,19 @@ struct server_task_result_slot_erase : server_task_result {
|
|||
virtual json to_json() override;
|
||||
};
|
||||
|
||||
struct server_task_result_control : server_task_result {
|
||||
bool success = false;
|
||||
std::string message; // optional detail when success is false
|
||||
|
||||
virtual json to_json() override {
|
||||
json out = json { { "success", success } };
|
||||
if (!message.empty()) {
|
||||
out["message"] = message;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
struct server_task_result_get_lora : server_task_result {
|
||||
struct lora {
|
||||
common_adapter_lora_info info;
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ int llama_server(int argc, char ** argv) {
|
|||
routes.post_completions = models_routes->proxy_post;
|
||||
routes.post_completions_oai = models_routes->proxy_post;
|
||||
routes.post_chat_completions = models_routes->proxy_post;
|
||||
routes.post_control = models_routes->proxy_post;
|
||||
routes.post_responses_oai = models_routes->proxy_post;
|
||||
routes.post_transcriptions_oai = models_routes->proxy_post;
|
||||
routes.post_anthropic_messages = models_routes->proxy_post;
|
||||
|
|
@ -185,6 +186,7 @@ int llama_server(int argc, char ** argv) {
|
|||
ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai));
|
||||
ctx_http.post("/chat/completions", ex_wrapper(routes.post_chat_completions));
|
||||
ctx_http.post("/v1/chat/completions", ex_wrapper(routes.post_chat_completions));
|
||||
ctx_http.post("/v1/chat/completions/control", ex_wrapper(routes.post_control));
|
||||
ctx_http.post("/v1/responses", ex_wrapper(routes.post_responses_oai));
|
||||
ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai));
|
||||
ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
VITE_PUBLIC_APP_NAME='llama-ui'
|
||||
# VITE_DEBUG='true'
|
||||
VITE_DEBUG='true'
|
||||
VITE_PUBLIC_SERVER_ORIGIN='http://localhost:8033'
|
||||
|
|
|
|||
|
|
@ -7,6 +7,9 @@ bun.lockb
|
|||
|
||||
# Miscellaneous
|
||||
/static/
|
||||
dist/
|
||||
.svelte-kit/
|
||||
build/
|
||||
|
||||
# Build output
|
||||
/dist/
|
||||
|
|
|
|||
|
|
@ -115,16 +115,18 @@ This starts:
|
|||
- **Vite dev server** at `http://localhost:5173` - The main UI frontend app
|
||||
- **Storybook** at `http://localhost:6006` - Component documentation
|
||||
|
||||
The Vite dev server proxies API requests to `http://localhost:8080` (default llama-server port):
|
||||
The Vite dev server proxies API requests to `SERVER_ORIGIN` (with fallback to default llama-server `8080` port):
|
||||
|
||||
```typescript
|
||||
// vite.config.ts proxy configuration
|
||||
proxy: {
|
||||
'/v1': 'http://localhost:8080',
|
||||
'/props': 'http://localhost:8080',
|
||||
'/slots': 'http://localhost:8080',
|
||||
'/models': 'http://localhost:8080'
|
||||
}
|
||||
'/v1': SERVER_ORIGIN,
|
||||
'/props': SERVER_ORIGIN,
|
||||
'/models': SERVER_ORIGIN,
|
||||
'/tools': SERVER_ORIGIN,
|
||||
'/slots': SERVER_ORIGIN,
|
||||
'/cors-proxy': SERVER_ORIGIN
|
||||
},
|
||||
```
|
||||
|
||||
### Development Workflow
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { defineConfig } from '@playwright/test';
|
|||
|
||||
export default defineConfig({
|
||||
webServer: {
|
||||
command: 'npm run build && http-server ../../build/tools/ui/dist -p 8181',
|
||||
command: 'npm run build && npx http-server ./dist -p 8181',
|
||||
port: 8181,
|
||||
timeout: 120000,
|
||||
reuseExistingServer: false
|
||||
|
|
|
|||
|
|
@ -111,6 +111,7 @@
|
|||
let preSelectedResourceUri = $state<string | undefined>(undefined);
|
||||
|
||||
let currentConfig = $derived(config());
|
||||
|
||||
let pasteLongTextToFileLength = $derived.by(() => {
|
||||
const n = Number(currentConfig.pasteLongTextToFileLen);
|
||||
return Number.isNaN(n) ? Number(SETTING_CONFIG_DEFAULT.pasteLongTextToFileLen) : n;
|
||||
|
|
@ -541,6 +542,7 @@
|
|||
canSend={canSubmit}
|
||||
{disabled}
|
||||
{isLoading}
|
||||
isReasoning={chatStore.isReasoning}
|
||||
{isRecording}
|
||||
{showAddButton}
|
||||
{showModelSelector}
|
||||
|
|
|
|||
|
|
@ -1,22 +1,18 @@
|
|||
<script lang="ts">
|
||||
import { Plus } from '@lucide/svelte';
|
||||
import { Plus, File, MessageSquare, Zap, FolderOpen } from '@lucide/svelte';
|
||||
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { buttonVariants } from '$lib/components/ui/button';
|
||||
import { cn } from '$lib/components/ui/utils';
|
||||
import {
|
||||
ATTACHMENT_FILE_ITEMS,
|
||||
ATTACHMENT_EXTRA_ITEMS,
|
||||
ATTACHMENT_MCP_ITEMS,
|
||||
ATTACHMENT_TOOLTIP_TEXT,
|
||||
TOOLTIP_DELAY_DURATION
|
||||
} from '$lib/constants';
|
||||
import { AttachmentMenuItemId } from '$lib/enums';
|
||||
import {
|
||||
ChatFormActionAddToolsSubmenu,
|
||||
ChatFormActionAddMcpServersSubmenu
|
||||
} from '$lib/components/app';
|
||||
|
||||
import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte';
|
||||
|
||||
interface Props {
|
||||
|
|
@ -97,107 +93,87 @@
|
|||
</Tooltip.Root>
|
||||
|
||||
<DropdownMenu.Content align="start" class="w-48">
|
||||
{#each ATTACHMENT_FILE_ITEMS as item (item.id)}
|
||||
{@const enabled = attachmentMenu.isItemEnabled(item.enabledWhen)}
|
||||
{#if enabled}
|
||||
<DropdownMenu.Item
|
||||
class="{item.class ?? ''} flex cursor-pointer items-center gap-2"
|
||||
onclick={() => attachmentMenu.callbacks[item.action]()}
|
||||
>
|
||||
<item.icon class="h-4 w-4" />
|
||||
<DropdownMenu.Sub>
|
||||
<DropdownMenu.SubTrigger class="flex cursor-pointer items-center gap-2">
|
||||
<File class="h-4 w-4" />
|
||||
|
||||
<span>{item.label}</span>
|
||||
</DropdownMenu.Item>
|
||||
{:else if item.disabledTooltip}
|
||||
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
|
||||
<Tooltip.Trigger tabindex={-1}>
|
||||
{#snippet child({ props })}
|
||||
<div {...props} class="cursor-default">
|
||||
<DropdownMenu.Item class="{item.class ?? ''} flex items-center gap-2" disabled>
|
||||
<item.icon class="h-4 w-4" />
|
||||
<span>Add files</span>
|
||||
</DropdownMenu.SubTrigger>
|
||||
|
||||
<span>{item.label}</span>
|
||||
</DropdownMenu.Item>
|
||||
</div>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content side="right">
|
||||
<p>{item.disabledTooltip}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
{/each}
|
||||
|
||||
{#if !attachmentMenu.isItemEnabled('hasVisionModality')}
|
||||
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
|
||||
<Tooltip.Trigger>
|
||||
{#snippet child({ props })}
|
||||
<DropdownMenu.SubContent class="w-48">
|
||||
{#each ATTACHMENT_FILE_ITEMS as item (item.id)}
|
||||
{@const enabled = attachmentMenu.isItemEnabled(item.enabledWhen)}
|
||||
{#if enabled}
|
||||
<DropdownMenu.Item
|
||||
{...props}
|
||||
class="flex cursor-pointer items-center gap-2"
|
||||
onclick={attachmentMenu.callbacks.onFileUpload}
|
||||
class="{item.class ?? ''} flex cursor-pointer items-center gap-2"
|
||||
onclick={() => attachmentMenu.callbacks[item.action]()}
|
||||
>
|
||||
{@const pdfItem = ATTACHMENT_FILE_ITEMS.find(
|
||||
(i) => i.id === AttachmentMenuItemId.PDF
|
||||
)}
|
||||
{#if pdfItem}
|
||||
<pdfItem.icon class="h-4 w-4" />
|
||||
<item.icon class="h-4 w-4" />
|
||||
|
||||
<span>{pdfItem.label}</span>
|
||||
{/if}
|
||||
<span>{item.label}</span>
|
||||
</DropdownMenu.Item>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
{:else if item.disabledTooltip}
|
||||
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
|
||||
<Tooltip.Trigger tabindex={-1}>
|
||||
{#snippet child({ props })}
|
||||
<div {...props} class="cursor-default">
|
||||
<DropdownMenu.Item
|
||||
class="{item.class ?? ''} flex items-center gap-2"
|
||||
disabled
|
||||
>
|
||||
<item.icon class="h-4 w-4" />
|
||||
|
||||
<Tooltip.Content side="right">
|
||||
<p>PDFs will be converted to text. Image-based PDFs may not work properly.</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
<span>{item.label}</span>
|
||||
</DropdownMenu.Item>
|
||||
</div>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<DropdownMenu.Separator />
|
||||
<Tooltip.Content side="right">
|
||||
<p>{item.disabledTooltip}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
{/each}
|
||||
</DropdownMenu.SubContent>
|
||||
</DropdownMenu.Sub>
|
||||
|
||||
{#each ATTACHMENT_EXTRA_ITEMS as item (item.id)}
|
||||
{#if item.id === AttachmentMenuItemId.SYSTEM_MESSAGE}
|
||||
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
|
||||
<Tooltip.Trigger>
|
||||
{#snippet child({ props })}
|
||||
<DropdownMenu.Item
|
||||
{...props}
|
||||
class="flex cursor-pointer items-center gap-2"
|
||||
onclick={() => attachmentMenu.callbacks[item.action]()}
|
||||
>
|
||||
<item.icon class="h-4 w-4" />
|
||||
<DropdownMenu.Item
|
||||
class="flex cursor-pointer items-center gap-2"
|
||||
onclick={onSystemPromptClick}
|
||||
>
|
||||
<MessageSquare class="h-4 w-4" />
|
||||
|
||||
<span>{item.label}</span>
|
||||
</DropdownMenu.Item>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content side="right">
|
||||
<p>{attachmentMenu.getSystemMessageTooltip()}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
{/each}
|
||||
<span>System Message</span>
|
||||
</DropdownMenu.Item>
|
||||
|
||||
<ChatFormActionAddToolsSubmenu />
|
||||
|
||||
<ChatFormActionAddMcpServersSubmenu onMcpSettingsClick={handleMcpSettingsClick} />
|
||||
|
||||
{#each ATTACHMENT_MCP_ITEMS as item (item.id)}
|
||||
{#if attachmentMenu.isItemVisible(item.visibleWhen)}
|
||||
<DropdownMenu.Item
|
||||
class="flex cursor-pointer items-center gap-2"
|
||||
onclick={() => attachmentMenu.callbacks[item.action]()}
|
||||
>
|
||||
<item.icon class="h-4 w-4" />
|
||||
{#if hasMcpPromptsSupport}
|
||||
<DropdownMenu.Separator />
|
||||
|
||||
<span>{item.label}</span>
|
||||
</DropdownMenu.Item>
|
||||
{/if}
|
||||
{/each}
|
||||
<DropdownMenu.Item
|
||||
class="flex cursor-pointer items-center gap-2"
|
||||
onclick={onMcpPromptClick}
|
||||
>
|
||||
<Zap class="h-4 w-4" />
|
||||
|
||||
<span>MCP Prompt</span>
|
||||
</DropdownMenu.Item>
|
||||
{/if}
|
||||
|
||||
{#if hasMcpResourcesSupport}
|
||||
<DropdownMenu.Item
|
||||
class="flex cursor-pointer items-center gap-2"
|
||||
onclick={onMcpResourcesClick}
|
||||
>
|
||||
<FolderOpen class="h-4 w-4" />
|
||||
|
||||
<span>MCP Resources</span>
|
||||
</DropdownMenu.Item>
|
||||
{/if}
|
||||
</DropdownMenu.Content>
|
||||
</DropdownMenu.Root>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -2,18 +2,19 @@
|
|||
import type { Snippet } from 'svelte';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import * as Sheet from '$lib/components/ui/sheet';
|
||||
import * as Collapsible from '$lib/components/ui/collapsible';
|
||||
import { File, MessageSquare, Zap, FolderOpen } from '@lucide/svelte';
|
||||
import { Switch } from '$lib/components/ui/switch';
|
||||
import { Checkbox } from '$lib/components/ui/checkbox';
|
||||
import { TOOLTIP_DELAY_DURATION } from '$lib/constants';
|
||||
import {
|
||||
ATTACHMENT_FILE_ITEMS,
|
||||
ATTACHMENT_EXTRA_ITEMS,
|
||||
ATTACHMENT_MCP_ITEMS
|
||||
} from '$lib/constants/attachment-menu';
|
||||
import { McpLogo } from '$lib/components/app';
|
||||
import { ATTACHMENT_FILE_ITEMS } from '$lib/constants/attachment-menu';
|
||||
import { useAttachmentMenu } from '$lib/hooks/use-attachment-menu.svelte';
|
||||
import { AttachmentMenuItemId } from '$lib/enums';
|
||||
import { PencilRuler } from '@lucide/svelte';
|
||||
import { ROUTES, SETTINGS_SECTION_SLUGS } from '$lib/constants/routes';
|
||||
import { RouterService } from '$lib/services/router.service';
|
||||
import { useToolsPanel } from '$lib/hooks/use-tools-panel.svelte';
|
||||
import { conversationsStore } from '$lib/stores/conversations.svelte';
|
||||
import { mcpStore } from '$lib/stores/mcp.svelte';
|
||||
import { McpLogo } from '$lib/components/app';
|
||||
import { PencilRuler, ChevronDown, ChevronRight } from '@lucide/svelte';
|
||||
import { HealthCheckStatus } from '$lib/enums';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
|
|
@ -46,6 +47,9 @@
|
|||
}: Props = $props();
|
||||
|
||||
let sheetOpen = $state(false);
|
||||
let filesExpanded = $state(true);
|
||||
let toolsExpanded = $state(false);
|
||||
let mcpExpanded = $state(false);
|
||||
|
||||
const attachmentMenu = useAttachmentMenu(
|
||||
() => ({
|
||||
|
|
@ -61,14 +65,22 @@
|
|||
}
|
||||
);
|
||||
|
||||
const toolsPanel = useToolsPanel();
|
||||
|
||||
const sheetItemClass =
|
||||
'flex w-full items-center gap-3 rounded-md px-3 py-2.5 text-left text-sm transition-colors hover:bg-accent active:bg-accent disabled:cursor-not-allowed disabled:opacity-50';
|
||||
|
||||
const sheetItemRowClass =
|
||||
'flex w-full items-center justify-between gap-2 rounded-md px-3 py-2 text-left text-sm transition-colors hover:bg-accent';
|
||||
|
||||
function getEnabledMcpServers() {
|
||||
return mcpStore.getServersSorted().filter((s) => s.enabled);
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="flex items-center gap-1 {className}">
|
||||
<Sheet.Root bind:open={sheetOpen}>
|
||||
{@render trigger({ disabled, onclick: () => (sheetOpen = true) })}
|
||||
<!-- <ChatFormActionAddButton {disabled} onclick={() => (sheetOpen = true)} /> -->
|
||||
|
||||
<Sheet.Content side="bottom" class="max-h-[85vh] gap-0 overflow-y-auto">
|
||||
<Sheet.Header>
|
||||
|
|
@ -80,110 +92,206 @@
|
|||
</Sheet.Header>
|
||||
|
||||
<div class="flex flex-col gap-1 px-1.5 pb-2">
|
||||
{#each ATTACHMENT_FILE_ITEMS as item (item.id)}
|
||||
{@const enabled = attachmentMenu.isItemEnabled(item.enabledWhen)}
|
||||
{#if enabled}
|
||||
<button
|
||||
type="button"
|
||||
class={sheetItemClass}
|
||||
onclick={() => attachmentMenu.callbacks[item.action]()}
|
||||
>
|
||||
<item.icon class="h-4 w-4 shrink-0" />
|
||||
<Collapsible.Root open={filesExpanded} onOpenChange={(open) => (filesExpanded = open)}>
|
||||
<Collapsible.Trigger class={sheetItemClass}>
|
||||
{#if filesExpanded}
|
||||
<ChevronDown class="h-4 w-4 shrink-0" />
|
||||
{:else}
|
||||
<ChevronRight class="h-4 w-4 shrink-0" />
|
||||
{/if}
|
||||
|
||||
<span>{item.label}</span>
|
||||
</button>
|
||||
{:else if item.disabledTooltip}
|
||||
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
|
||||
<Tooltip.Trigger>
|
||||
<button type="button" class={sheetItemClass} disabled>
|
||||
<item.icon class="h-4 w-4 shrink-0" />
|
||||
<File class="h-4 w-4 shrink-0" />
|
||||
|
||||
<span>{item.label}</span>
|
||||
</button>
|
||||
</Tooltip.Trigger>
|
||||
<span class="flex-1">Add files</span>
|
||||
</Collapsible.Trigger>
|
||||
|
||||
<Tooltip.Content side="right">
|
||||
<p>{item.disabledTooltip}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
{/each}
|
||||
<Collapsible.Content>
|
||||
<div class="flex flex-col gap-0.5 pl-4">
|
||||
{#each ATTACHMENT_FILE_ITEMS as item (item.id)}
|
||||
{@const enabled = attachmentMenu.isItemEnabled(item.enabledWhen)}
|
||||
{#if enabled}
|
||||
<button
|
||||
type="button"
|
||||
class={sheetItemClass}
|
||||
onclick={() => attachmentMenu.callbacks[item.action]()}
|
||||
>
|
||||
<item.icon class="h-4 w-4 shrink-0" />
|
||||
|
||||
<span>{item.label}</span>
|
||||
</button>
|
||||
{:else if item.disabledTooltip}
|
||||
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
|
||||
<Tooltip.Trigger>
|
||||
<button type="button" class={sheetItemClass} disabled>
|
||||
<item.icon class="h-4 w-4 shrink-0" />
|
||||
|
||||
<span>{item.label}</span>
|
||||
</button>
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content side="right">
|
||||
<p>{item.disabledTooltip}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
{/each}
|
||||
</div>
|
||||
</Collapsible.Content>
|
||||
</Collapsible.Root>
|
||||
|
||||
<Collapsible.Root open={mcpExpanded} onOpenChange={(open) => (mcpExpanded = open)}>
|
||||
<Collapsible.Trigger class={sheetItemClass}>
|
||||
{#if mcpExpanded}
|
||||
<ChevronDown class="h-4 w-4 shrink-0" />
|
||||
{:else}
|
||||
<ChevronRight class="h-4 w-4 shrink-0" />
|
||||
{/if}
|
||||
|
||||
<McpLogo class="inline h-4 w-4 shrink-0" />
|
||||
|
||||
<span class="flex-1">MCP Servers</span>
|
||||
|
||||
<span class="text-xs text-muted-foreground">
|
||||
{getEnabledMcpServers().length} server{getEnabledMcpServers().length !== 1 ? 's' : ''}
|
||||
</span>
|
||||
</Collapsible.Trigger>
|
||||
|
||||
<Collapsible.Content>
|
||||
<div class="flex flex-col gap-0.5 pl-4">
|
||||
{#each getEnabledMcpServers() as server (server.id)}
|
||||
{@const healthState = mcpStore.getHealthCheckState(server.id)}
|
||||
{@const hasError = healthState.status === HealthCheckStatus.ERROR}
|
||||
{@const displayName = mcpStore.getServerLabel(server)}
|
||||
{@const faviconUrl = mcpStore.getServerFavicon(server.id)}
|
||||
{@const isEnabled = conversationsStore.isMcpServerEnabledForChat(server.id)}
|
||||
|
||||
{#if !attachmentMenu.isItemEnabled('hasVisionModality')}
|
||||
{@const pdfItem = ATTACHMENT_FILE_ITEMS.find((i) => i.id === AttachmentMenuItemId.PDF)}
|
||||
{#if pdfItem}
|
||||
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
|
||||
<Tooltip.Trigger>
|
||||
<button
|
||||
type="button"
|
||||
class={sheetItemClass}
|
||||
onclick={() => attachmentMenu.callbacks[pdfItem.action]()}
|
||||
class={sheetItemRowClass}
|
||||
onclick={() => !hasError && conversationsStore.toggleMcpServerForChat(server.id)}
|
||||
disabled={hasError}
|
||||
>
|
||||
<pdfItem.icon class="h-4 w-4 shrink-0" />
|
||||
<div class="flex min-w-0 flex-1 items-center gap-2">
|
||||
{#if faviconUrl}
|
||||
<img
|
||||
src={faviconUrl}
|
||||
alt=""
|
||||
class="h-4 w-4 shrink-0 rounded-sm"
|
||||
onerror={(e) => {
|
||||
(e.currentTarget as HTMLImageElement).style.display = 'none';
|
||||
}}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<span>{pdfItem.label}</span>
|
||||
<span class="min-w-0 truncate text-sm">{displayName}</span>
|
||||
</div>
|
||||
|
||||
{#if hasError}
|
||||
<span
|
||||
class="shrink-0 rounded bg-destructive/15 px-1.5 py-0.5 text-xs text-destructive"
|
||||
>
|
||||
Error
|
||||
</span>
|
||||
{:else}
|
||||
<Switch
|
||||
checked={isEnabled}
|
||||
onCheckedChange={() => conversationsStore.toggleMcpServerForChat(server.id)}
|
||||
/>
|
||||
{/if}
|
||||
</button>
|
||||
</Tooltip.Trigger>
|
||||
{/each}
|
||||
|
||||
<Tooltip.Content side="right">
|
||||
<p>PDFs will be converted to text. Image-based PDFs may not work properly.</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
{#if getEnabledMcpServers().length === 0}
|
||||
<div class="px-3 py-2 text-center text-sm text-muted-foreground">
|
||||
No MCP servers configured
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</Collapsible.Content>
|
||||
</Collapsible.Root>
|
||||
|
||||
{#if toolsPanel.totalToolCount > 0}
|
||||
<Collapsible.Root open={toolsExpanded} onOpenChange={(open) => (toolsExpanded = open)}>
|
||||
<Collapsible.Trigger class={sheetItemClass}>
|
||||
{#if toolsExpanded}
|
||||
<ChevronDown class="h-4 w-4 shrink-0" />
|
||||
{:else}
|
||||
<ChevronRight class="h-4 w-4 shrink-0" />
|
||||
{/if}
|
||||
|
||||
<PencilRuler class="inline h-4 w-4 shrink-0" />
|
||||
|
||||
<span class="flex-1">Tools</span>
|
||||
|
||||
<span class="text-xs text-muted-foreground">
|
||||
{toolsPanel.totalToolCount} tool{toolsPanel.totalToolCount !== 1 ? 's' : ''}
|
||||
</span>
|
||||
</Collapsible.Trigger>
|
||||
|
||||
<Collapsible.Content>
|
||||
<div class="flex flex-col gap-0.5 pl-4">
|
||||
{#each toolsPanel.activeGroups as group (group.label)}
|
||||
{@const { checked, indeterminate } = toolsPanel.getGroupCheckedState(group)}
|
||||
{@const enabledCount = toolsPanel.getEnabledToolCount(group)}
|
||||
{@const favicon = toolsPanel.getFavicon(group)}
|
||||
|
||||
<button
|
||||
type="button"
|
||||
class={sheetItemRowClass}
|
||||
onclick={() => toolsPanel.toggleGroupByLabel(group.label)}
|
||||
>
|
||||
{#if favicon}
|
||||
<img
|
||||
src={favicon}
|
||||
alt=""
|
||||
class="h-4 w-4 shrink-0 rounded-sm"
|
||||
onerror={(e) => {
|
||||
(e.currentTarget as HTMLImageElement).style.display = 'none';
|
||||
}}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<span class="min-w-0 flex-1 truncate text-sm font-medium">{group.label}</span>
|
||||
|
||||
<span class="shrink-0 text-xs text-muted-foreground">
|
||||
{enabledCount}/{group.tools.length}
|
||||
</span>
|
||||
|
||||
<Checkbox
|
||||
{checked}
|
||||
{indeterminate}
|
||||
class="h-4 w-4 shrink-0"
|
||||
onclick={(e) => e.stopPropagation()}
|
||||
onCheckedChange={() => toolsPanel.toggleGroupByLabel(group.label)}
|
||||
/>
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
</Collapsible.Content>
|
||||
</Collapsible.Root>
|
||||
{/if}
|
||||
|
||||
{#each ATTACHMENT_EXTRA_ITEMS as item (item.id)}
|
||||
{#if item.id === AttachmentMenuItemId.SYSTEM_MESSAGE}
|
||||
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
|
||||
<Tooltip.Trigger>
|
||||
<button
|
||||
type="button"
|
||||
class={sheetItemClass}
|
||||
onclick={() => attachmentMenu.callbacks[item.action]()}
|
||||
>
|
||||
<item.icon class="h-4 w-4 shrink-0" />
|
||||
<button type="button" class={sheetItemClass} onclick={onSystemPromptClick}>
|
||||
<MessageSquare class="h-4 w-4 shrink-0" />
|
||||
|
||||
<span>{item.label}</span>
|
||||
</button>
|
||||
</Tooltip.Trigger>
|
||||
<span>System Message</span>
|
||||
</button>
|
||||
|
||||
<Tooltip.Content side="right">
|
||||
<p>{attachmentMenu.getSystemMessageTooltip()}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
{/each}
|
||||
{#if hasMcpPromptsSupport}
|
||||
<button type="button" class={sheetItemClass} onclick={onMcpPromptClick}>
|
||||
<Zap class="h-4 w-4 shrink-0" />
|
||||
|
||||
<div class="my-2 border-t"></div>
|
||||
<span>MCP Prompt</span>
|
||||
</button>
|
||||
{/if}
|
||||
|
||||
<a href={ROUTES.MCP_SERVERS} class="flex items-center gap-3 px-3 py-2">
|
||||
<McpLogo class="inline h-4 w-4" />
|
||||
{#if hasMcpResourcesSupport}
|
||||
<button type="button" class={sheetItemClass} onclick={onMcpResourcesClick}>
|
||||
<FolderOpen class="h-4 w-4 shrink-0" />
|
||||
|
||||
<span class="text-sm">MCP Servers</span>
|
||||
</a>
|
||||
|
||||
<a
|
||||
href={RouterService.settings(SETTINGS_SECTION_SLUGS.TOOLS)}
|
||||
class="flex items-center gap-3 px-3 py-2"
|
||||
>
|
||||
<PencilRuler class="inline h-4 w-4" />
|
||||
|
||||
<span class="text-sm">Tools</span>
|
||||
</a>
|
||||
|
||||
{#each ATTACHMENT_MCP_ITEMS as item (item.id)}
|
||||
{#if attachmentMenu.isItemVisible(item.visibleWhen)}
|
||||
<button
|
||||
type="button"
|
||||
class={sheetItemClass}
|
||||
onclick={() => attachmentMenu.callbacks[item.action]()}
|
||||
>
|
||||
<item.icon class="h-4 w-4 shrink-0" />
|
||||
|
||||
<span>{item.label}</span>
|
||||
</button>
|
||||
{/if}
|
||||
{/each}
|
||||
<span>MCP Resources</span>
|
||||
</button>
|
||||
{/if}
|
||||
</div>
|
||||
</Sheet.Content>
|
||||
</Sheet.Root>
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@
|
|||
<Checkbox
|
||||
{checked}
|
||||
{indeterminate}
|
||||
onCheckedChange={() => toolsStore.toggleGroup(group)}
|
||||
onCheckedChange={() => toolsPanel.toggleGroupByLabel(group.label)}
|
||||
class="mr-2 h-4 w-4 shrink-0"
|
||||
/>
|
||||
</Tooltip.Trigger>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
<script lang="ts">
|
||||
import { chatStore } from '$lib/stores/chat.svelte';
|
||||
import { modelsStore, modelOptions, selectedModelId } from '$lib/stores/models.svelte';
|
||||
import {
|
||||
modelsStore,
|
||||
modelOptions,
|
||||
selectedModelId,
|
||||
selectedModelName
|
||||
} from '$lib/stores/models.svelte';
|
||||
import { isRouterMode, serverError } from '$lib/stores/server.svelte';
|
||||
import { ModelsSelectorDropdown, ModelsSelectorSheet } from '$lib/components/app';
|
||||
import { isMobile } from '$lib/stores/viewport.svelte';
|
||||
|
|
@ -39,7 +44,18 @@
|
|||
|
||||
let lastSyncedConversationModel: string | null = null;
|
||||
|
||||
let selectorModel = $derived(conversationModel ?? modelsStore.selectedModelName ?? null);
|
||||
let selectorModel = $derived.by(() => {
|
||||
const storeModel = selectedModelName();
|
||||
if (storeModel && storeModel !== conversationModel) {
|
||||
return storeModel;
|
||||
}
|
||||
|
||||
if (conversationModel) {
|
||||
return conversationModel;
|
||||
}
|
||||
|
||||
return null;
|
||||
});
|
||||
|
||||
$effect(() => {
|
||||
if (conversationModel && conversationModel !== lastSyncedConversationModel) {
|
||||
|
|
@ -50,7 +66,6 @@
|
|||
modelsStore.selectedModelName = null;
|
||||
modelsStore.clearSelection();
|
||||
}
|
||||
|
||||
lastSyncedConversationModel = conversationModel;
|
||||
} else if (
|
||||
isRouter &&
|
||||
|
|
@ -60,9 +75,7 @@
|
|||
!conversationModel
|
||||
) {
|
||||
lastSyncedConversationModel = null;
|
||||
|
||||
const first = modelOptions().find((m) => modelsStore.loadedModelIds.includes(m.model));
|
||||
|
||||
if (first) modelsStore.selectModelById(first.id);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
<script lang="ts">
|
||||
import { Square } from '@lucide/svelte';
|
||||
import { Square, SkipForward } from '@lucide/svelte';
|
||||
import { Button } from '$lib/components/ui/button';
|
||||
import { ChatService } from '$lib/services';
|
||||
import {
|
||||
ChatFormActionsAdd,
|
||||
ChatFormActionModels,
|
||||
ChatFormActionRecord,
|
||||
ChatFormActionSubmit
|
||||
ChatFormActionSubmit,
|
||||
ChatFormReasoningToggle
|
||||
} from '$lib/components/app';
|
||||
import { FileTypeCategory } from '$lib/enums';
|
||||
import { mcpStore } from '$lib/stores/mcp.svelte';
|
||||
|
|
@ -21,6 +23,7 @@
|
|||
class?: string;
|
||||
disabled?: boolean;
|
||||
isLoading?: boolean;
|
||||
isReasoning?: boolean;
|
||||
isRecording?: boolean;
|
||||
showAddButton?: boolean;
|
||||
showModelSelector?: boolean;
|
||||
|
|
@ -39,6 +42,7 @@
|
|||
class: className = '',
|
||||
disabled = false,
|
||||
isLoading = false,
|
||||
isReasoning = false,
|
||||
isRecording = false,
|
||||
showAddButton = true,
|
||||
showModelSelector = true,
|
||||
|
|
@ -84,6 +88,11 @@
|
|||
export function openModelSelector() {
|
||||
selectorModelRef?.open();
|
||||
}
|
||||
// the streaming assistant message carries both the completion id and the model that
|
||||
// produced it, targeting reasoning control from the same source keeps them consistent
|
||||
let activeMessage = $derived(
|
||||
conversationsStore.activeMessages[conversationsStore.activeMessages.length - 1]
|
||||
);
|
||||
</script>
|
||||
|
||||
<div
|
||||
|
|
@ -91,7 +100,7 @@
|
|||
style="container-type: inline-size"
|
||||
>
|
||||
{#if showAddButton}
|
||||
<div class="mr-auto flex items-center gap-2">
|
||||
<div class="mr-auto flex items-center gap-3">
|
||||
<ChatFormActionsAdd
|
||||
{disabled}
|
||||
{hasAudioModality}
|
||||
|
|
@ -108,19 +117,38 @@
|
|||
</div>
|
||||
{/if}
|
||||
|
||||
{#if showModelSelector}
|
||||
<ChatFormActionModels
|
||||
{disabled}
|
||||
bind:this={selectorModelRef}
|
||||
bind:hasAudioModality
|
||||
bind:hasVideoModality
|
||||
bind:hasVisionModality
|
||||
bind:hasModelSelected
|
||||
bind:isSelectedModelInCache
|
||||
bind:submitTooltip
|
||||
forceForegroundText
|
||||
useGlobalSelection
|
||||
/>
|
||||
<div class="flex items-center gap-2">
|
||||
<ChatFormReasoningToggle />
|
||||
|
||||
{#if showModelSelector}
|
||||
<ChatFormActionModels
|
||||
{disabled}
|
||||
bind:this={selectorModelRef}
|
||||
bind:hasAudioModality
|
||||
bind:hasVideoModality
|
||||
bind:hasVisionModality
|
||||
bind:hasModelSelected
|
||||
bind:isSelectedModelInCache
|
||||
bind:submitTooltip
|
||||
forceForegroundText
|
||||
useGlobalSelection
|
||||
/>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if isReasoning}
|
||||
<Button
|
||||
type="button"
|
||||
variant="secondary"
|
||||
onclick={() =>
|
||||
ChatService.stopReasoning(activeMessage?.completionId ?? '', activeMessage?.model)}
|
||||
class="group h-8 w-8 rounded-full p-0"
|
||||
title="Skip reasoning"
|
||||
>
|
||||
<span class="sr-only">Skip reasoning</span>
|
||||
|
||||
<SkipForward class="h-4 w-4 stroke-muted-foreground group-hover:stroke-foreground" />
|
||||
</Button>
|
||||
{/if}
|
||||
|
||||
{#if isLoading && !canSubmit}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,132 @@
|
|||
<script lang="ts">
|
||||
import { Check, Info, Lightbulb, LightbulbOff } from '@lucide/svelte';
|
||||
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { ReasoningEffort, MessageRole } from '$lib/enums';
|
||||
import { REASONING_EFFORT_TOKENS } from '$lib/constants/reasoning-effort-tokens';
|
||||
import { REASONING_EFFORT_LEVELS } from '$lib/constants/reasoning-effort';
|
||||
import type { ReasoningEffortLevel } from '$lib/types';
|
||||
import {
|
||||
modelsStore,
|
||||
checkModelSupportsThinking,
|
||||
supportsThinking,
|
||||
propsCacheVersion,
|
||||
loadedModelIds
|
||||
} from '$lib/stores/models.svelte';
|
||||
import { chatStore } from '$lib/stores/chat.svelte';
|
||||
import { conversationsStore, activeMessages } from '$lib/stores/conversations.svelte';
|
||||
import { isRouterMode } from '$lib/stores/server.svelte';
|
||||
import type { DatabaseMessage } from '$lib/types/database';
|
||||
|
||||
let thinkingEnabled = $derived(conversationsStore.getThinkingEnabled());
|
||||
let currentEffort = $derived(conversationsStore.getReasoningEffort());
|
||||
let isOff = $derived(!thinkingEnabled);
|
||||
let subOpen = $state(false);
|
||||
|
||||
// Get conversation model from message history
|
||||
let conversationModel = $derived(
|
||||
chatStore.getConversationModel(activeMessages() as DatabaseMessage[])
|
||||
);
|
||||
|
||||
let modelSupportsThinkingFromMessages = $derived.by(() => {
|
||||
const modelId = isRouterMode() ? modelsStore.selectedModelName || conversationModel : null;
|
||||
if (!modelId) return false;
|
||||
|
||||
const messages = conversationsStore.activeMessages;
|
||||
|
||||
return messages.some(
|
||||
(m: DatabaseMessage) =>
|
||||
m.role === MessageRole.ASSISTANT && m.model === modelId && !!m.reasoningContent
|
||||
);
|
||||
});
|
||||
|
||||
let modelSupportsThinking = $derived.by(() => {
|
||||
loadedModelIds();
|
||||
propsCacheVersion();
|
||||
|
||||
if (isRouterMode()) {
|
||||
const modelId = modelsStore.selectedModelName || conversationModel;
|
||||
return checkModelSupportsThinking(modelId ?? '') || modelSupportsThinkingFromMessages;
|
||||
}
|
||||
|
||||
return supportsThinking() || modelSupportsThinkingFromMessages;
|
||||
});
|
||||
|
||||
function isSelected(item: ReasoningEffortLevel): boolean {
|
||||
if (item.isOff) return isOff;
|
||||
|
||||
return thinkingEnabled && currentEffort === item.value;
|
||||
}
|
||||
|
||||
function handleSelection(item: ReasoningEffortLevel) {
|
||||
if (item.isOff) {
|
||||
conversationsStore.setThinkingEnabled(false);
|
||||
} else {
|
||||
conversationsStore.setThinkingEnabled(true);
|
||||
conversationsStore.setReasoningEffort(item.value as ReasoningEffort);
|
||||
}
|
||||
subOpen = false;
|
||||
}
|
||||
</script>
|
||||
|
||||
{#if modelSupportsThinking}
|
||||
<DropdownMenu.Sub bind:open={subOpen}>
|
||||
<DropdownMenu.SubTrigger
|
||||
class="flex cursor-pointer items-center gap-2 rounded-md px-2.5 py-1.5 text-sm transition-colors outline-none hover:bg-accent focus:bg-accent"
|
||||
>
|
||||
{#if thinkingEnabled}
|
||||
<Lightbulb class="h-4 w-4 shrink-0 text-amber-400" />
|
||||
{:else}
|
||||
<LightbulbOff class="h-4 w-4 shrink-0 text-muted-foreground" />
|
||||
{/if}
|
||||
|
||||
<span class="flex-1">Thinking</span>
|
||||
|
||||
{#if thinkingEnabled}
|
||||
<span class="text-xs text-muted-foreground">{currentEffort}</span>
|
||||
{:else}
|
||||
<span class="text-xs text-muted-foreground">off</span>
|
||||
{/if}
|
||||
</DropdownMenu.SubTrigger>
|
||||
|
||||
<DropdownMenu.SubContent
|
||||
class="w-60 rounded-xl bg-popover p-3 text-popover-foreground shadow-md outline-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95"
|
||||
>
|
||||
{#each REASONING_EFFORT_LEVELS as level (level.value)}
|
||||
<button
|
||||
type="button"
|
||||
class="flex w-full cursor-pointer items-center gap-2 rounded-lg px-2.5 py-2 text-left text-sm transition-colors hover:bg-accent"
|
||||
class:bg-accent={isSelected(level)}
|
||||
onclick={() => handleSelection(level)}
|
||||
>
|
||||
{#if isSelected(level)}
|
||||
<Check class="h-4 w-4 shrink-0 text-foreground" />
|
||||
{:else}
|
||||
<div class="h-4 w-4 shrink-0"></div>
|
||||
{/if}
|
||||
|
||||
<span class="flex-1">{level.label}</span>
|
||||
|
||||
{#if !level.isOff}
|
||||
<span class="text-[11px] text-muted-foreground opacity-60">
|
||||
{REASONING_EFFORT_TOKENS[level.value] === -1
|
||||
? 'Unlimited'
|
||||
: `Max ${REASONING_EFFORT_TOKENS[level.value].toLocaleString()} tokens`}
|
||||
</span>
|
||||
{/if}
|
||||
|
||||
{#if level.hasInfo}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<Info class="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
|
||||
</Tooltip.Trigger>
|
||||
<Tooltip.Content side="left">
|
||||
<p>Maximum thinking effort with extended context usage</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
</button>
|
||||
{/each}
|
||||
</DropdownMenu.SubContent>
|
||||
</DropdownMenu.Sub>
|
||||
{/if}
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
<script lang="ts">
|
||||
import { Lightbulb, LightbulbOff, Check, Info } from '@lucide/svelte';
|
||||
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { ReasoningEffort, MessageRole } from '$lib/enums';
|
||||
import { REASONING_EFFORT_TOKENS } from '$lib/constants/reasoning-effort-tokens';
|
||||
import { REASONING_EFFORT_LEVELS } from '$lib/constants/reasoning-effort';
|
||||
import type { ReasoningEffortLevel } from '$lib/types';
|
||||
import {
|
||||
modelsStore,
|
||||
checkModelSupportsThinking,
|
||||
supportsThinking,
|
||||
propsCacheVersion,
|
||||
loadedModelIds
|
||||
} from '$lib/stores/models.svelte';
|
||||
import { chatStore } from '$lib/stores/chat.svelte';
|
||||
import { conversationsStore, activeMessages } from '$lib/stores/conversations.svelte';
|
||||
import { isRouterMode } from '$lib/stores/server.svelte';
|
||||
import type { DatabaseMessage } from '$lib/types/database';
|
||||
|
||||
let thinkingEnabled = $derived(conversationsStore.getThinkingEnabled());
|
||||
let currentEffort = $derived(conversationsStore.getReasoningEffort());
|
||||
let isOff = $derived(!thinkingEnabled);
|
||||
let tooltipText = $derived(thinkingEnabled ? `${currentEffort} Reasoning` : 'Disabled Reasoning');
|
||||
let subOpen = $state(false);
|
||||
|
||||
// Get conversation model from message history
|
||||
let conversationModel = $derived(
|
||||
chatStore.getConversationModel(activeMessages() as DatabaseMessage[])
|
||||
);
|
||||
|
||||
// Fallback: if model props aren't available, check if any assistant messages
|
||||
// for this model in the active conversation have reasoning content.
|
||||
let modelSupportsThinkingFromMessages = $derived.by(() => {
|
||||
const modelId = isRouterMode() ? modelsStore.selectedModelName || conversationModel : null;
|
||||
if (!modelId) return false;
|
||||
const messages = conversationsStore.activeMessages;
|
||||
return messages.some(
|
||||
(m: DatabaseMessage) =>
|
||||
m.role === MessageRole.ASSISTANT && m.model === modelId && !!m.reasoningContent
|
||||
);
|
||||
});
|
||||
|
||||
// Check if model supports thinking. Primary: chat template from /props.
|
||||
// Fallback: message history (reasoning content in assistant messages).
|
||||
let modelSupportsThinking = $derived.by(() => {
|
||||
loadedModelIds();
|
||||
propsCacheVersion();
|
||||
|
||||
if (isRouterMode()) {
|
||||
const modelId = modelsStore.selectedModelName || conversationModel;
|
||||
return checkModelSupportsThinking(modelId ?? '') || modelSupportsThinkingFromMessages;
|
||||
}
|
||||
|
||||
// In non-router mode, use the built-in supportsThinking
|
||||
return supportsThinking() || modelSupportsThinkingFromMessages;
|
||||
});
|
||||
|
||||
// Check if current item is selected
|
||||
function isSelected(item: ReasoningEffortLevel): boolean {
|
||||
if (item.isOff) {
|
||||
return isOff;
|
||||
}
|
||||
return thinkingEnabled && currentEffort === item.value;
|
||||
}
|
||||
|
||||
function handleSelection(item: ReasoningEffortLevel) {
|
||||
if (item.isOff) {
|
||||
conversationsStore.setThinkingEnabled(false);
|
||||
} else {
|
||||
conversationsStore.setThinkingEnabled(true);
|
||||
conversationsStore.setReasoningEffort(item.value as ReasoningEffort);
|
||||
}
|
||||
subOpen = false;
|
||||
}
|
||||
</script>
|
||||
|
||||
{#if modelSupportsThinking}
|
||||
<DropdownMenu.Root bind:open={subOpen}>
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<DropdownMenu.Trigger
|
||||
class={[
|
||||
'flex h-6 w-6 cursor-pointer items-center justify-center rounded-full p-0 transition-colors focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2',
|
||||
thinkingEnabled ? 'bg-amber-400/10 hover:bg-amber-400/20' : 'bg-muted'
|
||||
]}
|
||||
aria-label={`${tooltipText}. Click to configure.`}
|
||||
>
|
||||
{#if thinkingEnabled}
|
||||
<Lightbulb class="h-3 w-3 text-amber-400" />
|
||||
{:else}
|
||||
<LightbulbOff class="h-3 w-3 text-muted-foreground" />
|
||||
{/if}
|
||||
</DropdownMenu.Trigger>
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content>
|
||||
<p class="capitalize">{tooltipText}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
|
||||
<DropdownMenu.Content
|
||||
align="start"
|
||||
class="w-60 rounded-xl bg-popover p-3 text-popover-foreground shadow-md outline-none"
|
||||
>
|
||||
<div class="mb-2 px-2.5 text-sm font-medium">Reasoning effort</div>
|
||||
|
||||
{#each REASONING_EFFORT_LEVELS as level (level.value)}
|
||||
<button
|
||||
type="button"
|
||||
class="flex w-full cursor-pointer items-center gap-2 rounded-lg px-2.5 py-2 text-left text-sm transition-colors hover:bg-accent"
|
||||
class:bg-accent={isSelected(level)}
|
||||
onclick={() => handleSelection(level)}
|
||||
>
|
||||
{#if isSelected(level)}
|
||||
<Check class="h-4 w-4 shrink-0 text-foreground" />
|
||||
{:else}
|
||||
<div class="h-4 w-4 shrink-0"></div>
|
||||
{/if}
|
||||
|
||||
<span class="flex-1">{level.label}</span>
|
||||
|
||||
{#if !level.isOff}
|
||||
<span class="text-[11px] text-muted-foreground opacity-60">
|
||||
{REASONING_EFFORT_TOKENS[level.value] === -1
|
||||
? 'Unlimited'
|
||||
: `Max ${REASONING_EFFORT_TOKENS[level.value].toLocaleString()} tokens`}
|
||||
</span>
|
||||
{/if}
|
||||
|
||||
{#if level.hasInfo}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<Info class="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
|
||||
</Tooltip.Trigger>
|
||||
<Tooltip.Content side="left">
|
||||
<p>Maximum reasoning effort with extended context usage</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
</button>
|
||||
{/each}
|
||||
</DropdownMenu.Content>
|
||||
</DropdownMenu.Root>
|
||||
{/if}
|
||||
|
|
@ -240,6 +240,15 @@ export { default as ChatFormActionAddToolsSubmenu } from './ChatForm/ChatFormAct
|
|||
*/
|
||||
export { default as ChatFormActionAddMcpServersSubmenu } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddMcpServersSubmenu.svelte';
|
||||
|
||||
/**
|
||||
* **ChatFormReasoningToggle** - Thinking toggle button with effort dropdown
|
||||
*
|
||||
* A toggle button with lightbulb icon that indicates thinking status.
|
||||
* Shows the reasoning effort dropdown when clicked.
|
||||
* Only visible when the current model supports thinking.
|
||||
*/
|
||||
export { default as ChatFormReasoningToggle } from './ChatForm/ChatFormActions/ChatFormReasoningToggle.svelte';
|
||||
|
||||
/**
|
||||
* Hidden file input element for programmatic file selection.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
hideOrgName?: boolean;
|
||||
showRaw?: boolean;
|
||||
hideQuantization?: boolean;
|
||||
hideTags?: boolean;
|
||||
aliases?: string[];
|
||||
tags?: string[];
|
||||
class?: string;
|
||||
|
|
@ -17,7 +18,8 @@
|
|||
modelId,
|
||||
hideOrgName = false,
|
||||
showRaw = undefined,
|
||||
hideQuantization = false,
|
||||
hideQuantization,
|
||||
hideTags,
|
||||
aliases,
|
||||
tags,
|
||||
class: className = '',
|
||||
|
|
@ -31,6 +33,8 @@
|
|||
|
||||
let parsed = $derived(ModelsService.parseModelId(modelId));
|
||||
let resolvedShowRaw = $derived(showRaw ?? (config().showRawModelNames as boolean) ?? false);
|
||||
let resolvedHideQuantization = $derived(hideQuantization ?? !config().showModelQuantization);
|
||||
let resolvedHideTags = $derived(hideTags ?? !config().showModelTags);
|
||||
|
||||
let uniqueAliases = $derived([...new Set(aliases ?? [])]);
|
||||
let uniqueTags = $derived([...new Set([...(parsed.tags ?? []), ...(tags ?? [])])]);
|
||||
|
|
@ -53,7 +57,7 @@
|
|||
</span>
|
||||
{/if}
|
||||
|
||||
{#if parsed.quantization && !hideQuantization}
|
||||
{#if parsed.quantization && !resolvedHideQuantization}
|
||||
<span class={badgeClass}>
|
||||
{parsed.quantization}
|
||||
</span>
|
||||
|
|
@ -69,7 +73,7 @@
|
|||
{/each}
|
||||
{/if}
|
||||
|
||||
{#if uniqueTags.length > 0}
|
||||
{#if uniqueTags.length > 0 && !resolvedHideTags}
|
||||
{#each uniqueTags as tag (tag)}
|
||||
<span class={tagBadgeClass}>{tag}</span>
|
||||
{/each}
|
||||
|
|
|
|||
|
|
@ -106,9 +106,7 @@
|
|||
]}
|
||||
style="max-width: min(calc(100cqw - 10rem), 20rem)"
|
||||
>
|
||||
<Package class="h-3.5 w-3.5" />
|
||||
|
||||
<ModelId modelId={currentModel} class="min-w-0" hideQuantization />
|
||||
<Package class="h-3.5 w-3.5 shrink-0" />
|
||||
</span>
|
||||
{:else}
|
||||
<p class="text-xs text-muted-foreground">No models available.</p>
|
||||
|
|
@ -120,7 +118,7 @@
|
|||
<DropdownMenu.Root bind:open={isOpen} onOpenChange={ms.handleOpenChange}>
|
||||
<DropdownMenu.Trigger
|
||||
class={[
|
||||
`inline-grid cursor-pointer grid-cols-[1fr_auto_1fr] items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
|
||||
`inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
|
||||
!ms.isCurrentModelInCache
|
||||
? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'
|
||||
: forceForegroundText
|
||||
|
|
@ -133,7 +131,7 @@
|
|||
]}
|
||||
disabled={disabled || ms.updating}
|
||||
>
|
||||
<Package class="h-3.5 w-3.5" />
|
||||
<Package class="h-3.5 w-3.5 shrink-0" />
|
||||
|
||||
{#if selectedOption}
|
||||
<Tooltip.Root>
|
||||
|
|
@ -144,6 +142,7 @@
|
|||
modelId={selectedOption.model}
|
||||
class="min-w-0 overflow-hidden"
|
||||
hideOrgName={false}
|
||||
hideQuantization
|
||||
{...props}
|
||||
/>
|
||||
{/snippet}
|
||||
|
|
@ -158,9 +157,9 @@
|
|||
{/if}
|
||||
|
||||
{#if ms.updating || ms.isLoadingModel}
|
||||
<Loader2 class="h-3 w-3.5 animate-spin" />
|
||||
<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
|
||||
{:else}
|
||||
<ChevronDown class="h-3 w-3.5" />
|
||||
<ChevronDown class="h-3 w-3.5 shrink-0" />
|
||||
{/if}
|
||||
</DropdownMenu.Trigger>
|
||||
|
||||
|
|
@ -251,7 +250,7 @@
|
|||
onclick={() => ms.handleOpenChange(true)}
|
||||
disabled={disabled || ms.updating}
|
||||
>
|
||||
<Package class="h-3.5 w-3.5" />
|
||||
<Package class="h-3.5 w-3.5 shrink-0" />
|
||||
|
||||
{#if selectedOption}
|
||||
<Tooltip.Root>
|
||||
|
|
@ -262,6 +261,7 @@
|
|||
modelId={selectedOption.model}
|
||||
class="min-w-0 overflow-hidden"
|
||||
hideOrgName={false}
|
||||
hideQuantization
|
||||
{...props}
|
||||
/>
|
||||
{/snippet}
|
||||
|
|
@ -274,7 +274,7 @@
|
|||
{/if}
|
||||
|
||||
{#if ms.updating}
|
||||
<Loader2 class="h-3 w-3.5 animate-spin" />
|
||||
<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
|
||||
{/if}
|
||||
</button>
|
||||
{/if}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,7 @@
|
|||
DialogModelInformation,
|
||||
ModelId,
|
||||
ModelsSelectorList,
|
||||
SearchInput,
|
||||
TruncatedText
|
||||
SearchInput
|
||||
} from '$lib/components/app';
|
||||
|
||||
interface Props {
|
||||
|
|
@ -67,7 +66,7 @@
|
|||
<button
|
||||
type="button"
|
||||
class={[
|
||||
`inline-grid cursor-pointer grid-cols-[1fr_auto_1fr] items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
|
||||
`inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
|
||||
!ms.isCurrentModelInCache
|
||||
? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'
|
||||
: forceForegroundText
|
||||
|
|
@ -81,7 +80,7 @@
|
|||
disabled={disabled || ms.updating}
|
||||
onclick={() => ms.handleOpenChange(true)}
|
||||
>
|
||||
<Package class="h-3.5 w-3.5" />
|
||||
<Package class="h-3.5 w-3.5 shrink-0" />
|
||||
|
||||
{#if !selectedOption}
|
||||
<span class="min-w-0 font-medium">Select model</span>
|
||||
|
|
@ -90,14 +89,15 @@
|
|||
class="text-xs"
|
||||
modelId={selectedOption?.model || ''}
|
||||
hideQuantization
|
||||
hideTags
|
||||
hideOrgName
|
||||
/>
|
||||
{/if}
|
||||
|
||||
{#if ms.updating || ms.isLoadingModel}
|
||||
<Loader2 class="h-3 w-3.5 animate-spin" />
|
||||
<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
|
||||
{:else}
|
||||
<ChevronDown class="h-3 w-3.5" />
|
||||
<ChevronDown class="h-3 w-3.5 shrink-0" />
|
||||
{/if}
|
||||
</button>
|
||||
|
||||
|
|
@ -168,12 +168,12 @@
|
|||
onclick={() => ms.handleOpenChange(true)}
|
||||
disabled={disabled || ms.updating}
|
||||
>
|
||||
<Package class="h-3.5 w-3.5" />
|
||||
<Package class="h-3.5 w-3.5 shrink-0" />
|
||||
|
||||
<TruncatedText text={selectedOption?.model || ''} class="font-medium" />
|
||||
<ModelId modelId={selectedOption?.model || ''} class="font-medium" hideQuantization />
|
||||
|
||||
{#if ms.updating}
|
||||
<Loader2 class="h-3 w-3.5 animate-spin" />
|
||||
<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
|
||||
{/if}
|
||||
</button>
|
||||
{/if}
|
||||
|
|
|
|||
|
|
@ -74,8 +74,7 @@ export { default as ModelsSelectorOption } from './ModelsSelectorOption.svelte';
|
|||
*/
|
||||
export { default as ModelsSelectorSheet } from './ModelsSelectorSheet.svelte';
|
||||
|
||||
/**
|
||||
* **ModelBadge** - Model name display badge
|
||||
/** * **ModelBadge** - Model name display badge
|
||||
*
|
||||
* Compact badge showing current model name with package icon.
|
||||
* Only visible in single model mode. Supports tooltip and copy functionality.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,17 @@ export const API_MODELS = {
|
|||
UNLOAD: '/models/unload'
|
||||
};
|
||||
|
||||
// chat completion routes, the control route drives realtime inference (e.g. end reasoning)
|
||||
export const API_CHAT = {
|
||||
COMPLETIONS: './v1/chat/completions',
|
||||
CONTROL: './v1/chat/completions/control'
|
||||
};
|
||||
|
||||
// slot introspection, requires the --slots flag on the server
|
||||
export const API_SLOTS = {
|
||||
LIST: './slots'
|
||||
};
|
||||
|
||||
export const API_TOOLS = {
|
||||
LIST: '/tools',
|
||||
EXECUTE: '/tools'
|
||||
|
|
|
|||
|
|
@ -79,7 +79,9 @@ export const ATTACHMENT_FILE_ITEMS: AttachmentMenuItem[] = [
|
|||
}
|
||||
];
|
||||
|
||||
export const ATTACHMENT_EXTRA_ITEMS: AttachmentMenuItem[] = [
|
||||
export const ATTACHMENT_EXTRA_ITEMS: AttachmentMenuItem[] = [];
|
||||
|
||||
export const ATTACHMENT_PROMPT_ITEMS: AttachmentMenuItem[] = [
|
||||
{
|
||||
id: AttachmentMenuItemId.SYSTEM_MESSAGE,
|
||||
label: 'System Message',
|
||||
|
|
@ -87,10 +89,7 @@ export const ATTACHMENT_EXTRA_ITEMS: AttachmentMenuItem[] = [
|
|||
enabledWhen: AttachmentItemEnabledWhen.ALWAYS,
|
||||
hasEnabledTooltip: true,
|
||||
action: AttachmentAction.SYSTEM_PROMPT_CLICK
|
||||
}
|
||||
];
|
||||
|
||||
export const ATTACHMENT_MCP_ITEMS: AttachmentMenuItem[] = [
|
||||
},
|
||||
{
|
||||
id: AttachmentMenuItemId.MCP_PROMPT,
|
||||
label: 'MCP Prompt',
|
||||
|
|
@ -98,7 +97,10 @@ export const ATTACHMENT_MCP_ITEMS: AttachmentMenuItem[] = [
|
|||
enabledWhen: AttachmentItemEnabledWhen.ALWAYS,
|
||||
action: AttachmentAction.MCP_PROMPT_CLICK,
|
||||
visibleWhen: AttachmentItemVisibleWhen.HAS_MCP_PROMPTS_SUPPORT
|
||||
},
|
||||
}
|
||||
];
|
||||
|
||||
export const ATTACHMENT_MCP_ITEMS: AttachmentMenuItem[] = [
|
||||
{
|
||||
id: AttachmentMenuItemId.MCP_RESOURCES,
|
||||
label: 'MCP Resources',
|
||||
|
|
|
|||
7
tools/ui/src/lib/constants/control-actions.ts
Normal file
7
tools/ui/src/lib/constants/control-actions.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
// actions accepted by the realtime inference control endpoint (API_CHAT.CONTROL)
|
||||
// kept separate from the endpoint paths since these are protocol level verbs
|
||||
export const CONTROL_ACTION = {
|
||||
END_REASONING: 'reasoning_end'
|
||||
} as const;
|
||||
|
||||
export type ControlAction = (typeof CONTROL_ACTION)[keyof typeof CONTROL_ACTION];
|
||||
23
tools/ui/src/lib/constants/error.ts
Normal file
23
tools/ui/src/lib/constants/error.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
export const ERROR_MESSAGES = {
|
||||
NETWORK: {
|
||||
GENERIC: 'Failed to connect to server',
|
||||
NXDOMAIN: 'Server not found - check server address',
|
||||
REFUSED: 'Connection refused - server may be offline',
|
||||
TIMEOUT: 'Request timed out',
|
||||
UNREACHABLE: 'Server is not running or unreachable'
|
||||
},
|
||||
HTTP: {
|
||||
GENERIC: 'Request failed',
|
||||
ACCESS_DENIED: 'Access denied',
|
||||
INTERNAL_ERROR: 'Server error - check server logs',
|
||||
NOT_FOUND: 'Not found',
|
||||
TEMPORARILY_UNAVAILABLE: 'Server temporarily unavailable'
|
||||
}
|
||||
};
|
||||
|
||||
export const HTTP_CODE_TO_STRING: Record<string, string> = {
|
||||
401: ERROR_MESSAGES.HTTP.ACCESS_DENIED,
|
||||
403: ERROR_MESSAGES.HTTP.ACCESS_DENIED,
|
||||
500: ERROR_MESSAGES.HTTP.INTERNAL_ERROR,
|
||||
503: ERROR_MESSAGES.HTTP.TEMPORARILY_UNAVAILABLE
|
||||
};
|
||||
|
|
@ -5,6 +5,8 @@ export * from './agentic';
|
|||
export * from './api-endpoints';
|
||||
export * from './attachment-labels';
|
||||
export * from './database';
|
||||
export * from './reasoning-effort';
|
||||
export * from './reasoning-effort-tokens';
|
||||
export * from './storage';
|
||||
export * from './attachment-menu';
|
||||
export * from './auto-scroll';
|
||||
|
|
@ -15,6 +17,7 @@ export * from './cli-flags';
|
|||
export * from './code-blocks';
|
||||
export * from './code';
|
||||
export * from './context-keys';
|
||||
export * from './control-actions';
|
||||
export * from './css-classes';
|
||||
export * from './floating-ui-constraints';
|
||||
export * from './formatters';
|
||||
|
|
|
|||
12
tools/ui/src/lib/constants/reasoning-effort-tokens.ts
Normal file
12
tools/ui/src/lib/constants/reasoning-effort-tokens.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
import { ReasoningEffort } from '$lib/enums';
|
||||
|
||||
/**
|
||||
* Reasoning effort to token budget mapping.
|
||||
* Maps the ReasoningEffort enum values to concrete token counts for the server.
|
||||
*/
|
||||
export const REASONING_EFFORT_TOKENS: Record<string, number> = {
|
||||
[ReasoningEffort.LOW]: 512,
|
||||
[ReasoningEffort.MEDIUM]: 2048,
|
||||
[ReasoningEffort.HIGH]: 8192,
|
||||
[ReasoningEffort.MAX]: -1 // unlimited
|
||||
};
|
||||
21
tools/ui/src/lib/constants/reasoning-effort.ts
Normal file
21
tools/ui/src/lib/constants/reasoning-effort.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { ReasoningEffort } from '$lib/enums';
|
||||
import type { ReasoningEffortLevel } from '$lib/types';
|
||||
|
||||
/**
|
||||
* Reasoning effort UI labels.
|
||||
* Keys match the ReasoningEffort enum values for type-safe lookups.
|
||||
*/
|
||||
export const REASONING_EFFORT_LABELS: Record<string, string> = {
|
||||
[ReasoningEffort.LOW]: 'Low',
|
||||
[ReasoningEffort.MEDIUM]: 'Medium',
|
||||
[ReasoningEffort.HIGH]: 'High',
|
||||
[ReasoningEffort.MAX]: 'Max'
|
||||
};
|
||||
|
||||
export const REASONING_EFFORT_LEVELS: ReasoningEffortLevel[] = [
|
||||
{ value: 'off', label: 'Off', isOff: true },
|
||||
{ value: ReasoningEffort.LOW, label: 'Low' },
|
||||
{ value: ReasoningEffort.MEDIUM, label: 'Medium' },
|
||||
{ value: ReasoningEffort.HIGH, label: 'High' },
|
||||
{ value: ReasoningEffort.MAX, label: 'Max', hasInfo: true }
|
||||
];
|
||||
|
|
@ -29,6 +29,8 @@ export const SETTINGS_KEYS = {
|
|||
ALWAYS_SHOW_SIDEBAR_ON_DESKTOP: 'alwaysShowSidebarOnDesktop',
|
||||
FULL_HEIGHT_CODE_BLOCKS: 'fullHeightCodeBlocks',
|
||||
SHOW_RAW_MODEL_NAMES: 'showRawModelNames',
|
||||
SHOW_MODEL_QUANTIZATION: 'showModelQuantization',
|
||||
SHOW_MODEL_TAGS: 'showModelTags',
|
||||
SHOW_SYSTEM_MESSAGE: 'showSystemMessage',
|
||||
// Sampling
|
||||
TEMPERATURE: 'temperature',
|
||||
|
|
@ -64,6 +66,7 @@ export const SETTINGS_KEYS = {
|
|||
// Developer
|
||||
DISABLE_REASONING_PARSING: 'disableReasoningParsing',
|
||||
EXCLUDE_REASONING_FROM_CONTEXT: 'excludeReasoningFromContext',
|
||||
ENABLE_THINKING: 'enableThinking',
|
||||
SHOW_RAW_OUTPUT_SWITCH: 'showRawOutputSwitch',
|
||||
// PY_INTERPRETER_ENABLED: 'pyInterpreterEnabled',
|
||||
CUSTOM_JSON: 'customJson',
|
||||
|
|
|
|||
|
|
@ -330,6 +330,30 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
|
|||
paramType: SyncableParameterType.BOOLEAN
|
||||
}
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.SHOW_MODEL_QUANTIZATION,
|
||||
label: 'Show model quantization information',
|
||||
help: 'Display quantization badges (e.g. Q8_0, Q4_K_M) next to model names throughout the interface.',
|
||||
defaultValue: true,
|
||||
type: SettingsFieldType.CHECKBOX,
|
||||
section: SETTINGS_SECTION_SLUGS.DISPLAY,
|
||||
sync: {
|
||||
serverKey: SETTINGS_KEYS.SHOW_MODEL_QUANTIZATION,
|
||||
paramType: SyncableParameterType.BOOLEAN
|
||||
}
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.SHOW_MODEL_TAGS,
|
||||
label: 'Show model tags',
|
||||
help: 'Display model tags (e.g. "vision", "reasoning") next to model names throughout the interface.',
|
||||
defaultValue: true,
|
||||
type: SettingsFieldType.CHECKBOX,
|
||||
section: SETTINGS_SECTION_SLUGS.DISPLAY,
|
||||
sync: {
|
||||
serverKey: SETTINGS_KEYS.SHOW_MODEL_TAGS,
|
||||
paramType: SyncableParameterType.BOOLEAN
|
||||
}
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.ALWAYS_SHOW_AGENTIC_TURNS,
|
||||
label: 'Always show agentic turns in conversation',
|
||||
|
|
@ -646,6 +670,14 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
|
|||
paramType: SyncableParameterType.BOOLEAN
|
||||
}
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.ENABLE_THINKING,
|
||||
label: 'Enable thinking',
|
||||
help: 'Enable model thinking/reasoning for each request. When off, the model will skip the thinking phase and go straight to the response.',
|
||||
defaultValue: false,
|
||||
type: SettingsFieldType.CHECKBOX,
|
||||
section: SETTINGS_SECTION_SLUGS.DEVELOPER
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.SHOW_RAW_OUTPUT_SWITCH,
|
||||
label: 'Enable raw output toggle',
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ export const CONFIG_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.config`;
|
|||
export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTools`;
|
||||
export const FAVORITE_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.favoriteModels`;
|
||||
export const MCP_DEFAULT_ENABLED_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.mcpDefaultEnabled`;
|
||||
export const THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.thinkingEnabledDefault`;
|
||||
export const REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.reasoningEffortDefault`;
|
||||
export const USER_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.userOverrides`;
|
||||
|
||||
// Deprecated old key names (kept for backward compat while users migrate)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ export {
|
|||
ReasoningFormat
|
||||
} from './chat.enums';
|
||||
|
||||
export { ReasoningEffort } from './reasoning-effort.enums';
|
||||
|
||||
export {
|
||||
FileTypeCategory,
|
||||
FileTypeImage,
|
||||
|
|
|
|||
10
tools/ui/src/lib/enums/reasoning-effort.enums.ts
Normal file
10
tools/ui/src/lib/enums/reasoning-effort.enums.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
/**
|
||||
* Reasoning effort levels for thinking models.
|
||||
* These values are sent to the server and mapped to token budgets.
|
||||
*/
|
||||
export enum ReasoningEffort {
|
||||
LOW = 'low',
|
||||
MEDIUM = 'medium',
|
||||
HIGH = 'high',
|
||||
MAX = 'max'
|
||||
}
|
||||
|
|
@ -17,6 +17,8 @@ export interface UseToolsPanelReturn {
|
|||
getFavicon(group: { source: ToolSource; label: string }): string | null;
|
||||
isGroupDisabled(group: ToolGroup): boolean;
|
||||
toggleGroupExpanded(label: string): void;
|
||||
/** Toggle all tools in a group by label (avoids stale group object references). */
|
||||
toggleGroupByLabel(label: string): void;
|
||||
handleOpen(): void;
|
||||
}
|
||||
|
||||
|
|
@ -91,6 +93,13 @@ export function useToolsPanel(): UseToolsPanelReturn {
|
|||
}
|
||||
}
|
||||
|
||||
function toggleGroupByLabel(label: string): void {
|
||||
// Find current group by label to get up-to-date tool references
|
||||
const group = activeGroups.find((g) => g.label === label);
|
||||
if (!group) return;
|
||||
toolsStore.toggleGroup(group);
|
||||
}
|
||||
|
||||
function handleOpen(): void {
|
||||
if (toolsStore.builtinTools.length === 0 && !toolsStore.loading) {
|
||||
toolsStore.fetchBuiltinTools();
|
||||
|
|
@ -117,6 +126,7 @@ export function useToolsPanel(): UseToolsPanelReturn {
|
|||
getFavicon,
|
||||
isGroupDisabled,
|
||||
toggleGroupExpanded,
|
||||
toggleGroupByLabel,
|
||||
handleOpen
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,11 @@ import {
|
|||
ATTACHMENT_LABEL_MCP_PROMPT,
|
||||
ATTACHMENT_LABEL_MCP_RESOURCE,
|
||||
LEGACY_AGENTIC_REGEX,
|
||||
SETTINGS_KEYS
|
||||
REASONING_EFFORT_TOKENS,
|
||||
SETTINGS_KEYS,
|
||||
API_CHAT,
|
||||
API_SLOTS,
|
||||
CONTROL_ACTION
|
||||
} from '$lib/constants';
|
||||
import {
|
||||
AttachmentType,
|
||||
|
|
@ -126,6 +130,7 @@ export class ChatService {
|
|||
onReasoningChunk,
|
||||
onToolCallChunk,
|
||||
onModel,
|
||||
onCompletionId,
|
||||
onTimings,
|
||||
// Tools for function calling
|
||||
tools,
|
||||
|
|
@ -158,6 +163,8 @@ export class ChatService {
|
|||
// Config options
|
||||
disableReasoningParsing,
|
||||
excludeReasoningFromContext,
|
||||
enableThinking,
|
||||
reasoningEffort,
|
||||
continueFinalMessage
|
||||
} = options;
|
||||
|
||||
|
|
@ -239,6 +246,21 @@ export class ChatService {
|
|||
? ReasoningFormat.NONE
|
||||
: ReasoningFormat.AUTO;
|
||||
|
||||
const reasoningBudgetTokens =
|
||||
enableThinking && reasoningEffort ? (REASONING_EFFORT_TOKENS[reasoningEffort] ?? -1) : -1;
|
||||
|
||||
requestBody.chat_template_kwargs = {
|
||||
...(requestBody.chat_template_kwargs ?? {}),
|
||||
enable_thinking: enableThinking
|
||||
};
|
||||
|
||||
if (reasoningBudgetTokens >= 0) {
|
||||
requestBody.thinking_budget_tokens = reasoningBudgetTokens;
|
||||
}
|
||||
|
||||
// arms the budget sampler so reasoning can be ended at runtime via the control endpoint
|
||||
requestBody.reasoning_control = true;
|
||||
|
||||
if (continueFinalMessage) {
|
||||
requestBody.continue_final_message = true;
|
||||
requestBody.add_generation_prompt = false;
|
||||
|
|
@ -289,7 +311,7 @@ export class ChatService {
|
|||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`./v1/chat/completions`, {
|
||||
const response = await fetch(API_CHAT.COMPLETIONS, {
|
||||
method: 'POST',
|
||||
headers: getJsonHeaders(),
|
||||
body: JSON.stringify(requestBody),
|
||||
|
|
@ -315,6 +337,7 @@ export class ChatService {
|
|||
onReasoningChunk,
|
||||
onToolCallChunk,
|
||||
onModel,
|
||||
onCompletionId,
|
||||
onTimings,
|
||||
conversationId,
|
||||
signal
|
||||
|
|
@ -379,7 +402,7 @@ export class ChatService {
|
|||
*/
|
||||
static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise<boolean> {
|
||||
try {
|
||||
const url = model ? `./slots?model=${encodeURIComponent(model)}` : './slots';
|
||||
const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST;
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) return true;
|
||||
|
||||
|
|
@ -390,6 +413,50 @@ export class ChatService {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ends the current reasoning block of a running completion, targeted by its
|
||||
* chat completion id (streamed back as `id`). Matching the completion rather
|
||||
* than a slot index avoids a TOCTOU: a finished completion simply matches
|
||||
* nothing server side. The model is carried so the router forwards to the
|
||||
* right child, single model ignores it. Returns true on success.
|
||||
*/
|
||||
static async stopReasoning(completionId: string, model?: string | null): Promise<boolean> {
|
||||
if (!completionId) {
|
||||
console.error(
|
||||
'stopReasoning: no completion id for the active message, cannot target the running completion'
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const body: Record<string, unknown> = {
|
||||
id: completionId,
|
||||
action: CONTROL_ACTION.END_REASONING
|
||||
};
|
||||
if (model) body.model = model;
|
||||
|
||||
try {
|
||||
const res = await fetch(API_CHAT.CONTROL, {
|
||||
method: 'POST',
|
||||
headers: getJsonHeaders(),
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
|
||||
const data = await res.json().catch(() => null);
|
||||
if (!res.ok || data?.success !== true) {
|
||||
console.error('stopReasoning: control request failed', {
|
||||
status: res.status,
|
||||
completionId,
|
||||
response: data
|
||||
});
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('stopReasoning: control request threw', { completionId, error });
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a fire-and-forget request to pre-encode the conversation in the server's KV cache.
|
||||
* After a response completes, this re-submits the full conversation
|
||||
|
|
@ -457,7 +524,7 @@ export class ChatService {
|
|||
}
|
||||
|
||||
try {
|
||||
await fetch(`./v1/chat/completions`, {
|
||||
await fetch(API_CHAT.COMPLETIONS, {
|
||||
method: 'POST',
|
||||
headers: getJsonHeaders(),
|
||||
body: JSON.stringify(requestBody),
|
||||
|
|
@ -502,6 +569,7 @@ export class ChatService {
|
|||
onReasoningChunk?: (chunk: string) => void,
|
||||
onToolCallChunk?: (chunk: string) => void,
|
||||
onModel?: (model: string) => void,
|
||||
onCompletionId?: (id: string) => void,
|
||||
onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void,
|
||||
conversationId?: string,
|
||||
abortSignal?: AbortSignal
|
||||
|
|
@ -519,6 +587,7 @@ export class ChatService {
|
|||
let lastTimings: ChatMessageTimings | undefined;
|
||||
let streamFinished = false;
|
||||
let modelEmitted = false;
|
||||
let idEmitted = false;
|
||||
let toolCallIndexOffset = 0;
|
||||
let hasOpenToolCallBatch = false;
|
||||
|
||||
|
|
@ -607,6 +676,11 @@ export class ChatService {
|
|||
onModel?.(chunkModel);
|
||||
}
|
||||
|
||||
if (parsed.id && !idEmitted) {
|
||||
idEmitted = true;
|
||||
onCompletionId?.(parsed.id);
|
||||
}
|
||||
|
||||
if (promptProgress) {
|
||||
ChatService.notifyTimings(undefined, promptProgress, onTimings);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -488,6 +488,7 @@ class AgenticStore {
|
|||
onToolCallsStreaming,
|
||||
onAttachments,
|
||||
onModel,
|
||||
onCompletionId,
|
||||
onAssistantTurnComplete,
|
||||
createToolResultMessage,
|
||||
createAssistantMessage,
|
||||
|
|
@ -597,6 +598,7 @@ class AgenticStore {
|
|||
}
|
||||
},
|
||||
onModel,
|
||||
onCompletionId,
|
||||
onTimings: (timings?: ChatMessageTimings, progress?: ChatMessagePromptProgress) => {
|
||||
onTimings?.(timings, progress);
|
||||
if (timings) {
|
||||
|
|
|
|||
|
|
@ -63,7 +63,10 @@ class ChatStore {
|
|||
currentResponse = $state('');
|
||||
errorDialogState = $state<ErrorDialogState | null>(null);
|
||||
isLoading = $state(false);
|
||||
// true while the active conversation streams reasoning content but no visible content yet
|
||||
isReasoning = $state(false);
|
||||
chatLoadingStates = new SvelteMap<string, boolean>();
|
||||
chatReasoningStates = new SvelteMap<string, boolean>();
|
||||
chatStreamingStates = new SvelteMap<string, { response: string; messageId: string }>();
|
||||
private abortControllers = new SvelteMap<string, AbortController>();
|
||||
private preEncodeAbortController: AbortController | null = null;
|
||||
|
|
@ -94,6 +97,17 @@ class ChatStore {
|
|||
} else {
|
||||
this.chatLoadingStates.delete(convId);
|
||||
if (convId === conversationsStore.activeConversation?.id) this.isLoading = false;
|
||||
this.setChatReasoning(convId, false);
|
||||
}
|
||||
}
|
||||
|
||||
private setChatReasoning(convId: string, reasoning: boolean): void {
|
||||
if (reasoning) {
|
||||
this.chatReasoningStates.set(convId, true);
|
||||
if (convId === conversationsStore.activeConversation?.id) this.isReasoning = true;
|
||||
} else {
|
||||
this.chatReasoningStates.delete(convId);
|
||||
if (convId === conversationsStore.activeConversation?.id) this.isReasoning = false;
|
||||
}
|
||||
}
|
||||
private setChatStreaming(convId: string, response: string, messageId: string): void {
|
||||
|
|
@ -110,6 +124,7 @@ class ChatStore {
|
|||
}
|
||||
syncLoadingStateForChat(convId: string): void {
|
||||
this.isLoading = this.chatLoadingStates.get(convId) || false;
|
||||
this.isReasoning = this.chatReasoningStates.get(convId) || false;
|
||||
const s = this.chatStreamingStates.get(convId);
|
||||
this.currentResponse = s?.response || '';
|
||||
this.isStreamingActive = s !== undefined;
|
||||
|
|
@ -265,6 +280,10 @@ class ChatStore {
|
|||
return this.chatLoadingStates.get(convId) || false;
|
||||
}
|
||||
|
||||
isChatReasoningPublic(convId: string): boolean {
|
||||
return this.chatReasoningStates.get(convId) || false;
|
||||
}
|
||||
|
||||
private isChatLoadingInternal(convId: string): boolean {
|
||||
return this.chatLoadingStates.has(convId) || this.chatStreamingStates.has(convId);
|
||||
}
|
||||
|
|
@ -655,6 +674,17 @@ class ChatStore {
|
|||
}
|
||||
};
|
||||
|
||||
let completionIdRecorded = false;
|
||||
const recordCompletionId = (id: string): void => {
|
||||
if (!id || completionIdRecorded) return;
|
||||
completionIdRecorded = true;
|
||||
const idx = conversationsStore.findMessageIndex(currentMessageId);
|
||||
conversationsStore.updateMessageAtIndex(idx, { completionId: id });
|
||||
DatabaseService.updateMessage(currentMessageId, { completionId: id }).catch(() => {
|
||||
completionIdRecorded = false;
|
||||
});
|
||||
};
|
||||
|
||||
const updateStreamingUI = () => {
|
||||
this.setChatStreaming(convId, streamedContent, currentMessageId);
|
||||
const idx = conversationsStore.findMessageIndex(currentMessageId);
|
||||
|
|
@ -676,6 +706,7 @@ class ChatStore {
|
|||
onChunk: (chunk: string) => {
|
||||
streamedContent += chunk;
|
||||
updateStreamingUI();
|
||||
this.setChatReasoning(convId, false);
|
||||
},
|
||||
onReasoningChunk: (chunk: string) => {
|
||||
streamedReasoningContent += chunk;
|
||||
|
|
@ -685,6 +716,7 @@ class ChatStore {
|
|||
conversationsStore.updateMessageAtIndex(idx, {
|
||||
reasoningContent: streamedReasoningContent
|
||||
});
|
||||
this.setChatReasoning(convId, true);
|
||||
},
|
||||
onToolCallsStreaming: (toolCalls) => {
|
||||
const idx = conversationsStore.findMessageIndex(currentMessageId);
|
||||
|
|
@ -702,6 +734,7 @@ class ChatStore {
|
|||
DatabaseService.updateMessage(messageId, { extra: updatedExtras }).catch(console.error);
|
||||
},
|
||||
onModel: (modelName: string) => recordModel(modelName),
|
||||
onCompletionId: (id: string) => recordCompletionId(id),
|
||||
onTurnComplete: (intermediateTimings: ChatMessageTimings) => {
|
||||
// Update the first assistant message with cumulative agentic timings
|
||||
const idx = conversationsStore.findMessageIndex(assistantMessage.id);
|
||||
|
|
@ -887,6 +920,7 @@ class ChatStore {
|
|||
onChunk: streamCallbacks.onChunk,
|
||||
onReasoningChunk: streamCallbacks.onReasoningChunk,
|
||||
onModel: streamCallbacks.onModel,
|
||||
onCompletionId: streamCallbacks.onCompletionId,
|
||||
onTimings: streamCallbacks.onTimings,
|
||||
onComplete: async (
|
||||
finalContent?: string,
|
||||
|
|
@ -1373,6 +1407,7 @@ class ChatStore {
|
|||
appendedContent += chunk;
|
||||
hasReceivedContent = true;
|
||||
updateStreamingContent(originalContent + appendedContent);
|
||||
this.setChatReasoning(msg.convId, false);
|
||||
},
|
||||
onReasoningChunk: (chunk: string) => {
|
||||
appendedReasoning += chunk;
|
||||
|
|
@ -1382,6 +1417,7 @@ class ChatStore {
|
|||
conversationsStore.updateMessageAtIndex(idx, {
|
||||
reasoningContent: originalReasoning + appendedReasoning
|
||||
});
|
||||
this.setChatReasoning(msg.convId, true);
|
||||
},
|
||||
onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => {
|
||||
const tokensPerSecond =
|
||||
|
|
@ -1816,6 +1852,9 @@ class ChatStore {
|
|||
|
||||
if (currentConfig.excludeReasoningFromContext) apiOptions.excludeReasoningFromContext = true;
|
||||
|
||||
apiOptions.enableThinking = conversationsStore.getThinkingEnabled();
|
||||
apiOptions.reasoningEffort = conversationsStore.getReasoningEffort();
|
||||
|
||||
if (hasValue(currentConfig.temperature))
|
||||
apiOptions.temperature = Number(currentConfig.temperature);
|
||||
|
||||
|
|
@ -1924,6 +1963,7 @@ export const isChatLoading = (convId: string) => chatStore.isChatLoadingPublic(c
|
|||
export const isChatStreaming = () => chatStore.isStreaming();
|
||||
export const isEditing = () => chatStore.isEditing();
|
||||
export const isLoading = () => chatStore.isLoading;
|
||||
export const isReasoning = () => chatStore.isReasoning;
|
||||
export const pendingEditMessageId = () => chatStore.pendingEditMessageId;
|
||||
export const chatHasPendingMessage = (convId: string) => chatStore.hasPendingMessage(convId);
|
||||
export const chatPendingMessageContent = (convId: string) =>
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import { MigrationService } from '$lib/services/migration.service';
|
|||
import { config } from '$lib/stores/settings.svelte';
|
||||
import { filterByLeafNodeId, findLeafNode, generateConversationTitle } from '$lib/utils';
|
||||
import type { McpServerOverride } from '$lib/types/database';
|
||||
import { MessageRole, HtmlInputType, FileExtensionText } from '$lib/enums';
|
||||
import { MessageRole, HtmlInputType, FileExtensionText, ReasoningEffort } from '$lib/enums';
|
||||
import {
|
||||
ISO_DATE_TIME_SEPARATOR,
|
||||
ISO_DATE_TIME_SEPARATOR_REPLACEMENT,
|
||||
|
|
@ -38,7 +38,9 @@ import {
|
|||
ISO_TIME_SEPARATOR_REPLACEMENT,
|
||||
NON_ALPHANUMERIC_REGEX,
|
||||
MULTIPLE_UNDERSCORE_REGEX,
|
||||
MCP_DEFAULT_ENABLED_LOCALSTORAGE_KEY
|
||||
MCP_DEFAULT_ENABLED_LOCALSTORAGE_KEY,
|
||||
THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY,
|
||||
REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY
|
||||
} from '$lib/constants';
|
||||
|
||||
import { ROUTES } from '$lib/constants/routes';
|
||||
|
|
@ -74,6 +76,12 @@ class ConversationsStore {
|
|||
/** Pending MCP server overrides for new conversations (before first message) */
|
||||
pendingMcpServerOverrides = $state<McpServerOverride[]>(ConversationsStore.loadMcpDefaults());
|
||||
|
||||
/** Global (non-conversation-specific) thinking toggle default */
|
||||
pendingThinkingEnabled = $state(ConversationsStore.loadThinkingDefaults());
|
||||
|
||||
/** Global (non-conversation-specific) reasoning effort default */
|
||||
pendingReasoningEffort = $state<ReasoningEffort>(ConversationsStore.loadReasoningEffortDefault());
|
||||
|
||||
/** Load MCP default overrides from localStorage */
|
||||
private static loadMcpDefaults(): McpServerOverride[] {
|
||||
if (typeof globalThis.localStorage === 'undefined') return [];
|
||||
|
|
@ -104,6 +112,45 @@ class ConversationsStore {
|
|||
}
|
||||
}
|
||||
|
||||
/** Load thinking-enabled default from localStorage */
|
||||
private static loadThinkingDefaults(): boolean {
|
||||
if (typeof globalThis.localStorage === 'undefined') return false;
|
||||
try {
|
||||
const raw = localStorage.getItem(THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY);
|
||||
if (!raw) return false;
|
||||
const parsed = raw === 'true';
|
||||
return typeof parsed === 'boolean' ? parsed : false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Persist thinking-enabled default to localStorage */
|
||||
private saveThinkingDefaults(): void {
|
||||
if (typeof globalThis.localStorage === 'undefined') return;
|
||||
localStorage.setItem(
|
||||
THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY,
|
||||
this.pendingThinkingEnabled ? 'true' : 'false'
|
||||
);
|
||||
}
|
||||
|
||||
/** Load reasoning effort default from localStorage */
|
||||
private static loadReasoningEffortDefault(): ReasoningEffort {
|
||||
if (typeof globalThis.localStorage === 'undefined') return ReasoningEffort.MEDIUM;
|
||||
try {
|
||||
const raw = localStorage.getItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY);
|
||||
return (raw as ReasoningEffort) || ReasoningEffort.MEDIUM;
|
||||
} catch {
|
||||
return ReasoningEffort.MEDIUM;
|
||||
}
|
||||
}
|
||||
|
||||
/** Persist reasoning effort default to localStorage */
|
||||
private saveReasoningEffortDefaults(): void {
|
||||
if (typeof globalThis.localStorage === 'undefined') return;
|
||||
localStorage.setItem(REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, this.pendingReasoningEffort);
|
||||
}
|
||||
|
||||
/** Callback for title update confirmation dialog */
|
||||
titleUpdateConfirmationCallback?: (currentTitle: string, newTitle: string) => Promise<boolean>;
|
||||
|
||||
|
|
@ -253,6 +300,12 @@ class ConversationsStore {
|
|||
this.pendingMcpServerOverrides = [];
|
||||
}
|
||||
|
||||
// Inherit global thinking default into the new conversation
|
||||
conversation.thinkingEnabled = this.pendingThinkingEnabled;
|
||||
await DatabaseService.updateConversation(conversation.id, {
|
||||
thinkingEnabled: this.pendingThinkingEnabled
|
||||
});
|
||||
|
||||
this.conversations = [conversation, ...this.conversations];
|
||||
this.activeConversation = conversation;
|
||||
this.activeMessages = [];
|
||||
|
|
@ -276,6 +329,7 @@ class ConversationsStore {
|
|||
}
|
||||
|
||||
this.pendingMcpServerOverrides = [];
|
||||
this.pendingThinkingEnabled = false;
|
||||
this.activeConversation = conversation;
|
||||
|
||||
if (conversation.currNode) {
|
||||
|
|
@ -304,8 +358,9 @@ class ConversationsStore {
|
|||
clearActiveConversation(): void {
|
||||
this.activeConversation = null;
|
||||
this.activeMessages = [];
|
||||
// reload MCP defaults so new chats inherit persisted state
|
||||
// reload defaults so new chats inherit persisted state
|
||||
this.pendingMcpServerOverrides = ConversationsStore.loadMcpDefaults();
|
||||
this.pendingThinkingEnabled = ConversationsStore.loadThinkingDefaults();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -703,6 +758,84 @@ class ConversationsStore {
|
|||
this.saveMcpDefaults();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the effective thinking-enabled state for the active conversation.
|
||||
* Returns the conversation override if set, otherwise the global default.
|
||||
*/
|
||||
getThinkingEnabled(): boolean {
|
||||
if (this.activeConversation) {
|
||||
return this.activeConversation.thinkingEnabled ?? this.pendingThinkingEnabled;
|
||||
}
|
||||
return this.pendingThinkingEnabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the thinking-enabled state for the active conversation.
|
||||
* If no conversation exists, stores the global default.
|
||||
* @param enabled - The enabled state
|
||||
*/
|
||||
async setThinkingEnabled(enabled: boolean): Promise<void> {
|
||||
if (!this.activeConversation) {
|
||||
this.pendingThinkingEnabled = enabled;
|
||||
this.saveThinkingDefaults();
|
||||
return;
|
||||
}
|
||||
|
||||
this.activeConversation = {
|
||||
...this.activeConversation,
|
||||
thinkingEnabled: enabled
|
||||
};
|
||||
|
||||
await DatabaseService.updateConversation(this.activeConversation.id, {
|
||||
thinkingEnabled: enabled
|
||||
});
|
||||
|
||||
const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id);
|
||||
if (convIndex !== -1) {
|
||||
this.conversations[convIndex].thinkingEnabled = enabled;
|
||||
this.conversations = [...this.conversations];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the effective reasoning effort for the active conversation.
|
||||
* Returns the conversation override if set, otherwise the global default.
|
||||
*/
|
||||
getReasoningEffort(): ReasoningEffort {
|
||||
if (this.activeConversation) {
|
||||
return this.activeConversation.reasoningEffort ?? this.pendingReasoningEffort;
|
||||
}
|
||||
return this.pendingReasoningEffort;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the reasoning effort for the active conversation.
|
||||
* If no conversation exists, stores the global default.
|
||||
* @param effort - The effort level ('low' | 'medium' | 'high' | 'max')
|
||||
*/
|
||||
async setReasoningEffort(effort: ReasoningEffort): Promise<void> {
|
||||
if (!this.activeConversation) {
|
||||
this.pendingReasoningEffort = effort;
|
||||
this.saveReasoningEffortDefaults();
|
||||
return;
|
||||
}
|
||||
|
||||
this.activeConversation = {
|
||||
...this.activeConversation,
|
||||
reasoningEffort: effort
|
||||
};
|
||||
|
||||
await DatabaseService.updateConversation(this.activeConversation.id, {
|
||||
reasoningEffort: effort
|
||||
});
|
||||
|
||||
const convIndex = this.conversations.findIndex((c) => c.id === this.activeConversation!.id);
|
||||
if (convIndex !== -1) {
|
||||
this.conversations[convIndex].reasoningEffort = effort;
|
||||
this.conversations = [...this.conversations];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Forks a conversation at a specific message, creating a new conversation
|
||||
* containing messages from root up to the target message, then navigates to it.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@ import { ServerModelStatus, ModelModality } from '$lib/enums';
|
|||
import { ModelsService } from '$lib/services/models.service';
|
||||
import { PropsService } from '$lib/services/props.service';
|
||||
import { serverStore, isRouterMode } from '$lib/stores/server.svelte';
|
||||
import {
|
||||
detectThinkingSupport,
|
||||
detectThinkingSupportWithReason
|
||||
} from '$lib/utils/chat-template-thinking-detector';
|
||||
import { TTLCache } from '$lib/utils';
|
||||
import {
|
||||
MODEL_PROPS_CACHE_TTL_MS,
|
||||
|
|
@ -215,6 +219,67 @@ class ModelsStore {
|
|||
|
||||
return usage !== undefined && usage.size > 0;
|
||||
}
|
||||
//
|
||||
// Thinking Support Detection
|
||||
//
|
||||
|
||||
/**
|
||||
* Whether the selected model's chat template supports thinking/reasoning.
|
||||
* Uses heuristic detection on the model's chat_template from /props.
|
||||
*
|
||||
* - MODEL mode: uses serverStore.props.chat_template (single loaded model)
|
||||
* - ROUTER mode: fetches /props?model=<id> for the selected model (cached)
|
||||
*
|
||||
* Triggers an async fetch of model props if not yet cached in ROUTER mode.
|
||||
*/
|
||||
get supportsThinking(): boolean {
|
||||
const modelId = this.selectedModelName;
|
||||
if (!modelId) {
|
||||
if (!isRouterMode()) {
|
||||
return detectThinkingSupport(serverStore.props?.chat_template ?? '');
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isRouterMode() && !this.modelPropsCache.get(modelId)) {
|
||||
this.fetchModelProps(modelId);
|
||||
}
|
||||
const props = this.getModelProps(modelId);
|
||||
return detectThinkingSupport(props?.chat_template ?? '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a specific model supports thinking.
|
||||
* Fetches model props if not cached (in router mode).
|
||||
*/
|
||||
checkModelSupportsThinking(modelId: string): boolean {
|
||||
if (!modelId) return false;
|
||||
|
||||
if (isRouterMode() && !this.modelPropsCache.get(modelId)) {
|
||||
this.fetchModelProps(modelId);
|
||||
}
|
||||
|
||||
const props = this.getModelProps(modelId);
|
||||
return detectThinkingSupport(props?.chat_template ?? '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Detailed thinking support detection result with reason for debugging/UI.
|
||||
*/
|
||||
get thinkingSupportDetails(): { supported: boolean; reason: string } {
|
||||
const modelId = this.selectedModelName;
|
||||
if (!modelId) {
|
||||
if (!isRouterMode()) {
|
||||
return detectThinkingSupportWithReason(serverStore.props?.chat_template ?? '');
|
||||
}
|
||||
return { supported: false, reason: 'No model selected' };
|
||||
}
|
||||
if (isRouterMode() && !this.modelPropsCache.get(modelId)) {
|
||||
this.fetchModelProps(modelId);
|
||||
}
|
||||
const props = this.getModelProps(modelId);
|
||||
return detectThinkingSupportWithReason(props?.chat_template ?? '');
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
|
|
@ -362,6 +427,7 @@ class ModelsStore {
|
|||
try {
|
||||
const props = await PropsService.fetchForModel(modelId);
|
||||
this.modelPropsCache.set(modelId, props);
|
||||
this.propsCacheVersion++;
|
||||
return props;
|
||||
} catch (error) {
|
||||
console.warn(`Failed to fetch props for model ${modelId}:`, error);
|
||||
|
|
@ -755,3 +821,7 @@ export const propsCacheVersion = () => modelsStore.propsCacheVersion;
|
|||
export const singleModelName = () => modelsStore.singleModelName;
|
||||
export const selectedModelContextSize = () => modelsStore.selectedModelContextSize;
|
||||
export const favoriteModelIds = () => modelsStore.favoriteModelIds;
|
||||
export const supportsThinking = () => modelsStore.supportsThinking;
|
||||
export const checkModelSupportsThinking = (modelId: string) =>
|
||||
modelsStore.checkModelSupportsThinking(modelId);
|
||||
export const thinkingSupportDetails = () => modelsStore.thinkingSupportDetails;
|
||||
|
|
|
|||
|
|
@ -82,8 +82,8 @@ class ServerStore {
|
|||
this.props = props;
|
||||
this.error = null;
|
||||
this.detectRole(props);
|
||||
} catch (error) {
|
||||
this.error = this.getErrorMessage(error);
|
||||
} catch (error: unknown) {
|
||||
this.error = error instanceof Error ? error.message : String(error);
|
||||
console.error('Error fetching server properties:', error);
|
||||
} finally {
|
||||
this.loading = false;
|
||||
|
|
@ -95,32 +95,6 @@ class ServerStore {
|
|||
await fetchPromise;
|
||||
}
|
||||
|
||||
private getErrorMessage(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
const message = error.message || '';
|
||||
|
||||
if (error.name === 'TypeError' && message.includes('fetch')) {
|
||||
return 'Server is not running or unreachable';
|
||||
} else if (message.includes('ECONNREFUSED')) {
|
||||
return 'Connection refused - server may be offline';
|
||||
} else if (message.includes('ENOTFOUND')) {
|
||||
return 'Server not found - check server address';
|
||||
} else if (message.includes('ETIMEDOUT')) {
|
||||
return 'Request timed out';
|
||||
} else if (message.includes('503')) {
|
||||
return 'Server temporarily unavailable';
|
||||
} else if (message.includes('500')) {
|
||||
return 'Server error - check server logs';
|
||||
} else if (message.includes('404')) {
|
||||
return 'Server endpoint not found';
|
||||
} else if (message.includes('403') || message.includes('401')) {
|
||||
return 'Access denied';
|
||||
}
|
||||
}
|
||||
|
||||
return 'Failed to connect to server';
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.props = null;
|
||||
this.error = null;
|
||||
|
|
|
|||
1
tools/ui/src/lib/types/agentic.d.ts
vendored
1
tools/ui/src/lib/types/agentic.d.ts
vendored
|
|
@ -93,6 +93,7 @@ export interface AgenticFlowCallbacks {
|
|||
onAttachments?: (messageId: string, extras: DatabaseMessageExtra[]) => void;
|
||||
/** Model name detected from response */
|
||||
onModel?: (model: string) => void;
|
||||
onCompletionId?: (id: string) => void;
|
||||
/** Current assistant turn's streaming is complete - save to DB */
|
||||
onAssistantTurnComplete?: (
|
||||
content: string,
|
||||
|
|
|
|||
1
tools/ui/src/lib/types/api.d.ts
vendored
1
tools/ui/src/lib/types/api.d.ts
vendored
|
|
@ -271,6 +271,7 @@ export interface ApiChatCompletionToolCall extends ApiChatCompletionToolCallDelt
|
|||
}
|
||||
|
||||
export interface ApiChatCompletionStreamChunk {
|
||||
id?: string;
|
||||
object?: string;
|
||||
model?: string;
|
||||
choices: Array<{
|
||||
|
|
|
|||
1
tools/ui/src/lib/types/chat.d.ts
vendored
1
tools/ui/src/lib/types/chat.d.ts
vendored
|
|
@ -97,6 +97,7 @@ export interface ChatStreamCallbacks {
|
|||
onToolCallsStreaming?: (toolCalls: ApiChatCompletionToolCall[]) => void;
|
||||
onAttachments?: (messageId: string, extras: DatabaseMessageExtra[]) => void;
|
||||
onModel?: (model: string) => void;
|
||||
onCompletionId?: (id: string) => void;
|
||||
onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void;
|
||||
onAssistantTurnComplete?: (
|
||||
content: string,
|
||||
|
|
|
|||
6
tools/ui/src/lib/types/database.d.ts
vendored
6
tools/ui/src/lib/types/database.d.ts
vendored
|
|
@ -1,5 +1,5 @@
|
|||
import type { ChatMessageTimings, ChatRole, ChatMessageType } from '$lib/types/chat';
|
||||
import { AttachmentType } from '$lib/enums';
|
||||
import { AttachmentType, ReasoningEffort } from '$lib/enums';
|
||||
|
||||
export interface McpServerOverride {
|
||||
serverId: string;
|
||||
|
|
@ -12,6 +12,8 @@ export interface DatabaseConversation {
|
|||
lastModified: number;
|
||||
name: string;
|
||||
mcpServerOverrides?: McpServerOverride[];
|
||||
thinkingEnabled?: boolean;
|
||||
reasoningEffort?: ReasoningEffort;
|
||||
forkedFromConversationId?: string;
|
||||
}
|
||||
|
||||
|
|
@ -112,6 +114,8 @@ export interface DatabaseMessage {
|
|||
reasoningContent?: string;
|
||||
/** Serialized JSON array of tool calls made by assistant messages */
|
||||
toolCalls?: string;
|
||||
/** Chat completion id streamed by the server, used to target realtime control (e.g. end reasoning) */
|
||||
completionId?: string;
|
||||
/** Tool call ID for tool result messages (role: 'tool') */
|
||||
toolCallId?: string;
|
||||
children: string[];
|
||||
|
|
|
|||
|
|
@ -162,3 +162,6 @@ export type {
|
|||
|
||||
// Tools types
|
||||
export type { ToolEntry, ToolGroup } from './tools';
|
||||
|
||||
// Reasoning
|
||||
export type { ReasoningEffortLevel } from './reasoning';
|
||||
|
|
|
|||
6
tools/ui/src/lib/types/reasoning.ts
Normal file
6
tools/ui/src/lib/types/reasoning.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
export interface ReasoningEffortLevel {
|
||||
value: string;
|
||||
label: string;
|
||||
isOff?: boolean;
|
||||
hasInfo?: boolean;
|
||||
}
|
||||
12
tools/ui/src/lib/types/settings.d.ts
vendored
12
tools/ui/src/lib/types/settings.d.ts
vendored
|
|
@ -2,7 +2,12 @@ import type { SETTING_CONFIG_DEFAULT, SETTINGS_SECTION_TITLES } from '$lib/const
|
|||
import type { ChatMessagePromptProgress, ChatMessageTimings } from './chat';
|
||||
import type { OpenAIToolDefinition } from './mcp';
|
||||
import type { DatabaseMessageExtra } from './database';
|
||||
import type { ParameterSource, SyncableParameterType, SettingsFieldType } from '$lib/enums';
|
||||
import type {
|
||||
ParameterSource,
|
||||
ReasoningEffort,
|
||||
SyncableParameterType,
|
||||
SettingsFieldType
|
||||
} from '$lib/enums';
|
||||
import type { Icon } from '@lucide/svelte';
|
||||
import type { Component } from 'svelte';
|
||||
|
||||
|
|
@ -65,6 +70,10 @@ export interface SettingsChatServiceOptions {
|
|||
disableReasoningParsing?: boolean;
|
||||
// Strip reasoning content from context before sending
|
||||
excludeReasoningFromContext?: boolean;
|
||||
// Enable model thinking/reasoning via chat_template_kwargs
|
||||
enableThinking?: boolean;
|
||||
// Reasoning effort level (low/medium/high/max) for thinking models
|
||||
reasoningEffort?: ReasoningEffort;
|
||||
tools?: OpenAIToolDefinition[];
|
||||
// Generation parameters
|
||||
temperature?: number;
|
||||
|
|
@ -101,6 +110,7 @@ export interface SettingsChatServiceOptions {
|
|||
onToolCallChunk?: (chunk: string) => void;
|
||||
onAttachments?: (extras: DatabaseMessageExtra[]) => void;
|
||||
onModel?: (model: string) => void;
|
||||
onCompletionId?: (id: string) => void;
|
||||
onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void;
|
||||
onComplete?: (
|
||||
response: string,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { base } from '$app/paths';
|
||||
import { getJsonHeaders, getAuthHeaders } from './api-headers';
|
||||
import { UrlProtocol } from '$lib/enums';
|
||||
import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '$lib/constants/error';
|
||||
|
||||
/**
|
||||
* API Fetch Utilities
|
||||
|
|
@ -54,10 +55,15 @@ export async function apiFetch<T>(path: string, options: ApiFetchOptions = {}):
|
|||
? path
|
||||
: `${base}${path}`;
|
||||
|
||||
const response = await fetch(url, {
|
||||
...fetchOptions,
|
||||
headers
|
||||
});
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(url, {
|
||||
...fetchOptions,
|
||||
headers
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(beautifyNetworkError(e));
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const errorMessage = await parseErrorMessage(response);
|
||||
|
|
@ -101,10 +107,15 @@ export async function apiFetchWithParams<T>(
|
|||
const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders();
|
||||
const headers = { ...baseHeaders, ...customHeaders };
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
...fetchOptions,
|
||||
headers
|
||||
});
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(url.toString(), {
|
||||
...fetchOptions,
|
||||
headers
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(beautifyNetworkError(e));
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const errorMessage = await parseErrorMessage(response);
|
||||
|
|
@ -154,5 +165,37 @@ async function parseErrorMessage(response: Response): Promise<string> {
|
|||
// JSON parsing failed, use status text
|
||||
}
|
||||
|
||||
return `Request failed: ${response.status} ${response.statusText}`;
|
||||
const httpErrorStr = HTTP_CODE_TO_STRING[response.status];
|
||||
if (httpErrorStr) {
|
||||
return httpErrorStr;
|
||||
}
|
||||
|
||||
return `${ERROR_MESSAGES.HTTP.GENERIC}: ${response.status} ${response.statusText}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a network issue into a human-readable message.
|
||||
* @param throwable - The throwable raised during fetch operation
|
||||
* @returns Error in an human-readable format
|
||||
*/
|
||||
function beautifyNetworkError(throwable: unknown): string {
|
||||
let message;
|
||||
if (throwable instanceof Error) {
|
||||
message = throwable.message;
|
||||
if (throwable.name === 'TypeError' && message.includes('fetch')) {
|
||||
return ERROR_MESSAGES.NETWORK.UNREACHABLE;
|
||||
}
|
||||
} else {
|
||||
message = String(throwable);
|
||||
}
|
||||
|
||||
if (message.includes('ECONNREFUSED')) {
|
||||
return ERROR_MESSAGES.NETWORK.REFUSED;
|
||||
} else if (message.includes('ENOTFOUND')) {
|
||||
return ERROR_MESSAGES.NETWORK.NXDOMAIN;
|
||||
} else if (message.includes('ETIMEDOUT')) {
|
||||
return ERROR_MESSAGES.NETWORK.TIMEOUT;
|
||||
}
|
||||
|
||||
return `${ERROR_MESSAGES.NETWORK.GENERIC} (${message})`;
|
||||
}
|
||||
|
|
|
|||
86
tools/ui/src/lib/utils/chat-template-thinking-detector.ts
Normal file
86
tools/ui/src/lib/utils/chat-template-thinking-detector.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
/**
|
||||
* Detects whether a model's chat template supports thinking/reasoning control.
|
||||
*
|
||||
* The server "/props" endpoint does NOT expose a supports_thinking flag.
|
||||
* It is computed internally by common_chat_templates_support_enable_thinking
|
||||
* in common/chat.cpp. A proper server flag would make this unnecessary.
|
||||
*
|
||||
* Detection order (most reliable first):
|
||||
* 1. Thinking-control Jinja2 variables === pass-through via chat_template_kwargs
|
||||
* 2. Thinking-control Jinja2 conditionals === template-native on/off logic
|
||||
* 3. Paired thinking-content tag pairs === models that output special tags
|
||||
*/
|
||||
|
||||
const THINKING_KWARG_VARS = ['enable_thinking', 'reasoning_effort', 'thinking_budget'];
|
||||
|
||||
/**
|
||||
* Paired thinking-content tag patterns.
|
||||
*
|
||||
* Inspected: llama-cpp-deepseek-r1/v3, nim-nemotron-{3,4}-nano, qwen-qwq-32b,
|
||||
* qwen-3-32b, google-gemma-4-31b-it, kimikimi-k2-thinking, apertus-8b-instruct,
|
||||
* mistralai-Mistral-Small-3.2-24B, ByteDance-Seed-OSS.
|
||||
*
|
||||
* The self-closing entry is Kimi-K2, Gemma4 fixed-length pair,
|
||||
* where both tags always appear adjacent with no content between.
|
||||
*/
|
||||
const THINKING_TAG_PATTERNS: Array<[string, string | null]> = [
|
||||
['<think>', '</think>'],
|
||||
['<|channel>thought', '<|channel|>'],
|
||||
['<|think|>', '</|think|>'],
|
||||
['<seed:think|>', '</seed:think|>'],
|
||||
['<think></think>', null]
|
||||
];
|
||||
|
||||
const JINJA_THINKING_CONDITIONALS: RegExp[] = [
|
||||
// Matches: {% if enable thinking %}, {% if enable_thinking %}, {% if (enable_thinking is defined) %}
|
||||
// Handles: underscore-separated (enable_thinking), space-separated (enable thinking),
|
||||
// and optional parens/brackets before enable (if (enable_thinking )
|
||||
/\{%-?\s*if\s+\(?\s*\w*enable[\s_]+\w*(thinking|think|reasoning)/i,
|
||||
/\{%-?\s*if\s+\w*(thinking|reasoning)\s*(is not|==|!=)/i,
|
||||
/\{%-?\s*if\s+not\s+\w*enable/i,
|
||||
/\{%-?\s*if\s+ns\.enable_thinking/i
|
||||
];
|
||||
|
||||
/** Guards against false positives:
|
||||
* - Generic thought keyword (tool descriptions say "chain of thought")
|
||||
* - Qwen vertical-bar token (used for ALL tool calls, not thinking)
|
||||
*/
|
||||
export function detectThinkingSupport(t: string): boolean {
|
||||
if (!t) return false;
|
||||
for (const kwarg of THINKING_KWARG_VARS) {
|
||||
const regex = new RegExp(
|
||||
`(\\{\\{[^{}]*\\b${kwarg}\\b[^{}]*\\}\\}|\\{%[^{}]*\\b${kwarg}\\b[^{}]*%\\})`,
|
||||
'i'
|
||||
);
|
||||
if (regex.test(t)) return true;
|
||||
}
|
||||
for (const p of JINJA_THINKING_CONDITIONALS) {
|
||||
if (p.test(t)) return true;
|
||||
}
|
||||
for (const [s, e] of THINKING_TAG_PATTERNS) {
|
||||
if (t.includes(s) && (!e || t.includes(e))) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function detectThinkingSupportWithReason(t: string): { supported: boolean; reason: string } {
|
||||
if (!t) return { supported: false, reason: 'No chat template available' };
|
||||
for (const kwarg of THINKING_KWARG_VARS) {
|
||||
const regex = new RegExp(
|
||||
`(\\{\\{[^{}]*\\b${kwarg}\\b[^{}]*\\}\\}|\\{%[^{}]*\\b${kwarg}\\b[^{}]*%\\})`,
|
||||
'i'
|
||||
);
|
||||
if (regex.test(t)) {
|
||||
return { supported: true, reason: 'Found: ' + kwarg };
|
||||
}
|
||||
}
|
||||
for (const p of JINJA_THINKING_CONDITIONALS) {
|
||||
if (p.test(t)) return { supported: true, reason: 'Found: thinking conditional' };
|
||||
}
|
||||
for (const [s, e] of THINKING_TAG_PATTERNS) {
|
||||
if (t.includes(s) && (!e || t.includes(e))) {
|
||||
return { supported: true, reason: 'Found: ' + s + (e ? ' .. ' + e : ' (self)') };
|
||||
}
|
||||
}
|
||||
return { supported: false, reason: 'No thinking patterns found' };
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
|
||||
test('home page has expected h1', async ({ page }) => {
|
||||
test('home page loads correctly', async ({ page }) => {
|
||||
await page.goto('/');
|
||||
await expect(page.locator('h1').first()).toBeVisible();
|
||||
// Wait for the greeting to become visible (stores need time to initialize)
|
||||
await expect(page.locator('h1', { hasText: /Hello there/ })).toBeVisible();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import { llamaCppBuildPlugin } from './scripts/vite-plugin-llama-cpp-build';
|
|||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const SERVER_ORIGIN = import.meta.env?.VITE_PUBLIC_SERVER_ORIGIN || 'http://localhost:8080';
|
||||
|
||||
export default defineConfig({
|
||||
resolve: {
|
||||
alias: {
|
||||
|
|
@ -75,12 +77,12 @@ export default defineConfig({
|
|||
|
||||
server: {
|
||||
proxy: {
|
||||
'/v1': 'http://localhost:8080',
|
||||
'/props': 'http://localhost:8080',
|
||||
'/models': 'http://localhost:8080',
|
||||
'/tools': 'http://localhost:8080',
|
||||
'/slots': 'http://localhost:8080',
|
||||
'/cors-proxy': 'http://localhost:8080'
|
||||
'/v1': SERVER_ORIGIN,
|
||||
'/props': SERVER_ORIGIN,
|
||||
'/models': SERVER_ORIGIN,
|
||||
'/tools': SERVER_ORIGIN,
|
||||
'/slots': SERVER_ORIGIN,
|
||||
'/cors-proxy': SERVER_ORIGIN
|
||||
},
|
||||
headers: {
|
||||
'Cross-Origin-Embedder-Policy': 'require-corp',
|
||||
|
|
|
|||
36
vendor/cpp-httplib/httplib.cpp
vendored
36
vendor/cpp-httplib/httplib.cpp
vendored
|
|
@ -1832,7 +1832,7 @@ int getaddrinfo_with_timeout(const char *node, const char *service,
|
|||
|
||||
#ifdef _WIN32
|
||||
// Windows-specific implementation using GetAddrInfoEx with overlapped I/O
|
||||
OVERLAPPED overlapped = {0};
|
||||
OVERLAPPED overlapped = {};
|
||||
HANDLE event = CreateEventW(nullptr, TRUE, FALSE, nullptr);
|
||||
if (!event) { return EAI_FAIL; }
|
||||
|
||||
|
|
@ -1841,7 +1841,7 @@ int getaddrinfo_with_timeout(const char *node, const char *service,
|
|||
PADDRINFOEXW result_addrinfo = nullptr;
|
||||
HANDLE cancel_handle = nullptr;
|
||||
|
||||
ADDRINFOEXW hints_ex = {0};
|
||||
ADDRINFOEXW hints_ex = {};
|
||||
if (hints) {
|
||||
hints_ex.ai_flags = hints->ai_flags;
|
||||
hints_ex.ai_family = hints->ai_family;
|
||||
|
|
@ -9912,13 +9912,28 @@ bool ClientImpl::process_request(Stream &strm, Request &req,
|
|||
}
|
||||
#endif
|
||||
|
||||
// Handle Expect: 100-continue with timeout
|
||||
if (expect_100_continue && CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND > 0) {
|
||||
time_t sec = CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND / 1000;
|
||||
time_t usec = (CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND % 1000) * 1000;
|
||||
auto ret = detail::select_read(strm.socket(), sec, usec);
|
||||
if (ret <= 0) {
|
||||
// Timeout or error: send body anyway (server didn't respond in time)
|
||||
// Handle Expect: 100-continue.
|
||||
//
|
||||
// Wait for an interim/early response by attempting to read the status line
|
||||
// under a short timeout, instead of trusting raw socket readability. Over
|
||||
// TLS, post-handshake records (e.g. session tickets) make the socket
|
||||
// readable without any HTTP response being available; relying on
|
||||
// `select_read` there caused the body to be withheld forever and the
|
||||
// request to fail with `Read` (#2458). If no status line arrives within the
|
||||
// timeout, send the body anyway (matching curl's behavior).
|
||||
auto status_line_read = false;
|
||||
if (expect_100_continue && write_request_success) {
|
||||
if (CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND > 0) {
|
||||
time_t sec = CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND / 1000;
|
||||
time_t usec = (CPPHTTPLIB_EXPECT_100_TIMEOUT_MSECOND % 1000) * 1000;
|
||||
strm.set_read_timeout(sec, usec);
|
||||
status_line_read = read_response_line(strm, req, res, false);
|
||||
strm.set_read_timeout(read_timeout_sec_, read_timeout_usec_);
|
||||
}
|
||||
|
||||
if (!status_line_read) {
|
||||
// No interim response within the timeout: send the body and handle the
|
||||
// response as usual.
|
||||
if (!write_request_body(strm, req, error)) { return false; }
|
||||
expect_100_continue = false; // Switch to normal response handling
|
||||
}
|
||||
|
|
@ -9926,7 +9941,8 @@ bool ClientImpl::process_request(Stream &strm, Request &req,
|
|||
|
||||
// Receive response and headers
|
||||
// When using Expect: 100-continue, don't auto-skip `100 Continue` response
|
||||
if (!read_response_line(strm, req, res, !expect_100_continue) ||
|
||||
if ((!status_line_read &&
|
||||
!read_response_line(strm, req, res, !expect_100_continue)) ||
|
||||
!detail::read_headers(strm, res.headers)) {
|
||||
if (write_request_success) { error = Error::Read; }
|
||||
output_error_log(error, &req);
|
||||
|
|
|
|||
4
vendor/cpp-httplib/httplib.h
vendored
4
vendor/cpp-httplib/httplib.h
vendored
|
|
@ -8,8 +8,8 @@
|
|||
#ifndef CPPHTTPLIB_HTTPLIB_H
|
||||
#define CPPHTTPLIB_HTTPLIB_H
|
||||
|
||||
#define CPPHTTPLIB_VERSION "0.46.0"
|
||||
#define CPPHTTPLIB_VERSION_NUM "0x002e00"
|
||||
#define CPPHTTPLIB_VERSION "0.46.1"
|
||||
#define CPPHTTPLIB_VERSION_NUM "0x002e01"
|
||||
|
||||
#ifdef _WIN32
|
||||
#if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue