mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-03 21:23:38 +00:00
Merge branch 'upstream' into concedo_experimental
# Conflicts: # docs/backend/SYCL.md # ggml/src/ggml-cpu/ggml-cpu.c # ggml/src/ggml-opencl/CMakeLists.txt # ggml/src/ggml-opencl/ggml-opencl.cpp # ggml/src/ggml-opencl/kernels/cvt.cl # ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl # ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl # ggml/src/ggml-sycl/backend.hpp # ggml/src/ggml-sycl/common.hpp # ggml/src/ggml-sycl/dmmv.cpp # ggml/src/ggml-sycl/ggml-sycl.cpp # scripts/sync_vendor.py # tests/CMakeLists.txt # tests/test-alloc.cpp # tests/test-backend-ops.cpp # tests/test-chat.cpp # tests/test-gguf.cpp # tests/test-save-load-state.cpp
This commit is contained in:
Concedo
commit
fa21872f97
76 changed files with 1399 additions and 1110 deletions
|
|
@ -147,7 +147,8 @@ common_peg_arena autoparser::build_parser(const generation_params & inputs, cons
|
|||
} else {
|
||||
parser = content.build_parser(ctx);
|
||||
}
|
||||
return pure_content ? p.prefix(generation_prompt, reasoning.start) + parser : p.prefix(generation_prompt, reasoning.start) << parser;
|
||||
const std::string reasoning_start = trim_whitespace(reasoning.start);
|
||||
return pure_content ? p.prefix(generation_prompt, reasoning_start) + parser : p.prefix(generation_prompt, reasoning_start) << parser;
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -124,16 +124,16 @@ static std::vector<std::function<void(const common_chat_template & tmpl, autopar
|
|||
analysis.tools.format.section_end = "";
|
||||
analysis.tools.format.per_call_start = "<TOOLCALL>";
|
||||
analysis.tools.format.per_call_end = "</TOOLCALL>";
|
||||
analysis.tools.format.tools_array_wrapped = true;
|
||||
analysis.content.mode = content_mode::PLAIN;
|
||||
analysis.content.start = "";
|
||||
analysis.content.end = "";
|
||||
analysis.reasoning.mode = reasoning_mode::TAG_BASED;
|
||||
analysis.reasoning.start = "<think>\n\n";
|
||||
analysis.reasoning.start = "<think>\n";
|
||||
analysis.reasoning.end = "</think>";
|
||||
analysis.assistant_start = "<SPECIAL_11>Assistant";
|
||||
analysis.user_start = "<SPECIAL_11>User";
|
||||
analysis.preserved_tokens.clear();
|
||||
analysis.preserved_tokens.push_back("<SPECIAL_12>");
|
||||
analysis.preserved_tokens.push_back("<SPECIAL_11>");
|
||||
analysis.preserved_tokens.push_back("</think>");
|
||||
analysis.preserved_tokens.push_back("<TOOLCALL>");
|
||||
|
|
|
|||
|
|
@ -1082,6 +1082,9 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *);
|
|||
struct common_prompt_checkpoint {
|
||||
int64_t n_tokens;
|
||||
|
||||
// (optional) id of the task that created the checkpoint
|
||||
int id_task = -1;
|
||||
|
||||
llama_pos pos_min;
|
||||
llama_pos pos_max;
|
||||
|
||||
|
|
|
|||
|
|
@ -7765,7 +7765,7 @@ Current version indicated by LITEVER below.
|
|||
const codeContainer = btn.parentElement.querySelector('pre code');
|
||||
let innercode = codeContainer.innerText;
|
||||
//remove common language descriptiors from the start
|
||||
let langsmatched = ["matlab","jsonc","powershell","ps1","haskell","hs","vbnet","vb","apache","apacheconf","makefile","mk","ini","protobuf","proto","typescript","tsx","markdown","md","mkdown","mkd","python","py","javascript","js","jsx","html","xhtml","xml","css","json","typescript","ts","tsx","bash","sh","zsh","java","csharp","cs","c","h","cpp","hpp","php","sql","ruby","rb","go","golang","kotlin","kt","swift","rust","rs","r","dart","scala","dockerfile","docker","yaml","yml","ini","toml","perl","pl","shell","console","powershell","ps1","lua","typescript","ts"];
|
||||
let langsmatched = ["matlab","jsonc","powershell","ps1","haskell","hs","vbnet","vb","apache","apacheconf","makefile","mk","ini","protobuf","proto","typescript","tsx","markdown","md","mkdown","mkd","python","py","javascript","js","jsx","html","xhtml","xml","css","json","typescript","ts","tsx","bash","sh","zsh","java","csharp","cs","c","h","cpp","hpp","php","sql","ruby","rb","go","golang","kotlin","kt","swift","rust","rs","r","dart","scala","dockerfile","docker","yaml","yml","ini","toml","perl","pl","shell","console","powershell","ps1","lua","typescript","ts","patch","diff"];
|
||||
for(let i = 0; i < langsmatched.length; ++i) {
|
||||
let matcher = langsmatched[i]+"\n";
|
||||
if (innercode.startsWith(matcher)) {
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ extern "C" {
|
|||
|
||||
#define RPC_PROTO_MAJOR_VERSION 4
|
||||
#define RPC_PROTO_MINOR_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 1
|
||||
#define RPC_PROTO_PATCH_VERSION 2
|
||||
|
||||
#ifdef __cplusplus
|
||||
static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
static_assert(GGML_OP_COUNT == 98, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
#endif
|
||||
|
||||
#define GGML_RPC_MAX_SERVERS 16
|
||||
|
|
|
|||
|
|
@ -576,6 +576,7 @@ extern "C" {
|
|||
GGML_OP_RWKV_WKV7,
|
||||
GGML_OP_SOLVE_TRI,
|
||||
GGML_OP_GATED_DELTA_NET,
|
||||
GGML_OP_LIGHTNING_INDEXER,
|
||||
|
||||
GGML_OP_UNARY,
|
||||
|
||||
|
|
@ -2593,6 +2594,24 @@ extern "C" {
|
|||
struct ggml_tensor * state,
|
||||
int64_t K);
|
||||
|
||||
// DSA lightning indexer
|
||||
//
|
||||
// q: [n_embd_idx, n_head_idx, n_batch, ne3 ]
|
||||
// k: [n_embd_idx, 1, n_kv, ne3 ]
|
||||
// weights: [n_head_idx, n_batch, 1, ne3 ] !! prescaled !!
|
||||
// mask: [n_kv, n_batch, 1, ne33] !! f16 !!
|
||||
// res: [n_kv, n_batch, 1, ne3 ]
|
||||
//
|
||||
// broadcast:
|
||||
// ne3 % ne33 == 0
|
||||
//
|
||||
GGML_API struct ggml_tensor * ggml_lightning_indexer(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * q,
|
||||
struct ggml_tensor * k,
|
||||
struct ggml_tensor * weights,
|
||||
struct ggml_tensor * mask);
|
||||
|
||||
// custom operators
|
||||
|
||||
typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
|
||||
|
|
|
|||
|
|
@ -125,12 +125,13 @@ extern "C" {
|
|||
// get ith C string from array with given key_id
|
||||
GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i);
|
||||
|
||||
GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx);
|
||||
GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found
|
||||
GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id);
|
||||
GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id);
|
||||
GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id);
|
||||
GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id);
|
||||
GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx);
|
||||
GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found
|
||||
GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id);
|
||||
GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id);
|
||||
GGML_API const int64_t * gguf_get_tensor_ne (const struct gguf_context * ctx, int64_t tensor_id); // returns ne, an array of GGML_MAX_DIMS elements; ne[dim] is 1 for dim >= n_dims
|
||||
GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id);
|
||||
GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id);
|
||||
|
||||
// removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist)
|
||||
GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key);
|
||||
|
|
|
|||
|
|
@ -2832,6 +2832,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
|
|||
{
|
||||
ggml_compute_forward_gated_delta_net(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
{
|
||||
ggml_compute_forward_lightning_indexer(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_MAP_CUSTOM1:
|
||||
{
|
||||
ggml_compute_forward_map_custom1(params, tensor);
|
||||
|
|
@ -3193,6 +3197,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
|||
case GGML_OP_FLASH_ATTN_BACK:
|
||||
case GGML_OP_SSM_CONV:
|
||||
case GGML_OP_SSM_SCAN:
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
{
|
||||
n_tasks = n_threads;
|
||||
} break;
|
||||
|
|
@ -3814,6 +3819,12 @@ struct ggml_cplan ggml_graph_plan(
|
|||
// to prepare magnitude and phase for inverted FFTs.
|
||||
cur = ggml_type_size(node->type)*(n_tasks + node->ne[0] * n_tasks * 4);
|
||||
} break;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
{
|
||||
// temp buffer for dequantizing lightning indexer keys
|
||||
const int64_t ne10 = node->src[1]->ne[0];
|
||||
cur += sizeof(float)*ne10*n_tasks;
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11568,3 +11568,87 @@ void ggml_compute_forward_fwht(const ggml_compute_params * params, ggml_tensor *
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ggml_compute_forward_lightning_indexer
|
||||
|
||||
void ggml_compute_forward_lightning_indexer(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
const ggml_tensor * q = dst->src[0];
|
||||
const ggml_tensor * k = dst->src[1];
|
||||
const ggml_tensor * w = dst->src[2]; // weights
|
||||
const ggml_tensor * m = dst->src[3]; // mask
|
||||
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT( q->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT( w->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT( m->type == GGML_TYPE_F16);
|
||||
|
||||
GGML_TENSOR_LOCALS(int64_t, neq, q, ne)
|
||||
GGML_TENSOR_LOCALS(size_t, nbq, q, nb)
|
||||
GGML_TENSOR_LOCALS(int64_t, nek, k, ne)
|
||||
GGML_TENSOR_LOCALS(size_t, nbk, k, nb)
|
||||
GGML_TENSOR_LOCALS(int64_t, new, w, ne)
|
||||
GGML_TENSOR_LOCALS(size_t, nbw, w, nb)
|
||||
GGML_TENSOR_LOCALS(int64_t, nem, m, ne)
|
||||
GGML_TENSOR_LOCALS(size_t, nbm, m, nb)
|
||||
GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
|
||||
GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
|
||||
|
||||
GGML_ASSERT( nb0 == ggml_type_size(dst->type));
|
||||
GGML_ASSERT(nbq0 == ggml_type_size( q->type));
|
||||
GGML_ASSERT(nbk0 == ggml_type_size( k->type));
|
||||
GGML_ASSERT(nbw0 == ggml_type_size( w->type));
|
||||
GGML_ASSERT(nbm0 == ggml_type_size( m->type));
|
||||
|
||||
const int n_embd = q->ne[0];
|
||||
const int n_head = q->ne[1];
|
||||
const int n_tokens = q->ne[2];
|
||||
const int n_stream = q->ne[3];
|
||||
const int n_kv = k->ne[2];
|
||||
|
||||
ggml_to_float_t const k_to_float = ggml_get_type_traits(k->type)->to_float;
|
||||
GGML_ASSERT((k->type == GGML_TYPE_F32 || k_to_float) && "lightning indexer: unsupported K-type");
|
||||
|
||||
const int nr = n_kv;
|
||||
const int ith = params->ith;
|
||||
const int nth = params->nth;
|
||||
|
||||
// (temporary) buffer for K converted to float
|
||||
float * k_row_f32 = (float *) params->wdata + ith*(1*n_embd + CACHE_LINE_SIZE_F32);
|
||||
|
||||
// rows per thread
|
||||
const int dr = (nr + nth - 1)/nth;
|
||||
|
||||
// row range for this thread
|
||||
const int ir0 = dr*ith;
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
for (int s = 0; s < n_stream; ++s) {
|
||||
for (int t = 0; t < n_tokens; ++t) {
|
||||
const float * w_row = (float *) ((char *) w->data + t*nbw1 + s*nbw3);
|
||||
const ggml_fp16_t * m_row = (ggml_fp16_t *) ((char *) m->data + t*nbm1 + (s%nem3)*nbm3);
|
||||
float * dst_row = (float *) ((char *) dst->data + t*nb1 + s*nb3 );
|
||||
for (int ik = ir0; ik < ir1; ++ik) {
|
||||
char * k_row = (char *) k->data + ik*nbk2 + s*nbk3;
|
||||
if (k_to_float) {
|
||||
k_to_float(k_row, k_row_f32, n_embd);
|
||||
} else {
|
||||
k_row_f32 = (float *) k_row;
|
||||
}
|
||||
float score = 0.0f;
|
||||
for (int h = 0; h < n_head; ++h) {
|
||||
// dot product of q and k for head h
|
||||
float qk = 0.0f;
|
||||
const float * q_row = (float *) ((char *) q->data + h*nbq1 + t*nbq2 + s*nbq3);
|
||||
ggml_vec_dot_f32(n_embd, &qk, 0, q_row, 0, k_row_f32, 0, 1);
|
||||
// ReLU and weights (prescaled)
|
||||
score += MAX(qk, 0.0f) * w_row[h];
|
||||
}
|
||||
// apply mask
|
||||
dst_row[ik] = score + GGML_CPU_FP16_TO_FP32(m_row[ik]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ void ggml_compute_forward_rwkv_wkv7(const struct ggml_compute_params * params, s
|
|||
void ggml_compute_forward_solve_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_gla(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_gated_delta_net(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_lightning_indexer(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_map_custom1(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_map_custom2(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_map_custom3(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
|
|
|
|||
|
|
@ -4506,7 +4506,14 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k
|
|||
static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
|
||||
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
|
||||
ggml_cuda_set_device(ctx->device);
|
||||
CUDA_CHECK(cudaMemGetInfo(free, total));
|
||||
cudaError_t err = cudaMemGetInfo(free, total);
|
||||
if (err != cudaSuccess) {
|
||||
(void)cudaGetLastError();
|
||||
GGML_LOG_WARN("%s: cudaMemGetInfo failed (%s), returning 0/0\n", __func__, cudaGetErrorString(err));
|
||||
*free = 0;
|
||||
*total = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/17368
|
||||
#if defined(__linux__)
|
||||
|
|
|
|||
|
|
@ -6524,6 +6524,14 @@ static vk_device ggml_vk_get_device(size_t idx) {
|
|||
device->mul_mat_id_m[i] = true;
|
||||
device->mul_mat_id_s[i] = false;
|
||||
break;
|
||||
case VK_VENDOR_ID_QUALCOMM:
|
||||
device->mul_mat_l[i] = false;
|
||||
device->mul_mat_m[i] = true;
|
||||
device->mul_mat_s[i] = true;
|
||||
device->mul_mat_id_l[i] = false;
|
||||
device->mul_mat_id_m[i] = true;
|
||||
device->mul_mat_id_s[i] = true;
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
device->mul_mat_l[i] = true;
|
||||
|
|
|
|||
|
|
@ -1095,6 +1095,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
|
|||
"RWKV_WKV7",
|
||||
"SOLVE_TRI",
|
||||
"GATED_DELTA_NET",
|
||||
"LIGHTNING_INDEXER",
|
||||
|
||||
"UNARY",
|
||||
|
||||
|
|
@ -1112,7 +1113,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
|
|||
"GLU",
|
||||
};
|
||||
|
||||
static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97");
|
||||
static_assert(GGML_OP_COUNT == 98, "GGML_OP_COUNT != 98");
|
||||
|
||||
static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
||||
"none",
|
||||
|
|
@ -1206,6 +1207,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
|||
"rwkv_wkv7(r, w, k, v, a, b, s)",
|
||||
"A X = B, A triangular, solve X",
|
||||
"gated_delta_net(q, k, v, g, beta, s)",
|
||||
"lightning_indexer(q, k, weights, mask)",
|
||||
|
||||
"unary(x)",
|
||||
|
||||
|
|
@ -1223,7 +1225,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
|||
"glu(x)",
|
||||
};
|
||||
|
||||
static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97");
|
||||
static_assert(GGML_OP_COUNT == 98, "GGML_OP_COUNT != 98");
|
||||
|
||||
static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
|
||||
|
||||
|
|
@ -6303,6 +6305,42 @@ struct ggml_tensor * ggml_gated_delta_net(
|
|||
return result;
|
||||
}
|
||||
|
||||
// ggml_lightning_indexer
|
||||
|
||||
struct ggml_tensor * ggml_lightning_indexer(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * q,
|
||||
struct ggml_tensor * k,
|
||||
struct ggml_tensor * weights,
|
||||
struct ggml_tensor * mask) {
|
||||
|
||||
GGML_ASSERT( q->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT( weights->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT( mask->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT( q->ne[0] == k->ne[0]);
|
||||
GGML_ASSERT( mask->ne[0] == k->ne[2]);
|
||||
GGML_ASSERT( q->ne[1] == weights->ne[0]);
|
||||
GGML_ASSERT( k->ne[1] == 1);
|
||||
GGML_ASSERT( mask->ne[1] == q->ne[2]);
|
||||
GGML_ASSERT( q->ne[2] == weights->ne[1]);
|
||||
GGML_ASSERT(weights->ne[2] == 1);
|
||||
GGML_ASSERT( mask->ne[2] == 1);
|
||||
GGML_ASSERT( q->ne[3] == k->ne[3]);
|
||||
GGML_ASSERT( k->ne[3] == weights->ne[3]);
|
||||
GGML_ASSERT(weights->ne[3] % mask->ne[3] == 0);
|
||||
|
||||
int64_t ne[4] = { k->ne[2], q->ne[2], 1, q->ne[3] };
|
||||
struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
|
||||
|
||||
result->op = GGML_OP_LIGHTNING_INDEXER;
|
||||
result->src[0] = q;
|
||||
result->src[1] = k;
|
||||
result->src[2] = weights;
|
||||
result->src[3] = mask;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct ggml_hash_set ggml_hash_set_new(size_t size) {
|
||||
|
|
|
|||
|
|
@ -587,6 +587,10 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr
|
|||
GGML_LOG_ERROR("%s: encountered bad_alloc error while reading key %" PRIi64 "\n", __func__, i);
|
||||
ok = false;
|
||||
}
|
||||
if (ok && key.empty()) {
|
||||
GGML_LOG_ERROR("%s: key %" PRIi64 " is empty\n", __func__, i);
|
||||
ok = false;
|
||||
}
|
||||
for (size_t j = 0; ok && j < ctx->kv.size(); ++j) {
|
||||
if (key == ctx->kv[j].key) {
|
||||
GGML_LOG_ERROR("%s: duplicate key '%s' for tensors %zu and %" PRIi64 " \n", __func__, key.c_str(), j, i);
|
||||
|
|
@ -1226,6 +1230,11 @@ const char * gguf_get_tensor_name(const struct gguf_context * ctx, int64_t tenso
|
|||
return ctx->info[tensor_id].t.name;
|
||||
}
|
||||
|
||||
const int64_t * gguf_get_tensor_ne(const struct gguf_context * ctx, int64_t tensor_id) {
|
||||
GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
|
||||
return ctx->info[tensor_id].t.ne;
|
||||
}
|
||||
|
||||
enum ggml_type gguf_get_tensor_type(const struct gguf_context * ctx, int64_t tensor_id) {
|
||||
GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
|
||||
return ctx->info[tensor_id].t.type;
|
||||
|
|
|
|||
|
|
@ -58,6 +58,12 @@ static const llm_fused_op_probe llm_fused_op_gdn_ch_probe = {
|
|||
/*.n_tokens_per_seq =*/ 16,
|
||||
};
|
||||
|
||||
static const llm_fused_op_probe llm_fused_op_lid_probe = {
|
||||
/*.op =*/ LLM_FUSED_OP_LIGHTNING_INDEXER,
|
||||
/*.name =*/ "Lightning Indexer",
|
||||
/*.n_tokens_per_seq =*/ 1,
|
||||
};
|
||||
|
||||
llama_context::llama_context(
|
||||
const llama_model & model,
|
||||
llama_context_params params) :
|
||||
|
|
@ -229,6 +235,9 @@ llama_context::llama_context(
|
|||
cparams.fused_gdn_ch = true;
|
||||
cparams.auto_fgdn = true;
|
||||
|
||||
cparams.fused_lid = true;
|
||||
cparams.auto_flid = true;
|
||||
|
||||
// with causal attention, the batch size is limited by the context size
|
||||
cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;
|
||||
|
||||
|
|
@ -530,6 +539,12 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3
|
|||
resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch);
|
||||
cparams.auto_fgdn = false;
|
||||
}
|
||||
|
||||
if (cparams.auto_flid) {
|
||||
LLAMA_LOG_INFO("%s: resolving fused Lightning Indexer support:\n", func);
|
||||
resolve(llm_fused_op_lid_probe, cparams.fused_lid);
|
||||
cparams.auto_flid = false;
|
||||
}
|
||||
}
|
||||
|
||||
void llama_context::sched_reserve() {
|
||||
|
|
|
|||
|
|
@ -41,6 +41,8 @@ struct llama_cparams {
|
|||
bool fused_gdn_ar; // use fused gated delta net (autoregressive)
|
||||
bool fused_gdn_ch; // use fused gated delta net (chunked)
|
||||
bool auto_fgdn;
|
||||
bool fused_lid; // use fused lightning indexer
|
||||
bool auto_flid;
|
||||
bool no_perf;
|
||||
bool warmup; // TODO: remove [TAG_LLAMA_GRAPH_NO_WARMUP]
|
||||
bool op_offload;
|
||||
|
|
|
|||
|
|
@ -843,7 +843,7 @@ static void dsv4_build_comp_inputs(
|
|||
GGML_ASSERT(n_stream > 0);
|
||||
GGML_ASSERT(n_tokens%n_stream == 0);
|
||||
|
||||
inp.kq_mask = ggml_new_tensor_4d(ctx, cparams.flash_attn && strcmp(name, "lid") != 0 ? GGML_TYPE_F16 : GGML_TYPE_F32, plan.n_kv, n_tokens/n_stream, 1, n_stream);
|
||||
inp.kq_mask = ggml_new_tensor_4d(ctx, (strcmp(name, "lid") != 0 && cparams.flash_attn) || (strcmp(name, "lid") == 0 && cparams.fused_lid) ? GGML_TYPE_F16 : GGML_TYPE_F32, plan.n_kv, n_tokens/n_stream, 1, n_stream);
|
||||
ggml_set_input(inp.kq_mask);
|
||||
ggml_set_name(inp.kq_mask, (std::string("dsv4_") + name + "_kq_mask").c_str());
|
||||
}
|
||||
|
|
@ -3026,9 +3026,9 @@ llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const {
|
|||
{
|
||||
inp->self_k_idxs_lid = mctx_cur->get_lid()->build_input_k_idxs(ctx0, ubatch);
|
||||
|
||||
// ensure F32 mask
|
||||
// ensure that mask type matches fused lightning indexer use (requires f16 mask)
|
||||
auto cparams_copy = cparams;
|
||||
cparams_copy.flash_attn = false;
|
||||
cparams_copy.flash_attn = cparams.fused_lid;
|
||||
|
||||
inp->self_kq_mask_lid = build_attn_inp_kq_mask(ctx0, mctx_cur->get_lid(), ubatch, cparams_copy);
|
||||
inp->self_kq_mask_lid_cnv = inp->self_kq_mask_lid;
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ enum llm_fused_op {
|
|||
LLM_FUSED_OP_FLASH_ATTN,
|
||||
LLM_FUSED_OP_GDN_AR,
|
||||
LLM_FUSED_OP_GDN_CH,
|
||||
LLM_FUSED_OP_LIGHTNING_INDEXER,
|
||||
};
|
||||
|
||||
enum llm_ffn_op_type : int {
|
||||
|
|
|
|||
|
|
@ -29,6 +29,15 @@ static uint32_t dsv4_comp_size(uint32_t kv_size, uint32_t ratio) {
|
|||
return std::max<uint32_t>(1, (kv_size + ratio - 1)/ratio);
|
||||
}
|
||||
|
||||
static void dsv4_clear_tensor_stream(ggml_tensor * tensor, uint32_t stream) {
|
||||
GGML_ASSERT(ggml_is_contiguous(tensor));
|
||||
GGML_ASSERT(tensor->ne[3] == 1);
|
||||
GGML_ASSERT(stream < (uint32_t) tensor->ne[2]);
|
||||
|
||||
const size_t stream_size = tensor->nb[2];
|
||||
ggml_backend_tensor_memset(tensor, 0, stream*stream_size, stream_size);
|
||||
}
|
||||
|
||||
static int64_t dsv4_stream_offset(uint32_t n_stream, llama_seq_id seq_id, uint32_t size) {
|
||||
if (n_stream <= 1) {
|
||||
return 0;
|
||||
|
|
@ -781,11 +790,20 @@ llama_dsv4_comp_state::llama_dsv4_comp_state(
|
|||
__func__, name, ratio, state_size, n_embd_state, n_stream, layers.size(), total_size()/1024.0/1024.0);
|
||||
}
|
||||
|
||||
void llama_dsv4_comp_state::clear(bool data) {
|
||||
void llama_dsv4_comp_state::clear(llama_seq_id seq_id, bool data) {
|
||||
if (!data) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (seq_id >= 0) {
|
||||
GGML_ASSERT((uint32_t) seq_id < n_stream);
|
||||
for (const auto & layer : layers) {
|
||||
dsv4_clear_tensor_stream(layer.kv, (uint32_t) seq_id);
|
||||
dsv4_clear_tensor_stream(layer.score, (uint32_t) seq_id);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto & [_, buf] : ctxs_bufs) {
|
||||
ggml_backend_buffer_clear(buf.get(), 0);
|
||||
}
|
||||
|
|
@ -1034,7 +1052,7 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
|
|||
// graph does not necessarily overwrite; uninitialized buffer contents would
|
||||
// otherwise leak in (instance-specific garbage) and corrupt recall. Zero all
|
||||
// compressed buffers up front so reads of un-written rows are deterministic.
|
||||
clear_compressed(true);
|
||||
clear_compressed(-1, true);
|
||||
}
|
||||
|
||||
llama_memory_context_ptr llama_kv_cache_dsv4::init_batch(
|
||||
|
|
@ -1147,7 +1165,7 @@ bool llama_kv_cache_dsv4::get_can_shift() const {
|
|||
|
||||
void llama_kv_cache_dsv4::clear(bool data) {
|
||||
kv_raw->clear(data);
|
||||
clear_compressed(true); // DSV4 compressed buffers must never expose stale/uninit rows
|
||||
clear_compressed(-1, true); // DSV4 compressed buffers must never expose stale/uninit rows
|
||||
}
|
||||
|
||||
bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
|
||||
|
|
@ -1169,7 +1187,7 @@ bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1
|
|||
const bool res = kv_raw->seq_rm(seq_id, p0, p1);
|
||||
|
||||
if (res) {
|
||||
clear_compressed(true);
|
||||
clear_compressed(seq_id, true);
|
||||
}
|
||||
|
||||
return res;
|
||||
|
|
@ -1177,22 +1195,29 @@ bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1
|
|||
|
||||
void llama_kv_cache_dsv4::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
|
||||
kv_raw->seq_cp(seq_id_src, seq_id_dst, p0, p1);
|
||||
clear_compressed(true);
|
||||
}
|
||||
|
||||
void llama_kv_cache_dsv4::seq_keep(llama_seq_id seq_id) {
|
||||
GGML_ASSERT(seq_id >= 0 && (uint32_t) seq_id < n_seq_max);
|
||||
|
||||
kv_raw->seq_keep(seq_id);
|
||||
clear_compressed(true);
|
||||
|
||||
for (llama_seq_id id = 0; id < (llama_seq_id) n_seq_max; ++id) {
|
||||
if (id == seq_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
kv_raw->seq_rm(id, -1, -1);
|
||||
clear_compressed(id, true);
|
||||
}
|
||||
}
|
||||
|
||||
void llama_kv_cache_dsv4::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {
|
||||
kv_raw->seq_add(seq_id, p0, p1, shift);
|
||||
clear_compressed(true);
|
||||
}
|
||||
|
||||
void llama_kv_cache_dsv4::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {
|
||||
kv_raw->seq_div(seq_id, p0, p1, d);
|
||||
clear_compressed(true);
|
||||
}
|
||||
|
||||
llama_pos llama_kv_cache_dsv4::seq_pos_min(llama_seq_id seq_id) const {
|
||||
|
|
@ -1328,13 +1353,32 @@ llama_dsv4_comp_state * llama_kv_cache_dsv4::get_lid_state() const {
|
|||
return lid_state.get();
|
||||
}
|
||||
|
||||
void llama_kv_cache_dsv4::clear_compressed(bool data) {
|
||||
kv_csa->clear(data);
|
||||
kv_hca->clear(data);
|
||||
kv_lid->clear(data);
|
||||
csa_state->clear(data);
|
||||
hca_state->clear(data);
|
||||
lid_state->clear(data);
|
||||
void llama_kv_cache_dsv4::clear_compressed(llama_seq_id seq_id, bool data) {
|
||||
if (seq_id < 0) {
|
||||
kv_csa->clear(data);
|
||||
kv_hca->clear(data);
|
||||
kv_lid->clear(data);
|
||||
} else {
|
||||
GGML_ASSERT((uint32_t) seq_id < n_seq_max);
|
||||
|
||||
const auto clear_seq = [seq_id, data](llama_kv_cache * kv) {
|
||||
kv->seq_rm(seq_id, -1, -1);
|
||||
|
||||
if (data) {
|
||||
for (uint32_t il : kv->get_layer_ids()) {
|
||||
dsv4_clear_tensor_stream(kv->get_k_storage(il), (uint32_t) seq_id);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
clear_seq(kv_csa.get());
|
||||
clear_seq(kv_hca.get());
|
||||
clear_seq(kv_lid.get());
|
||||
}
|
||||
|
||||
csa_state->clear(seq_id, data);
|
||||
hca_state->clear(seq_id, data);
|
||||
lid_state->clear(seq_id, data);
|
||||
}
|
||||
|
||||
//
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ public:
|
|||
const char * name,
|
||||
const llama_memory_i::layer_filter_cb & filter);
|
||||
|
||||
void clear(bool data);
|
||||
void clear(llama_seq_id seq_id, bool data);
|
||||
|
||||
uint32_t get_ratio() const;
|
||||
uint32_t get_state_size() const;
|
||||
|
|
@ -67,6 +67,8 @@ private:
|
|||
// DSV4 uses a normal raw/SWA token cache plus compressed K-only block caches.
|
||||
// The compressed caches are storage only; DSV4-specific visibility and block
|
||||
// planning are handled by llama_kv_cache_dsv4_context / llm_graph_input_dsv4.
|
||||
// FIXME: currently the cache only supports non-unified mode even if unified flag is passed
|
||||
// FIXME: we currently conflate token_pos and buffer contents. See https://github.com/ggml-org/llama.cpp/pull/25521#discussion_r3558173819
|
||||
|
||||
class llama_kv_cache_dsv4 : public llama_memory_i {
|
||||
public:
|
||||
|
|
@ -146,7 +148,7 @@ private:
|
|||
std::unique_ptr<llama_dsv4_comp_state> hca_state;
|
||||
std::unique_ptr<llama_dsv4_comp_state> lid_state;
|
||||
|
||||
void clear_compressed(bool data);
|
||||
void clear_compressed(llama_seq_id seq_id, bool data);
|
||||
};
|
||||
|
||||
// DSV4 raw attention only uses the SWA half of kv_raw. The base half is kept
|
||||
|
|
|
|||
|
|
@ -451,8 +451,7 @@ llama_model * llama_model_create(llm_arch arch, const llama_model_params & param
|
|||
|
||||
if (model != nullptr) {
|
||||
model->arch = arch;
|
||||
auto & devices = model->devices;
|
||||
if (!devices.empty() && devices[0].is_meta && !llm_arch_supports_sm_tensor(arch)) {
|
||||
if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR && !llm_arch_supports_sm_tensor(arch)) {
|
||||
throw std::runtime_error(std::string("LLAMA_SPLIT_MODE_TENSOR not implemented for architecture '") + llm_arch_name(arch) + "'");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -301,43 +301,50 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_
|
|||
indexer_q = ggml_view_4d(ctx0, indexer_q, indexer_q->ne[0], indexer_q->ne[1], indexer_q->ne[2]/n_stream, n_stream, indexer_q->nb[1], indexer_q->nb[2], indexer_q->nb[3]/n_stream, 0);
|
||||
indexer_weights = ggml_view_4d(ctx0, indexer_weights, indexer_weights->ne[0], indexer_weights->ne[1]/n_stream, indexer_weights->ne[2], n_stream, indexer_weights->nb[1], indexer_weights->nb[2]/n_stream, indexer_weights->nb[3]/n_stream, 0);
|
||||
|
||||
// calculate indexer kq
|
||||
indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
|
||||
cb(indexer_q, "indexer_q", il);
|
||||
indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
|
||||
cb(indexer_k, "indexer_k", il);
|
||||
|
||||
ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
|
||||
cb(indexer_kq, "indexer_kq", il);
|
||||
|
||||
// ReLU requires contiguous tensors
|
||||
indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
|
||||
cb(indexer_kq, "indexer_kq", il);
|
||||
|
||||
// apply ReLU
|
||||
ggml_tensor * indexer_score = ggml_relu(ctx0, indexer_kq);
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
|
||||
// pre-scale weights to avoid scaling operations on huge indexer_score tensor
|
||||
indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f / sqrtf(float(n_embd_indexer_head * n_indexer_head)));
|
||||
cb(indexer_weights, "indexer_weights", il);
|
||||
|
||||
// multiply scores by indexer weights
|
||||
indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
ggml_tensor * indexer_score = nullptr;
|
||||
if (cparams.fused_lid) {
|
||||
indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->get_kq_mask_lid());
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
|
||||
} else {
|
||||
// calculate indexer kq
|
||||
indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
|
||||
cb(indexer_q, "indexer_q", il);
|
||||
indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
|
||||
cb(indexer_k, "indexer_k", il);
|
||||
|
||||
// sum by q n_indexer_head dimension
|
||||
indexer_score = ggml_sum_rows(ctx0, indexer_score);
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
|
||||
cb(indexer_kq, "indexer_kq", il);
|
||||
|
||||
// permute result to match KQ mask
|
||||
indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
// ReLU requires contiguous tensors
|
||||
indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
|
||||
cb(indexer_kq, "indexer_kq", il);
|
||||
|
||||
// mask indexer scores
|
||||
ggml_tensor * indexer_kq_mask = inp_attn_dsa->get_kq_mask_lid();
|
||||
indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
// apply ReLU
|
||||
indexer_score = ggml_relu(ctx0, indexer_kq);
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
|
||||
// multiply scores by indexer weights
|
||||
indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
|
||||
// sum by q n_indexer_head dimension
|
||||
indexer_score = ggml_sum_rows(ctx0, indexer_score);
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
|
||||
// permute result to match KQ mask
|
||||
indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
|
||||
// mask indexer scores
|
||||
ggml_tensor * indexer_kq_mask = inp_attn_dsa->get_kq_mask_lid();
|
||||
indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask);
|
||||
cb(indexer_score, "indexer_score", il);
|
||||
}
|
||||
|
||||
// get indices of top k indexer scores
|
||||
uint32_t n_top_k = indexer_score->ne[0] < n_indexer_top_k ? indexer_score->ne[0] : n_indexer_top_k;
|
||||
|
|
|
|||
|
|
@ -556,25 +556,32 @@ ggml_tensor * llama_model_deepseek4::graph::build_lid_top_k(
|
|||
indexer_weights->ne[0], indexer_weights->ne[1]/n_stream, indexer_weights->ne[2], n_stream,
|
||||
indexer_weights->nb[1], indexer_weights->nb[2]/n_stream, indexer_weights->nb[3]/n_stream, 0);
|
||||
|
||||
indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
|
||||
cb(indexer_q, "lid_q", il);
|
||||
indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
|
||||
cb(indexer_k, "lid_k", il);
|
||||
ggml_tensor * indexer_score = nullptr;
|
||||
if (cparams.fused_lid) {
|
||||
indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_lid.kq_mask);
|
||||
cb(indexer_score, "lid_score_masked", il);
|
||||
res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il});
|
||||
} else {
|
||||
indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3);
|
||||
cb(indexer_q, "lid_q", il);
|
||||
indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3);
|
||||
cb(indexer_k, "lid_k", il);
|
||||
|
||||
ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
|
||||
cb(indexer_kq, "lid_kq", il);
|
||||
ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q);
|
||||
cb(indexer_kq, "lid_kq", il);
|
||||
|
||||
indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
|
||||
cb(indexer_kq, "lid_kq", il);
|
||||
indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3));
|
||||
cb(indexer_kq, "lid_kq", il);
|
||||
|
||||
ggml_tensor * indexer_score = ggml_relu(ctx0, indexer_kq);
|
||||
indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
|
||||
indexer_score = ggml_sum_rows(ctx0, indexer_score);
|
||||
indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
|
||||
cb(indexer_score, "lid_score", il);
|
||||
indexer_score = ggml_relu(ctx0, indexer_kq);
|
||||
indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights);
|
||||
indexer_score = ggml_sum_rows(ctx0, indexer_score);
|
||||
indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3));
|
||||
cb(indexer_score, "lid_score", il);
|
||||
|
||||
indexer_score = ggml_add(ctx0, indexer_score, inp_lid.kq_mask);
|
||||
cb(indexer_score, "lid_score_masked", il);
|
||||
indexer_score = ggml_add(ctx0, indexer_score, inp_lid.kq_mask);
|
||||
cb(indexer_score, "lid_score_masked", il);
|
||||
}
|
||||
|
||||
const uint32_t n_top_k = indexer_score->ne[0] < hparams.indexer_top_k ? indexer_score->ne[0] : hparams.indexer_top_k;
|
||||
ggml_tensor * top_k = ggml_cont(ctx0, ggml_top_k(ctx0, indexer_score, n_top_k));
|
||||
|
|
|
|||
|
|
@ -250,7 +250,8 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
|
|||
LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str());
|
||||
|
||||
mtmd_input_text text;
|
||||
text.text = formatted_chat.c_str();
|
||||
text.text = formatted_chat.data();
|
||||
text.text_len = formatted_chat.size();
|
||||
text.add_special = add_bos;
|
||||
text.parse_special = true;
|
||||
|
||||
|
|
|
|||
|
|
@ -809,7 +809,7 @@ void mtmd_free(mtmd_context * ctx) {
|
|||
struct mtmd_tokenizer {
|
||||
mtmd_context * ctx;
|
||||
|
||||
std::string input_text;
|
||||
std::string input_text; // note: can contain null bytes; do not use c_str()
|
||||
bool add_special;
|
||||
bool parse_special;
|
||||
const llama_vocab * vocab;
|
||||
|
|
@ -839,9 +839,10 @@ struct mtmd_tokenizer {
|
|||
size_t n_bitmaps) : ctx(ctx) {
|
||||
add_special = text->add_special;
|
||||
parse_special = text->parse_special;
|
||||
input_text = text->text;
|
||||
vocab = ctx->vocab;
|
||||
|
||||
input_text.assign(text->text, text->text_len);
|
||||
|
||||
std::vector<const mtmd_bitmap *> bitmaps(bmps, bmps + n_bitmaps);
|
||||
auto parts_str = split_text(input_text, ctx->media_marker);
|
||||
size_t i_bm = 0;
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ struct mtmd_batch;
|
|||
|
||||
struct mtmd_input_text {
|
||||
const char * text;
|
||||
size_t text_len;
|
||||
bool add_special;
|
||||
bool parse_special;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -126,15 +126,15 @@ It is opt in via the `X-Conversation-Id` header on `POST /v1/chat/completions`.
|
|||
|
||||
The feature lives entirely in `server-stream.{h,cpp}` and rests on three types:
|
||||
|
||||
- `stream_session`: a bounded ring buffer (4 MiB cap, oldest bytes drop first) plus a condvar. `append` pushes raw SSE bytes, `read_from` drains from any offset and blocks for live bytes or finalize, `finalize` wakes readers, `cancel` stops the producer. One conv maps to at most one live session.
|
||||
- `stream_session`: a bounded ring buffer (4 MiB cap, oldest bytes drop first) plus a condvar. `append` pushes raw SSE bytes, `read_from` drains from any offset and blocks for live bytes or finalize, `finalize` wakes readers, `cancel` sets the flag the producer polls. One conv maps to at most one live session.
|
||||
- `stream_session_manager`: a file-static singleton (`g_stream_sessions`) inside `server-stream.cpp`, owns all sessions keyed by conv id, enforces the one conv one session invariant via `create_or_replace`, and runs a GC thread that drops completed sessions past their TTL. Exposed to main only through `server_stream_session_manager_start/stop`.
|
||||
- `stream_pipe_producer` / `stream_pipe_consumer`: the write and read ends. The producer owns the session lifetime and finalizes it on destruction; the consumer is read only and never finalizes, so a reader detaching cannot kill a running generation.
|
||||
|
||||
The implementation is hidden in `server-stream.cpp` (pimpl). The header exposes only the route handler factories, `server_stream_session_attach_pipe`, `server_stream_aware_should_stop`, `server_stream_conv_id_from_headers` and the GC lifecycle; the session, manager and consumer types stay in the `.cpp`.
|
||||
The implementation is hidden in `server-stream.cpp` (pimpl). The header exposes only the route handler factories, the `server_res_spipe` response base, `server_stream_conv_id_from_headers` and the GC lifecycle; the session, manager, consumer and the `server_stream_create_spipe` factory stay in the `.cpp`.
|
||||
|
||||
Producer side: `server_res_generator` attaches a producer pipe when the header is present. The HTTP content provider mirrors every chunk into the ring before writing it to the socket. While a pipe is attached, `server_stream_aware_should_stop` ignores peer disconnect, so a dropped socket does not stop generation: only an explicit `DELETE` does. When the peer leaves early, `on_complete` calls `close()`, which drains the rest of the generation into the ring on the http worker.
|
||||
Producer side: `server_res_generator` extends `server_res_spipe`, which keeps all spipe logic out of the generic `server_http_res`. `set_req` attaches a producer when the header is present, and the wrapped `next` tees each chunk into the ring before the socket, so a chunk lost to a dead wire is already buffered. While attached, `should_stop` ignores peer disconnect: only a `DELETE` stops generation. On an early peer drop, `on_complete` drains the tail into the ring on the http worker.
|
||||
|
||||
Lifetime safety: the producer pipe holds a shared `alive` flag also captured by the session cancel hook. `~server_res_generator` calls `cleanup()` to clear that hook while the reader is still alive, so a `cancel` arriving during teardown can never call `stop()` on a freed response. This ordering is the most fragile part of the feature: finalizing or destroying the producer before `cleanup()` runs reintroduces a use after free.
|
||||
Lifetime safety: the session holds no back reference to the response, so `spipe` is a plain `unique_ptr` touched only by the http worker. `cancel` raises an atomic the producer polls; the producer finalizes the session from its destructor, which also runs `~server_response_reader::stop()` to cancel the generation at the queue level. A `DELETE` stops work by raising the flag and letting the worker unwind.
|
||||
|
||||
Consumer side: `GET /v1/stream/<conv_id>?from=N` opens a `text/event-stream` that replays buffered bytes from offset `N` and blocks for live bytes, so the browser reattaches like a fresh EventSource. An offset below the dropped prefix returns 400.
|
||||
|
||||
|
|
@ -235,6 +235,29 @@ That requires `JSON.stringify` when formatted to message content:
|
|||
}
|
||||
```
|
||||
|
||||
Set `stream: true` in the request body to stream a tool's output as it runs, instead of waiting for it to finish. Only certain tools accept this (for ex. `exec_shell_command`);
|
||||
returns 404 if tool doesn't support it.
|
||||
|
||||
Response is SSE stream, one `data: <json>` line per chunk:
|
||||
|
||||
```json
|
||||
{"chunk": "hello\n"}
|
||||
```
|
||||
|
||||
followed by a final event once the tool returns:
|
||||
|
||||
```json
|
||||
{"done": true}
|
||||
```
|
||||
|
||||
or, if `invoke()` threw:
|
||||
|
||||
```json
|
||||
{"done": true, "error": "..."}
|
||||
```
|
||||
|
||||
There is no `[DONE]` sentinel (unlike `/chat/completions`), the stream ends after the `done`
|
||||
|
||||
### Router mode: how child <--> router communicates
|
||||
|
||||
Upon spawning a new child process using `subprocess`, both child and router listen to the stdout/stderr (combined)
|
||||
|
|
|
|||
|
|
@ -431,22 +431,70 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
|
|||
std::string tool_use_id = json_value(block, "tool_use_id", std::string());
|
||||
|
||||
auto result_content = json_value(block, "content", json());
|
||||
std::string result_text;
|
||||
if (result_content.is_string()) {
|
||||
result_text = result_content.get<std::string>();
|
||||
tool_results.push_back({
|
||||
{"role", "tool"},
|
||||
{"tool_call_id", tool_use_id},
|
||||
{"content", result_content.get<std::string>()}
|
||||
});
|
||||
} else if (result_content.is_array()) {
|
||||
// Single-pass: build both text and content_parts, decide format at the end
|
||||
std::string result_text;
|
||||
json content_parts = json::array();
|
||||
bool has_images = false;
|
||||
|
||||
for (const auto & c : result_content) {
|
||||
if (json_value(c, "type", std::string()) == "text") {
|
||||
result_text += json_value(c, "text", std::string());
|
||||
std::string c_type = json_value(c, "type", std::string());
|
||||
if (c_type == "text") {
|
||||
std::string text = json_value(c, "text", std::string());
|
||||
result_text += text;
|
||||
content_parts.push_back({
|
||||
{"type", "text"},
|
||||
{"text", text}
|
||||
});
|
||||
} else if (c_type == "image") {
|
||||
has_images = true;
|
||||
json source = json_value(c, "source", json::object());
|
||||
std::string source_type = json_value(source, "type", std::string());
|
||||
if (source_type == "base64") {
|
||||
std::string media_type = json_value(source, "media_type", std::string("image/jpeg"));
|
||||
std::string data = json_value(source, "data", std::string());
|
||||
std::string url = "data:" + media_type + ";base64," + data;
|
||||
content_parts.push_back({
|
||||
{"type", "image_url"},
|
||||
{"image_url", {{"url", url}}}
|
||||
});
|
||||
} else if (source_type == "url") {
|
||||
content_parts.push_back({
|
||||
{"type", "image_url"},
|
||||
{"image_url", {{"url", json_value(source, "url", std::string())}}}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tool_results.push_back({
|
||||
{"role", "tool"},
|
||||
{"tool_call_id", tool_use_id},
|
||||
{"content", result_text}
|
||||
});
|
||||
if (!has_images) {
|
||||
// Text-only: collapse to a plain string for maximum compatibility
|
||||
tool_results.push_back({
|
||||
{"role", "tool"},
|
||||
{"tool_call_id", tool_use_id},
|
||||
{"content", result_text}
|
||||
});
|
||||
} else {
|
||||
// Mixed or image-only: use array content parts (OpenAI multimodal tool format)
|
||||
tool_results.push_back({
|
||||
{"role", "tool"},
|
||||
{"tool_call_id", tool_use_id},
|
||||
{"content", content_parts}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
tool_results.push_back({
|
||||
{"role", "tool"},
|
||||
{"tool_call_id", tool_use_id},
|
||||
{"content", ""}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -705,7 +705,8 @@ server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & promp
|
|||
std::vector<server_tokens> inputs;
|
||||
// multimodal
|
||||
mtmd_input_text inp_txt = {
|
||||
prompt.c_str(),
|
||||
prompt.data(),
|
||||
prompt.size(),
|
||||
/* add_special */ true,
|
||||
/* parse_special */ true,
|
||||
};
|
||||
|
|
@ -1116,7 +1117,8 @@ json oaicompat_chat_params_parse(
|
|||
|
||||
// Reasoning budget: pass parameters through to sampling layer
|
||||
{
|
||||
int reasoning_budget = json_value(body, "thinking_budget_tokens", -1);
|
||||
int reasoning_budget = json_value(body, "reasoning_budget_tokens",
|
||||
json_value(body, "thinking_budget_tokens", -1));
|
||||
if (reasoning_budget == -1) {
|
||||
reasoning_budget = opt.reasoning_budget;
|
||||
}
|
||||
|
|
@ -1125,7 +1127,7 @@ json oaicompat_chat_params_parse(
|
|||
llama_params["reasoning_budget_tokens"] = reasoning_budget;
|
||||
llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag;
|
||||
llama_params["reasoning_budget_end_tag"] = chat_params.thinking_end_tag;
|
||||
llama_params["reasoning_budget_message"] = opt.reasoning_budget_message;
|
||||
llama_params["reasoning_budget_message"] = json_value(body, "reasoning_budget_message", opt.reasoning_budget_message);
|
||||
llama_params["reasoning_control"] = json_value(body, "reasoning_control", false);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2290,6 +2290,24 @@ private:
|
|||
|
||||
// n_tokens_cur: the number of tokens added to the batch for the current slot
|
||||
void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) {
|
||||
const int id_task = slot.task->id;
|
||||
|
||||
// evict checkpoints within min-step of a previous checkpoint, unless they were
|
||||
// created by the current task
|
||||
int64_t last = -1;
|
||||
for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) {
|
||||
if (it->id_task != id_task && last >= 0 && it->n_tokens <= last + params_base.checkpoint_min_step) {
|
||||
SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",
|
||||
it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024);
|
||||
|
||||
it = slot.prompt.checkpoints.erase(it);
|
||||
continue;
|
||||
}
|
||||
|
||||
last = it->n_tokens;
|
||||
++it;
|
||||
}
|
||||
|
||||
while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) {
|
||||
// make room for the new checkpoint, if needed
|
||||
const auto & cur = slot.prompt.checkpoints.front();
|
||||
|
|
@ -2302,6 +2320,8 @@ private:
|
|||
|
||||
auto & cur = slot.prompt.checkpoints.emplace_back();
|
||||
|
||||
cur.id_task = id_task;
|
||||
|
||||
// [TAG_CHECKPOINTS_FIX_POS_MIN]
|
||||
// TODO: here we incorrectly deterimne that the saved checkpoint data covers the [pos_min, pos_max] range
|
||||
// this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225
|
||||
|
|
@ -3511,7 +3531,10 @@ private:
|
|||
do_checkpoint = do_checkpoint && !has_mtmd;
|
||||
|
||||
// no need to create checkpoints that are too close together, unless it's the last user message
|
||||
do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || is_last_user_message || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
|
||||
do_checkpoint = do_checkpoint && (
|
||||
slot.prompt.checkpoints.empty() ||
|
||||
is_last_user_message || near_prompt_end ||
|
||||
n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
|
||||
SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max);
|
||||
|
||||
// note: we create the checkpoint before calling llama_decode(), so the current batch is not
|
||||
|
|
@ -3979,11 +4002,9 @@ server_context_meta server_context::get_meta() const {
|
|||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
// generator-like API for HTTP response generation
|
||||
// may have bypass_sleep = true if the task does not use ctx_server
|
||||
struct server_res_generator : server_http_res {
|
||||
struct server_res_generator : server_res_spipe {
|
||||
server_response_reader rd;
|
||||
server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false)
|
||||
: rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) {
|
||||
|
|
@ -3993,15 +4014,6 @@ struct server_res_generator : server_http_res {
|
|||
queue_tasks.wait_until_no_sleep();
|
||||
}
|
||||
}
|
||||
~server_res_generator() override {
|
||||
// cleanup() must run while rd is still alive (rd is destroyed after this body returns)
|
||||
if (spipe) {
|
||||
spipe->cleanup();
|
||||
}
|
||||
}
|
||||
void stop() override {
|
||||
rd.stop();
|
||||
}
|
||||
void ok(const json & response_data) {
|
||||
status = 200;
|
||||
data = safe_json_to_str(response_data);
|
||||
|
|
@ -4039,6 +4051,8 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
|
|||
auto & rd = res->rd;
|
||||
auto & params = this->params;
|
||||
|
||||
res->set_req(&req); // will also set spipe if needed
|
||||
|
||||
int32_t sse_ping_interval = params.sse_ping_interval;
|
||||
|
||||
try {
|
||||
|
|
@ -4181,7 +4195,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
|
|||
}
|
||||
res->status = 200;
|
||||
res->content_type = "text/event-stream";
|
||||
res->next = [res_this = res.get(), res_type, sse_ping_interval, &req](std::string & output) -> bool {
|
||||
res->set_next([res_this = res.get(), res_type, sse_ping_interval](std::string & output) -> bool {
|
||||
static auto format_error = [](task_response_type res_type, const json & res_json) {
|
||||
if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) {
|
||||
return format_anthropic_sse({
|
||||
|
|
@ -4193,7 +4207,9 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
|
|||
}
|
||||
};
|
||||
|
||||
auto effective_should_stop = server_stream_aware_should_stop(res_this, req.should_stop);
|
||||
auto effective_should_stop = [&res_this]() {
|
||||
return res_this->should_stop();
|
||||
};
|
||||
|
||||
try {
|
||||
if (effective_should_stop()) {
|
||||
|
|
@ -4284,13 +4300,9 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
|
|||
// terminate on exception
|
||||
return false;
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// attach a producer pipe to the response when X-Conversation-Id is present.
|
||||
// the pipe mirrors SSE chunks into the ring buffer and wires up the cancel hook.
|
||||
server_stream_session_attach_pipe(*res, req.headers);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
#include "common.h"
|
||||
#include "http.h"
|
||||
#include "server-http.h"
|
||||
#include "server-stream.h"
|
||||
#include "server-common.h"
|
||||
#include "ui.h"
|
||||
|
||||
|
|
@ -530,33 +529,20 @@ static void process_handler_response(server_http_req_ptr && request, server_http
|
|||
std::string chunk;
|
||||
const bool has_next = response->next(chunk);
|
||||
if (!chunk.empty()) {
|
||||
// mirror into the ring buffer first, the session must reflect every SSE chunk
|
||||
// whether or not the wire write below succeeds
|
||||
if (response->spipe) {
|
||||
response->spipe->write(chunk.data(), chunk.size());
|
||||
}
|
||||
if (!sink.write(chunk.data(), chunk.size())) {
|
||||
// peer is gone, stop the wire path here
|
||||
return false;
|
||||
}
|
||||
SRV_DBG("http: streamed chunk: %s\n", chunk.c_str());
|
||||
}
|
||||
if (!has_next) {
|
||||
// producer reached its natural end on the wire, a later close() skips the drain
|
||||
if (response->spipe) {
|
||||
response->spipe->done();
|
||||
}
|
||||
sink.done();
|
||||
SRV_DBG("%s", "http: stream ended\n");
|
||||
}
|
||||
return has_next;
|
||||
};
|
||||
const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable {
|
||||
// on a dropped peer, close() drains the rest of the generation into the ring buffer
|
||||
if (response->spipe) {
|
||||
response->spipe->close();
|
||||
}
|
||||
response.reset(); // spipe destructor finalizes the session if attached
|
||||
response->on_complete();
|
||||
response.reset();
|
||||
request.reset();
|
||||
};
|
||||
res.set_chunked_content_provider(content_type, chunked_content_provider, on_complete);
|
||||
|
|
@ -564,6 +550,7 @@ static void process_handler_response(server_http_req_ptr && request, server_http
|
|||
res.status = response->status;
|
||||
set_headers(res, response->headers);
|
||||
res.set_content(response->data, response->content_type);
|
||||
response->on_complete();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@
|
|||
#include <unordered_map>
|
||||
|
||||
struct common_params;
|
||||
struct stream_pipe_producer; // defined in server-stream.h
|
||||
|
||||
// generator-like API for HTTP response generation
|
||||
// this object response with one of the 2 modes:
|
||||
|
|
@ -25,19 +24,13 @@ struct server_http_res {
|
|||
std::string data;
|
||||
std::map<std::string, std::string> headers;
|
||||
|
||||
// if set, the stream survives a client disconnect: the producer pipe keeps draining into the
|
||||
// ring buffer and finalizes the session on destruction, so no explicit on_stream_end is needed.
|
||||
// shared_ptr (not unique_ptr) so the forward-declared type is safe to delete here.
|
||||
std::shared_ptr<stream_pipe_producer> spipe;
|
||||
|
||||
std::function<bool(std::string &)> next = nullptr;
|
||||
bool is_stream() const {
|
||||
return next != nullptr;
|
||||
}
|
||||
|
||||
// called when the session is cancelled (e.g. DELETE /v1/stream/<conv_id>).
|
||||
// server_res_generator overrides this to stop its reader; the default is a no-op.
|
||||
virtual void stop() {}
|
||||
// fired before req and res are destroyed
|
||||
virtual void on_complete() {}
|
||||
|
||||
virtual ~server_http_res() = default;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -219,13 +219,14 @@ void server_model_meta::update_caps() {
|
|||
"LLAMA_ARG_MODEL_URL",
|
||||
"LLAMA_ARG_MMPROJ",
|
||||
"LLAMA_ARG_MMPROJ_URL",
|
||||
"LLAMA_ARG_MMPROJ_AUTO",
|
||||
"LLAMA_ARG_HF_REPO",
|
||||
"LLAMA_ARG_HF_REPO_FILE",
|
||||
});
|
||||
params.offline = true;
|
||||
common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
|
||||
common_models_handler_apply(handler, params); // note: this won't download the model because offline=true
|
||||
if (params.mmproj.path.empty()) {
|
||||
if (params.no_mmproj || params.mmproj.path.empty()) {
|
||||
multimodal = { false, false };
|
||||
} else {
|
||||
multimodal = mtmd_get_cap_from_file(params.mmproj.path.c_str());
|
||||
|
|
|
|||
|
|
@ -96,8 +96,6 @@ struct stream_session {
|
|||
size_t dropped_prefix() const; // bytes evicted from the front due to cap
|
||||
int64_t completed_at() const; // 0 while alive, unix seconds after finalize
|
||||
|
||||
void set_stop_producer(std::function<void()> fn);
|
||||
|
||||
void cancel();
|
||||
|
||||
private:
|
||||
|
|
@ -109,7 +107,6 @@ private:
|
|||
bool done;
|
||||
std::atomic<bool> cancelled; // polled lock-free by the should_stop closure, no mu
|
||||
int64_t completed_ts;
|
||||
std::function<void()> stop_producer;
|
||||
};
|
||||
stream_session::stream_session(std::string conversation_id_, size_t max_bytes_)
|
||||
: conversation_id(std::move(conversation_id_))
|
||||
|
|
@ -217,26 +214,10 @@ int64_t stream_session::completed_at() const {
|
|||
return completed_ts;
|
||||
}
|
||||
|
||||
void stream_session::set_stop_producer(std::function<void()> fn) {
|
||||
std::lock_guard<std::mutex> lock(mu);
|
||||
stop_producer = std::move(fn);
|
||||
}
|
||||
|
||||
void stream_session::cancel() {
|
||||
// flip cancelled first so the producer-side server_stream_aware_should_stop can break out of the
|
||||
// recv() wait even if remove_waiting_task_ids does not notify the condvar (the cancel task
|
||||
// posted by rd.stop() will eventually notify, but we do not want to depend on that timing)
|
||||
// the should_stop closure on both the producer and any HTTP reader polls is_cancelled()
|
||||
// so flipping this is the only signal needed to unwind both sides
|
||||
cancelled.store(true, std::memory_order_release);
|
||||
// copy the hook under the lock then invoke outside, the producer side may grab queue locks
|
||||
// and we do not want to hold our mu across that path
|
||||
std::function<void()> fn;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mu);
|
||||
fn = stop_producer;
|
||||
}
|
||||
if (fn) {
|
||||
fn();
|
||||
}
|
||||
}
|
||||
|
||||
bool stream_session::is_cancelled() const {
|
||||
|
|
@ -325,8 +306,10 @@ void stream_session_manager::evict_and_cancel(const std::string & conversation_i
|
|||
s = it->second;
|
||||
sessions.erase(it);
|
||||
}
|
||||
// signal the producer side first so the inference is cancelled at the queue level,
|
||||
// then finalize, which wakes any pending HTTP reader and lets the drain exit naturally
|
||||
// cancel first so the producer's on_complete() drain loop and any pending HTTP reader
|
||||
// observe is_cancelled() and stop pulling further output, then finalize to wake readers
|
||||
// blocked in read_from(). note: this does not interrupt the underlying generation itself,
|
||||
// which keeps running to its own natural stop condition (EOS/max_tokens)
|
||||
s->cancel();
|
||||
s->finalize();
|
||||
}
|
||||
|
|
@ -431,65 +414,15 @@ stream_pipe_producer::stream_pipe_producer(stream_session_ptr session)
|
|||
}
|
||||
|
||||
stream_pipe_producer::~stream_pipe_producer() {
|
||||
cleanup();
|
||||
session_->finalize();
|
||||
}
|
||||
|
||||
void stream_pipe_producer::cleanup() {
|
||||
if (!alive_) {
|
||||
return;
|
||||
}
|
||||
alive_->store(false, std::memory_order_release);
|
||||
session_->set_stop_producer(nullptr);
|
||||
alive_.reset();
|
||||
}
|
||||
|
||||
bool stream_pipe_producer::write(const char * data, size_t len) {
|
||||
return session_->append(data, len);
|
||||
}
|
||||
|
||||
void stream_pipe_producer::done() {
|
||||
done_ = true;
|
||||
}
|
||||
|
||||
void stream_pipe_producer::close() {
|
||||
// httplib bails its content provider the moment is_peer_alive() goes false, so pump the rest
|
||||
// of the generation into the ring buffer here. a DELETE flips is_cancelled and cuts it short
|
||||
if (done_ || session_->is_cancelled()) {
|
||||
SRV_TRC("stream_pipe close: skip drain (done=%d cancelled=%d) conv=%s\n",
|
||||
done_ ? 1 : 0, session_->is_cancelled() ? 1 : 0, session_->conversation_id.c_str());
|
||||
return;
|
||||
}
|
||||
SRV_TRC("stream_pipe close: draining conv=%s\n", session_->conversation_id.c_str());
|
||||
size_t drained = 0;
|
||||
std::string chunk;
|
||||
while (true) {
|
||||
chunk.clear();
|
||||
bool has_next = res_->next(chunk);
|
||||
if (!chunk.empty()) {
|
||||
write(chunk.data(), chunk.size());
|
||||
drained += chunk.size();
|
||||
}
|
||||
if (!has_next) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
SRV_TRC("stream_pipe close: drain ended conv=%s bytes=%zu\n", session_->conversation_id.c_str(), drained);
|
||||
}
|
||||
|
||||
std::shared_ptr<stream_pipe_producer> stream_pipe_producer::create(stream_session_ptr session,
|
||||
server_http_res & res) {
|
||||
auto alive = std::make_shared<std::atomic<bool>>(true);
|
||||
auto * res_ptr = &res;
|
||||
session->set_stop_producer([alive, res_ptr]() {
|
||||
if (alive->load(std::memory_order_acquire)) {
|
||||
res_ptr->stop();
|
||||
}
|
||||
});
|
||||
auto pipe = std::shared_ptr<stream_pipe_producer>(new stream_pipe_producer(std::move(session)));
|
||||
pipe->alive_ = std::move(alive);
|
||||
pipe->res_ = res_ptr;
|
||||
return pipe;
|
||||
stream_pipe_producer * stream_pipe_producer::create(stream_session_ptr session) {
|
||||
return new stream_pipe_producer(std::move(session));
|
||||
}
|
||||
|
||||
// stream_pipe_consumer
|
||||
|
|
@ -661,21 +594,68 @@ std::string server_stream_conv_id_from_headers(const std::map<std::string, std::
|
|||
return std::string();
|
||||
}
|
||||
|
||||
void server_stream_session_attach_pipe(server_http_res & res, const std::map<std::string, std::string> & headers) {
|
||||
static stream_pipe_producer * server_stream_create_spipe(const std::map<std::string, std::string> & headers) {
|
||||
std::string conversation_id = server_stream_conv_id_from_headers(headers);
|
||||
SRV_TRC("conv_id=%s (empty=%d)\n", conversation_id.c_str(), conversation_id.empty() ? 1 : 0);
|
||||
if (conversation_id.empty()) {
|
||||
return;
|
||||
return nullptr;
|
||||
}
|
||||
auto session = g_stream_sessions.create_or_replace(conversation_id);
|
||||
res.spipe = stream_pipe_producer::create(session, res);
|
||||
return stream_pipe_producer::create(session);
|
||||
}
|
||||
|
||||
std::function<bool()> server_stream_aware_should_stop(server_http_res * res, std::function<bool()> fallback) {
|
||||
return [res, fallback = std::move(fallback)]() -> bool {
|
||||
if (res->spipe) {
|
||||
return res->spipe->is_cancelled();
|
||||
//
|
||||
// server_res_spipe
|
||||
//
|
||||
|
||||
void server_res_spipe::set_req(const server_http_req * req) {
|
||||
this->req = req;
|
||||
// optionally attach spipe to the response when X-Conversation-Id is present
|
||||
spipe.reset(server_stream_create_spipe(req->headers));
|
||||
}
|
||||
|
||||
bool server_res_spipe::conn_alive() {
|
||||
GGML_ASSERT(req != nullptr);
|
||||
return !req->should_stop();
|
||||
}
|
||||
|
||||
bool server_res_spipe::should_stop() {
|
||||
if (spipe) {
|
||||
// note: if DELETE /v1/stream/<conv_id> is called, is_cancelled() will be true
|
||||
return spipe->is_cancelled();
|
||||
} else {
|
||||
return !conn_alive();
|
||||
}
|
||||
}
|
||||
|
||||
void server_res_spipe::on_complete() {
|
||||
if (!spipe || next_finished) {
|
||||
return;
|
||||
}
|
||||
std::string chunk;
|
||||
while (!spipe->is_cancelled()) {
|
||||
chunk.clear();
|
||||
bool has_next = next_orig(chunk);
|
||||
if (!chunk.empty()) {
|
||||
spipe->write(chunk.data(), chunk.size());
|
||||
}
|
||||
return fallback();
|
||||
if (!has_next) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void server_res_spipe::set_next(std::function<bool(std::string &)> next_fn) {
|
||||
next_orig = std::move(next_fn);
|
||||
next = [this](std::string & out) {
|
||||
bool has_next = next_orig(out);
|
||||
if (spipe) {
|
||||
// if spipe is set, tee-style pipe input to both HTTP and spipe
|
||||
spipe->write(out.data(), out.size());
|
||||
}
|
||||
if (!has_next) {
|
||||
next_finished = true;
|
||||
}
|
||||
return has_next;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,36 +30,15 @@ protected:
|
|||
|
||||
// producer end: writes chunks into the ring buffer and owns the session lifetime, finalizing it
|
||||
// on destruction.
|
||||
//
|
||||
// lifetime safety: holds a shared_ptr<atomic<bool>> alive also captured by the session's
|
||||
// stop_producer hook. cleanup() sets alive=false and clears the hook; it must run while the
|
||||
// response the hook calls stop() on is still alive. ~server_res_generator() does this explicitly.
|
||||
struct stream_pipe_producer : stream_pipe {
|
||||
~stream_pipe_producer() override;
|
||||
|
||||
bool write(const char * data, size_t len);
|
||||
|
||||
// mark the natural end on the wire so a later close() is a no-op
|
||||
void done();
|
||||
|
||||
// on a peer drop, pump the response next() into the ring buffer until done. runs on the http
|
||||
// worker from on_complete, no-op after done() or cancel
|
||||
void close();
|
||||
|
||||
// disarm the stop hook and drop the alive guard, must run while the response the hook
|
||||
// references is still alive. idempotent, the destructor calls it too
|
||||
void cleanup();
|
||||
|
||||
// res.stop() is invoked when the session is cancelled, the alive guard ensures stop() is not
|
||||
// called after cleanup() has run
|
||||
static std::shared_ptr<stream_pipe_producer> create(stream_session_ptr session, server_http_res & res);
|
||||
static stream_pipe_producer * create(stream_session_ptr session);
|
||||
|
||||
private:
|
||||
explicit stream_pipe_producer(stream_session_ptr session);
|
||||
|
||||
bool done_ = false;
|
||||
std::shared_ptr<std::atomic<bool>> alive_;
|
||||
server_http_res * res_ = nullptr;
|
||||
};
|
||||
|
||||
void server_stream_session_manager_start();
|
||||
|
|
@ -73,10 +52,22 @@ server_http_context::handler_t server_stream_make_delete_handler();
|
|||
// extract the X-Conversation-Id header value (case-insensitive), empty when absent
|
||||
std::string server_stream_conv_id_from_headers(const std::map<std::string, std::string> & headers);
|
||||
|
||||
// on an X-Conversation-Id header, create or replace the session and attach a producer pipe to res
|
||||
void server_stream_session_attach_pipe(server_http_res & res, const std::map<std::string, std::string> & headers);
|
||||
// implement tee-style pipe (spipe) for "stream replay" functionality
|
||||
struct server_res_spipe : server_http_res {
|
||||
private:
|
||||
// if set, the stream survives a client disconnect:
|
||||
// connection kept alive, output is forwarded to spipe and reuse later
|
||||
std::unique_ptr<stream_pipe_producer> spipe;
|
||||
// if spipe is set, use this next_orig to implement tee-style pipe
|
||||
std::function<bool(std::string &)> next_orig;
|
||||
const server_http_req * req = nullptr;
|
||||
// set once next_orig reports no more data, so on_complete() doesn't re-drain a finished stream
|
||||
bool next_finished = false;
|
||||
|
||||
// should_stop closure that ignores peer disconnect when a pipe is attached, so only an explicit
|
||||
// DELETE stops the producer and generation keeps flowing into the ring buffer. without a pipe it
|
||||
// delegates to fallback, the legacy non-resumable flow
|
||||
std::function<bool()> server_stream_aware_should_stop(server_http_res * res, std::function<bool()> fallback);
|
||||
public:
|
||||
void set_req(const server_http_req * req);
|
||||
bool conn_alive();
|
||||
bool should_stop();
|
||||
void on_complete() override;
|
||||
void set_next(std::function<bool(std::string &)> next_fn);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
#include <climits>
|
||||
#include <algorithm>
|
||||
#include <unordered_set>
|
||||
#include <functional>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
|
|
@ -51,7 +52,13 @@ public:
|
|||
virtual bool write_file(const std::string & path, const std::string & content) const = 0;
|
||||
// paths relative to `base`, '/'-separated; sets `err` if `base` isn't a directory
|
||||
virtual std::vector<std::string> list_files(const std::string & base, std::string & err) const = 0;
|
||||
virtual exec_result run(const std::vector<std::string> & args, size_t max_output, int timeout_secs) const = 0;
|
||||
// on_chunk, if set, is called with each chunk of output as it is read (before truncation cuts in);
|
||||
// returning false terminates the process early (e.g. the client disconnected)
|
||||
virtual exec_result run(
|
||||
const std::vector<std::string> & args,
|
||||
size_t max_output,
|
||||
int timeout_secs,
|
||||
const std::function<bool(const std::string &)> & on_chunk = nullptr) const = 0;
|
||||
};
|
||||
|
||||
class tools_io_basic : public tools_io {
|
||||
|
|
@ -123,7 +130,11 @@ public:
|
|||
return list_files_fallback(base);
|
||||
}
|
||||
|
||||
exec_result run(const std::vector<std::string> & args, size_t max_output, int timeout_secs) const override {
|
||||
exec_result run(
|
||||
const std::vector<std::string> & args,
|
||||
size_t max_output,
|
||||
int timeout_secs,
|
||||
const std::function<bool(const std::string &)> & on_chunk = nullptr) const override {
|
||||
exec_result res;
|
||||
|
||||
subprocess_s proc;
|
||||
|
|
@ -164,8 +175,14 @@ public:
|
|||
size_t len = strlen(buf);
|
||||
if (output.size() + len <= max_output) {
|
||||
output.append(buf, len);
|
||||
if (on_chunk && !on_chunk(std::string(buf, len))) {
|
||||
subprocess_terminate(&proc);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
output.append(buf, max_output - output.size());
|
||||
size_t remaining = max_output - output.size();
|
||||
output.append(buf, remaining);
|
||||
if (on_chunk && remaining > 0) on_chunk(std::string(buf, remaining));
|
||||
truncated = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -287,7 +304,7 @@ struct server_tool_read_file : server_tool {
|
|||
};
|
||||
}
|
||||
|
||||
json invoke(json params) const override {
|
||||
json invoke(json params, server_tool::stream *) const override {
|
||||
std::string path = params.at("path").get<std::string>();
|
||||
int start_line = json_value(params, "start_line", 1);
|
||||
int end_line = json_value(params, "end_line", -1); // -1 = no limit
|
||||
|
|
@ -376,7 +393,7 @@ struct server_tool_file_glob_search : server_tool {
|
|||
};
|
||||
}
|
||||
|
||||
json invoke(json params) const override {
|
||||
json invoke(json params, server_tool::stream *) const override {
|
||||
std::string base = params.at("path").get<std::string>();
|
||||
std::string include = json_value(params, "include", std::string("**"));
|
||||
std::string exclude = json_value(params, "exclude", std::string(""));
|
||||
|
|
@ -457,7 +474,7 @@ struct server_tool_grep_search : server_tool {
|
|||
};
|
||||
}
|
||||
|
||||
json invoke(json params) const override {
|
||||
json invoke(json params, server_tool::stream *) const override {
|
||||
std::string path = params.at("path").get<std::string>();
|
||||
std::string pat_str = params.at("pattern").get<std::string>();
|
||||
std::string include = json_value(params, "include", std::string("**"));
|
||||
|
|
@ -577,6 +594,7 @@ struct server_tool_exec_shell_command : server_tool {
|
|||
name = "exec_shell_command";
|
||||
display_name = "Execute shell command";
|
||||
permission_write = true;
|
||||
support_stream = true;
|
||||
}
|
||||
|
||||
json get_definition() const override {
|
||||
|
|
@ -598,7 +616,7 @@ struct server_tool_exec_shell_command : server_tool {
|
|||
};
|
||||
}
|
||||
|
||||
json invoke(json params) const override {
|
||||
json invoke(json params, server_tool::stream * st) const override {
|
||||
std::string command = params.at("command").get<std::string>();
|
||||
int timeout = json_value(params, "timeout", 10);
|
||||
size_t max_output = (size_t) json_value(params, "max_output_size", (int) SERVER_TOOL_EXEC_SHELL_COMMAND_MAX_OUTPUT_SIZE);
|
||||
|
|
@ -612,7 +630,24 @@ struct server_tool_exec_shell_command : server_tool {
|
|||
std::vector<std::string> args = {"sh", "-c", command};
|
||||
#endif
|
||||
|
||||
auto io = make_tools_io(params);
|
||||
auto io = make_tools_io(params);
|
||||
|
||||
if (st) {
|
||||
auto res = io->run(args, max_output, timeout, [st](const std::string & chunk) {
|
||||
st->push(chunk);
|
||||
return !st->alive || st->alive();
|
||||
});
|
||||
if (st->alive && !st->alive()) {
|
||||
return json();
|
||||
}
|
||||
std::string tail = string_format("\n[exit code: %d]", res.exit_code);
|
||||
if (res.timed_out) {
|
||||
tail += " [exit due to timed out]";
|
||||
}
|
||||
st->push(tail);
|
||||
return json();
|
||||
}
|
||||
|
||||
auto res = io->run(args, max_output, timeout);
|
||||
|
||||
std::string text_output = res.output;
|
||||
|
|
@ -654,7 +689,7 @@ struct server_tool_write_file : server_tool {
|
|||
};
|
||||
}
|
||||
|
||||
json invoke(json params) const override {
|
||||
json invoke(json params, server_tool::stream *) const override {
|
||||
std::string path = params.at("path").get<std::string>();
|
||||
std::string content = params.at("content").get<std::string>();
|
||||
|
||||
|
|
@ -710,7 +745,7 @@ struct server_tool_edit_file : server_tool {
|
|||
};
|
||||
}
|
||||
|
||||
json invoke(json params) const override {
|
||||
json invoke(json params, server_tool::stream *) const override {
|
||||
std::string path = params.at("path").get<std::string>();
|
||||
const json & edits_json = params.at("edits");
|
||||
|
||||
|
|
@ -1018,7 +1053,7 @@ struct server_tool_get_datetime : server_tool {
|
|||
};
|
||||
}
|
||||
|
||||
json invoke(json) const override {
|
||||
json invoke(json, server_tool::stream *) const override {
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto time = std::chrono::system_clock::to_time_t(now);
|
||||
|
||||
|
|
@ -1026,6 +1061,59 @@ struct server_tool_get_datetime : server_tool {
|
|||
}
|
||||
};
|
||||
|
||||
struct server_tool_stream_result : server_task_result {
|
||||
std::string chunk;
|
||||
bool done = false;
|
||||
std::string error_msg;
|
||||
|
||||
json to_json() override {
|
||||
if (!done) {
|
||||
return {{"chunk", chunk}};
|
||||
} else {
|
||||
json result = {{"done", true}};
|
||||
if (!error_msg.empty()) {
|
||||
result["error"] = error_msg;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void server_tool::stream::push(const std::string & chunk) {
|
||||
if (chunk.empty()) return;
|
||||
auto r = std::make_unique<server_tool_stream_result>();
|
||||
r->id = id;
|
||||
r->chunk = chunk;
|
||||
qr.send(std::move(r));
|
||||
}
|
||||
|
||||
struct server_tools_res : server_http_res {
|
||||
std::thread worker;
|
||||
server_response * qr = nullptr; // set only for streaming responses
|
||||
int id = -1;
|
||||
|
||||
~server_tools_res() override {
|
||||
if (worker.joinable()) {
|
||||
worker.join();
|
||||
}
|
||||
if (qr) {
|
||||
qr->remove_waiting_task_id(id);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static server_tool & find_tool(std::vector<std::unique_ptr<server_tool>> & tools, const std::string & name, bool require_stream) {
|
||||
for (auto & t : tools) {
|
||||
if (t->name == name) {
|
||||
if (require_stream && !t->support_stream) {
|
||||
throw std::invalid_argument(string_format("tool \"%s\" does not support stream = true", name.c_str()));
|
||||
}
|
||||
return *t;
|
||||
}
|
||||
}
|
||||
throw std::invalid_argument(string_format("unknown tool \"%s\"", name.c_str()));
|
||||
}
|
||||
|
||||
//
|
||||
// public API
|
||||
//
|
||||
|
|
@ -1090,16 +1178,63 @@ void server_tools::setup(const std::vector<std::string> & enabled_tools) {
|
|||
};
|
||||
|
||||
handle_post = [this](const server_http_req & req) -> server_http_res_ptr {
|
||||
auto res = std::make_unique<server_http_res>();
|
||||
auto res = std::make_unique<server_tools_res>();
|
||||
try {
|
||||
json body = json::parse(req.body);
|
||||
std::string tool_name = body.at("tool").get<std::string>();
|
||||
json params = body.value("params", json::object());
|
||||
json result = invoke(tool_name, params);
|
||||
res->data = safe_json_to_str(result);
|
||||
bool stream = body.value("stream", false);
|
||||
|
||||
server_tool & tool = find_tool(tools, tool_name, stream);
|
||||
|
||||
if (stream) {
|
||||
int id = res_id.fetch_add(1);
|
||||
queue_res.add_waiting_task_id(id);
|
||||
res->qr = &queue_res;
|
||||
res->id = id;
|
||||
|
||||
res->worker = std::thread([this, id, &req, &tool, params]() mutable {
|
||||
server_tool::stream st{queue_res, id, [&req]() {
|
||||
return !req.should_stop();
|
||||
}};
|
||||
|
||||
auto done = std::make_unique<server_tool_stream_result>();
|
||||
try {
|
||||
tool.invoke(params, &st);
|
||||
} catch (const std::exception & e) {
|
||||
done->error_msg = e.what();
|
||||
} catch (...) {
|
||||
done->error_msg = "An unknown error occurred";
|
||||
}
|
||||
done->id = st.id;
|
||||
done->done = true;
|
||||
st.qr.send(std::move(done));
|
||||
});
|
||||
|
||||
res->content_type = "text/event-stream";
|
||||
res->status = 200;
|
||||
res->next = [this, id](std::string & output) -> bool {
|
||||
auto result = queue_res.recv(id);
|
||||
auto * r = dynamic_cast<server_tool_stream_result *>(result.get());
|
||||
GGML_ASSERT(r != nullptr);
|
||||
output = "data: " + safe_json_to_str(r->to_json()) + "\n\n";
|
||||
if (r->done) {
|
||||
queue_res.remove_waiting_task_id(id);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
} else {
|
||||
json result = tool.invoke(params, nullptr);
|
||||
res->status = 200;
|
||||
res->data = safe_json_to_str(result);
|
||||
}
|
||||
} catch (const json::exception & e) {
|
||||
res->status = 400;
|
||||
res->data = safe_json_to_str(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));
|
||||
} catch (const std::invalid_argument & e) {
|
||||
res->status = 404;
|
||||
res->data = safe_json_to_str(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));
|
||||
} catch (const std::exception & e) {
|
||||
SRV_ERR("got exception: %s\n", e.what());
|
||||
res->status = 500;
|
||||
|
|
@ -1108,12 +1243,3 @@ void server_tools::setup(const std::vector<std::string> & enabled_tools) {
|
|||
return res;
|
||||
};
|
||||
}
|
||||
|
||||
json server_tools::invoke(const std::string & name, const json & params) {
|
||||
for (auto & t : tools) {
|
||||
if (t->name == name) {
|
||||
return t->invoke(params);
|
||||
}
|
||||
}
|
||||
return {{"error", "unknown tool: " + name}};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,15 +2,27 @@
|
|||
|
||||
#include "server-common.h"
|
||||
#include "server-http.h"
|
||||
#include "server-queue.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <functional>
|
||||
|
||||
struct server_tool {
|
||||
std::string name;
|
||||
std::string display_name;
|
||||
bool permission_write = false;
|
||||
bool support_stream = false; // if true, output can be streamed
|
||||
|
||||
virtual ~server_tool() = default;
|
||||
virtual json get_definition() const = 0;
|
||||
virtual json invoke(json params) const = 0;
|
||||
|
||||
struct stream {
|
||||
server_response & qr;
|
||||
int id;
|
||||
std::function<bool()> alive;
|
||||
void push(const std::string & chunk);
|
||||
};
|
||||
virtual json invoke(json params, stream * st = nullptr) const = 0;
|
||||
|
||||
json to_json() const;
|
||||
};
|
||||
|
|
@ -18,8 +30,11 @@ struct server_tool {
|
|||
struct server_tools {
|
||||
std::vector<std::unique_ptr<server_tool>> tools;
|
||||
|
||||
// for streaming
|
||||
server_response queue_res;
|
||||
std::atomic<int> res_id{0};
|
||||
|
||||
void setup(const std::vector<std::string> & enabled_tools);
|
||||
json invoke(const std::string & name, const json & params);
|
||||
|
||||
server_http_context::handler_t handle_get;
|
||||
server_http_context::handler_t handle_post;
|
||||
|
|
|
|||
|
|
@ -402,6 +402,65 @@ def test_anthropic_tool_result_with_text():
|
|||
assert len(res.body["content"]) > 0
|
||||
|
||||
|
||||
def test_anthropic_tool_result_with_image():
|
||||
"""Test tool result containing mixed text and image blocks
|
||||
|
||||
Verifies that image blocks inside Anthropic tool_result content are
|
||||
properly converted to OpenAI image_url format rather than being
|
||||
silently dropped. With a non-multimodal model, the converted image
|
||||
triggers a clear error message instead of being ignored.
|
||||
"""
|
||||
server.jinja = True
|
||||
server.start()
|
||||
|
||||
# Small 1x1 red PNG image in base64 (same as vision tests)
|
||||
red_pixel_png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="
|
||||
|
||||
res = server.make_request("POST", "/v1/messages", data={
|
||||
"model": "test",
|
||||
"max_tokens": 100,
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is in this image?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "tool_1",
|
||||
"name": "read",
|
||||
"input": {"file": "test.png"}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "tool_1",
|
||||
"content": [
|
||||
{"type": "text", "text": "File: test.png"},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/png",
|
||||
"data": red_pixel_png
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
# Without the fix, image block would cause "unsupported content[].type"
|
||||
# With the fix, image is converted to image_url but tinyllama doesn't support images
|
||||
assert res.status_code == 500
|
||||
assert "image input is not supported" in res.body.get("error", {}).get("message", "").lower()
|
||||
|
||||
|
||||
def test_anthropic_tool_result_error():
|
||||
"""Test tool result with error flag"""
|
||||
server.jinja = True
|
||||
|
|
|
|||
|
|
@ -105,6 +105,24 @@ def test_tools_builtin_edit_file_rejects_non_unique_old_text():
|
|||
os.remove(log_path)
|
||||
|
||||
|
||||
def test_tools_builtin_exec_shell_command_stream():
|
||||
global server
|
||||
server.start()
|
||||
|
||||
events = list(server.make_stream_request("POST", "/tools", data={
|
||||
"tool": "exec_shell_command",
|
||||
"params": {"command": "echo hello"},
|
||||
"stream": True,
|
||||
}))
|
||||
|
||||
assert len(events) >= 2
|
||||
assert events[-1]["done"] is True
|
||||
assert not events[-1].get("error")
|
||||
chunks = "".join(e["chunk"] for e in events[:-1])
|
||||
assert "hello" in chunks
|
||||
assert "[exit code: 0]" in chunks
|
||||
|
||||
|
||||
def test_tools_builtin_edit_file_rejects_overlapping_edits():
|
||||
global server
|
||||
server.start()
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
let { onMcpSettingsClick }: Props = $props();
|
||||
|
||||
let mcpSearchQuery = $state('');
|
||||
let allMcpServers = $derived(mcpStore.getServersSorted());
|
||||
let allMcpServers = $derived(mcpStore.getServers());
|
||||
let mcpServers = $derived(mcpStore.visibleMcpServers);
|
||||
let hasMcpServers = $derived(mcpServers.length > 0);
|
||||
// let hasAnyMcpServers = $derived(allMcpServers.length > 0);
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
import { useToolsPanel } from '$lib/hooks/use-tools-panel.svelte';
|
||||
|
||||
const toolsPanel = useToolsPanel();
|
||||
const hasMcpServersAvailable = $derived(mcpStore.getServersSorted().length > 0);
|
||||
const hasMcpServersAvailable = $derived(mcpStore.getServers().length > 0);
|
||||
</script>
|
||||
|
||||
<DropdownMenu.Sub onOpenChange={(open) => open && toolsPanel.handleOpen()}>
|
||||
|
|
|
|||
|
|
@ -322,7 +322,7 @@
|
|||
}
|
||||
|
||||
let filteredPrompts = $derived.by(() => {
|
||||
const sortedServers = mcpStore.getServersSorted();
|
||||
const sortedServers = mcpStore.getServers();
|
||||
const serverOrderMap = new Map(sortedServers.map((server, index) => [server.id, index]));
|
||||
|
||||
const sortedPrompts = [...prompts].sort((a, b) => {
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@
|
|||
}
|
||||
|
||||
let filteredResources = $derived.by(() => {
|
||||
const sortedServers = mcpStore.getServersSorted();
|
||||
const sortedServers = mcpStore.getServers();
|
||||
const serverOrderMap = new Map(sortedServers.map((server, index) => [server.id, index]));
|
||||
|
||||
const sortedResources = [...resources].sort((a, b) => {
|
||||
|
|
|
|||
|
|
@ -1,210 +0,0 @@
|
|||
<script lang="ts">
|
||||
import { Button } from '$lib/components/ui/button';
|
||||
import * as Card from '$lib/components/ui/card';
|
||||
import * as Dialog from '$lib/components/ui/dialog';
|
||||
import { fly } from 'svelte/transition';
|
||||
import { McpServerCardCompact, McpServerForm } from '$lib/components/app/mcp';
|
||||
import { RECOMMENDED_MCP_SERVERS, SETTINGS_KEYS } from '$lib/constants';
|
||||
import { conversationsStore } from '$lib/stores/conversations.svelte';
|
||||
import { mcpStore } from '$lib/stores/mcp.svelte';
|
||||
import { settingsStore } from '$lib/stores/settings.svelte';
|
||||
import { uuid } from '$lib/utils';
|
||||
import { MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY, MCP_SERVER_ID_PREFIX } from '$lib/constants';
|
||||
import type { MCPServerSettingsEntry } from '$lib/types';
|
||||
import { Plus } from '@lucide/svelte';
|
||||
|
||||
interface Props {
|
||||
open: boolean;
|
||||
onOpenChange?: (open: boolean) => void;
|
||||
}
|
||||
|
||||
let { open = $bindable(), onOpenChange }: Props = $props();
|
||||
|
||||
let selected = $state<Record<string, boolean>>(
|
||||
Object.fromEntries(RECOMMENDED_MCP_SERVERS.map((server) => [server.id, false]))
|
||||
);
|
||||
|
||||
let addedServers = $state<MCPServerSettingsEntry[]>([]);
|
||||
let didAddAny = $state(false);
|
||||
|
||||
let selectedRecommendedCount = $derived.by(
|
||||
() => RECOMMENDED_MCP_SERVERS.filter((server) => selected[server.id]).length
|
||||
);
|
||||
|
||||
let footerLabel = $derived.by(() => {
|
||||
const recommended = selectedRecommendedCount;
|
||||
const custom = addedServers.length;
|
||||
const total = recommended + custom;
|
||||
|
||||
if (total === 0) return 'Continue';
|
||||
if (recommended === 0) return custom === 1 ? 'Add server' : `Add ${custom} servers`;
|
||||
if (custom === 0) return recommended === 1 ? 'Add server' : `Add ${recommended} servers`;
|
||||
return `Add ${recommended} servers and ${custom} custom`;
|
||||
});
|
||||
|
||||
let showAddForm = $state(false);
|
||||
let newServerUrl = $state('');
|
||||
let newServerHeaders = $state('');
|
||||
let newServerUrlError = $derived.by(() => {
|
||||
if (!newServerUrl.trim()) return 'URL is required';
|
||||
try {
|
||||
new URL(newServerUrl);
|
||||
|
||||
return null;
|
||||
} catch {
|
||||
return 'Invalid URL format';
|
||||
}
|
||||
});
|
||||
|
||||
function handleOpenChange(value: boolean) {
|
||||
if (!value) {
|
||||
showAddForm = false;
|
||||
newServerUrl = '';
|
||||
newServerHeaders = '';
|
||||
|
||||
if (!didAddAny) {
|
||||
settingsStore.updateConfig(SETTINGS_KEYS.MCP_SERVERS, []);
|
||||
}
|
||||
|
||||
localStorage.setItem(MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY, 'true');
|
||||
addedServers = [];
|
||||
didAddAny = false;
|
||||
}
|
||||
open = value;
|
||||
onOpenChange?.(value);
|
||||
}
|
||||
|
||||
function resetAddForm() {
|
||||
showAddForm = false;
|
||||
newServerUrl = '';
|
||||
newServerHeaders = '';
|
||||
}
|
||||
|
||||
function enableSelected() {
|
||||
didAddAny = true;
|
||||
localStorage.setItem(MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY, 'true');
|
||||
|
||||
for (const server of RECOMMENDED_MCP_SERVERS) {
|
||||
if (selected[server.id]) {
|
||||
const existing = mcpStore.getServerById(server.id);
|
||||
if (existing) {
|
||||
mcpStore.updateServer(server.id, { enabled: true });
|
||||
} else {
|
||||
mcpStore.addServer({
|
||||
id: server.id,
|
||||
enabled: true,
|
||||
url: server.url,
|
||||
name: server.name
|
||||
});
|
||||
}
|
||||
conversationsStore.setMcpServerOverride(server.id, true);
|
||||
}
|
||||
}
|
||||
handleOpenChange(false);
|
||||
}
|
||||
|
||||
function saveNewServer() {
|
||||
if (newServerUrlError) return;
|
||||
|
||||
didAddAny = true;
|
||||
|
||||
const newServerId = uuid() ?? `${MCP_SERVER_ID_PREFIX}-${Date.now()}`;
|
||||
|
||||
localStorage.setItem(MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY, 'true');
|
||||
|
||||
const newServer = mcpStore.addServer({
|
||||
id: newServerId,
|
||||
enabled: true,
|
||||
url: newServerUrl.trim(),
|
||||
headers: newServerHeaders.trim() || undefined
|
||||
});
|
||||
|
||||
conversationsStore.setMcpServerOverride(newServerId, true);
|
||||
|
||||
if (newServer) {
|
||||
addedServers = [...addedServers, newServer];
|
||||
}
|
||||
|
||||
resetAddForm();
|
||||
}
|
||||
</script>
|
||||
|
||||
<Dialog.Root bind:open onOpenChange={handleOpenChange}>
|
||||
<Dialog.Content class="sm:max-w-lg">
|
||||
<Dialog.Header>
|
||||
<Dialog.Title>Do more with MCP</Dialog.Title>
|
||||
<Dialog.Description>
|
||||
Power-up your experience by adding tools, resources and more capabilities provided by MCP
|
||||
servers.
|
||||
</Dialog.Description>
|
||||
</Dialog.Header>
|
||||
|
||||
<div class="max-h-[60vh] space-y-4 overflow-y-auto py-4" in:fly={{ y: 16, duration: 300 }}>
|
||||
<h3 class="text-sm font-semibold">Quickly get started with</h3>
|
||||
|
||||
{#each RECOMMENDED_MCP_SERVERS as server (server.id)}
|
||||
<McpServerCardCompact
|
||||
{server}
|
||||
enabled={selected[server.id]}
|
||||
onToggle={(enabled) => (selected[server.id] = enabled)}
|
||||
/>
|
||||
{/each}
|
||||
|
||||
{#if addedServers.length > 0}
|
||||
{#each addedServers as server (server.id)}
|
||||
<McpServerCardCompact {server} enabled={true} />
|
||||
{/each}
|
||||
{/if}
|
||||
|
||||
{#if showAddForm}
|
||||
<Card.Root class="gap-3! bg-muted/30 p-4">
|
||||
<McpServerForm
|
||||
url={newServerUrl}
|
||||
headers={newServerHeaders}
|
||||
onUrlChange={(v) => (newServerUrl = v)}
|
||||
onHeadersChange={(v) => (newServerHeaders = v)}
|
||||
urlError={newServerUrl ? newServerUrlError : null}
|
||||
id="recommendation-new-server"
|
||||
/>
|
||||
|
||||
<div class="flex justify-end gap-2 pt-2">
|
||||
<Button variant="secondary" size="sm" onclick={resetAddForm}>Cancel</Button>
|
||||
|
||||
<Button
|
||||
variant="default"
|
||||
size="sm"
|
||||
onclick={saveNewServer}
|
||||
disabled={!!newServerUrlError}
|
||||
aria-label="Save"
|
||||
>
|
||||
Add
|
||||
</Button>
|
||||
</div>
|
||||
</Card.Root>
|
||||
{:else}
|
||||
<Card.Root class="gap-0 border-dashed bg-muted/30 p-0 transition-colors hover:bg-muted/50">
|
||||
<button
|
||||
type="button"
|
||||
class="flex w-full items-center justify-center gap-2 rounded-lg p-6 text-sm text-muted-foreground transition-colors hover:text-foreground"
|
||||
onclick={() => (showAddForm = true)}
|
||||
aria-label="Add your own MCP server"
|
||||
>
|
||||
<Plus class="h-4 w-4" />
|
||||
<span>Add your own server</span>
|
||||
</button>
|
||||
</Card.Root>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<Dialog.Footer>
|
||||
<Button variant="secondary" size="sm" onclick={() => handleOpenChange(false)}>Not now</Button>
|
||||
|
||||
<Button
|
||||
variant="default"
|
||||
size="sm"
|
||||
onclick={enableSelected}
|
||||
disabled={footerLabel === 'Continue'}>{footerLabel}</Button
|
||||
>
|
||||
</Dialog.Footer>
|
||||
</Dialog.Content>
|
||||
</Dialog.Root>
|
||||
|
|
@ -18,15 +18,6 @@
|
|||
*/
|
||||
export { default as DialogMcpServerAddNew } from './DialogMcpServerAddNew.svelte';
|
||||
|
||||
/**
|
||||
* **DialogMcpServerRecommendations** - Suggested MCP servers opt-in dialog
|
||||
*
|
||||
* Prompts the user to enable pre-defined recommended MCP servers on first launch.
|
||||
* Shows one switch per suggested server and persists the choice as a per-chat
|
||||
* override so the selected servers become available in conversations.
|
||||
*/
|
||||
export { default as DialogMcpServerRecommendations } from './DialogMcpServerRecommendations.svelte';
|
||||
|
||||
/**
|
||||
* **DialogExportSettings** - Settings export dialog with sensitive data warning
|
||||
*
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
let { class: className = '', onclick }: Props = $props();
|
||||
|
||||
let mcpServers = $derived(mcpStore.getServersSorted().filter((s) => s.enabled));
|
||||
let mcpServers = $derived(mcpStore.getServers().filter((s) => s.enabled));
|
||||
let enabledMcpServersForChat = $derived(
|
||||
mcpServers.filter((s) => conversationsStore.isMcpServerEnabledForChat(s.id) && s.url.trim())
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,156 +0,0 @@
|
|||
<script lang="ts">
|
||||
import * as Card from '$lib/components/ui/card';
|
||||
import { Badge } from '$lib/components/ui/badge';
|
||||
import { Skeleton } from '$lib/components/ui/skeleton';
|
||||
import { Switch } from '$lib/components/ui/switch';
|
||||
import * as Tooltip from '$lib/components/ui/tooltip';
|
||||
import { McpServerIdentity } from '$lib/components/app/mcp';
|
||||
import { mcpStore } from '$lib/stores/mcp.svelte';
|
||||
import { HealthCheckStatus } from '$lib/enums';
|
||||
import type { MCPServerDisplayInfo, HealthCheckState, MCPServerSettingsEntry } from '$lib/types';
|
||||
import { onMount } from 'svelte';
|
||||
import { MCP_CARD_VISIBLE_TOOL_LIMIT, NEWLINE } from '$lib/constants';
|
||||
|
||||
interface Props {
|
||||
server: MCPServerDisplayInfo & { description?: string };
|
||||
enabled?: boolean;
|
||||
onToggle?: (enabled: boolean) => void;
|
||||
}
|
||||
|
||||
let { server, enabled = false, onToggle }: Props = $props();
|
||||
|
||||
onMount(() => {
|
||||
const state = mcpStore.getHealthCheckState(server.id);
|
||||
|
||||
if (state.status === HealthCheckStatus.IDLE) {
|
||||
mcpStore.runHealthCheck(server as MCPServerSettingsEntry).catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
let healthState = $derived<HealthCheckState>(mcpStore.getHealthCheckState(server.id));
|
||||
let displayName = $derived(mcpStore.getServerLabel(server));
|
||||
let faviconUrl = $derived(mcpStore.getServerFavicon(server.id));
|
||||
let isIdle = $derived(healthState.status === HealthCheckStatus.IDLE);
|
||||
let isHealthChecking = $derived(healthState.status === HealthCheckStatus.CONNECTING);
|
||||
let isError = $derived(healthState.status === HealthCheckStatus.ERROR);
|
||||
let errorMessage = $derived(
|
||||
healthState.status === HealthCheckStatus.ERROR ? healthState.message : undefined
|
||||
);
|
||||
let serverInfo = $derived(
|
||||
healthState.status === HealthCheckStatus.SUCCESS ? healthState.serverInfo : undefined
|
||||
);
|
||||
let tools = $derived(healthState.status === HealthCheckStatus.SUCCESS ? healthState.tools : []);
|
||||
let instructions = $derived(
|
||||
healthState.status === HealthCheckStatus.SUCCESS ? healthState.instructions : undefined
|
||||
);
|
||||
let showSkeleton = $derived(isIdle || isHealthChecking);
|
||||
|
||||
// Curated descriptions get two lines; instructions fallback is one line so the
|
||||
// compact card stays scannable.
|
||||
let description = $derived.by(() => {
|
||||
if (server.description) {
|
||||
return { text: server.description, lines: 2 };
|
||||
}
|
||||
if (!instructions) return null;
|
||||
const firstLine = instructions.split(NEWLINE).find((line: string) => line.trim().length > 0);
|
||||
const trimmed = firstLine?.trim();
|
||||
return trimmed ? { text: trimmed, lines: 1 } : null;
|
||||
});
|
||||
|
||||
let visibleTools = $derived(tools.slice(0, MCP_CARD_VISIBLE_TOOL_LIMIT));
|
||||
let hiddenTools = $derived(tools.slice(MCP_CARD_VISIBLE_TOOL_LIMIT));
|
||||
let hiddenToolCount = $derived(hiddenTools.length);
|
||||
|
||||
function handleToggle(checked: boolean) {
|
||||
onToggle?.(checked);
|
||||
}
|
||||
</script>
|
||||
|
||||
<Card.Root class="!gap-3 bg-muted/30 p-4">
|
||||
<div class="flex items-start justify-between gap-3">
|
||||
<div class="min-w-0 flex-1">
|
||||
{#if showSkeleton}
|
||||
<span class="flex min-w-0 items-center gap-1.5">
|
||||
<Skeleton class="h-5 w-5 rounded" />
|
||||
<Skeleton class="h-4 w-32" />
|
||||
</span>
|
||||
{:else}
|
||||
<McpServerIdentity
|
||||
{displayName}
|
||||
{faviconUrl}
|
||||
{serverInfo}
|
||||
iconClass="h-5 w-5"
|
||||
iconRounded="rounded"
|
||||
nameClass="font-medium"
|
||||
/>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<Switch checked={enabled} disabled={isError || showSkeleton} onCheckedChange={handleToggle} />
|
||||
</div>
|
||||
|
||||
{#if isError && errorMessage}
|
||||
<p class="text-xs text-destructive">{errorMessage}</p>
|
||||
{/if}
|
||||
|
||||
{#if showSkeleton}
|
||||
<div class="space-y-1.5">
|
||||
<Skeleton class="h-3 w-full max-w-md" />
|
||||
</div>
|
||||
|
||||
<div class="flex flex-wrap items-center gap-1.5">
|
||||
<Skeleton class="h-5 w-16 rounded-full" />
|
||||
<Skeleton class="h-5 w-20 rounded-full" />
|
||||
<Skeleton class="h-5 w-24 rounded-full" />
|
||||
<Skeleton class="h-5 w-14 rounded-full" />
|
||||
</div>
|
||||
{:else}
|
||||
{#if description}
|
||||
{#if description.lines === 2}
|
||||
<p class="line-clamp-2 text-xs text-muted-foreground" title={description.text}>
|
||||
{description.text}
|
||||
</p>
|
||||
{:else}
|
||||
<p class="line-clamp-1 truncate text-xs text-muted-foreground" title={description.text}>
|
||||
{description.text}
|
||||
</p>
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
{#if tools.length > 0}
|
||||
<div class="flex flex-wrap items-center gap-1.5">
|
||||
{#each visibleTools as tool (tool.name)}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<Badge variant="secondary" class="h-5 max-w-40 px-2 text-[11px]">
|
||||
<span class="block min-w-0 flex-1 truncate">{tool.name}</span>
|
||||
</Badge>
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content>
|
||||
<p class="max-w-xs text-xs">
|
||||
{tool.description ?? 'No description'}
|
||||
</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/each}
|
||||
|
||||
{#if hiddenToolCount > 0}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<Badge variant="secondary" class="h-5 px-2 text-[11px] text-muted-foreground">
|
||||
+ {hiddenToolCount} more tools
|
||||
</Badge>
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content class="max-w-md">
|
||||
<p class="text-xs">
|
||||
{hiddenTools.map((tool) => tool.name).join(', ')}
|
||||
</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
{/if}
|
||||
</Card.Root>
|
||||
|
|
@ -180,16 +180,6 @@ export { default as McpServerCardDeleteDialog } from './McpServerCard/McpServerC
|
|||
/** Skeleton loading state for server card during health checks. */
|
||||
export { default as McpServerCardSkeleton } from './McpServerCardSkeleton.svelte';
|
||||
|
||||
/**
|
||||
* **McpServerCardCompact** - Condensed MCP server card
|
||||
*
|
||||
* Compact alternative to McpServerCard tailored for picker-style UIs.
|
||||
* Shows the server identity, status, and a flex-wrapped list of available tools.
|
||||
* Tool names are rendered as badges; hovering a badge shows its description in a tooltip.
|
||||
* Does not show connection logs or server instructions.
|
||||
*/
|
||||
export { default as McpServerCardCompact } from './McpServerCard/McpServerCardCompact.svelte';
|
||||
|
||||
/**
|
||||
* **McpServerIdentity** - Server identity display (icon, name, version)
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
<script lang="ts">
|
||||
import { X, Plus } from '@lucide/svelte';
|
||||
import { Button } from '$lib/components/ui/button';
|
||||
import { mcpStore } from '$lib/stores/mcp.svelte';
|
||||
import { conversationsStore } from '$lib/stores/conversations.svelte';
|
||||
import { toolsStore } from '$lib/stores/tools.svelte';
|
||||
import { Button } from '$lib/components/ui/button';
|
||||
import * as Empty from '$lib/components/ui/empty';
|
||||
import { ActionIcon, McpServerCard, McpServerCardSkeleton } from '$lib/components/app';
|
||||
import { DialogMcpServerAddNew } from '$lib/components/app/dialogs';
|
||||
import { HealthCheckStatus } from '$lib/enums';
|
||||
|
|
@ -23,7 +24,6 @@
|
|||
|
||||
let servers = $derived(mcpStore.visibleMcpServers);
|
||||
|
||||
let initialLoadComplete = $state(false);
|
||||
let isAddingServer = $state(false);
|
||||
|
||||
let previousRouteId = $state<string | null>(null);
|
||||
|
|
@ -55,26 +55,16 @@
|
|||
}
|
||||
});
|
||||
|
||||
$effect(() => {
|
||||
if (initialLoadComplete) return;
|
||||
|
||||
const allChecked =
|
||||
servers.length > 0 &&
|
||||
servers.every((server) => {
|
||||
const state = mcpStore.getHealthCheckState(server.id);
|
||||
|
||||
return (
|
||||
state.status === HealthCheckStatus.SUCCESS || state.status === HealthCheckStatus.ERROR
|
||||
);
|
||||
});
|
||||
|
||||
if (allChecked) {
|
||||
initialLoadComplete = true;
|
||||
}
|
||||
});
|
||||
// Each card decides for itself whether to render based on its own
|
||||
// health-check state, so adding a server only flashes the new card
|
||||
// (not every other already-loaded card) until its health check resolves.
|
||||
function isServerPending(serverId: string): boolean {
|
||||
const status = mcpStore.getHealthCheckState(serverId).status;
|
||||
return status === HealthCheckStatus.IDLE || status === HealthCheckStatus.CONNECTING;
|
||||
}
|
||||
</script>
|
||||
|
||||
<div in:fade={{ duration: 150 }}>
|
||||
<div in:fade={{ duration: 150 }} class="flex min-h-[calc(100dvh-4rem)] flex-col">
|
||||
<div class="fixed top-4.5 right-4 z-50 md:hidden">
|
||||
<ActionIcon icon={X} tooltip="Close" onclick={handleClose} />
|
||||
</div>
|
||||
|
|
@ -87,53 +77,78 @@
|
|||
|
||||
<h1 class="text-lg font-semibold md:text-2xl">MCP Servers</h1>
|
||||
</div>
|
||||
|
||||
<Button
|
||||
variant="outline"
|
||||
size="lg"
|
||||
class="shrink-0 fixed md:static bottom-6 right-6"
|
||||
onclick={() => (isAddingServer = true)}
|
||||
>
|
||||
<Plus class="h-4 w-4" />
|
||||
|
||||
Add New Server
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<DialogMcpServerAddNew bind:open={isAddingServer} />
|
||||
|
||||
<div class="grid gap-5 md:space-y-4 {className}">
|
||||
{#if servers.length === 0 && !isAddingServer}
|
||||
<div class="rounded-md border border-dashed p-4 text-sm text-muted-foreground">
|
||||
No MCP Servers configured yet. Add one to enable agentic features.
|
||||
</div>
|
||||
{/if}
|
||||
{#if servers.length === 0}
|
||||
<div class="flex flex-1 items-center justify-center py-16">
|
||||
<Empty.Root class="max-w-md">
|
||||
<Empty.Header>
|
||||
<Empty.Media variant="icon">
|
||||
<Plus />
|
||||
</Empty.Media>
|
||||
|
||||
{#if servers.length > 0}
|
||||
<div
|
||||
class="grid gap-3"
|
||||
style="grid-template-columns: repeat(auto-fill, minmax(min(32rem, calc(100dvw - 2rem)), 1fr));"
|
||||
>
|
||||
{#each servers as server (server.id)}
|
||||
{#if !initialLoadComplete}
|
||||
<McpServerCardSkeleton />
|
||||
{:else}
|
||||
<McpServerCard
|
||||
{server}
|
||||
enabled={conversationsStore.isMcpServerEnabledForChat(server.id)}
|
||||
onToggle={async () => {
|
||||
const wasEnabled = conversationsStore.isMcpServerEnabledForChat(server.id);
|
||||
await conversationsStore.toggleMcpServerForChat(server.id);
|
||||
if (!wasEnabled) {
|
||||
toolsStore.enableAllToolsForServer(server.id);
|
||||
}
|
||||
}}
|
||||
onUpdate={(updates) => mcpStore.updateServer(server.id, updates)}
|
||||
onDelete={() => mcpStore.removeServer(server.id)}
|
||||
/>
|
||||
{/if}
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
<Empty.Title>Add your first MCP server</Empty.Title>
|
||||
|
||||
<Empty.Description>Connect a remote MCP server by URL.</Empty.Description>
|
||||
</Empty.Header>
|
||||
|
||||
<Empty.Content>
|
||||
<Button size="sm" onclick={() => (isAddingServer = true)}>
|
||||
<Plus />
|
||||
|
||||
Add New Server
|
||||
</Button>
|
||||
</Empty.Content>
|
||||
</Empty.Root>
|
||||
</div>
|
||||
{:else}
|
||||
<div
|
||||
class="grid gap-3 {className}"
|
||||
style="grid-template-columns: repeat(auto-fill, minmax(min(32rem, calc(100dvw - 2rem)), 1fr));"
|
||||
>
|
||||
{#each servers as server (server.id)}
|
||||
{#if isServerPending(server.id)}
|
||||
<McpServerCardSkeleton />
|
||||
{:else}
|
||||
<McpServerCard
|
||||
{server}
|
||||
enabled={conversationsStore.isMcpServerEnabledForChat(server.id)}
|
||||
onToggle={async () => {
|
||||
const wasEnabled = conversationsStore.isMcpServerEnabledForChat(server.id);
|
||||
await conversationsStore.toggleMcpServerForChat(server.id);
|
||||
if (!wasEnabled) {
|
||||
toolsStore.enableAllToolsForServer(server.id);
|
||||
}
|
||||
}}
|
||||
onUpdate={(updates) => mcpStore.updateServer(server.id, updates)}
|
||||
onDelete={() => mcpStore.removeServer(server.id)}
|
||||
/>
|
||||
{/if}
|
||||
{/each}
|
||||
|
||||
{#if !isAddingServer}
|
||||
<Empty.Root class="border">
|
||||
<Empty.Header>
|
||||
<Empty.Media variant="icon">
|
||||
<Plus />
|
||||
</Empty.Media>
|
||||
|
||||
<Empty.Title>Add another MCP server</Empty.Title>
|
||||
|
||||
<Empty.Description>Connect a remote MCP server by URL.</Empty.Description>
|
||||
</Empty.Header>
|
||||
|
||||
<Empty.Content>
|
||||
<Button size="sm" onclick={() => (isAddingServer = true)}>
|
||||
<Plus />
|
||||
|
||||
Add New Server
|
||||
</Button>
|
||||
</Empty.Content>
|
||||
</Empty.Root>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
|
|
|||
23
tools/ui/src/lib/components/ui/empty/empty-content.svelte
Normal file
23
tools/ui/src/lib/components/ui/empty/empty-content.svelte
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
<script lang="ts">
|
||||
import { cn, type WithElementRef } from '$lib/components/ui/utils.js';
|
||||
import type { HTMLAttributes } from 'svelte/elements';
|
||||
|
||||
let {
|
||||
ref = $bindable(null),
|
||||
class: className,
|
||||
children,
|
||||
...restProps
|
||||
}: WithElementRef<HTMLAttributes<HTMLDivElement>> = $props();
|
||||
</script>
|
||||
|
||||
<div
|
||||
bind:this={ref}
|
||||
data-slot="empty-content"
|
||||
class={cn(
|
||||
'gap-2.5 text-sm flex w-full max-w-sm min-w-0 flex-col items-center text-balance',
|
||||
className
|
||||
)}
|
||||
{...restProps}
|
||||
>
|
||||
{@render children?.()}
|
||||
</div>
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
<script lang="ts">
|
||||
import { cn, type WithElementRef } from '$lib/components/ui/utils.js';
|
||||
import type { HTMLAttributes } from 'svelte/elements';
|
||||
|
||||
let {
|
||||
ref = $bindable(null),
|
||||
class: className,
|
||||
children,
|
||||
...restProps
|
||||
}: WithElementRef<HTMLAttributes<HTMLDivElement>> = $props();
|
||||
</script>
|
||||
|
||||
<div
|
||||
bind:this={ref}
|
||||
data-slot="empty-description"
|
||||
class={cn(
|
||||
'text-sm/relaxed text-muted-foreground [&>a:hover]:text-primary text-sm/relaxed [&>a]:underline [&>a]:underline-offset-4',
|
||||
className
|
||||
)}
|
||||
{...restProps}
|
||||
>
|
||||
{@render children?.()}
|
||||
</div>
|
||||
20
tools/ui/src/lib/components/ui/empty/empty-header.svelte
Normal file
20
tools/ui/src/lib/components/ui/empty/empty-header.svelte
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
<script lang="ts">
|
||||
import { cn, type WithElementRef } from '$lib/components/ui/utils.js';
|
||||
import type { HTMLAttributes } from 'svelte/elements';
|
||||
|
||||
let {
|
||||
ref = $bindable(null),
|
||||
class: className,
|
||||
children,
|
||||
...restProps
|
||||
}: WithElementRef<HTMLAttributes<HTMLDivElement>> = $props();
|
||||
</script>
|
||||
|
||||
<div
|
||||
bind:this={ref}
|
||||
data-slot="empty-header"
|
||||
class={cn('gap-2 flex max-w-sm flex-col items-center', className)}
|
||||
{...restProps}
|
||||
>
|
||||
{@render children?.()}
|
||||
</div>
|
||||
41
tools/ui/src/lib/components/ui/empty/empty-media.svelte
Normal file
41
tools/ui/src/lib/components/ui/empty/empty-media.svelte
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
<script lang="ts" module>
|
||||
import { tv, type VariantProps } from 'tailwind-variants';
|
||||
|
||||
export const emptyMediaVariants = tv({
|
||||
base: 'mb-2 flex shrink-0 items-center justify-center [&_svg]:pointer-events-none [&_svg]:shrink-0',
|
||||
variants: {
|
||||
variant: {
|
||||
default: 'bg-transparent',
|
||||
icon: "bg-muted text-foreground flex size-8 shrink-0 items-center justify-center rounded-lg [&_svg:not([class*='size-'])]:size-4"
|
||||
}
|
||||
},
|
||||
defaultVariants: {
|
||||
variant: 'default'
|
||||
}
|
||||
});
|
||||
|
||||
export type EmptyMediaVariant = VariantProps<typeof emptyMediaVariants>['variant'];
|
||||
</script>
|
||||
|
||||
<script lang="ts">
|
||||
import { cn, type WithElementRef } from '$lib/components/ui/utils.js';
|
||||
import type { HTMLAttributes } from 'svelte/elements';
|
||||
|
||||
let {
|
||||
ref = $bindable(null),
|
||||
class: className,
|
||||
children,
|
||||
variant = 'default',
|
||||
...restProps
|
||||
}: WithElementRef<HTMLAttributes<HTMLDivElement>> & { variant?: EmptyMediaVariant } = $props();
|
||||
</script>
|
||||
|
||||
<div
|
||||
bind:this={ref}
|
||||
data-slot="empty-icon"
|
||||
data-variant={variant}
|
||||
class={cn(emptyMediaVariants({ variant }), className)}
|
||||
{...restProps}
|
||||
>
|
||||
{@render children?.()}
|
||||
</div>
|
||||
20
tools/ui/src/lib/components/ui/empty/empty-title.svelte
Normal file
20
tools/ui/src/lib/components/ui/empty/empty-title.svelte
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
<script lang="ts">
|
||||
import { cn, type WithElementRef } from '$lib/components/ui/utils.js';
|
||||
import type { HTMLAttributes } from 'svelte/elements';
|
||||
|
||||
let {
|
||||
ref = $bindable(null),
|
||||
class: className,
|
||||
children,
|
||||
...restProps
|
||||
}: WithElementRef<HTMLAttributes<HTMLDivElement>> = $props();
|
||||
</script>
|
||||
|
||||
<div
|
||||
bind:this={ref}
|
||||
data-slot="empty-title"
|
||||
class={cn('text-sm font-medium tracking-tight', className)}
|
||||
{...restProps}
|
||||
>
|
||||
{@render children?.()}
|
||||
</div>
|
||||
23
tools/ui/src/lib/components/ui/empty/empty.svelte
Normal file
23
tools/ui/src/lib/components/ui/empty/empty.svelte
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
<script lang="ts">
|
||||
import { cn, type WithElementRef } from '$lib/components/ui/utils.js';
|
||||
import type { HTMLAttributes } from 'svelte/elements';
|
||||
|
||||
let {
|
||||
ref = $bindable(null),
|
||||
class: className,
|
||||
children,
|
||||
...restProps
|
||||
}: WithElementRef<HTMLAttributes<HTMLDivElement>> = $props();
|
||||
</script>
|
||||
|
||||
<div
|
||||
bind:this={ref}
|
||||
data-slot="empty"
|
||||
class={cn(
|
||||
'gap-4 rounded-xl border-dashed p-6 flex w-full min-w-0 flex-1 flex-col items-center justify-center text-center text-balance',
|
||||
className
|
||||
)}
|
||||
{...restProps}
|
||||
>
|
||||
{@render children?.()}
|
||||
</div>
|
||||
22
tools/ui/src/lib/components/ui/empty/index.ts
Normal file
22
tools/ui/src/lib/components/ui/empty/index.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import Root from './empty.svelte';
|
||||
import Header from './empty-header.svelte';
|
||||
import Media from './empty-media.svelte';
|
||||
import Title from './empty-title.svelte';
|
||||
import Description from './empty-description.svelte';
|
||||
import Content from './empty-content.svelte';
|
||||
|
||||
export {
|
||||
Root,
|
||||
Header,
|
||||
Media,
|
||||
Title,
|
||||
Description,
|
||||
Content,
|
||||
//
|
||||
Root as Empty,
|
||||
Header as EmptyHeader,
|
||||
Media as EmptyMedia,
|
||||
Title as EmptyTitle,
|
||||
Description as EmptyDescription,
|
||||
Content as EmptyContent
|
||||
};
|
||||
|
|
@ -8,7 +8,6 @@ export * from './attachment-labels';
|
|||
export * from './database';
|
||||
export * from './reasoning-effort';
|
||||
export * from './reasoning-effort-tokens';
|
||||
export * from './recommended-mcp-servers';
|
||||
export * from './storage';
|
||||
export * from './attachment-menu';
|
||||
export * from './auto-scroll';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,2 @@
|
|||
export const MCP_SERVER_URL_PLACEHOLDER = 'https://mcp.example.com/sse';
|
||||
export const MIN_AUTOCOMPLETE_INPUT_LENGTH = 1;
|
||||
/** Number of tools shown on the compact MCP server card before collapsing to a "+ N more" badge */
|
||||
export const MCP_CARD_VISIBLE_TOOL_LIMIT = 4;
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
import { DEFAULT_MCP_CONFIG } from './mcp';
|
||||
import type { RecommendedMCPServer } from '$lib/types';
|
||||
|
||||
/**
|
||||
* Pre-defined recommended MCP servers.
|
||||
*
|
||||
* Servers are enabled by default, but they are not turned on for individual
|
||||
* conversations until the user explicitly enables them (so their tools are
|
||||
* disabled by default).
|
||||
*/
|
||||
export const RECOMMENDED_MCP_SERVERS: RecommendedMCPServer[] = [
|
||||
{
|
||||
id: 'exa-web-search',
|
||||
name: 'Exa Web Search',
|
||||
description: 'Search the web and retrieve relevant content.',
|
||||
url: 'https://mcp.exa.ai/mcp',
|
||||
enabled: true,
|
||||
requestTimeoutSeconds: DEFAULT_MCP_CONFIG.requestTimeoutSeconds
|
||||
},
|
||||
{
|
||||
id: 'huggingface-mcp',
|
||||
name: 'Hugging Face',
|
||||
description:
|
||||
'Browse models, datasets, spaces and machine learning papers from the Hugging Face hub.',
|
||||
url: 'https://huggingface.co/mcp',
|
||||
enabled: true,
|
||||
requestTimeoutSeconds: DEFAULT_MCP_CONFIG.requestTimeoutSeconds
|
||||
}
|
||||
];
|
||||
|
||||
export const RECOMMENDED_MCP_SERVER_IDS = new Set(
|
||||
RECOMMENDED_MCP_SERVERS.map((server) => server.id)
|
||||
);
|
||||
|
||||
export const RECOMMENDED_MCP_SERVERS_OPTIN_DIALOG_DELAY = 1000;
|
||||
|
|
@ -58,7 +58,6 @@ export const SETTINGS_KEYS = {
|
|||
// MCP
|
||||
MCP_SERVERS: 'mcpServers',
|
||||
MCP_REQUEST_TIMEOUT_SECONDS: 'mcpRequestTimeoutSeconds',
|
||||
MCP_DEFAULT_SERVER_OVERRIDES: 'mcpDefaultServerOverrides',
|
||||
AGENTIC_MAX_TURNS: 'agenticMaxTurns',
|
||||
AGENTIC_MAX_TOOL_PREVIEW_LINES: 'agenticMaxToolPreviewLines',
|
||||
SHOW_TOOL_CALL_IN_PROGRESS: 'showToolCallInProgress',
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import McpLogo from '$lib/components/app/mcp/McpLogo.svelte';
|
|||
import { SETTINGS_KEYS } from './settings-keys';
|
||||
import { ROUTES, SETTINGS_SECTION_SLUGS } from './routes';
|
||||
import { TITLE_GENERATION } from './title-generation';
|
||||
import { RECOMMENDED_MCP_SERVERS } from './recommended-mcp-servers';
|
||||
|
||||
export const SETTINGS_SECTION_TITLES = {
|
||||
GENERAL: 'General',
|
||||
|
|
@ -775,16 +774,9 @@ const NON_UI_SETTINGS: SettingsEntry[] = [
|
|||
key: SETTINGS_KEYS.MCP_SERVERS,
|
||||
label: 'MCP servers',
|
||||
help: 'Configure MCP servers as a JSON list. Use the form in the MCP Client settings section to edit.',
|
||||
defaultValue: JSON.stringify(RECOMMENDED_MCP_SERVERS),
|
||||
defaultValue: '[]',
|
||||
type: SettingsFieldType.INPUT,
|
||||
sync: { serverKey: SETTINGS_KEYS.MCP_SERVERS, paramType: SyncableParameterType.STRING }
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.MCP_DEFAULT_SERVER_OVERRIDES,
|
||||
label: 'MCP default server overrides',
|
||||
help: 'Per-server enable/disable defaults inherited by new chats. JSON-serialized list of {serverId, enabled} entries.',
|
||||
defaultValue: '[]',
|
||||
type: SettingsFieldType.INPUT
|
||||
}
|
||||
// {
|
||||
// key: SETTINGS_KEYS.PY_INTERPRETER_ENABLED,
|
||||
|
|
|
|||
|
|
@ -22,8 +22,6 @@ export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTool
|
|||
export const DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledToolKeys`;
|
||||
export const FAVORITE_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.favoriteModels`;
|
||||
export const REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.reasoningEffortDefault`;
|
||||
/** Set when user has interacted with the MCP server recommendations dialog (checked servers, added custom server, or dismissed) */
|
||||
export const MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.mcpServersSetupDone`;
|
||||
export const USER_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.userOverrides`;
|
||||
|
||||
/** Key prefix for per-conversation resumable stream state, conversationId is appended */
|
||||
|
|
|
|||
|
|
@ -1,80 +0,0 @@
|
|||
import { browser } from '$app/environment';
|
||||
import {
|
||||
MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY,
|
||||
RECOMMENDED_MCP_SERVER_IDS,
|
||||
RECOMMENDED_MCP_SERVERS_OPTIN_DIALOG_DELAY
|
||||
} from '$lib/constants';
|
||||
import { mcpStore } from '$lib/stores/mcp.svelte';
|
||||
|
||||
/**
|
||||
* First-run opt-in dialog for the recommended MCP servers.
|
||||
*
|
||||
* Owns the dismissed / open / trigger-timeout state and the effect that
|
||||
* schedules the dialog. Reads opt-in status and the configured server list
|
||||
* from `mcpStore`, so callers don't need to recompute on their side.
|
||||
*/
|
||||
export function useMcpRecommendations() {
|
||||
let dismissed = $state(
|
||||
browser && localStorage.getItem(MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY) === 'true'
|
||||
);
|
||||
let open = $state(false);
|
||||
let checked = $state(false);
|
||||
let triggerTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
function dismiss() {
|
||||
if (browser) {
|
||||
localStorage.setItem(MCP_SERVERS_ADDED_TO_CHAT_LOCALSTORAGE_KEY, 'true');
|
||||
}
|
||||
dismissed = true;
|
||||
open = false;
|
||||
if (triggerTimeout) {
|
||||
clearTimeout(triggerTimeout);
|
||||
triggerTimeout = null;
|
||||
}
|
||||
}
|
||||
|
||||
function handleOpenChange(next: boolean) {
|
||||
open = next;
|
||||
if (!next) dismiss();
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
if (!browser) return;
|
||||
|
||||
if (open || dismissed) {
|
||||
if (triggerTimeout) {
|
||||
clearTimeout(triggerTimeout);
|
||||
triggerTimeout = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Already evaluated once this session; leave any pending trigger alone so
|
||||
// it can still fire later. Setting `checked = true` below re-runs this
|
||||
// effect, and we must not wipe the timeout that was just scheduled.
|
||||
if (checked) return;
|
||||
|
||||
const hasRecommendations = mcpStore
|
||||
.getServers()
|
||||
.some((server) => RECOMMENDED_MCP_SERVER_IDS.has(server.id));
|
||||
|
||||
if (hasRecommendations) {
|
||||
triggerTimeout = setTimeout(() => {
|
||||
open = true;
|
||||
}, RECOMMENDED_MCP_SERVERS_OPTIN_DIALOG_DELAY);
|
||||
}
|
||||
|
||||
checked = true;
|
||||
});
|
||||
|
||||
return {
|
||||
get open() {
|
||||
return open;
|
||||
},
|
||||
get dismissed() {
|
||||
return dismissed;
|
||||
},
|
||||
dismiss,
|
||||
handleOpenChange
|
||||
};
|
||||
}
|
||||
|
|
@ -95,7 +95,7 @@ export function useToolsPanel(): UseToolsPanelReturn {
|
|||
if (toolsStore.builtinTools.length === 0 && !toolsStore.loading) {
|
||||
toolsStore.fetchBuiltinTools();
|
||||
}
|
||||
mcpStore.runHealthChecksForServers(mcpStore.getServersSorted().filter((s) => s.enabled));
|
||||
mcpStore.runHealthChecksForServers(mcpStore.getServers().filter((s) => s.enabled));
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -522,7 +522,7 @@ const mcpDefaultEnabledMigration: Migration = {
|
|||
const config = configRaw ? JSON.parse(configRaw) : {};
|
||||
|
||||
// Don't overwrite an existing config entry — current data wins.
|
||||
if (SETTINGS_KEYS.MCP_DEFAULT_SERVER_OVERRIDES in config) {
|
||||
if (MCP_DEFAULT_OVERRIDES_LEGACY_KEY in config) {
|
||||
if (import.meta.env.DEV && import.meta.env.VITE_DEBUG)
|
||||
console.log('[Migration] MCP default enabled: config already has overrides, skipping');
|
||||
return;
|
||||
|
|
@ -543,7 +543,7 @@ const mcpDefaultEnabledMigration: Migration = {
|
|||
return;
|
||||
}
|
||||
|
||||
config[SETTINGS_KEYS.MCP_DEFAULT_SERVER_OVERRIDES] = raw;
|
||||
config[MCP_DEFAULT_OVERRIDES_LEGACY_KEY] = raw;
|
||||
localStorage.setItem(CONFIG_LOCALSTORAGE_KEY, JSON.stringify(config));
|
||||
|
||||
if (import.meta.env.DEV && import.meta.env.VITE_DEBUG)
|
||||
|
|
@ -586,6 +586,83 @@ const configTypesMigration: Migration = {
|
|||
}
|
||||
};
|
||||
|
||||
const MCP_DEFAULT_OVERRIDES_LEGACY_KEY = `${STORAGE_APP_NAME}.mcpDefaultServerOverrides`;
|
||||
const MCP_DEFAULT_OVERRIDES_MERGE_MIGRATION_ID = 'mcp-default-overrides-merge-v1';
|
||||
|
||||
/**
|
||||
* Folds `mcpDefaultServerOverrides` (the legacy "default for new chats" list,
|
||||
* JSON-encoded as `[{ serverId, enabled }, ...]`) into `mcpServers[i].enabled`.
|
||||
* The legacy override key is intentionally left in the config so a downgrade
|
||||
* keeps reading it. Runs after `mcpDefaultEnabledMigration` so any legacy
|
||||
* standalone overrides are already inside the config.
|
||||
*/
|
||||
const mcpDefaultOverridesMergeMigration: Migration = {
|
||||
id: MCP_DEFAULT_OVERRIDES_MERGE_MIGRATION_ID,
|
||||
description:
|
||||
'Merge mcpDefaultServerOverrides entries onto mcpServers[i].enabled (preserves legacy key)',
|
||||
|
||||
async run(): Promise<void> {
|
||||
const configRaw = localStorage.getItem(CONFIG_LOCALSTORAGE_KEY);
|
||||
if (configRaw === null) return;
|
||||
|
||||
const config = JSON.parse(configRaw);
|
||||
const raw = config[MCP_DEFAULT_OVERRIDES_LEGACY_KEY];
|
||||
|
||||
if (typeof raw !== 'string' || raw.length === 0) {
|
||||
if (import.meta.env.DEV && import.meta.env.VITE_DEBUG)
|
||||
console.log('[Migration] MCP default overrides merge: nothing to merge');
|
||||
return;
|
||||
}
|
||||
|
||||
let overrides: { serverId: string; enabled: boolean }[];
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return;
|
||||
overrides = parsed.filter(
|
||||
(o) =>
|
||||
typeof o === 'object' &&
|
||||
o !== null &&
|
||||
typeof (o as Record<string, unknown>).serverId === 'string' &&
|
||||
typeof (o as Record<string, unknown>).enabled === 'boolean'
|
||||
) as { serverId: string; enabled: boolean }[];
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
const serversRaw = config[SETTINGS_KEYS.MCP_SERVERS];
|
||||
let servers: { id: string; enabled?: boolean }[];
|
||||
try {
|
||||
servers = typeof serversRaw === 'string' ? JSON.parse(serversRaw) : [];
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Array.isArray(servers)) servers = [];
|
||||
|
||||
let serversChanged = false;
|
||||
const knownIds = new Set(servers.map((s) => s.id));
|
||||
for (const override of overrides) {
|
||||
if (!knownIds.has(override.serverId)) continue;
|
||||
const index = servers.findIndex((s) => s.id === override.serverId);
|
||||
|
||||
if (index >= 0 && servers[index].enabled !== override.enabled) {
|
||||
servers[index] = { ...servers[index], enabled: override.enabled };
|
||||
serversChanged = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (serversChanged) {
|
||||
config[SETTINGS_KEYS.MCP_SERVERS] = JSON.stringify(servers);
|
||||
localStorage.setItem(CONFIG_LOCALSTORAGE_KEY, JSON.stringify(config));
|
||||
}
|
||||
|
||||
if (import.meta.env.DEV && import.meta.env.VITE_DEBUG)
|
||||
console.log(
|
||||
`[Migration] MCP default overrides merge: applied=${overrides.length} serversChanged=${serversChanged} (legacy key preserved)`
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const migrations: Migration[] = [
|
||||
localStorageMigration,
|
||||
idxdbMigration,
|
||||
|
|
@ -593,6 +670,7 @@ const migrations: Migration[] = [
|
|||
themeMigration,
|
||||
customJsonKeyMigration,
|
||||
mcpDefaultEnabledMigration,
|
||||
mcpDefaultOverridesMergeMigration,
|
||||
configTypesMigration
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,8 @@ import { browser } from '$app/environment';
|
|||
import { toast } from 'svelte-sonner';
|
||||
import { DatabaseService } from '$lib/services/database.service';
|
||||
import { MigrationService } from '$lib/services/migration.service';
|
||||
import { config, settingsStore } from '$lib/stores/settings.svelte';
|
||||
import { config } from '$lib/stores/settings.svelte';
|
||||
import { mcpStore } from '$lib/stores/mcp.svelte';
|
||||
import { filterByLeafNodeId, findLeafNode, generateConversationTitle } from '$lib/utils';
|
||||
import type { McpServerOverride } from '$lib/types/database';
|
||||
import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate';
|
||||
|
|
@ -46,7 +47,6 @@ import {
|
|||
ISO_TIME_SEPARATOR_REPLACEMENT,
|
||||
NON_ALPHANUMERIC_REGEX,
|
||||
MULTIPLE_UNDERSCORE_REGEX,
|
||||
SETTINGS_KEYS,
|
||||
REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY
|
||||
} from '$lib/constants';
|
||||
|
||||
|
|
@ -80,9 +80,6 @@ class ConversationsStore {
|
|||
/** Whether the store has been initialized */
|
||||
isInitialized = $state(false);
|
||||
|
||||
/** Pending MCP server overrides for new conversations (before first message) */
|
||||
pendingMcpServerOverrides = $state<McpServerOverride[]>(ConversationsStore.loadMcpDefaults());
|
||||
|
||||
/** Global (non-conversation-specific) thinking toggle default, derived from reasoning effort */
|
||||
pendingThinkingEnabled = $state(false);
|
||||
|
||||
|
|
@ -94,28 +91,6 @@ class ConversationsStore {
|
|||
/** Last non-off reasoning effort, restored when re-enabling thinking globally */
|
||||
private lastNonOffEffort: ReasoningEffort | null = null;
|
||||
|
||||
private static loadMcpDefaults(): McpServerOverride[] {
|
||||
const raw = config()[SETTINGS_KEYS.MCP_DEFAULT_SERVER_OVERRIDES];
|
||||
if (typeof raw !== 'string' || raw.length === 0) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed.filter(
|
||||
(o: unknown) => typeof o === 'object' && o !== null && 'serverId' in o && 'enabled' in o
|
||||
) as McpServerOverride[];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private saveMcpDefaults(): void {
|
||||
const plain = this.pendingMcpServerOverrides.map((o) => ({
|
||||
serverId: o.serverId,
|
||||
enabled: o.enabled
|
||||
}));
|
||||
settingsStore.updateConfig(SETTINGS_KEYS.MCP_DEFAULT_SERVER_OVERRIDES, JSON.stringify(plain));
|
||||
}
|
||||
|
||||
/** Load reasoning effort default from localStorage */
|
||||
private static loadReasoningEffortDefault(): ReasoningEffort | ReasoningEffort.OFF {
|
||||
if (typeof globalThis.localStorage === 'undefined') return ReasoningEffort.OFF;
|
||||
|
|
@ -162,11 +137,6 @@ class ConversationsStore {
|
|||
|
||||
try {
|
||||
await MigrationService.runAllMigrations();
|
||||
|
||||
// Re-read defaults after migrations: a migration may have populated
|
||||
// the settings config (e.g. moved legacy MCP overrides into it).
|
||||
this.pendingMcpServerOverrides = ConversationsStore.loadMcpDefaults();
|
||||
|
||||
await this.loadConversations();
|
||||
this.isInitialized = true;
|
||||
} catch (error) {
|
||||
|
|
@ -273,18 +243,9 @@ class ConversationsStore {
|
|||
const conversationName = name || `Chat ${new Date().toLocaleString()}`;
|
||||
const conversation = await DatabaseService.createConversation(conversationName);
|
||||
|
||||
if (this.pendingMcpServerOverrides.length > 0) {
|
||||
// Deep clone to plain objects (Svelte 5 $state uses Proxies which can't be cloned to IndexedDB)
|
||||
const plainOverrides = this.pendingMcpServerOverrides.map((o) => ({
|
||||
serverId: o.serverId,
|
||||
enabled: o.enabled
|
||||
}));
|
||||
conversation.mcpServerOverrides = plainOverrides;
|
||||
await DatabaseService.updateConversation(conversation.id, {
|
||||
mcpServerOverrides: plainOverrides
|
||||
});
|
||||
this.pendingMcpServerOverrides = [];
|
||||
}
|
||||
// New conversations inherit per-server enabled defaults directly from
|
||||
// `mcpServers[i].enabled` (see #checkServerEnabled). No per-conversation
|
||||
// override list needs to be seeded.
|
||||
|
||||
// Inherit global thinking/reasoning defaults into the new conversation
|
||||
const thinkingEnabled = this.getThinkingEnabled();
|
||||
|
|
@ -321,7 +282,6 @@ class ConversationsStore {
|
|||
return false;
|
||||
}
|
||||
|
||||
this.pendingMcpServerOverrides = [];
|
||||
this.activeConversation = conversation;
|
||||
|
||||
if (conversation.currNode) {
|
||||
|
|
@ -351,7 +311,6 @@ class ConversationsStore {
|
|||
this.activeConversation = null;
|
||||
this.activeMessages = [];
|
||||
// reload defaults so new chats inherit persisted state
|
||||
this.pendingMcpServerOverrides = ConversationsStore.loadMcpDefaults();
|
||||
this.pendingReasoningEffort = ConversationsStore.loadReasoningEffortDefault();
|
||||
}
|
||||
|
||||
|
|
@ -641,11 +600,30 @@ class ConversationsStore {
|
|||
*
|
||||
*/
|
||||
|
||||
/**
|
||||
/**
|
||||
* Resolve the per-server enabled value when no active conversation exists.
|
||||
* The default for new chats is the server's own `enabled` flag in `mcpServers`.
|
||||
*/
|
||||
#getDefaultOverrideForNoConversation(serverId: string): McpServerOverride | undefined {
|
||||
const server = mcpStore.getServers().find((s) => s.id === serverId);
|
||||
if (!server) return undefined;
|
||||
return { serverId, enabled: server.enabled };
|
||||
}
|
||||
|
||||
/**
|
||||
* Default overrides for new chats are derived from `mcpServers[i].enabled`,
|
||||
* so the global on/off state lives in one place.
|
||||
*/
|
||||
#getAllDefaultOverridesForNoConversation(): McpServerOverride[] {
|
||||
return mcpStore.getServers().map((s) => ({ serverId: s.id, enabled: s.enabled }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets MCP server override for a specific server in the active conversation.
|
||||
* Falls back to pending overrides if no active conversation exists.
|
||||
* Falls back to `mcpServers[i].enabled` if no active conversation exists.
|
||||
* @param serverId - The server ID to check
|
||||
* @returns The override if set, undefined if using global setting
|
||||
* @returns The override if set, undefined if no matching server
|
||||
*/
|
||||
getMcpServerOverride(serverId: string): McpServerOverride | undefined {
|
||||
if (this.activeConversation) {
|
||||
|
|
@ -653,18 +631,18 @@ class ConversationsStore {
|
|||
(o: McpServerOverride) => o.serverId === serverId
|
||||
);
|
||||
}
|
||||
return this.pendingMcpServerOverrides.find((o) => o.serverId === serverId);
|
||||
return this.#getDefaultOverrideForNoConversation(serverId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all MCP server overrides for the current conversation.
|
||||
* Returns pending overrides if no active conversation.
|
||||
* When no active conversation, derives from `mcpServers[i].enabled`.
|
||||
*/
|
||||
getAllMcpServerOverrides(): McpServerOverride[] {
|
||||
if (this.activeConversation?.mcpServerOverrides) {
|
||||
return this.activeConversation.mcpServerOverrides;
|
||||
}
|
||||
return this.pendingMcpServerOverrides;
|
||||
return this.#getAllDefaultOverridesForNoConversation();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -679,13 +657,16 @@ class ConversationsStore {
|
|||
|
||||
/**
|
||||
* Sets or removes MCP server override for the active conversation.
|
||||
* If no conversation exists, stores as pending override.
|
||||
* If no conversation exists, persists `enabled` onto `mcpServers[i].enabled`
|
||||
* (the single source of truth for new-chat defaults).
|
||||
* @param serverId - The server ID to override
|
||||
* @param enabled - The enabled state, or undefined to remove override
|
||||
* @param enabled - The enabled state, or undefined to remove per-conversation override
|
||||
*/
|
||||
async setMcpServerOverride(serverId: string, enabled: boolean | undefined): Promise<void> {
|
||||
if (!this.activeConversation) {
|
||||
this.setPendingMcpServerOverride(serverId, enabled);
|
||||
if (enabled !== undefined) {
|
||||
mcpStore.updateServer(serverId, { enabled });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -729,29 +710,6 @@ class ConversationsStore {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets or removes a pending MCP server override (for new conversations).
|
||||
*/
|
||||
private setPendingMcpServerOverride(serverId: string, enabled: boolean | undefined): void {
|
||||
if (enabled === undefined) {
|
||||
this.pendingMcpServerOverrides = this.pendingMcpServerOverrides.filter(
|
||||
(o) => o.serverId !== serverId
|
||||
);
|
||||
} else {
|
||||
const existingIndex = this.pendingMcpServerOverrides.findIndex(
|
||||
(o) => o.serverId === serverId
|
||||
);
|
||||
if (existingIndex >= 0) {
|
||||
const newOverrides = [...this.pendingMcpServerOverrides];
|
||||
newOverrides[existingIndex] = { serverId, enabled };
|
||||
this.pendingMcpServerOverrides = newOverrides;
|
||||
} else {
|
||||
this.pendingMcpServerOverrides = [...this.pendingMcpServerOverrides, { serverId, enabled }];
|
||||
}
|
||||
}
|
||||
this.saveMcpDefaults();
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggles MCP server enabled state for the active conversation.
|
||||
* @param serverId - The server ID to toggle
|
||||
|
|
@ -769,14 +727,6 @@ class ConversationsStore {
|
|||
await this.setMcpServerOverride(serverId, undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all pending MCP server overrides.
|
||||
*/
|
||||
clearPendingMcpServerOverrides(): void {
|
||||
this.pendingMcpServerOverrides = [];
|
||||
this.saveMcpDefaults();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the effective thinking-enabled state for the active conversation.
|
||||
* Returns the conversation override if set, otherwise the global default.
|
||||
|
|
|
|||
|
|
@ -470,18 +470,12 @@ class MCPStore {
|
|||
}
|
||||
}
|
||||
|
||||
// Fallback: try favicon from root domain
|
||||
const fallbackUrl = this.#getServerFaviconFallback(server.url);
|
||||
if (fallbackUrl) {
|
||||
return fallbackUrl;
|
||||
}
|
||||
|
||||
return null;
|
||||
return this.#getServerFaviconFallback(server.url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a fallback favicon URL from the MCP server URL.
|
||||
* e.g. https://mcp.exa.ai/mcp -> https://exa.ai/favicon.ico
|
||||
* e.g. https://mcp.example.com/sse -> https://example.com/favicon.ico
|
||||
*/
|
||||
#getServerFaviconFallback(serverUrl: string): string | null {
|
||||
try {
|
||||
|
|
@ -505,27 +499,6 @@ class MCPStore {
|
|||
return null;
|
||||
}
|
||||
|
||||
isAnyServerLoading(): boolean {
|
||||
return this.getServers().some((s) => {
|
||||
const state = this.getHealthCheckState(s.id);
|
||||
|
||||
return (
|
||||
state.status === HealthCheckStatus.IDLE || state.status === HealthCheckStatus.CONNECTING
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
getServersSorted(): MCPServerSettingsEntry[] {
|
||||
const servers = this.getServers();
|
||||
if (this.isAnyServerLoading()) {
|
||||
return servers;
|
||||
}
|
||||
|
||||
return [...servers].sort((a, b) =>
|
||||
this.getServerLabel(a).localeCompare(this.getServerLabel(b))
|
||||
);
|
||||
}
|
||||
|
||||
addServer(
|
||||
serverData: Omit<MCPServerSettingsEntry, 'id' | 'requestTimeoutSeconds'> & { id?: string }
|
||||
): MCPServerSettingsEntry {
|
||||
|
|
@ -579,10 +552,11 @@ class MCPStore {
|
|||
}
|
||||
|
||||
/**
|
||||
* MCP servers selectable in chat-add UIs and the settings page.
|
||||
* MCP servers selectable in chat-add UIs and the settings page,
|
||||
* in the order they were added to the config.
|
||||
*/
|
||||
get visibleMcpServers(): MCPServerSettingsEntry[] {
|
||||
return this.getServersSorted().filter((server) => server.enabled);
|
||||
return this.getServers().filter((server) => server.enabled);
|
||||
}
|
||||
|
||||
async ensureInitialized(perChatOverrides?: McpServerOverride[]): Promise<boolean> {
|
||||
|
|
|
|||
|
|
@ -128,7 +128,6 @@ export type {
|
|||
MCPClientConfig,
|
||||
MCPServerSettingsEntry,
|
||||
MCPServerDisplayInfo,
|
||||
RecommendedMCPServer,
|
||||
MCPToolCall,
|
||||
OpenAIToolDefinition,
|
||||
ServerStatus,
|
||||
|
|
|
|||
9
tools/ui/src/lib/types/mcp.d.ts
vendored
9
tools/ui/src/lib/types/mcp.d.ts
vendored
|
|
@ -226,15 +226,6 @@ export type MCPServerSettingsEntry = MCPServerDisplayInfo & {
|
|||
useProxy?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Pre-defined recommended MCP server shown to the user in onboarding/picker UIs.
|
||||
*/
|
||||
export interface RecommendedMCPServer extends MCPServerDisplayInfo {
|
||||
description: string;
|
||||
enabled: boolean;
|
||||
requestTimeoutSeconds: number;
|
||||
}
|
||||
|
||||
export interface MCPHostManagerConfig {
|
||||
servers: MCPClientConfig['servers'];
|
||||
clientInfo?: Implementation;
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
import { onMount } from 'svelte';
|
||||
|
||||
import { SidebarNavigation, DialogConversationTitleUpdate } from '$lib/components/app';
|
||||
import { DialogMcpServerRecommendations } from '$lib/components/app/dialogs';
|
||||
import { PwaMetaTags, PwaRefreshAlert } from '$lib/components/pwa';
|
||||
import { pwaAssetsHead } from 'virtual:pwa-assets/head';
|
||||
|
||||
|
|
@ -27,7 +26,6 @@
|
|||
import { FAVICON_PATHS, FAVICON_SELECTORS } from '$lib/constants/pwa';
|
||||
import { useKeyboardShortcuts } from '$lib/hooks/use-keyboard-shortcuts.svelte';
|
||||
import { usePwa } from '$lib/hooks/use-pwa.svelte';
|
||||
import { useMcpRecommendations } from '$lib/hooks/use-mcp-recommendations.svelte';
|
||||
import { conversations } from '$lib/stores/conversations.svelte';
|
||||
import { isMobile } from '$lib/stores/viewport.svelte';
|
||||
import { theme } from '$lib/stores/theme.svelte';
|
||||
|
|
@ -39,8 +37,6 @@
|
|||
let innerHeight = $state<number | undefined>();
|
||||
let innerWidth = $state(browser ? window.innerWidth : 0);
|
||||
|
||||
const mcpRecommendations = useMcpRecommendations();
|
||||
|
||||
let chatSidebar:
|
||||
| {
|
||||
activateSearchMode?: () => void;
|
||||
|
|
@ -239,7 +235,10 @@
|
|||
});
|
||||
|
||||
// Background MCP server health checks on app load
|
||||
// Fetch enabled servers from settings and run health checks in background
|
||||
// Fetch enabled servers from settings and run health checks in background.
|
||||
// Only IDLE servers are checked; already-resolved (SUCCESS / ERROR) servers
|
||||
// keep their existing state, so adding or removing a server does not flash
|
||||
// every other card back through skeleton state.
|
||||
$effect(() => {
|
||||
if (!browser) return;
|
||||
|
||||
|
|
@ -251,7 +250,7 @@
|
|||
if (enabledServers.length > 0) {
|
||||
untrack(() => {
|
||||
// Run health checks in background (don't await)
|
||||
mcpStore.runHealthChecksForServers(enabledServers, false).catch((error) => {
|
||||
mcpStore.runHealthChecksForServers(enabledServers, true).catch((error) => {
|
||||
console.warn('[layout] MCP health checks failed:', error);
|
||||
});
|
||||
});
|
||||
|
|
@ -325,11 +324,6 @@
|
|||
onConfirm={handleTitleUpdateConfirm}
|
||||
onCancel={handleTitleUpdateCancel}
|
||||
/>
|
||||
|
||||
<DialogMcpServerRecommendations
|
||||
open={mcpRecommendations.open}
|
||||
onOpenChange={mcpRecommendations.handleOpenChange}
|
||||
/>
|
||||
</Tooltip.Provider>
|
||||
|
||||
<!-- PWA update prompt + version -->
|
||||
|
|
|
|||
158
tools/ui/tests/unit/mcp-default-overrides-merge.test.ts
Normal file
158
tools/ui/tests/unit/mcp-default-overrides-merge.test.ts
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { STORAGE_APP_NAME, CONFIG_LOCALSTORAGE_KEY } from '$lib/constants';
|
||||
|
||||
// node env unit project has no DOM, install a minimal localStorage backed by a Map
|
||||
beforeAll(() => {
|
||||
const store = new Map<string, string>();
|
||||
const polyfill: Storage = {
|
||||
get length() {
|
||||
return store.size;
|
||||
},
|
||||
clear: () => store.clear(),
|
||||
getItem: (k) => (store.has(k) ? store.get(k)! : null),
|
||||
key: (i) => Array.from(store.keys())[i] ?? null,
|
||||
removeItem: (k) => {
|
||||
store.delete(k);
|
||||
},
|
||||
setItem: (k, v) => {
|
||||
store.set(k, String(v));
|
||||
}
|
||||
};
|
||||
(globalThis as unknown as { localStorage: Storage }).localStorage = polyfill;
|
||||
});
|
||||
|
||||
/**
|
||||
* Migration `mcp-default-overrides-merge-v1` folds the values of the parallel
|
||||
* `mcpDefaultServerOverrides` config entry onto `mcpServers[i].enabled` (the
|
||||
* single source of truth for new-chat defaults). The legacy key is kept on
|
||||
* disk for downgrade compatibility.
|
||||
*/
|
||||
describe('mcp-default-overrides-merge-v1 migration', () => {
|
||||
const MIGRATION_STATE_KEY = `${STORAGE_APP_NAME}.migration-state`;
|
||||
const MCP_DEFAULT_OVERRIDES_KEY = `${STORAGE_APP_NAME}.mcpDefaultServerOverrides`;
|
||||
|
||||
beforeEach(async () => {
|
||||
localStorage.clear();
|
||||
// Reset the migration run counter so `runAllMigrations` is guaranteed to execute.
|
||||
await import('$lib/services/migration.service').then((mod) =>
|
||||
mod.MigrationService.resetState()
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
localStorage.clear();
|
||||
});
|
||||
|
||||
async function runMigrations() {
|
||||
const { MigrationService } = await import('$lib/services/migration.service');
|
||||
await MigrationService.runAllMigrations();
|
||||
}
|
||||
|
||||
function readConfig(): Record<string, unknown> {
|
||||
const raw = localStorage.getItem(CONFIG_LOCALSTORAGE_KEY);
|
||||
|
||||
return raw ? (JSON.parse(raw) as Record<string, unknown>) : {};
|
||||
}
|
||||
|
||||
function writeConfig(config: Record<string, unknown>) {
|
||||
localStorage.setItem(CONFIG_LOCALSTORAGE_KEY, JSON.stringify(config));
|
||||
}
|
||||
|
||||
it('applies matching overrides onto mcpServers[i].enabled and preserves the legacy key', async () => {
|
||||
writeConfig({
|
||||
mcpServers: JSON.stringify([
|
||||
{ id: 'exa', enabled: false, url: 'https://mcp.exa.ai/mcp' },
|
||||
{ id: 'hf', enabled: false, url: 'https://huggingface.co/mcp' }
|
||||
]),
|
||||
[MCP_DEFAULT_OVERRIDES_KEY]: JSON.stringify([
|
||||
{ serverId: 'exa', enabled: true },
|
||||
{ serverId: 'hf', enabled: false }
|
||||
])
|
||||
});
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const after = readConfig();
|
||||
const servers = JSON.parse(after.mcpServers as string) as Array<{
|
||||
id: string;
|
||||
enabled: boolean;
|
||||
}>;
|
||||
|
||||
expect(servers.find((s) => s.id === 'exa')?.enabled).toBe(true);
|
||||
expect(servers.find((s) => s.id === 'hf')?.enabled).toBe(false);
|
||||
expect(MCP_DEFAULT_OVERRIDES_KEY in after).toBe(true);
|
||||
});
|
||||
|
||||
it('skips override ids that do not match any configured server', async () => {
|
||||
writeConfig({
|
||||
mcpServers: JSON.stringify([{ id: 'exa', enabled: false, url: 'https://mcp.exa.ai/mcp' }]),
|
||||
[MCP_DEFAULT_OVERRIDES_KEY]: JSON.stringify([
|
||||
{ serverId: 'orphan', enabled: true },
|
||||
{ serverId: 'exa', enabled: true }
|
||||
])
|
||||
});
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const after = readConfig();
|
||||
const servers = JSON.parse(after.mcpServers as string) as Array<{
|
||||
id: string;
|
||||
enabled: boolean;
|
||||
}>;
|
||||
|
||||
expect(servers).toHaveLength(1);
|
||||
expect(servers[0].enabled).toBe(true);
|
||||
expect(MCP_DEFAULT_OVERRIDES_KEY in after).toBe(true);
|
||||
});
|
||||
|
||||
it('is a no-op when there are no legacy overrides', async () => {
|
||||
writeConfig({
|
||||
mcpServers: JSON.stringify([{ id: 'exa', enabled: true, url: 'https://mcp.exa.ai/mcp' }])
|
||||
});
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const after = readConfig();
|
||||
const servers = JSON.parse(after.mcpServers as string) as Array<{
|
||||
id: string;
|
||||
enabled: boolean;
|
||||
}>;
|
||||
|
||||
expect(servers[0].enabled).toBe(true);
|
||||
expect(MCP_DEFAULT_OVERRIDES_KEY in after).toBe(false);
|
||||
});
|
||||
|
||||
it('does not rewrite mcpServers when override.enabled already matches', async () => {
|
||||
const originalServers = JSON.stringify([
|
||||
{ id: 'exa', enabled: true, url: 'https://mcp.exa.ai/mcp' }
|
||||
]);
|
||||
|
||||
writeConfig({
|
||||
mcpServers: originalServers,
|
||||
[MCP_DEFAULT_OVERRIDES_KEY]: JSON.stringify([{ serverId: 'exa', enabled: true }])
|
||||
});
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const after = readConfig();
|
||||
expect(after.mcpServers).toBe(originalServers);
|
||||
expect(MCP_DEFAULT_OVERRIDES_KEY in after).toBe(true);
|
||||
});
|
||||
|
||||
it('records itself as completed so subsequent loads do not re-run', async () => {
|
||||
writeConfig({
|
||||
mcpServers: JSON.stringify([{ id: 'exa', enabled: false, url: 'https://mcp.exa.ai/mcp' }]),
|
||||
[MCP_DEFAULT_OVERRIDES_KEY]: JSON.stringify([{ serverId: 'exa', enabled: true }])
|
||||
});
|
||||
|
||||
const { MigrationService } = await import('$lib/services/migration.service');
|
||||
|
||||
await MigrationService.runAllMigrations();
|
||||
|
||||
const stateRaw = localStorage.getItem(MIGRATION_STATE_KEY);
|
||||
expect(stateRaw).not.toBeNull();
|
||||
const state = JSON.parse(stateRaw!) as { completed: string[]; failed: string[] };
|
||||
expect(state.completed).toContain('mcp-default-overrides-merge-v1');
|
||||
expect(state.failed).not.toContain('mcp-default-overrides-merge-v1');
|
||||
});
|
||||
});
|
||||
19
tools/ui/tests/unit/mcp-servers-default.test.ts
Normal file
19
tools/ui/tests/unit/mcp-servers-default.test.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { SETTINGS_KEYS } from '$lib/constants/settings-keys';
|
||||
|
||||
/**
|
||||
* Default-value policy for the `MCP_SERVERS` setting.
|
||||
*
|
||||
* Earlier versions of the UI preloaded a hard-coded list of suggested
|
||||
* MCP servers into this setting on first install. That caused silent
|
||||
* third-party HTTP requests at app load (see issue #25509) and a popup
|
||||
* "recommendation" dialog (see issue #25274). New users must now opt
|
||||
* in explicitly when adding a server, so the default is an empty list.
|
||||
*/
|
||||
describe('MCP_SERVERS default value', () => {
|
||||
it('does not preload any servers in the MCP_SERVERS setting default', async () => {
|
||||
const { SETTING_CONFIG_DEFAULT } = await import('$lib/constants/settings-registry');
|
||||
|
||||
expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.MCP_SERVERS]).toBe('[]');
|
||||
}, 15000);
|
||||
});
|
||||
|
|
@ -5,11 +5,10 @@ import { DEFAULT_MCP_CONFIG, MCP_SERVER_ID_PREFIX } from '$lib/constants/mcp';
|
|||
/**
|
||||
* Tests for the mcpServers settings parser.
|
||||
*
|
||||
* The branch seeds the MCP servers setting with a default value of
|
||||
* `JSON.stringify(RECOMMENDED_MCP_SERVERS)`, so the parser has to be
|
||||
* resilient to anything that may live in the user's localStorage: malformed
|
||||
* JSON, wrong shapes, missing fields, falsy-but-not-zero numbers, and entry
|
||||
* arrays that have been mutated by the user via the settings form.
|
||||
* The parser has to be resilient to anything that may live in the
|
||||
* user's localStorage: malformed JSON, wrong shapes, missing fields,
|
||||
* falsy-but-not-zero numbers, and entry arrays that have been mutated
|
||||
* by the user via the settings form.
|
||||
*/
|
||||
describe('parseMcpServerSettings', () => {
|
||||
it('returns an empty array for falsy or whitespace-only input', () => {
|
||||
|
|
|
|||
|
|
@ -1,90 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
RECOMMENDED_MCP_SERVER_IDS,
|
||||
RECOMMENDED_MCP_SERVERS
|
||||
} from '$lib/constants/recommended-mcp-servers';
|
||||
import { parseMcpServerSettings } from '$lib/utils/mcp';
|
||||
import { DEFAULT_MCP_CONFIG, MCP_SERVER_ID_PREFIX } from '$lib/constants/mcp';
|
||||
|
||||
/**
|
||||
* Tests for the predefined recommended MCP servers.
|
||||
*
|
||||
* These are surfaced to first-time users via
|
||||
* DialogMcpServerRecommendations and used as the default value of the MCP
|
||||
* servers setting, so a regression that breaks the round-trip through the
|
||||
* settings parser would silently break onboarding for new users.
|
||||
*/
|
||||
describe('RECOMMENDED_MCP_SERVERS', () => {
|
||||
it('lists at least one entry and uses stable, unique ids', () => {
|
||||
expect(RECOMMENDED_MCP_SERVERS.length).toBeGreaterThan(0);
|
||||
|
||||
const ids = RECOMMENDED_MCP_SERVERS.map((server) => server.id);
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
|
||||
for (const id of ids) {
|
||||
expect(id).toMatch(/^[a-z0-9-]+$/);
|
||||
expect(id.toLowerCase()).not.toContain(MCP_SERVER_ID_PREFIX.toLowerCase());
|
||||
}
|
||||
});
|
||||
|
||||
it('requires a name, description and url for every entry', () => {
|
||||
for (const server of RECOMMENDED_MCP_SERVERS) {
|
||||
expect(server.name?.trim().length ?? 0).toBeGreaterThan(0);
|
||||
expect(server.description.trim().length).toBeGreaterThan(0);
|
||||
expect(server.url.trim().length).toBeGreaterThan(0);
|
||||
expect(() => new URL(server.url)).not.toThrow();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('RECOMMENDED_MCP_SERVER_IDS', () => {
|
||||
it('matches the ids declared in RECOMMENDED_MCP_SERVERS', () => {
|
||||
expect(RECOMMENDED_MCP_SERVER_IDS.size).toBe(RECOMMENDED_MCP_SERVERS.length);
|
||||
|
||||
for (const server of RECOMMENDED_MCP_SERVERS) {
|
||||
expect(RECOMMENDED_MCP_SERVER_IDS.has(server.id)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('recommended-mcp-servers default value', () => {
|
||||
it('round-trips cleanly through parseMcpServerSettings', () => {
|
||||
const serialized = JSON.stringify(RECOMMENDED_MCP_SERVERS);
|
||||
const parsed = parseMcpServerSettings(serialized);
|
||||
|
||||
expect(parsed).toHaveLength(RECOMMENDED_MCP_SERVERS.length);
|
||||
|
||||
for (let index = 0; index < RECOMMENDED_MCP_SERVERS.length; index++) {
|
||||
const source = RECOMMENDED_MCP_SERVERS[index];
|
||||
const entry = parsed[index];
|
||||
|
||||
expect(entry).toBeDefined();
|
||||
expect(entry?.id).toBe(source.id);
|
||||
expect(entry?.url).toBe(source.url);
|
||||
expect(entry?.enabled).toBe(source.enabled);
|
||||
expect(entry?.requestTimeoutSeconds).toBe(source.requestTimeoutSeconds);
|
||||
expect(entry?.name).toBe(source.name);
|
||||
|
||||
// Headers and useProxy are not set on recommended servers; the
|
||||
// parser must fall back to the inactive defaults rather than
|
||||
// surfacing undefined-boundary states.
|
||||
expect(entry?.headers).toBeUndefined();
|
||||
expect(entry?.useProxy).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('uses the global default timeout when one is not specified on an entry', () => {
|
||||
const sourceOnlyRequired = {
|
||||
id: 'roundtrip-only',
|
||||
name: 'Only required fields',
|
||||
url: 'https://example.test/mcp',
|
||||
description: 'Smoke entry for parser roundtrip with default timeout.',
|
||||
enabled: true
|
||||
};
|
||||
|
||||
const parsed = parseMcpServerSettings(JSON.stringify([sourceOnlyRequired]));
|
||||
const entry = parsed[0];
|
||||
|
||||
expect(entry?.requestTimeoutSeconds).toBe(DEFAULT_MCP_CONFIG.requestTimeoutSeconds);
|
||||
});
|
||||
});
|
||||
35
vendor/cpp-httplib/httplib.cpp
vendored
35
vendor/cpp-httplib/httplib.cpp
vendored
|
|
@ -3705,6 +3705,12 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider,
|
|||
// Trailer
|
||||
if (trailer) {
|
||||
for (const auto &kv : *trailer) {
|
||||
// Skip fields with invalid names or values to prevent response
|
||||
// splitting via CR/LF injection, matching set_header().
|
||||
if (!fields::is_field_name(kv.first) ||
|
||||
!fields::is_field_value(kv.second)) {
|
||||
continue;
|
||||
}
|
||||
std::string field_line = kv.first + ": " + kv.second + "\r\n";
|
||||
if (!write_data(strm, field_line.data(), field_line.size())) {
|
||||
ok = false;
|
||||
|
|
@ -8301,8 +8307,8 @@ void Server::apply_ranges(const Request &req, Response &res,
|
|||
}
|
||||
}
|
||||
|
||||
auto length = std::to_string(res.body.size());
|
||||
res.set_header("Content-Length", length);
|
||||
res.content_length_ = res.body.size();
|
||||
res.set_header("Content-Length", std::to_string(res.content_length_));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -10270,6 +10276,11 @@ Result ClientImpl::Get(const std::string &path,
|
|||
return Get(path, Headers(), std::move(progress));
|
||||
}
|
||||
|
||||
Result ClientImpl::Get(const std::string &path, const Params ¶ms,
|
||||
DownloadProgress progress) {
|
||||
return Get(path, params, Headers(), std::move(progress));
|
||||
}
|
||||
|
||||
Result ClientImpl::Get(const std::string &path, const Params ¶ms,
|
||||
const Headers &headers,
|
||||
DownloadProgress progress) {
|
||||
|
|
@ -11348,6 +11359,10 @@ Result Client::Get(const std::string &path, const Headers &headers,
|
|||
return cli_->Get(path, headers, std::move(response_handler),
|
||||
std::move(content_receiver), std::move(progress));
|
||||
}
|
||||
Result Client::Get(const std::string &path, const Params ¶ms,
|
||||
DownloadProgress progress) {
|
||||
return cli_->Get(path, params, std::move(progress));
|
||||
}
|
||||
Result Client::Get(const std::string &path, const Params ¶ms,
|
||||
const Headers &headers, DownloadProgress progress) {
|
||||
return cli_->Get(path, params, headers, std::move(progress));
|
||||
|
|
@ -12076,11 +12091,18 @@ bool SSLServer::update_certs_pem(const char *cert_pem,
|
|||
|
||||
// SSL HTTP client implementation
|
||||
SSLClient::~SSLClient() {
|
||||
if (ctx_) { tls::free_context(ctx_); }
|
||||
// Make sure to shut down SSL since shutdown_ssl will resolve to the
|
||||
// base function rather than the derived function once we get to the
|
||||
// base class destructor, and won't free the SSL (causing a leak).
|
||||
// This must happen before the context is freed below: some backends
|
||||
// (e.g. mbedTLS) have the SSL session borrow a raw pointer into the
|
||||
// context, so freeing the context first leaves close_notify reading
|
||||
// freed memory.
|
||||
shutdown_ssl_impl(socket_, true);
|
||||
if (ctx_) {
|
||||
tls::free_context(ctx_);
|
||||
ctx_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
bool SSLClient::is_valid() const { return ctx_ != nullptr; }
|
||||
|
|
@ -16501,6 +16523,11 @@ WebSocketClient::~WebSocketClient() {
|
|||
bool WebSocketClient::is_valid() const { return is_valid_; }
|
||||
|
||||
void WebSocketClient::shutdown_and_close() {
|
||||
// Send the close frame while the TLS session is still alive: ws_ holds an
|
||||
// SSLSocketStream that keeps a raw pointer to tls_session_, so the session
|
||||
// must outlive ws_->close() and ws_.reset() to avoid a use-after-free.
|
||||
if (ws_ && ws_->is_open()) { ws_->close(); }
|
||||
ws_.reset();
|
||||
#ifdef CPPHTTPLIB_SSL_ENABLED
|
||||
if (is_ssl_) {
|
||||
if (tls_session_) {
|
||||
|
|
@ -16510,8 +16537,6 @@ void WebSocketClient::shutdown_and_close() {
|
|||
}
|
||||
}
|
||||
#endif
|
||||
if (ws_ && ws_->is_open()) { ws_->close(); }
|
||||
ws_.reset();
|
||||
if (sock_ != INVALID_SOCKET) {
|
||||
detail::shutdown_socket(sock_);
|
||||
detail::close_socket(sock_);
|
||||
|
|
|
|||
6
vendor/cpp-httplib/httplib.h
vendored
6
vendor/cpp-httplib/httplib.h
vendored
|
|
@ -8,8 +8,8 @@
|
|||
#ifndef CPPHTTPLIB_HTTPLIB_H
|
||||
#define CPPHTTPLIB_HTTPLIB_H
|
||||
|
||||
#define CPPHTTPLIB_VERSION "0.49.0"
|
||||
#define CPPHTTPLIB_VERSION_NUM "0x003100"
|
||||
#define CPPHTTPLIB_VERSION "0.50.1"
|
||||
#define CPPHTTPLIB_VERSION_NUM "0x003201"
|
||||
|
||||
#ifdef _WIN32
|
||||
#if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00
|
||||
|
|
@ -2219,6 +2219,7 @@ public:
|
|||
Result Get(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Params ¶ms, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Params ¶ms, const Headers &headers, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr);
|
||||
|
|
@ -2602,6 +2603,7 @@ public:
|
|||
Result Get(const std::string &path, const Headers &headers, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Params ¶ms, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Params ¶ms, const Headers &headers, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ContentReceiver content_receiver, DownloadProgress progress = nullptr);
|
||||
Result Get(const std::string &path, const Params ¶ms, const Headers &headers, ResponseHandler response_handler, ContentReceiver content_receiver, DownloadProgress progress = nullptr);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue