mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-22 06:45:11 +00:00
Merge branch 'upstream' into concedo_experimental
# Conflicts: # .github/workflows/ui-publish.yml # CODEOWNERS # docs/ops.md # ggml/CMakeLists.txt # ggml/src/CMakeLists.txt # ggml/src/ggml-hexagon/htp/CMakeLists.txt # ggml/src/ggml-hexagon/htp/argsort-ops.c # scripts/sync-ggml.last # tools/mtmd/tests/test-deepseek-ocr.py
This commit is contained in:
Concedo
commit
f57cd915a9
32 changed files with 1234 additions and 650 deletions
|
|
@ -489,12 +489,15 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
task.opts = opts;
|
||||
tasks.push_back(task);
|
||||
}
|
||||
|
||||
bool had_spec_url = false;
|
||||
if (!params.speculative.draft.mparams.url.empty()) {
|
||||
common_download_task task;
|
||||
task.url = params.speculative.draft.mparams.url;
|
||||
task.local_path = params.speculative.draft.mparams.path;
|
||||
task.opts = opts;
|
||||
tasks.push_back(task);
|
||||
had_spec_url = true;
|
||||
}
|
||||
|
||||
// handle hf_plan tasks
|
||||
|
|
@ -514,6 +517,18 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
});
|
||||
}
|
||||
};
|
||||
|
||||
// handle plan_spec (e.g. --spec-draft-hf)
|
||||
if (!plan_spec.model_files.empty() && !had_spec_url) {
|
||||
add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams);
|
||||
had_spec_url = true;
|
||||
}
|
||||
|
||||
// handle vocoder plan (e.g. --hf-repo-v)
|
||||
if (!plan_voc.model_files.empty()) {
|
||||
add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model);
|
||||
}
|
||||
|
||||
if (!plan.model_files.empty()) {
|
||||
add_tasks(plan.model_files, plan.primary, params.model);
|
||||
}
|
||||
|
|
@ -522,7 +537,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
params.mmproj.path = hf_cache::finalize_file(plan.mmproj);
|
||||
});
|
||||
}
|
||||
if (!plan.mtp.local_path.empty()) {
|
||||
if (!plan.mtp.local_path.empty() && !had_spec_url) {
|
||||
tasks.emplace_back(plan.mtp, opts, [&]() {
|
||||
// only fall back to the discovered MTP head when no draft was explicitly provided
|
||||
if (params.speculative.draft.mparams.empty()) {
|
||||
|
|
@ -541,16 +556,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
});
|
||||
}
|
||||
|
||||
// handle plan_spec (e.g. --spec-draft-hf)
|
||||
if (!plan_spec.model_files.empty()) {
|
||||
add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams);
|
||||
}
|
||||
|
||||
// handle vocoder plan (e.g. --hf-repo-v)
|
||||
if (!plan_voc.model_files.empty()) {
|
||||
add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model);
|
||||
}
|
||||
|
||||
// run all tasks in parallel
|
||||
if (!params.offline) {
|
||||
// if duplicated files are found, only download once (but still call on_done for each task)
|
||||
|
|
@ -563,6 +568,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
}
|
||||
std::vector<common_download_task> unique_tasks_vec;
|
||||
for (auto & pair : unique_tasks) {
|
||||
LOG_DBG("download task: %s -> %s\n", pair.second->url.c_str(), pair.second->local_path.c_str());
|
||||
unique_tasks_vec.push_back(*pair.second);
|
||||
}
|
||||
common_download_run_tasks(unique_tasks_vec);
|
||||
|
|
@ -3037,7 +3043,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
{"--tools"}, "TOOL1,TOOL2,...",
|
||||
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
|
||||
"specify \"all\" to enable all tools\n"
|
||||
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, apply_diff, get_datetime",
|
||||
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.server_tools = parse_csv_row(value);
|
||||
}
|
||||
|
|
|
|||
28
ggml/include/ggml-et.h
Normal file
28
ggml/include/ggml-et.h
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
#pragma once
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define GGML_ET_NAME "ET"
|
||||
|
||||
// backend API
|
||||
GGML_BACKEND_API ggml_guid_t ggml_backend_et_guid(void);
|
||||
GGML_BACKEND_API ggml_backend_t ggml_backend_et_init(size_t devidx);
|
||||
|
||||
GGML_BACKEND_API bool ggml_backend_is_et(ggml_backend_t backend);
|
||||
GGML_BACKEND_API int ggml_backend_et_get_device_count(void);
|
||||
GGML_BACKEND_API void ggml_backend_et_get_device_description(int devidx, char * description, size_t description_size);
|
||||
GGML_BACKEND_API void ggml_backend_et_get_device_memory(int devidx, size_t * free, size_t * total);
|
||||
|
||||
GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_et_buffer_type(size_t dev_num);
|
||||
GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_et_host_buffer_type(void);
|
||||
|
||||
GGML_BACKEND_API ggml_backend_reg_t ggml_backend_et_reg(void);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
@ -86,6 +86,10 @@
|
|||
#include "ggml-openvino.h"
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_ET
|
||||
#include "ggml-et.h"
|
||||
#endif
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
static std::string path_str(const fs::path & path) {
|
||||
|
|
@ -161,6 +165,9 @@ struct ggml_backend_registry {
|
|||
#ifdef GGML_USE_OPENVINO
|
||||
register_backend(ggml_backend_openvino_reg());
|
||||
#endif
|
||||
#ifdef GGML_USE_ET
|
||||
register_backend(ggml_backend_et_reg());
|
||||
#endif
|
||||
#ifdef GGML_USE_CPU
|
||||
register_backend(ggml_backend_cpu_reg());
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -263,13 +263,13 @@ void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
|||
const uint8x16_t raw16 = vcombine_u8(raw, raw);
|
||||
|
||||
// First 16 elements: replicate bytes 0-3, shift, mask, subtract 1
|
||||
uint8x16_t bytes0 = vqtbl1q_u8(raw16, idx_lo);
|
||||
uint8x16_t bytes0 = ggml_vqtbl1q_u8(raw16, idx_lo);
|
||||
int8x16_t qv0 = vsubq_s8(
|
||||
vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes0, shifts), mask2)),
|
||||
one);
|
||||
|
||||
// Second 16 elements: replicate bytes 4-7, shift, mask, subtract 1
|
||||
uint8x16_t bytes1 = vqtbl1q_u8(raw16, idx_hi);
|
||||
uint8x16_t bytes1 = ggml_vqtbl1q_u8(raw16, idx_hi);
|
||||
int8x16_t qv1 = vsubq_s8(
|
||||
vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes1, shifts), mask2)),
|
||||
one);
|
||||
|
|
|
|||
|
|
@ -646,7 +646,7 @@ static void dsv4_set_kq_mask(
|
|||
return;
|
||||
}
|
||||
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(n_stream > 0);
|
||||
GGML_ASSERT(n_tokens%n_stream == 0);
|
||||
GGML_ASSERT(dst->ne[0] == plan.n_kv);
|
||||
|
|
@ -656,13 +656,27 @@ static void dsv4_set_kq_mask(
|
|||
GGML_ASSERT((int64_t) plan.n_visible.size() == (int64_t) n_tokens);
|
||||
GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
|
||||
|
||||
float * data = (float *) dst->data;
|
||||
if (dst->type == GGML_TYPE_F32) {
|
||||
float * data = (float *) dst->data;
|
||||
|
||||
for (int64_t i = 0; i < (int64_t) n_tokens; ++i) {
|
||||
const int32_t n_visible = plan.n_visible[i];
|
||||
for (int64_t i = 0; i < (int64_t) n_tokens; ++i) {
|
||||
const int32_t n_visible = plan.n_visible[i];
|
||||
|
||||
for (int64_t j = 0; j < dst->ne[0]; ++j) {
|
||||
data[i*dst->ne[0] + j] = j < n_visible ? 0.0f : -INFINITY;
|
||||
for (int64_t j = 0; j < dst->ne[0]; ++j) {
|
||||
data[i*dst->ne[0] + j] = j < n_visible ? 0.0f : -INFINITY;
|
||||
}
|
||||
}
|
||||
} else if (dst->type == GGML_TYPE_F16) {
|
||||
ggml_fp16_t * data = (ggml_fp16_t *) dst->data;
|
||||
const ggml_fp16_t fp16_ninf = llama_cast<ggml_fp16_t>(-INFINITY);
|
||||
const ggml_fp16_t fp16_zero = llama_cast<ggml_fp16_t>(0.0f);
|
||||
|
||||
for (int64_t i = 0; i < (int64_t) n_tokens; ++i) {
|
||||
const int32_t n_visible = plan.n_visible[i];
|
||||
|
||||
for (int64_t j = 0; j < dst->ne[0]; ++j) {
|
||||
data[i*dst->ne[0] + j] = j < n_visible ? fp16_zero : fp16_ninf;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -679,8 +693,7 @@ static ggml_tensor * dsv4_build_raw_kq_mask(
|
|||
GGML_ASSERT(n_stream > 0);
|
||||
GGML_ASSERT(n_tokens%n_stream == 0);
|
||||
|
||||
const bool use_fattn = cparams.flash_attn && (!cparams.kv_unified || n_stream == 1);
|
||||
const auto type = use_fattn ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
const auto type = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
|
||||
ggml_tensor * res = ggml_new_tensor_4d(ctx, type, n_kv, n_tokens/n_stream, 1, n_stream);
|
||||
ggml_set_input(res);
|
||||
|
|
@ -815,6 +828,7 @@ static void dsv4_build_comp_inputs(
|
|||
llm_graph_input_dsv4::comp_input & inp,
|
||||
const llama_kv_cache_dsv4_context::comp_plan & plan,
|
||||
const char * name,
|
||||
const llama_cparams & cparams,
|
||||
int64_t n_stream) {
|
||||
inp.state_pos = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_pos.size(), std::string("dsv4_") + name + "_state_pos");
|
||||
inp.state_persist_src_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_persist_src_idxs.size(), std::string("dsv4_") + name + "_state_persist_src_idxs");
|
||||
|
|
@ -829,7 +843,7 @@ static void dsv4_build_comp_inputs(
|
|||
GGML_ASSERT(n_stream > 0);
|
||||
GGML_ASSERT(n_tokens%n_stream == 0);
|
||||
|
||||
inp.kq_mask = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, plan.n_kv, n_tokens/n_stream, 1, n_stream);
|
||||
inp.kq_mask = ggml_new_tensor_4d(ctx, cparams.flash_attn && strcmp(name, "lid") != 0 ? GGML_TYPE_F16 : GGML_TYPE_F32, plan.n_kv, n_tokens/n_stream, 1, n_stream);
|
||||
ggml_set_input(inp.kq_mask);
|
||||
ggml_set_name(inp.kq_mask, (std::string("dsv4_") + name + "_kq_mask").c_str());
|
||||
}
|
||||
|
|
@ -3077,9 +3091,9 @@ llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const {
|
|||
inp_raw->self_k_rot = raw_ctx->build_input_k_rot(ctx0);
|
||||
auto inp = std::make_unique<llm_graph_input_dsv4>(cparams, std::move(inp_raw), mctx_cur);
|
||||
|
||||
dsv4_build_comp_inputs(ctx0, inp->inp_csa, mctx_cur->get_csa_plan(ubatch), "csa", n_stream);
|
||||
dsv4_build_comp_inputs(ctx0, inp->inp_hca, mctx_cur->get_hca_plan(ubatch), "hca", n_stream);
|
||||
dsv4_build_comp_inputs(ctx0, inp->inp_lid, mctx_cur->get_lid_plan(ubatch), "lid", n_stream);
|
||||
dsv4_build_comp_inputs(ctx0, inp->inp_csa, mctx_cur->get_csa_plan(ubatch), "csa", cparams, n_stream);
|
||||
dsv4_build_comp_inputs(ctx0, inp->inp_hca, mctx_cur->get_hca_plan(ubatch), "hca", cparams, n_stream);
|
||||
dsv4_build_comp_inputs(ctx0, inp->inp_lid, mctx_cur->get_lid_plan(ubatch), "lid", cparams, n_stream);
|
||||
inp->inp_csa.k_rot = mctx_cur->get_csa()->build_input_k_rot(ctx0);
|
||||
inp->inp_hca.k_rot = mctx_cur->get_hca()->build_input_k_rot(ctx0);
|
||||
inp->inp_lid.k_rot = mctx_cur->get_lid()->build_input_k_rot(ctx0);
|
||||
|
|
|
|||
|
|
@ -474,38 +474,38 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
|
|||
const llama_hparams & hparams = ud->model->hparams;
|
||||
const std::string tensor_name = tensor->name;
|
||||
|
||||
const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");
|
||||
const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");
|
||||
const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");
|
||||
const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");
|
||||
const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");
|
||||
const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");
|
||||
const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");
|
||||
const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");
|
||||
const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");
|
||||
const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
|
||||
const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");
|
||||
const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");
|
||||
static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");
|
||||
static const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");
|
||||
static const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");
|
||||
static const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");
|
||||
static const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");
|
||||
static const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");
|
||||
static const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");
|
||||
static const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");
|
||||
static const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");
|
||||
static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
|
||||
static const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");
|
||||
static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");
|
||||
|
||||
const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");
|
||||
const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");
|
||||
const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");
|
||||
const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");
|
||||
const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");
|
||||
const std::regex pattern_r_cache ("cache_r_l\\d*");
|
||||
const std::regex pattern_s_cache ("cache_s_l\\d*");
|
||||
const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");
|
||||
const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");
|
||||
static const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");
|
||||
static const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");
|
||||
static const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");
|
||||
static const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");
|
||||
static const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");
|
||||
static const std::regex pattern_r_cache ("cache_r_l\\d*");
|
||||
static const std::regex pattern_s_cache ("cache_s_l\\d*");
|
||||
static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");
|
||||
static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");
|
||||
|
||||
const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");
|
||||
const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");
|
||||
const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
|
||||
const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
|
||||
const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");
|
||||
const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");
|
||||
static const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");
|
||||
static const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");
|
||||
static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
|
||||
static const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
|
||||
static const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");
|
||||
static const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");
|
||||
|
||||
const std::regex pattern_output_weight("output\\.weight");
|
||||
const std::regex pattern_output_bias ("output\\.bias");
|
||||
static const std::regex pattern_output_weight("output\\.weight");
|
||||
static const std::regex pattern_output_bias ("output\\.bias");
|
||||
|
||||
struct tensor_config {
|
||||
ggml_backend_meta_split_axis axis;
|
||||
|
|
|
|||
|
|
@ -184,32 +184,6 @@ static ggml_tensor * dsv4_with_zero_dep(ggml_context * ctx, ggml_tensor * t, ggm
|
|||
return ggml_add(ctx, t, zero);
|
||||
}
|
||||
|
||||
// Raw SWA K is stored once, but compressed K/masks can carry a stream axis.
|
||||
// Repeat raw K at graph build time before concatenating raw and compressed K.
|
||||
static ggml_tensor * dsv4_repeat_streams(ggml_context * ctx, ggml_tensor * t, int64_t n_stream) {
|
||||
if (t->ne[3] == n_stream) {
|
||||
return t;
|
||||
}
|
||||
|
||||
GGML_ASSERT(t->ne[3] == 1);
|
||||
return ggml_repeat_4d(ctx, t, t->ne[0], t->ne[1], t->ne[2], n_stream);
|
||||
}
|
||||
|
||||
static ggml_tensor * dsv4_build_kq_zero_bias(
|
||||
ggml_context * ctx,
|
||||
const llama_cparams & cparams,
|
||||
ggml_tensor * kq_mask,
|
||||
int64_t n_head) {
|
||||
if (!cparams.kv_unified || !cparams.flash_attn || kq_mask->ne[3] == 1) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Keep multi-stream unified DSV4 on the explicit attention path.
|
||||
ggml_tensor * res = ggml_new_tensor_4d(ctx, GGML_TYPE_F32,
|
||||
kq_mask->ne[0], kq_mask->ne[1], n_head, kq_mask->ne[3]);
|
||||
return ggml_fill(ctx, res, 0.0f);
|
||||
}
|
||||
|
||||
static constexpr int64_t DSV4_CSA_RATIO = 4;
|
||||
static constexpr int64_t DSV4_HCA_RATIO = 128;
|
||||
|
||||
|
|
@ -624,7 +598,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_top_k_mask(
|
|||
ggml_tensor * top_k_3d = ggml_view_4d(ctx0, top_k, top_k->ne[0], top_k->ne[1], top_k->ne[3], 1,
|
||||
top_k->nb[1], top_k->nb[2], top_k->ne[3]*top_k->nb[3], 0);
|
||||
|
||||
ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]);
|
||||
ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]);
|
||||
zeros = ggml_fill(ctx0, zeros, 0.0f);
|
||||
|
||||
ggml_tensor * kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, top_k_3d);
|
||||
|
|
@ -681,26 +655,16 @@ ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention(
|
|||
csa_k->nb[1], csa_k->nb[2], csa_k->nb[3], 0);
|
||||
cb(csa_k, "csa_comp_k", il);
|
||||
|
||||
raw_k = dsv4_repeat_streams(ctx0, raw_k, csa_k->ne[3]);
|
||||
|
||||
ggml_tensor * k_all = ggml_concat(ctx0, raw_k, csa_k, 2);
|
||||
cb(k_all, "csa_k_all", il);
|
||||
|
||||
ggml_tensor * raw_mask = inp_attn->get_kq_mask();
|
||||
ggml_tensor * csa_mask = build_top_k_mask(inp_csa.kq_mask, top_k, "csa_top_k_mask", il);
|
||||
const bool use_fattn = cparams.flash_attn && (!cparams.kv_unified || csa_mask->ne[3] == 1);
|
||||
if (use_fattn && csa_mask->type != GGML_TYPE_F16) {
|
||||
csa_mask = ggml_cast(ctx0, csa_mask, GGML_TYPE_F16);
|
||||
}
|
||||
if (raw_mask->type != csa_mask->type) {
|
||||
raw_mask = ggml_cast(ctx0, raw_mask, csa_mask->type);
|
||||
}
|
||||
|
||||
ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, csa_mask, 0);
|
||||
cb(kq_mask, "csa_lid_kq_mask", il);
|
||||
|
||||
ggml_tensor * kq_b = dsv4_build_kq_zero_bias(ctx0, cparams, kq_mask, q->ne[1]);
|
||||
ggml_tensor * out = build_attn_mha(q, k_all, k_all, kq_b, kq_mask, sinks, nullptr, kq_scale, il);
|
||||
ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il);
|
||||
if (k_rot) {
|
||||
out = llama_mul_mat_hadamard(ctx0, out, k_rot);
|
||||
}
|
||||
|
|
@ -746,26 +710,16 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_attention(
|
|||
hca_k->nb[1], hca_k->nb[2], hca_k->nb[3], 0);
|
||||
cb(hca_k, "hca_comp_k", il);
|
||||
|
||||
raw_k = dsv4_repeat_streams(ctx0, raw_k, hca_k->ne[3]);
|
||||
|
||||
ggml_tensor * k_all = ggml_concat(ctx0, raw_k, hca_k, 2);
|
||||
cb(k_all, "hca_k_all", il);
|
||||
|
||||
ggml_tensor * raw_mask = inp_attn->get_kq_mask();
|
||||
ggml_tensor * hca_mask = inp_hca.kq_mask;
|
||||
const bool use_fattn = cparams.flash_attn && (!cparams.kv_unified || hca_mask->ne[3] == 1);
|
||||
if (use_fattn && hca_mask->type != GGML_TYPE_F16) {
|
||||
hca_mask = ggml_cast(ctx0, hca_mask, GGML_TYPE_F16);
|
||||
}
|
||||
if (raw_mask->type != hca_mask->type) {
|
||||
raw_mask = ggml_cast(ctx0, raw_mask, hca_mask->type);
|
||||
}
|
||||
|
||||
ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, hca_mask, 0);
|
||||
cb(kq_mask, "hca_kq_mask", il);
|
||||
|
||||
ggml_tensor * kq_b = dsv4_build_kq_zero_bias(ctx0, cparams, kq_mask, q->ne[1]);
|
||||
ggml_tensor * out = build_attn_mha(q, k_all, k_all, kq_b, kq_mask, sinks, nullptr, kq_scale, il);
|
||||
ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il);
|
||||
if (k_rot) {
|
||||
out = llama_mul_mat_hadamard(ctx0, out, k_rot);
|
||||
}
|
||||
|
|
@ -800,10 +754,8 @@ ggml_tensor * llama_model_deepseek4::graph::build_raw_attention(
|
|||
ggml_tensor * kq_mask = inp_attn->get_kq_mask();
|
||||
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
k = dsv4_repeat_streams(ctx0, k, kq_mask->ne[3]);
|
||||
|
||||
ggml_tensor * kq_b = dsv4_build_kq_zero_bias(ctx0, cparams, kq_mask, q->ne[1]);
|
||||
ggml_tensor * out = build_attn_mha(q, k, k, kq_b, kq_mask, sinks, nullptr, kq_scale, il);
|
||||
ggml_tensor * out = build_attn_mha(q, k, k, nullptr, kq_mask, sinks, nullptr, kq_scale, il);
|
||||
if (k_rot) {
|
||||
out = llama_mul_mat_hadamard(ctx0, out, k_rot);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -153,9 +153,19 @@ bool cli_context::init() {
|
|||
|
||||
if (use_external_server) {
|
||||
spinner.reset();
|
||||
if (!list_and_ask_models()) {
|
||||
try {
|
||||
if (!list_and_ask_models()) {
|
||||
return false;
|
||||
}
|
||||
} catch (const json::parse_error & e) {
|
||||
ui::show_error(e.what());
|
||||
ui::show_message("This might be caused by an incorrect server-base endpoint URL");
|
||||
return false;
|
||||
} catch (const std::exception & e) {
|
||||
ui::show_error(e.what());
|
||||
return false;
|
||||
}
|
||||
|
||||
// restore the spinner for the next step
|
||||
spinner.emplace("Waiting for server...");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ struct clip_graph {
|
|||
const clip_hparams & hparams;
|
||||
projector_type proj_type;
|
||||
|
||||
// we only support single image per batch
|
||||
const clip_image_f32 & img;
|
||||
const clip_image_f32 & img; // for backward compat
|
||||
const clip_image_f32_batch * img_batch = nullptr;
|
||||
|
||||
const int patch_size;
|
||||
const int n_patches_x;
|
||||
|
|
@ -63,6 +63,12 @@ struct clip_graph {
|
|||
//
|
||||
void cb(ggml_tensor * cur0, const char * name, int il) const;
|
||||
|
||||
const clip_image_f32 & get_img(size_t idx) const {
|
||||
GGML_ASSERT(img_batch);
|
||||
GGML_ASSERT(idx < img_batch->entries.size());
|
||||
return img_batch->entries[idx];
|
||||
}
|
||||
|
||||
// siglip2 naflex
|
||||
ggml_tensor * resize_position_embeddings(uint32_t interpolation_mode = DEFAULT_INTERPOLATION_MODE);
|
||||
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ struct clip_hparams {
|
|||
std::vector<clip_image_size> image_res_candidates;
|
||||
int32_t preproc_min_tiles = 0;
|
||||
int32_t preproc_max_tiles = 0;
|
||||
int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr)
|
||||
resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
|
||||
resize_algo image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
|
||||
pad_style image_pad_rf = PAD_CEIL; // padding style for the refined image (e.g. llava-1.6)
|
||||
|
|
|
|||
|
|
@ -1072,6 +1072,8 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
|
|||
GGML_ABORT("missing cgraph builder");
|
||||
}
|
||||
|
||||
builder->img_batch = &imgs;
|
||||
|
||||
// TODO [QWEN_VIDEO]: improve this in the future
|
||||
builder->n_batch = imgs.entries.size();
|
||||
|
||||
|
|
@ -1656,7 +1658,16 @@ struct clip_model_loader {
|
|||
get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true);
|
||||
get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true);
|
||||
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
|
||||
hparams.preproc_min_tiles = 2;
|
||||
if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR) {
|
||||
hparams.preproc_max_tiles = 9;
|
||||
hparams.preproc_tile_size = 640;
|
||||
// the CLIP/ViT body runs its layernorms at 1e-5 (the SAM stage uses 1e-6)
|
||||
hparams.eps = 1e-5f;
|
||||
}
|
||||
if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) {
|
||||
hparams.preproc_max_tiles = 6;
|
||||
hparams.preproc_tile_size = 768;
|
||||
// qwen2 encoder is GQA, requires KEY_N_HEAD_KV
|
||||
get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
|
||||
}
|
||||
|
|
@ -3332,6 +3343,9 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
|
|||
return (img->nx() / params.patch_size) / 2;
|
||||
case PROJECTOR_TYPE_STEP3VL:
|
||||
return img->nx() / (params.patch_size * params.n_merge);
|
||||
case PROJECTOR_TYPE_DEEPSEEKOCR:
|
||||
case PROJECTOR_TYPE_DEEPSEEKOCR2:
|
||||
return (img->nx() / params.patch_size) / 4;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -3541,10 +3555,17 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
|
|||
// E.g., 64x64 -> 16x16 patches
|
||||
n_patches /= 16;
|
||||
|
||||
// build_global_local_features adds image newlines and view separator
|
||||
// Formula: h*(w+1) + 1 where h = w = sqrt(n_patches)
|
||||
int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
|
||||
n_patches = h * (h + 1) + 1;
|
||||
if (img->add_viewsep) {
|
||||
// global view: one image-newline per token-row + trailing view separator
|
||||
const int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
|
||||
n_patches = h * (h + 1) + 1;
|
||||
} else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) {
|
||||
// tile row: one image-newline per token-row
|
||||
const int grid_w = img->ny() / img->nx();
|
||||
const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4)
|
||||
const int h = tile_patches;
|
||||
n_patches = (tile_patches * grid_w + 1) * h;
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
{
|
||||
|
|
@ -4184,7 +4205,10 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
|
|||
case PROJECTOR_TYPE_DEEPSEEKOCR:
|
||||
case PROJECTOR_TYPE_DEEPSEEKOCR2:
|
||||
{
|
||||
GGML_ASSERT(pos_w == pos_h);
|
||||
GGML_ASSERT(
|
||||
(pos_w == pos_h) // overview image
|
||||
|| (pos_h >= pos_w && pos_h % pos_w == 0) // tile images
|
||||
);
|
||||
|
||||
const int window = hparams.attn_window_size;
|
||||
const int pos = pos_w;
|
||||
|
|
|
|||
|
|
@ -96,6 +96,8 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
|
|||
const int n_heads = hparams.sam_n_head;
|
||||
const int d_heads = n_embd / n_heads;
|
||||
const int window = hparams.attn_window_size;
|
||||
// SAM stage runs its layernorms at 1e-6
|
||||
const float sam_eps = 1e-6f;
|
||||
|
||||
ggml_tensor * inpL;
|
||||
|
||||
|
|
@ -134,7 +136,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
|
|||
ggml_tensor * shortcut = cur;
|
||||
|
||||
// layernorm1
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, sam_eps, il);
|
||||
|
||||
const int64_t w0 = cur->ne[1];
|
||||
const int64_t h0 = cur->ne[2];
|
||||
|
|
@ -214,7 +216,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
|
|||
ggml_tensor * inpFF = cur;
|
||||
|
||||
// layernorm2
|
||||
cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, sam_eps, il);
|
||||
|
||||
// ffn
|
||||
cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b,
|
||||
|
|
@ -229,12 +231,12 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
|
|||
|
||||
cur = ggml_conv_2d(ctx0, model.neck_0_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, hparams.eps, -1);
|
||||
cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, sam_eps, -1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = ggml_conv_2d(ctx0, model.neck_2_w, cur, 1, 1, 1, 1, 1, 1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, hparams.eps, -1);
|
||||
cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, sam_eps, -1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = ggml_conv_2d(ctx0, model.net_2, cur, 2, 2, 1, 1, 1, 1);
|
||||
|
|
@ -248,8 +250,40 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
|
|||
ggml_cgraph * clip_graph_deepseekocr::build() {
|
||||
// patch embedding
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
bool is_overview = img.add_viewsep;
|
||||
int n_tiles_per_row = 0;
|
||||
|
||||
// note: we expect either a batch of rows or a batch of overviews, but not a mix of both
|
||||
|
||||
if (!is_overview) {
|
||||
// handle the case where we have a batch of rows
|
||||
// sanity check
|
||||
for (auto & entry : img_batch->entries) {
|
||||
if (entry.add_viewsep) {
|
||||
throw std::runtime_error("DeepSeek-OCR: mixed overview and non-overview images in batch");
|
||||
}
|
||||
if (entry.nx() != img.nx() || entry.ny() != img.ny()) {
|
||||
throw std::runtime_error("DeepSeek-OCR: mixed image sizes in batch");
|
||||
}
|
||||
}
|
||||
|
||||
GGML_ASSERT(img.ny() >= img.nx());
|
||||
GGML_ASSERT(img.ny() % img.nx() == 0);
|
||||
n_tiles_per_row = img.ny() / img.nx();
|
||||
|
||||
// input shape: [tile_size, tile_size * n_tiles_per_row, 3]
|
||||
// we want to reshape it to [tile_size, tile_size, 3, n_tiles_per_row]
|
||||
inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), n_tiles_per_row, 3);
|
||||
inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 1, 3, 2));
|
||||
}
|
||||
|
||||
ggml_tensor * sam_out = build_sam(inp_raw);
|
||||
|
||||
if (!is_overview) {
|
||||
n_batch = n_tiles_per_row;
|
||||
}
|
||||
|
||||
const int clip_n_patches = sam_out->ne[0] * sam_out->ne[1];
|
||||
|
||||
ggml_tensor * clip_out;
|
||||
|
|
@ -257,7 +291,9 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
|
|||
{
|
||||
ggml_tensor * inp;
|
||||
|
||||
inp = ggml_reshape_2d(ctx0, sam_out, clip_n_patches, sam_out->ne[2]);
|
||||
// sam_out: [patch_h, patch_w, n_embd, n_batch]
|
||||
// -> [n_embd, clip_n_patches, n_batch]
|
||||
inp = ggml_reshape_3d(ctx0, sam_out, clip_n_patches, sam_out->ne[2], sam_out->ne[3]);
|
||||
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
|
||||
|
||||
ggml_tensor * new_pos_embd = model.position_embeddings;
|
||||
|
|
@ -281,8 +317,11 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
|
|||
n_pos = tgt_size * tgt_size + 1;
|
||||
}
|
||||
|
||||
// add CLS token
|
||||
inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
|
||||
// add CLS token per batch item
|
||||
// inp: [n_embd, clip_n_patches, n_batch]
|
||||
// class_embedding: [n_embd] -> [n_embd, 1, n_batch]
|
||||
ggml_tensor * cls_embd = ggml_repeat_4d(ctx0, model.class_embedding, n_embd, 1, n_batch, 1);
|
||||
inp = ggml_concat(ctx0, cls_embd, inp, 1);
|
||||
|
||||
// for selecting learned pos embd, used by ViT
|
||||
ggml_tensor * positions = ggml_cast(ctx0, ggml_arange(ctx0, 0, n_pos, 1), GGML_TYPE_I32);
|
||||
|
|
@ -294,25 +333,56 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
|
|||
clip_out = cur;
|
||||
}
|
||||
|
||||
// sam_out: [patch_h, patch_w, n_embd, n_batch]
|
||||
// -> [n_embd, clip_n_patches, n_batch]
|
||||
sam_out = ggml_cont(ctx0, ggml_permute(ctx0, sam_out, 1, 2, 0, 3));
|
||||
sam_out = ggml_reshape_2d(ctx0, sam_out, sam_out->ne[0], clip_n_patches);
|
||||
clip_out = ggml_view_2d(ctx0, clip_out, n_embd, clip_n_patches, clip_out->nb[1], clip_out->nb[1]);
|
||||
sam_out = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0], clip_n_patches, n_batch);
|
||||
|
||||
// clip_out: [n_embd, n_pos, n_batch] where n_pos = clip_n_patches + 1 (CLS)
|
||||
// strip CLS token: skip first position, view only the patch tokens
|
||||
clip_out = ggml_view_3d(ctx0, clip_out, n_embd, clip_n_patches, n_batch,
|
||||
clip_out->nb[1], clip_out->nb[2], clip_out->nb[1]);
|
||||
|
||||
ggml_tensor * cur;
|
||||
cur = ggml_concat(ctx0, clip_out, sam_out, 0);
|
||||
cur = ggml_mul_mat(ctx0, model.mm_fc_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_fc_b);
|
||||
|
||||
const auto h = static_cast<int>(std::sqrt(static_cast<float>(cur->ne[1])));
|
||||
const auto w = h;
|
||||
const auto n_dim = cur->ne[0];
|
||||
if (is_overview) {
|
||||
// global view: weave one newline per row + trailing view separator
|
||||
const auto h = static_cast<int>(std::sqrt(static_cast<float>(cur->ne[1])));
|
||||
const auto w = h;
|
||||
const auto n_dim = cur->ne[0];
|
||||
|
||||
ggml_tensor * imgnl;
|
||||
ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
|
||||
cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
|
||||
cur = ggml_concat(ctx0, cur, model.view_seperator, 1); // (n_dim, h*(w+1) + 1)
|
||||
} else {
|
||||
// tile row: interleave tiles within each row, add newline per row
|
||||
const int grid_x = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
|
||||
const int grid_y = grid_x;
|
||||
const auto n_dim = cur->ne[0];
|
||||
|
||||
imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
|
||||
cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
|
||||
cur = ggml_concat(ctx0, cur, model.view_seperator, 1); // (n_dim, h*(w+1) + 1)
|
||||
// (n_dim, clip_n_patches, n_batch) -> (n_dim, grid_x, grid_y, n_batch)
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x, grid_y, n_batch);
|
||||
|
||||
// tiles: re-order from A.row0 A.row1 B.row0 B.row1 ...
|
||||
// to A.row0 B.row0 A.row1 B.row1 ...
|
||||
// then add nl: A.row0 B.row0 [nl] A.row1 B.row1 [nl] ...
|
||||
// interleave tiles: (n_dim, grid_x, grid_y, n_batch) -> (n_dim, grid_x, n_batch, grid_y)
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 1, 3, 2));
|
||||
|
||||
// merge: (n_dim, grid_x, n_batch, grid_y) -> (n_dim, grid_x*n_batch, grid_y, 1)
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_batch, grid_y, 1);
|
||||
|
||||
// append newline per row: (n_dim, grid_x*n_batch+1, grid_y, 1)
|
||||
ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, 1);
|
||||
cur = ggml_concat(ctx0, cur, imgnl, 1);
|
||||
|
||||
// flatten: (n_dim, (grid_x*n_batch+1)*grid_y)
|
||||
cur = ggml_reshape_2d(ctx0, cur, n_dim, (grid_x * n_batch + 1) * grid_y);
|
||||
}
|
||||
|
||||
cb(cur, "dsocr_output", -1);
|
||||
|
||||
|
|
|
|||
|
|
@ -127,6 +127,7 @@ struct clip_graph_deepseekocr : clip_graph {
|
|||
clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
|
||||
// bool support_batch() const override { return true; } // TODO: support batch for DeepSeek-OCR v1
|
||||
};
|
||||
|
||||
struct clip_graph_deepseekocr2 : clip_graph_deepseekocr {
|
||||
|
|
|
|||
|
|
@ -1107,44 +1107,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_i
|
|||
// mtmd_image_preprocessor_deepseekocr
|
||||
//
|
||||
|
||||
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
|
||||
static constexpr int native_resolutions[] = { 1024 /* base */, 1280 /* large */ };
|
||||
// TODO: support 512 (tiny) and 640 (small) once we have eval data for them
|
||||
|
||||
const int64_t orig_area = static_cast<int64_t>(img.get_size().area());
|
||||
|
||||
size_t mode_i = 0;
|
||||
int64_t min_diff = std::numeric_limits<int64_t>::max();
|
||||
for (size_t i = 0; i < std::size(native_resolutions); i++) {
|
||||
const int64_t r = native_resolutions[i];
|
||||
const int64_t diff = std::abs(orig_area - r * r);
|
||||
if (diff < min_diff) {
|
||||
mode_i = i;
|
||||
min_diff = diff;
|
||||
}
|
||||
}
|
||||
const int image_size = native_resolutions[mode_i];
|
||||
|
||||
// Aspect-preserving fit-and-pad. Pillow bicubic + PAD_NEAREST for
|
||||
// byte-parity with the upstream deepseek-ai/DeepSeek-OCR HF preprocessor.
|
||||
clip_image_u8 padded;
|
||||
img_tool::resize(img, padded, {image_size, image_size}, RESIZE_ALGO_BICUBIC_PILLOW,
|
||||
PAD_NEAREST, hparams.image_pad_color);
|
||||
mtmd_image_preproc_out output;
|
||||
output.append_overview(hparams, padded, true);
|
||||
output.grid_x = 0;
|
||||
output.grid_y = 0;
|
||||
// TODO @ngxson : support slicing for DeepSeek-OCR, to do in another PR
|
||||
return output;
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_image_preprocessor_deepseekocr2
|
||||
//
|
||||
|
||||
// candidate tile grids (cols, rows) with min_tiles <= cols*rows <= max_tiles
|
||||
// sorted by tile count
|
||||
std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr2::get_target_ratios() {
|
||||
std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr::get_target_ratios() const {
|
||||
std::vector<clip_image_size> ratios;
|
||||
for (int n = min_tiles; n <= max_tiles; n++) {
|
||||
for (int w = 1; w <= n; w++) {
|
||||
|
|
@ -1171,13 +1134,11 @@ std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr2::get_target_ra
|
|||
return ratios;
|
||||
}
|
||||
|
||||
// pick the grid whose aspect ratio is closest to the image
|
||||
// on a tie, prefer the larger grid when the image fits
|
||||
clip_image_size mtmd_image_preprocessor_deepseekocr2::find_closest_aspect_ratio(
|
||||
clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio(
|
||||
float aspect_ratio,
|
||||
const std::vector<clip_image_size> & target_ratios,
|
||||
int width,
|
||||
int height) {
|
||||
int height) const {
|
||||
float best_ratio_diff = std::numeric_limits<float>::max();
|
||||
clip_image_size best_ratio = { 1, 1 };
|
||||
const float area = static_cast<float>(width * height);
|
||||
|
|
@ -1198,37 +1159,69 @@ clip_image_size mtmd_image_preprocessor_deepseekocr2::find_closest_aspect_ratio(
|
|||
return best_ratio;
|
||||
}
|
||||
|
||||
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr2::preprocess(const clip_image_u8 & img) {
|
||||
// emit 768x768 local tiles when the image is larger than a tile in either
|
||||
// dimension, then always a 1024x1024 global view. order: [tiles..., global].
|
||||
|
||||
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
|
||||
mtmd_image_preproc_out output;
|
||||
int grid_w = 0;
|
||||
int grid_h = 0;
|
||||
const auto img_size = img.get_size();
|
||||
|
||||
// global view: aspect-preserving fit-and-pad to base_size
|
||||
clip_image_u8 padded;
|
||||
img_tool::resize(img, padded,
|
||||
{ base_size, base_size },
|
||||
RESIZE_ALGO_BICUBIC_PILLOW,
|
||||
PAD_NEAREST,
|
||||
hparams.image_pad_color);
|
||||
output.append_overview(hparams, padded, true);
|
||||
output.overview.add_viewsep = true;
|
||||
|
||||
// if this condition doesn't hold, the output is overview only, no tiles
|
||||
if (img_size.width > tile_size || img_size.height > tile_size) {
|
||||
const float aspect_ratio = static_cast<float>(img_size.width) / img_size.height;
|
||||
const auto target_ratios = get_target_ratios();
|
||||
const clip_image_size grid = find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height);
|
||||
const clip_image_size grid =
|
||||
find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height);
|
||||
grid_w = grid.width;
|
||||
grid_h = grid.height;
|
||||
|
||||
// stretch onto the grid (no aspect preserve), then crop tiles row-major.
|
||||
clip_image_u8 refined;
|
||||
img_tool::resize(img, refined, { tile_size * grid.width, tile_size * grid.height },
|
||||
RESIZE_ALGO_BICUBIC_PILLOW, PAD_NONE);
|
||||
img_tool::resize(img, refined, { tile_size * grid_w, tile_size * grid_h }, RESIZE_ALGO_BICUBIC_PILLOW,
|
||||
PAD_NONE);
|
||||
|
||||
for (int row = 0; row < grid.height; row++) {
|
||||
for (int col = 0; col < grid.width; col++) {
|
||||
clip_image_u8 tile;
|
||||
img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size);
|
||||
output.append(hparams, tile, true);
|
||||
for (int row = 0; row < grid_h; row++) {
|
||||
if (fuse_row) {
|
||||
// concat all tiles in this row into a single image, along the H axis
|
||||
// output image size: w = tile_size, h = tile_size * grid_w
|
||||
// this is to ensure the whole row is always processed together
|
||||
clip_image_u8 row_img;
|
||||
row_img.set_size({tile_size, tile_size * grid_w}, false);
|
||||
for (int col = 0; col < grid_w; col++) {
|
||||
for (int py = 0; py < tile_size; py++) {
|
||||
for (int px = 0; px < tile_size; px++) {
|
||||
row_img.set_pixel(px, col * tile_size + py,
|
||||
refined.get_pixel(col * tile_size + px, row * tile_size + py));
|
||||
}
|
||||
}
|
||||
}
|
||||
output.append(hparams, row_img, true);
|
||||
} else {
|
||||
for (int col = 0; col < grid_w; col++) {
|
||||
clip_image_u8 tile;
|
||||
img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size);
|
||||
output.append(hparams, tile, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (fuse_row) {
|
||||
grid_w = 1; // each fused row is one image; a single output column
|
||||
}
|
||||
}
|
||||
|
||||
// global view: aspect-preserving fit-and-pad to base_size.
|
||||
clip_image_u8 padded;
|
||||
img_tool::resize(img, padded, { base_size, base_size }, RESIZE_ALGO_BICUBIC_PILLOW,
|
||||
PAD_NEAREST, hparams.image_pad_color);
|
||||
output.append_overview(hparams, padded, true);
|
||||
output.overview.add_viewsep = true;
|
||||
LOG_DBG("%s: grid size: %d x %d (%d tiles) + global view\n", __func__, grid_w, grid_h, grid_w * grid_h);
|
||||
LOG_DBG("%s: overview size: %d x %d\n", __func__, padded.get_size().width, padded.get_size().height);
|
||||
|
||||
output.grid_x = grid_w;
|
||||
output.grid_y = grid_h;
|
||||
return output;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -160,29 +160,29 @@ struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd {
|
|||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
|
||||
};
|
||||
|
||||
// DeepSeek-OCR (v1/v2) global view + optional local tile grid
|
||||
struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
|
||||
};
|
||||
|
||||
// DeepSeek-OCR-2: a 1024x1024 global view, plus InternVL-style 768x768 local
|
||||
// tiles when the image is larger than a tile in either dimension.
|
||||
struct mtmd_image_preprocessor_deepseekocr2 : mtmd_image_preprocessor {
|
||||
static constexpr int base_size = 1024; // global view
|
||||
static constexpr int tile_size = 768; // local tile
|
||||
static constexpr int min_tiles = 2;
|
||||
static constexpr int max_tiles = 6;
|
||||
|
||||
mtmd_image_preprocessor_deepseekocr2(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx)
|
||||
: mtmd_image_preprocessor(ctx),
|
||||
fuse_row(clip_get_projector_type(ctx) == PROJECTOR_TYPE_DEEPSEEKOCR),
|
||||
base_size(hparams.image_size),
|
||||
tile_size(hparams.preproc_tile_size),
|
||||
min_tiles(hparams.preproc_min_tiles),
|
||||
max_tiles(hparams.preproc_max_tiles) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
|
||||
|
||||
private:
|
||||
static std::vector<clip_image_size> get_target_ratios();
|
||||
static clip_image_size find_closest_aspect_ratio(
|
||||
float aspect_ratio,
|
||||
const std::vector<clip_image_size> & target_ratios,
|
||||
int width,
|
||||
int height);
|
||||
bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate
|
||||
int base_size; // global view
|
||||
int tile_size; // each tile
|
||||
int min_tiles;
|
||||
int max_tiles;
|
||||
|
||||
std::vector<clip_image_size> get_target_ratios() const;
|
||||
clip_image_size find_closest_aspect_ratio(
|
||||
float aspect_ratio,
|
||||
const std::vector<clip_image_size> & target_ratios,
|
||||
int width, int height) const;
|
||||
};
|
||||
|
||||
// custom image preprocessing for Step3VL
|
||||
|
|
|
|||
|
|
@ -618,15 +618,10 @@ struct mtmd_context {
|
|||
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DEEPSEEKOCR:
|
||||
{
|
||||
img_end = "\n"; // prevent empty batch on llama-server
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
|
||||
ov_img_first = false;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DEEPSEEKOCR2:
|
||||
{
|
||||
img_end = "\n"; // prevent empty batch on llama-server
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr2>(ctx_v);
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
|
||||
ov_img_first = false;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_HUNYUANVL:
|
||||
|
|
@ -1132,6 +1127,7 @@ struct mtmd_tokenizer {
|
|||
|
||||
// add slices (or tiles)
|
||||
if (!chunks.empty()) {
|
||||
LOG_DBG("%s: adding %d slices (%d rows x %d cols)\n", __func__, (int)chunks.size(), n_row, n_col);
|
||||
GGML_ASSERT((int)chunks.size() == n_row * n_col);
|
||||
add_text(ctx->tok_slices_start);
|
||||
for (int y = 0; y < n_row; y++) {
|
||||
|
|
@ -1174,7 +1170,6 @@ struct mtmd_tokenizer {
|
|||
cur.entries.emplace_back(std::move(ov_chunk));
|
||||
add_text(ctx->tok_ov_img_end);
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
if (preproc_out.entries.size() == 0) {
|
||||
|
|
|
|||
|
|
@ -175,6 +175,15 @@ bool server_http_context::init(const common_params & params) {
|
|||
// Middlewares
|
||||
//
|
||||
|
||||
// Frontend paths - all embedded UI assets
|
||||
static const std::unordered_set<std::string> frontend_paths = []() {
|
||||
std::unordered_set<std::string> paths { "/" };
|
||||
for (const llama_ui_asset & a : llama_ui_get_assets()) {
|
||||
paths.insert("/" + a.name);
|
||||
}
|
||||
return paths;
|
||||
}();
|
||||
|
||||
// Public endpoints - API routes plus all embedded UI assets
|
||||
static const std::unordered_set<std::string> get_public_endpoints = []() {
|
||||
std::unordered_set<std::string> endpoints {
|
||||
|
|
@ -182,11 +191,8 @@ bool server_http_context::init(const common_params & params) {
|
|||
"/v1/health",
|
||||
"/models",
|
||||
"/v1/models",
|
||||
"/",
|
||||
};
|
||||
for (const llama_ui_asset & a : llama_ui_get_assets()) {
|
||||
endpoints.insert("/" + a.name);
|
||||
}
|
||||
endpoints.insert(frontend_paths.begin(), frontend_paths.end());
|
||||
return endpoints;
|
||||
}();
|
||||
|
||||
|
|
@ -239,18 +245,9 @@ bool server_http_context::init(const common_params & params) {
|
|||
|
||||
auto middleware_server_state = [this](const httplib::Request & req, httplib::Response & res) {
|
||||
if (!is_ready.load()) {
|
||||
#if defined(LLAMA_UI_HAS_ASSETS)
|
||||
if (const auto tmp = string_split<std::string>(req.path, '.');
|
||||
req.path == "/" || (!tmp.empty() && tmp.back() == "html")) {
|
||||
if (const llama_ui_asset * a = llama_ui_find_asset("loading.html")) {
|
||||
res.status = 503;
|
||||
res.set_content(reinterpret_cast<const char*>(a->data), a->size, "text/html; charset=utf-8");
|
||||
return false;
|
||||
}
|
||||
if (frontend_paths.count(req.path)) {
|
||||
return true; // frontend asset, allow it to load and show "loading"
|
||||
}
|
||||
#else
|
||||
(void)req;
|
||||
#endif
|
||||
// no endpoints are allowed to be accessed when the server is not ready
|
||||
// this is to prevent any data races or inconsistent states
|
||||
res.status = 503;
|
||||
|
|
|
|||
|
|
@ -568,10 +568,16 @@ static void handle_with_catch(const char * name, std::function<void()> func) {
|
|||
}
|
||||
}
|
||||
|
||||
// treat a null value as absent so clients can send null to request the server default
|
||||
static bool has_value(const json & data, const char * n) {
|
||||
auto it = data.find(n);
|
||||
return it != data.end() && !it->is_null();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void field_num<T>::eval(field_eval_context & ctx, const json & data) {
|
||||
for (const auto & n : name) {
|
||||
if (data.contains(n)) {
|
||||
if (has_value(data, n)) {
|
||||
handle_with_catch(n, [&]() {
|
||||
if (custom_handler) {
|
||||
custom_handler(ctx, data);
|
||||
|
|
@ -593,7 +599,7 @@ void field_num<T>::eval(field_eval_context & ctx, const json & data) {
|
|||
void field_str::eval(field_eval_context & ctx, const json & data) {
|
||||
GGML_ASSERT(custom_handler);
|
||||
for (const auto & n : name) {
|
||||
if (data.contains(n)) {
|
||||
if (has_value(data, n)) {
|
||||
handle_with_catch(n, [&]() {
|
||||
custom_handler(ctx, data);
|
||||
});
|
||||
|
|
@ -604,7 +610,7 @@ void field_str::eval(field_eval_context & ctx, const json & data) {
|
|||
|
||||
void field_bool::eval(field_eval_context & ctx, const json & data) {
|
||||
for (const auto & n : name) {
|
||||
if (data.contains(n)) {
|
||||
if (has_value(data, n)) {
|
||||
handle_with_catch(n, [&]() {
|
||||
if (custom_handler) {
|
||||
custom_handler(ctx, data);
|
||||
|
|
@ -620,7 +626,7 @@ void field_bool::eval(field_eval_context & ctx, const json & data) {
|
|||
void field_json::eval(field_eval_context & ctx, const json & data) {
|
||||
GGML_ASSERT(custom_handler);
|
||||
for (const auto & n : name) {
|
||||
if (data.contains(n)) {
|
||||
if (has_value(data, n)) {
|
||||
handle_with_catch(n, [&]() {
|
||||
custom_handler(ctx, data);
|
||||
});
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -9,10 +9,10 @@ struct server_tool {
|
|||
bool permission_write = false;
|
||||
|
||||
virtual ~server_tool() = default;
|
||||
virtual json get_definition() = 0;
|
||||
virtual json invoke(json params) = 0;
|
||||
virtual json get_definition() const = 0;
|
||||
virtual json invoke(json params) const = 0;
|
||||
|
||||
json to_json();
|
||||
json to_json() const;
|
||||
};
|
||||
|
||||
struct server_tools {
|
||||
|
|
|
|||
125
tools/server/tests/unit/test_tools_builtin.py
Executable file
125
tools/server/tests/unit/test_tools_builtin.py
Executable file
|
|
@ -0,0 +1,125 @@
|
|||
import os
|
||||
|
||||
import pytest
|
||||
from utils import *
|
||||
|
||||
server: ServerProcess
|
||||
|
||||
# project root, used as the search directory for grep_search/file_glob_search
|
||||
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
|
||||
|
||||
# marker for the grep_search test to find in this file
|
||||
GREP_MARKER = "llama_cpp_test_tools_builtin_marker_grep_search"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def create_server():
|
||||
global server
|
||||
server = ServerPreset.router()
|
||||
server.server_tools = "all"
|
||||
|
||||
|
||||
def call_tool(name: str, params: dict) -> dict:
|
||||
res = server.make_request("POST", "/tools", data={"tool": name, "params": params})
|
||||
assert res.status_code == 200, res.body
|
||||
assert "error" not in res.body, res.body
|
||||
return res.body
|
||||
|
||||
|
||||
def call_tool_expect_error(name: str, params: dict) -> str:
|
||||
res = server.make_request("POST", "/tools", data={"tool": name, "params": params})
|
||||
assert res.status_code == 200, res.body
|
||||
assert "error" in res.body, res.body
|
||||
return res.body["error"]
|
||||
|
||||
|
||||
def test_tools_builtin_grep_search():
|
||||
global server
|
||||
server.start()
|
||||
|
||||
res = call_tool("grep_search", {
|
||||
"path": PROJECT_ROOT,
|
||||
"pattern": GREP_MARKER,
|
||||
"include": "test_tools_builtin.py", # bare pattern -> matches basename at any depth
|
||||
})
|
||||
text = res["plain_text_response"]
|
||||
assert "test_tools_builtin.py" in text
|
||||
assert GREP_MARKER in text
|
||||
assert "Total matches: 1" in text
|
||||
|
||||
|
||||
def test_tools_builtin_read_file():
|
||||
global server
|
||||
server.start()
|
||||
|
||||
this_file = os.path.join(PROJECT_ROOT, "tools", "server", "tests", "unit", "test_tools_builtin.py")
|
||||
res = call_tool("read_file", {"path": this_file})
|
||||
text = res["plain_text_response"]
|
||||
assert GREP_MARKER in text
|
||||
assert "def test_tools_builtin_read_file" in text
|
||||
|
||||
|
||||
def test_tools_builtin_write_then_edit_file():
|
||||
global server
|
||||
server.start()
|
||||
|
||||
log_path = os.path.join(PROJECT_ROOT, "test.log")
|
||||
try:
|
||||
write_res = call_tool("write_file", {"path": log_path, "content": "line1\nline2\nline3\n"})
|
||||
assert write_res["result"] == "file written successfully"
|
||||
|
||||
read_before = call_tool("read_file", {"path": log_path})
|
||||
assert read_before["plain_text_response"] == "line1\nline2\nline3\n"
|
||||
|
||||
edit_res = call_tool("edit_file", {
|
||||
"path": log_path,
|
||||
"edits": [
|
||||
{"old_text": "line2", "new_text": "line2-edited"},
|
||||
{"old_text": "line3\n", "new_text": "line3\nline4\n"},
|
||||
],
|
||||
})
|
||||
assert edit_res["result"] == "file edited successfully"
|
||||
assert edit_res["edits_applied"] == 2
|
||||
|
||||
read_after = call_tool("read_file", {"path": log_path})
|
||||
assert read_after["plain_text_response"] == "line1\nline2-edited\nline3\nline4\n"
|
||||
finally:
|
||||
if os.path.exists(log_path):
|
||||
os.remove(log_path)
|
||||
|
||||
|
||||
def test_tools_builtin_edit_file_rejects_non_unique_old_text():
|
||||
global server
|
||||
server.start()
|
||||
|
||||
log_path = os.path.join(PROJECT_ROOT, "test.log")
|
||||
try:
|
||||
call_tool("write_file", {"path": log_path, "content": "dup\ndup\n"})
|
||||
err = call_tool_expect_error("edit_file", {
|
||||
"path": log_path,
|
||||
"edits": [{"old_text": "dup", "new_text": "changed"}],
|
||||
})
|
||||
assert "unique" in err
|
||||
finally:
|
||||
if os.path.exists(log_path):
|
||||
os.remove(log_path)
|
||||
|
||||
|
||||
def test_tools_builtin_edit_file_rejects_overlapping_edits():
|
||||
global server
|
||||
server.start()
|
||||
|
||||
log_path = os.path.join(PROJECT_ROOT, "test.log")
|
||||
try:
|
||||
call_tool("write_file", {"path": log_path, "content": "line1\nline2\n"})
|
||||
err = call_tool_expect_error("edit_file", {
|
||||
"path": log_path,
|
||||
"edits": [
|
||||
{"old_text": "line1\nline2", "new_text": "a"},
|
||||
{"old_text": "line2", "new_text": "b"},
|
||||
],
|
||||
})
|
||||
assert "overlap" in err
|
||||
finally:
|
||||
if os.path.exists(log_path):
|
||||
os.remove(log_path)
|
||||
|
|
@ -113,6 +113,7 @@ class ServerProcess:
|
|||
ui_mcp_proxy: bool = False
|
||||
backend_sampling: bool = False
|
||||
gcp_compat: bool = False
|
||||
server_tools: str | None = None
|
||||
|
||||
# session variables
|
||||
process: subprocess.Popen | None = None
|
||||
|
|
@ -256,6 +257,8 @@ class ServerProcess:
|
|||
server_args.append("--no-cache-idle-slots")
|
||||
if self.ui_mcp_proxy:
|
||||
server_args.append("--ui-mcp-proxy")
|
||||
if self.server_tools:
|
||||
server_args.extend(["--tools", self.server_tools])
|
||||
if self.backend_sampling:
|
||||
server_args.append("--backend_sampling")
|
||||
if self.gcp_compat:
|
||||
|
|
|
|||
|
|
@ -187,7 +187,6 @@ int main(int argc, char ** argv) {
|
|||
struct required_check { const char * label; match_fn match; bool found; };
|
||||
required_check checks[] = {
|
||||
{ "index.html", exact("index.html"), false },
|
||||
{ "loading.html", exact("loading.html"), false },
|
||||
{ "manifest.webmanifest", exact("manifest.webmanifest"), false },
|
||||
{ "sw.js", exact("sw.js"), false },
|
||||
{ "build.json", exact("build.json"), false },
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
<script lang="ts">
|
||||
import { AlertTriangle, RefreshCw } from '@lucide/svelte';
|
||||
import { AlertTriangle, Loader2, RefreshCw } from '@lucide/svelte';
|
||||
import { fadeInView } from '$lib/actions/fade-in-view.svelte';
|
||||
import * as Alert from '$lib/components/ui/alert';
|
||||
import { serverError, serverLoading, serverStore } from '$lib/stores/server.svelte';
|
||||
import { serverError, serverLoading, serverStatus, serverStore } from '$lib/stores/server.svelte';
|
||||
|
||||
let hasError = $derived(!!serverError());
|
||||
let isLoadingModel = $derived(serverStatus() === 503);
|
||||
</script>
|
||||
|
||||
{#if hasError}
|
||||
|
|
@ -12,23 +13,31 @@
|
|||
class="pointer-events-auto mx-auto mb-4 max-w-[48rem] px-1"
|
||||
use:fadeInView={{ y: 10, duration: 250 }}
|
||||
>
|
||||
<Alert.Root variant="destructive">
|
||||
<AlertTriangle class="h-4 w-4" />
|
||||
<Alert.Root variant={isLoadingModel ? 'default' : 'destructive'}>
|
||||
{#if isLoadingModel}
|
||||
<Loader2 class="h-4 w-4 animate-spin" />
|
||||
{:else}
|
||||
<AlertTriangle class="h-4 w-4" />
|
||||
{/if}
|
||||
|
||||
<Alert.Title class="flex items-center justify-between">
|
||||
<span>Server unavailable</span>
|
||||
<span>{isLoadingModel ? 'Loading model' : 'Server unavailable'}</span>
|
||||
|
||||
<button
|
||||
onclick={() => serverStore.fetch()}
|
||||
disabled={serverLoading()}
|
||||
class="flex items-center gap-1.5 rounded-lg bg-destructive/20 px-2 py-1 text-xs font-medium hover:bg-destructive/30 disabled:opacity-50"
|
||||
>
|
||||
<RefreshCw class="h-3 w-3 {serverLoading() ? 'animate-spin' : ''}" />
|
||||
{serverLoading() ? 'Retrying...' : 'Retry'}
|
||||
</button>
|
||||
{#if !isLoadingModel}
|
||||
<button
|
||||
onclick={() => serverStore.fetch()}
|
||||
disabled={serverLoading()}
|
||||
class="flex items-center gap-1.5 rounded-lg bg-destructive/20 px-2 py-1 text-xs font-medium hover:bg-destructive/30 disabled:opacity-50"
|
||||
>
|
||||
<RefreshCw class="h-3 w-3 {serverLoading() ? 'animate-spin' : ''}" />
|
||||
{serverLoading() ? 'Retrying...' : 'Retry'}
|
||||
</button>
|
||||
{/if}
|
||||
</Alert.Title>
|
||||
|
||||
<Alert.Description>{serverError()}</Alert.Description>
|
||||
{#if !isLoadingModel}
|
||||
<Alert.Description>{serverError()}</Alert.Description>
|
||||
{/if}
|
||||
</Alert.Root>
|
||||
</div>
|
||||
{/if}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
let {
|
||||
ref = $bindable(null),
|
||||
class: className,
|
||||
sideOffset = 0,
|
||||
sideOffset = 4,
|
||||
side = 'top',
|
||||
children,
|
||||
arrowClasses,
|
||||
|
|
|
|||
|
|
@ -258,12 +258,6 @@ export const GLOB_PATTERNS: string[] = [
|
|||
'**/*.{js,css,html,ico,svg,png,webp,woff,woff2,json,webmanifest}'
|
||||
];
|
||||
|
||||
// loading.html is the model loading page served by llama-server itself.
|
||||
// The SvelteKit PWA manifest transform strips the html extension from every
|
||||
// precache entry to match clean URLs, but loading.html is a plain static asset
|
||||
// with no clean URL, so static servers answer 404 and the SW install fails.
|
||||
export const GLOB_IGNORES: string[] = ['**/loading.html'];
|
||||
|
||||
export const SW_CONFIG = {
|
||||
CHECK_INTERVAL_MS: 60000,
|
||||
UPDATE_FETCH_OPTIONS: {
|
||||
|
|
@ -317,7 +311,6 @@ export const SVELTEKIT_PWA_OPTIONS: SvelteKitPWAOptions = {
|
|||
// Uses '**/' because SvelteKit outputs files under _app/immutable/
|
||||
// subdirectories.
|
||||
globPatterns: GLOB_PATTERNS,
|
||||
globIgnores: GLOB_IGNORES,
|
||||
maximumFileSizeToCacheInBytes: CACHE_SETTINGS.MAX_FILE_SIZE_BYTES,
|
||||
|
||||
// Prevent @vite-pwa/sveltekit from auto-adding a NavigationRoute by
|
||||
|
|
|
|||
|
|
@ -1115,21 +1115,18 @@ class ConversationsStore {
|
|||
}
|
||||
|
||||
/**
|
||||
* Downloads a conversation as JSON file.
|
||||
* Downloads a single conversation as a JSONL file, serializing the full message tree.
|
||||
* @param convId - The conversation ID to download
|
||||
*/
|
||||
async downloadConversation(convId: string): Promise<void> {
|
||||
let conversation: DatabaseConversation | null;
|
||||
let messages: DatabaseMessage[];
|
||||
const conversation =
|
||||
this.activeConversation?.id === convId
|
||||
? this.activeConversation
|
||||
: await DatabaseService.getConversation(convId);
|
||||
|
||||
if (this.activeConversation?.id === convId) {
|
||||
conversation = this.activeConversation;
|
||||
messages = this.activeMessages;
|
||||
} else {
|
||||
conversation = await DatabaseService.getConversation(convId);
|
||||
if (!conversation) return;
|
||||
messages = await DatabaseService.getConversationMessages(convId);
|
||||
}
|
||||
if (!conversation) return;
|
||||
|
||||
const messages = await DatabaseService.getConversationMessages(convId);
|
||||
|
||||
this.downloadConversationFile({ conv: conversation, messages });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -145,6 +145,10 @@ class ModelsStore {
|
|||
*/
|
||||
|
||||
getModelModalities(modelId: string): ModelModalities | null {
|
||||
if (!isRouterMode() && serverStore.props?.modalities) {
|
||||
return this.buildModalities(serverStore.props.modalities);
|
||||
}
|
||||
|
||||
const model = this.models.find((m) => m.model === modelId || m.id === modelId);
|
||||
if (model?.modalities) {
|
||||
return model.modalities;
|
||||
|
|
@ -629,7 +633,12 @@ class ModelsStore {
|
|||
}
|
||||
|
||||
findModelByName(modelName: string): ModelOption | null {
|
||||
return this.models.find((model) => model.model === modelName) ?? null;
|
||||
return (
|
||||
this.models.find(
|
||||
(model) =>
|
||||
model.model === modelName || model.id === modelName || model.aliases?.includes(modelName)
|
||||
) ?? null
|
||||
);
|
||||
}
|
||||
|
||||
findModelById(modelId: string): ModelOption | null {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import { PropsService } from '$lib/services/props.service';
|
||||
import { ServerRole } from '$lib/enums';
|
||||
import { ApiError } from '$lib/utils/api-fetch';
|
||||
|
||||
const LOADING_RETRY_INTERVAL_MS = 1000;
|
||||
|
||||
/**
|
||||
* serverStore - Server connection state, configuration, and role detection
|
||||
|
|
@ -29,8 +32,10 @@ class ServerStore {
|
|||
props = $state<ApiLlamaCppServerProps | null>(null);
|
||||
loading = $state(false);
|
||||
error = $state<string | null>(null);
|
||||
status = $state<number | null>(null);
|
||||
role = $state<ServerRole | null>(null);
|
||||
private fetchPromise: Promise<void> | null = null;
|
||||
private retryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
/**
|
||||
*
|
||||
|
|
@ -70,23 +75,43 @@ class ServerStore {
|
|||
*
|
||||
*/
|
||||
|
||||
async fetch(): Promise<void> {
|
||||
/**
|
||||
* @param background - Set by the automatic "still loading" poll. Skips the
|
||||
* `loading` flag flip so the UI doesn't bounce between the full loading
|
||||
* splash and the chat screen every retry tick.
|
||||
*/
|
||||
async fetch({ background = false }: { background?: boolean } = {}): Promise<void> {
|
||||
if (this.fetchPromise) return this.fetchPromise;
|
||||
|
||||
this.loading = true;
|
||||
this.error = null;
|
||||
this.clearRetryTimer();
|
||||
if (!background) {
|
||||
this.loading = true;
|
||||
}
|
||||
// Don't clear an existing "still loading" error before a retry -
|
||||
// doing so would unmount/remount the error banner every second.
|
||||
if (this.status !== 503) {
|
||||
this.error = null;
|
||||
}
|
||||
|
||||
const fetchPromise = (async () => {
|
||||
try {
|
||||
const props = await PropsService.fetch();
|
||||
this.props = props;
|
||||
this.error = null;
|
||||
this.status = null;
|
||||
this.detectRole(props);
|
||||
} catch (error: unknown) {
|
||||
this.error = error instanceof Error ? error.message : String(error);
|
||||
this.status = error instanceof ApiError ? error.status : null;
|
||||
console.error('Error fetching server properties:', error);
|
||||
|
||||
if (this.status === 503) {
|
||||
this.scheduleRetry();
|
||||
}
|
||||
} finally {
|
||||
this.loading = false;
|
||||
if (!background) {
|
||||
this.loading = false;
|
||||
}
|
||||
this.fetchPromise = null;
|
||||
}
|
||||
})();
|
||||
|
|
@ -96,13 +121,30 @@ class ServerStore {
|
|||
}
|
||||
|
||||
clear(): void {
|
||||
this.clearRetryTimer();
|
||||
this.props = null;
|
||||
this.error = null;
|
||||
this.status = null;
|
||||
this.loading = false;
|
||||
this.role = null;
|
||||
this.fetchPromise = null;
|
||||
}
|
||||
|
||||
private scheduleRetry(): void {
|
||||
if (this.retryTimer) return;
|
||||
this.retryTimer = setTimeout(() => {
|
||||
this.retryTimer = null;
|
||||
this.fetch({ background: true });
|
||||
}, LOADING_RETRY_INTERVAL_MS);
|
||||
}
|
||||
|
||||
private clearRetryTimer(): void {
|
||||
if (this.retryTimer) {
|
||||
clearTimeout(this.retryTimer);
|
||||
this.retryTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
|
|
@ -125,6 +167,7 @@ export const serverStore = new ServerStore();
|
|||
export const serverProps = () => serverStore.props;
|
||||
export const serverLoading = () => serverStore.loading;
|
||||
export const serverError = () => serverStore.error;
|
||||
export const serverStatus = () => serverStore.status;
|
||||
export const serverRole = () => serverStore.role;
|
||||
export const defaultParams = () => serverStore.defaultParams;
|
||||
export const contextSize = () => serverStore.contextSize;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,21 @@ import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '$lib/constants/error';
|
|||
* - Base path resolution
|
||||
*/
|
||||
|
||||
/**
|
||||
* Error thrown when an API request fails, carrying the HTTP status code
|
||||
* so callers can distinguish e.g. a 503 "still loading" response from a
|
||||
* genuine failure.
|
||||
*/
|
||||
export class ApiError extends Error {
|
||||
status: number;
|
||||
|
||||
constructor(message: string, status: number) {
|
||||
super(message);
|
||||
this.name = 'ApiError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export interface ApiFetchOptions extends Omit<RequestInit, 'headers'> {
|
||||
/**
|
||||
* Use auth-only headers (no Content-Type).
|
||||
|
|
@ -67,7 +82,7 @@ export async function apiFetch<T>(path: string, options: ApiFetchOptions = {}):
|
|||
|
||||
if (!response.ok) {
|
||||
const errorMessage = await parseErrorMessage(response);
|
||||
throw new Error(errorMessage);
|
||||
throw new ApiError(errorMessage, response.status);
|
||||
}
|
||||
|
||||
return response.json() as Promise<T>;
|
||||
|
|
@ -119,7 +134,7 @@ export async function apiFetchWithParams<T>(
|
|||
|
||||
if (!response.ok) {
|
||||
const errorMessage = await parseErrorMessage(response);
|
||||
throw new Error(errorMessage);
|
||||
throw new ApiError(errorMessage, response.status);
|
||||
}
|
||||
|
||||
return response.json() as Promise<T>;
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="refresh" content="5">
|
||||
</head>
|
||||
<body>
|
||||
<div id="loading">
|
||||
The model is loading. Please wait.<br/>
|
||||
The user interface will appear soon.
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -189,9 +189,5 @@ describe('PWA Build Output', () => {
|
|||
expect(existsSync(resolve(DIST_DIR, 'pwa-192x192.png'))).toBeTruthy();
|
||||
expect(existsSync(resolve(DIST_DIR, 'pwa-512x512.png'))).toBeTruthy();
|
||||
});
|
||||
|
||||
it('has loading.html fallback page', () => {
|
||||
expect(existsSync(resolve(DIST_DIR, 'loading.html'))).toBeTruthy();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue