diff --git a/conversion/base.py b/conversion/base.py index 8f6b3519c..6aca7f1d3 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1861,6 +1861,9 @@ class TextModel(ModelBase): if chkhsh == "972da7b59cec44d1f0a490a86c96df53859e486e481563e5dddac155013d87ac": # ref: https://huggingface.co/poolside/Laguna-XS.2 res = "laguna" + if chkhsh == "653660222fb704f61cbf2b618a8ae6502b7f8b20c980f9a5de07ed78e13319cd": + # ref: https://huggingface.co/ufakai/ufakzeka-1 + res = "ufakzeka" if res is None: logger.warning("\n") diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 3a15a6fca..24b9bc075 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -163,6 +163,7 @@ models = [ {"name": "granite-embed-multi-311m", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-embedding-311m-multilingual-r2", }, {"name": "mellum2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base"}, {"name": "laguna", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/poolside/Laguna-XS.2", }, + {"name": "ufakzeka", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ufakai/ufakzeka-1", }, ] # some models are known to be broken upstream, so we will skip them as exceptions diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index e5d93dadf..ebbc14c71 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1637,7 +1637,10 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { ggml_backend_synchronize(sched->backends[i]); } - ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids); + if (!ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) { + GGML_LOG_ERROR("%s: failed to reserve graph buffers\n", __func__); + return false; + } if (!ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) { GGML_LOG_ERROR("%s: failed to allocate graph\n", __func__); return false; diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 7b96bbdd4..70cd172a1 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1331,7 +1331,9 @@ UseGgmlGemm1:; const size_t nbw3 = nbw2*ne12; assert(params->wsize >= ne13*nbw3); - GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); + // the F16 path below writes plain floats into wdata, so it needs an F32 vec_dot_type + GGML_ASSERT(src1->type == GGML_TYPE_F32 || vec_dot_type == GGML_TYPE_F32); #if 0 for (int64_t i13 = 0; i13 < ne13; ++i13) { @@ -1350,9 +1352,15 @@ UseGgmlGemm1:; size_t bs = ggml_blck_size(vec_dot_type); int64_t ne10_block_start = (ith * ne10/bs) / nth; int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth; - from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10), - (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0), - (ne10_block_end - ne10_block_start) * bs); + const char * src1_block = (const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10; + char * dst_block = wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0; + const int64_t n_block = (ne10_block_end - ne10_block_start) * bs; + + if (src1->type == GGML_TYPE_F32) { + from_float((const float *) src1_block, dst_block, n_block); + } else { + ggml_cpu_fp16_to_fp32((const ggml_fp16_t *) src1_block, (float *) dst_block, n_block); + } } } } diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index ac91080e6..c4f889ce3 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -451,6 +451,10 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st op->type != GGML_TYPE_IQ1_S && op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float case GGML_OP_MUL_MAT: + if (ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD && + src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) { + return src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16; + } return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type; case GGML_OP_SOFT_MAX_BACK: { if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) { diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 23001254c..ba00a0a73 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -12015,11 +12015,20 @@ void ggml_compute_forward_opt_step_sgd(const ggml_compute_params * params, ggml_ } } -static void ggml_compute_forward_fwht_f32(const ggml_compute_params * params, ggml_tensor * dst) { +static inline float ggml_fwht_load(const float value) { + return value; +} + +static inline float ggml_fwht_load(const ggml_fp16_t value) { + return ggml_fp16_to_fp32(value); +} + +template +static void ggml_compute_forward_fwht_impl(const ggml_compute_params * params, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == (std::is_same_v ? GGML_TYPE_F32 : GGML_TYPE_F16)); GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_TENSOR_BINARY_OP_LOCALS @@ -12046,11 +12055,11 @@ static void ggml_compute_forward_fwht_f32(const ggml_compute_params * params, gg const int64_t i12 = (r - i13 * ne11 * ne12) / ne11; const int64_t i11 = r - i13 * ne11 * ne12 - i12 * ne11; - const float * src_row = (const float *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13); + const src_t * src_row = (const src_t *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13); float * dst_row = (float *) ((char *) dst->data + i11 * nb1 + i12 * nb2 + i13 * nb3); for (int64_t j = 0; j < n; j++) { - dst_row[j] = src_row[j] * scale; + dst_row[j] = ggml_fwht_load(src_row[j]) * scale; } // Scalar passes @@ -12097,12 +12106,17 @@ void ggml_compute_forward_fwht(const ggml_compute_params * params, ggml_tensor * switch (src1->type) { case GGML_TYPE_F32: { - ggml_compute_forward_fwht_f32(params, dst); + ggml_compute_forward_fwht_impl(params, dst); + } + break; + case GGML_TYPE_F16: + { + ggml_compute_forward_fwht_impl(params, dst); } break; default: { - GGML_ABORT("fatal error - fwht is F32 only"); + GGML_ABORT("fatal error - fwht supports F32 and F16 input"); } } } diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index fb7283483..6c29c7e3c 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1498,10 +1498,14 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std: case GGML_TYPE_Q4_K: block_a_size = std430_size({{16, 4}, {fp2_size, fp2_align}}); break; // qs[4] + dm(vec2) case GGML_TYPE_Q5_K: block_a_size = std430_size({{32, 4}, {fp2_size, fp2_align}}); break; // qs[8] + dm(vec2) case GGML_TYPE_Q6_K: block_a_size = std430_size({{32, 4}, {fp2_size, fp2_align}}); break; // qs[8] + d_scales(vec2) + case GGML_TYPE_IQ3_S: block_a_size = std430_size({{32, 4}, {fp_size, fp_align}}); break; // qs[8] + d default: return false; } + // IQ3_S also copies its 512-entry grid into shared memory (types.glsl, init_iq_shmem) + const uint32_t lut_size = (src0_type == GGML_TYPE_IQ3_S) ? 4*512 : 0; + // block_b_cache: { int32_t qs[8]; FLOAT_TYPEV2 ds; } const uint32_t block_b_size = std430_size({{32, 4}, {fp2_size, fp2_align}}); @@ -1517,7 +1521,7 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std: const uint32_t warps = warptile[0] / warptile[10]; const uint32_t ballots_sh = mul_mat_id ? (warps * 4u * (uint32_t)sizeof(uint32_t)) : 0u; - const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + ballots_sh; + const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + ballots_sh + lut_size; const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), " @@ -1794,10 +1798,10 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } // The q8_1 mmq path has its own (larger) shmem layout, check it separately. - // K-quants use the _int_k warptiles, others use _int. + // K-quants and IQ3_S use the _int_k warptiles, others use _int. const bool is_k_quant = (t == GGML_TYPE_Q2_K || t == GGML_TYPE_Q3_K || t == GGML_TYPE_Q4_K || t == GGML_TYPE_Q5_K || - t == GGML_TYPE_Q6_K); + t == GGML_TYPE_Q6_K || t == GGML_TYPE_IQ3_S); const auto & s_int = is_k_quant ? s_warptile_mmq_int_k : s_warptile_mmq_int; const auto & m_int = is_k_quant ? m_warptile_mmq_int_k : m_warptile_mmq_int; const auto & l_int = is_k_quant ? l_warptile_mmq_int_k : l_warptile_mmq_int; @@ -2461,6 +2465,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_len, matmul_q4_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_len, matmul_q5_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_len, matmul_q6_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_IQ3_S, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_iq3_s_q8_1", matmul_iq3_s_q8_1_len, matmul_iq3_s_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); } #endif @@ -2497,6 +2502,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q4_k_q8_1", matmul_id_subgroup_q4_k_q8_1_len, matmul_id_subgroup_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q5_k_q8_1", matmul_id_subgroup_q5_k_q8_1_len, matmul_id_subgroup_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q6_k_q8_1", matmul_id_subgroup_q6_k_q8_1_len, matmul_id_subgroup_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_IQ3_S, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_iq3_s_q8_1", matmul_id_subgroup_iq3_s_q8_1_len, matmul_id_subgroup_iq3_s_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); } #endif } else { @@ -2532,6 +2538,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q4_k_q8_1", matmul_id_q4_k_q8_1_len, matmul_id_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q5_k_q8_1", matmul_id_q5_k_q8_1_len, matmul_id_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q6_k_q8_1", matmul_id_q6_k_q8_1_len, matmul_id_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_IQ3_S, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_iq3_s_q8_1", matmul_id_iq3_s_q8_1_len, matmul_id_iq3_s_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } #endif } @@ -2568,6 +2575,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_fp32_len, matmul_q4_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_fp32_len, matmul_q5_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_fp32_len, matmul_q6_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_IQ3_S, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_iq3_s_q8_1", matmul_iq3_s_q8_1_fp32_len, matmul_iq3_s_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); } #endif @@ -7069,7 +7077,9 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& // n_as counts, n_as offsets, one total, then one packed row id per (expert, token). // Hoisting requires 16-bit indices for the packing and a table that fits one binding. const uint64_t hoisted_row_id_words = 2 * n_as + 1 + nei0 * nei1; - const bool hoist_row_ids = n_as <= 256 && nei0 <= 0xffff && nei1 <= 0xffff && + // 1024 matches MAX_EXPERTS in count_experts.comp and LLAMA_MAX_EXPERTS. It costs + // 3 * 1024 * 4 = 12 KiB of shared memory, within the 16 KiB Vulkan guarantees. + const bool hoist_row_ids = n_as <= 1024 && nei0 <= 0xffff && nei1 <= 0xffff && hoisted_row_id_words * sizeof(uint32_t) <= ctx->device->properties.limits.maxStorageBufferRange; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp index ef659959d..06a50181c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp @@ -30,9 +30,14 @@ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {uint data_a[];}; layout (binding = 1) writeonly buffer D {uint data_d[];}; -shared uint vals[BLOCK_SIZE]; -shared uint offsets[BLOCK_SIZE]; -shared uint cursors[BLOCK_SIZE]; +// Upper bound on n_experts for the hoisted row-id path. Must match the limit in +// ggml_vk_mul_mat_id_q_f16 (hoist_row_ids). The non-hoisted reduction below only +// needs BLOCK_SIZE entries. +#define MAX_EXPERTS 1024 + +shared uint vals[MAX_EXPERTS]; +shared uint offsets[MAX_EXPERTS]; +shared uint cursors[MAX_EXPERTS]; // data_d layout when p.hoist_row_ids is set: // [0, n_experts) per-expert row count @@ -46,8 +51,8 @@ void main() { const uint tid = gl_LocalInvocationID.x; if (p.hoist_row_ids != 0) { - if (tid < p.n_experts) { - vals[tid] = 0; + for (uint e = tid; e < p.n_experts; e += BLOCK_SIZE) { + vals[e] = 0; } barrier(); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl index 24da4f715..5fc4d3db4 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl @@ -454,6 +454,55 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { } #endif +#if defined(DATA_A_IQ3_S) +// 2-byte loads for IQ3_S blocks (110 bytes) +void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { + const uint ib_k = ib / 8; + const uint ib32 = ib % 8; + + // grid indices for qs[2 * iqs] and qs[2 * iqs + 1] + const uint qs = uint(data_a_packed16[ib_k].qs[ib32 * 4 + iqs]); + // their two high index bits + const uint qh = uint(data_a_packed16[ib_k].qh[ib32 / 2]) >> ((ib32 & 1) * 8 + 2 * iqs); + // one sign bit per value, 8 values + const uint signs = uint(data_a_packed16[ib_k].signs[ib32 * 2 + iqs / 2]) >> ((iqs & 1) * 8); + + // grid holds 4 values of 1..15, one per byte + const ivec4 vals0 = ivec4(unpack8(iq3s_grid[( qs & 0xFF) | ((qh & 1) << 8)])); + const ivec4 vals1 = ivec4(unpack8(iq3s_grid[((qs >> 8) & 0xFF) | ((qh & 2) << 7)])); + + // negate with (v ^ -s) - -s to avoid branches + const ivec4 m0 = -(ivec4(signs, signs >> 1, signs >> 2, signs >> 3) & 1); + const ivec4 m1 = -(ivec4(signs >> 4, signs >> 5, signs >> 6, signs >> 7) & 1); + + buf_a[buf_ib].qs[2 * iqs ] = pack32(i8vec4((vals0 ^ m0) - m0)); + buf_a[buf_ib].qs[2 * iqs + 1] = pack32(i8vec4((vals1 ^ m1) - m1)); + + if (iqs == 0) { + const uint scale = (uint(data_a_packed16[ib_k].scales[ib32 / 4]) >> ((ib32 & 3) * 4)) & 0xF; + + buf_a[buf_ib].d = FLOAT_TYPE(float(data_a_packed16[ib_k].d) * float(1 + 2 * scale)); + } +} + +void block_a_to_registers(const uint reg_ib, const uint buf_ib) { + cache_a[reg_ib].d = buf_a[buf_ib].d; + + [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) { + cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs]; + } +} + +ACC_TYPE mmq_dot_product(const uint ib_a) { + int32_t q_sum = 0; + [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) { + q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]); + } + + return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum)); +} +#endif + void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) { if (is_in_bounds) { const uint ib_outer = ib / 4; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl index 2b7adcb6c..7632e457a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl @@ -59,6 +59,12 @@ struct block_a_cache { int32_t qs[8]; FLOAT_TYPE d; }; +#elif defined(DATA_A_IQ3_S) +#define QUANT_R_MMQ 2 +struct block_a_cache { + int32_t qs[8]; + FLOAT_TYPE d; +}; #elif defined(DATA_A_Q2_K) #define QUANT_R_MMQ 4 struct block_a_cache { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index a37e72656..09b0861f8 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -650,7 +650,7 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c }; #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4")) { + if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4" || tname == "iq3_s")) { string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc); } #endif diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 36b4c3190..27c126b36 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -5876,7 +5876,7 @@ GGML_QUANT_SIZES: dict[GGMLQuantizationType, tuple[int, int]] = { GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16), GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16), GGMLQuantizationType.Q8_0: (32, 2 + 32), - GGMLQuantizationType.Q8_1: (32, 4 + 4 + 32), + GGMLQuantizationType.Q8_1: (32, 2 + 2 + 32), GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4), GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12), GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12), diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 0a27367c9..0f5155b2e 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -15,26 +15,11 @@ bool llama_model_saver_supports_arch(llm_arch arch) { switch (arch) { - case LLM_ARCH_PLAMO3: - case LLM_ARCH_GEMMA3: case LLM_ARCH_GEMMA3N: - case LLM_ARCH_COHERE2: - case LLM_ARCH_COHERE2MOE: - case LLM_ARCH_OLMO2: case LLM_ARCH_BITNET: case LLM_ARCH_T5: - case LLM_ARCH_EXAONE_MOE: - case LLM_ARCH_AFMOE: case LLM_ARCH_APERTUS: - case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: - case LLM_ARCH_SPARK2_5: - case LLM_ARCH_MUSE_GLIMMER: - case LLM_ARCH_MELLUM: - case LLM_ARCH_LAGUNA: - case LLM_ARCH_GRANITE_SWA: - case LLM_ARCH_DOTS3NOTE: // TODO: need to handle SWA pattern and MLA+SWA config - case LLM_ARCH_MAPLE: return false; default: return true; @@ -290,7 +275,11 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts); add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true); add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); - // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???); + if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { + // never collapsed to a scalar: the loaders read a scalar as a period + add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector( + hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all)); + } add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale); add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale); @@ -300,6 +289,9 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl); add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa); add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa); + add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, hparams.n_embd_head_k_mla_swa); + add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa); + add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, hparams.n_lora_kv_swa); add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 6f963eb83..713f03fd5 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -3407,6 +3407,15 @@ ggml_tensor * llama_model_base::create_tensor(const LLM_TN_IMPL & tn, const std: } void llama_model_base::create_tensor_gate_up_exps(llama_layer & layer, int bid, int64_t n_embd_, int64_t n_ff_, int64_t n_expert_, int flags) { + if (flags & TENSOR_SKIP) { + const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP; + + create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", bid), {n_embd_, n_ff_ * 2, n_expert_}, skip | TENSOR_SKIP_IF_VIRTUAL); + create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", bid), {n_embd_, n_ff_, n_expert_}, skip); + create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", bid), {n_embd_, n_ff_, n_expert_}, skip); + return; + } + layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", bid), {n_embd_, n_ff_ * 2, n_expert_}, TENSOR_NOT_REQUIRED); if (layer.ffn_gate_up_exps == nullptr) { layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", bid), {n_embd_, n_ff_, n_expert_}, flags); @@ -3452,6 +3461,15 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, } } +void llama_model_base::load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first) { + if (ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, false)) { + return; + } + + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern, false); + hparams.set_swa_pattern(n_pattern, dense_first); +} + const int32_t * llama_model_target_layer_ids(const struct llama_model * model) { const auto & v = model->target_layer_ids; return v.empty() ? nullptr : v.data(); diff --git a/src/llama-model.h b/src/llama-model.h index d61afa2cb..984b2cf38 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -814,6 +814,9 @@ struct llama_model_base : public llama_model { int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_, int flags); + // helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern + void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false); + void load_stats (llama_model_loader & ml) override; void load_hparams(llama_model_loader & ml) override; void load_vocab (llama_model_loader & ml) override; diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 4e10fb4f9..b84005995 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -713,6 +713,12 @@ struct llm_tokenizer_bpe : llm_tokenizer { "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1}| ?[^\\s\\p{L}\\p{N}\\r\\n]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", }; break; + case LLAMA_VOCAB_PRE_TYPE_UFAKZEKA: + regex_exprs = { + // Qwen2 pattern without the English contraction group, so Turkish apostrophe suffixes stay attached + "[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + break; case LLAMA_VOCAB_PRE_TYPE_GROK_2: regex_exprs = { // original regex from tokenizer.json @@ -2612,6 +2618,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "kimi-k2") { pre_type = LLAMA_VOCAB_PRE_TYPE_KIMI_K2; clean_spaces = false; + } else if ( + tokenizer_pre == "ufakzeka") { + pre_type = LLAMA_VOCAB_PRE_TYPE_UFAKZEKA; + clean_spaces = false; } else if ( tokenizer_pre == "grok-2") { pre_type = LLAMA_VOCAB_PRE_TYPE_GROK_2; diff --git a/src/llama-vocab.h b/src/llama-vocab.h index c2c822594..24d2ea00b 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -68,6 +68,7 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, LLAMA_VOCAB_PRE_TYPE_HY_V4 = 57, LLAMA_VOCAB_PRE_TYPE_SPARK2_5 = 58, + LLAMA_VOCAB_PRE_TYPE_UFAKZEKA = 59, }; struct LLM_KV; diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp index cf0220367..08c22b6ed 100644 --- a/src/models/afmoe.cpp +++ b/src/models/afmoe.cpp @@ -14,9 +14,7 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) { // Pattern: 3 sliding - 1 full (global_attn_every_n_layers = 4) if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cpp index e2b366256..7ad5244e4 100644 --- a/src/models/cohere2.cpp +++ b/src/models/cohere2.cpp @@ -2,9 +2,7 @@ void llama_model_cohere2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp index 5e02cd56e..7704cbb87 100644 --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -25,12 +25,7 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) { } hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { - hparams.set_swa_pattern(swa_period, true); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4, true); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp index 976ee050a..840da5f67 100644 --- a/src/models/exaone-moe.cpp +++ b/src/models/exaone-moe.cpp @@ -3,9 +3,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 128; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp index 9ba978956..c1c55957a 100644 --- a/src/models/exaone4.cpp +++ b/src/models/exaone4.cpp @@ -4,9 +4,7 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_layer() == 64) { // 32B hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cpp index 80ed3b1a4..6c97883d8 100644 --- a/src/models/gemma-embedding.cpp +++ b/src/models/gemma-embedding.cpp @@ -2,9 +2,7 @@ void llama_model_gemma_embedding::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC; - uint32_t swa_period = 6; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 6); hparams.causal_attn = false; // embeddings do not use causal attention diff --git a/src/models/gemma2.cpp b/src/models/gemma2.cpp index 2fbfb15a9..9e5243465 100644 --- a/src/models/gemma2.cpp +++ b/src/models/gemma2.cpp @@ -3,9 +3,7 @@ void llama_model_gemma2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; // default value of gemma 2 - uint32_t swa_period = 2; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 2); hparams.attn_soft_cap = true; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp index 690194529..f99bbaacd 100644 --- a/src/models/gemma3.cpp +++ b/src/models/gemma3.cpp @@ -4,9 +4,7 @@ void llama_model_gemma3::load_arch_hparams(llama_model_loader & ml) { const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 6; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 6); ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); } else { diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index bb628203a..4d47ddc62 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -1,10 +1,8 @@ #include "models.h" void llama_model_gemma3n::load_arch_hparams(llama_model_loader & ml) { - uint32_t swa_period = 5; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 5); hparams.n_layer_kv_from_start = 20; hparams.f_attention_scale = 1.0f; diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp index 556400bfc..2bc4fa8a0 100644 --- a/src/models/laguna.cpp +++ b/src/models/laguna.cpp @@ -36,9 +36,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, /*dense_first=*/true); // XS.2: FULL at il%4==0 + load_swa_pattern(ml, 4, /*dense_first=*/true); // XS.2: FULL at il%4==0 // Per-layer-type RoPE: full layers use YaRN θ=500000 over 64 dims; // SWA layers use default RoPE θ=10000 over 128 dims. Base load_hparams diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp index 8a812beff..4f79b4d1e 100644 --- a/src/models/llama4.cpp +++ b/src/models/llama4.cpp @@ -16,9 +16,7 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) { hparams.f_attn_temp_scale = 0.1f; hparams.f_attn_temp_offset = 1.0f; - uint32_t swa_period = 4; // pattern: 3 chunked - 1 full - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); // pattern: 3 chunked - 1 full hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp index 872a9c8f5..3ab79747a 100644 --- a/src/models/mellum.cpp +++ b/src/models/mellum.cpp @@ -8,13 +8,7 @@ void llama_model_mellum::load_arch_hparams(llama_model_loader & ml) { if (hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - const auto res = ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - if (res) { - hparams.set_swa_pattern(swa_period); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cpp index f3e9407e0..b7542d59b 100644 --- a/src/models/modern-bert.cpp +++ b/src/models/modern-bert.cpp @@ -5,9 +5,7 @@ void llama_model_modern_bert::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - uint32_t swa_period = 3; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, true); + load_swa_pattern(ml, 3, true); } else { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } diff --git a/src/models/muse-glimmer.cpp b/src/models/muse-glimmer.cpp index 0e9415308..0e5f75ebe 100644 --- a/src/models/muse-glimmer.cpp +++ b/src/models/muse-glimmer.cpp @@ -10,12 +10,7 @@ void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { - hparams.set_swa_pattern(swa_period); - } else { - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); - } + load_swa_pattern(ml, 4); switch (hparams.n_layer()) { case 52: type = LLM_TYPE_30B; break; diff --git a/src/models/nemotron-h-moe.cpp b/src/models/nemotron-h-moe.cpp index 4d03f49e0..b4fb25430 100644 --- a/src/models/nemotron-h-moe.cpp +++ b/src/models/nemotron-h-moe.cpp @@ -100,6 +100,18 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, cons ggml_tensor * router_logits = build_lora_mm(layer.ffn_gate_inp, cur); cb(router_logits, "mtp_ffn_moe_logits", il); + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, NULL, layer.ffn_up_shexp_s, + NULL, NULL, NULL, + layer.ffn_down_shexp, NULL, layer.ffn_down_shexp_s, + NULL, + LLM_FFN_RELU_SQR, LLM_FFN_PAR, il); + cb(ffn_shexp, "mtp_ffn_shexp", il); + + if (layer.ffn_latent_down) { + cur = ggml_mul_mat(ctx0, layer.ffn_latent_down, cur); + } + ggml_tensor * moe_out = build_moe_ffn(cur, layer.ffn_gate_inp, @@ -118,13 +130,9 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, cons layer.ffn_down_exps_s); cb(moe_out, "mtp_ffn_moe_out", il); - ggml_tensor * ffn_shexp = build_ffn(cur, - layer.ffn_up_shexp, NULL, layer.ffn_up_shexp_s, - NULL, NULL, NULL, - layer.ffn_down_shexp, NULL, layer.ffn_down_shexp_s, - NULL, - LLM_FFN_RELU_SQR, LLM_FFN_PAR, il); - cb(ffn_shexp, "mtp_ffn_shexp", il); + if (layer.ffn_latent_up) { + moe_out = ggml_mul_mat(ctx0, layer.ffn_latent_up, moe_out); + } cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "mtp_ffn_out", il); diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp index 24ed9a673..be27650b0 100644 --- a/src/models/nemotron-h.cpp +++ b/src/models/nemotron-h.cpp @@ -172,6 +172,8 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) { layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, mtp_flags); layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, mtp_flags); layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, mtp_flags); + layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, mtp_flags | TENSOR_NOT_REQUIRED); + layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, mtp_flags | TENSOR_NOT_REQUIRED); layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, moe_n_embd, n_expert}, mtp_flags); layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp, n_expert}, mtp_flags); layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, mtp_flags); diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp index 05b9394b8..fe5dc88d4 100644 --- a/src/models/olmo2.cpp +++ b/src/models/olmo2.cpp @@ -6,9 +6,7 @@ void llama_model_olmo2::load_arch_hparams(llama_model_loader & ml) { const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 4); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = 1.0; // See olmo2.cpp diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp index c9f9b677d..39660a325 100644 --- a/src/models/openai-moe.cpp +++ b/src/models/openai-moe.cpp @@ -6,9 +6,7 @@ void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 2; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 2); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp index 16d0b1dce..f8235f8a8 100644 --- a/src/models/plamo3.cpp +++ b/src/models/plamo3.cpp @@ -6,9 +6,7 @@ void llama_model_plamo3::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - uint32_t swa_period = 8; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period); + load_swa_pattern(ml, 8); } else { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp index 680ffb8fd..555f8b718 100644 --- a/src/models/smallthinker.cpp +++ b/src/models/smallthinker.cpp @@ -6,9 +6,7 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) { if (found_swa && hparams.n_swa > 0) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); - hparams.set_swa_pattern(swa_period, true); + load_swa_pattern(ml, 4, true); hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 64b78984a..f08e59282 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -4528,7 +4528,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { // build the inference graph ggml_backend_sched_reset(ctx->sched.get()); ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs, params)->build(); - ggml_backend_sched_alloc_graph(ctx->sched.get(), gf); + if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) { + LOG_ERR("%s: failed to allocate compute graph\n", __func__); + return false; + } // set inputs const auto & model = ctx->model; diff --git a/tools/rpc/rpc-server.cpp b/tools/rpc/rpc-server.cpp index 08e680391..2db5e52ab 100644 --- a/tools/rpc/rpc-server.cpp +++ b/tools/rpc/rpc-server.cpp @@ -1,3 +1,4 @@ +#include "ggml-backend.h" #include "ggml-rpc.h" #ifdef _WIN32 # define NOMINMAX @@ -270,7 +271,8 @@ static std::vector get_devices(const rpc_server_params & par if (devices.empty()) { for (size_t i = 0; i < ggml_backend_dev_count(); i++) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); - if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) { + enum ggml_backend_dev_type dev_type = ggml_backend_dev_type(dev); + if (dev_type != GGML_BACKEND_DEVICE_TYPE_CPU && dev_type != GGML_BACKEND_DEVICE_TYPE_ACCEL) { devices.push_back(dev); } } diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte index 9c3a9e891..1ce731845 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte @@ -1,6 +1,10 @@