vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-19 13:25:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge commit '11b068d066' into concedo_experimental

Browse source 
# Conflicts:
#	CONTRIBUTING.md
#	docs/backend/SYCL.md
#	docs/install.md
#	docs/speculative.md
#	ggml/src/ggml-hip/CMakeLists.txt
#	ggml/src/ggml-opencl/ggml-opencl.cpp
#	ggml/src/ggml-sycl/common.hpp
#	ggml/src/ggml-sycl/element_wise.cpp
#	ggml/src/ggml-sycl/fattn-onednn.cpp
#	ggml/src/ggml-sycl/ggml-sycl.cpp
#	ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
#	ggml/src/ggml-webgpu/ggml-webgpu.cpp
#	ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl
#	ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl
#	tests/test-backend-ops.cpp
#	tests/test-chat.cpp
#	tests/test-llama-archs.cpp
#	tools/cli/README.md
#	tools/llama-bench/llama-bench.cpp
#	tools/mtmd/CMakeLists.txt
#	tools/server/README.md
This commit is contained in:
Concedo 2026-07-31 17:06:19 +08:00
commit 98926f27c1
81 changed files with 5314 additions and 518 deletions
Download patch file Download diff file Expand all files Collapse all files

381
tools/mtmd/clip.cpp
View file

@ -61,11 +61,13 @@
#include "models/minicpmv.cpp"
#include "models/minimax-m3.cpp"
#include "models/paddleocr.cpp"
#include "models/parakeet.cpp"
#include "models/pixtral.cpp"
#include "models/qwen2vl.cpp"
#include "models/qwen3vl.cpp"
#include "models/mimovl.cpp"
#include "models/qwen3a.cpp"
#include "models/mimovl.cpp"
#include "models/mimo-audio.cpp"
#include "models/step3vl.cpp"
#include "models/siglip.cpp"
#include "models/whisper-enc.cpp"
@ -389,6 +391,11 @@ ggml_tensor * clip_graph::build_vit(
auto & layer = model.layers[il];
ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
ggml_tensor * attn_mask = opts.attn_mask;
if (opts.attn_mask_layers.size() > (size_t) il) {
attn_mask = opts.attn_mask_layers[il];
}
// layernorm1
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
cb(cur, "layer_inp_normed", il);
@ -501,7 +508,7 @@ ggml_tensor * clip_graph::build_vit(
// build_attn returns a flat 2D [n_embd, n_pos*B]
cur = build_attn(layer.o_w, layer.o_b,
Qcur, Kcur, Vcur, opts.attn_mask, kq_scale, il);
Qcur, Kcur, Vcur, attn_mask, kq_scale, il);
cb(cur, "attn_out", il);
}
@ -520,6 +527,10 @@ ggml_tensor * clip_graph::build_vit(
inpL = cur; // inpL = residual, cur = hidden_states
if (opts.callback_layer_out) {
opts.callback_layer_out(cur, il);
}
cb(cur, "ffn_inp", il);
// layernorm2 (pre-ffn norm)
@ -568,7 +579,7 @@ ggml_tensor * clip_graph::build_vit(
}
// post-layernorm
if (model.post_ln_w) {
if (model.post_ln_w && !opts.skip_post_ln) {
inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1);
}
@ -1061,6 +1072,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_qwen3a>(ctx, img);
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
builder = std::make_unique<clip_graph_mimo_audio>(ctx, img);
} break;
case PROJECTOR_TYPE_YOUTUVL:
{
builder = std::make_unique<clip_graph_youtuvl>(ctx, img);
@ -1069,6 +1084,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_yasa2>(ctx, img);
} break;
case PROJECTOR_TYPE_PARAKEET:
{
builder = std::make_unique<clip_graph_parakeet>(ctx, img);
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
builder = std::make_unique<clip_graph_granite4_vision>(ctx, img);
@ -1415,6 +1434,20 @@ struct clip_model_loader {
{
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
} break;
case PROJECTOR_TYPE_PARAKEET:
{
get_u32(KEY_AUDIO_SUBSAMPLING_FACTOR, hparams.subsampling_factor);
GGML_ASSERT(hparams.subsampling_factor == 8 &&
"subsampling_factor must match the conv strides in clip_graph_parakeet::build()");
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size);
GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 &&
"audio_conv_kernel_size must be a positive odd integer");
hparams.audio_chunk_len = 0;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 512;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_IDEFICS3:
{
// use default llava-uhd preprocessing params
@ -1652,6 +1685,45 @@ struct clip_model_loader {
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
get_u32(KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers, false);
get_arr_int(KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size, false);
if (hparams.rvq_num_quantizers <= 0) {
throw std::runtime_error(string_format("%s: mimo_audio: missing %s\n", __func__, KEY_A_RVQ_NUM_QUANTIZERS));
}
if ((int) hparams.rvq_codebook_size.size() != hparams.rvq_num_quantizers) {
throw std::runtime_error(string_format(
"%s: mimo_audio: %s length (%zu) must equal %s (%d)\n", __func__,
KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size.size(),
KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers));
}
hparams.ffn_op = FFN_GELU_ERF; // PyTorch F.gelu default (approximate="none")
hparams.rope_theta = 10000.0f;
// audio preprocessing params (mel spectrogram)
hparams.audio_sample_rate = 24000;
hparams.audio_n_fft = 960;
hparams.audio_window_len = 960;
hparams.audio_hop_len = 240;
get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size);
std::vector<int> wa_pattern;
get_arr_int(KEY_A_WA_PATTERN_MODE, wa_pattern, true);
if ((int) wa_pattern.size() != hparams.n_layer) {
throw std::runtime_error(string_format(
"%s: mimo_audio: %s length (%zu) must equal n_layer (%d)\n", __func__,
KEY_A_WA_PATTERN_MODE, wa_pattern.size(), hparams.n_layer));
}
hparams.wa_pattern_mode.assign(wa_pattern.begin(), wa_pattern.end());
get_u32(KEY_A_LOCAL_BLOCK_COUNT, hparams.audio_local_n_layer);
get_u32(KEY_A_LOCAL_GROUP_SIZE, hparams.audio_local_group_size);
if (hparams.audio_local_group_size <= 0) {
throw std::runtime_error(string_format(
"%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE));
}
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
hparams.n_merge = 2;
@ -1923,16 +1995,46 @@ struct clip_model_loader {
return cur;
};
auto get_scalar = [&](const std::string & name, float default_val) {
auto get_vector = [&](const std::string & name) {
std::vector<float> result;
auto it = tensor_offset.find(name);
if (it == tensor_offset.end()) {
return result;
}
const int64_t idx = gguf_find_tensor(ctx_gguf.get(), name.c_str());
if (idx < 0) {
throw std::runtime_error(string_format("%s: failed to find tensor %s\n", __func__, name.c_str()));
}
if (const auto type = gguf_get_tensor_type(ctx_gguf.get(), idx); type != GGML_TYPE_F32) {
throw std::runtime_error(string_format("%s: %s must be %s, was %s\n", __func__,
name.c_str(), ggml_type_name(GGML_TYPE_F32), ggml_type_name(type)));
}
const size_t n_bytes = gguf_get_tensor_size(ctx_gguf.get(), idx);
if (n_bytes == 0) {
throw std::runtime_error(string_format("%s: tensor %s is empty\n", __func__, name.c_str()));
}
const size_t n_elems = n_bytes / sizeof(float);
result.resize(n_elems);
fin.seekg(it->second, std::ios::beg);
fin.read(reinterpret_cast<char*>(result.data()), n_bytes);
return result;
};
auto get_scalar = [&](const std::string & name, float default_val) {
auto v = get_vector(name);
if (v.empty()) {
return default_val;
}
size_t offset = it->second;
fin.seekg(offset, std::ios::beg);
float value;
fin.read(reinterpret_cast<char*>(&value), sizeof(float));
return value;
if (v.size() != 1) {
throw std::runtime_error(string_format("%s: expected scalar tensor '%s' but got %d elements\n",
__func__, name.c_str(), (int) v.size()));
}
return v[0];
};
model.class_embedding = get_tensor(TN_CLASS_EMBD, false);
@ -2526,6 +2628,54 @@ struct clip_model_loader {
model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias"));
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias"));
model.downsample_conv_w = get_tensor(string_format(TN_A_DOWNSAMPLE_CONV, "weight"));
model.downsample_norm_w = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "weight"));
model.downsample_norm_b = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "bias"));
model.rvq_codebook = get_tensor(string_format(TN_A_RVQ_CODEBOOK, "weight"), false);
model.mm_a_code_embd = get_tensor(string_format(TN_MM_A_CODE_EMBD, "weight"), false);
if (!model.rvq_codebook || !model.mm_a_code_embd) {
throw std::runtime_error(string_format("%s: mimo_audio: missing %s or %s\n", __func__,
TN_A_RVQ_CODEBOOK, TN_MM_A_CODE_EMBD));
}
// hparams.rvq_codebook_size comes from GGUF metadata and is independent of the
// tensors' actual shapes - bound it so codebook/code_embd views built from it
// (mimo-audio.cpp) can never read past either tensor's allocated bins/vocab.
for (int32_t bins : hparams.rvq_codebook_size) {
if (bins <= 0 || bins > model.rvq_codebook->ne[1] || bins > model.mm_a_code_embd->ne[1]) {
throw std::runtime_error(string_format(
"%s: mimo_audio: %s entry (%d) out of range for codebook/code_embd tensors\n",
__func__, KEY_A_RVQ_CODEBOOK_SIZE, bins));
}
}
// LLM-side connector: input_local_transformer + projection
model.mm_a_local_layers.resize(hparams.audio_local_n_layer);
for (int il = 0; il < hparams.audio_local_n_layer; il++) {
auto & layer = model.mm_a_local_layers[il];
layer.q_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q, il, "weight"));
layer.q_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q, il, "bias"));
layer.k_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K, il, "weight"));
layer.k_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K, il, "bias"));
layer.v_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V, il, "weight"));
layer.v_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V, il, "bias"));
layer.o_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_OUT, il, "weight"));
layer.ff_gate_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_GATE, il, "weight"));
layer.ff_up_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_UP, il, "weight"));
layer.ff_down_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_DOWN, il, "weight"));
layer.ln_1_w = get_tensor(string_format(TN_MM_A_LOCAL_LN1, il, "weight"));
layer.ln_2_w = get_tensor(string_format(TN_MM_A_LOCAL_LN2, il, "weight"));
}
model.mm_a_local_norm_w = get_tensor(string_format(TN_MM_A_LOCAL_NORM, "weight"));
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
} break;
case PROJECTOR_TYPE_VOXTRAL:
{
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
@ -2782,6 +2932,68 @@ struct clip_model_loader {
layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"));
}
} break;
case PROJECTOR_TYPE_PARAKEET:
{
hparams.mel_filters = get_vector(TN_MEL_FILTERS);
hparams.window = get_vector(TN_WINDOW);
// Subsampling layers (conv1d)
for (int i : {0, 2, 3, 5, 6}) {
model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
}
model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
// Projection layers
model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"), false);
model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"), false);
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"), false);
// Encoder layers
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = model.layers[il];
// Attention (from shared above)
// Relative position encoding
layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight"));
layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il));
layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il));
// Convolution module
layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"));
layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"), false);
layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight"));
layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"), false);
layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight"));
layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias"));
layer.conv_norm_mean = get_tensor(string_format(TN_CONV_NORM_MEAN, prefix, il));
layer.conv_norm_var = get_tensor(string_format(TN_CONV_NORM_VAR, prefix, il));
layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"));
layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"), false);
// Feed-forward networks
layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight"));
layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias"));
layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight"));
layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias"));
layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight"));
layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false);
layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight"));
layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false);
// Layer norms
layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight"));
layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias"));
}
model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight"));
@ -3627,10 +3839,23 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
}
n_patches = n;
} break;
case PROJECTOR_TYPE_PARAKEET:
{
n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor;
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
n_patches = img->nx(); // no downsampling: one token per raw waveform frame
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
// conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2)
int n = img->nx();
n = (n - 1) / 2 + 1; // conv1 + conv2
n = (n - 2) / 2 + 1; // downsample conv
const int group_size = params.audio_local_group_size;
n_patches = (n + group_size - 1) / group_size;
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
const int ws = ctx->model.hparams.audio_proj_window_size;
@ -4458,6 +4683,58 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
set_input_f32("pos_emb", pos_emb);
}
} break;
case PROJECTOR_TYPE_MIMO_AUDIO:
{
GGML_ASSERT(imgs.entries.size() == 1);
const int n_frames = imgs.entries.front().nx();
const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length
std::vector<int32_t> positions(n_pos);
for (int i = 0; i < n_pos; i++) {
positions[i] = i;
}
set_input_i32("mimo_audio_positions", positions);
const int window = hparams.attn_window_size;
GGML_ASSERT(window > 0);
const float neg_inf = std::numeric_limits<float>::lowest();
std::vector<float> full_mask((size_t) n_pos * n_pos);
std::vector<float> window_mask((size_t) n_pos * n_pos);
for (int q = 0; q < n_pos; q++) {
for (int k = 0; k < n_pos; k++) {
const bool causal_ok = k <= q;
full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf;
window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf;
}
}
set_input_f32("mimo_audio_full_mask", full_mask);
set_input_f32("mimo_audio_window_mask", window_mask);
// input_local_transformer: block-diagonal mask + in-group positions
{
const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0)
const int group_size = hparams.audio_local_group_size;
GGML_ASSERT(group_size > 0);
const int n_groups = (n_pos_ds + group_size - 1) / group_size;
const int n_padded = n_groups * group_size;
std::vector<int32_t> local_positions(n_padded);
for (int i = 0; i < n_padded; i++) {
local_positions[i] = i % group_size;
}
set_input_i32("mimo_audio_local_positions", local_positions);
std::vector<float> local_mask((size_t) n_padded * n_padded);
for (int q = 0; q < n_padded; q++) {
for (int k = 0; k < n_padded; k++) {
const bool same_group = (q / group_size) == (k / group_size);
local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf;
}
}
set_input_f32("mimo_audio_local_mask", local_mask);
}
} break;
case PROJECTOR_TYPE_LFM2A:
{
GGML_ASSERT(imgs.entries.size() == 1);
@ -4479,6 +4756,88 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
}
set_input_f32("pos_emb", pos_emb);
} break;
case PROJECTOR_TYPE_PARAKEET:
{
GGML_ASSERT(imgs.entries.size() == 1);
struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask");
const int n_q = attn_mask->ne[1];
const int n_k = attn_mask->ne[0];
const int n_frames = imgs.entries.front().nx();
const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor;
const float mask_value = -1e30f;
std::vector<float> mask_data(n_q * n_k);
if (n_k == n_q) {
// full attention: mask keys that are padding
for (int q = 0; q < n_q; ++q) {
for (int k = 0; k < n_k; ++k) {
mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f;
}
}
} else {
// local attention: mask keys outside the valid window
const int att_left = n_k / 2;
for (int q = 0; q < n_q; ++q) {
for (int k = 0; k < n_k; ++k) {
const int key = q - att_left + k;
mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value;
}
}
}
set_input_f32(attn_mask->name, mask_data);
// local attention skew mask: zeroes out the probs that were
// computed for keys outside the valid sliding window.
if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) {
const int lm_k = local_mask->ne[0];
const int lm_q = local_mask->ne[1];
const int window_size = lm_k - lm_q + 1;
std::vector<float> lm_data(lm_q * lm_k);
for (int q = 0; q < lm_q; ++q) {
for (int k = 0; k < lm_k; ++k) {
const int rel = k - q;
lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f;
}
}
set_input_f32(local_mask->name, lm_data);
}
// Generate rotation frequencies for relative positional encoding.
{
const int n_state = hparams.n_embd;
const int d_half = n_state / 2;
const float log_10000 = logf(10000.0f);
std::vector<float> freqs(d_half);
for (int k = 0; k < d_half; ++k) {
freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state)));
}
set_input_f32("pos_freqs", freqs);
}
// Generate relative positional distance values which scaled by
// the frequency to produce the angles for sin/cos.
{
// window_size is only known after graph construction since it depends on
// n_time from the conv output, so we read it back from the graph tensor.
struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions");
const int window_size = rel_pos->ne[1];
std::vector<float> pos(window_size);
// local attention: window is fixed at [att_left, att_right]
// full attention: window covers the full sequence, centered
if (ggml_graph_get_tensor(gf, "local_mask")) {
const int att_left = window_size / 2;
for (int t = 0; t < window_size; ++t) {
pos[t] = float(att_left - t);
}
} else {
const int n_time = (window_size + 1) / 2;
for (int t = 0; t < window_size; ++t) {
pos[t] = float(n_time - 1 - t);
}
}
set_input_f32(rel_pos->name, pos);
}
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
const int context_size = ctx->model.hparams.audio_chunk_size;
@ -4760,6 +5119,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim;
case PROJECTOR_TYPE_GLM4V:
return ctx->model.mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_MIMO_AUDIO:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_PARAKEET:
return ctx->model.mm_1_w->ne[1];
default:
GGML_ABORT("Unknown projector type");
}