vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-10-03 03:25:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

mtmd: add various sanity checks (#29276)

Browse source
This commit is contained in:
Xuan-Son Nguyen 2026-09-22 17:53:05 +02:00 • committed by GitHub
parent 0f8a414b75
commit 73c941b111
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 33 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

27
tools/mtmd/clip.cpp
View file

@ -1408,9 +1408,14 @@ struct clip_model_loader {
}
// Load the vision/audio feature layer indices if they are explicitly provided
// NOTE: gguf conversions should standardize the values of the vision feature layer to
// be non-negative, since we use -1 to mark values as unset here.
// NOTE: gguf conversions should standardize the values of the vision feature layer to be non-negative, since we use -1 to mark values as unset here.
get_arr_int(string_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false);
for (const auto & v : hparams.feature_layers) {
if (v > (int) hparams.n_layer) {
throw std::runtime_error(string_format("%s: feature layer index %d is out of range (n_layer: %d)",
__func__, v, hparams.n_layer));
}
}
// model-specific params
switch (model.proj_type) {
@ -1456,7 +1461,12 @@ struct clip_model_loader {
std::vector<int> wa_layer_indexes_vec;
get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false);
if (!wa_layer_indexes_vec.empty()) {
hparams.insert_layer_id = wa_layer_indexes_vec[0];
const int insert_lid = wa_layer_indexes_vec[0];
if (insert_lid < 0 || insert_lid >= (int) hparams.n_layer) {
throw std::runtime_error(string_format("%s: layer index %d is out of range (n_layer: %d)",
__func__, insert_lid, hparams.n_layer));
}
hparams.insert_layer_id = insert_lid;
}
} break;
case PROJECTOR_TYPE_INTERNVL:
@ -3226,6 +3236,7 @@ struct clip_model_loader {
model.pos_embed = get_tensor(string_format(TN_SAM_POS_EMBD, "weight"));
model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight"));
model.patch_embed_proj_b = get_tensor(string_format(TN_SAM_PATCH_EMBD, "bias"));
model.n_sam_layers = hparams.sam_n_layer;
model.sam_layers.resize(model.n_sam_layers);
for (int il = 0; il < model.n_sam_layers; ++il) {
auto & layer = model.sam_layers[il];
@ -4652,8 +4663,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
// -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit
// -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316
std::vector<int32_t> positions(pos_h * pos_w);
int bucket_coords_h[1024];
int bucket_coords_w[1024];
// note: sized by the actual patch counts; a tall/wide image produces more
// than 1024 patches per side and a fixed [1024] array would be overrun
std::vector<int> bucket_coords_h(pos_h);
std::vector<int> bucket_coords_w(pos_w);
for (int i = 0; i < pos_h; i++){
bucket_coords_h[i] = std::floor(70.0*i/pos_h);
}
@ -4696,8 +4709,8 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
// SigLIP position buckets (same as resampler path)
std::vector<int32_t> positions(pos_h * pos_w);
int bucket_coords_h[1024];
int bucket_coords_w[1024];
std::vector<int> bucket_coords_h(pos_h);
std::vector<int> bucket_coords_w(pos_w);
for (int i = 0; i < pos_h; i++){
bucket_coords_h[i] = std::floor(70.0*i/pos_h);
}

20
tools/mtmd/mtmd-image.cpp
View file

@ -294,8 +294,8 @@ private:
support = filter_support * filterscale; // Widen filter when downsampling
ksize = static_cast<int>(std::ceil(support)) * 2 + 1; // Total pixels in kernel
std::vector<double> pre_weights(outSize * ksize); // Temporary weights
bounds.resize(outSize * 2);
std::vector<double> pre_weights((size_t) outSize * ksize); // Temporary weights
bounds.resize((size_t) outSize * 2);
// For each output pixel, compute its filter coefficients
@ -322,20 +322,20 @@ private:
for (x = 0; x < xmax; x++) {
// Distance from input pixel center to output pixel center in input space
double w = resample_filter((x + xmin - center + 0.5) * ss);
pre_weights[xx * ksize + x] = w;
pre_weights[(size_t) xx * ksize + x] = w;
ww += w; // Accumulate for normalization
}
// Normalize weights to sum to 1.0 (preserves brightness)
for (x = 0; x < xmax; x++) {
if (ww != 0.0) {
pre_weights[xx * ksize + x] /= ww;
pre_weights[(size_t) xx * ksize + x] /= ww;
}
}
// Zero-pad remaining kernel positions
for (; x < ksize; x++) {
pre_weights[xx * ksize + x] = 0;
pre_weights[(size_t) xx * ksize + x] = 0;
}
// Store input pixel range for this output pixel
@ -345,11 +345,11 @@ private:
// Convert floating-point coefficients to fixed-point integers
// Formula: int32 = round(float * 2^PRECISION_BITS)
weights.resize(outSize * ksize);
weights.resize((size_t) outSize * ksize);
const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS
for (int i = 0; i < outSize * ksize; i++) {
for (size_t i = 0; i < (size_t) outSize * ksize; i++) {
// Pillow adds +/- 0.5 then truncates toward zero; std::round would round twice
const double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5);
weights[i] = static_cast<int32_t>(rounded);
@ -442,6 +442,12 @@ private:
const int src_width = img.get_size().width;
const int src_height = img.get_size().height;
// sanity check on the target size
if (target_width <= 0 || target_width > 65536 || target_height <= 0 || target_height > 65536) {
throw std::runtime_error("resize target " + std::to_string(target_width) + "x" +
std::to_string(target_height) + " is out of range (max 65536)");
}
bool need_horizontal = (target_width != src_width);
bool need_vertical = (target_height != src_height);