diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 1783c7ddf..feceb7ff7 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1408,9 +1408,14 @@ struct clip_model_loader { } // Load the vision/audio feature layer indices if they are explicitly provided - // NOTE: gguf conversions should standardize the values of the vision feature layer to - // be non-negative, since we use -1 to mark values as unset here. + // NOTE: gguf conversions should standardize the values of the vision feature layer to be non-negative, since we use -1 to mark values as unset here. get_arr_int(string_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false); + for (const auto & v : hparams.feature_layers) { + if (v > (int) hparams.n_layer) { + throw std::runtime_error(string_format("%s: feature layer index %d is out of range (n_layer: %d)", + __func__, v, hparams.n_layer)); + } + } // model-specific params switch (model.proj_type) { @@ -1456,7 +1461,12 @@ struct clip_model_loader { std::vector wa_layer_indexes_vec; get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false); if (!wa_layer_indexes_vec.empty()) { - hparams.insert_layer_id = wa_layer_indexes_vec[0]; + const int insert_lid = wa_layer_indexes_vec[0]; + if (insert_lid < 0 || insert_lid >= (int) hparams.n_layer) { + throw std::runtime_error(string_format("%s: layer index %d is out of range (n_layer: %d)", + __func__, insert_lid, hparams.n_layer)); + } + hparams.insert_layer_id = insert_lid; } } break; case PROJECTOR_TYPE_INTERNVL: @@ -3226,6 +3236,7 @@ struct clip_model_loader { model.pos_embed = get_tensor(string_format(TN_SAM_POS_EMBD, "weight")); model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight")); model.patch_embed_proj_b = get_tensor(string_format(TN_SAM_PATCH_EMBD, "bias")); + model.n_sam_layers = hparams.sam_n_layer; model.sam_layers.resize(model.n_sam_layers); for (int il = 0; il < model.n_sam_layers; ++il) { auto & layer = model.sam_layers[il]; @@ -4652,8 +4663,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit // -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316 std::vector positions(pos_h * pos_w); - int bucket_coords_h[1024]; - int bucket_coords_w[1024]; + // note: sized by the actual patch counts; a tall/wide image produces more + // than 1024 patches per side and a fixed [1024] array would be overrun + std::vector bucket_coords_h(pos_h); + std::vector bucket_coords_w(pos_w); for (int i = 0; i < pos_h; i++){ bucket_coords_h[i] = std::floor(70.0*i/pos_h); } @@ -4696,8 +4709,8 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { // SigLIP position buckets (same as resampler path) std::vector positions(pos_h * pos_w); - int bucket_coords_h[1024]; - int bucket_coords_w[1024]; + std::vector bucket_coords_h(pos_h); + std::vector bucket_coords_w(pos_w); for (int i = 0; i < pos_h; i++){ bucket_coords_h[i] = std::floor(70.0*i/pos_h); } diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index c11d35c87..9b2e98862 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -294,8 +294,8 @@ private: support = filter_support * filterscale; // Widen filter when downsampling ksize = static_cast(std::ceil(support)) * 2 + 1; // Total pixels in kernel - std::vector pre_weights(outSize * ksize); // Temporary weights - bounds.resize(outSize * 2); + std::vector pre_weights((size_t) outSize * ksize); // Temporary weights + bounds.resize((size_t) outSize * 2); // For each output pixel, compute its filter coefficients @@ -322,20 +322,20 @@ private: for (x = 0; x < xmax; x++) { // Distance from input pixel center to output pixel center in input space double w = resample_filter((x + xmin - center + 0.5) * ss); - pre_weights[xx * ksize + x] = w; + pre_weights[(size_t) xx * ksize + x] = w; ww += w; // Accumulate for normalization } // Normalize weights to sum to 1.0 (preserves brightness) for (x = 0; x < xmax; x++) { if (ww != 0.0) { - pre_weights[xx * ksize + x] /= ww; + pre_weights[(size_t) xx * ksize + x] /= ww; } } // Zero-pad remaining kernel positions for (; x < ksize; x++) { - pre_weights[xx * ksize + x] = 0; + pre_weights[(size_t) xx * ksize + x] = 0; } // Store input pixel range for this output pixel @@ -345,11 +345,11 @@ private: // Convert floating-point coefficients to fixed-point integers // Formula: int32 = round(float * 2^PRECISION_BITS) - weights.resize(outSize * ksize); + weights.resize((size_t) outSize * ksize); const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS - for (int i = 0; i < outSize * ksize; i++) { + for (size_t i = 0; i < (size_t) outSize * ksize; i++) { // Pillow adds +/- 0.5 then truncates toward zero; std::round would round twice const double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5); weights[i] = static_cast(rounded); @@ -442,6 +442,12 @@ private: const int src_width = img.get_size().width; const int src_height = img.get_size().height; + // sanity check on the target size + if (target_width <= 0 || target_width > 65536 || target_height <= 0 || target_height > 65536) { + throw std::runtime_error("resize target " + std::to_string(target_width) + "x" + + std::to_string(target_height) + " is out of range (max 65536)"); + } + bool need_horizontal = (target_width != src_width); bool need_vertical = (target_height != src_height);