diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index dfa3e5e2c..2d0f24874 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -66,6 +66,31 @@ MAKE_TEST(test_image_preprocessor_longest_edge) { } } +MAKE_TEST(test_image_preprocessor_dyn_size) { + // { patch_size, image size, expected size } + // a side that is an even number plus .5 merged patches rounds down, like Python round() in smart_resize + const std::vector> cases = { + { 16, { 768, 400 }, { 768, 384 } }, + { 16, { 800, 336 }, { 800, 320 } }, + { 16, { 400, 400 }, { 384, 384 } }, + { 16, { 640, 240 }, { 640, 256 } }, + { 14, { 966, 560 }, { 952, 560 } }, + }; + + for (const auto & [patch_size, input, expected] : cases) { + clip_hparams hparams; + hparams.patch_size = patch_size; + hparams.n_merge = 2; + hparams.image_round_half_even = true; + hparams.set_limit_image_tokens(8, 4096); + + const clip_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input); + const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height); + t.assert_equal(name + " width", expected.width, actual.width); + t.assert_equal(name + " height", expected.height, actual.height); + } +} + MAKE_TEST(test_image_preprocessor_lfm2) { clip_hparams hparams; hparams.patch_size = 16; diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 92ffe5e54..fefbd4f46 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -65,6 +65,7 @@ struct clip_hparams { int32_t image_max_pixels = -1; resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC; pad_style image_resize_pad = PAD_CEIL; // padding style when resizing + bool image_round_half_even = false; // smart_resize rounds .5 to even, like Python round() in the HF processor std::array image_pad_color = {0, 0, 0}; // (preprocessor) for llava-uhd style models diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 793b8cd1a..faeb37702 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1634,6 +1634,8 @@ struct clip_model_loader { case PROJECTOR_TYPE_DOTS_OCR: { hparams.rope_theta = 10000.0f; + hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding + hparams.image_round_half_even = true; // smart_resize uses Python round() get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); @@ -1750,6 +1752,8 @@ struct clip_model_loader { { hparams.n_merge = 2; // default value for Qwen 2 and 2.5 hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding + hparams.image_round_half_even = true; // smart_resize uses Python round() get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it // ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json @@ -1825,6 +1829,8 @@ struct clip_model_loader { { hparams.n_merge = 2; // spatial_merge_size hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding + hparams.image_round_half_even = true; // smart_resize uses Python round() get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv); // 1D banded sliding-window radius (visual_token_window_size); required @@ -2092,6 +2098,8 @@ struct clip_model_loader { case PROJECTOR_TYPE_EXAONE4_5: { hparams.n_merge = 2; + hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding + hparams.image_round_half_even = true; // smart_resize uses Python round() get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false); get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 5debfe6f0..e56f33e77 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -115,6 +115,7 @@ struct img_tool { int max_pixels = 0; // 0 = disabled // applied before min/max_pixels, so min_pixels can push an edge back above longest_edge int longest_edge = 0; // 0 = disabled + bool round_half_even = false; // Python round() instead of std::round }; // calculate the size of the **resized** image, while preserving the aspect ratio and @@ -127,7 +128,10 @@ struct img_tool { return {0, 0}; } - auto round_by_factor = [f = opts.align_size](float x) { return static_cast(std::round(x / static_cast(f))) * f; }; + auto round_by_factor = [f = opts.align_size, even = opts.round_half_even](float x) { + const float q = x / static_cast(f); + return static_cast(even ? std::nearbyint(q) : std::round(q)) * f; + }; auto ceil_by_factor = [f = opts.align_size](float x) { return static_cast(std::ceil(x / static_cast(f))) * f; }; auto floor_by_factor = [f = opts.align_size](float x) { return static_cast(std::floor(x / static_cast(f))) * f; }; @@ -767,20 +771,24 @@ mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip // mtmd_image_preprocessor_dyn_size // -mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const { +clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) { GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0); - clip_image_u8 resized_image; - const clip_image_size original_size = img.get_size(); // the original pixtral model doesn't have n_merge const int cur_merge = hparams.n_merge; - const clip_image_size target_size = img_tool::calc_size_preserved_ratio( + return img_tool::calc_size_preserved_ratio( original_size, { - /* align_size */ hparams.patch_size * cur_merge, - /* min_pixels */ hparams.image_min_pixels, - /* max_pixels */ hparams.image_max_pixels, - /* longest_edge */ 0, + /* align_size */ hparams.patch_size * cur_merge, + /* min_pixels */ hparams.image_min_pixels, + /* max_pixels */ hparams.image_max_pixels, + /* longest_edge */ 0, + /* round_half_even */ hparams.image_round_half_even, }); +} + +mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const { + clip_image_u8 resized_image; + const clip_image_size target_size = get_target_size(hparams, img.get_size()); img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, hparams.image_resize_pad, diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 1798ce51d..26193e4e4 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -131,6 +131,7 @@ struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor { // this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor { mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} + static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size); mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; };