diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 6ad60213e..536dfaa80 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -5,6 +5,7 @@ #include "mtmd-internal.h" #include +#include #include #include #include @@ -95,6 +96,45 @@ static std::string image_size_to_str(const clip_image_size & size) { return std::to_string(size.width) + "x" + std::to_string(size.height); } +static std::string resize_algo_to_str(resize_algo algo) { + std::string str; + switch (algo & ~RESIZE_ALGO_FLAG_TORCH) { + case RESIZE_ALGO_BILINEAR: str = "bilinear"; break; + case RESIZE_ALGO_BICUBIC: str = "bicubic"; break; + case RESIZE_ALGO_LANCZOS: str = "lanczos"; break; + default: str = "unknown"; break; + } + if (algo & RESIZE_ALGO_FLAG_TORCH) { + str += "_torch"; + } + return str; +} + +// pseudo-random pixels: s = s * 1103515245 + 12345 from s = 1, one byte (s >> 16) & 255 per channel +static clip_image_u8 make_test_image(const clip_image_size & size) { + clip_image_u8 img; + img.set_size(size, false); + uint32_t s = 1; + auto next = [&s]() { + s = s * 1103515245u + 12345u; + return static_cast(s >> 16); + }; + for (int y = 0; y < size.height; y++) { + for (int x = 0; x < size.width; x++) { + img.set_pixel(x, y, { next(), next(), next() }); + } + } + return img; +} + +// FNV-1a of the pixels +static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 2166136261u) { + for (uint8_t v : img.get_ro_buf()) { + hash = (hash ^ v) * 16777619u; + } + return hash; +} + MAKE_TEST(test_image_preprocessor_lfm2) { clip_hparams hparams; hparams.patch_size = 16; @@ -158,6 +198,42 @@ MAKE_TEST(test_image_preprocessor_lfm2) { std::string(mtmd_image_preprocessor_lfm2::should_tile(hparams, { 1024, 977 }) ? "tiled" : "single")); } +MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) { + clip_hparams hparams; + hparams.patch_size = 16; + hparams.n_merge = 2; + hparams.set_limit_image_tokens(64, 256); + hparams.preproc_min_tiles = 2; + hparams.preproc_max_tiles = 10; + hparams.preproc_tile_size = 512; + hparams.preproc_max_pixels_tolerance = 2.0f; + // Lfm2VlImageProcessor stretches the image to the tile grid + hparams.image_pad_rf = PAD_NONE; + + // { resize algo, image size, expected hash of the tiles and the thumbnail from Lfm2VlImageProcessor (torchvision 0.29, native uint8 resize: arm64, or x86 with AVX2/AVX512) } + const std::vector> cases = { + { RESIZE_ALGO_BILINEAR, { 620, 410 }, 0xf5e9c941 }, // 608x416 + { RESIZE_ALGO_BILINEAR, { 2000, 1500 }, 0x3331f56c }, // 3x2 tiles, 576x416 thumbnail + { RESIZE_ALGO_BICUBIC, { 620, 410 }, 0x2b34eac9 }, + { RESIZE_ALGO_BICUBIC, { 2000, 1500 }, 0xb035d942 }, + }; + + for (const auto & [base_algo, size, expected] : cases) { + const resize_algo algo = static_cast(base_algo | RESIZE_ALGO_FLAG_TORCH); + hparams.image_resize_algo_rf = algo; + hparams.image_resize_algo_ov = algo; + const mtmd_image_preprocessor_lfm2 preproc(hparams); + const auto sliced = preproc.slice_image(make_test_image(size), preproc.get_slice_instructions(size)); + + uint32_t hash = 2166136261u; + for (const auto & slice : sliced.slices) { + hash = hash_pixels(slice, hash); + } + hash = hash_pixels(sliced.overview, hash); + t.assert_equal("pixels of " + std::to_string(size.width) + "x" + std::to_string(size.height) + " with resize algo " + resize_algo_to_str(algo), expected, hash); + } +} + // // mtmd temporal merge // diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 350c8fc9d..1f319843b 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -29,11 +29,14 @@ enum patch_merge_type { PATCH_MERGE_SPATIAL_UNPAD, }; -// all algos are Pillow-compatible (matching PIL.Image.resize output) +// the low bits select the algo, Pillow-compatible (matching PIL.Image.resize output) +// RESIZE_ALGO_FLAG_TORCH matches torchvision's antialiased resize of uint8 images on the CPU (int16 weights), bilinear and bicubic only enum resize_algo { RESIZE_ALGO_BILINEAR, RESIZE_ALGO_BICUBIC, RESIZE_ALGO_LANCZOS, + + RESIZE_ALGO_FLAG_TORCH = 1 << 8, }; // Padding style for img_tool::resize @@ -207,6 +210,11 @@ struct clip_hparams { int32_t custom_image_min_tokens = -1; int32_t custom_image_max_tokens = -1; + void set_image_resize_algo(resize_algo algo, bool torch = false) { + GGML_ASSERT(!(torch && algo == RESIZE_ALGO_LANCZOS) && "torchvision has no lanczos"); + image_resize_algo = static_cast(torch ? algo | RESIZE_ALGO_FLAG_TORCH : algo); + } + void set_limit_image_tokens(int n_tokens_min, int n_tokens_max) { const int patch_area = patch_size * patch_size * n_merge * n_merge; image_min_pixels = (custom_image_min_tokens > 0 ? custom_image_min_tokens : n_tokens_min) * patch_area; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 950d8fde0..2ab55c52a 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1589,12 +1589,13 @@ struct clip_model_loader { // default for older GGUFs std::string resize_algo = "bilinear"; get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo, false); + // Lfm2VlImageProcessor and d1-omni vision.py resize bilinear and bicubic with torchvision if (resize_algo == "bilinear") { - hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; + hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true); } else if (resize_algo == "bicubic") { - hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true); } else if (resize_algo == "lanczos") { - hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; + hparams.set_image_resize_algo(RESIZE_ALGO_LANCZOS); } else { throw std::runtime_error(string_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str())); } diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index ab6067b97..cf6e9a8ac 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -205,6 +205,8 @@ private: // 1. Separable filtering: horizontal pass followed by vertical pass // 2. Pre-computes normalized filter coefficients for each output pixel // 3. Fixed-point integer arithmetic (22 fractional bits) for speed and determinism + // + // RESIZE_ALGO_FLAG_TORCH is the same, except for the int16 weights of torchvision (aten/src/ATen/native/cpu/UpSampleKernel.cpp) static bool resize_pillow( const clip_image_u8 & img, clip_image_u8 & dst, @@ -215,6 +217,13 @@ private: // This allows encoding fractional weights as integers: weight * 2^22 const int PRECISION_BITS = 32 - 8 - 2; + const bool int16_weights = algo & RESIZE_ALGO_FLAG_TORCH; + algo = static_cast(algo & ~RESIZE_ALGO_FLAG_TORCH); + + if (int16_weights && algo == RESIZE_ALGO_LANCZOS) { + GGML_ABORT("unsupported resize algo: torchvision has no lanczos"); + } + // Filter support radius double filter_support; switch (algo) { @@ -222,11 +231,11 @@ private: case RESIZE_ALGO_BICUBIC: filter_support = 2.0; break; case RESIZE_ALGO_LANCZOS: filter_support = 3.0; break; default: - throw std::runtime_error("Unsupported resize algorithm"); + GGML_ABORT("unsupported resize algo: %d", (int) algo); } // Returns filter weight for distance x from pixel center - // Note: for bicubic, Pillow uses a = -0.5 while GGML/PyTorch use a = -0.75 + // Note: for bicubic, Pillow (and PyTorch with antialias) uses a = -0.5 while GGML (and PyTorch without antialias) uses a = -0.75 auto resample_filter = [algo](double x) -> double { if (algo == RESIZE_ALGO_LANCZOS) { if (-3.0 <= x && x < 3.0) { @@ -351,12 +360,24 @@ private: // Formula: int32 = round(float * 2^PRECISION_BITS) weights.resize((size_t) outSize * ksize); - const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS + // torchvision rounds the weights to int16, with as many fractional bits as the largest weight allows; + // scaled back to PRECISION_BITS, they give the same output in the passes below + int precision = PRECISION_BITS; + if (int16_weights) { + const double weight_max = *std::max_element(pre_weights.begin(), pre_weights.end()); + for (precision = 0; precision < PRECISION_BITS; precision++) { + if (static_cast(0.5 + weight_max * (1 << (precision + 1))) >= (1 << 15)) { + break; + } + } + } + + const double fxp_scale = std::ldexp(1.0, precision); // 1.0 * 2^precision for (size_t i = 0; i < (size_t) outSize * ksize; i++) { // Pillow adds +/- 0.5 then truncates toward zero; std::round would round twice const double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5); - weights[i] = static_cast(rounded); + weights[i] = static_cast(rounded) * (1 << (PRECISION_BITS - precision)); } return ksize; diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index bc90a76c6..cb66ac2ea 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -31,6 +31,7 @@ struct mtmd_image_preprocessor { const clip_hparams & hparams; mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {} + mtmd_image_preprocessor(const clip_hparams & hparams): hparams(hparams) {} virtual ~mtmd_image_preprocessor() = default; virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0; @@ -59,6 +60,7 @@ struct mtmd_image_preprocessor { */ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} + mtmd_image_preprocessor_llava_uhd(const clip_hparams & hparams) : mtmd_image_preprocessor(hparams) {} mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; struct slice_coordinates {