diff --git a/conversion/__init__.py b/conversion/__init__.py index efa537b0f..bddfe3c1f 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -200,7 +200,9 @@ TEXT_MODEL_MAP: dict[str, str] = { "MiniMaxM3SparseForCausalLM": "minimax", "MiniMaxM3SparseForConditionalGeneration": "minimax", "Ministral3ForCausalLM": "mistral3", + "Ministral3Model": "mistral3", "Mistral3ForConditionalGeneration": "mistral3", + "Mistral3Model": "mistral3", "VoxtralRealtimeForConditionalGeneration": "mistral3", "MistralForCausalLM": "llama", "MixtralForCausalLM": "llama", @@ -358,6 +360,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "MiniCPMV4_6ForConditionalGeneration": "minicpm", "MiniCPMV4_7ForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", + "Mistral3Model": "llava", "NemotronH_Omni_Reasoning_V3": "nemotron", "NemotronH_Nano_VL_V2": "nemotron", "MuseGlimmerForConditionalGeneration": "muse_glimmer", diff --git a/conversion/llava.py b/conversion/llava.py index 98a004f98..e193f0b95 100644 --- a/conversion/llava.py +++ b/conversion/llava.py @@ -2,7 +2,7 @@ from __future__ import annotations import json -from typing import Iterable, TYPE_CHECKING +from typing import Callable, Iterable, TYPE_CHECKING if TYPE_CHECKING: from torch import Tensor @@ -128,3 +128,24 @@ class LlavaVisionModel(MmprojModel): yield from super().modify_tensors(img_break_embd, name, bid) return # skip other tensors + + +@ModelBase.register("Mistral3Model") +@ModelBase.example("nvidia/Nemotron-3.5-Embed-1B-BF16") +class Mistral3EmbeddingVisionModel(LlavaVisionModel): + def __init__(self, dir_model, *args, **kwargs): + hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) + text_config = hparams.get("text_config", hparams) + if text_config.get("is_causal") is not False: + raise NotImplementedError("Expected a Nemotron Embed model for Mistral3Model, but is_causal is not False") + super().__init__(dir_model, *args, **kwargs) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + if item[0].endswith("embed_tokens.weight"): + return ModelBase.filter_tensors(item) + return super().filter_tensors(item) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_vision_preproc_image_size(self.preprocessor_config["size"]["longest_edge"]) diff --git a/conversion/mistral3.py b/conversion/mistral3.py index 661475251..106c329e7 100644 --- a/conversion/mistral3.py +++ b/conversion/mistral3.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from torch import Tensor -from .base import ModelBase, TextModel, gguf +from .base import ModelBase, TextModel, get_model_architecture, gguf from .deepseek import DeepseekV2Model from .llama import LlamaModel @@ -13,9 +13,11 @@ from .llama import LlamaModel @ModelBase.register( "Mistral3ForConditionalGeneration", + "Mistral3Model", "Ministral3ForCausalLM", + "Ministral3Model", ) -@ModelBase.example("mistralai/Mistral-Small-3.1-24B-Instruct-2503", "hf-tiny-v2/tiny-random-Ministral3ForCausalLM") +@ModelBase.example("mistralai/Mistral-Small-3.1-24B-Instruct-2503", "hf-tiny-v2/tiny-random-Ministral3ForCausalLM", "nvidia/Nemotron-3-Embed-1B-BF16", "nvidia/Nemotron-3.5-Embed-1B-BF16") class Mistral3Model(TextModel): class Ministral3Model(LlamaModel): model_arch = gguf.MODEL_ARCH.MISTRAL3 @@ -29,6 +31,18 @@ class Mistral3Model(TextModel): self.gguf_writer.add_rope_scaling_yarn_log_mul(rope_params["mscale_all_dim"]) self.gguf_writer.add_attn_temperature_scale(rope_params["llama_4_scaling_beta"]) + class Mistral3EmbedNemotronModel(Ministral3Model): + model_arch = gguf.MODEL_ARCH.MISTRAL3 + + def set_vocab(self): + super().set_vocab() + self.gguf_writer.add_add_bos_token(False) + self.gguf_writer.add_add_eos_token(False) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self._try_set_pooling_type() + class Mistral4Model(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.MISTRAL4 skip_mtp = False # model contains no MTP layers, so no need to skip @@ -42,12 +56,20 @@ class Mistral3Model(TextModel): model_arch = gguf.MODEL_ARCH.MISTRAL3 # unused impl: TextModel - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, dir_model, *args, **kwargs): + hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) + hf_arch = get_model_architecture(hparams, self.model_type) + if hf_arch in ("Ministral3Model", "Mistral3Model"): + text_config = hparams.get("text_config", hparams) + if text_config.get("is_causal") is not False: + raise NotImplementedError(f"Expected a Nemotron Embed model for {hf_arch}, but is_causal is not False") + super().__init__(dir_model, *args, **kwargs) if self.hparams.get("model_type") == "mistral4": - self.impl = Mistral3Model.Mistral4Model(*args, **kwargs) + self.impl = Mistral3Model.Mistral4Model(dir_model, *args, **kwargs) + elif self.hf_arch in ("Ministral3Model", "Mistral3Model"): + self.impl = Mistral3Model.Mistral3EmbedNemotronModel(dir_model, *args, **kwargs) else: - self.impl = Mistral3Model.Ministral3Model(*args, **kwargs) + self.impl = Mistral3Model.Ministral3Model(dir_model, *args, **kwargs) def set_vocab(self): self.impl.set_vocab() diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 64e006e8d..5caa93b42 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -66,6 +66,7 @@ struct clip_hparams { resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC; pad_style image_resize_pad = PAD_CEIL; // padding style when resizing bool image_round_half_even = false; // smart_resize rounds .5 to even, like Python round() in the HF processor + bool image_floor_before_align = false; std::array image_pad_color = {0, 0, 0}; // (preprocessor) for llava-uhd style models diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 26941c9a8..df61f82f0 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1625,7 +1625,23 @@ struct clip_model_loader { hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; hparams.rope_theta = 10000.0f; get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); - hparams.set_limit_image_tokens(8, 1024); + get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); + if (hparams.image_longest_edge > 0) { + const int64_t patch_size = int64_t(hparams.patch_size) * hparams.n_merge; + if (patch_size <= 0) { + throw std::runtime_error("Pixtral merged patch size must be positive"); + } + const int64_t n_tokens_per_side = + (hparams.image_longest_edge + patch_size - 1) / patch_size; + if (n_tokens_per_side * patch_size > std::sqrt(std::numeric_limits::max())) { + throw std::runtime_error("Pixtral preprocessing image size is too large"); + } + hparams.image_resize_pad = PAD_NONE; + hparams.image_floor_before_align = true; + hparams.set_limit_image_tokens(0, n_tokens_per_side * n_tokens_per_side); + } else { + hparams.set_limit_image_tokens(8, 1024); + } hparams.set_warmup_n_tokens(256); // avoid OOM on warmup } break; case PROJECTOR_TYPE_LIGHTONOCR: diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index e0cd0bf1d..ab6067b97 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -873,16 +873,28 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) { GGML_ASSERT(hparams.image_longest_edge > 0); + clip_image_size size = original_size; + if (hparams.image_floor_before_align) { + const double ratio = double(std::max(size.width, size.height)) / hparams.image_longest_edge; + if (ratio > 1.0) { + size.width = static_cast(std::floor(size.width / ratio)); + size.height = static_cast(std::floor(size.height / ratio)); + if (size.width <= 0 || size.height <= 0) { + throw std::runtime_error("image dimensions must be positive after resizing"); + } + } + } + // the original pixtral model doesn't have n_merge const int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge; return img_tool::calc_size_preserved_ratio( - original_size, + size, { /* align_size */ hparams.patch_size * cur_merge, /* min_pixels */ std::max(0, hparams.image_min_pixels), /* max_pixels */ std::max(0, hparams.image_max_pixels), /* longest_edge */ std::min(hparams.image_longest_edge, - std::max(original_size.width, original_size.height)), + std::max(size.width, size.height)), }); } diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 26d5ee410..b3ac1979e 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -804,7 +804,11 @@ struct mtmd_context { { // https://github.com/huggingface/transformers/blob/1cd110c6cb6a6237614130c470e9a902dbc1a4bd/docs/source/en/model_doc/pixtral.md img_end = "[IMG_END]"; - image_preproc = std::make_unique(ctx_v); + if (clip_get_hparams(ctx_v)->image_longest_edge > 0) { + image_preproc = std::make_unique(ctx_v); + } else { + image_preproc = std::make_unique(ctx_v); + } } break; case PROJECTOR_TYPE_PHI4: {