diff --git a/conversion/__init__.py b/conversion/__init__.py index 8921f53e4..5f252b490 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -196,6 +196,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "MiniCPMForCausalLM": "minicpm", "MiniCPMV4_6ForConditionalGeneration": "minicpm", "MiniCPMV4_7ForConditionalGeneration": "minicpm", + "MiniCPMOcrForConditionalGeneration": "minicpm", "MiniMaxText01ForCausalLM": "minimax", "MiniMaxM1ForCausalLM": "minimax", "MiniMaxM2ForCausalLM": "minimax", @@ -361,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "MiniMaxM3SparseForConditionalGeneration": "minimax", "MiniCPMV4_6ForConditionalGeneration": "minicpm", "MiniCPMV4_7ForConditionalGeneration": "minicpm", + "MiniCPMOcrForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", "Mistral3Model": "llava", "NemotronH_Omni_Reasoning_V3": "nemotron", diff --git a/conversion/minicpm.py b/conversion/minicpm.py index 937ac033a..d8fceb0c0 100644 --- a/conversion/minicpm.py +++ b/conversion/minicpm.py @@ -272,3 +272,18 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel): yield self.format_tensor_name(gguf.MODEL_TENSOR.V_TOK_EMBD_SEP, suffix=""), data_torch[ids] return yield from super().modify_tensors(data_torch, name, bid) + + +# MiniCPM-OCR 1.0: the 4.6 stack with model_type minicpm_ocr. + +@ModelBase.register("MiniCPMOcrForConditionalGeneration") +@ModelBase.example("openbmb/MiniCPM-OCR-1.0") +class MiniCPMOCR1_0TextModel(MiniCPMV4_6TextModel): + model_arch = gguf.MODEL_ARCH.QWEN35 + no_mtp = True + + +@ModelBase.register("MiniCPMOcrForConditionalGeneration") +@ModelBase.example("openbmb/MiniCPM-OCR-1.0") +class MiniCPMOCR1_0VisionModel(MiniCPMV4_6VisionModel): + projector_type = gguf.VisionProjectorType.MINICPM_OCR_1_0 diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 0b95580bd..b8cfa3a85 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -6452,6 +6452,7 @@ class VisionProjectorType: MINIMAXM3 = "minimax_m3" MINICPMV4_6 = "minicpmv4_6" MINICPMV4_7 = "minicpmv4_7" + MINICPM_OCR_1_0 = "minicpm_ocr_1_0" GRANITE_SPEECH = "granite_speech" # audio MIMOVL = "mimovl" MIMO_AUDIO = "mimo_audio" diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 6ca465280..5583a7c57 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -537,6 +537,7 @@ enum projector_type { PROJECTOR_TYPE_EXAONE4_5, PROJECTOR_TYPE_MINICPMV4_6, PROJECTOR_TYPE_MINICPMV4_7, + PROJECTOR_TYPE_MINICPM_OCR_1_0, PROJECTOR_TYPE_GRANITE_SPEECH, PROJECTOR_TYPE_MIMOVL, PROJECTOR_TYPE_MINIMAX_M3, @@ -611,6 +612,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"}, { PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"}, { PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"}, + { PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"}, { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, { PROJECTOR_TYPE_MIMOVL, "mimovl"}, { PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"}, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 869e3fff8..5737400df 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1031,6 +1031,7 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: { builder = std::make_unique(ctx, img); } break; @@ -1496,6 +1497,7 @@ struct clip_model_loader { } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: { // MiniCPM-V 4.6/4.7 unified merger projector // ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension @@ -2527,6 +2529,7 @@ struct clip_model_loader { || model.proj_type == PROJECTOR_TYPE_MINICPMV || model.proj_type == PROJECTOR_TYPE_MINICPMV4_6 || model.proj_type == PROJECTOR_TYPE_MINICPMV4_7 + || model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0 ) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd; if (is_ffn_swapped) { // swap up and down weights @@ -2630,6 +2633,7 @@ struct clip_model_loader { } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: { const bool merger_required = hparams.n_merge == 4; auto get_merger_tensor = [&](const std::string & name, bool required = true) { @@ -4348,6 +4352,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_STEP3VL: case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: return img->nx() / (params.patch_size * params.n_merge); case PROJECTOR_TYPE_DEEPSEEKOCR: case PROJECTOR_TYPE_DEEPSEEKOCR2: @@ -4379,6 +4384,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_STEP3VL: case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: return img->ny() / (params.patch_size * params.n_merge); default: break; @@ -4445,6 +4451,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: { n_patches /= params.n_merge * params.n_merge; } break; @@ -5169,6 +5176,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: { const bool is_4x = hparams.n_merge == 2; @@ -6437,6 +6445,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_model_proj->ne[0]; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: return ctx->model.mm_ffn_down_w->ne[1]; case PROJECTOR_TYPE_GLM_EDGE: return ctx->model.mm_model_mlp_3_w->ne[1]; diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 3aea23ced..bc90a76c6 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -190,6 +190,17 @@ protected: } }; +// MiniCPM-OCR 1.0: find_best_resize aligns to patch_size * n_merge (28 in 4x). +struct mtmd_image_preprocessor_minicpm_ocr_1_0 : mtmd_image_preprocessor_minicpmv { + using mtmd_image_preprocessor_minicpmv::mtmd_image_preprocessor_minicpmv; + +protected: + int get_slice_align() const override { + const int merge = hparams.n_merge > 0 ? hparams.n_merge : 2; + return hparams.patch_size * merge; + } +}; + // custom llava-uhd slicing logic for LFM2 // tiling params (min/max tiles, tile size, pixels tolerance) come from hparams // ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 9460f15fe..9a1663015 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -722,6 +722,7 @@ struct mtmd_context { } break; case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: + case PROJECTOR_TYPE_MINICPM_OCR_1_0: { slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_6; tok_ov_img_start = {lookup_token("")}; @@ -732,7 +733,11 @@ struct mtmd_context { tok_row_end_trail = false; // no trailing end-of-row token ov_img_first = true; use_image_id = true; - image_preproc = std::make_unique(ctx_v); + if (clip_get_projector_type(ctx_v) == PROJECTOR_TYPE_MINICPM_OCR_1_0) { + image_preproc = std::make_unique(ctx_v); + } else { + image_preproc = std::make_unique(ctx_v); + } } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: