diff --git a/conversion/__init__.py b/conversion/__init__.py index bddfe3c1f..369ce7afd 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -17,6 +17,7 @@ __all__ = [ TEXT_MODEL_MAP: dict[str, str] = { + "NEOChatModel": "sensenova_u1", "AfmoeForCausalLM": "afmoe", "LagunaForCausalLM": "laguna", "ApertusForCausalLM": "llama", @@ -367,6 +368,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "MuseSparkForConditionalGeneration": "muse_spark", "PaddleOCRVisionModel": "ernie", "Phi4ForCausalLMV": "phi", + "NEOVisionModel": "sensenova_u1", "Qwen2AudioForConditionalGeneration": "ultravox", "Qwen2VLForConditionalGeneration": "qwenvl", "Qwen2VLModel": "qwenvl", diff --git a/conversion/base.py b/conversion/base.py index dec1d4322..2cbaed700 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -3143,7 +3143,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st # Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older # Kimi-Linear-48B architecture and cannot load K3 (no attention residuals, # latent MoE, situ, ...). Route on the top-level architecture instead. - if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"): + if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"): return arch # if "architectures" is found in the sub-config, use that instead diff --git a/conversion/sensenova_u1.py b/conversion/sensenova_u1.py new file mode 100644 index 000000000..bde5f9d9b --- /dev/null +++ b/conversion/sensenova_u1.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from pathlib import Path + +import torch + +from .base import MmprojModel, ModelBase, gguf +from .qwen import Qwen2Model + + +@ModelBase.example("sensenova/SenseNova-U1.5-8B-MoT") +@ModelBase.register("NEOChatModel") +class SenseNovaU1Model(Qwen2Model): + model_arch = gguf.MODEL_ARCH.SENSENOVA_U1 + + @classmethod + def filter_tensors(cls, item): + name, _ = item + if not name.startswith("language_model.") or "_mot_gen" in name: + return None + return super().filter_tensors(item) + + def set_gguf_parameters(self): + if int(self.hparams.get("num_experts") or 0) > 1: + raise ValueError("SenseNova U1 conversion currently supports the dense understanding branch") + if self.rope_parameters.get("rope_type", "default") not in ("default", None): + raise ValueError("SenseNova U1 scaled RoPE is not supported") + if self.hparams.get("attention_bias", False): + raise ValueError("SenseNova U1 attention bias is not supported") + if self.hparams.get("use_sliding_window", False): + raise ValueError("SenseNova U1 sliding attention is not supported") + super().set_gguf_parameters() + self.gguf_writer.add_rope_freq_base_spatial(self.hparams["rope_theta_hw"]) + + def get_tensors(self): + for name, gen in self.model_tensors.items(): + if name.endswith((".q_norm_hw.weight", ".k_norm_hw.weight")): + continue + data = gen() + if name.endswith((".q_norm.weight", ".k_norm.weight")): + spatial = self.model_tensors[name.replace("_norm.weight", "_norm_hw.weight")]() + # Keep one weight vector; the graph normalizes its two halves independently. + data = torch.cat((data, spatial), dim=0) + yield name, data + + +@ModelBase.register("NEOVisionModel") +@ModelBase.example("sensenova/SenseNova-U1.5-8B-MoT") +class SenseNovaU1VisionModel(MmprojModel): + def __init__(self, dir_model: Path, *args, **kwargs): + hparams = kwargs.pop("hparams", None) + if hparams is None: + hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) + else: + hparams = dict(hparams) + + vision_config = dict(hparams["vision_config"]) + # NEOVisionModel has no transformer blocks + # these defaults satisfy the generic mmproj metadata contract without replacing checkpoint values + vision_config.setdefault("image_size", 0) + vision_config.setdefault("intermediate_size", 0) + vision_config.setdefault("num_hidden_layers", 0) + vision_config.setdefault("num_attention_heads", 1) + hparams["vision_config"] = vision_config + super().__init__(dir_model, *args, hparams=hparams, **kwargs) + + self.preprocessor_config.setdefault("image_mean", [0.485, 0.456, 0.406]) + self.preprocessor_config.setdefault("image_std", [0.229, 0.224, 0.225]) + + @classmethod + def filter_tensors(cls, item): + name, gen = item + if not name.startswith("vision_model."): + return None + return super().filter_tensors((name, gen)) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_clip_projector_type("sensenova_u1") + vision_config = self.hparams["vision_config"] + # These defaults match the known U1.5 checkpoint when older configs omit fields. + hidden_size = int(vision_config.get("hidden_size", 1024)) + min_pixels = int(vision_config.get("min_pixels", 65536)) + max_pixels = int(vision_config.get("max_pixels", 16777216)) + downsample_ratio = float(vision_config.get("downsample_ratio", 0.5)) + if downsample_ratio <= 0: + raise ValueError(f"SenseNova U1 downsample_ratio must be positive, got {downsample_ratio}") + merge_size = round(1.0 / downsample_ratio) + if abs((1.0 / downsample_ratio) - merge_size) > 1e-6: + raise ValueError(f"SenseNova U1 downsample_ratio must have an integer reciprocal, got {downsample_ratio}") + norm_eps = float(vision_config.get("layer_norm_eps", vision_config.get("rms_norm_eps", 1e-6))) + self.gguf_writer.add_vision_head_dim(hidden_size) + self.gguf_writer.add_vision_attention_layernorm_eps(norm_eps) + self.gguf_writer.add_vision_min_pixels(min_pixels) + self.gguf_writer.add_vision_max_pixels(max_pixels) + self.gguf_writer.add_vision_spatial_merge_size(merge_size) + + def modify_tensors(self, data_torch, name, bid): + tensor_names = { + "vision_model.embeddings.patch_embedding.weight": "v.patch_embd.weight", + "vision_model.embeddings.patch_embedding.bias": "v.patch_embd.bias", + "vision_model.embeddings.dense_embedding.weight": self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0), + "vision_model.embeddings.dense_embedding.bias": self.format_tensor_name( + gguf.MODEL_TENSOR.V_MMPROJ, 0, suffix=".bias" + ), + } + if name not in tensor_names: + raise ValueError(f"Unexpected SenseNova U1 vision tensor: {name!r}") + return [(tensor_names[name], data_torch)] + + def tensor_force_quant(self, name, new_name, bid, n_dims): + if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0): + return gguf.GGMLQuantizationType.F16 if self.ftype == gguf.LlamaFileType.MOSTLY_F16 else gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 2840348cb..619a15fc9 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -263,6 +263,7 @@ class Keys: DIMENSION_SECTIONS = "{arch}.rope.dimension_sections" SECTION_ORDER = "{arch}.rope.section_order" FREQ_BASE = "{arch}.rope.freq_base" + FREQ_BASE_SPATIAL = "{arch}.rope.freq_base_spatial" FREQ_BASE_SWA = "{arch}.rope.freq_base_swa" SCALING_TYPE = "{arch}.rope.scaling.type" SCALING_FACTOR = "{arch}.rope.scaling.factor" @@ -573,6 +574,7 @@ class MODEL_ARCH(IntEnum): QWEN2 = auto() QWEN2MOE = auto() QWEN2VL = auto() + SENSENOVA_U1 = auto() QWEN3 = auto() QWEN3MOE = auto() QWEN3NEXT = auto() @@ -1390,6 +1392,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.QWEN2: "qwen2", MODEL_ARCH.QWEN2MOE: "qwen2moe", MODEL_ARCH.QWEN2VL: "qwen2vl", + MODEL_ARCH.SENSENOVA_U1: "sensenova_u1", MODEL_ARCH.QWEN3: "qwen3", MODEL_ARCH.QWEN3MOE: "qwen3moe", MODEL_ARCH.QWEN3NEXT: "qwen3next", @@ -2916,6 +2919,24 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP, ], + MODEL_ARCH.SENSENOVA_U1: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], MODEL_ARCH.QWEN3: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -6485,6 +6506,7 @@ KEY_ATTENTION_LAYERNORM_RMS_EPS = Keys.Attention.LAYERNORM_RMS_EPS # RoPE KEY_ROPE_DIMENSION_COUNT = Keys.Rope.DIMENSION_COUNT KEY_ROPE_FREQ_BASE = Keys.Rope.FREQ_BASE +KEY_ROPE_FREQ_BASE_SPATIAL = Keys.Rope.FREQ_BASE_SPATIAL KEY_ROPE_SCALING_TYPE = Keys.Rope.SCALING_TYPE KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index e8a96e571..d00392211 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1167,6 +1167,9 @@ class GGUFWriter: def add_rope_freq_base(self, value: float) -> None: self.add_float32(Keys.Rope.FREQ_BASE.format(arch=self.arch), value) + def add_rope_freq_base_spatial(self, value: float) -> None: + self.add_float32(Keys.Rope.FREQ_BASE_SPATIAL.format(arch=self.arch), value) + def add_rope_scaling_type(self, value: RopeScalingType) -> None: self.add_string(Keys.Rope.SCALING_TYPE.format(arch=self.arch), value.value) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 4534be22f..1c11536b5 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -33,6 +33,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_QWEN2, "qwen2" }, { LLM_ARCH_QWEN2MOE, "qwen2moe" }, { LLM_ARCH_QWEN2VL, "qwen2vl" }, + { LLM_ARCH_SENSENOVA_U1, "sensenova_u1" }, { LLM_ARCH_QWEN3, "qwen3" }, { LLM_ARCH_QWEN3MOE, "qwen3moe" }, { LLM_ARCH_QWEN3NEXT, "qwen3next" }, @@ -339,6 +340,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ROPE_DIMENSION_SECTIONS, "%s.rope.dimension_sections" }, { LLM_KV_ROPE_SECTION_ORDER, "%s.rope.section_order" }, { LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" }, + { LLM_KV_ROPE_FREQ_BASE_SPATIAL, "%s.rope.freq_base_spatial" }, { LLM_KV_ROPE_FREQ_BASE_SWA, "%s.rope.freq_base_swa" }, { LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" }, { LLM_KV_ROPE_SCALING_TYPE, "%s.rope.scaling.type" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 5a8ca4250..afceb8ea6 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -38,6 +38,7 @@ enum llm_arch { LLM_ARCH_QWEN2, LLM_ARCH_QWEN2MOE, LLM_ARCH_QWEN2VL, + LLM_ARCH_SENSENOVA_U1, LLM_ARCH_QWEN3, LLM_ARCH_QWEN3MOE, LLM_ARCH_QWEN3NEXT, @@ -344,6 +345,7 @@ enum llm_kv { LLM_KV_ROPE_DIMENSION_SECTIONS, LLM_KV_ROPE_SECTION_ORDER, LLM_KV_ROPE_FREQ_BASE, + LLM_KV_ROPE_FREQ_BASE_SPATIAL, LLM_KV_ROPE_FREQ_BASE_SWA, LLM_KV_ROPE_SCALE_LINEAR, LLM_KV_ROPE_SCALING_TYPE, diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 46b1d2ae2..d620fcd9c 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -15,6 +15,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) { switch (arch) { + case LLM_ARCH_SENSENOVA_U1: case LLM_ARCH_GEMMA3N: case LLM_ARCH_BITNET: case LLM_ARCH_T5: diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 6712f0f22..3797c567d 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -110,6 +110,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_qwen2vl(params); case LLM_ARCH_QWEN2MOE: return new llama_model_qwen2moe(params); + case LLM_ARCH_SENSENOVA_U1: + return new llama_model_sensenova_u1(params); case LLM_ARCH_QWEN3: return new llama_model_qwen3(params); case LLM_ARCH_QWEN3MOE: @@ -3640,6 +3642,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { // DSV4 DSpark drafters use DeepSeek-V4's normal RoPE; legacy DFlash backbones are NeoX return model->hparams.dsv4_hc_mult > 0 ? LLAMA_ROPE_TYPE_NORM : LLAMA_ROPE_TYPE_NEOX; + case LLM_ARCH_SENSENOVA_U1: case LLM_ARCH_QWEN2VL: case LLM_ARCH_PADDLEOCR: return LLAMA_ROPE_TYPE_MROPE; diff --git a/src/models/models.h b/src/models/models.h index 9aa450dea..47b0203da 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -614,6 +614,21 @@ struct llama_model_qwen2moe : public llama_model_base { }; +struct llama_model_sensenova_u1 : public llama_model_base { + llama_model_sensenova_u1(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + float rope_freq_base_spatial = 10000.0f; + + struct graph : public llm_graph_context { + graph(const llama_model_sensenova_u1 & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_qwen3 : public llama_model_base { llama_model_qwen3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/src/models/sensenova-u1.cpp b/src/models/sensenova-u1.cpp new file mode 100644 index 000000000..c42c604e8 --- /dev/null +++ b/src/models/sensenova-u1.cpp @@ -0,0 +1,190 @@ +#include "models.h" + +namespace { +class llm_graph_input_pos_sensenova_u1 : public llm_graph_input_pos { +public: + llm_graph_input_pos_sensenova_u1() : llm_graph_input_pos(4) {} + + void set_input(const llama_ubatch * ubatch) override { + if (ubatch->pos && pos) { + const int64_t n_tokens = ubatch->n_tokens; + + // U1 expects text-token positions in the temporal plane only: [t, 0, 0, 0]. + // The generic M-RoPE input uses [t, t, t, 0], so U1 needs a custom input class. + if (ubatch->token) { + std::vector pos_data(n_tokens*n_pos_per_embd, 0); + for (int i = 0; i < n_tokens; ++i) { + pos_data[i] = ubatch->pos[i]; + } + ggml_backend_tensor_set(pos, pos_data.data(), 0, pos_data.size()*ggml_element_size(pos)); + } else { + ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos)); + } + } + } +}; +} + +void llama_model_sensenova_u1::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SPATIAL, rope_freq_base_spatial); + if (hparams.n_embd_head_k() % 4 != 0 || hparams.n_embd_head_v() != hparams.n_embd_head_k()) { + throw std::runtime_error("SenseNova U1 requires equal Q/K/V head sizes divisible by four"); + } + switch (hparams.n_layer()) { + case 42: type = LLM_TYPE_8B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_sensenova_u1::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + // output + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } +} + +std::unique_ptr llama_model_sensenova_u1::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_sensenova_u1::graph::graph(const llama_model_sensenova_u1 & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + ggml_tensor * cur; + ggml_tensor * inpL; + + auto normalize = [&](ggml_tensor * x, ggml_tensor * weight) { + x = ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps); + + return ggml_mul(ctx0, x, weight); + }; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * pos[3]{}; + auto inp = std::make_unique(); + auto & inp_pos = inp->pos; + inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, (int64_t)n_tokens*hparams.n_pos_per_embd()); + ggml_set_input(inp_pos); + res->add_input(std::move(inp)); + + for (int axis = 0; axis < 3; ++axis) { + pos[axis] = ggml_view_1d(ctx0, inp_pos, n_tokens, axis*n_tokens*sizeof(int32_t)); + } + + auto normalize_and_rotate = [&](ggml_tensor * x, ggml_tensor * weight) { + const int64_t heads = x->ne[1]; + x = ggml_reshape_4d(ctx0, x, n_embd_head/2, 2, heads, n_tokens); + x = normalize(x, ggml_reshape_2d(ctx0, weight, n_embd_head/2, 2)); + x = ggml_reshape_3d(ctx0, x, n_embd_head, heads, n_tokens); + + // Normalize temporal and spatial halves separately; rotate time, height, then width. + for (int axis = 0; axis < 3; ++axis) { + const int dims = axis == 0 ? n_embd_head/2 : n_embd_head/4; + const int offset = axis == 0 ? 0 : n_embd_head/2 + (axis - 1)*n_embd_head/4; + const float base = axis == 0 ? freq_base : model.rope_freq_base_spatial; + x = ggml_rope_ext(ctx0, x, pos[axis], nullptr, dims, GGML_ROPE_TYPE_NEOX, + n_ctx_orig, base, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); + ggml_rope_set_offset(x, offset); + } + return x; + }; + + auto * inp_attn = build_attn_inp_kv(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + + ggml_tensor * inpSA = inpL; + + // norm + cur = normalize(inpL, model.layers[il].attn_norm); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, n_head_kv, il); + + Qcur = normalize_and_rotate(Qcur, model.layers[il].attn_q_norm); + Kcur = normalize_and_rotate(Kcur, model.layers[il].attn_k_norm); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); + } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = normalize(ffn_inp, model.layers[il].ffn_norm); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + cur = inpL; + + cur = normalize(cur, model.output_norm); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur, model.output_s); + + cb(cur, "result_output", -1); + res->t_logits = cur; +} diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 21d27e17d..c86b1315a 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -329,6 +329,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { // partial rotary: n_rot must not exceed the indexer key length (64) ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64)); } + if (arch == LLM_ARCH_SENSENOVA_U1) { + ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SPATIAL, 10000.0f); + } ms.add_kv(LLM_KV_ATTENTION_CLAMP_KQV, 1.0f); ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, 1e-5f); ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f); diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 398b73904..bd7fb7356 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -55,6 +55,7 @@ add_library(mtmd models/paddleocr.cpp models/pixtral.cpp models/qwen2vl.cpp + models/sensenova-u1.cpp models/minimax-m3.cpp models/qwen3vl.cpp models/ling3vl.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 71d349cc5..6ca465280 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -547,6 +547,7 @@ enum projector_type { PROJECTOR_TYPE_POCKETTTS_SPKENC, PROJECTOR_TYPE_POCKETTTS_GEN, PROJECTOR_TYPE_MUSE_GLIMMER, + PROJECTOR_TYPE_SENSENOVA_U1, PROJECTOR_TYPE_COHERE2V, PROJECTOR_TYPE_MUSE_SPARK_AUDIO, PROJECTOR_TYPE_UNKNOWN, @@ -621,6 +622,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"}, { PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"}, { PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"}, + { PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"}, { PROJECTOR_TYPE_COHERE2V, "cohere2v"}, { PROJECTOR_TYPE_MUSE_SPARK_AUDIO, "muse-spark-audio"}, }; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 359ec5ac3..869e3fff8 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -997,6 +997,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_LING3VL: { builder = std::make_unique(ctx, img); @@ -1795,6 +1799,20 @@ struct clip_model_loader { LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__); } } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + hparams.n_merge = 2; + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels); + hparams.rope_theta = 10000.0f; + hparams.set_limit_image_tokens(16, 4096); + if (hparams.n_merge != 2) { + throw std::runtime_error(string_format("%s: SenseNova U1 requires spatial_merge_size=2, got %u", __func__, hparams.n_merge)); + } + hparams.set_warmup_n_tokens(16*16); + } break; case PROJECTOR_TYPE_MINIMAX_M3: { hparams.n_merge = 2; // spatial_merge_size @@ -3837,6 +3855,11 @@ struct clip_model_loader { } } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); + } break; default: GGML_ASSERT(false && "unknown projector type"); } @@ -4311,6 +4334,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_SENSENOVA_U1: case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: @@ -4341,6 +4365,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_SENSENOVA_U1: case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: @@ -4426,6 +4451,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_SENSENOVA_U1: case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: @@ -5525,6 +5551,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } set_input_i32("pos_w", pos_data); } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + const int n_patches_per_row = image_size_width / patch_size; + std::vector pos_data(n_pos); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i / n_patches_per_row; + } + set_input_i32("pos_y", pos_data); + for (int i = 0; i < n_pos; i++) { + pos_data[i] = i % n_patches_per_row; + } + set_input_i32("pos_x", pos_data); + } break; case PROJECTOR_TYPE_DEEPSEEK4V: { // set the 2D positions (mrope layout, only the first 2 channels are used) @@ -6411,6 +6450,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_JANUS_PRO: case PROJECTOR_TYPE_YOUTUVL: return ctx->model.mm_1_b->ne[0]; + case PROJECTOR_TYPE_SENSENOVA_U1: + return ctx->model.hparams.projection_dim; case PROJECTOR_TYPE_QWEN3VL: // main path + deepstack paths return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers); diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 294448605..55e525cdc 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -50,6 +50,11 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl { ggml_cgraph * build() override; }; +struct clip_graph_sensenova_u1 : clip_graph { + clip_graph_sensenova_u1(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_ling3vl : clip_graph_qwen3vl { clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/sensenova-u1.cpp b/tools/mtmd/models/sensenova-u1.cpp new file mode 100644 index 000000000..57875c6ed --- /dev/null +++ b/tools/mtmd/models/sensenova-u1.cpp @@ -0,0 +1,40 @@ +#include "models.h" + +ggml_cgraph * clip_graph_sensenova_u1::build() { + GGML_ASSERT(model.patch_embeddings_0 != nullptr); + GGML_ASSERT(model.patch_bias != nullptr); + GGML_ASSERT(model.mm_0_w != nullptr); + GGML_ASSERT(model.mm_0_b != nullptr); + GGML_ASSERT(hparams.n_merge == 2); + + ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); + ggml_set_name(pos_x, "pos_x"); + ggml_set_input(pos_x); + + ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); + ggml_set_name(pos_y, "pos_y"); + ggml_set_input(pos_y); + + ggml_tensor * inp = build_inp(); + inp = ggml_gelu(ctx0, inp); + + // Apply adjacent-pair 2D RoPE to patch embeddings before downsampling. + inp = ggml_reshape_4d(ctx0, inp, n_embd, 1, n_patches, n_batch); + inp = build_rope_2d(ctx0, inp, pos_x, pos_y, hparams.rope_theta, false); + inp = ggml_reshape_3d(ctx0, inp, n_embd, n_patches, n_batch); + cb(inp, "patch_embd_rope", -1); + + inp = ggml_reshape_4d(ctx0, inp, n_embd, n_patches_x, n_patches_y, n_batch); + inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3)); + inp = ggml_conv_2d(ctx0, model.mm_0_w, inp, + hparams.n_merge, hparams.n_merge, 0, 0, 1, 1); + + const int n_output_patches = n_patches / (hparams.n_merge * hparams.n_merge); + inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 2, 0, 3)); + inp = ggml_reshape_3d(ctx0, inp, n_mmproj_embd, n_output_patches, n_batch); + inp = ggml_add(ctx0, inp, model.mm_0_b); + cb(inp, "proj_out", -1); + + ggml_build_forward_expand(gf, inp); + return gf; +} diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 9e2c5f3c9..9460f15fe 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -225,6 +225,7 @@ enum mtmd_pos_type { MTMD_POS_TYPE_NORMAL, // number of positions equals to number of tokens MTMD_POS_TYPE_MROPE, // qwen-vl mrope style, each image takes max(t,h,w) position indexes MTMD_POS_TYPE_HUNYUANVL, // HunyuanVL mrope + BOI/EOI/newline layout with XD-RoPE dim-3 + MTMD_POS_TYPE_SENSENOVA_U1, // U1 uses zero-based H/W and one temporal position per image MTMD_POS_TYPE_CANVAS, // MiniCPM-V 4.7: overview + slices in one chunk, sharing one 2D canvas (see mtmd_image_tokens::canvas_tile_grid) MTMD_POS_TYPE_COUNT, // for validation }; @@ -751,6 +752,12 @@ struct mtmd_context { img_end = "]<]end of image[>["; image_preproc = std::make_unique(ctx_v); } break; + case PROJECTOR_TYPE_SENSENOVA_U1: + { + img_beg = ""; + img_end = ""; + image_preproc = std::make_unique(ctx_v); + } break; case PROJECTOR_TYPE_MUSE_GLIMMER: { img_beg = "<|image_start|>"; @@ -1615,10 +1622,15 @@ struct mtmd_tokenizer { image_tokens->ny = 1; } image_tokens->pos = ctx->pos_type; + // U1 uses M-RoPE, but unlike Qwen its image H/W coordinates are + // zero-based and the whole image occupies one temporal position. + if (ctx->proj_type_v() == PROJECTOR_TYPE_SENSENOVA_U1) { + image_tokens->pos = MTMD_POS_TYPE_SENSENOVA_U1; + } // HunyuanVL wraps the image grid with BOI/EOI and adds one newline per row, // and uses XD-RoPE (dim-3 = image index). Override the position type so that // n_tokens() and mtmd_image_tokens_get_decoder_pos pick the HunyuanVL layout. - if (ctx->proj_type_v() == PROJECTOR_TYPE_HUNYUANVL) { + else if (ctx->proj_type_v() == PROJECTOR_TYPE_HUNYUANVL) { image_tokens->pos = MTMD_POS_TYPE_HUNYUANVL; image_tokens->image_idx = n_images_added; GGML_ASSERT(n_tokens == (size_t)image_tokens->n_tokens()); @@ -2398,6 +2410,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk } case PROJECTOR_TYPE_GEMMA4UV: case PROJECTOR_TYPE_GEMMA3: + case PROJECTOR_TYPE_SENSENOVA_U1: case PROJECTOR_TYPE_DEEPSEEK4V: return true; default: @@ -2761,6 +2774,13 @@ mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * ima pos.y = pos_0 + i; pos.z = pos_0 + i; } break; + case MTMD_POS_TYPE_SENSENOVA_U1: + { + pos.t = pos_0; + pos.x = i % image_tokens->nx; + pos.y = i / image_tokens->nx; + pos.z = 0; + } break; case MTMD_POS_TYPE_HUNYUANVL: { // HunyuanVL layout: [BOI] [row0 tokens + newline] ... [row(ny-1) tokens + newline] [EOI] @@ -2813,6 +2833,8 @@ llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens) { return std::max(image_tokens->nx, image_tokens->ny); case MTMD_POS_TYPE_NORMAL: return image_tokens->n_tokens(); + case MTMD_POS_TYPE_SENSENOVA_U1: + return 1; case MTMD_POS_TYPE_HUNYUANVL: // HunyuanVL: the sequential (dim-0) position advances by the full token count // (includes BOI/EOI and row newline tokens), not by max(nx, ny)