From abb76ab380c7f3f731e365a859fb4cc64e939bb8 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 29 Oct 2026 09:13:41 +0100 Subject: [PATCH] model : Support Limite 1B - Violetto (#29433) * gguf-py : add Limite arch Assisted-by: Claude Opus 5.5 Signed-off-by: Giuseppe Scrivano * convert : add Limite support Assisted-by: Claude Opus 5.5 Signed-off-by: Giuseppe Scrivano * llama : add Limite support Assisted-by: Claude Opus 5.5 Signed-off-by: Giuseppe Scrivano --------- Signed-off-by: Giuseppe Scrivano --- conversion/__init__.py | 1 + conversion/limite.py | 184 ++++++++++++++++++++++++++ gguf-py/gguf/constants.py | 42 ++++++ gguf-py/gguf/gguf_writer.py | 15 +++ src/llama-arch.cpp | 22 ++++ src/llama-arch.h | 13 ++ src/llama-hparams.h | 7 + src/llama-model-loader.cpp | 1 + src/llama-model-saver.cpp | 12 ++ src/llama-model.cpp | 3 + src/llama-model.h | 12 ++ src/llama-quant.cpp | 7 + src/models/limite.cpp | 252 ++++++++++++++++++++++++++++++++++++ src/models/models.h | 13 ++ tests/test-llama-archs.cpp | 16 ++- 15 files changed, 598 insertions(+), 2 deletions(-) create mode 100644 conversion/limite.py create mode 100644 src/models/limite.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index 4e4fae608..d4e9f0441 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -176,6 +176,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Lfm3ForCausalLM": "lfm3", "Lfm3OmniForConditionalGeneration": "lfm3", "Lfm3VlForConditionalGeneration": "lfm3", + "LimiteForCausalLM": "limite", "Llama4ForCausalLM": "llama", "Llama4ForConditionalGeneration": "llama", "LlamaBidirectionalModel": "llama", diff --git a/conversion/limite.py b/conversion/limite.py new file mode 100644 index 000000000..7d413302b --- /dev/null +++ b/conversion/limite.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +from typing import Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf + + +@ModelBase.register("LimiteForCausalLM") +@ModelBase.example("paradigma-inc/limite-1b-violetto") +class LimiteModel(TextModel): + model_arch = gguf.MODEL_ARCH.LIMITE + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hp = self.hparams + + expected = { + "pos_mode": "rope", + "qk_norm": "rms_pre_rope", + "rope_style": "interleaved_pairs_odd_lane_sign_flip", + "rope_per_layer": False, + "rms_norm_has_weight": False, + "rms_norm_eps_mode": "torch_finfo_default", + "mlp_type": "swiglu", + "attn_gate_applied": "per_head_before_o_proj", + "ve_head_slice": "first_num_key_value_heads", + "ve_applied_before_qk_norm": True, + "global_nope": True, + "global_window": -1, + "mudd_r_site": "resid", + "tie_word_embeddings": True, + "final_softcap": 0.0, + } + for key, value in expected.items(): + if hp.get(key) != value: + raise NotImplementedError(f"Limite: unsupported {key}={hp.get(key)!r}, expected {value!r}") + if hp["softcap_logits"]["kind"] != "sigmoid": + raise NotImplementedError(f"Limite: unsupported softcap_logits kind {hp['softcap_logits']['kind']!r}") + + self.n_head = hp["num_attention_heads"] + self.n_head_kv = hp["num_key_value_heads"] + self.head_dim = hp["head_dim"] + self.n_pairs = hp["rope_n_pairs"] + + self.mudd_taps: dict[int, list[int]] = {} + if hp.get("mudd"): + if not hp.get("mudd_mlp"): + raise NotImplementedError("Limite: mudd without mudd_mlp is not supported") + self.mudd_taps = {int(il): [int(t) for t in taps] for il, taps in hp["mudd_tap_idx"].items()} + assert all(len(taps) == hp["mudd_taps"] for taps in self.mudd_taps.values()) + + def set_vocab(self): + self._set_vocab_gpt2() + + def set_gguf_parameters(self): + super().set_gguf_parameters() + hp = self.hparams + + self.gguf_writer.add_key_length(self.head_dim) + self.gguf_writer.add_value_length(self.head_dim) + + # torch rms_norm with eps=None uses the fp32 epsilon, even for bf16 inputs + self.gguf_writer.add_layer_norm_rms_eps(torch.finfo(torch.float32).eps) + self.gguf_writer.add_attention_scale(hp["attention_softmax_scale"]) + self.gguf_writer.add_xsa_eps(hp["xsa_normalize_eps"] if hp.get("xsa") else 0.0) + + # HF freq_i = base^(-i/(n_pairs-1)), ggml uses freq_base^(-i/n_pairs) + self.gguf_writer.add_rope_dimension_count(2 * self.n_pairs) + self.gguf_writer.add_rope_freq_base(hp["rope_base_local"] ** (self.n_pairs / (self.n_pairs - 1))) + + # HF keeps keys k >= q - window, so a window of W spans W + 1 tokens + global_layers = set(hp["global_layers"]) + self.gguf_writer.add_sliding_window(hp["sliding_window"] + 1) + self.gguf_writer.add_sliding_window_pattern([il not in global_layers for il in range(self.block_count)]) + + cap = hp["softcap_logits"] + self.gguf_writer.add_final_logit_sigmoid_capping(cap["a"], cap["b"], cap["c"]) + + if self.mudd_taps: + n_taps = hp["mudd_taps"] + tap_indices = [-1] * (self.block_count * n_taps) + for il, taps in self.mudd_taps.items(): + tap_indices[il * n_taps:(il + 1) * n_taps] = taps + self.gguf_writer.add_mudd_feed_forward_length(hp["mudd_inter"]) + self.gguf_writer.add_mudd_tap_count(n_taps) + self.gguf_writer.add_mudd_tap_indices(tap_indices) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # tiny mixing weights, kept at full precision as in the reference + # the mudd tensors come from a global source tensor, so take the block id from the new name + if new_name.startswith("blk."): + bid = int(new_name.split(".")[1]) + for key in (gguf.MODEL_TENSOR.ATTN_GATE, gguf.MODEL_TENSOR.ATTN_VE_GATE, gguf.MODEL_TENSOR.MUDD_DOWN, gguf.MODEL_TENSOR.MUDD_UP): + if self.match_model_tensor_name(new_name, key, bid): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def _swap_rope_pairs(self, w: Tensor, n_head: int) -> Tensor: + # HF rotates each pair by -theta; swapping the pair lanes gives the ggml +theta rotation. + # q and k are both swapped, so the attention scores do not change. + idx = torch.arange(self.head_dim) + idx[:2 * self.n_pairs] = idx[:2 * self.n_pairs].view(-1, 2).flip(-1).reshape(-1) + w = w.reshape(n_head, self.head_dim, -1) + return w[:, idx, :].reshape(n_head * self.head_dim, -1) + + def _pad_cols(self, w: Tensor) -> Tensor: + # gates read only the first channels of the input, pad them to n_embd + return torch.nn.functional.pad(w, (0, self.hparams["hidden_size"] - w.shape[-1])) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + hp = self.hparams + T = gguf.MODEL_TENSOR + + def load(n: str) -> Tensor: + return self.model_tensors[n]().float() + + if name == "model.value_embeds.weight": + ve = data_torch.reshape(data_torch.shape[0], -1, hp["ve_dim"])[:, :self.n_head_kv] + ve = torch.nn.functional.pad(ve, (0, self.head_dim - hp["ve_dim"])) + yield self.format_tensor_name(T.VALUE_EMBD), ve.reshape(ve.shape[0], -1) * hp["ve_gate_scale"] + return + + if name == "model.mudd.dense1": + for il in self.mudd_taps: + yield self.format_tensor_name(T.MUDD_DOWN, il), data_torch + return + if name == "model.mudd.dense2": + # rows [0, n) mix the attention input, rows [n, 2n) mix the residual + dense2_mlp = load("model.mudd.dense2_mlp") + bias = load("model.mudd.bias") + bias_mlp = load("model.mudd.bias_mlp") + for il, taps in self.mudd_taps.items(): + n = len(taps) + w = torch.cat([data_torch[il, :n], dense2_mlp[il, :n]], 0) + b = torch.cat([bias[il, :n], bias_mlp[il, :n]], 0) + yield self.format_tensor_name(T.MUDD_UP, il), w + yield self.format_tensor_name(T.MUDD_UP, il, ".bias"), b + return + if name.startswith("model.mudd."): + return + + if bid is None: + yield from super().modify_tensors(data_torch, name, bid) + return + + prefix = f"model.layers.{bid}." + name = name.removeprefix(prefix) + + if name in ("self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight"): + data_torch = data_torch * load(prefix + "self_attn.qkv_scale") + if name == "self_attn.q_proj.weight": + data_torch = self._swap_rope_pairs(data_torch, self.n_head) + elif name == "self_attn.k_proj.weight": + data_torch = self._swap_rope_pairs(data_torch, self.n_head_kv) + elif name == "self_attn.o_proj.weight": + data_torch = data_torch * load(prefix + "self_attn.o_scale") * load(prefix + "post_lambda_attn") * hp["attn_gate_scale"] + elif name == "mlp.down_proj.weight": + data_torch = data_torch * load(prefix + "post_lambda_mlp") + elif name == "self_attn.attn_gate": + yield self.format_tensor_name(T.ATTN_GATE, bid), self._pad_cols(data_torch) + return + elif name == "self_attn.ve_gate": + yield self.format_tensor_name(T.ATTN_VE_GATE, bid), self._pad_cols(data_torch[:self.n_head_kv]) + return + elif name == "self_attn.xsa_alpha": + if hp.get("xsa") and bid in hp["xsa_layers"]: + yield self.format_tensor_name(T.ATTN_XSA_ALPHA, bid), torch.tanh(data_torch) + return + elif name == "resid_lambda_attn": + yield self.format_tensor_name(T.ATTN_RESID_SCALE, bid), data_torch.reshape(1) + return + elif name == "resid_lambda_mlp": + yield self.format_tensor_name(T.FFN_RESID_SCALE, bid), data_torch.reshape(1) + return + elif name in ("self_attn.qkv_scale", "self_attn.o_scale", "post_lambda_attn", "post_lambda_mlp"): + # folded into the projections above + return + + yield from super().modify_tensors(data_torch, prefix + name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index e28fdf605..1ca2415cd 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -143,6 +143,7 @@ class Keys: ATTN_LOGIT_SOFTCAPPING = "{arch}.attn_logit_softcapping" ROUTER_LOGIT_SOFTCAPPING = "{arch}.router_logit_softcapping" FINAL_LOGIT_SOFTCAPPING = "{arch}.final_logit_softcapping" + FINAL_LOGIT_SIGMOID_CAPPING = "{arch}.final_logit_sigmoid_capping" SWIN_NORM = "{arch}.swin_norm" RESCALE_EVERY_N_LAYERS = "{arch}.rescale_every_n_layers" TIME_MIX_EXTRA_DIM = "{arch}.time_mix_extra_dim" @@ -227,6 +228,7 @@ class Keys: ROPE_PATTERN = "{arch}.attention.rope_pattern" VALUE_EXPERT_COUNT = "{arch}.attention.value_expert_count" VALUE_EXPERT_USED_COUNT = "{arch}.attention.value_expert_used_count" + XSA_EPS = "{arch}.attention.xsa_epsilon" class Indexer: HEAD_COUNT = "{arch}.attention.indexer.head_count" @@ -301,6 +303,11 @@ class Keys: LLM_KV_SPLIT_COUNT = "split.count" LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count" + class MUDD: + FEED_FORWARD_LENGTH = "{arch}.mudd.feed_forward_length" + TAP_COUNT = "{arch}.mudd.tap_count" + TAP_INDICES = "{arch}.mudd.tap_indices" + class HRM: LAYERS_PER_STACK = "{arch}.hrm.layers_per_stack" H_CYCLES = "{arch}.hrm.h_cycles" @@ -731,6 +738,7 @@ class MODEL_ARCH(IntEnum): POCKETTTS = auto() K2HORIZON = auto() KOLIBRI1 = auto() + LIMITE = auto() class VISION_PROJECTOR_TYPE(IntEnum): @@ -865,6 +873,13 @@ class MODEL_TENSOR(IntEnum): FFN_ROUTED_DOWN = auto() # Kimi K3 (latent MoE: hidden -> latent) FFN_ROUTED_UP = auto() # Kimi K3 (latent MoE: latent -> hidden) FFN_ROUTED_NORM = auto() # Kimi K3 (latent MoE: norm on expert output) + VALUE_EMBD = auto() # limite + ATTN_VE_GATE = auto() # limite + ATTN_XSA_ALPHA = auto() # limite + ATTN_RESID_SCALE = auto() # limite + FFN_RESID_SCALE = auto() # limite + MUDD_DOWN = auto() # limite + MUDD_UP = auto() # limite TIME_MIX_W0 = auto() TIME_MIX_W1 = auto() TIME_MIX_W2 = auto() @@ -1577,6 +1592,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.POCKETTTS: "pockettts", MODEL_ARCH.K2HORIZON: "k2-horizon", MODEL_ARCH.KOLIBRI1: "kolibri1", + MODEL_ARCH.LIMITE: "limite", } VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = { @@ -1709,6 +1725,13 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.FFN_ROUTED_DOWN: "blk.{bid}.ffn_routed_down", # Kimi K3 MODEL_TENSOR.FFN_ROUTED_UP: "blk.{bid}.ffn_routed_up", # Kimi K3 MODEL_TENSOR.FFN_ROUTED_NORM: "blk.{bid}.ffn_routed_norm", # Kimi K3 + MODEL_TENSOR.VALUE_EMBD: "value_embd", # limite + MODEL_TENSOR.ATTN_VE_GATE: "blk.{bid}.attn_ve_gate", # limite + MODEL_TENSOR.ATTN_XSA_ALPHA: "blk.{bid}.attn_xsa_alpha", # limite + MODEL_TENSOR.ATTN_RESID_SCALE: "blk.{bid}.attn_resid_scale", # limite + MODEL_TENSOR.FFN_RESID_SCALE: "blk.{bid}.ffn_resid_scale", # limite + MODEL_TENSOR.MUDD_DOWN: "blk.{bid}.mudd_down", # limite + MODEL_TENSOR.MUDD_UP: "blk.{bid}.mudd_up", # limite MODEL_TENSOR.TIME_MIX_W0: "blk.{bid}.time_mix_w0", MODEL_TENSOR.TIME_MIX_W1: "blk.{bid}.time_mix_w1", MODEL_TENSOR.TIME_MIX_W2: "blk.{bid}.time_mix_w2", @@ -6288,6 +6311,25 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_UP_SHEXP, MODEL_TENSOR.FFN_DOWN_SHEXP, ], + MODEL_ARCH.LIMITE: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.VALUE_EMBD, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.ATTN_VE_GATE, + MODEL_TENSOR.ATTN_XSA_ALPHA, + MODEL_TENSOR.ATTN_RESID_SCALE, + MODEL_TENSOR.FFN_RESID_SCALE, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.MUDD_DOWN, + MODEL_TENSOR.MUDD_UP, + ], } # tensors that will not be serialized diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index efdf8f09c..b3dabc0bd 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -876,6 +876,9 @@ class GGUFWriter: def add_final_logit_softcapping(self, value: float) -> None: self.add_float32(Keys.LLM.FINAL_LOGIT_SOFTCAPPING.format(arch=self.arch), value) + def add_final_logit_sigmoid_capping(self, scale: float, shift: float, temperature: float) -> None: + self.add_array(Keys.LLM.FINAL_LOGIT_SIGMOID_CAPPING.format(arch=self.arch), [scale, shift, temperature]) + def add_expert_count(self, count: int) -> None: self.add_uint32(Keys.LLM.EXPERT_COUNT.format(arch=self.arch), count) @@ -948,6 +951,15 @@ class GGUFWriter: def add_embedding_scale(self, value: float) -> None: self.add_float32(Keys.LLM.EMBEDDING_SCALE.format(arch=self.arch), value) + def add_mudd_feed_forward_length(self, value: int) -> None: + self.add_uint32(Keys.MUDD.FEED_FORWARD_LENGTH.format(arch=self.arch), value) + + def add_mudd_tap_count(self, value: int) -> None: + self.add_uint32(Keys.MUDD.TAP_COUNT.format(arch=self.arch), value) + + def add_mudd_tap_indices(self, value: Sequence[int]) -> None: + self.add_array(Keys.MUDD.TAP_INDICES.format(arch=self.arch), value) + def add_hrm_layers_per_stack(self, value: int) -> None: self.add_uint32(Keys.HRM.LAYERS_PER_STACK.format(arch=self.arch), value) @@ -1151,6 +1163,9 @@ class GGUFWriter: def add_attention_scale(self, value: float) -> None: self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value) + def add_xsa_eps(self, value: float) -> None: + self.add_float32(Keys.Attention.XSA_EPS.format(arch=self.arch), value) + def add_attn_output_scale(self, value: float) -> None: self.add_float32(Keys.Attention.OUTPUT_SCALE.format(arch=self.arch), value) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 497e42692..3a19646eb 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -145,6 +145,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_APERTUS, "apertus" }, { LLM_ARCH_MINIMAX_01, "minimax-01" }, { LLM_ARCH_HRM_TEXT, "hrm_text" }, + { LLM_ARCH_LIMITE, "limite" }, { LLM_ARCH_MINIMAX_M2, "minimax-m2" }, { LLM_ARCH_MINIMAX_M3, "minimax-m3" }, { LLM_ARCH_COGVLM, "cogvlm" }, @@ -248,6 +249,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTN_LOGIT_SOFTCAPPING, "%s.attn_logit_softcapping" }, { LLM_KV_ROUTER_LOGIT_SOFTCAPPING, "%s.router_logit_softcapping" }, { LLM_KV_FINAL_LOGIT_SOFTCAPPING, "%s.final_logit_softcapping" }, + { LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, "%s.final_logit_sigmoid_capping" }, { LLM_KV_SWIN_NORM, "%s.swin_norm" }, { LLM_KV_RESCALE_EVERY_N_LAYERS, "%s.rescale_every_n_layers" }, { LLM_KV_TIME_MIX_EXTRA_DIM, "%s.time_mix_extra_dim" }, @@ -268,6 +270,9 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" }, { LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" }, { LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" }, + { LLM_KV_MUDD_FEED_FORWARD_LENGTH, "%s.mudd.feed_forward_length" }, + { LLM_KV_MUDD_TAP_COUNT, "%s.mudd.tap_count" }, + { LLM_KV_MUDD_TAP_INDICES, "%s.mudd.tap_indices" }, { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, @@ -292,6 +297,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_ROPE_PATTERN, "%s.attention.rope_pattern" }, { LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, "%s.attention.value_expert_count" }, { LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, "%s.attention.value_expert_used_count" }, + { LLM_KV_ATTENTION_XSA_EPS, "%s.attention.xsa_epsilon" }, { LLM_KV_ATTENTION_SCALE, "%s.attention.scale" }, { LLM_KV_ATTENTION_OUTPUT_SCALE, "%s.attention.output_scale" }, @@ -605,6 +611,13 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_FFN_ROUTED_DOWN, "blk.%d.ffn_routed_down" }, { LLM_TENSOR_FFN_ROUTED_UP, "blk.%d.ffn_routed_up" }, { LLM_TENSOR_FFN_ROUTED_NORM, "blk.%d.ffn_routed_norm" }, + { LLM_TENSOR_VALUE_EMBD, "value_embd" }, + { LLM_TENSOR_ATTN_VE_GATE, "blk.%d.attn_ve_gate" }, + { LLM_TENSOR_ATTN_XSA_ALPHA, "blk.%d.attn_xsa_alpha" }, + { LLM_TENSOR_ATTN_RESID_SCALE, "blk.%d.attn_resid_scale" }, + { LLM_TENSOR_FFN_RESID_SCALE, "blk.%d.ffn_resid_scale" }, + { LLM_TENSOR_MUDD_DOWN, "blk.%d.mudd_down" }, + { LLM_TENSOR_MUDD_UP, "blk.%d.mudd_up" }, { LLM_TENSOR_SSM_G_B, "blk.%d.ssm_g_b" }, { LLM_TENSOR_SSM_NORM, "blk.%d.ssm_norm" }, { LLM_TENSOR_ATTN_Q_A_NORM, "blk.%d.attn_q_a_norm" }, @@ -998,6 +1011,13 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_FFN_ROUTED_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_FFN_ROUTED_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_FFN_ROUTED_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_VALUE_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, + {LLM_TENSOR_ATTN_VE_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_XSA_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ATTN_RESID_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_FFN_RESID_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_MUDD_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_MUDD_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_SSM_G_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_TIME_MIX_LERP_X, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_TIME_MIX_LN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, @@ -1315,6 +1335,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) { case LLM_ARCH_EAGLE3: case LLM_ARCH_DFLASH: case LLM_ARCH_GEMMA4_ASSISTANT: + case LLM_ARCH_LIMITE: return false; default: return true; @@ -1357,6 +1378,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_KIMI_K3: case LLM_ARCH_GLM5_NEXT: case LLM_ARCH_QWEN3TTS: + case LLM_ARCH_LIMITE: return false; default: return true; diff --git a/src/llama-arch.h b/src/llama-arch.h index eca6d18af..554156813 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -176,6 +176,7 @@ enum llm_arch { LLM_ARCH_K2_HORIZON, LLM_ARCH_BEAM, LLM_ARCH_KOLIBRI1, + LLM_ARCH_LIMITE, LLM_ARCH_UNKNOWN, }; @@ -252,6 +253,7 @@ enum llm_kv { LLM_KV_ATTN_LOGIT_SOFTCAPPING, LLM_KV_ROUTER_LOGIT_SOFTCAPPING, LLM_KV_FINAL_LOGIT_SOFTCAPPING, + LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, LLM_KV_SWIN_NORM, LLM_KV_RESCALE_EVERY_N_LAYERS, LLM_KV_TIME_MIX_EXTRA_DIM, @@ -273,6 +275,9 @@ enum llm_kv { LLM_KV_HRM_L_CYCLES, LLM_KV_HRM_PREFIX_LM, LLM_KV_REALTIME_DELAY_TOKENS, + LLM_KV_MUDD_FEED_FORWARD_LENGTH, + LLM_KV_MUDD_TAP_COUNT, + LLM_KV_MUDD_TAP_INDICES, LLM_KV_ATTENTION_HEAD_COUNT, LLM_KV_ATTENTION_HEAD_COUNT_KV, @@ -298,6 +303,7 @@ enum llm_kv { LLM_KV_ATTENTION_ROPE_PATTERN, LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, + LLM_KV_ATTENTION_XSA_EPS, LLM_KV_ATTENTION_OUTPUT_SCALE, LLM_KV_ATTENTION_VALUE_SCALE, @@ -612,6 +618,13 @@ enum llm_tensor { LLM_TENSOR_FFN_ROUTED_DOWN, // kimi-k3: latent MoE down LLM_TENSOR_FFN_ROUTED_UP, // kimi-k3: latent MoE up LLM_TENSOR_FFN_ROUTED_NORM, // kimi-k3: latent MoE norm + LLM_TENSOR_VALUE_EMBD, // limite: value embeddings + LLM_TENSOR_ATTN_VE_GATE, // limite: value embedding gate + LLM_TENSOR_ATTN_XSA_ALPHA, // limite: exclusive self attention + LLM_TENSOR_ATTN_RESID_SCALE, // limite: residual scale (attn) + LLM_TENSOR_FFN_RESID_SCALE, // limite: residual scale (ffn) + LLM_TENSOR_MUDD_DOWN, // limite: dense connection mixer + LLM_TENSOR_MUDD_UP, // limite: dense connection mixer LLM_TENSOR_TIME_MIX_W0, LLM_TENSOR_TIME_MIX_W1, LLM_TENSOR_TIME_MIX_W2, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 2c464ad30..2ac7bdeae 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -227,6 +227,13 @@ struct llama_hparams { uint32_t n_hrm_l_cycles = 0; bool hrm_prefix_lm = false; + // limite + float f_xsa_eps = 0.0f; + std::array final_logit_sigmoid_capping = {}; // scale, shift, temperature + uint32_t n_mudd_ff = 0; + uint32_t n_mudd_taps = 0; + std::array mudd_tap_idx; // [n_layer][n_mudd_taps], -1 = no mixing + bool ssm_dt_b_c_rms = false; float f_clamp_kqv = 0.0f; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index a963f2480..a71e3017a 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -411,6 +411,7 @@ namespace GGUFMeta { template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); + template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 1160e88a2..93836ed96 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -244,6 +244,16 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles); add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles); add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm); + if (hparams.final_logit_sigmoid_capping[0] != 0.0f) { + add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector( + hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end())); + } + add_kv(LLM_KV_MUDD_FEED_FORWARD_LENGTH, hparams.n_mudd_ff); + add_kv(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps); + if (hparams.n_mudd_taps > 0) { + add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector( + hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.begin() + hparams.n_layer()*hparams.n_mudd_taps)); + } add_kv(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count); add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step); // add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, ???); // saved as LLM_KV_ATTENTION_RECURRENT_LAYERS instead @@ -277,6 +287,7 @@ void llama_model_saver::add_kv_from_model() { hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all)); } add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); + add_kv(LLM_KV_ATTENTION_XSA_EPS, hparams.f_xsa_eps); add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale); add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale); add_kv(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length); @@ -501,6 +512,7 @@ void llama_model_saver::add_tensors_from_model() { add_tensor(model->hc_head_base); add_tensor(model->hc_head_scale); add_tensor(model->per_layer_tok_embd); + add_tensor(model->value_embd); add_tensor(model->hc_head_norm); add_tensor(model->hc_head_down); add_tensor(model->hc_head_up); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 81af8903f..8966bc946 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -332,6 +332,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_hrm_text(params); case LLM_ARCH_BEAM: return new llama_model_beam(params); + case LLM_ARCH_LIMITE: + return new llama_model_limite(params); case LLM_ARCH_COGVLM: return new llama_model_cogvlm(params); case LLM_ARCH_PANGU_EMBED: @@ -3575,6 +3577,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_DOTS3NOTE: case LLM_ARCH_NANBEIGE: case LLM_ARCH_POCKETTTS: + case LLM_ARCH_LIMITE: return LLAMA_ROPE_TYPE_NORM; case LLM_ARCH_BAILINGMOE3: // VL files carry mrope sections; text-only files keep NORM rope diff --git a/src/llama-model.h b/src/llama-model.h index 222238efe..3af8b054f 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -638,6 +638,15 @@ struct llama_layer { // gemma4 layer output scale, reused for talkie embedding skip scale struct ggml_tensor * out_scale = nullptr; + // limite + struct ggml_tensor * attn_ve_gate = nullptr; + struct ggml_tensor * attn_xsa_alpha = nullptr; + struct ggml_tensor * attn_resid_scale = nullptr; + struct ggml_tensor * ffn_resid_scale = nullptr; + struct ggml_tensor * mudd_down = nullptr; + struct ggml_tensor * mudd_up = nullptr; + struct ggml_tensor * mudd_up_b = nullptr; + struct llama_layer_posnet posnet; struct llama_layer_convnext convnext; @@ -783,6 +792,9 @@ struct llama_model { struct ggml_tensor * altup_unembd_proj = nullptr; struct ggml_tensor * per_layer_tok_embd = nullptr; + // limite value embeddings + struct ggml_tensor * value_embd = nullptr; + struct ggml_tensor * hc_head_norm = nullptr; struct ggml_tensor * hc_head_down = nullptr; struct ggml_tensor * hc_head_up = nullptr; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 9ddd6a63a..9f8aa8984 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -333,6 +333,13 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param // these are not too big so keep them as it is quantize &= name.find("per_layer_model_proj") == std::string::npos; + // limite: very small gates and dense connection mixers + if (arch == LLM_ARCH_LIMITE) { + quantize &= name.find("attn_gate.weight") == std::string::npos; + quantize &= name.find("attn_ve_gate.weight") == std::string::npos; + quantize &= name.find("mudd_") == std::string::npos; + } + // do not quantize positional embeddings and token types (BERT) quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD, "weight"); quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, "weight"); diff --git a/src/models/limite.cpp b/src/models/limite.cpp new file mode 100644 index 000000000..63f7c8288 --- /dev/null +++ b/src/models/limite.cpp @@ -0,0 +1,252 @@ +#include "models.h" + +// Limite: pre-norm transformer with value embeddings, exclusive self attention (XSA), +// per-head output gates, learned residual scales and dense (MUDD) cross-layer connections. +// ref: https://github.com/paradigma-inc/limite-violetto + +void llama_model_limite::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale); + ml.get_key(LLM_KV_ATTENTION_XSA_EPS, hparams.f_xsa_eps, false); + ml.get_arr(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, hparams.final_logit_sigmoid_capping, false); + + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + load_swa_pattern(ml, 4); + + std::fill(hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.end(), -1); + ml.get_key(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps, false); + if (hparams.n_mudd_taps > 0) { + ml.get_key(LLM_KV_MUDD_FEED_FORWARD_LENGTH, hparams.n_mudd_ff); + ml.get_arr(LLM_KV_MUDD_TAP_INDICES, hparams.mudd_tap_idx); + } + + switch (hparams.n_layer()) { + case 48: type = LLM_TYPE_1B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_limite::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + value_embd = create_tensor(tn(LLM_TENSOR_VALUE_EMBD, "weight"), {n_embd_v_gqa, n_vocab}, 0); + + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + // if output is NULL, init from the input tok embed + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + const int64_t n_taps = hparams.n_mudd_taps; + const int64_t n_mudd = hparams.n_mudd_ff; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, TENSOR_NOT_REQUIRED); + layer.attn_ve_gate = create_tensor(tn(LLM_TENSOR_ATTN_VE_GATE, "weight", i), {n_embd, n_head_kv}, TENSOR_NOT_REQUIRED); + layer.attn_xsa_alpha = create_tensor(tn(LLM_TENSOR_ATTN_XSA_ALPHA, "weight", i), {n_head}, TENSOR_NOT_REQUIRED); + + layer.attn_resid_scale = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE, "weight", i), {1}, 0); + layer.ffn_resid_scale = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE, "weight", i), {1}, 0); + + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + + if (n_taps > 0 && hparams.mudd_tap_idx[i*n_taps] >= 0) { + // rows [0, n_taps) mix the attention input, rows [n_taps, 2*n_taps) mix the residual + layer.mudd_down = create_tensor(tn(LLM_TENSOR_MUDD_DOWN, "weight", i), {n_embd, n_mudd}, 0); + layer.mudd_up = create_tensor(tn(LLM_TENSOR_MUDD_UP, "weight", i), {n_mudd, 2*n_taps}, 0); + layer.mudd_up_b = create_tensor(tn(LLM_TENSOR_MUDD_UP, "bias", i), {2*n_taps}, 0); + } + } +} + +std::unique_ptr llama_model_limite::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_limite::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_k(); + const int64_t n_taps = hparams.n_mudd_taps; + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_v()); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // value embeddings are looked up by token id, embeddings input is not supported + ggml_tensor * inp_ve = ggml_get_rows(ctx0, model.value_embd, res->t_inp_tokens); + inp_ve = ggml_reshape_3d(ctx0, inp_ve, n_embd_head, n_head_kv, n_tokens); + cb(inp_ve, "inp_ve", -1); + + inpL = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1); + cb(inpL, "inp_norm", -1); + + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // hist[il] is the input of layer il, read by the dense connections + std::vector hist(n_layer, nullptr); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + hist[il] = inpL; + + ggml_tensor * resid = inpL; + + if (layer.mudd_up) { + const int32_t * taps = hparams.mudd_tap_idx.data() + il*n_taps; + + ggml_tensor * w = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, il); + w = ggml_gelu_erf(ctx0, build_lora_mm(layer.mudd_down, w)); + w = ggml_add(ctx0, build_lora_mm(layer.mudd_up, w), layer.mudd_up_b); + cb(w, "mudd_w", il); + + ggml_tensor * stack = nullptr; + for (int64_t j = 0; j < n_taps; ++j) { + GGML_ASSERT(taps[j] >= 0 && taps[j] <= il); + ggml_tensor * h = ggml_reshape_3d(ctx0, hist[taps[j]], n_embd, 1, n_tokens); + stack = stack ? ggml_concat(ctx0, stack, h, 1) : h; + } + + auto mix = [&](size_t offset) { + if (n_taps == 4) { + return ggml_dsv4_hc_pre(ctx0, stack, ggml_cont(ctx0, ggml_view_2d(ctx0, w, n_taps, n_tokens, w->nb[1], offset))); + } + ggml_tensor * wj = ggml_view_3d(ctx0, w, 1, n_taps, n_tokens, w->nb[0], w->nb[1], offset); + ggml_tensor * m = ggml_cont(ctx0, ggml_permute(ctx0, ggml_mul(ctx0, stack, wj), 1, 0, 2, 3)); + return ggml_reshape_2d(ctx0, ggml_sum_rows(ctx0, m), n_embd, n_tokens); + }; + + cur = mix(0); + resid = mix(n_taps*w->nb[0]); + cb(cur, "mudd_attn", il); + cb(resid, "mudd_resid", il); + } else { + cur = inpL; + } + + ggml_tensor * attn_in = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, il); + cb(attn_in, "attn_norm", il); + + // self-attention + { + auto [Qcur, Kcur, Vcur] = build_qkv(layer, attn_in, n_embd_head, n_head, n_head_kv, il); + + if (layer.attn_ve_gate) { + ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(layer.attn_ve_gate, attn_in)); + gate = ggml_reshape_3d(ctx0, gate, 1, n_head_kv, n_tokens); + + Vcur = ggml_add(ctx0, Vcur, ggml_mul(ctx0, inp_ve, gate)); + cb(Vcur, "Vcur_ve", il); + } + + Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps); + Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps); + cb(Qcur, "Qcur_normed", il); + cb(Kcur, "Kcur_normed", il); + + // the full attention layers use no positional encoding + if (hparams.is_swa(il)) { + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + } + + cur = build_attn(inp_attn, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, hparams.f_attention_scale, il); + cb(cur, "attn_raw", il); + + if (layer.attn_xsa_alpha) { + // remove from each head output its projection on the token's own value + const int64_t n_group = n_head/n_head_kv; + + ggml_tensor * y = ggml_reshape_4d(ctx0, cur, n_embd_head, n_group, n_head_kv, n_tokens); + ggml_tensor * vn = ggml_l2_norm(ctx0, Vcur, hparams.f_xsa_eps); + vn = ggml_reshape_4d(ctx0, vn, n_embd_head, 1, n_head_kv, n_tokens); + vn = ggml_repeat(ctx0, vn, y); + + ggml_tensor * proj = ggml_sum_rows(ctx0, ggml_mul(ctx0, y, vn)); + proj = ggml_mul(ctx0, proj, ggml_reshape_4d(ctx0, layer.attn_xsa_alpha, 1, n_group, n_head_kv, 1)); + + cur = ggml_sub(ctx0, y, ggml_mul(ctx0, vn, proj)); + cb(cur, "attn_xsa", il); + } + + cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head, n_tokens); + + if (layer.wqkv_gate) { + ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, attn_in)); + cur = ggml_mul(ctx0, cur, ggml_reshape_3d(ctx0, gate, 1, n_head, n_tokens)); + cb(cur, "attn_gated", il); + } + + cur = ggml_reshape_2d(ctx0, cur, n_embd_head*n_head, n_tokens); + cur = build_lora_mm(layer.wo, cur, layer.wo_s); + cb(cur, "attn_out", il); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + resid = ggml_get_rows(ctx0, resid, inp_out_ids); + } + + cur = ggml_add(ctx0, ggml_mul(ctx0, resid, layer.attn_resid_scale), cur); + cb(cur, "attn_resid", il); + + ggml_tensor * ffn_inp = cur; + + cur = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + layer.ffn_up, nullptr, nullptr, + layer.ffn_gate, nullptr, nullptr, + layer.ffn_down, nullptr, nullptr, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, ggml_mul(ctx0, ffn_inp, layer.ffn_resid_scale), cur); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + + // scale * sigmoid((x + shift) / temperature) + const auto & cap = hparams.final_logit_sigmoid_capping; + if (cap[0] != 0.0f && cap[2] != 0.0f) { + cur = ggml_scale_bias(ctx0, cur, 1.0f/cap[2], cap[1]/cap[2]); + cur = ggml_sigmoid(ctx0, cur); + cur = ggml_scale(ctx0, cur, cap[0]); + } + + cb(cur, "result_output", -1); + res->t_logits = cur; +} diff --git a/src/models/models.h b/src/models/models.h index 6c938c458..0fe40f4d0 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2056,6 +2056,19 @@ struct llama_model_plm : public llama_model_base { }; +struct llama_model_limite : public llama_model_base { + llama_model_limite(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_hrm_text : public llama_model_base { llama_model_hrm_text(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index aff6a7c90..333551157 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -269,6 +269,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_embd = 160; // exercise per-head tensor split granularity with head size 80 } else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) { n_head = 4; + } else if (arch == LLM_ARCH_LIMITE) { + n_head = 4; + n_layer = 4; // dense connections on the last two layers } else if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA @@ -297,7 +300,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_QWEN3) { n_head_kv = 1; // MQA coverage } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER || - arch == LLM_ARCH_KOLIBRI1) { + arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) { n_head_kv = 2; // GQA coverage } if (arch == LLM_ARCH_BEAM) { @@ -458,7 +461,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); } - if (arch == LLM_ARCH_KOLIBRI1) { + if (arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) { ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, false); } @@ -594,6 +597,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f); } + if (arch == LLM_ARCH_LIMITE) { + ms.add_kv(LLM_KV_ATTENTION_SCALE, 0.1f); + ms.add_kv(LLM_KV_ATTENTION_XSA_EPS, 1e-4f); + ms.add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector({23.0f, 5.0f, 7.5f})); + ms.add_kv(LLM_KV_MUDD_FEED_FORWARD_LENGTH, uint32_t(8)); + ms.add_kv(LLM_KV_MUDD_TAP_COUNT, uint32_t(2)); + ms.add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector({-1, -1, -1, -1, 0, 2, 1, 3})); + } + // dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids { std::vector tokenizer_list(n_vocab);