model : Support Limite 1B - Violetto (#29433)
* gguf-py : add Limite arch Assisted-by: Claude Opus 5.5 Signed-off-by: Giuseppe Scrivano <giuseppe@scrivano.org> * convert : add Limite support Assisted-by: Claude Opus 5.5 Signed-off-by: Giuseppe Scrivano <giuseppe@scrivano.org> * llama : add Limite support Assisted-by: Claude Opus 5.5 Signed-off-by: Giuseppe Scrivano <giuseppe@scrivano.org> --------- Signed-off-by: Giuseppe Scrivano <giuseppe@scrivano.org>
This commit is contained in:
Giuseppe Scrivano authored and 
GitHub committed
1 parent
41be0bdc98
commit
abb76ab380
15 files changed
+598
-2
No files matched your search
@@ -176,6 +176,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Lfm3ForCausalLM": "lfm3",
|
||||
"Lfm3OmniForConditionalGeneration": "lfm3",
|
||||
"Lfm3VlForConditionalGeneration": "lfm3",
|
||||
"LimiteForCausalLM": "limite",
|
||||
"Llama4ForCausalLM": "llama",
|
||||
"Llama4ForConditionalGeneration": "llama",
|
||||
"LlamaBidirectionalModel": "llama",
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("LimiteForCausalLM")
|
||||
@ModelBase.example("paradigma-inc/limite-1b-violetto")
|
||||
class LimiteModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LIMITE
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
hp = self.hparams
|
||||
|
||||
expected = {
|
||||
"pos_mode": "rope",
|
||||
"qk_norm": "rms_pre_rope",
|
||||
"rope_style": "interleaved_pairs_odd_lane_sign_flip",
|
||||
"rope_per_layer": False,
|
||||
"rms_norm_has_weight": False,
|
||||
"rms_norm_eps_mode": "torch_finfo_default",
|
||||
"mlp_type": "swiglu",
|
||||
"attn_gate_applied": "per_head_before_o_proj",
|
||||
"ve_head_slice": "first_num_key_value_heads",
|
||||
"ve_applied_before_qk_norm": True,
|
||||
"global_nope": True,
|
||||
"global_window": -1,
|
||||
"mudd_r_site": "resid",
|
||||
"tie_word_embeddings": True,
|
||||
"final_softcap": 0.0,
|
||||
}
|
||||
for key, value in expected.items():
|
||||
if hp.get(key) != value:
|
||||
raise NotImplementedError(f"Limite: unsupported {key}={hp.get(key)!r}, expected {value!r}")
|
||||
if hp["softcap_logits"]["kind"] != "sigmoid":
|
||||
raise NotImplementedError(f"Limite: unsupported softcap_logits kind {hp['softcap_logits']['kind']!r}")
|
||||
|
||||
self.n_head = hp["num_attention_heads"]
|
||||
self.n_head_kv = hp["num_key_value_heads"]
|
||||
self.head_dim = hp["head_dim"]
|
||||
self.n_pairs = hp["rope_n_pairs"]
|
||||
|
||||
self.mudd_taps: dict[int, list[int]] = {}
|
||||
if hp.get("mudd"):
|
||||
if not hp.get("mudd_mlp"):
|
||||
raise NotImplementedError("Limite: mudd without mudd_mlp is not supported")
|
||||
self.mudd_taps = {int(il): [int(t) for t in taps] for il, taps in hp["mudd_tap_idx"].items()}
|
||||
assert all(len(taps) == hp["mudd_taps"] for taps in self.mudd_taps.values())
|
||||
|
||||
def set_vocab(self):
|
||||
self._set_vocab_gpt2()
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
hp = self.hparams
|
||||
|
||||
self.gguf_writer.add_key_length(self.head_dim)
|
||||
self.gguf_writer.add_value_length(self.head_dim)
|
||||
|
||||
# torch rms_norm with eps=None uses the fp32 epsilon, even for bf16 inputs
|
||||
self.gguf_writer.add_layer_norm_rms_eps(torch.finfo(torch.float32).eps)
|
||||
self.gguf_writer.add_attention_scale(hp["attention_softmax_scale"])
|
||||
self.gguf_writer.add_xsa_eps(hp["xsa_normalize_eps"] if hp.get("xsa") else 0.0)
|
||||
|
||||
# HF freq_i = base^(-i/(n_pairs-1)), ggml uses freq_base^(-i/n_pairs)
|
||||
self.gguf_writer.add_rope_dimension_count(2 * self.n_pairs)
|
||||
self.gguf_writer.add_rope_freq_base(hp["rope_base_local"] ** (self.n_pairs / (self.n_pairs - 1)))
|
||||
|
||||
# HF keeps keys k >= q - window, so a window of W spans W + 1 tokens
|
||||
global_layers = set(hp["global_layers"])
|
||||
self.gguf_writer.add_sliding_window(hp["sliding_window"] + 1)
|
||||
self.gguf_writer.add_sliding_window_pattern([il not in global_layers for il in range(self.block_count)])
|
||||
|
||||
cap = hp["softcap_logits"]
|
||||
self.gguf_writer.add_final_logit_sigmoid_capping(cap["a"], cap["b"], cap["c"])
|
||||
|
||||
if self.mudd_taps:
|
||||
n_taps = hp["mudd_taps"]
|
||||
tap_indices = [-1] * (self.block_count * n_taps)
|
||||
for il, taps in self.mudd_taps.items():
|
||||
tap_indices[il * n_taps:(il + 1) * n_taps] = taps
|
||||
self.gguf_writer.add_mudd_feed_forward_length(hp["mudd_inter"])
|
||||
self.gguf_writer.add_mudd_tap_count(n_taps)
|
||||
self.gguf_writer.add_mudd_tap_indices(tap_indices)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# tiny mixing weights, kept at full precision as in the reference
|
||||
# the mudd tensors come from a global source tensor, so take the block id from the new name
|
||||
if new_name.startswith("blk."):
|
||||
bid = int(new_name.split(".")[1])
|
||||
for key in (gguf.MODEL_TENSOR.ATTN_GATE, gguf.MODEL_TENSOR.ATTN_VE_GATE, gguf.MODEL_TENSOR.MUDD_DOWN, gguf.MODEL_TENSOR.MUDD_UP):
|
||||
if self.match_model_tensor_name(new_name, key, bid):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def _swap_rope_pairs(self, w: Tensor, n_head: int) -> Tensor:
|
||||
# HF rotates each pair by -theta; swapping the pair lanes gives the ggml +theta rotation.
|
||||
# q and k are both swapped, so the attention scores do not change.
|
||||
idx = torch.arange(self.head_dim)
|
||||
idx[:2 * self.n_pairs] = idx[:2 * self.n_pairs].view(-1, 2).flip(-1).reshape(-1)
|
||||
w = w.reshape(n_head, self.head_dim, -1)
|
||||
return w[:, idx, :].reshape(n_head * self.head_dim, -1)
|
||||
|
||||
def _pad_cols(self, w: Tensor) -> Tensor:
|
||||
# gates read only the first channels of the input, pad them to n_embd
|
||||
return torch.nn.functional.pad(w, (0, self.hparams["hidden_size"] - w.shape[-1]))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
hp = self.hparams
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def load(n: str) -> Tensor:
|
||||
return self.model_tensors[n]().float()
|
||||
|
||||
if name == "model.value_embeds.weight":
|
||||
ve = data_torch.reshape(data_torch.shape[0], -1, hp["ve_dim"])[:, :self.n_head_kv]
|
||||
ve = torch.nn.functional.pad(ve, (0, self.head_dim - hp["ve_dim"]))
|
||||
yield self.format_tensor_name(T.VALUE_EMBD), ve.reshape(ve.shape[0], -1) * hp["ve_gate_scale"]
|
||||
return
|
||||
|
||||
if name == "model.mudd.dense1":
|
||||
for il in self.mudd_taps:
|
||||
yield self.format_tensor_name(T.MUDD_DOWN, il), data_torch
|
||||
return
|
||||
if name == "model.mudd.dense2":
|
||||
# rows [0, n) mix the attention input, rows [n, 2n) mix the residual
|
||||
dense2_mlp = load("model.mudd.dense2_mlp")
|
||||
bias = load("model.mudd.bias")
|
||||
bias_mlp = load("model.mudd.bias_mlp")
|
||||
for il, taps in self.mudd_taps.items():
|
||||
n = len(taps)
|
||||
w = torch.cat([data_torch[il, :n], dense2_mlp[il, :n]], 0)
|
||||
b = torch.cat([bias[il, :n], bias_mlp[il, :n]], 0)
|
||||
yield self.format_tensor_name(T.MUDD_UP, il), w
|
||||
yield self.format_tensor_name(T.MUDD_UP, il, ".bias"), b
|
||||
return
|
||||
if name.startswith("model.mudd."):
|
||||
return
|
||||
|
||||
if bid is None:
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
return
|
||||
|
||||
prefix = f"model.layers.{bid}."
|
||||
name = name.removeprefix(prefix)
|
||||
|
||||
if name in ("self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight"):
|
||||
data_torch = data_torch * load(prefix + "self_attn.qkv_scale")
|
||||
if name == "self_attn.q_proj.weight":
|
||||
data_torch = self._swap_rope_pairs(data_torch, self.n_head)
|
||||
elif name == "self_attn.k_proj.weight":
|
||||
data_torch = self._swap_rope_pairs(data_torch, self.n_head_kv)
|
||||
elif name == "self_attn.o_proj.weight":
|
||||
data_torch = data_torch * load(prefix + "self_attn.o_scale") * load(prefix + "post_lambda_attn") * hp["attn_gate_scale"]
|
||||
elif name == "mlp.down_proj.weight":
|
||||
data_torch = data_torch * load(prefix + "post_lambda_mlp")
|
||||
elif name == "self_attn.attn_gate":
|
||||
yield self.format_tensor_name(T.ATTN_GATE, bid), self._pad_cols(data_torch)
|
||||
return
|
||||
elif name == "self_attn.ve_gate":
|
||||
yield self.format_tensor_name(T.ATTN_VE_GATE, bid), self._pad_cols(data_torch[:self.n_head_kv])
|
||||
return
|
||||
elif name == "self_attn.xsa_alpha":
|
||||
if hp.get("xsa") and bid in hp["xsa_layers"]:
|
||||
yield self.format_tensor_name(T.ATTN_XSA_ALPHA, bid), torch.tanh(data_torch)
|
||||
return
|
||||
elif name == "resid_lambda_attn":
|
||||
yield self.format_tensor_name(T.ATTN_RESID_SCALE, bid), data_torch.reshape(1)
|
||||
return
|
||||
elif name == "resid_lambda_mlp":
|
||||
yield self.format_tensor_name(T.FFN_RESID_SCALE, bid), data_torch.reshape(1)
|
||||
return
|
||||
elif name in ("self_attn.qkv_scale", "self_attn.o_scale", "post_lambda_attn", "post_lambda_mlp"):
|
||||
# folded into the projections above
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, prefix + name, bid)
|
||||
@@ -143,6 +143,7 @@ class Keys:
|
||||
ATTN_LOGIT_SOFTCAPPING = "{arch}.attn_logit_softcapping"
|
||||
ROUTER_LOGIT_SOFTCAPPING = "{arch}.router_logit_softcapping"
|
||||
FINAL_LOGIT_SOFTCAPPING = "{arch}.final_logit_softcapping"
|
||||
FINAL_LOGIT_SIGMOID_CAPPING = "{arch}.final_logit_sigmoid_capping"
|
||||
SWIN_NORM = "{arch}.swin_norm"
|
||||
RESCALE_EVERY_N_LAYERS = "{arch}.rescale_every_n_layers"
|
||||
TIME_MIX_EXTRA_DIM = "{arch}.time_mix_extra_dim"
|
||||
@@ -227,6 +228,7 @@ class Keys:
|
||||
ROPE_PATTERN = "{arch}.attention.rope_pattern"
|
||||
VALUE_EXPERT_COUNT = "{arch}.attention.value_expert_count"
|
||||
VALUE_EXPERT_USED_COUNT = "{arch}.attention.value_expert_used_count"
|
||||
XSA_EPS = "{arch}.attention.xsa_epsilon"
|
||||
|
||||
class Indexer:
|
||||
HEAD_COUNT = "{arch}.attention.indexer.head_count"
|
||||
@@ -301,6 +303,11 @@ class Keys:
|
||||
LLM_KV_SPLIT_COUNT = "split.count"
|
||||
LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"
|
||||
|
||||
class MUDD:
|
||||
FEED_FORWARD_LENGTH = "{arch}.mudd.feed_forward_length"
|
||||
TAP_COUNT = "{arch}.mudd.tap_count"
|
||||
TAP_INDICES = "{arch}.mudd.tap_indices"
|
||||
|
||||
class HRM:
|
||||
LAYERS_PER_STACK = "{arch}.hrm.layers_per_stack"
|
||||
H_CYCLES = "{arch}.hrm.h_cycles"
|
||||
@@ -731,6 +738,7 @@ class MODEL_ARCH(IntEnum):
|
||||
POCKETTTS = auto()
|
||||
K2HORIZON = auto()
|
||||
KOLIBRI1 = auto()
|
||||
LIMITE = auto()
|
||||
|
||||
|
||||
class VISION_PROJECTOR_TYPE(IntEnum):
|
||||
@@ -865,6 +873,13 @@ class MODEL_TENSOR(IntEnum):
|
||||
FFN_ROUTED_DOWN = auto() # Kimi K3 (latent MoE: hidden -> latent)
|
||||
FFN_ROUTED_UP = auto() # Kimi K3 (latent MoE: latent -> hidden)
|
||||
FFN_ROUTED_NORM = auto() # Kimi K3 (latent MoE: norm on expert output)
|
||||
VALUE_EMBD = auto() # limite
|
||||
ATTN_VE_GATE = auto() # limite
|
||||
ATTN_XSA_ALPHA = auto() # limite
|
||||
ATTN_RESID_SCALE = auto() # limite
|
||||
FFN_RESID_SCALE = auto() # limite
|
||||
MUDD_DOWN = auto() # limite
|
||||
MUDD_UP = auto() # limite
|
||||
TIME_MIX_W0 = auto()
|
||||
TIME_MIX_W1 = auto()
|
||||
TIME_MIX_W2 = auto()
|
||||
@@ -1577,6 +1592,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
|
||||
MODEL_ARCH.POCKETTTS: "pockettts",
|
||||
MODEL_ARCH.K2HORIZON: "k2-horizon",
|
||||
MODEL_ARCH.KOLIBRI1: "kolibri1",
|
||||
MODEL_ARCH.LIMITE: "limite",
|
||||
}
|
||||
|
||||
VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
|
||||
@@ -1709,6 +1725,13 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.FFN_ROUTED_DOWN: "blk.{bid}.ffn_routed_down", # Kimi K3
|
||||
MODEL_TENSOR.FFN_ROUTED_UP: "blk.{bid}.ffn_routed_up", # Kimi K3
|
||||
MODEL_TENSOR.FFN_ROUTED_NORM: "blk.{bid}.ffn_routed_norm", # Kimi K3
|
||||
MODEL_TENSOR.VALUE_EMBD: "value_embd", # limite
|
||||
MODEL_TENSOR.ATTN_VE_GATE: "blk.{bid}.attn_ve_gate", # limite
|
||||
MODEL_TENSOR.ATTN_XSA_ALPHA: "blk.{bid}.attn_xsa_alpha", # limite
|
||||
MODEL_TENSOR.ATTN_RESID_SCALE: "blk.{bid}.attn_resid_scale", # limite
|
||||
MODEL_TENSOR.FFN_RESID_SCALE: "blk.{bid}.ffn_resid_scale", # limite
|
||||
MODEL_TENSOR.MUDD_DOWN: "blk.{bid}.mudd_down", # limite
|
||||
MODEL_TENSOR.MUDD_UP: "blk.{bid}.mudd_up", # limite
|
||||
MODEL_TENSOR.TIME_MIX_W0: "blk.{bid}.time_mix_w0",
|
||||
MODEL_TENSOR.TIME_MIX_W1: "blk.{bid}.time_mix_w1",
|
||||
MODEL_TENSOR.TIME_MIX_W2: "blk.{bid}.time_mix_w2",
|
||||
@@ -6288,6 +6311,25 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.FFN_UP_SHEXP,
|
||||
MODEL_TENSOR.FFN_DOWN_SHEXP,
|
||||
],
|
||||
MODEL_ARCH.LIMITE: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.VALUE_EMBD,
|
||||
MODEL_TENSOR.OUTPUT,
|
||||
MODEL_TENSOR.ATTN_Q,
|
||||
MODEL_TENSOR.ATTN_K,
|
||||
MODEL_TENSOR.ATTN_V,
|
||||
MODEL_TENSOR.ATTN_OUT,
|
||||
MODEL_TENSOR.ATTN_GATE,
|
||||
MODEL_TENSOR.ATTN_VE_GATE,
|
||||
MODEL_TENSOR.ATTN_XSA_ALPHA,
|
||||
MODEL_TENSOR.ATTN_RESID_SCALE,
|
||||
MODEL_TENSOR.FFN_RESID_SCALE,
|
||||
MODEL_TENSOR.FFN_GATE,
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
MODEL_TENSOR.MUDD_DOWN,
|
||||
MODEL_TENSOR.MUDD_UP,
|
||||
],
|
||||
}
|
||||
|
||||
# tensors that will not be serialized
|
||||
|
||||
@@ -876,6 +876,9 @@ class GGUFWriter:
|
||||
def add_final_logit_softcapping(self, value: float) -> None:
|
||||
self.add_float32(Keys.LLM.FINAL_LOGIT_SOFTCAPPING.format(arch=self.arch), value)
|
||||
|
||||
def add_final_logit_sigmoid_capping(self, scale: float, shift: float, temperature: float) -> None:
|
||||
self.add_array(Keys.LLM.FINAL_LOGIT_SIGMOID_CAPPING.format(arch=self.arch), [scale, shift, temperature])
|
||||
|
||||
def add_expert_count(self, count: int) -> None:
|
||||
self.add_uint32(Keys.LLM.EXPERT_COUNT.format(arch=self.arch), count)
|
||||
|
||||
@@ -948,6 +951,15 @@ class GGUFWriter:
|
||||
def add_embedding_scale(self, value: float) -> None:
|
||||
self.add_float32(Keys.LLM.EMBEDDING_SCALE.format(arch=self.arch), value)
|
||||
|
||||
def add_mudd_feed_forward_length(self, value: int) -> None:
|
||||
self.add_uint32(Keys.MUDD.FEED_FORWARD_LENGTH.format(arch=self.arch), value)
|
||||
|
||||
def add_mudd_tap_count(self, value: int) -> None:
|
||||
self.add_uint32(Keys.MUDD.TAP_COUNT.format(arch=self.arch), value)
|
||||
|
||||
def add_mudd_tap_indices(self, value: Sequence[int]) -> None:
|
||||
self.add_array(Keys.MUDD.TAP_INDICES.format(arch=self.arch), value)
|
||||
|
||||
def add_hrm_layers_per_stack(self, value: int) -> None:
|
||||
self.add_uint32(Keys.HRM.LAYERS_PER_STACK.format(arch=self.arch), value)
|
||||
|
||||
@@ -1151,6 +1163,9 @@ class GGUFWriter:
|
||||
def add_attention_scale(self, value: float) -> None:
|
||||
self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value)
|
||||
|
||||
def add_xsa_eps(self, value: float) -> None:
|
||||
self.add_float32(Keys.Attention.XSA_EPS.format(arch=self.arch), value)
|
||||
|
||||
def add_attn_output_scale(self, value: float) -> None:
|
||||
self.add_float32(Keys.Attention.OUTPUT_SCALE.format(arch=self.arch), value)
|
||||
|
||||
|
||||
@@ -145,6 +145,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
|
||||
{ LLM_ARCH_APERTUS, "apertus" },
|
||||
{ LLM_ARCH_MINIMAX_01, "minimax-01" },
|
||||
{ LLM_ARCH_HRM_TEXT, "hrm_text" },
|
||||
{ LLM_ARCH_LIMITE, "limite" },
|
||||
{ LLM_ARCH_MINIMAX_M2, "minimax-m2" },
|
||||
{ LLM_ARCH_MINIMAX_M3, "minimax-m3" },
|
||||
{ LLM_ARCH_COGVLM, "cogvlm" },
|
||||
@@ -248,6 +249,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_ATTN_LOGIT_SOFTCAPPING, "%s.attn_logit_softcapping" },
|
||||
{ LLM_KV_ROUTER_LOGIT_SOFTCAPPING, "%s.router_logit_softcapping" },
|
||||
{ LLM_KV_FINAL_LOGIT_SOFTCAPPING, "%s.final_logit_softcapping" },
|
||||
{ LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, "%s.final_logit_sigmoid_capping" },
|
||||
{ LLM_KV_SWIN_NORM, "%s.swin_norm" },
|
||||
{ LLM_KV_RESCALE_EVERY_N_LAYERS, "%s.rescale_every_n_layers" },
|
||||
{ LLM_KV_TIME_MIX_EXTRA_DIM, "%s.time_mix_extra_dim" },
|
||||
@@ -268,6 +270,9 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" },
|
||||
{ LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" },
|
||||
{ LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" },
|
||||
{ LLM_KV_MUDD_FEED_FORWARD_LENGTH, "%s.mudd.feed_forward_length" },
|
||||
{ LLM_KV_MUDD_TAP_COUNT, "%s.mudd.tap_count" },
|
||||
{ LLM_KV_MUDD_TAP_INDICES, "%s.mudd.tap_indices" },
|
||||
|
||||
{ LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },
|
||||
{ LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },
|
||||
@@ -292,6 +297,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_ATTENTION_ROPE_PATTERN, "%s.attention.rope_pattern" },
|
||||
{ LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, "%s.attention.value_expert_count" },
|
||||
{ LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, "%s.attention.value_expert_used_count" },
|
||||
{ LLM_KV_ATTENTION_XSA_EPS, "%s.attention.xsa_epsilon" },
|
||||
|
||||
{ LLM_KV_ATTENTION_SCALE, "%s.attention.scale" },
|
||||
{ LLM_KV_ATTENTION_OUTPUT_SCALE, "%s.attention.output_scale" },
|
||||
@@ -605,6 +611,13 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
|
||||
{ LLM_TENSOR_FFN_ROUTED_DOWN, "blk.%d.ffn_routed_down" },
|
||||
{ LLM_TENSOR_FFN_ROUTED_UP, "blk.%d.ffn_routed_up" },
|
||||
{ LLM_TENSOR_FFN_ROUTED_NORM, "blk.%d.ffn_routed_norm" },
|
||||
{ LLM_TENSOR_VALUE_EMBD, "value_embd" },
|
||||
{ LLM_TENSOR_ATTN_VE_GATE, "blk.%d.attn_ve_gate" },
|
||||
{ LLM_TENSOR_ATTN_XSA_ALPHA, "blk.%d.attn_xsa_alpha" },
|
||||
{ LLM_TENSOR_ATTN_RESID_SCALE, "blk.%d.attn_resid_scale" },
|
||||
{ LLM_TENSOR_FFN_RESID_SCALE, "blk.%d.ffn_resid_scale" },
|
||||
{ LLM_TENSOR_MUDD_DOWN, "blk.%d.mudd_down" },
|
||||
{ LLM_TENSOR_MUDD_UP, "blk.%d.mudd_up" },
|
||||
{ LLM_TENSOR_SSM_G_B, "blk.%d.ssm_g_b" },
|
||||
{ LLM_TENSOR_SSM_NORM, "blk.%d.ssm_norm" },
|
||||
{ LLM_TENSOR_ATTN_Q_A_NORM, "blk.%d.attn_q_a_norm" },
|
||||
@@ -998,6 +1011,13 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
|
||||
{LLM_TENSOR_FFN_ROUTED_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_FFN_ROUTED_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_FFN_ROUTED_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_VALUE_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
|
||||
{LLM_TENSOR_ATTN_VE_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_ATTN_XSA_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_ATTN_RESID_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_FFN_RESID_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_MUDD_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_MUDD_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_SSM_G_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_TIME_MIX_LERP_X, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_TIME_MIX_LN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
@@ -1315,6 +1335,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) {
|
||||
case LLM_ARCH_EAGLE3:
|
||||
case LLM_ARCH_DFLASH:
|
||||
case LLM_ARCH_GEMMA4_ASSISTANT:
|
||||
case LLM_ARCH_LIMITE:
|
||||
return false;
|
||||
default:
|
||||
return true;
|
||||
@@ -1357,6 +1378,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
|
||||
case LLM_ARCH_KIMI_K3:
|
||||
case LLM_ARCH_GLM5_NEXT:
|
||||
case LLM_ARCH_QWEN3TTS:
|
||||
case LLM_ARCH_LIMITE:
|
||||
return false;
|
||||
default:
|
||||
return true;
|
||||
|
||||
@@ -176,6 +176,7 @@ enum llm_arch {
|
||||
LLM_ARCH_K2_HORIZON,
|
||||
LLM_ARCH_BEAM,
|
||||
LLM_ARCH_KOLIBRI1,
|
||||
LLM_ARCH_LIMITE,
|
||||
LLM_ARCH_UNKNOWN,
|
||||
};
|
||||
|
||||
@@ -252,6 +253,7 @@ enum llm_kv {
|
||||
LLM_KV_ATTN_LOGIT_SOFTCAPPING,
|
||||
LLM_KV_ROUTER_LOGIT_SOFTCAPPING,
|
||||
LLM_KV_FINAL_LOGIT_SOFTCAPPING,
|
||||
LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING,
|
||||
LLM_KV_SWIN_NORM,
|
||||
LLM_KV_RESCALE_EVERY_N_LAYERS,
|
||||
LLM_KV_TIME_MIX_EXTRA_DIM,
|
||||
@@ -273,6 +275,9 @@ enum llm_kv {
|
||||
LLM_KV_HRM_L_CYCLES,
|
||||
LLM_KV_HRM_PREFIX_LM,
|
||||
LLM_KV_REALTIME_DELAY_TOKENS,
|
||||
LLM_KV_MUDD_FEED_FORWARD_LENGTH,
|
||||
LLM_KV_MUDD_TAP_COUNT,
|
||||
LLM_KV_MUDD_TAP_INDICES,
|
||||
|
||||
LLM_KV_ATTENTION_HEAD_COUNT,
|
||||
LLM_KV_ATTENTION_HEAD_COUNT_KV,
|
||||
@@ -298,6 +303,7 @@ enum llm_kv {
|
||||
LLM_KV_ATTENTION_ROPE_PATTERN,
|
||||
LLM_KV_ATTENTION_VALUE_EXPERT_COUNT,
|
||||
LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT,
|
||||
LLM_KV_ATTENTION_XSA_EPS,
|
||||
|
||||
LLM_KV_ATTENTION_OUTPUT_SCALE,
|
||||
LLM_KV_ATTENTION_VALUE_SCALE,
|
||||
@@ -612,6 +618,13 @@ enum llm_tensor {
|
||||
LLM_TENSOR_FFN_ROUTED_DOWN, // kimi-k3: latent MoE down
|
||||
LLM_TENSOR_FFN_ROUTED_UP, // kimi-k3: latent MoE up
|
||||
LLM_TENSOR_FFN_ROUTED_NORM, // kimi-k3: latent MoE norm
|
||||
LLM_TENSOR_VALUE_EMBD, // limite: value embeddings
|
||||
LLM_TENSOR_ATTN_VE_GATE, // limite: value embedding gate
|
||||
LLM_TENSOR_ATTN_XSA_ALPHA, // limite: exclusive self attention
|
||||
LLM_TENSOR_ATTN_RESID_SCALE, // limite: residual scale (attn)
|
||||
LLM_TENSOR_FFN_RESID_SCALE, // limite: residual scale (ffn)
|
||||
LLM_TENSOR_MUDD_DOWN, // limite: dense connection mixer
|
||||
LLM_TENSOR_MUDD_UP, // limite: dense connection mixer
|
||||
LLM_TENSOR_TIME_MIX_W0,
|
||||
LLM_TENSOR_TIME_MIX_W1,
|
||||
LLM_TENSOR_TIME_MIX_W2,
|
||||
|
||||
@@ -227,6 +227,13 @@ struct llama_hparams {
|
||||
uint32_t n_hrm_l_cycles = 0;
|
||||
bool hrm_prefix_lm = false;
|
||||
|
||||
// limite
|
||||
float f_xsa_eps = 0.0f;
|
||||
std::array<float, 3> final_logit_sigmoid_capping = {}; // scale, shift, temperature
|
||||
uint32_t n_mudd_ff = 0;
|
||||
uint32_t n_mudd_taps = 0;
|
||||
std::array<int32_t, LLAMA_MAX_LAYERS> mudd_tap_idx; // [n_layer][n_mudd_taps], -1 = no mixing
|
||||
|
||||
bool ssm_dt_b_c_rms = false;
|
||||
|
||||
float f_clamp_kqv = 0.0f;
|
||||
|
||||
@@ -411,6 +411,7 @@ namespace GGUFMeta {
|
||||
|
||||
template bool llama_model_loader::get_arr<std::vector<std::string>>(enum llm_kv kid, std::vector<std::string> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::array<int32_t, 512>>(enum llm_kv kid, std::array<int32_t, 512> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::array<float, 3>>(enum llm_kv kid, std::array<float, 3> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::vector<int32_t>>(enum llm_kv kid, std::vector<int32_t> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::array<uint32_t, LLAMA_MAX_LAYERS>>(enum llm_kv kid, std::array<uint32_t, LLAMA_MAX_LAYERS> & result, bool required);
|
||||
template bool llama_model_loader::get_arr<std::vector<uint32_t>>(enum llm_kv kid, std::vector<uint32_t> & result, bool required);
|
||||
|
||||
@@ -244,6 +244,16 @@ void llama_model_saver::add_kv_from_model() {
|
||||
add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles);
|
||||
add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles);
|
||||
add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm);
|
||||
if (hparams.final_logit_sigmoid_capping[0] != 0.0f) {
|
||||
add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>(
|
||||
hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end()));
|
||||
}
|
||||
add_kv(LLM_KV_MUDD_FEED_FORWARD_LENGTH, hparams.n_mudd_ff);
|
||||
add_kv(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps);
|
||||
if (hparams.n_mudd_taps > 0) {
|
||||
add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector<int32_t>(
|
||||
hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.begin() + hparams.n_layer()*hparams.n_mudd_taps));
|
||||
}
|
||||
add_kv(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count);
|
||||
add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step);
|
||||
// add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, ???); // saved as LLM_KV_ATTENTION_RECURRENT_LAYERS instead
|
||||
@@ -277,6 +287,7 @@ void llama_model_saver::add_kv_from_model() {
|
||||
hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all));
|
||||
}
|
||||
add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);
|
||||
add_kv(LLM_KV_ATTENTION_XSA_EPS, hparams.f_xsa_eps);
|
||||
add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale);
|
||||
add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale);
|
||||
add_kv(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length);
|
||||
@@ -501,6 +512,7 @@ void llama_model_saver::add_tensors_from_model() {
|
||||
add_tensor(model->hc_head_base);
|
||||
add_tensor(model->hc_head_scale);
|
||||
add_tensor(model->per_layer_tok_embd);
|
||||
add_tensor(model->value_embd);
|
||||
add_tensor(model->hc_head_norm);
|
||||
add_tensor(model->hc_head_down);
|
||||
add_tensor(model->hc_head_up);
|
||||
|
||||
@@ -332,6 +332,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
|
||||
return new llama_model_hrm_text(params);
|
||||
case LLM_ARCH_BEAM:
|
||||
return new llama_model_beam(params);
|
||||
case LLM_ARCH_LIMITE:
|
||||
return new llama_model_limite(params);
|
||||
case LLM_ARCH_COGVLM:
|
||||
return new llama_model_cogvlm(params);
|
||||
case LLM_ARCH_PANGU_EMBED:
|
||||
@@ -3575,6 +3577,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
|
||||
case LLM_ARCH_DOTS3NOTE:
|
||||
case LLM_ARCH_NANBEIGE:
|
||||
case LLM_ARCH_POCKETTTS:
|
||||
case LLM_ARCH_LIMITE:
|
||||
return LLAMA_ROPE_TYPE_NORM;
|
||||
case LLM_ARCH_BAILINGMOE3:
|
||||
// VL files carry mrope sections; text-only files keep NORM rope
|
||||
|
||||
@@ -638,6 +638,15 @@ struct llama_layer {
|
||||
// gemma4 layer output scale, reused for talkie embedding skip scale
|
||||
struct ggml_tensor * out_scale = nullptr;
|
||||
|
||||
// limite
|
||||
struct ggml_tensor * attn_ve_gate = nullptr;
|
||||
struct ggml_tensor * attn_xsa_alpha = nullptr;
|
||||
struct ggml_tensor * attn_resid_scale = nullptr;
|
||||
struct ggml_tensor * ffn_resid_scale = nullptr;
|
||||
struct ggml_tensor * mudd_down = nullptr;
|
||||
struct ggml_tensor * mudd_up = nullptr;
|
||||
struct ggml_tensor * mudd_up_b = nullptr;
|
||||
|
||||
struct llama_layer_posnet posnet;
|
||||
|
||||
struct llama_layer_convnext convnext;
|
||||
@@ -783,6 +792,9 @@ struct llama_model {
|
||||
struct ggml_tensor * altup_unembd_proj = nullptr;
|
||||
struct ggml_tensor * per_layer_tok_embd = nullptr;
|
||||
|
||||
// limite value embeddings
|
||||
struct ggml_tensor * value_embd = nullptr;
|
||||
|
||||
struct ggml_tensor * hc_head_norm = nullptr;
|
||||
struct ggml_tensor * hc_head_down = nullptr;
|
||||
struct ggml_tensor * hc_head_up = nullptr;
|
||||
|
||||
@@ -333,6 +333,13 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
|
||||
// these are not too big so keep them as it is
|
||||
quantize &= name.find("per_layer_model_proj") == std::string::npos;
|
||||
|
||||
// limite: very small gates and dense connection mixers
|
||||
if (arch == LLM_ARCH_LIMITE) {
|
||||
quantize &= name.find("attn_gate.weight") == std::string::npos;
|
||||
quantize &= name.find("attn_ve_gate.weight") == std::string::npos;
|
||||
quantize &= name.find("mudd_") == std::string::npos;
|
||||
}
|
||||
|
||||
// do not quantize positional embeddings and token types (BERT)
|
||||
quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD, "weight");
|
||||
quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, "weight");
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
#include "models.h"
|
||||
|
||||
// Limite: pre-norm transformer with value embeddings, exclusive self attention (XSA),
|
||||
// per-head output gates, learned residual scales and dense (MUDD) cross-layer connections.
|
||||
// ref: https://github.com/paradigma-inc/limite-violetto
|
||||
|
||||
void llama_model_limite::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);
|
||||
ml.get_key(LLM_KV_ATTENTION_XSA_EPS, hparams.f_xsa_eps, false);
|
||||
ml.get_arr(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, hparams.final_logit_sigmoid_capping, false);
|
||||
|
||||
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
||||
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
||||
load_swa_pattern(ml, 4);
|
||||
|
||||
std::fill(hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.end(), -1);
|
||||
ml.get_key(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps, false);
|
||||
if (hparams.n_mudd_taps > 0) {
|
||||
ml.get_key(LLM_KV_MUDD_FEED_FORWARD_LENGTH, hparams.n_mudd_ff);
|
||||
ml.get_arr(LLM_KV_MUDD_TAP_INDICES, hparams.mudd_tap_idx);
|
||||
}
|
||||
|
||||
switch (hparams.n_layer()) {
|
||||
case 48: type = LLM_TYPE_1B; break;
|
||||
default: type = LLM_TYPE_UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
void llama_model_limite::load_arch_tensors(llama_model_loader &) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
value_embd = create_tensor(tn(LLM_TENSOR_VALUE_EMBD, "weight"), {n_embd_v_gqa, n_vocab}, 0);
|
||||
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
|
||||
// if output is NULL, init from the input tok embed
|
||||
if (output == NULL) {
|
||||
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
|
||||
}
|
||||
|
||||
const int64_t n_taps = hparams.n_mudd_taps;
|
||||
const int64_t n_mudd = hparams.n_mudd_ff;
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & layer = layers[i];
|
||||
|
||||
create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
|
||||
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
|
||||
|
||||
layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, TENSOR_NOT_REQUIRED);
|
||||
layer.attn_ve_gate = create_tensor(tn(LLM_TENSOR_ATTN_VE_GATE, "weight", i), {n_embd, n_head_kv}, TENSOR_NOT_REQUIRED);
|
||||
layer.attn_xsa_alpha = create_tensor(tn(LLM_TENSOR_ATTN_XSA_ALPHA, "weight", i), {n_head}, TENSOR_NOT_REQUIRED);
|
||||
|
||||
layer.attn_resid_scale = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE, "weight", i), {1}, 0);
|
||||
layer.ffn_resid_scale = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE, "weight", i), {1}, 0);
|
||||
|
||||
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
|
||||
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
|
||||
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
|
||||
|
||||
if (n_taps > 0 && hparams.mudd_tap_idx[i*n_taps] >= 0) {
|
||||
// rows [0, n_taps) mix the attention input, rows [n_taps, 2*n_taps) mix the residual
|
||||
layer.mudd_down = create_tensor(tn(LLM_TENSOR_MUDD_DOWN, "weight", i), {n_embd, n_mudd}, 0);
|
||||
layer.mudd_up = create_tensor(tn(LLM_TENSOR_MUDD_UP, "weight", i), {n_mudd, 2*n_taps}, 0);
|
||||
layer.mudd_up_b = create_tensor(tn(LLM_TENSOR_MUDD_UP, "bias", i), {2*n_taps}, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_limite::build_arch_graph(const llm_graph_params & params) const {
|
||||
return std::make_unique<graph>(*this, params);
|
||||
}
|
||||
|
||||
llama_model_limite::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
|
||||
const int64_t n_embd_head = hparams.n_embd_head_k();
|
||||
const int64_t n_taps = hparams.n_mudd_taps;
|
||||
|
||||
GGML_ASSERT(n_embd_head == hparams.n_embd_head_v());
|
||||
|
||||
ggml_tensor * cur;
|
||||
ggml_tensor * inpL;
|
||||
|
||||
inpL = build_inp_embd(model.tok_embd);
|
||||
|
||||
// value embeddings are looked up by token id, embeddings input is not supported
|
||||
ggml_tensor * inp_ve = ggml_get_rows(ctx0, model.value_embd, res->t_inp_tokens);
|
||||
inp_ve = ggml_reshape_3d(ctx0, inp_ve, n_embd_head, n_head_kv, n_tokens);
|
||||
cb(inp_ve, "inp_ve", -1);
|
||||
|
||||
inpL = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1);
|
||||
cb(inpL, "inp_norm", -1);
|
||||
|
||||
ggml_tensor * inp_pos = build_inp_pos();
|
||||
|
||||
auto * inp_attn = build_attn_inp_kv_iswa();
|
||||
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
// hist[il] is the input of layer il, read by the dense connections
|
||||
std::vector<ggml_tensor *> hist(n_layer, nullptr);
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const auto & layer = model.layers[il];
|
||||
|
||||
hist[il] = inpL;
|
||||
|
||||
ggml_tensor * resid = inpL;
|
||||
|
||||
if (layer.mudd_up) {
|
||||
const int32_t * taps = hparams.mudd_tap_idx.data() + il*n_taps;
|
||||
|
||||
ggml_tensor * w = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, il);
|
||||
w = ggml_gelu_erf(ctx0, build_lora_mm(layer.mudd_down, w));
|
||||
w = ggml_add(ctx0, build_lora_mm(layer.mudd_up, w), layer.mudd_up_b);
|
||||
cb(w, "mudd_w", il);
|
||||
|
||||
ggml_tensor * stack = nullptr;
|
||||
for (int64_t j = 0; j < n_taps; ++j) {
|
||||
GGML_ASSERT(taps[j] >= 0 && taps[j] <= il);
|
||||
ggml_tensor * h = ggml_reshape_3d(ctx0, hist[taps[j]], n_embd, 1, n_tokens);
|
||||
stack = stack ? ggml_concat(ctx0, stack, h, 1) : h;
|
||||
}
|
||||
|
||||
auto mix = [&](size_t offset) {
|
||||
if (n_taps == 4) {
|
||||
return ggml_dsv4_hc_pre(ctx0, stack, ggml_cont(ctx0, ggml_view_2d(ctx0, w, n_taps, n_tokens, w->nb[1], offset)));
|
||||
}
|
||||
ggml_tensor * wj = ggml_view_3d(ctx0, w, 1, n_taps, n_tokens, w->nb[0], w->nb[1], offset);
|
||||
ggml_tensor * m = ggml_cont(ctx0, ggml_permute(ctx0, ggml_mul(ctx0, stack, wj), 1, 0, 2, 3));
|
||||
return ggml_reshape_2d(ctx0, ggml_sum_rows(ctx0, m), n_embd, n_tokens);
|
||||
};
|
||||
|
||||
cur = mix(0);
|
||||
resid = mix(n_taps*w->nb[0]);
|
||||
cb(cur, "mudd_attn", il);
|
||||
cb(resid, "mudd_resid", il);
|
||||
} else {
|
||||
cur = inpL;
|
||||
}
|
||||
|
||||
ggml_tensor * attn_in = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, il);
|
||||
cb(attn_in, "attn_norm", il);
|
||||
|
||||
// self-attention
|
||||
{
|
||||
auto [Qcur, Kcur, Vcur] = build_qkv(layer, attn_in, n_embd_head, n_head, n_head_kv, il);
|
||||
|
||||
if (layer.attn_ve_gate) {
|
||||
ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(layer.attn_ve_gate, attn_in));
|
||||
gate = ggml_reshape_3d(ctx0, gate, 1, n_head_kv, n_tokens);
|
||||
|
||||
Vcur = ggml_add(ctx0, Vcur, ggml_mul(ctx0, inp_ve, gate));
|
||||
cb(Vcur, "Vcur_ve", il);
|
||||
}
|
||||
|
||||
Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);
|
||||
Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);
|
||||
cb(Qcur, "Qcur_normed", il);
|
||||
cb(Kcur, "Kcur_normed", il);
|
||||
|
||||
// the full attention layers use no positional encoding
|
||||
if (hparams.is_swa(il)) {
|
||||
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
}
|
||||
|
||||
cur = build_attn(inp_attn,
|
||||
nullptr, nullptr, nullptr,
|
||||
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, hparams.f_attention_scale, il);
|
||||
cb(cur, "attn_raw", il);
|
||||
|
||||
if (layer.attn_xsa_alpha) {
|
||||
// remove from each head output its projection on the token's own value
|
||||
const int64_t n_group = n_head/n_head_kv;
|
||||
|
||||
ggml_tensor * y = ggml_reshape_4d(ctx0, cur, n_embd_head, n_group, n_head_kv, n_tokens);
|
||||
ggml_tensor * vn = ggml_l2_norm(ctx0, Vcur, hparams.f_xsa_eps);
|
||||
vn = ggml_reshape_4d(ctx0, vn, n_embd_head, 1, n_head_kv, n_tokens);
|
||||
vn = ggml_repeat(ctx0, vn, y);
|
||||
|
||||
ggml_tensor * proj = ggml_sum_rows(ctx0, ggml_mul(ctx0, y, vn));
|
||||
proj = ggml_mul(ctx0, proj, ggml_reshape_4d(ctx0, layer.attn_xsa_alpha, 1, n_group, n_head_kv, 1));
|
||||
|
||||
cur = ggml_sub(ctx0, y, ggml_mul(ctx0, vn, proj));
|
||||
cb(cur, "attn_xsa", il);
|
||||
}
|
||||
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head, n_tokens);
|
||||
|
||||
if (layer.wqkv_gate) {
|
||||
ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, attn_in));
|
||||
cur = ggml_mul(ctx0, cur, ggml_reshape_3d(ctx0, gate, 1, n_head, n_tokens));
|
||||
cb(cur, "attn_gated", il);
|
||||
}
|
||||
|
||||
cur = ggml_reshape_2d(ctx0, cur, n_embd_head*n_head, n_tokens);
|
||||
cur = build_lora_mm(layer.wo, cur, layer.wo_s);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
if (il == n_layer - 1 && inp_out_ids) {
|
||||
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
||||
resid = ggml_get_rows(ctx0, resid, inp_out_ids);
|
||||
}
|
||||
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, resid, layer.attn_resid_scale), cur);
|
||||
cb(cur, "attn_resid", il);
|
||||
|
||||
ggml_tensor * ffn_inp = cur;
|
||||
|
||||
cur = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, il);
|
||||
cb(cur, "ffn_norm", il);
|
||||
|
||||
cur = build_ffn(cur,
|
||||
layer.ffn_up, nullptr, nullptr,
|
||||
layer.ffn_gate, nullptr, nullptr,
|
||||
layer.ffn_down, nullptr, nullptr,
|
||||
nullptr,
|
||||
LLM_FFN_SILU, LLM_FFN_PAR, il);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, ffn_inp, layer.ffn_resid_scale), cur);
|
||||
|
||||
cur = build_cvec(cur, il);
|
||||
cb(cur, "l_out", il);
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
cur = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1);
|
||||
|
||||
cb(cur, "result_norm", -1);
|
||||
res->t_embd = cur;
|
||||
|
||||
cur = build_lora_mm(model.output, cur, model.output_s);
|
||||
|
||||
// scale * sigmoid((x + shift) / temperature)
|
||||
const auto & cap = hparams.final_logit_sigmoid_capping;
|
||||
if (cap[0] != 0.0f && cap[2] != 0.0f) {
|
||||
cur = ggml_scale_bias(ctx0, cur, 1.0f/cap[2], cap[1]/cap[2]);
|
||||
cur = ggml_sigmoid(ctx0, cur);
|
||||
cur = ggml_scale(ctx0, cur, cap[0]);
|
||||
}
|
||||
|
||||
cb(cur, "result_output", -1);
|
||||
res->t_logits = cur;
|
||||
}
|
||||
@@ -2056,6 +2056,19 @@ struct llama_model_plm : public llama_model_base {
|
||||
};
|
||||
|
||||
|
||||
struct llama_model_limite : public llama_model_base {
|
||||
llama_model_limite(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
void load_arch_tensors(llama_model_loader & ml) override;
|
||||
|
||||
struct graph : public llm_graph_context {
|
||||
graph(const llama_model & model, const llm_graph_params & params);
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
};
|
||||
|
||||
|
||||
struct llama_model_hrm_text : public llama_model_base {
|
||||
llama_model_hrm_text(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
|
||||
@@ -269,6 +269,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
n_embd = 160; // exercise per-head tensor split granularity with head size 80
|
||||
} else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {
|
||||
n_head = 4;
|
||||
} else if (arch == LLM_ARCH_LIMITE) {
|
||||
n_head = 4;
|
||||
n_layer = 4; // dense connections on the last two layers
|
||||
} else if (arch == LLM_ARCH_DEEPSEEK2
|
||||
|| arch == LLM_ARCH_DEEPSEEK32
|
||||
|| arch == LLM_ARCH_GLM_DSA
|
||||
@@ -297,7 +300,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
if (arch == LLM_ARCH_QWEN3) {
|
||||
n_head_kv = 1; // MQA coverage
|
||||
} else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||
|
||||
arch == LLM_ARCH_KOLIBRI1) {
|
||||
arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {
|
||||
n_head_kv = 2; // GQA coverage
|
||||
}
|
||||
if (arch == LLM_ARCH_BEAM) {
|
||||
@@ -458,7 +461,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
|
||||
}
|
||||
|
||||
if (arch == LLM_ARCH_KOLIBRI1) {
|
||||
if (arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {
|
||||
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, false);
|
||||
}
|
||||
|
||||
@@ -594,6 +597,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);
|
||||
}
|
||||
|
||||
if (arch == LLM_ARCH_LIMITE) {
|
||||
ms.add_kv(LLM_KV_ATTENTION_SCALE, 0.1f);
|
||||
ms.add_kv(LLM_KV_ATTENTION_XSA_EPS, 1e-4f);
|
||||
ms.add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>({23.0f, 5.0f, 7.5f}));
|
||||
ms.add_kv(LLM_KV_MUDD_FEED_FORWARD_LENGTH, uint32_t(8));
|
||||
ms.add_kv(LLM_KV_MUDD_TAP_COUNT, uint32_t(2));
|
||||
ms.add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector<int32_t>({-1, -1, -1, -1, 0, 2, 1, 3}));
|
||||
}
|
||||
|
||||
// dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids
|
||||
{
|
||||
std::vector<std::string> tokenizer_list(n_vocab);
|
||||
|
||||
Reference in new issue
Block a user