diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 946de5347..0f8e2ffee 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -803,9 +803,6 @@ class DeepseekV4Model(TextModel): self._dsv4_mxfp4_generated = True return () - def _format_dsv4_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> str: - return self.format_tensor_name(key, bid, suffix) - def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]: root_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { "embed.weight": (gguf.MODEL_TENSOR.TOKEN_EMBD, ".weight"), @@ -879,17 +876,17 @@ class DeepseekV4Model(TextModel): def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name): - return [] + return # hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]: - return [] + return tensor_key, suffix = self._map_dsv4_tensor_name(name, bid) if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID: - return [] + return - return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)] + yield self.format_tensor_name(tensor_key, bid, suffix), data_torch def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: del bid # unused diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index 4b87d5c28..53980c1d0 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -145,12 +145,6 @@ class DeepseekV41Model(DeepseekV4Model): # num_hidden_layers lives in text_config, which is merged only after indexing; MTP stages are skipped anyway return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id) - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - if item[0].startswith("vision."): - return None - return super().filter_tensors(item) - def set_gguf_parameters(self): super().set_gguf_parameters() hp = self.hparams @@ -282,12 +276,13 @@ class DeepseekV41Model(DeepseekV4Model): def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name.endswith(".engram.k_weight"): - return [] # folded into engram_gate + return # folded into engram_gate if name.endswith(".engram.q_weight"): # Engram gate weight = q_weight * k_weight (bf16 * bf16 is exact in f32) k = self.model_tensors[name.removesuffix("q_weight") + "k_weight"] - return [(self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), data_torch.float() * k().float())] - return super().modify_tensors(data_torch, name, bid) + yield self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), data_torch.float() * k().float() + return + yield from super().modify_tensors(data_torch, name, bid) def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: if new_name.endswith((".engram_gate.weight", ".ffn_gate_inp.weight")) or name in self._dsv4_f32_tensors: @@ -411,13 +406,15 @@ class DeepseekV41DSparkModel(DeepseekV41Model): def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name.endswith(".ffn.gate.bias_vl"): - return [] + return if name in self._DSPARK_ROOT: - return [(self.format_tensor_name(self._DSPARK_ROOT[name], None), data_torch)] + yield self.format_tensor_name(self._DSPARK_ROOT[name], None), data_torch + return m = re.match(r"layers\.\d+\.(main_proj\.weight|main_norm\.weight|norm\.weight)$", name) if m is not None: - return [(self.format_tensor_name(self._DSPARK_ROOT[m.group(1)], None), data_torch)] - return super().modify_tensors(data_torch, name, bid) + yield self.format_tensor_name(self._DSPARK_ROOT[m.group(1)], None), data_torch + return + yield from super().modify_tensors(data_torch, name, bid) def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: if new_name == "conf_proj.weight": diff --git a/src/llama-hparams.h b/src/llama-hparams.h index a3bf9e688..760bb8ac6 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -358,6 +358,8 @@ struct llama_hparams { llama_token ple_embd_token = LLAMA_TOKEN_NULL; // hash constant sets, stored back to back: 1 shared by all PLE layers (qwen4exp), or 1 per PLE layer in layer order (deepseek41) uint32_t ple_n_modules = 1; + // deepseek41 Engram: the token that pads the n-gram history, an index into llama_model::engram_token_map + uint32_t engram_pad_id = 0; // the file lists PLE layer indices, so this is never a per-layer gguf array and can hold one bit per layer std::bitset is_ple_impl; // the hash multipliers reach ~2e13 and have to stay 64-bit diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index c18d913da..6186f9d34 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -428,8 +428,11 @@ static void dsv4_state_read_k_cache( dsv4_state_read_tensor_streams(io, kv->get_k_storage(il), kv_size, n_rows_ref, s0, ns); } - // refill the mirrors from their source rows just read - std::vector buf; + if (n_rows_ref == 0) { + return; + } + + // refill the mirrors from their sources; a mirror gets the same rows as its source, so a whole copy is exact for (uint32_t il : kv->get_layer_ids()) { const int32_t src = kv_src[il]; if (src == (int32_t) il) { @@ -437,20 +440,11 @@ static void dsv4_state_read_k_cache( } ggml_tensor * t_src = kv->get_k_storage(src); ggml_tensor * t_dst = kv->get_k_storage(il); - GGML_ASSERT(t_src->type == t_dst->type && t_src->ne[0] == t_dst->ne[0]); + if (t_src->type != t_dst->type || !ggml_are_same_shape(t_src, t_dst)) { + throw std::runtime_error("DSV4 K-cache mirror layout mismatch"); + } - const size_t row_size = ggml_row_size(t_src->type, t_src->ne[0]); - const size_t stream_stride = (size_t) kv_size*row_size; - const size_t size = (size_t) n_rows_ref*row_size; - if (size == 0) { - continue; - } - buf.resize(size); - for (uint32_t s = 0; s < ns; ++s) { - const size_t offset = (size_t) (s0 + s)*stream_stride; - ggml_backend_tensor_get(t_src, buf.data(), offset, size); - ggml_backend_tensor_set(t_dst, buf.data(), offset, size); - } + ggml_backend_tensor_copy(t_src, t_dst); } } @@ -1352,10 +1346,10 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( // crosses devices. no mirrors without offload const uint32_t n_layer = model.hparams.n_layer(); comp_read.assign(n_layer, -1); - comp_kv_src.resize(n_layer); + comp_il_src.resize(n_layer); for (uint32_t il = 0; il < n_layer; ++il) { const int32_t src = is_v41 ? model.hparams.dsv41_kv_src[il] : (int32_t) il; - comp_kv_src[il] = src >= 0 ? src : (int32_t) il; + comp_il_src[il] = src >= 0 ? src : (int32_t) il; if (model.hparams.dsv4_compress_ratios[il] == 0) { continue; } @@ -1366,7 +1360,7 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( // the first layer of this source on this device holds the mirror comp_read[il] = il; for (int32_t j = src + 1; j < (int32_t) il; ++j) { - if (comp_read[j] == j && comp_kv_src[j] == src && model.dev_layer(j) == model.dev_layer(il)) { + if (comp_read[j] == j && comp_il_src[j] == src && model.dev_layer(j) == model.dev_layer(il)) { comp_read[il] = j; break; } @@ -1811,9 +1805,9 @@ void llama_kv_cache_dsv4::state_write(llama_io_write_i & io, llama_seq_id seq_id const uint32_t n_rows_lid = seq_id >= 0 ? dsv4_state_n_used_k_rows(pos_max, lid_state->get_ratio(), kv_lid->get_size()) : kv_lid->get_size(); - dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags, n_rows_csa, comp_kv_src); - dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags, n_rows_hca, comp_kv_src); - dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid, comp_kv_src); + dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags, n_rows_csa, comp_il_src); + dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags, n_rows_hca, comp_il_src); + dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid, comp_il_src); } csa_state->state_write(io, seq_id, flags, rs_idx); @@ -1851,9 +1845,9 @@ void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id, if (!partial_only) { clear_compressed(seq_id, true); - dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags, comp_kv_src); - dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags, comp_kv_src); - dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags, comp_kv_src); + dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags, comp_il_src); + dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags, comp_il_src); + dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags, comp_il_src); } csa_state->state_read(io, seq_id, flags); diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h index 99ac73ddb..bfc52947f 100644 --- a/src/llama-kv-cache-dsv4.h +++ b/src/llama-kv-cache-dsv4.h @@ -170,7 +170,7 @@ private: std::vector rs_idx; // [n_layer] storage layer -> layer whose rows it holds (itself, or the source of a mirror) - std::vector comp_kv_src; + std::vector comp_il_src; // [n_layer] layer -> layer whose compressed-KV storage it reads (itself, its V4.1 kv source or a mirror), -1 = none std::vector comp_read; diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 752e6361f..56a42c756 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -8,7 +8,6 @@ #include "llama-hparams.h" #include "llama-model.h" #include "llama-vocab.h" -#include "models/models.h" #include #include @@ -342,8 +341,6 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank); if (model->arch == LLM_ARCH_DEEPSEEK41) { - const auto * m41 = dynamic_cast(model); - GGML_ASSERT(m41 != nullptr); std::vector kv_src; std::vector idx_src; for (uint32_t il = 0; il < hparams.n_layer(); ++il) { @@ -362,8 +359,8 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, hparams.dsv41_cand_block_size); } if (hparams.ple_n_heads > 0) { - add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, m41->engram_pad_id); - add_kv(LLM_KV_ENGRAM_TOKEN_MAP, m41->engram_token_map); + add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id); + add_kv(LLM_KV_ENGRAM_TOKEN_MAP, model->engram_token_map); } } diff --git a/src/llama-model.h b/src/llama-model.h index 01fbda319..ccc2602e5 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -734,6 +734,9 @@ struct llama_model { std::vector classifier_labels; std::vector decision_labels; + // deepseek41 Engram: token id -> id of its normalized form, hashed into the n-grams + std::vector engram_token_map; + struct ggml_tensor * tok_embd = nullptr; struct ggml_tensor * type_embd = nullptr; struct ggml_tensor * pos_embd = nullptr; diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 7dc5ee3a5..13f6a5acc 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -350,7 +350,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_pre( ggml_tensor ** post, ggml_tensor ** comb, int il) const { - return build_hc_pre(x, build_hc_mixes(x, hc_fn, hc_scale, hc_base, post, comb, false, il), il); + return build_hc_pre(x, build_hc_mixes(x, hc_fn, hc_scale, hc_base, post, comb, il), il); } ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes( @@ -360,7 +360,6 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes( ggml_tensor * hc_base, ggml_tensor ** post, ggml_tensor ** comb, - bool f32_src, int il) const { const int64_t hc = hparams.dsv4_hc_mult; const int64_t hc_dim = hc*n_embd; @@ -373,10 +372,6 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes( ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc_dim, nt); ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, norm_rms_eps); ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); - if (f32_src && !ggml_is_quantized(hc_fn->type)) { - // F32 weights (model.py hc_pre runs in fp32): request F32 inputs, the mixes are near-tie sensitive - ggml_prec_set_src(mixes, GGML_PREC_F32, 1); - } cb(mixes, "hc_mixes", il); ggml_tensor * scale_pre = dsv4_view_1d(ctx0, hc_scale, 1, 0); diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index 49e95f54c..6a9003d42 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -11,9 +11,7 @@ #include // ref: deepseek-ai/DeepSeek-V4.1-Flash-1017 inference/model.py, engram.py -// compressed layers attend over [window | all compressed rows] with top-k as a mask. with flash attention the default -// is sparse: fused indexer, FA bounded to window + top-k, decode-sized ubatches gather only the selected rows (same -// keys and mask, different key order). LLAMA_DSV41_SPARSE=0 restores the reference path +// LLAMA_DSV41_SPARSE=0 disables the sparse attention path (see build_attention41) static constexpr int64_t DSV41_ENGRAM_DEAD = -1; @@ -96,7 +94,7 @@ void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) { ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, mult); ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, offsets); ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, primes); - ml.get_key(LLM_KV_ENGRAM_PAD_TOKEN_ID, engram_pad_id); + ml.get_key(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id); ml.get_arr(LLM_KV_ENGRAM_TOKEN_MAP, engram_token_map); const size_t n_e = e_layers.size(); @@ -105,7 +103,7 @@ void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) { if (n_cols == 0 || hparams.ple_head_dim == 0 || mult.size() != n_e*n_gram || primes.size() != n_e*n_cols || offsets.size() != n_e*n_cols || mult.size() > LLAMA_MAX_PLE_NGRAM || primes.size() > LLAMA_MAX_PLE_HEADS || - engram_pad_id >= engram_token_map.size()) { + hparams.engram_pad_id >= engram_token_map.size()) { throw std::runtime_error("DeepSeek-V4.1 engram metadata is inconsistent"); } for (size_t e = 0; e < n_e; ++e) { @@ -247,7 +245,7 @@ void llama_model_deepseek41::engram_hash(int e, const llama_ubatch & ubatch, con const uint64_t * mult = hparams.ple_layer_multipliers.data() + (size_t) e*n_gram; const uint32_t * primes = hparams.ple_head_vocab_sizes.data() + (size_t) e*n_cols; - const int64_t pad = engram_token_map[engram_pad_id]; + const int64_t pad = engram_token_map[hparams.engram_pad_id]; const uint32_t * offs = hparams.ple_head_offsets.data() + (size_t) e*n_cols; const auto cid = [&](llama_token t) -> int64_t { @@ -279,8 +277,7 @@ void llama_model_deepseek41::engram_hash(int e, const llama_ubatch & ubatch, con } } -// cand_pin: +1e30 (the reference uses +inf) on the block holding each query's newest compressed row, so a partly -// filled block always survives; a finite pin keeps fully masked blocks at -inf instead of NaN +// cand_pin: +1e30 on the block with the newest compressed row of each query, so a partly filled block is always kept; the reference uses +inf, but a finite pin keeps fully masked blocks at -inf instead of NaN class llm_graph_input_dsv41 : public llm_graph_input_i { public: llm_graph_input_dsv41(const llama_model_deepseek41 & model, const llama_kv_cache_dsv4_context * mctx) : @@ -535,8 +532,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( if (hparams.dsv41_topk_src[il] == il) { ggml_tensor * ik = ggml_view_4d(ctx0, comp_all, n_idx_embd, 1, n_comp, n_stream, comp_all->nb[1], comp_all->nb[2], comp_all->nb[3], ggml_row_size(comp_all->type, n_embd_head)); - // the fused indexer reads the strided view in place, the unfused path needs it contiguous; quantized - // caches are dequantized straight from the strided view (CUDA has no same-type copy of quantized views) + // the fused indexer reads the strided view in place, the unfused path needs it contiguous; CUDA has no same-type copy of quantized views, so dequantize them const bool fused_idx = sparse && cparams.fused_lid && ci.kq_mask->type == GGML_TYPE_F16; if (ggml_is_quantized(ik->type)) { ik = ggml_cast(ctx0, ik, GGML_TYPE_F32); @@ -551,8 +547,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( ggml_tensor * iw = ggml_scale(ctx0, build_lora_mm(layer.indexer_proj, cur), 1.0f/sqrtf(float(n_idx_embd*n_idx_head))); iw = ggml_view_4d(ctx0, iw, iw->ne[0], nt/n_stream, 1, n_stream, iw->nb[1], iw->nb[1]*(nt/n_stream), iw->nb[1]*(nt/n_stream), 0); - // chunk queries so the [n_ih, tokens, n_comp] scores stay near 2^26 elements (at most 64 chunks); the fused - // indexer has no such intermediate and runs as one chunk + // chunk queries so the [n_ih, tokens, n_comp] scores stay near 2^26 elements (at most 64 chunks); the fused indexer runs as one chunk const int64_t n_tok = nt/n_stream; GGML_ASSERT(ci.kq_mask->ne[1] == n_tok); const int64_t chunk = fused_idx ? n_tok : @@ -582,9 +577,6 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( ggml_tensor * iqc = ggml_view_4d(ctx0, iq, iq->ne[0], iq->ne[1], nc, n_stream, iq->nb[1], iq->nb[2], iq->nb[3], t0*iq->nb[2]); ggml_tensor * kq = ggml_mul_mat(ctx0, ik_p, ggml_permute(ctx0, iqc, 0, 2, 1, 3)); - if (!ggml_is_quantized(ik->type)) { - ggml_prec_set_src(kq, GGML_PREC_F32, 1); - } kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); score = ggml_sum_rows(ctx0, ggml_mul(ctx0, ggml_relu(ctx0, kq), rows(iw))); ggml_tensor * mask = rows(ci.kq_mask); @@ -656,14 +648,12 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( const int64_t k_sel = top_k->ne[0]; const int64_t n_raw = raw_k->ne[2]; - // few queries per stream: gather each query's top-k rows and attend over [window | selected], one FA stream - // per query, instead of a full n_comp mask per layer. same keys and mask as dense, different key order + // few queries per stream: attend over [window | top-k rows of the query], one FA stream per query; same keys and mask as dense, different key order const bool gather = sparse && cparams.flash_attn && raw_mask->type == GGML_TYPE_F16 && ci.kq_mask->type == GGML_TYPE_F16 && ((n_stream == 1 && nt <= 32) || nt == n_stream); if (gather) { - // ggml_top_k returns the rows in no particular order (CUB DeviceTopK can vary between runs) and FA sums the - // gathered keys in that order: sort the rows once per top-k source so the result is deterministic + // ggml_top_k returns the rows in no fixed order and FA sums the keys in that order: sort them so the result is deterministic ggml_tensor *& sel = gather_sel_of[topk_src]; if (!sel) { GGML_ASSERT(n_comp <= (INT64_C(1) << 24)); // row ids are exact in F32 @@ -714,8 +704,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( } ggml_tensor * k_all = ggml_concat(ctx0, raw_k, comp_k, 2); - // mask: window row, then comp mask at the selected rows (-inf where top-k picked an invisible row); - // one per top-k source, shared by its layers + // window mask, then the comp mask at the selected rows (-inf where top-k picked an invisible row); one per top-k source ggml_tensor *& kq_mask = gather_mask_of[topk_src]; if (!kq_mask) { ggml_tensor * sel_mask = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, ci.kq_mask, 1, n_comp, nt), sel); @@ -730,8 +719,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( ggml_tensor * comp_k = ggml_view_4d(ctx0, comp_all, n_embd_head, 1, n_comp, n_stream, comp_all->nb[1], comp_all->nb[2], comp_all->nb[3], 0); if (k_rot) { - // compressed rows are stored unrotated: rotate in F32, then cast to the cache type (dequantized from - // the strided view: CUDA has no same-type quantized copy) + // compressed rows are stored unrotated: rotate in F32, then cast to the cache type comp_k = comp_k->type != GGML_TYPE_F32 ? ggml_cast(ctx0, comp_k, GGML_TYPE_F32) : ggml_cont(ctx0, comp_k); comp_k = llama_mul_mat_hadamard(ctx0, comp_k, k_rot); if (comp_k->type != raw_k->type) { @@ -862,7 +850,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g ggml_tensor * post = nullptr; ggml_tensor * comb = nullptr; - ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, true, il); + ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); // the first layer's attention collapses with the identity pre-mix: copy 0 cur = pre_prev ? build_hc_pre(inpL, pre_prev, il) : ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0)); @@ -877,7 +865,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g cb(inpL, "hc_attn_post", il); residual = inpL; - pre_prev = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, true, il); + pre_prev = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); cur = build_hc_pre(inpL, attn_pre, il); cb(cur, "hc_ffn_pre", il); @@ -889,12 +877,9 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); - // F32 router inputs: expert selection is near-tie sensitive + // F32 router accumulation: expert selection is near-tie sensitive ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur); ggml_prec_set_acc(logits, GGML_PREC_F32); - if (!ggml_is_quantized(layer.ffn_gate_inp->type)) { - ggml_prec_set_src(logits, GGML_PREC_F32, 1); - } cb(logits, "ffn_moe_logits", il); // image (embedding) tokens use the vision router bias, as DeepSeek-V4 diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 8b6d6abdb..6bbb534f0 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -1406,7 +1406,7 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll ggml_tensor * post = nullptr; ggml_tensor * comb = nullptr; - ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, false, il); + ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); ggml_tensor * cur; if (pre_prev == nullptr) { @@ -1465,7 +1465,7 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll cb(inpL, "hc_attn_post", il); residual = inpL; - ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, false, il); + ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); cur = build_hc_pre(inpL, attn_pre, il); cb(cur, "hc_ffn_pre", il); diff --git a/src/models/models.h b/src/models/models.h index 59a4e5703..37dbacdb9 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1376,7 +1376,6 @@ struct llama_model_deepseek4 : public llama_model_base { ggml_tensor * hc_base, ggml_tensor ** post, ggml_tensor ** comb, - bool f32_src, int il) const; ggml_tensor * build_hc_post( @@ -1498,11 +1497,7 @@ struct llama_model_deepseek41 : public llama_model_deepseek4 { void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; - // Engram: the hash constants are hparams.ple_* (one module per Engram layer), hashed on the host (ggml has no - // int64 mul/xor/mod); the token map and its pad id are V4.1 only - uint32_t engram_pad_id = 0; - std::vector engram_token_map; - + // Engram: the hash constants are hparams.ple_* (one module per Engram layer), hashed on the host (ggml has no int64 mul/xor/mod) int engram_index(int il) const; void engram_hash(int e, const llama_ubatch & ubatch, const std::vector & prev, std::vector & out) const; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index fe3a67c64..3e9439252 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9996,6 +9996,7 @@ static std::vector> make_test_cases_eval() { std::default_random_engine rng(0); // unary ops + // TODO: add GGML_UNARY_OP_SOFTPLUS test that is sensitive to using `log1p(exp(x))` vs `log(exp(x) + 1)` for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { for (int v : {0, 1}) { for (int op = 0; op < GGML_UNARY_OP_COUNT; op++) {