diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index a08e173e9..36f3850d5 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -199,6 +199,9 @@ void llama_model_saver::add_kv_from_model() { if (hparams.n_embd_out_impl > 0) { add_kv(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl); } + if (hparams.n_embd_per_layer > 0) { + add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer); + } add_kv(LLM_KV_BLOCK_COUNT, hparams.n_layer_all); add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true); @@ -307,7 +310,9 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, hparams.indexer_kpool_select_tail); add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks); add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, true); - if (hparams.n_layer_kv_from_start < 0) { + if (hparams.n_layer_kv_from_start >= 0) { + add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(hparams.n_layer_all - hparams.n_layer_kv_from_start)); + } else { for (uint32_t il = 0; il < hparams.n_layer(); ++il) { if (hparams.kv_source(il) != il) { add_kv(LLM_KV_ATTENTION_KV_SOURCE_MAP, hparams.kv_source_impl, true); @@ -520,6 +525,8 @@ void llama_model_saver::add_tensors_from_model() { add_tensor(model->hc_head_base); add_tensor(model->hc_head_scale); add_tensor(model->per_layer_tok_embd); + add_tensor(model->per_layer_model_proj); + add_tensor(model->per_layer_proj_norm); add_tensor(model->value_embd); add_tensor(model->hc_head_norm); add_tensor(model->hc_head_down); diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index e08d0c81b..569947a7f 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -81,6 +81,8 @@ gemma2 ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 gemma3 ,0 ,any ,RMS_NORM+MUL , 9 gemma3 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +gemma4 ,0 ,any ,RMS_NORM+MUL , 47 +gemma4 ,0 ,any ,RMS_NORM+MUL+ADD , 20 glm-dsa ,0 ,any ,ADD+ADD , 1 glm-dsa ,0 ,any ,MUL+ADD , 2 glm-dsa ,0 ,any ,NORM+MUL+ADD , 2 diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index d82d6db79..312733ecd 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -243,10 +243,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_head = 4; n_layer = 4; // dense and MoE layers, each with local and full attention } else if (arch == LLM_ARCH_GEMMA4) { - n_embd = 128; - n_head = 2; - n_ff = 192; - n_layer = 5; // need at least 5 for swa_pattern (every 5th is full_attention) + // TODO: expand coverage to both GQA and MQA (see https://github.com/ggml-org/llama.cpp/pull/31401#pullrequestreview-5710165130) + n_head = 4; + n_layer = 10; // last 5 layers reuse KV, both SWA and full attention } else if (arch == LLM_ARCH_GEMMA3N) { n_embd = 64; n_head = 1; @@ -305,7 +304,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_QWEN3) { n_head_kv = 1; // MQA coverage } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER || - arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) { + arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4) { n_head_kv = 2; // GQA coverage } if (arch == LLM_ARCH_BEAM) { @@ -433,12 +432,17 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); } else if (arch == LLM_ARCH_GEMMA4) { ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, n_embd/2); - ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(0)); + ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(5)); ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, n_embd_head); ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, n_embd_head); ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f); // SWA pattern: every 5th layer is full attention (matches E2B layer_types) - ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); + std::vector pattern; + pattern.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + pattern.push_back((il + 1) % 5 != 0); + } + ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) { @@ -1001,9 +1005,6 @@ static bool arch_supported(const llm_arch arch) { if (arch == LLM_ARCH_WAVTOKENIZER_DEC) { return false; // FIXME CUDA backend crashes. } - if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { - return false; // FIXME @ngxson - } if (arch == LLM_ARCH_GRANITE_SWITCH) { return false; // FIXME adapter fixture } @@ -1075,10 +1076,7 @@ static int save_models(const std::string & arch_filter, const size_t seed, const if (!arch_matches(arch_filter, arch)) { continue; } - if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { - continue; // FIXME: ISWA KV cache initialization needs more fixture params - } - if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) { + if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) { continue; } for (bool moe : {false, true}) { @@ -1235,10 +1233,7 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con if (!arch_matches(arch_filter, arch)) { continue; } - if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { - continue; // FIXME: ISWA KV cache initialization needs more fixture params - } - if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) { + if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) { continue; }