diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 4cb04097e..65c42f85f 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -853,6 +853,13 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
             return {blck_size_perf};
         }
 
+        if (ud->model->arch == LLM_ARCH_STEP35 || ud->model->arch == LLM_ARCH_LAGUNA) {
+            GGML_ASSERT(segments.size() == 1);
+            if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {
+                return {hparams.n_head(il) / hparams.n_head_kv(il)};
+            }
+        }
+
         // everything else
         GGML_ASSERT(segments.size() == 1);
         return {1};