From eb182d85c0e304a69ecc0be0c5e12884691c116a Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Wed, 21 Oct 2026 14:02:29 +0530 Subject: [PATCH] llama : place d2t by its set_rows use so it can stay on the GPU (#30912) --- src/llama-arch.cpp | 2 +- src/llama-model-loader.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 1b954f176..4534be22f 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1110,7 +1110,7 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_MASKED_EMBD_ORDERING, {LLM_TENSOR_LAYER_INPUT, GGML_OP_NONE}}, // eagle3 {LLM_TENSOR_FC, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, - {LLM_TENSOR_D2T, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}}, + {LLM_TENSOR_D2T, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_SET_ROWS}}, // dspark {LLM_TENSOR_DSPARK_MARKOV_W1, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_DSPARK_MARKOV_W2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index bc93c6f9e..9dc366fda 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -951,6 +951,13 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512); op_tensor = ggml_get_rows(ctx, w, b); } break; + case GGML_OP_SET_ROWS: + { + // w holds the row indices (e.g. d2t) + ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, w->ne[0]); + ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, w->ne[0]); + op_tensor = ggml_set_rows(ctx, a, b, w); + } break; case GGML_OP_MUL_MAT: { ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]);