From ace20f679e30a39c09a7b17bec14199855469633 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 13 Oct 2026 16:34:10 +0200 Subject: [PATCH] graph: pad the KQ bias of the no-cache attention (#30446) The no-cache attention pads K, V and the mask to 256, but T5 also adds a relative position bias of [n_tokens, n_tokens], so the add with KQ fails unless n_tokens divides the padded width. Pad the bias with zeros up to the KV dimension of the mask, like K and V. --- src/llama-graph.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 9039e47fd..f69fe63f8 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -3056,12 +3056,16 @@ ggml_tensor * llm_graph_context::build_attn( ggml_tensor * k = k_cur; ggml_tensor * v = v_cur; - // the KV values are padded with zeros up to the KV dimension of the mask + // the KV values and the KQ bias are padded with zeros up to the KV dimension of the mask const int64_t n_kv_pad = kq_mask->ne[0] - k->ne[2]; k = ggml_pad(ctx0, k, 0, 0, n_kv_pad, 0); v = ggml_pad(ctx0, v, 0, 0, n_kv_pad, 0); + if (kq_b) { + kq_b = ggml_pad(ctx0, kq_b, n_kv_pad, 0, 0, 0); + } + ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il); cb(cur, "kqv_out", il);