diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index c7e8ea85d..45e6ae902 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -2397,7 +2397,7 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg } } - GGML_ASSERT(htp_kparams_solve(sess, node.node, node.kernel_params)); + GGML_ASSERT(htp_kparams_solve(sess, node)); if (node.opcode == HTP_OP_CPY && node.kparams()->total_elems == 0) { continue; } diff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cpp index 65aacf73f..ba2dd6410 100644 --- a/ggml/src/ggml-hexagon/htp-fusion.cpp +++ b/ggml/src/ggml-hexagon/htp-fusion.cpp @@ -192,15 +192,20 @@ static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & pre if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &src2)) return false; if (src2->type != GGML_TYPE_F32) return false; - const ggml_tensor * src0 = prev.src0(); - const ggml_tensor * src1 = prev.src1(); - const ggml_tensor * dst = cur.dst(); - if (!ggml_are_same_shape(prev.dst(), dst)) return false; + const ggml_tensor * src0 = prev.node->src[0]; + const ggml_tensor * src1 = prev.node->src[1]; + const ggml_tensor * dst = cur.node; + if (!ggml_are_same_shape(prev.node, dst)) return false; + + const bool prev_collapsed = htp_matmul_can_collapse(src0, src1, prev.node); + const bool cur_collapsed = htp_matmul_add_can_collapse(src0, src1, src2, dst); + if (prev_collapsed != cur_collapsed) { + return false; + } const auto * orig_kparams = prev.kparams(); struct htp_mm_kernel_params kparams; - if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams) || - orig_kparams->collapse != kparams.collapse) { + if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams)) { return false; } @@ -222,6 +227,7 @@ static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & pre prev.opcode = HTP_OP_MUL_MAT_ADD; prev.inputs = { src0, src1, src2 }; prev.outputs = { dst }; + htp_opnode_collapse(sess, prev); prev.fused.push_back(cur.node); prev.set_kparams(kparams); return true; @@ -233,36 +239,31 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev const uint32_t curr_n = (uint32_t) prev.outputs.size(); if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) return false; - const ggml_tensor * w0 = prev.inputs[0]; - const ggml_tensor * x = prev.inputs[curr_n]; - const ggml_tensor * w_in = cur.src0(); - const ggml_tensor * x_in = cur.src1(); - const ggml_tensor * d_in = cur.dst(); + const ggml_tensor * w0 = prev.inputs[0]; + const ggml_tensor * x_orig = prev.node->src[1]; + const ggml_tensor * w_in = cur.node->src[0]; + const ggml_tensor * x_in = cur.node->src[1]; + const ggml_tensor * d_in = cur.node; auto * orig_kparams = prev.kparams(); const auto * cur_kparams = cur.kparams(); - if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || + if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 || htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) || orig_kparams->n_hmx != cur_kparams->n_hmx) { return false; } - const bool collapse = orig_kparams->collapse && htp_matmul_can_collapse(w_in, x, d_in); - if ((x->ne[2] > 1 || x->ne[3] > 1) && !collapse) return false; + const bool collapse = htp_matmul_can_collapse(w0, x_orig, prev.node) && htp_matmul_can_collapse(w_in, x_in, d_in); + if ((x_orig->ne[2] > 1 || x_orig->ne[3] > 1) && !collapse) return false; if (prev.opcode == HTP_OP_MUL_MAT_NX) { orig_kparams->n_weights = curr_n + 1; } else { struct htp_mm_kernel_params kparams; - if (collapse) { - const ggml_tensor x_collapsed = htp_tensor_collapse_rows(x); - if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false; - kparams.collapse = 1; - } else if (!htp_kparams_solve_mmnx(sess, w0, x, curr_n + 1, &kparams)) { - return false; - } + const ggml_tensor x_collapsed = collapse ? htp_tensor_collapse_rows(x_orig) : *x_orig; + if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false; if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) return false; prev.set_kparams(kparams); @@ -270,7 +271,7 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev } prev.inputs.insert(prev.inputs.begin() + curr_n, w_in); - prev.outputs.push_back(d_in); + prev.outputs.push_back(collapse ? prev.add_dummy(htp_tensor_collapse_rows(d_in)) : d_in); prev.fused.push_back(cur.node); return true; } @@ -415,15 +416,13 @@ static bool fuse_unary_mul_glu(const ggml_hexagon_session * sess, htp_opnode & p const ggml_tensor * dst = cur.dst(); if (htp_inputs_overlap_unsafe(prev, dst, up)) return false; - struct htp_glu_kernel_params kparams; - if (!htp_kparams_solve_unary_mul_glu(sess, glu_op, gate, up, dst, &kparams)) return false; - prev.opcode = glu_op; prev.inputs = { gate, up }; prev.outputs = { dst }; + if (!htp_kparams_solve(sess, prev)) return false; + prev.fused.push_back(cur.node); memset(prev.params, 0, sizeof(prev.params)); - prev.set_kparams(kparams); return true; } diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index a8f01a4c4..b8d1aee3d 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -1,6 +1,7 @@ #define _USE_MATH_DEFINES #include "htp-kparams.h" +#include "htp-opnode.h" #include "ggml-impl.h" #include @@ -1023,14 +1024,12 @@ static bool htp_kparams_solve_matmul( if (htp_matmul_can_collapse(src0, src1, dst)) { const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1); const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst); - const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams); - kparams->collapse = 1; - return ok; + return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams); } return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, kparams); } -static bool htp_matmul_add_can_collapse( +bool htp_matmul_add_can_collapse( const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * src2, @@ -1068,9 +1067,7 @@ bool htp_kparams_solve_matmul_add( : (src2 ? *src2 : ggml_tensor{}); const struct ggml_tensor * p_src2 = src2 ? &src2_collapsed : nullptr; const size_t src2_size = p_src2 ? hex_round_up(ggml_nbytes(p_src2), 128) : 0; - const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams); - kparams->collapse = 1; - return ok; + return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams); } const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0; return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams); @@ -1405,47 +1402,103 @@ static bool htp_kparams_solve_binary( } // collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector -static void htp_unary_collapse_dims(struct ggml_tensor * src, struct ggml_tensor * dst, size_t elem_size) { - if (src->nb[0] != elem_size || dst->nb[0] != elem_size) { +static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_threads) { + if (n_tensors == 0 || !tensors[0]) { return; } + for (size_t t = 0; t < n_tensors; t++) { + if (!tensors[t] || tensors[t]->nb[0] != elem_size) { + return; + } + if (t > 0 && !ggml_are_same_shape(tensors[0], tensors[t])) { + return; + } + } - while (src->ne[1] > 1 || src->ne[2] > 1 || src->ne[3] > 1) { - const size_t row_size = (size_t) src->ne[0] * elem_size; - const bool rows_contig = src->nb[1] == row_size && dst->nb[1] == (size_t) dst->ne[0] * elem_size; - const bool can_merge = (src->ne[1] == 1 && dst->ne[1] == 1) || - (rows_contig && (src->ne[0] == 1 || (row_size < 128 && row_size * src->ne[1] <= 2048))); + struct ggml_tensor * t0 = tensors[0]; + + while (t0->ne[1] > 1 || t0->ne[2] > 1 || t0->ne[3] > 1) { + const size_t row_size = (size_t) t0->ne[0] * elem_size; + bool rows_contig = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->nb[1] != (size_t) tensors[t]->ne[0] * elem_size) { + rows_contig = false; + break; + } + } + bool all_ne1_one = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->ne[1] != 1) { + all_ne1_one = false; + break; + } + } + const bool can_merge = all_ne1_one || + (rows_contig && (t0->ne[0] == 1 || (row_size < 128 && row_size * t0->ne[1] <= 2048))); if (!can_merge) { break; } - src->ne[0] *= src->ne[1]; - src->ne[1] = src->ne[2]; - src->ne[2] = src->ne[3]; - src->ne[3] = 1; - dst->ne[0] *= dst->ne[1]; - dst->ne[1] = dst->ne[2]; - dst->ne[2] = dst->ne[3]; - dst->ne[3] = 1; - src->nb[1] = src->nb[2]; - src->nb[2] = src->nb[3]; - dst->nb[1] = dst->nb[2]; - dst->nb[2] = dst->nb[3]; + for (size_t t = 0; t < n_tensors; t++) { + tensors[t]->ne[0] *= tensors[t]->ne[1]; + tensors[t]->ne[1] = tensors[t]->ne[2]; + tensors[t]->ne[2] = tensors[t]->ne[3]; + tensors[t]->ne[3] = 1; + tensors[t]->nb[1] = tensors[t]->nb[2]; + tensors[t]->nb[2] = tensors[t]->nb[3]; + tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2]; + } } // ne1 too big to merge in full: merge it in chunks of k rows (k divides ne1) - const size_t row_size = (size_t) src->ne[0] * elem_size; - if (src->ne[1] > 1 && row_size < 128 && src->nb[1] == row_size && dst->nb[1] == row_size) { + const size_t row_size = (size_t) t0->ne[0] * elem_size; + bool rows_contig = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->nb[1] != row_size) { + rows_contig = false; + break; + } + } + if (t0->ne[1] > 1 && row_size < 128 && rows_contig) { int64_t k = 2048 / row_size; - while (k > 1 && src->ne[1] % k != 0) { + while (k > 1 && t0->ne[1] % k != 0) { k--; } if (k > 1) { - src->ne[0] *= k; - src->ne[1] /= k; - src->nb[1] *= k; - dst->ne[0] *= k; - dst->ne[1] /= k; - dst->nb[1] *= k; + for (size_t t = 0; t < n_tensors; t++) { + tensors[t]->ne[0] *= k; + tensors[t]->ne[1] /= k; + tensors[t]->nb[1] *= k; + tensors[t]->nb[2] = tensors[t]->nb[1] * tensors[t]->ne[1]; + tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2]; + } + } + } + + // nrows too small to saturate threads: split ne0 into k rows (k divides ne0) + const size_t row_size_curr = (size_t) t0->ne[0] * elem_size; + const uint32_t nrows = (uint32_t) (t0->ne[1] * t0->ne[2] * t0->ne[3]); + rows_contig = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->ne[1] != 1 && tensors[t]->nb[1] != row_size_curr) { + rows_contig = false; + break; + } + } + if (nrows < n_threads && rows_contig && row_size_curr >= 4096) { + const size_t vec_elems = 128 / elem_size; + int64_t k = std::max(row_size_curr / 16384, (n_threads * 4 + nrows - 1) / nrows); + k = std::min(k, row_size_curr / 2048); + while (k > 1 && t0->ne[0] % (k * vec_elems) != 0) { + k--; + } + if (k > 1) { + for (size_t t = 0; t < n_tensors; t++) { + tensors[t]->ne[0] /= k; + tensors[t]->ne[1] *= k; + tensors[t]->nb[1] = tensors[t]->ne[0] * elem_size; + tensors[t]->nb[2] = tensors[t]->nb[1] * tensors[t]->ne[1]; + tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2]; + } } } } @@ -1507,19 +1560,6 @@ static bool htp_kparams_solve_unary_impl( const size_t elem_size = ggml_type_size(src0->type); - struct ggml_tensor src0_c = *src0; - struct ggml_tensor dst_c = *dst; - if (htp_unary_is_elementwise(op)) { - htp_unary_collapse_dims(&src0_c, &dst_c, elem_size); - src0 = &src0_c; - dst = &dst_c; - for (int i = 0; i < HTP_OP_MAX_DIMS; i++) { - kparams->ne[i] = (uint32_t) src0->ne[i]; - kparams->src0_nb[i] = (uint32_t) src0->nb[i]; - kparams->dst_nb[i] = (uint32_t) dst->nb[i]; - } - } - const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type); const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; @@ -1533,9 +1573,14 @@ static bool htp_kparams_solve_unary_impl( const bool broadcast_weight = (op == HTP_OP_RMS_NORM_MUL) && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); kparams->broadcast_weight = broadcast_weight; - const bool src0_contig = ggml_is_contiguous_1(src0); - const bool src1_contig = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true; - const uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1]; + const bool src0_contig = ggml_is_contiguous_1(src0); + const bool src1_contig = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true; + const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + const uint32_t min_block_rows = (uint32_t) (std::max(1, 2048 / dst_data_row_size)); + uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1]; + if (src0_nrows_per_thread >= 2 * min_block_rows) { + max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_thread / 4)); + } kparams->src0_contig = src0_contig ? 1 : 0; kparams->src1_contig = src1_contig ? 1 : 0; @@ -2000,11 +2045,17 @@ bool htp_kparams_solve_unary_mul_glu( const size_t vtcm_size = htp_session_vtcm_size(sess); const size_t row_size_aligned = hex_round_up(dst_row_size, 128); - const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned)); + uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned)); if (vtcm_row_per_thread < 2) { return false; } + const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + if (src0_nrows_per_thread >= 2 && vtcm_row_per_thread > src0_nrows_per_thread) { + uint32_t target_rpt = (src0_nrows_per_thread >= 4) ? (src0_nrows_per_thread / 2) & ~1u : 2; + vtcm_row_per_thread = (std::max)(2u, (std::min)(vtcm_row_per_thread, target_rpt)); + } + struct htp_glu_vtcm_layout L; htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, vtcm_row_per_thread); @@ -3246,12 +3297,131 @@ static inline bool htp_kparams_invoke( return SolveFn(sess, op, kparams ? (KParams *) kparams : &tmp); } +void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & node) { + const uint32_t n_threads = htp_session_n_threads(sess); + + if (htp_op_is_unary(node.opcode) && htp_unary_is_elementwise(node.opcode)) { + if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) { + return; + } + if (!ggml_are_same_shape(node.inputs[0], node.outputs[0])) { + return; + } + const size_t elem_size = ggml_type_size(node.inputs[0]->type); + struct ggml_tensor src0_c = *node.inputs[0]; + struct ggml_tensor dst_c = *node.outputs[0]; + struct ggml_tensor * tensors[2] = { &src0_c, &dst_c }; + htp_collapse_dims(tensors, 2, elem_size, n_threads); + if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { + node.inputs[0] = node.add_dummy(src0_c); + node.outputs[0] = node.add_dummy(dst_c); + } + return; + } + + if (htp_op_is_glu(node.opcode)) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + if (!ggml_are_same_shape(node.inputs[0], node.outputs[0]) || + !ggml_are_same_shape(node.inputs[1], node.outputs[0])) { + return; + } + const size_t elem_size = ggml_type_size(node.inputs[0]->type); + struct ggml_tensor src0_c = *node.inputs[0]; + struct ggml_tensor src1_c = *node.inputs[1]; + struct ggml_tensor dst_c = *node.outputs[0]; + struct ggml_tensor * tensors[3] = { &src0_c, &src1_c, &dst_c }; + htp_collapse_dims(tensors, 3, elem_size, n_threads); + if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { + node.inputs[0] = node.add_dummy(src0_c); + node.inputs[1] = node.add_dummy(src1_c); + node.outputs[0] = node.add_dummy(dst_c); + } + return; + } + + if (node.opcode == HTP_OP_MUL_MAT) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * src0 = node.inputs[0]; + const struct ggml_tensor * src1 = node.inputs[1]; + const struct ggml_tensor * dst = node.outputs[0]; + if (htp_matmul_can_collapse(src0, src1, dst)) { + node.inputs[1] = node.add_dummy(htp_tensor_collapse_rows(src1)); + node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst)); + } + return; + } + + if (node.opcode == HTP_OP_MUL_MAT_ADD) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * src0 = node.inputs[0]; + const struct ggml_tensor * src1 = node.inputs[1]; + const struct ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const struct ggml_tensor * dst = node.outputs[0]; + if (htp_matmul_add_can_collapse(src0, src1, src2, dst)) { + node.inputs[1] = node.add_dummy(htp_tensor_collapse_rows(src1)); + node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst)); + if (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) { + node.inputs[2] = node.add_dummy(htp_tensor_collapse_rows(src2)); + } + } + return; + } +} + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + htp_opnode & node +) { + htp_opnode_collapse(sess, node); + + const ggml_tensor * src0 = node.inputs.empty() ? nullptr : node.inputs[0]; + const ggml_tensor * src1 = node.inputs.size() > 1 ? node.inputs[1] : nullptr; + const ggml_tensor * dst = node.dst(); + + if (htp_op_is_unary(node.opcode)) { + return htp_kparams_solve_unary_impl(sess, node.opcode, src0, src1, dst, node.kparams()); + } + if (htp_op_is_glu(node.opcode)) { + return htp_kparams_solve_unary_mul_glu(sess, node.opcode, src0, src1, dst, node.kparams()); + } + if (node.opcode == HTP_OP_MUL_MAT) { + if (!htp_matmul_check_types(src0, src1, dst)) { + return false; + } + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, node.kparams()); + } + if (node.opcode == HTP_OP_MUL_MAT_ADD) { + const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0; + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, node.kparams()); + } + + return htp_kparams_solve(sess, node.node, node.kernel_params); +} + bool htp_kparams_solve( const struct ggml_hexagon_session * sess, const struct ggml_tensor * op, void * kparams ) { const htp_op_code opcode = htp_op_remap(op); + if (htp_op_is_unary(opcode) || htp_op_is_glu(opcode)) { + htp_opnode tmp(opcode, op); + if (!htp_kparams_solve(sess, tmp)) { + return false; + } + if (kparams) { + memcpy(kparams, tmp.kernel_params, sizeof(tmp.kernel_params)); + } + return true; + } + switch (opcode) { case HTP_OP_MUL_MAT: case HTP_OP_MUL_MAT_ID: diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h index 0871209e7..0bde658dc 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.h +++ b/ggml/src/ggml-hexagon/htp-kparams.h @@ -34,6 +34,7 @@ bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride); bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst); +bool htp_matmul_add_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * src2, const struct ggml_tensor * dst); ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t); template @@ -45,6 +46,18 @@ static inline bool htp_kparams_check(Args &&... args) { return SolveFn(std::forward(args)..., &kparams); } +struct htp_opnode; + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + struct htp_opnode & node +); + +void htp_opnode_collapse( + const struct ggml_hexagon_session * sess, + struct htp_opnode & node +); + bool htp_kparams_solve( const struct ggml_hexagon_session * sess, const struct ggml_tensor * op, diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index eb40fa642..784bb37b9 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -3775,16 +3775,6 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k return HTP_STATUS_OK; } -static inline void htp_mm_tensor_collapse_rows(struct htp_tensor * c, const struct htp_tensor * t, uint32_t stride) { - *c = *t; - c->ne[1] = t->ne[1] * t->ne[2] * t->ne[3]; - c->ne[2] = 1; - c->ne[3] = 1; - c->nb[1] = stride; - c->nb[2] = c->nb[1] * c->ne[1]; - c->nb[3] = c->nb[2]; -} - static int op_matmul_impl(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -3801,36 +3791,6 @@ static int op_matmul_impl(struct htp_ops_context * octx) { } int op_matmul(struct htp_ops_context * octx) { - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - - if (kparams->collapse) { - const struct htp_tensor * act = octx->src[1]; - const struct htp_tensor * dst = octx->dst; - const uint32_t s_act = (act->ne[1] > 1) ? act->nb[1] : ((act->ne[2] > 1) ? act->nb[2] : act->nb[3]); - const uint32_t sd = (dst->ne[1] > 1) ? dst->nb[1] : ((dst->ne[2] > 1) ? dst->nb[2] : dst->nb[3]); - struct htp_tensor act_collapsed, dst_collapsed; - htp_mm_tensor_collapse_rows(&act_collapsed, act, s_act); - htp_mm_tensor_collapse_rows(&dst_collapsed, dst, sd); - octx->src[1] = &act_collapsed; - octx->dst = &dst_collapsed; - - struct htp_tensor src2_collapsed; - const struct htp_tensor * src2 = octx->src[2]; - if (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) { - const uint32_t s2 = (src2->ne[1] > 1) ? src2->nb[1] : ((src2->ne[2] > 1) ? src2->nb[2] : src2->nb[3]); - htp_mm_tensor_collapse_rows(&src2_collapsed, src2, s2); - octx->src[2] = &src2_collapsed; - } - - const int status = op_matmul_impl(octx); - octx->src[1] = act; - octx->dst = dst; - if (src2) { - octx->src[2] = src2; - } - return status; - } - return op_matmul_impl(octx); } @@ -4562,35 +4522,5 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) { } int op_matmul_nx(struct htp_ops_context * octx) { - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - - if (kparams->collapse) { - const uint32_t n_weights = kparams->n_weights; - const struct htp_tensor * act = octx->src[n_weights]; - const uint32_t s1 = (act->ne[1] > 1) ? act->nb[1] : ((act->ne[2] > 1) ? act->nb[2] : act->nb[3]); - struct htp_tensor act_collapsed; - struct htp_tensor dsts_collapsed[HTP_OP_MAX_OUTPUTS]; - const struct htp_tensor * orig_dsts[HTP_OP_MAX_OUTPUTS]; - - htp_mm_tensor_collapse_rows(&act_collapsed, act, s1); - octx->src[n_weights] = &act_collapsed; - - for (uint32_t p = 0; p < n_weights; p++) { - orig_dsts[p] = octx->dsts[p]; - const struct htp_tensor * d = octx->dsts[p]; - const uint32_t sd = (d->ne[1] > 1) ? d->nb[1] : ((d->ne[2] > 1) ? d->nb[2] : d->nb[3]); - htp_mm_tensor_collapse_rows(&dsts_collapsed[p], d, sd); - octx->dsts[p] = &dsts_collapsed[p]; - } - - const int status = op_matmul_nx_impl(octx); - - octx->src[n_weights] = act; - for (uint32_t p = 0; p < n_weights; p++) { - octx->dsts[p] = orig_dsts[p]; - } - return status; - } - return op_matmul_nx_impl(octx); } diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index 474c2dbc0..4971ed958 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -61,7 +61,7 @@ enum htp_mm_kernel_type { struct htp_mm_kernel_params { uint8_t kernel_type; // enum htp_mm_kernel_type uint8_t pipeline; // 1 = pipelined execution, 0 = standard - uint8_t collapse; // 1 = collapse outer dims into 2D, 0 = standard + uint8_t unused; // reserved uint8_t n_hmx; // 1 = use HMX, 0 = use HVX uint8_t n_threads; // Number of threads to spawn diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 194569ab5..3b63ac674 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -1984,29 +1984,8 @@ static int execute_op_unary(struct htp_ops_context * octx) { int op_unary(struct htp_ops_context * octx) { switch (octx->src[0]->type) { case HTP_TYPE_F32: - case HTP_TYPE_F16: { - if (!htp_unary_is_elementwise(octx->op)) { - return execute_op_unary(octx); - } - - const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; - - struct htp_tensor src0_c = *octx->src[0]; - struct htp_tensor dst_c = *octx->dst; - memcpy(src0_c.ne, kparams->ne, sizeof(src0_c.ne)); - memcpy(dst_c.ne, kparams->ne, sizeof(dst_c.ne)); - memcpy(src0_c.nb, kparams->src0_nb, sizeof(src0_c.nb)); - memcpy(dst_c.nb, kparams->dst_nb, sizeof(dst_c.nb)); - - const struct htp_tensor * orig_src0 = octx->src[0]; - const struct htp_tensor * orig_dst = octx->dst; - octx->src[0] = &src0_c; - octx->dst = &dst_c; - const int status = execute_op_unary(octx); - octx->src[0] = orig_src0; - octx->dst = orig_dst; - return status; - } + case HTP_TYPE_F16: + return execute_op_unary(octx); default: return HTP_STATUS_NO_SUPPORT; diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index b824544ff..713e40d50 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -24,11 +24,6 @@ struct htp_unary_kernel_params { struct fastdiv_values div_ne02; struct fastdiv_values div_ne012; struct fastdiv_values div_tpr; - - // collapsed shape of elementwise ops - uint32_t ne[HTP_OP_MAX_DIMS]; - uint32_t src0_nb[HTP_OP_MAX_DIMS]; - uint32_t dst_nb[HTP_OP_MAX_DIMS]; }; #if defined(__cplusplus) @@ -72,6 +67,9 @@ static inline bool htp_op_is_unary(uint32_t opcode) { } static inline bool htp_unary_is_elementwise(uint32_t opcode) { + if (!htp_op_is_unary(opcode)) { + return false; + } switch (opcode) { case HTP_OP_NORM: case HTP_OP_RMS_NORM: diff --git a/scripts/snapdragon/ggml-hexagon-profile.py b/scripts/snapdragon/ggml-hexagon-profile.py index 4ac227678..7c95ec8cb 100755 --- a/scripts/snapdragon/ggml-hexagon-profile.py +++ b/scripts/snapdragon/ggml-hexagon-profile.py @@ -215,6 +215,7 @@ def parse_log(file_path, pmu_index=None, limit=None, device_filter=None, op_filt 'unwrapped_cycles_start': unwrapped_cycles_start, 'pmu_val': pmu_val, 'evt_val': evt_val, + 'mhz': float(op_match.group('mhz')) if op_match.group('mhz') else None, 'abs_usec': abs_usec, 'trace_events': [], 'device': device @@ -630,6 +631,165 @@ def generate_report(ops, top_n, width_overrides, sort_col, pmu_name=None): logger.info("| " + " | ".join(row_vals) + " |") +def has_perf_print(file_path): + if file_path == "-": + return True + with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: + return any("common_perf_print" in line for line in f) + + +def pct_delta(a, b): + return (b - a) * 100.0 / a if a else None + + +def fmt_pct(p): + return f"{p:+.2f}%" if p is not None else "n/a" + + +def print_table(title, cols, rows, width_overrides): + # cols: list of (cli name, header); rows: list of dicts keyed by cli name + logger.info(f"\n# {title}\n") + if not rows: + logger.info("None.") + return + + headers, keys, widths = [], [], [] + for key, header in cols: + natural_width = max([len(str(row[key])) for row in rows] + [len(header)]) + target_width = width_overrides.get(key, natural_width) + if target_width == 0: + continue + headers.append(header) + keys.append(key) + widths.append(target_width) + + logger.info("| " + " | ".join(f"{h:<{widths[i]}}" for i, h in enumerate(headers)) + " |") + logger.info("| " + " | ".join("-" * widths[i] for i in range(len(headers))) + " |") + for row in rows: + row_vals = [] + for i, key in enumerate(keys): + val = str(row[key]) + if len(val) > widths[i]: + val = val[:widths[i] - 3] + "..." + row_vals.append(f"{val:<{widths[i]}}") + logger.info("| " + " | ".join(row_vals) + " |") + + +def generate_compare_report(ops_a, ops_b, file_a, file_b, top_n, width_overrides, min_delta): + def summarize(ops): + batches = [o for o in ops if o['name'] == "OPBATCH"] + others = [o for o in ops if o['name'] != "OPBATCH"] + mhz = [o['mhz'] for o in others if o['mhz'] is not None] + return { + 'n_ops': len(others), + 'tot_usec': sum(o['usec'] for o in others), + 'n_batch': len(batches), + 'batch_usec': sum(o['usec'] for o in batches), + 'avg_mhz': statistics.mean(mhz) if mhz else 0.0, + } + + sa, sb = summarize(ops_a), summarize(ops_b) + + logger.info("\n# Compare Summary (OPBATCH excluded from op totals)\n") + logger.info(f" A (baseline) : {file_a}") + logger.info(f" B (new) : {file_b}") + logger.info(" Deltas are B - A: negative means B is faster\n") + logger.info(f" Ops : A {sa['n_ops']} B {sb['n_ops']}") + logger.info(f" Tot op usec : A {sa['tot_usec']} B {sb['tot_usec']} delta {sb['tot_usec'] - sa['tot_usec']:+d} ({fmt_pct(pct_delta(sa['tot_usec'], sb['tot_usec']))})") + logger.info(f" OPBATCH : A {sa['n_batch']} / {sa['batch_usec']} usec B {sb['n_batch']} / {sb['batch_usec']} usec") + logger.info(f" Avg op mhz : A {sa['avg_mhz']:.1f} B {sb['avg_mhz']:.1f}") + + if sa['n_batch'] != sb['n_batch']: + logger.warning(f" WARNING: OPBATCH counts differ (A {sa['n_batch']}, B {sb['n_batch']}), one log may be incomplete") + for label, path in (("A", file_a), ("B", file_b)): + if not has_perf_print(path): + logger.warning(f" WARNING: log {label} has no common_perf_print line, run may be incomplete") + + # Per-op-name rollup + by_name = defaultdict(lambda: [0, 0, 0, 0]) + for idx, ops in ((0, ops_a), (1, ops_b)): + for op in ops: + if op['name'] == "OPBATCH": + continue + by_name[op['name']][idx] += 1 + by_name[op['name']][idx + 2] += op['usec'] + + rows = [] + for name, (cnt_a, cnt_b, tot_a, tot_b) in by_name.items(): + p = pct_delta(tot_a, tot_b) + if min_delta and p is not None and abs(p) < min_delta: + continue + rows.append({'op': name, 'count-a': cnt_a, 'count-b': cnt_b, 'tot-usec-a': tot_a, 'tot-usec-b': tot_b, + 'delta-usec': f"{tot_b - tot_a:+d}", 'delta-pct': fmt_pct(p), '_sort': abs(tot_b - tot_a)}) + rows.sort(key=lambda r: r['_sort'], reverse=True) + + print_table("Per-Op Rollup (delta = B - A, sorted by abs delta usec)", [ + ("op", "Op"), ("count-a", "Count A"), ("count-b", "Count B"), ("tot-usec-a", "Tot usec A"), + ("tot-usec-b", "Tot usec B"), ("delta-usec", "Delta usec"), ("delta-pct", "Delta %")], rows, width_overrides) + + # Per-group compare keyed on (op, dims, dtypes) + def group(ops): + grouped = defaultdict(list) + for op in ops: + if op['name'] != "OPBATCH": + grouped[(op['name'], op['dims'], op['types'])].append(op['usec']) + return grouped + + ga, gb = group(ops_a), group(ops_b) + + rows = [] + for key in ga.keys() & gb.keys(): + ua, ub = ga[key], gb[key] + tot_a, tot_b = sum(ua), sum(ub) + avg_a, avg_b = tot_a / len(ua), tot_b / len(ub) + p = pct_delta(avg_a, avg_b) + if min_delta and p is not None and abs(p) < min_delta: + continue + rows.append({'op': key[0], 'dims': key[1], 'dtypes': key[2], 'count-a': len(ua), 'count-b': len(ub), + 'avg-usec-a': f"{avg_a:.2f}", 'avg-usec-b': f"{avg_b:.2f}", 'delta-pct': fmt_pct(p), + 'tot-usec-a': tot_a, 'tot-usec-b': tot_b, 'delta-usec': f"{tot_b - tot_a:+d}", '_sort': abs(tot_b - tot_a)}) + rows.sort(key=lambda r: r['_sort'], reverse=True) + + print_table(f"Per-Group Compare (delta = B - A, top {top_n} sorted by abs delta usec)", [ + ("op", "Op"), ("dims", "Dims"), ("dtypes", "DTypes"), ("count-a", "Count A"), ("count-b", "Count B"), + ("avg-usec-a", "Avg usec A"), ("avg-usec-b", "Avg usec B"), ("delta-pct", "Delta %"), + ("tot-usec-a", "Tot usec A"), ("tot-usec-b", "Tot usec B"), ("delta-usec", "Delta usec")], rows[:top_n], width_overrides) + + # Groups present in only one log + for label, g, other in (("A", ga, gb), ("B", gb, ga)): + rows = [{'op': k[0], 'dims': k[1], 'dtypes': k[2], 'count': len(v), 'tot-usec': sum(v)} for k, v in g.items() if k not in other] + rows.sort(key=lambda r: r['tot-usec'], reverse=True) + print_table(f"Groups only in {label} (Top {top_n} sorted by tot usec)", [ + ("op", "Op"), ("dims", "Dims"), ("dtypes", "DTypes"), ("count", "Count"), ("tot-usec", "Tot usec")], rows[:top_n], width_overrides) + + +def filter_ops(ops, args): + if args.filter: + try: + filter_re = re.compile(args.filter) + except re.error as e: + logger.error(f"Invalid regex filter: {e}") + sys.exit(1) + ops = [op for op in ops if filter_re.search(op['op_text'])] + + if args.head is not None or args.tail is not None: + ops_by_dev = defaultdict(list) + for op in ops: + ops_by_dev[op['device']].append(op) + + filtered_ops = [] + for dev in sorted(ops_by_dev.keys()): + dev_ops = ops_by_dev[dev] + if args.head is not None: + dev_ops = dev_ops[:args.head] + elif args.tail is not None: + dev_ops = dev_ops[-args.tail:] + filtered_ops.extend(dev_ops) + ops = filtered_ops + + return ops + + def main(): parser = argparse.ArgumentParser(description="Post-process Op profile info.") parser.add_argument("logfile") @@ -642,6 +802,8 @@ def main(): help="Output ASCII art event summary or thread idle bubble analysis (default: summary)") parser.add_argument("--filter", type=str, help="Regex filter matching against the original profile-op line") parser.add_argument("--device", type=str, help="Device to filter by (e.g. HTP0, HTP0:0) or 'split' to generate separate reports per device") + parser.add_argument("--compare", type=str, metavar="LOGFILE", help="Compare against another log (logfile is baseline A, this is B)") + parser.add_argument("--min-delta", type=float, default=0, metavar="PCT", help="In compare mode hide rows with abs(Delta %%) below PCT") group = parser.add_mutually_exclusive_group() group.add_argument("--head", type=int, help="Limit to first N ops") @@ -655,6 +817,10 @@ def main(): logger.error(f"Cannot sort by '{args.sort}' without --pmu-index.") sys.exit(1) + if args.compare and (args.timeline or args.device == "split"): + logger.error("--compare cannot be used with --timeline or --device split.") + sys.exit(1) + overrides = {} if args.width: for w in args.width: @@ -681,6 +847,13 @@ def main(): if args.device and args.device != "split": ops = [op for op in ops if device_matches(op['device'], args.device)] + if args.compare: + ops_b = parse_log(args.compare, pmu_index=args.pmu_index, limit=limit, device_filter=device_filter, op_filter_re=op_filter_re) + if device_filter: + ops_b = [op for op in ops_b if device_matches(op['device'], device_filter)] + generate_compare_report(filter_ops(ops, args), filter_ops(ops_b, args), args.logfile, args.compare, args.top, overrides, args.min_delta) + return + if args.device == "split": unique_devices = sorted(list(set(op['device'] for op in ops))) for dev in unique_devices: @@ -712,28 +885,7 @@ def main(): else: generate_report(dev_ops, args.top, overrides, args.sort, pmu_name=final_pmu_name) else: - if args.filter: - try: - filter_re = re.compile(args.filter) - except re.error as e: - logger.error(f"Invalid regex filter: {e}") - sys.exit(1) - ops = [op for op in ops if filter_re.search(op['op_text'])] - - if args.head is not None or args.tail is not None: - ops_by_dev = defaultdict(list) - for op in ops: - ops_by_dev[op['device']].append(op) - - filtered_ops = [] - for dev in sorted(ops_by_dev.keys()): - dev_ops = ops_by_dev[dev] - if args.head is not None: - dev_ops = dev_ops[:args.head] - elif args.tail is not None: - dev_ops = dev_ops[-args.tail:] - filtered_ops.extend(dev_ops) - ops = filtered_ops + ops = filter_ops(ops, args) if args.timeline: for op in ops: