diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 6e714ff3b..605c23c68 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -5359,10 +5359,38 @@ static void ggml_hexagon_precompute_fused_mmidnx_params( kparams->n_weights = n_weights; } +static bool ggml_hexagon_op_mul_mat_use_fwht(const struct ggml_tensor * dst) { + if (dst->op != GGML_OP_MUL_MAT || ggml_get_op_params_i32(dst, 1) != GGML_HINT_SRC0_IS_HADAMARD) { + return false; + } + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + const int64_t n = src1->ne[0]; + if (n < 32 || (n & (n - 1)) != 0) { + return false; + } + + return src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && + (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) && + ggml_is_contiguous_rows(src1) && ggml_is_contiguous(dst) && + ggml_are_same_shape(src1, dst); +} + static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) { const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; + if (ggml_get_op_params_i32(dst, 1) == GGML_HINT_SRC0_IS_HADAMARD) { + if (!ggml_hexagon_op_mul_mat_use_fwht(dst)) { + return false; + } + struct htp_unary_kernel_params ukparams; + ggml_hexagon_precompute_unary_params(sess, HTP_OP_HADAMARD, src1, nullptr, dst, &ukparams); + return ukparams.vtcm_row_per_thread > 0 && (size_t) ukparams.vtcm_size <= sess->vtcm_size; + } + if (dst->type != GGML_TYPE_F32) { return false; } @@ -5921,8 +5949,9 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * const struct ggml_tensor * src1 = op->src[1]; // indices const struct ggml_tensor * dst = op; - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && - src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32 && + src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && + src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K ) { return false; } @@ -6666,7 +6695,11 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) { static htp_op_code op_remap_to_htp(const ggml_tensor * t) { switch (t->op) { case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT; - case GGML_OP_MUL_MAT: return HTP_OP_MUL_MAT; + case GGML_OP_MUL_MAT: + if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) { + return HTP_OP_HADAMARD; + } + return HTP_OP_MUL_MAT; case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID; case GGML_OP_MUL: return HTP_OP_MUL; case GGML_OP_ADD: return HTP_OP_ADD; @@ -6795,6 +6828,7 @@ static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const st static bool is_mergeable_mul_mat(const ggml_tensor * t) { if (t->op != GGML_OP_MUL_MAT) return false; + if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) return false; const ggml_tensor * src0 = t->src[0]; const ggml_tensor * src1 = t->src[1]; @@ -6897,7 +6931,9 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE; - } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT || graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { + } else if ((graph->nodes[i]->op == GGML_OP_MUL_MAT && + ggml_get_op_params_i32(graph->nodes[i], 1) != GGML_HINT_SRC0_IS_HADAMARD) || + graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { if ((i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_ADD && ggml_can_fuse(graph, i, { graph->nodes[i]->op, GGML_OP_ADD })) || ggml_node_has_n_uses(graph, i, 1)) { extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE; diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index d62ce1d00..57cd5f208 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -47,13 +47,19 @@ struct htp_opnode { void init(ggml_tensor * node) { this->node = node; if (this->node) { - this->name = ggml_op_desc(this->node); + if (this->node->op == GGML_OP_MUL_MAT && + ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) { + this->name = "MUL_MAT_HADAMARD"; + this->inputs.push_back(this->node->src[1]); + } else { + this->name = ggml_op_desc(this->node); - // Build inputs (preserving optional nullptrs) - int n_inputs = n_active_src(this->node); - this->inputs.resize(n_inputs, nullptr); - for (int i = 0; i < n_inputs; i++) { - this->inputs[i] = this->node->src[i]; + // Build inputs (preserving optional nullptrs) + int n_inputs = n_active_src(this->node); + this->inputs.resize(n_inputs, nullptr); + for (int i = 0; i < n_inputs; i++) { + this->inputs[i] = this->node->src[i]; + } } // Build outputs @@ -124,6 +130,10 @@ struct htp_opnode { bool stackable() const { switch (this->op()) { case GGML_OP_MUL_MAT: + if (ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) { + return false; + } + return ggml_is_quantized(this->src0()->type); case GGML_OP_MUL_MAT_ID: return ggml_is_quantized(this->src0()->type); default: diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index 27948c37a..78eb1bb50 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -288,7 +288,7 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat } const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); - const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS; + const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type); const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread; uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread; @@ -450,7 +450,9 @@ int op_get_rows(struct htp_ops_context * octx) { break; case HTP_GET_ROWS_KERNEL_TILED: switch (octx->src[0]->type) { - case HTP_TYPE_F16: grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break; + case HTP_TYPE_F16: grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break; + case HTP_TYPE_Q1_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q1_0_f32; break; + case HTP_TYPE_Q2_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q2_0_f32; break; case HTP_TYPE_Q8_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q8_0_f32; break; case HTP_TYPE_Q4_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_0_f32; break; case HTP_TYPE_Q4_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_k_f32; break; @@ -466,6 +468,8 @@ int op_get_rows(struct htp_ops_context * octx) { switch (octx->src[0]->type) { case HTP_TYPE_F16: grctx.compute_flat = hvx_dequantize_row_flat_f16_f32; break; case HTP_TYPE_BF16: grctx.compute_flat = hvx_dequantize_row_flat_bf16_f32; break; + case HTP_TYPE_Q1_0: grctx.compute_flat = hvx_dequantize_row_flat_q1_0_f32; break; + case HTP_TYPE_Q2_0: grctx.compute_flat = hvx_dequantize_row_flat_q2_0_f32; break; case HTP_TYPE_Q8_0: grctx.compute_flat = hvx_dequantize_row_flat_q8_0_f32; break; case HTP_TYPE_Q4_0: grctx.compute_flat = hvx_dequantize_row_flat_q4_0_f32; break; case HTP_TYPE_Q4_K: grctx.compute_flat = hvx_dequantize_row_flat_q4_k_f32; break; diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h index f44e58d16..94f3c15f2 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h @@ -64,7 +64,7 @@ static inline void htp_get_rows_vtcm_layout_build( size_t src0_row_size = 0; if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) { const size_t tile_stride = htp_tensor_get_tile_padded_size(type); - const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS; + const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(type); src0_row_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride; } else { src0_row_size = htp_tensor_get_row_size(type, ne00); diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index 3fc558119..9b4b86d3b 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -114,6 +114,7 @@ enum htp_op_code { HTP_OP_POOL_1D, HTP_OP_UNARY_GELU_QUICK, HTP_OP_IM2COL_3D, + HTP_OP_HADAMARD, HTP_OP_INVALID }; diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h index 842001c85..9b2e13133 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h @@ -29,29 +29,27 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); - HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); - HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); - HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); + + HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); + HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); + HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); + HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); + vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); + + HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); + HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); - HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); - HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); - - HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); - HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); - - HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); - HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); - - HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); + HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); + HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); + HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); - vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); - vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); + vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); @@ -81,25 +79,10 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ HVX_Vector voff01_hf = hvx_vec_f32_to_f16(voff0_sf, voff1_sf); HVX_Vector voff23_hf = hvx_vec_f32_to_f16(voff2_sf, voff3_sf); - HVX_Vector r_scale[4] = { - hvx_vec_repl_f16(vd01_hf), - hvx_vec_repl_f16(Q6_V_vror_VR(vd01_hf, 64)), - hvx_vec_repl_f16(vd23_hf), - hvx_vec_repl_f16(Q6_V_vror_VR(vd23_hf, 64)), - }; - HVX_Vector r_offset[4] = { - hvx_vec_repl_f16(voff01_hf), - hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 64)), - hvx_vec_repl_f16(voff23_hf), - hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 64)), - }; + int32_t __attribute__((aligned(128))) ws[32]; if (sums16) { // |sum of 16 int8| <= 2032 fits a halfword - int32_t __attribute__((aligned(128))) ws[32]; *(HVX_Vector *) ws = v_sums; - for (int b = 0; b < 4; b++) { - r_offset[b] = Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF)); - } } static const uint8_t __attribute__((aligned(128))) repl[128] = { @@ -114,8 +97,15 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ }; HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; + #pragma unroll for (int b = 0; b < 4; b++) { HVX_Vector v_act = Q6_V_vror_VR(vx_i8, b * 32); + HVX_Vector vd_b = (b < 2) ? vd01_hf : vd23_hf; + HVX_Vector voff_b = (b < 2) ? voff01_hf : voff23_hf; + HVX_Vector r_scale = hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(vd_b, 64) : vd_b); + HVX_Vector r_offset = sums16 + ? Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF)) + : hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(voff_b, 64) : voff_b); HVX_Vector r0 = Q6_V_vdelta_VV(v_act, v_repl_ctrl); HVX_Vector r1 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 4), v_repl_ctrl); @@ -135,8 +125,8 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ dst[5] = r5; dst[6] = r6; dst[7] = r7; - dst[8] = r_scale[b]; - dst[9] = r_offset[b]; + dst[8] = r_scale; + dst[9] = r_offset; } } @@ -932,7 +922,7 @@ static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; - HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector v_sum_qf = Q6_V_vzero(); HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); @@ -951,9 +941,10 @@ static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const } HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); - v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf)); + v_sum_qf = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf)); } + HVX_Vector v_sum_float = Q6_Vsf_equals_Vqf32(v_sum_qf); if (sz) { hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); } else { @@ -966,16 +957,16 @@ static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, float const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; - HVX_Vector v_sum_float_c0 = Q6_V_vzero(); - HVX_Vector v_sum_float_c1 = Q6_V_vzero(); - HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); - HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + HVX_Vector v_sum_qf_c0 = Q6_V_vzero(); + HVX_Vector v_sum_qf_c1 = Q6_V_vzero(); + HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0; for (uint32_t kt = 0; kt < n_k_tiles; kt++) { const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0)); - HVX_Vector v_tile_c0 = Q6_V_vzero(); - HVX_Vector v_tile_c1 = Q6_V_vzero(); + HVX_Vector v_tile_qf_c0 = Q6_V_vzero(); + HVX_Vector v_tile_qf_c1 = Q6_V_vzero(); for (int quarter = 0; quarter < 4; quarter++) { const uint32_t act_tile = 4 * kt + quarter; @@ -987,15 +978,17 @@ static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, float HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8])); HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8])); - v_tile_c0 = hvx_vec_add_f32_f32(v_tile_c0, hvx_vec_mul_f32_f32(v_sum_sf_c0, v_da_sf_c0)); - v_tile_c1 = hvx_vec_add_f32_f32(v_tile_c1, hvx_vec_mul_f32_f32(v_sum_sf_c1, v_da_sf_c1)); + v_tile_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c0, v_da_sf_c0)); + v_tile_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c1, v_da_sf_c1)); } HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(v_tile_c0, v_dw_sf)); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(v_tile_c1, v_dw_sf)); + v_sum_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c0, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf)); + v_sum_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c1, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf)); } + HVX_Vector v_sum_float_c0 = Q6_Vsf_equals_Vqf32(v_sum_qf_c0); + HVX_Vector v_sum_float_c1 = Q6_Vsf_equals_Vqf32(v_sum_qf_c1); if (sz0) { hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); } else { diff --git a/ggml/src/ggml-hexagon/htp/hvx-quant.h b/ggml/src/ggml-hexagon/htp/hvx-quant.h index b7837967a..b0ca6ecf6 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-quant.h +++ b/ggml/src/ggml-hexagon/htp/hvx-quant.h @@ -191,6 +191,64 @@ static inline void hvx_dequantize_row_flat_bf16_f32(float * restrict dst_ptr, co } } +static inline void hvx_dequantize_row_flat_q1_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + static const uint32_t __attribute__((aligned(128))) bit_mask[32] = { + 1u << 0, 1u << 1, 1u << 2, 1u << 3, 1u << 4, 1u << 5, 1u << 6, 1u << 7, + 1u << 8, 1u << 9, 1u << 10, 1u << 11, 1u << 12, 1u << 13, 1u << 14, 1u << 15, + 1u << 16, 1u << 17, 1u << 18, 1u << 19, 1u << 20, 1u << 21, 1u << 22, 1u << 23, + 1u << 24, 1u << 25, 1u << 26, 1u << 27, 1u << 28, 1u << 29, 1u << 30, 1u << 31, + }; + const int nb = n / QK1_0; + const block_q1_0 * src = (const block_q1_0 *) src_ptr; + const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask; + const HVX_Vector v_zero = Q6_V_vzero(); + const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000); + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16)); + HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit); + HVX_Vector v_qs = *(const HVX_UVector *) src[i].qs; + + float * block_dst = dst_ptr + i * QK1_0; + #pragma unroll + for (int q = 0; q < 4; q++) { + HVX_Vector v_bits = hvx_vec_repl_u32(q == 0 ? v_qs : Q6_V_vror_VR(v_qs, 4 * q)); + HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero); + hvx_vmem(block_dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd); + } + } +} + +static inline void hvx_dequantize_row_flat_q2_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK2_0; + const block_q2_0 * src = (const block_q2_0 *) src_ptr; + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector v_one = Q6_Vb_vsplat_R(1); + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16)); + + HVX_Vector vq = *(const HVX_UVector *) src[i].qs; + HVX_Vector v0 = Q6_V_vand_VV(vq, mask_03); + HVX_Vector v1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 2), mask_03); + HVX_Vector v2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 4), mask_03); + HVX_Vector v3 = Q6_Vub_vlsr_VubR(vq, 6); + + HVX_Vector v01 = Q6_V_lo_W(Q6_W_vshuff_VVR(v1, v0, -1)); + HVX_Vector v23 = Q6_V_lo_W(Q6_W_vshuff_VVR(v3, v2, -1)); + HVX_Vector v64 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(v23, v01, -2)), v_one); + + HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v64); + HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + + float * block_dst = dst_ptr + i * QK2_0; + hvx_vmem(block_dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), vd); + hvx_vmem(block_dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), vd); + } +} + static inline void hvx_dequantize_row_flat_q4_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { const int nb = n / QK4_0; const block_q4_0 * src = (const block_q4_0 *) src_ptr; @@ -603,4 +661,66 @@ static inline void hvx_dequantize_row_tiled_f16_f32(float * dst_spad, const uint } } +static inline void hvx_dequantize_row_tiled_q1_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + static const uint32_t __attribute__((aligned(128))) bit_mask[32] = { + 1u << 0, 1u << 8, 1u << 16, 1u << 24, + 1u << 1, 1u << 9, 1u << 17, 1u << 25, + 1u << 2, 1u << 10, 1u << 18, 1u << 26, + 1u << 3, 1u << 11, 1u << 19, 1u << 27, + 1u << 4, 1u << 12, 1u << 20, 1u << 28, + 1u << 5, 1u << 13, 1u << 21, 1u << 29, + 1u << 6, 1u << 14, 1u << 22, 1u << 30, + 1u << 7, 1u << 15, 1u << 23, 1u << 31, + }; + const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask; + const HVX_Vector v_zero = Q6_V_vzero(); + const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q1_0; + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit); + + #pragma unroll + for (int q = 0; q < 4; ++q) { + const HVX_Vector v_bits = hvx_vec_repl_u32(Q6_V_vror_VR(hvx_vmem(tile + q * VLEN), 4 * row)); + const HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero); + hvx_vmem(dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd); + } + } +} + +static inline void hvx_dequantize_row_tiled_q2_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector v_one = Q6_Vb_vsplat_R(1); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q2_0; + + const HVX_Vector v_packed = hvx_dequantize_tiled_f16_gather4(tile, 4 * row); + const HVX_Vector p0 = Q6_V_vand_VV(v_packed, mask_03); + const HVX_Vector p1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 2), mask_03); + const HVX_Vector p2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 4), mask_03); + const HVX_Vector p3 = Q6_Vub_vlsr_VubR(v_packed, 6); + + const HVX_Vector p01 = Q6_V_lo_W(Q6_W_vshuff_VVR(p1, p0, -4)); + const HVX_Vector p23 = Q6_V_lo_W(Q6_W_vshuff_VVR(p3, p2, -4)); + const HVX_Vector vq = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(p23, p01, -8)), v_one); + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + hvx_vmem(dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + hvx_vmem(dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), scale); + } +} + #endif // HVX_QUANT_H diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 2374add3f..b72ef2b75 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -940,6 +940,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_UNARY_RELU: case HTP_OP_UNARY_STEP: case HTP_OP_L2_NORM: + case HTP_OP_HADAMARD: return op_unary(octx); case HTP_OP_GLU_SWIGLU: diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 37df31ddb..ea3d92ad0 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -751,6 +751,143 @@ static void l2_norm_f32(const void * restrict src, } } +static inline HVX_VectorPair hvx_fwht_stage_pair(HVX_Vector v0, HVX_Vector v1, int32_t b) { + HVX_VectorPair p = Q6_W_vshuff_VVR(v1, v0, b); + HVX_Vector s = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p)); + HVX_Vector d = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p)); + return Q6_W_vshuff_VVR(d, s, b); +} + +static inline HVX_VectorPair hvx_fwht32_pair(HVX_Vector v0_sf, HVX_Vector v1_sf, HVX_Vector v_scale) { + HVX_Vector v0 = Q6_Vqf32_vmpy_VsfVsf(v0_sf, v_scale); + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(v1_sf, v_scale); + HVX_VectorPair p; + p = hvx_fwht_stage_pair(v0, v1, 4); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 8); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 16); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 32); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 64); + return p; +} + +static inline void hvx_fwht_store32(HVX_Vector * restrict v_dst, HVX_VectorPair p) { + v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); +} + +static inline void hvx_fwht_store64(HVX_Vector * restrict v_dst, HVX_VectorPair p) { + HVX_Vector u = Q6_V_lo_W(p); + HVX_Vector v = Q6_V_hi_W(p); + v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(u, v)); + v_dst[1] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(u, v)); +} + +static inline void hvx_fwht_tail_f32(HVX_Vector * restrict v_dst, uint32_t nvec) { + uint32_t len_vec = 2; + for (; len_vec * 2 < nvec; len_vec <<= 2) { + const uint32_t step = len_vec * 4; + for (uint32_t i = 0; i < nvec; i += step) { + for (uint32_t j = 0; j < len_vec; j++) { + HVX_Vector a0 = v_dst[i + j]; + HVX_Vector a1 = v_dst[i + j + len_vec]; + HVX_Vector a2 = v_dst[i + j + 2 * len_vec]; + HVX_Vector a3 = v_dst[i + j + 3 * len_vec]; + + HVX_Vector t0 = Q6_Vqf32_vadd_VsfVsf(a0, a1); + HVX_Vector t1 = Q6_Vqf32_vsub_VsfVsf(a0, a1); + HVX_Vector t2 = Q6_Vqf32_vadd_VsfVsf(a2, a3); + HVX_Vector t3 = Q6_Vqf32_vsub_VsfVsf(a2, a3); + + v_dst[i + j] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t0, t2)); + v_dst[i + j + len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t1, t3)); + v_dst[i + j + 2 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t0, t2)); + v_dst[i + j + 3 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t1, t3)); + } + } + } + + if (len_vec < nvec) { + for (uint32_t i = 0; i < nvec; i += 2 * len_vec) { + for (uint32_t j = 0; j < len_vec; j++) { + HVX_Vector u = v_dst[i + j]; + HVX_Vector v = v_dst[i + len_vec + j]; + v_dst[i + j] = hvx_vec_add_f32_f32(u, v); + v_dst[i + len_vec + j] = hvx_vec_sub_f32_f32(u, v); + } + } + } +} + +static inline void hvx_fwht_row_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const uint32_t nvec = n / VLEN_FP32; + + if (nvec == 1) { + HVX_VectorPair p = hvx_fwht32_pair(v_src[0], v_src[0], v_scale); + hvx_fwht_store32(v_dst, p); + return; + } + + for (uint32_t i = 0; i < nvec; i += 2) { + HVX_VectorPair p = hvx_fwht32_pair(v_src[i], v_src[i + 1], v_scale); + hvx_fwht_store64(&v_dst[i], p); + } + + hvx_fwht_tail_f32(v_dst, nvec); +} + +static inline void hvx_fwht_row_f16(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const uint32_t nvec = n / VLEN_FP32; + + if (nvec == 1) { + HVX_Vector v0 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src[0])); + HVX_VectorPair p = hvx_fwht32_pair(v0, v0, v_scale); + hvx_fwht_store32(v_dst, p); + return; + } + + const uint32_t nvec_f16 = n / VLEN_FP16; + for (uint32_t i = 0; i < nvec_f16; i++) { + HVX_VectorPair f32p = hvx_vec_f16_to_f32(v_src[i]); + HVX_VectorPair p = hvx_fwht32_pair(Q6_V_lo_W(f32p), Q6_V_hi_W(f32p), v_scale); + hvx_fwht_store64(&v_dst[2 * i], p); + } + + hvx_fwht_tail_f32(v_dst, nvec); +} + +static void hadamard_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned); + + hvx_fwht_row_f32(src_row, dst_row, ne0, v_scale); + } +} + +static void hadamard_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned); + + hvx_fwht_row_f16(src_row, dst_row, ne0, v_scale); + } +} + static void tanh_f32(const void * restrict src, void * restrict dst, const uint32_t num_rows, @@ -1644,6 +1781,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_STEP: op_type = is_f16 ? "step-f16" : "step-f32"; break; case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break; case HTP_OP_TRI: op_type = "tri-f32"; break; + case HTP_OP_HADAMARD: op_type = is_f16 ? "hadamard-f16" : "hadamard-f32"; break; default: FARF(ERROR, "Unsupported unary Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; @@ -1663,6 +1801,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_ABS: case HTP_OP_UNARY_LOG: case HTP_OP_UNARY_STEP: + case HTP_OP_HADAMARD: break; default: FARF(ERROR, "unary-%s: not supported for F16\n", op_type); @@ -1677,16 +1816,17 @@ static int execute_op_unary(struct htp_ops_context * octx) { } const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); - const size_t src0_data_row_size = src0->ne[0] * elem_size; - const size_t dst_data_row_size = dst->ne[0] * elem_size; + const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); + const size_t dst_elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float); + const size_t src0_data_row_size = src0->ne[0] * src0_elem_size; + const size_t dst_data_row_size = dst->ne[0] * dst_elem_size; uint32_t row_start = 0; uint32_t nrows = src0_nrows; if (octx->ctx->mdev.count > 1) { uint32_t rows_per_chunk = 0; - htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk); + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst_elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk); const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); row_start = range.start; nrows = range.count; @@ -1813,6 +1953,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_ABS: compute_func = (void *) abs_f16; break; case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break; case HTP_OP_UNARY_STEP: compute_func = (void *) step_f16; break; + case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f16; break; default: break; } } else { @@ -1843,6 +1984,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break; case HTP_OP_UNARY_STEP: compute_func = (void *) step_f32; break; case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break; + case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f32; break; case HTP_OP_TRI: task_func = unary_thread_tri_f32; compute_func = (void *) tri_f32; diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 7e5779e5a..902028635 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -64,6 +64,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) { case HTP_OP_UNARY_STEP: case HTP_OP_L2_NORM: case HTP_OP_TRI: + case HTP_OP_HADAMARD: return true; default: return false; @@ -94,8 +95,9 @@ static inline void htp_unary_vtcm_layout_build( uint32_t * out_col_tile, uint32_t * out_vtcm_row_per_thread ) { + const size_t dst_elem_size = (op == HTP_OP_HADAMARD) ? sizeof(float) : elem_size; const size_t src0_data_row_size = ne00 * elem_size; - const size_t dst_data_row_size = ne10 * elem_size; + const size_t dst_data_row_size = ne10 * dst_elem_size; const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); @@ -131,7 +133,8 @@ static inline void htp_unary_vtcm_layout_build( } const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM || - op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM); + op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM || + op == HTP_OP_HADAMARD); // The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*); // F16 has no tiled kernels, so it must stay on the row-block path like reduction ops. // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM