diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index d347757e1..8327c52e6 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -650,9 +650,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name) # Fetch KleidiAI sources: include(FetchContent) - set(KLEIDIAI_COMMIT_TAG "v1.31.0") + set(KLEIDIAI_COMMIT_TAG "v1.32.0") set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz") - set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "930495f3aeb16b25dc18a2bde45924f4") + set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "05f1953428ffe8e8c7f585dc6d508887") set(KLEIDIAI_FETCH_ARGS URL ${KLEIDIAI_DOWNLOAD_URL} diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cpp index 89e95968c..24aa28244 100644 --- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp @@ -201,85 +201,117 @@ static struct kai_matmul_uker_config q4_k_matmul_config() { return config; } -template -static size_t q4_k_get_m_step() { - const struct kai_matmul_uker_api api = GetApi(); - const struct kai_matmul_uker_config config = q4_k_matmul_config(); - return api.get_step(&config).m; +static struct kai_matmul_uker_config default_matmul_config() { + return {}; } -template -static size_t q4_k_get_n_step() { - const struct kai_matmul_uker_api api = GetApi(); - const struct kai_matmul_uker_config config = q4_k_matmul_config(); - return api.get_step(&config).n; -} +template< + struct kai_matmul_uker_api (*GetApi)(void), + struct kai_matmul_uker_config (*GetConfig)(void), + size_t MR, size_t NR, size_t KR, size_t SR> +struct matmul_uker_adapter { + static size_t get_m_step() { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + return api.get_step(&config).m; + } -template -static size_t q4_k_get_constant() { - return Value; -} + static size_t get_n_step() { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + return api.get_step(&config).n; + } -template -static size_t q4_k_get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) { - const struct kai_matmul_uker_api api = GetApi(); - const struct kai_matmul_uker_config config = q4_k_matmul_config(); - const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx }; - const struct kai_matmul_uker_dst_stride_args stride = { dst_stride }; - return api.get_dst_offset(&config, &index, &stride); -} + static size_t get_mr() { + if constexpr (MR == 0) { + return get_m_step(); + } + return MR; + } -template -static size_t q4_k_get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) { - GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + static size_t get_nr() { + if constexpr (NR == 0) { + return get_n_step(); + } + return NR; + } - const struct kai_matmul_uker_api api = GetApi(); - const struct kai_matmul_uker_config config = q4_k_matmul_config(); - const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k }; - const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 }; - const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape); - return api.get_lhs_offset(&config, &index, &stride); -} + static size_t get_kr() { + return KR; + } -template -static size_t q4_k_get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) { - GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + static size_t get_sr() { + return SR; + } - const struct kai_matmul_uker_api api = GetApi(); - const struct kai_matmul_uker_config config = q4_k_matmul_config(); - const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k }; - const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 }; - const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape); - return api.get_rhs_offset(&config, &index, &stride); -} + static size_t get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx }; + const struct kai_matmul_uker_dst_stride_args stride = { dst_stride }; + return api.get_dst_offset(&config, &index, &stride); + } -template -static void q4_k_run_matmul( - size_t m, size_t n, size_t k, size_t bl, - const void * lhs_packed, const void * rhs_packed, - void * dst, size_t dst_stride_row, size_t dst_stride_col, - float clamp_min, float clamp_max) { - GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); - GGML_ASSERT(dst_stride_col == sizeof(float)); + static size_t get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + GGML_ASSERT(bl == config.format.bl); + const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k }; + const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 }; + const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape); + return api.get_lhs_offset(&config, &index, &stride); + } - const struct kai_matmul_uker_api api = GetApi(); - const struct kai_matmul_uker_config config = q4_k_matmul_config(); - const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k }; - const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k }; + static size_t get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + GGML_ASSERT(bl == config.format.bl); + const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k }; + const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 }; + const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape); + return api.get_rhs_offset(&config, &index, &stride); + } - struct kai_matmul_uker_args args = {}; - args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP; - args.shape = { m, n, k }; - args.operand.lhs.ptr = lhs_packed; - args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape); - args.operand.rhs.ptr = rhs_packed; - args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape); - args.operand.dst.ptr = dst; - args.operand.dst.stride.m = dst_stride_row; - args.activation.clamp.min_ptr = &clamp_min; - args.activation.clamp.max_ptr = &clamp_max; - api.run(&config, &args); -} + static void run( + size_t m, size_t n, size_t k, size_t bl, + const void * lhs_packed, const void * rhs_packed, + void * dst, size_t dst_stride_row, size_t dst_stride_col, + float clamp_min, float clamp_max) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + GGML_ASSERT(bl == config.format.bl); + GGML_ASSERT(dst_stride_col == sizeof(float)); + const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k }; + const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k }; + + struct kai_matmul_uker_args args = {}; + args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP; + args.shape = { m, n, k }; + args.operand.lhs.ptr = lhs_packed; + args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape); + args.operand.rhs.ptr = rhs_packed; + args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape); + args.operand.dst.ptr = dst; + args.operand.dst.stride.m = dst_stride_row; + args.activation.clamp.min_ptr = &clamp_min; + args.activation.clamp.max_ptr = &clamp_max; + api.run(&config, &args); + } +}; + +using q4_k_gemm_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa, + q4_k_matmul_config, 0, 0, 4, 2>; +using q4_k_gemv_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot, + q4_k_matmul_config, 0, 0, 4, 2>; +// GEMM and GEMV share the RHS layout only at SVE256 (NR=8, KR=8, SR=1). +using sve_q8_gemm_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_qai8dxp4x8sf32_qsi8cxp4vsx8sf32bf32_16x4vs_sve_i8mm, + default_matmul_config, 4, 0, 8, 1>; +using sve_q8_gemv_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp8x8_1x8_sve_dot, + default_matmul_config, 1, 8, 8, 1>; static struct kai_matmul_pack_lhs_uker_config q4_k_lhs_pack_config() { struct kai_matmul_pack_lhs_uker_config config = {}; @@ -628,7 +660,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .packed_size_ex = */ &rhs_ps_fn5, /* .pack_func_ex = */ &rhs_pack_fn12, }, - /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, + /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD, /* .lhs_type = */ GGML_TYPE_F32, /* .rhs_type = */ GGML_TYPE_Q4_0, /* .op_type = */ GGML_TYPE_F32, @@ -737,16 +769,16 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = { { /* SME2 GEMM */ { - /* .get_m_step = */ &q4_k_get_m_step, - /* .get_n_step = */ &q4_k_get_n_step, - /* .get_mr = */ &q4_k_get_m_step, - /* .get_nr = */ &q4_k_get_n_step, - /* .get_kr = */ &q4_k_get_constant<4>, - /* .get_sr = */ &q4_k_get_constant<2>, - /* .get_dst_offset = */ &q4_k_get_dst_offset, - /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset, - /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset, - /* .run_kernel_ex = */ &q4_k_run_matmul, + /* .get_m_step = */ q4_k_gemm_adapter::get_m_step, + /* .get_n_step = */ q4_k_gemm_adapter::get_n_step, + /* .get_mr = */ q4_k_gemm_adapter::get_mr, + /* .get_nr = */ q4_k_gemm_adapter::get_nr, + /* .get_kr = */ q4_k_gemm_adapter::get_kr, + /* .get_sr = */ q4_k_gemm_adapter::get_sr, + /* .get_dst_offset = */ q4_k_gemm_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ q4_k_gemm_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ q4_k_gemm_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ q4_k_gemm_adapter::run, }, { /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon, @@ -756,16 +788,16 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = { }, /* SME2 GEMV */ { - /* .get_m_step = */ &q4_k_get_m_step, - /* .get_n_step = */ &q4_k_get_n_step, - /* .get_mr = */ &q4_k_get_m_step, - /* .get_nr = */ &q4_k_get_n_step, - /* .get_kr = */ &q4_k_get_constant<4>, - /* .get_sr = */ &q4_k_get_constant<2>, - /* .get_dst_offset = */ &q4_k_get_dst_offset, - /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset, - /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset, - /* .run_kernel_ex = */ &q4_k_run_matmul, + /* .get_m_step = */ q4_k_gemv_adapter::get_m_step, + /* .get_n_step = */ q4_k_gemv_adapter::get_n_step, + /* .get_mr = */ q4_k_gemv_adapter::get_mr, + /* .get_nr = */ q4_k_gemv_adapter::get_nr, + /* .get_kr = */ q4_k_gemv_adapter::get_kr, + /* .get_sr = */ q4_k_gemv_adapter::get_sr, + /* .get_dst_offset = */ q4_k_gemv_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ q4_k_gemv_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ q4_k_gemv_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ q4_k_gemv_adapter::run, }, { /* .get_offset = */ q4_k_lhs_get_offset, @@ -883,6 +915,54 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .rhs_type = */ GGML_TYPE_Q8_0, /* .op_type = */ GGML_TYPE_F32, }, + { + /* SVE I8MM GEMM */ + { + /* .get_m_step = */ sve_q8_gemm_adapter::get_m_step, + /* .get_n_step = */ sve_q8_gemm_adapter::get_n_step, + /* .get_mr = */ sve_q8_gemm_adapter::get_mr, + /* .get_nr = */ sve_q8_gemm_adapter::get_nr, + /* .get_kr = */ sve_q8_gemm_adapter::get_kr, + /* .get_sr = */ sve_q8_gemm_adapter::get_sr, + /* .get_dst_offset = */ sve_q8_gemm_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ sve_q8_gemm_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ sve_q8_gemm_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ sve_q8_gemm_adapter::run, + }, + /* .gemm_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32, + /* .get_packed_offset_ex = */ &lhs_offs_fn5, + /* .packed_size_ex = */ &lhs_ps_fn5, + /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, + }, + /* SVE DOT GEMV */ + { + /* .get_m_step = */ sve_q8_gemv_adapter::get_m_step, + /* .get_n_step = */ sve_q8_gemv_adapter::get_n_step, + /* .get_mr = */ sve_q8_gemv_adapter::get_mr, + /* .get_nr = */ sve_q8_gemv_adapter::get_nr, + /* .get_kr = */ sve_q8_gemv_adapter::get_kr, + /* .get_sr = */ sve_q8_gemv_adapter::get_sr, + /* .get_dst_offset = */ sve_q8_gemv_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ sve_q8_gemv_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ sve_q8_gemv_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ sve_q8_gemv_adapter::run, + }, + /* .gemv_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32, + /* .get_packed_offset_ex = */ &lhs_offs_fn5, + /* .packed_size_ex = */ &lhs_ps_fn5, + /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, + }, + /* .rhs_info = */ { + /* .packed_size_ex = */ &rhs_ps_fn5, + /* .pack_func_ex = */ &rhs_pack_scale_fn12, + }, + /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD, + /* .lhs_type = */ GGML_TYPE_F32, + /* .rhs_type = */ GGML_TYPE_Q8_0, + /* .op_type = */ GGML_TYPE_F32, + }, { /* I8MM GEMM */ { diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.h b/ggml/src/ggml-cpu/kleidiai/kernels.h index 308ea9451..ac5d5d302 100644 --- a/ggml/src/ggml-cpu/kleidiai/kernels.h +++ b/ggml/src/ggml-cpu/kleidiai/kernels.h @@ -13,7 +13,8 @@ enum cpu_feature { CPU_FEATURE_SVE = 4, CPU_FEATURE_SME = 8, CPU_FEATURE_SME2 = 16, - CPU_FEATURE_FP16 = 32 + CPU_FEATURE_FP16 = 32, + CPU_FEATURE_SVEI8MM = 64 }; inline cpu_feature& operator|=(cpu_feature& lhs, cpu_feature rhs) { diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp index dd9bcbf63..57a4a5662 100644 --- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp @@ -319,8 +319,10 @@ static void init_kleidiai_context(void) { size_t detected_smcus = 0; + // The fixed-width SVE kernels require 32-byte vectors, including the Q8 shared RHS layout. ctx.features = (runtime_feat.has_dotprod ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) | (runtime_feat.has_i8mm ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) | + (runtime_feat.has_svei8mm ? CPU_FEATURE_SVEI8MM : CPU_FEATURE_NONE) | (runtime_feat.has_fp16 ? CPU_FEATURE_FP16 : CPU_FEATURE_NONE) | (runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE); @@ -1616,6 +1618,11 @@ public: const size_t kr = kernel->get_kr(); const size_t sr = kernel->get_sr(); const ggml_type rhs_type = kernels->rhs_type; + if (rhs_type == GGML_TYPE_Q8_0 && (kernels->required_cpu & CPU_FEATURE_SVE)) { + GGML_ASSERT(nr == kernels->gemv.get_nr()); + GGML_ASSERT(kr == kernels->gemv.get_kr()); + GGML_ASSERT(sr == kernels->gemv.get_sr()); + } const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL : diff --git a/ggml/src/ggml-feats.h b/ggml/src/ggml-feats.h index 79a0afd87..c45e4ad08 100644 --- a/ggml/src/ggml-feats.h +++ b/ggml/src/ggml-feats.h @@ -22,6 +22,10 @@ #define HWCAP2_I8MM (1ULL << 13) #endif +#if !defined(HWCAP2_SVEI8MM) +#define HWCAP2_SVEI8MM (1ULL << 9) +#endif + #if !defined(HWCAP_ASIMDDP) #define HWCAP_ASIMDDP (1 << 20) #endif @@ -63,6 +67,10 @@ #define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47 #endif +#if !defined(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE 57 +#endif + #if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) #define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66 #endif @@ -87,6 +95,7 @@ typedef struct ggml_feats_arch64_runtime { bool has_sve; bool has_sve2; bool has_i8mm; + bool has_svei8mm; bool has_sme; bool has_sme2; int sve_cnt; @@ -104,6 +113,7 @@ static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) { runtime_feat.has_sve = !!(hwcap & HWCAP_SVE); runtime_feat.has_sve2 = !!(hwcap2 & HWCAP2_SVE2); runtime_feat.has_i8mm = !!(hwcap2 & HWCAP2_I8MM); + runtime_feat.has_svei8mm = !!(hwcap2 & HWCAP2_SVEI8MM); runtime_feat.has_sme = !!(hwcap2 & HWCAP2_SME); runtime_feat.has_sme2 = !!(hwcap2 & HWCAP2_SME2); @@ -153,6 +163,7 @@ static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) { runtime_feat.has_sve = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0; runtime_feat.has_sve2 = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0; runtime_feat.has_i8mm = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_svei8mm = IsProcessorFeaturePresent(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE) != 0; runtime_feat.has_sme = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0; runtime_feat.has_sme2 = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0;