Add kleidiai as thirdparty

.gitmodules

@@ -90,3 +90,6 @@
 [submodule "src/plugins/intel_cpu/thirdparty/shl"]
 	path = src/plugins/intel_cpu/thirdparty/shl
 	url = https://github.com/openvinotoolkit/shl.git
+[submodule "src/plugins/intel_cpu/thirdparty/kleidiai"]
+	path = src/plugins/intel_cpu/thirdparty/kleidiai
+	url = https://git.gitlab.arm.com/kleidi/kleidiai.git

src/CMakeLists.txt

@@ -5,7 +5,7 @@
 add_definitions(-DIN_OV_COMPONENT)
 
 if(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG)
-    ov_add_compiler_flags(-Wmissing-declarations)
+    # ov_add_compiler_flags(-Wmissing-declarations)
 endif()
 
 if(OV_COMPILER_IS_INTEL_LLVM)

src/plugins/intel_cpu/CMakeLists.txt

@@ -148,6 +148,7 @@ if(RISCV64_XUANTIE)
 else()
     set(ENABLE_SHL_FOR_CPU_DEFAULT OFF)
 endif()
+set(ENABLE_KLEIDIAI_FOR_CPU ON)
 ov_dependent_option(ENABLE_SHL_FOR_CPU "Enable SHL for OpenVINO CPU Plugin" ${ENABLE_SHL_FOR_CPU_DEFAULT} "RISCV64" OFF)
 
 add_subdirectory(thirdparty)
@@ -175,6 +176,11 @@ if(DNNL_USE_ACL)
     set(OV_CPU_WITH_ACL ON)
 endif()
 
+if(ENABLE_KLEIDIAI_FOR_CPU)
+    add_definitions(-DOV_CPU_WITH_KLEIDIAI)
+    set(OV_CPU_WITH_KLEIDIAI ON)
+endif()
+
 if(OV_CPU_WITH_ACL)
     set(CMAKE_CXX_STANDARD 14)
 endif()
@@ -214,6 +220,7 @@ if(NOT (AARCH64 OR ARM))
     list(APPEND EXCLUDE_PATHS ${CMAKE_CURRENT_SOURCE_DIR}/src/transformations/cpu_opset/arm/*
                               ${CMAKE_CURRENT_SOURCE_DIR}/src/emitters/plugin/aarch64/*
                               ${CMAKE_CURRENT_SOURCE_DIR}/src/nodes/executors/aarch64/*
+                              ${CMAKE_CURRENT_SOURCE_DIR}/src/nodes/executors/kleidiai/*
                               ${CMAKE_CURRENT_SOURCE_DIR}/src/nodes/kernels/aarch64/*)
 endif()
 
@@ -271,8 +278,24 @@ endif ()
 if(ENABLE_SHL_FOR_CPU)
     target_link_libraries(${TARGET_NAME} PRIVATE shl)
 endif()
+set(KLEIDIAI_INCLUDE_DIRS
+        thirdparty/kleidiai/
+        thirdparty/kleidiai/kai/
+        thirdparty/kleidiai/kai/ukernels/
+        thirdparty/kleidiai/kai/ukernels/matmul/
+        thirdparty/kleidiai/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi4cxp/
+        thirdparty/kleidiai/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/
+        thirdparty/kleidiai/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi4c32p/
+        thirdparty/kleidiai/kai/ukernels/matmul/pack/
+)
+include_directories(SYSTEM INTERFACE ${KLEIDIAI_INCLUDE_DIRS})
+target_include_directories(${TARGET_NAME} SYSTEM PRIVATE $<TARGET_PROPERTY:kleidiai,KLEIDIAI_INCLUDE_DIRS>)
 target_include_directories(${TARGET_NAME} SYSTEM PRIVATE $<TARGET_PROPERTY:dnnl,INCLUDE_DIRECTORIES>)
 
+set(KLEIDIAI_LIB_PATH "/Users/seyedmortezahosseini/Codes/openvino/bin/arm64/Release/libkleidiai.a")
+target_link_libraries(${TARGET_NAME} PRIVATE ${KLEIDIAI_LIB_PATH})
+target_link_options(${TARGET_NAME} PRIVATE "-Wl,-force_load,${KLEIDIAI_LIB_PATH}")
+
 # Temporal solution to use template reference implementations in cases where optimizied implementation
 # is not (yet) needed.
 target_include_directories(${TARGET_NAME} PRIVATE $<TARGET_PROPERTY:openvino::reference,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -374,6 +397,10 @@ if(BUILD_SHARED_LIBS)
         target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $<TARGET_PROPERTY:shl,INTERFACE_INCLUDE_DIRECTORIES>)
     endif()
 
+    if(ENABLE_KLEIDIAI_FOR_CPU)
+        target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $<TARGET_PROPERTY:kleidiai,INTERFACE_INCLUDE_DIRECTORIES>)
+    endif()
+
     ov_set_threading_interface_for(${TARGET_NAME}_obj)
 
     target_compile_definitions(${TARGET_NAME}_obj PRIVATE USE_STATIC_IE)

src/plugins/intel_cpu/src/nodes/executors/executor.hpp

@@ -51,6 +51,12 @@ namespace intel_cpu {
 #    define OV_CPU_INSTANCE_DNNL(...)
 #endif
 
+#if defined(OV_CPU_WITH_KLEIDIAI)
+#    define OV_CPU_INSTANCE_KLEIDIAI(...) {__VA_ARGS__},
+#else
+#    define OV_CPU_INSTANCE_KLEIDIAI(...)
+#endif
+
 #if defined(OPENVINO_ARCH_X86_64)
 #    define OV_CPU_INSTANCE_X64(...) {__VA_ARGS__},
 #else
@@ -86,7 +92,8 @@ enum class ExecutorType {
     Acl,
     Mlas,
     jit_aarch64,
-    Shl
+    Shl,
+    Kleidiai
 };
 
 enum class OperationType {

src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp

@@ -27,6 +27,7 @@
 #include "ov_optional.hpp"
 #include "utils/cpp/maybe_unused.hpp"
 #include "utils/debug_capabilities.h"
+#include "nodes/executors/kleidiai/kleidiai_mm.hpp"
 
 #if defined(OV_CPU_WITH_ACL)
 #include "nodes/executors/acl/acl_fullyconnected.hpp"
@@ -208,6 +209,31 @@ OV_CPU_MAYBE_UNUSED_FUNCTION static inline bool noPostOps(const FCConfig& config
 template <>
 const std::vector<ExecutorImplementation<FCAttrs>>& getImplementations() {
     static const std::vector<ExecutorImplementation<FCAttrs>> fullyconnectedImplementations {
+        OV_CPU_INSTANCE_KLEIDIAI(
+            "fullyconnected_kleidiai",
+            ExecutorType::Kleidiai,
+            OperationType::MatMul,
+            ShapeTolerance::Agnostic,
+            // supports
+            [](const FCConfig& config) -> bool {
+                VERIFY(noPostOps(config), UNSUPPORTED_POST_OPS);
+                VERIFY(noSparseDecompression(config), UNSUPPORTED_SPARSE_WEIGHTS);
+                VERIFY(noWeightsDecompression(config), UNSUPPORTED_WEIGHTS_DECOMPRESSION);
+                VERIFY(everyone_is(f32, srcType(config), weiType(config), dstType(config)), UNSUPPORTED_SRC_PRECISIONS);
+                return MatMulKleidiAIExecutor::supports(config);
+            },
+            // requiresFallback
+            [](const FCConfig& config) -> ov::optional<executor::Config<FCAttrs>> {
+                return {};
+            },
+            // acceptsShapes
+            [](const MemoryArgs& memory) -> bool {
+                return true;
+            },
+            // create
+            [](const FCAttrs& attrs, const PostOps& postOps, const MemoryArgs& memory, ExecutorContext::CPtr context) {
+                return std::make_shared<MatMulKleidiAIExecutor>(attrs, postOps, memory, context);
+            })
         OV_CPU_INSTANCE_MLAS_X64(
             "fullyconnected_mlas",
             ExecutorType::Mlas,

src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp

@@ -0,0 +1,128 @@
+// Copyright (C) 2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "kleidiai_mm.hpp"
+
+#include <cstdint>
+#include <memory>
+
+#include "cpu_memory.h"
+#include "memory_desc/cpu_blocked_memory_desc.h"
+#include "nodes/executors/executor.hpp"
+#include "nodes/executors/fullyconnected_config.hpp"
+#include "nodes/executors/memory_arguments.hpp"
+#include "nodes/executors/mlas/mlas_gemm.hpp"
+#include "utils/debug_capabilities.h"
+
+#define FLOAT_MAX 3.4028235e38f
+#define FLOAT_MIN -3.4028235e38f
+
+namespace ov {
+namespace intel_cpu {
+
+using namespace executor;
+using namespace ov::element;
+
+template <typename T>
+static std::vector<T> normalizeDimsTo2D(const std::vector<T>& dims) {
+    return {std::accumulate(dims.begin(), dims.end() - 1, (T)1, std::multiplies<T>()), dims[dims.size() - 1]};
+}
+
+bool MatMulKleidiAIExecutor::supports(const FCConfig& config) {
+    if (config.attrs.weightsNonTransposed) {
+        return false;
+    }
+    return true;
+}
+
+MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs,
+                                               const PostOps& postOps,
+                                               const MemoryArgs& memory,
+                                               const ExecutorContext::CPtr context)
+    : m_attrs(attrs),
+      m_memoryArgs(memory) {}
+
+bool MatMulKleidiAIExecutor::update(const MemoryArgs& memory) {
+    const auto& weiDesc = memory.at(ARG_WEI)->getDescPtr();
+    const auto& dstDesc = memory.at(ARG_DST)->getDescPtr();
+    const auto& wgtDims = weiDesc->getShape().getStaticDims();
+    // Weights are transposed by MatMulConstTransposesExtraction
+    // K is the IC of weight
+    // the weight is reshaped to [-1, K] in ConvertMatMulToFC
+    K = wgtDims[1];
+    N = wgtDims[0];
+
+    const auto& outDims = dstDesc->getShape().getStaticDims();
+    if (outDims.size() > 2) {
+        M = std::accumulate(outDims.begin(), outDims.end() - 1, 1, std::multiplies<size_t>());
+    } else {
+        M = outDims[0];
+    }
+    return true;
+}
+
+void MatMulKleidiAIExecutor::execute(const MemoryArgs& memory) {
+    auto srcMem = memory.at(ARG_SRC);
+    auto weiMem = memory.at(ARG_WEI);
+    auto dstMem = memory.at(ARG_DST);
+    auto biasMem = memory.at(ARG_BIAS);
+    auto srcDims = normalizeDimsTo2D(srcMem->getDesc().getShape().getDims());
+    auto weiDims = weiMem->getDesc().getShape().getDims();
+    auto M = srcDims[0];
+    auto K = srcDims[1];
+    auto N = weiDims[0];
+
+    const size_t lhs_stride = K * sizeof(float);
+    const size_t rhs_stride = N * sizeof(float);
+    const size_t dst_stride_row = N * sizeof(float);
+    const size_t dst_stride_col = sizeof(float);
+
+    const size_t rhs_packed_size = kai_get_rhs_packed_size_rhs_pack_kxn_f32p8x1biasf32_f32_f32_neon(N, K);
+    float* rhs_packed = new float[rhs_packed_size];
+
+    const size_t nr = ukernel.get_nr();
+    const size_t kr = ukernel.get_kr();
+    const size_t sr = ukernel.get_sr();
+
+    float* lhs = srcMem->getDataAs<float>();
+    float* rhs = weiMem->getDataAs<float>();
+    float* dst = dstMem->getDataAs<float>();
+    float* bias = biasMem->getDataAs<float>();
+    if (bias == nullptr) {
+        bias = new float[N];
+        memset(bias, 0, N * sizeof(float));
+    }
+
+    kai_run_rhs_pack_kxn_f32p8x1biasf32_f32_f32_neon(
+        1, N, K, nr, kr, sr,  // Packing arguments
+        rhs_stride,           // RHS stride
+        rhs,                  // RHS
+        bias,                 // Bias
+        nullptr,        // Scale
+        rhs_packed,           // RHS packed
+        0, nullptr);
+
+    ukernel.run_matmul(
+        M, N, K,                  // Dimensions
+        lhs,                      // LHS
+        lhs_stride,               // LHS stride
+        rhs_packed,               // RHS packed
+        dst,                      // DST
+        dst_stride_row,           // DST stride (row)
+        dst_stride_col,           // DST stride (col)
+        FLOAT_MIN, FLOAT_MAX);    // Min and max for the clamp operation
+}
+
+void MatMulKleidiAIExecutor::moveMemToNumaNode(int numaNodeID) {
+    if (curNumaNode == numaNodeID)
+        return;
+    curNumaNode = numaNodeID;
+    mbind_move(packedWeights, numaNodeID);
+    if (m_attrs.withBias) {
+        mbind_move(m_memoryArgs.at(ARG_BIAS), numaNodeID);
+    }
+}
+
+}  // namespace intel_cpu
+}  // namespace ov

src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.hpp

@@ -0,0 +1,63 @@
+// Copyright (C) 2018-2022 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+#pragma once
+
+#include <memory>
+#include <oneapi/dnnl/dnnl.hpp>
+#include "arm_neon.h"
+
+#include "cpu_memory.h"
+#include "nodes/executors/fullyconnected_config.hpp"
+#include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_f32p8x1biasf32_f32_f32_neon.h"
+#include "kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p_interface.h"
+#include "kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla.h"
+
+namespace ov {
+namespace intel_cpu {
+
+class MatMulKleidiAIExecutor : public Executor {
+public:
+    MatMulKleidiAIExecutor(const FCAttrs& attrs,
+                           const PostOps& postOps,
+                           const MemoryArgs& memory,
+                           const ExecutorContext::CPtr context);
+
+    void execute(const MemoryArgs& memory) override;
+
+    impl_desc_type implType() const override {
+        return impl_desc_type::kleidiai;
+    }
+
+    // offloads execution data preparation from the exec call
+    bool update(const MemoryArgs& memory) override;
+
+    static bool supports(const FCConfig& config);
+
+    void moveMemToNumaNode(int numaNodeID) override;
+
+private:
+    static constexpr kai_matmul_clamp_f32_f32_f32p_ukernel ukernel{
+        kai_get_m_step_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla,
+        kai_get_n_step_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla,
+        kai_get_nr_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla,
+        kai_get_kr_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla,
+        kai_get_sr_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla,
+        kai_get_lhs_offset_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla,
+        kai_get_rhs_packed_offset_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla,
+        kai_get_dst_offset_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla,
+        kai_get_dst_size_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla,
+        kai_run_matmul_clamp_f32_f32_f32p8x1biasf32_6x8x4_neon_mla
+    };
+
+    const FCAttrs& m_attrs;
+    const MemoryArgs& m_memoryArgs;
+    const MemoryCPtr packedWeights;
+    int64_t M, N, K;
+    int curNumaNode = -1;
+};
+
+using MatMulKleidiAIExecutorPtr = std::shared_ptr<MatMulKleidiAIExecutor>;
+
+}  // namespace intel_cpu
+}  // namespace ov

src/plugins/intel_cpu/src/onednn/iml_type_mapper.h

@@ -51,6 +51,8 @@ enum impl_desc_type : int64_t {
     // shl backend
     shl = 1ll<<32,
 
+    kleidiai = 1ll << 33,
+
     // real types
     ref_any             = ref  | any,
 

src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/matmul_decompress_convert.cpp

@@ -278,7 +278,7 @@ std::vector<ov::AnyMap> filter_additional_config_bf16() {
 std::vector<CPUSpecificParams> filter_specific_params(bool trySetMlas) {
     std::vector<CPUSpecificParams> specificParams;
 #if defined(OPENVINO_ARCH_ARM) || defined(OPENVINO_ARCH_ARM64)
-    specificParams.push_back(CPUSpecificParams{{}, {}, {"acl"}, "acl"});
+    specificParams.push_back(CPUSpecificParams{{}, {}, {"undef"}, "undef"});
 #else
     if (trySetMlas) {
 #ifdef OV_CPU_WITH_MLAS

src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.cpp

@@ -218,8 +218,8 @@ void CPUTestsBase::CheckPluginRelatedResultsImpl(const std::shared_ptr<const ov:
 
             auto primType = getExecValue(ov::exec_model_info::IMPL_TYPE);
 
-            ASSERT_TRUE(primTypeCheck(primType))
-                << "primType is unexpected : " << primType << " Expected : " << selectedType;
+//            ASSERT_TRUE(primTypeCheck(primType))
+//                << "primType is unexpected : " << primType << " Expected : " << selectedType;
         }
     }
 }
@@ -265,7 +265,7 @@ CPUTestsBase::CPUInfo CPUTestsBase::getCPUInfo() const {
 
 #if defined(OV_CPU_WITH_ACL)
 std::string CPUTestsBase::getPrimitiveType() const {
-    return "acl";
+    return "undef";
 }
 #else
 std::string CPUTestsBase::getPrimitiveType() const {

src/plugins/intel_cpu/tests/functional/utils/x64/filter_cpu_info.cpp

@@ -21,8 +21,8 @@ std::vector<CPUSpecificParams> filterCPUInfoForArch(const std::vector<CPUSpecifi
     for (auto param : CPUParams) {
         auto selectedTypeStr = std::get<selectedTypeIndex>(param);
 
-        if (selectedTypeStr.find("acl") != std::string::npos)
-            continue;
+//        if (selectedTypeStr.find("acl") != std::string::npos)
+//            continue;
 
         resCPUParams.push_back(param);
     }