Support for ARM requires non-accelerated CRC. (eBay#264)

.gitignore

@@ -105,3 +105,4 @@ cmake-*/**
 # Visual Studio
 CMakeSettings.json
 .vs/**
+.cache

conanfile.py

@@ -5,7 +5,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "4.9.3"
+    version = "4.9.4"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"
@@ -59,7 +59,8 @@ def requirements(self):
         self.requires("sisl/[>=10.3]")
 
         self.requires("farmhash/cci.20190513@")
-        self.requires("isa-l/2.30.0")
+        if self.settings.arch in ['x86', 'x86_64']:
+            self.requires("isa-l/2.30.0")
         self.requires("spdk/21.07.y")
 
     def build(self):

src/CMakeLists.txt

@@ -3,25 +3,29 @@ cmake_minimum_required(VERSION 3.13)
 set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 
 find_package(Threads)
-find_library(LIB_AIO aio REQUIRED)
-find_package(isa-l REQUIRED)
-find_package(iomgr REQUIRED)
-find_package(farmhash REQUIRED)
-find_package(GTest REQUIRED)
-
-set (COMMON_DEPS
-        iomgr::iomgr
-        farmhash::farmhash
-        isa-l::isa-l
-        sisl::sisl
-    )
-
-set(COMMON_TEST_DEPS
-        ${COMMON_DEPS}
-        ${spdk_LIBRARY_LIST}
-        ${dpdk_LIBRARY_LIST}
-        GTest::gmock
-    )
+find_library(LIB_AIO aio QUIET REQUIRED)
+find_package(isa-l QUIET)
+find_package(iomgr QUIET REQUIRED)
+find_package(farmhash QUIET REQUIRED)
+find_package(GTest QUIET REQUIRED)
+
+list(APPEND COMMON_DEPS
+    iomgr::iomgr
+    farmhash::farmhash
+    sisl::sisl
+)
+if (${isa-l_FOUND})
+    list(APPEND COMMON_DEPS isa-l::isa-l)
+else ()
+    add_flags("-DNO_ISAL")
+endif()
+
+list(APPEND COMMON_TEST_DEPS
+    ${COMMON_DEPS}
+    ${spdk_LIBRARY_LIST}
+    ${dpdk_LIBRARY_LIST}
+    GTest::gmock
+)
 
 include_directories (BEFORE lib/)
 include_directories (BEFORE include/)
@@ -49,6 +53,7 @@ set(HOMESTORE_OBJECTS
     $<TARGET_OBJECTS:hs_datasvc>
     $<TARGET_OBJECTS:hs_replication>
     lib/homestore.cpp
+    lib/crc.cpp
     #$<TARGET_OBJECTS:hs_cp>
     #$<TARGET_OBJECTS:indx_mgr>
 )

src/include/homestore/btree/detail/btree_node.hpp

@@ -24,8 +24,7 @@
 #include <sisl/utility/obj_life_counter.hpp>
 #include "btree_internal.hpp"
 #include <homestore/btree/btree_kv.hpp>
-// #include <iomgr/iomgr_flip.hpp>
-#include <isa-l/crc.h>
+#include <homestore/crc.h>
 
 namespace homestore {
 ENUM(locktype_t, uint8_t, NONE, READ, WRITE)

src/include/homestore/crc.h

@@ -0,0 +1,16 @@
+#pragma once
+
+// Only x86 and x86_64 supported by Intel Storage Acceleration library
+#ifndef NO_ISAL
+#include <isa-l/crc.h>
+
+#else
+
+extern "C" {
+// crc16_t10dif reference function, slow crc16 from the definition.
+uint16_t crc16_t10dif(uint16_t seed, const unsigned char* buf, uint64_t len);
+
+// crc32_ieee reference function, slow crc32 from the definition.
+uint32_t crc32_ieee(uint32_t seed, const unsigned char* buf, uint64_t len);
+}
+#endif

src/lib/crc.cpp

@@ -0,0 +1,43 @@
+// Only x86 and x86_64 supported by Intel Storage Acceleration library
+#ifdef NO_ISAL
+
+#include <cstdint>
+#include <cstddef>
+extern "C" {
+#define MAX_ITER 8
+
+// crc16_t10dif reference function, slow crc16 from the definition.
+uint16_t crc16_t10dif(uint16_t seed, const unsigned char* buf, uint64_t len) {
+    size_t rem = seed;
+    unsigned int i, j;
+
+    uint16_t poly = 0x8bb7; // t10dif standard
+
+    for (i = 0; i < len; i++) {
+        rem = rem ^ (buf[i] << 8);
+        for (j = 0; j < MAX_ITER; j++) {
+            rem = rem << 1;
+            rem = (rem & 0x10000) ? rem ^ poly : rem;
+        }
+    }
+    return rem;
+}
+
+// crc32_ieee reference function, slow crc32 from the definition.
+uint32_t crc32_ieee(uint32_t seed, const unsigned char* buf, uint64_t len) {
+    uint64_t rem = ~seed;
+    unsigned int i, j;
+
+    uint32_t poly = 0x04C11DB7; // IEEE standard
+
+    for (i = 0; i < len; i++) {
+        rem = rem ^ ((uint64_t)buf[i] << 24);
+        for (j = 0; j < MAX_ITER; j++) {
+            rem = rem << 1;
+            rem = (rem & 0x100000000ULL) ? rem ^ poly : rem;
+        }
+    }
+    return ~rem;
+}
+}
+#endif

src/lib/device/device.h

@@ -17,7 +17,7 @@
 #include <map>
 #include <vector>
 
-#include <isa-l/crc.h>
+#include <homestore/crc.h>
 #include <iomgr/iomgr.hpp>
 #include <sisl/fds/sparse_vector.hpp>
 #include <homestore/homestore_decl.hpp>

src/lib/device/device_manager.cpp

@@ -15,7 +15,7 @@
 #include <vector>
 
 #include <iomgr/iomgr.hpp>
-#include <isa-l/crc.h>
+#include <homestore/crc.h>
 #include <sisl/logging/logging.h>
 
 #include <boost/uuid/random_generator.hpp>

src/lib/device/physical_dev.cpp

@@ -22,7 +22,6 @@
 #include <iomgr/iomgr.hpp>
 #include <iomgr/iomgr_flip.hpp>
 #include <sisl/fds/utils.hpp>
-#include <isa-l/crc.h>
 
 #include <homestore/homestore_decl.hpp>
 #include "device/chunk.h"

src/lib/device/physical_dev.hpp

@@ -25,7 +25,7 @@
 
 #include <boost/icl/split_interval_set.hpp>
 #include <nlohmann/json.hpp>
-#include <isa-l/crc.h>
+#include <homestore/crc.h>
 #include <sisl/metrics/metrics.hpp>
 #include <sisl/logging/logging.h>
 #include <homestore/homestore_decl.hpp>

src/lib/logstore/log_dev.cpp

@@ -19,7 +19,6 @@
 #include <iterator>
 
 #include <sisl/fds/vector_pool.hpp>
-#include <isa-l/crc.h>
 #include <iomgr/iomgr_flip.hpp>
 
 #include <homestore/logstore_service.hpp>

src/lib/logstore/log_group.cpp

@@ -15,8 +15,6 @@
  *********************************************************************************/
 #include <cstring>
 
-#include <isa-l/crc.h>
-
 #include <homestore/logstore/log_store.hpp>
 #include "common/homestore_assert.hpp"
 #include "log_dev.hpp"

src/lib/logstore/log_stream.cpp

@@ -13,8 +13,6 @@
  * specific language governing permissions and limitations under the License.
  *
  *********************************************************************************/
-#include <isa-l/crc.h>
-
 #include "device/chunk.h"
 #include "common/homestore_assert.hpp"
 #include "common/homestore_config.hpp"

src/lib/meta/meta_blk_service.cpp

@@ -24,7 +24,6 @@
 
 #include <sisl/fds/compress.hpp>
 #include <sisl/fds/utils.hpp>
-#include <isa-l/crc.h>
 #include <iomgr/iomgr_flip.hpp>
 
 #include <homestore/meta_service.hpp>