From d56a1dfb76bb0d3f29c096ded7dbc584ec0db3b2 Mon Sep 17 00:00:00 2001 From: Brian Szmyd Date: Wed, 20 Dec 2023 19:47:06 +0000 Subject: [PATCH 1/4] Support for ARM requires non-accelerated CRC. --- .gitignore | 1 + conanfile.py | 5 ++- src/CMakeLists.txt | 42 ++++++++++--------- .../homestore/btree/detail/btree_node.hpp | 4 ++ src/lib/common/crc.h | 39 +++++++++++++++++ src/lib/device/device.h | 4 ++ src/lib/device/device_manager.cpp | 4 ++ src/lib/device/physical_dev.cpp | 1 - src/lib/device/physical_dev.hpp | 4 ++ src/lib/logstore/log_dev.cpp | 1 - src/lib/logstore/log_group.cpp | 3 ++ src/lib/logstore/log_stream.cpp | 2 - src/lib/meta/meta_blk_service.cpp | 1 - 13 files changed, 85 insertions(+), 26 deletions(-) create mode 100644 src/lib/common/crc.h diff --git a/.gitignore b/.gitignore index 97b58ba32..816ffced6 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,4 @@ cmake-*/** # Visual Studio CMakeSettings.json .vs/** +.cache diff --git a/conanfile.py b/conanfile.py index 1d2ad18cb..bf4ffbe20 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "4.9.3" + version = "4.9.4" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" @@ -59,7 +59,8 @@ def requirements(self): self.requires("sisl/[>=10.3]") self.requires("farmhash/cci.20190513@") - self.requires("isa-l/2.30.0") + if self.settings.arch in ['x86', 'x86_64']: + self.requires("isa-l/2.30.0") self.requires("spdk/21.07.y") def build(self): diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 30329c54e..8c2b8b06d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,25 +3,29 @@ cmake_minimum_required(VERSION 3.13) set(CMAKE_THREAD_PREFER_PTHREAD TRUE) find_package(Threads) -find_library(LIB_AIO aio REQUIRED) -find_package(isa-l REQUIRED) -find_package(iomgr REQUIRED) -find_package(farmhash REQUIRED) -find_package(GTest REQUIRED) - -set (COMMON_DEPS - iomgr::iomgr - farmhash::farmhash - isa-l::isa-l - sisl::sisl - ) - -set(COMMON_TEST_DEPS - ${COMMON_DEPS} - ${spdk_LIBRARY_LIST} - ${dpdk_LIBRARY_LIST} - GTest::gmock - ) +find_library(LIB_AIO aio QUIET REQUIRED) +find_package(isa-l QUIET) +find_package(iomgr QUIET REQUIRED) +find_package(farmhash QUIET REQUIRED) +find_package(GTest QUIET REQUIRED) + +list(APPEND COMMON_DEPS + iomgr::iomgr + farmhash::farmhash + sisl::sisl +) +if (${isa-l_FOUND}) + list(APPEND COMMON_DEPS isa-l::isa-l) +else () + add_flags("-DNO_ISAL") +endif() + +list(APPEND COMMON_TEST_DEPS + ${COMMON_DEPS} + ${spdk_LIBRARY_LIST} + ${dpdk_LIBRARY_LIST} + GTest::gmock +) include_directories (BEFORE lib/) include_directories (BEFORE include/) diff --git a/src/include/homestore/btree/detail/btree_node.hpp b/src/include/homestore/btree/detail/btree_node.hpp index 8f713e534..895e377a7 100644 --- a/src/include/homestore/btree/detail/btree_node.hpp +++ b/src/include/homestore/btree/detail/btree_node.hpp @@ -25,7 +25,11 @@ #include "btree_internal.hpp" #include // #include +#ifndef NO_ISAL #include +#else +#include "common/crc.h" +#endif namespace homestore { ENUM(locktype_t, uint8_t, NONE, READ, WRITE) diff --git a/src/lib/common/crc.h b/src/lib/common/crc.h new file mode 100644 index 000000000..656a60ada --- /dev/null +++ b/src/lib/common/crc.h @@ -0,0 +1,39 @@ +#pragma once + +#define MAX_ITER 8 + +extern "C" { +// crc16_t10dif reference function, slow crc16 from the definition. +static inline uint16_t crc16_t10dif(uint16_t seed, const unsigned char* buf, uint64_t len) { + size_t rem = seed; + unsigned int i, j; + + uint16_t poly = 0x8bb7; // t10dif standard + + for (i = 0; i < len; i++) { + rem = rem ^ (buf[i] << 8); + for (j = 0; j < MAX_ITER; j++) { + rem = rem << 1; + rem = (rem & 0x10000) ? rem ^ poly : rem; + } + } + return rem; +} + +// crc32_ieee reference function, slow crc32 from the definition. +static inline uint32_t crc32_ieee(uint32_t seed, const unsigned char* buf, uint64_t len) { + uint64_t rem = ~seed; + unsigned int i, j; + + uint32_t poly = 0x04C11DB7; // IEEE standard + + for (i = 0; i < len; i++) { + rem = rem ^ ((uint64_t)buf[i] << 24); + for (j = 0; j < MAX_ITER; j++) { + rem = rem << 1; + rem = (rem & 0x100000000ULL) ? rem ^ poly : rem; + } + } + return ~rem; +} +} diff --git a/src/lib/device/device.h b/src/lib/device/device.h index ad8749576..31efd8e43 100644 --- a/src/lib/device/device.h +++ b/src/lib/device/device.h @@ -17,7 +17,11 @@ #include #include +#ifndef NO_ISAL #include +#else +#include "common/crc.h" +#endif #include #include #include diff --git a/src/lib/device/device_manager.cpp b/src/lib/device/device_manager.cpp index ecab5562c..53bd926cd 100644 --- a/src/lib/device/device_manager.cpp +++ b/src/lib/device/device_manager.cpp @@ -15,7 +15,11 @@ #include #include +#ifndef NO_ISAL #include +#else +#include "common/crc.h" +#endif #include #include diff --git a/src/lib/device/physical_dev.cpp b/src/lib/device/physical_dev.cpp index 33f243824..3f250372b 100644 --- a/src/lib/device/physical_dev.cpp +++ b/src/lib/device/physical_dev.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include "device/chunk.h" diff --git a/src/lib/device/physical_dev.hpp b/src/lib/device/physical_dev.hpp index 951e61f34..60b36b5c1 100644 --- a/src/lib/device/physical_dev.hpp +++ b/src/lib/device/physical_dev.hpp @@ -25,7 +25,11 @@ #include #include +#ifndef NO_ISAL #include +#else +#include "common/crc.h" +#endif #include #include #include diff --git a/src/lib/logstore/log_dev.cpp b/src/lib/logstore/log_dev.cpp index b8ea1e2ef..c6d035d09 100644 --- a/src/lib/logstore/log_dev.cpp +++ b/src/lib/logstore/log_dev.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include diff --git a/src/lib/logstore/log_group.cpp b/src/lib/logstore/log_group.cpp index 7c68e581d..13392c708 100644 --- a/src/lib/logstore/log_group.cpp +++ b/src/lib/logstore/log_group.cpp @@ -15,7 +15,10 @@ *********************************************************************************/ #include +#ifndef NO_ISAL #include +#else +#endif #include #include "common/homestore_assert.hpp" diff --git a/src/lib/logstore/log_stream.cpp b/src/lib/logstore/log_stream.cpp index d7c0ce8e2..70712d942 100644 --- a/src/lib/logstore/log_stream.cpp +++ b/src/lib/logstore/log_stream.cpp @@ -13,8 +13,6 @@ * specific language governing permissions and limitations under the License. * *********************************************************************************/ -#include - #include "device/chunk.h" #include "common/homestore_assert.hpp" #include "common/homestore_config.hpp" diff --git a/src/lib/meta/meta_blk_service.cpp b/src/lib/meta/meta_blk_service.cpp index 03cfd1ae7..993e65099 100644 --- a/src/lib/meta/meta_blk_service.cpp +++ b/src/lib/meta/meta_blk_service.cpp @@ -24,7 +24,6 @@ #include #include -#include #include #include From 843a22e7df7d2d9f6810a29cd469d531a80a0136 Mon Sep 17 00:00:00 2001 From: Brian Szmyd Date: Wed, 20 Dec 2023 19:54:37 +0000 Subject: [PATCH 2/4] Move macro test to common area. --- src/include/homestore/btree/detail/btree_node.hpp | 5 ----- src/lib/common/crc.h | 7 +++++++ src/lib/device/device.h | 4 ---- src/lib/device/device_manager.cpp | 4 ---- src/lib/device/physical_dev.hpp | 4 ---- src/lib/logstore/log_group.cpp | 5 ----- 6 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/include/homestore/btree/detail/btree_node.hpp b/src/include/homestore/btree/detail/btree_node.hpp index 895e377a7..b4f7dab65 100644 --- a/src/include/homestore/btree/detail/btree_node.hpp +++ b/src/include/homestore/btree/detail/btree_node.hpp @@ -24,12 +24,7 @@ #include #include "btree_internal.hpp" #include -// #include -#ifndef NO_ISAL -#include -#else #include "common/crc.h" -#endif namespace homestore { ENUM(locktype_t, uint8_t, NONE, READ, WRITE) diff --git a/src/lib/common/crc.h b/src/lib/common/crc.h index 656a60ada..ea893d52e 100644 --- a/src/lib/common/crc.h +++ b/src/lib/common/crc.h @@ -1,5 +1,11 @@ #pragma once +// Only x86 and x86_64 supported by Intel Storage Acceleration library +#ifndef NO_ISAL +#include + +#else + #define MAX_ITER 8 extern "C" { @@ -37,3 +43,4 @@ static inline uint32_t crc32_ieee(uint32_t seed, const unsigned char* buf, uint6 return ~rem; } } +#endif diff --git a/src/lib/device/device.h b/src/lib/device/device.h index 31efd8e43..42dce472d 100644 --- a/src/lib/device/device.h +++ b/src/lib/device/device.h @@ -17,11 +17,7 @@ #include #include -#ifndef NO_ISAL -#include -#else #include "common/crc.h" -#endif #include #include #include diff --git a/src/lib/device/device_manager.cpp b/src/lib/device/device_manager.cpp index 53bd926cd..9376b8005 100644 --- a/src/lib/device/device_manager.cpp +++ b/src/lib/device/device_manager.cpp @@ -15,11 +15,7 @@ #include #include -#ifndef NO_ISAL -#include -#else #include "common/crc.h" -#endif #include #include diff --git a/src/lib/device/physical_dev.hpp b/src/lib/device/physical_dev.hpp index 60b36b5c1..20e41a02e 100644 --- a/src/lib/device/physical_dev.hpp +++ b/src/lib/device/physical_dev.hpp @@ -25,11 +25,7 @@ #include #include -#ifndef NO_ISAL -#include -#else #include "common/crc.h" -#endif #include #include #include diff --git a/src/lib/logstore/log_group.cpp b/src/lib/logstore/log_group.cpp index 13392c708..a94ff7fa1 100644 --- a/src/lib/logstore/log_group.cpp +++ b/src/lib/logstore/log_group.cpp @@ -15,11 +15,6 @@ *********************************************************************************/ #include -#ifndef NO_ISAL -#include -#else -#endif - #include #include "common/homestore_assert.hpp" #include "log_dev.hpp" From a492d075da4bb967df4cc447009222480d328b4c Mon Sep 17 00:00:00 2001 From: Brian Szmyd Date: Wed, 20 Dec 2023 20:01:32 +0000 Subject: [PATCH 3/4] crc.h needs to be public, included by btree. --- src/include/homestore/btree/detail/btree_node.hpp | 2 +- src/{lib/common => include/homestore}/crc.h | 0 src/lib/device/device.h | 2 +- src/lib/device/device_manager.cpp | 2 +- src/lib/device/physical_dev.hpp | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename src/{lib/common => include/homestore}/crc.h (100%) diff --git a/src/include/homestore/btree/detail/btree_node.hpp b/src/include/homestore/btree/detail/btree_node.hpp index b4f7dab65..6428e9c06 100644 --- a/src/include/homestore/btree/detail/btree_node.hpp +++ b/src/include/homestore/btree/detail/btree_node.hpp @@ -24,7 +24,7 @@ #include #include "btree_internal.hpp" #include -#include "common/crc.h" +#include namespace homestore { ENUM(locktype_t, uint8_t, NONE, READ, WRITE) diff --git a/src/lib/common/crc.h b/src/include/homestore/crc.h similarity index 100% rename from src/lib/common/crc.h rename to src/include/homestore/crc.h diff --git a/src/lib/device/device.h b/src/lib/device/device.h index 42dce472d..59500e1ad 100644 --- a/src/lib/device/device.h +++ b/src/lib/device/device.h @@ -17,7 +17,7 @@ #include #include -#include "common/crc.h" +#include #include #include #include diff --git a/src/lib/device/device_manager.cpp b/src/lib/device/device_manager.cpp index 9376b8005..0ae9d7d6f 100644 --- a/src/lib/device/device_manager.cpp +++ b/src/lib/device/device_manager.cpp @@ -15,7 +15,7 @@ #include #include -#include "common/crc.h" +#include #include #include diff --git a/src/lib/device/physical_dev.hpp b/src/lib/device/physical_dev.hpp index 20e41a02e..eb5cb52c9 100644 --- a/src/lib/device/physical_dev.hpp +++ b/src/lib/device/physical_dev.hpp @@ -25,7 +25,7 @@ #include #include -#include "common/crc.h" +#include #include #include #include From b21efc6b5bc93e471e245206083d32443474a591 Mon Sep 17 00:00:00 2001 From: Brian Szmyd Date: Wed, 20 Dec 2023 20:20:57 +0000 Subject: [PATCH 4/4] Need internal linkage for crc routines. --- src/CMakeLists.txt | 1 + src/include/homestore/crc.h | 34 ++--------------------------- src/lib/crc.cpp | 43 +++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 32 deletions(-) create mode 100644 src/lib/crc.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8c2b8b06d..6a7109d6c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -53,6 +53,7 @@ set(HOMESTORE_OBJECTS $ $ lib/homestore.cpp + lib/crc.cpp #$ #$ ) diff --git a/src/include/homestore/crc.h b/src/include/homestore/crc.h index ea893d52e..9c21f3d6c 100644 --- a/src/include/homestore/crc.h +++ b/src/include/homestore/crc.h @@ -6,41 +6,11 @@ #else -#define MAX_ITER 8 - extern "C" { // crc16_t10dif reference function, slow crc16 from the definition. -static inline uint16_t crc16_t10dif(uint16_t seed, const unsigned char* buf, uint64_t len) { - size_t rem = seed; - unsigned int i, j; - - uint16_t poly = 0x8bb7; // t10dif standard - - for (i = 0; i < len; i++) { - rem = rem ^ (buf[i] << 8); - for (j = 0; j < MAX_ITER; j++) { - rem = rem << 1; - rem = (rem & 0x10000) ? rem ^ poly : rem; - } - } - return rem; -} +uint16_t crc16_t10dif(uint16_t seed, const unsigned char* buf, uint64_t len); // crc32_ieee reference function, slow crc32 from the definition. -static inline uint32_t crc32_ieee(uint32_t seed, const unsigned char* buf, uint64_t len) { - uint64_t rem = ~seed; - unsigned int i, j; - - uint32_t poly = 0x04C11DB7; // IEEE standard - - for (i = 0; i < len; i++) { - rem = rem ^ ((uint64_t)buf[i] << 24); - for (j = 0; j < MAX_ITER; j++) { - rem = rem << 1; - rem = (rem & 0x100000000ULL) ? rem ^ poly : rem; - } - } - return ~rem; -} +uint32_t crc32_ieee(uint32_t seed, const unsigned char* buf, uint64_t len); } #endif diff --git a/src/lib/crc.cpp b/src/lib/crc.cpp new file mode 100644 index 000000000..9b1b296a1 --- /dev/null +++ b/src/lib/crc.cpp @@ -0,0 +1,43 @@ +// Only x86 and x86_64 supported by Intel Storage Acceleration library +#ifdef NO_ISAL + +#include +#include +extern "C" { +#define MAX_ITER 8 + +// crc16_t10dif reference function, slow crc16 from the definition. +uint16_t crc16_t10dif(uint16_t seed, const unsigned char* buf, uint64_t len) { + size_t rem = seed; + unsigned int i, j; + + uint16_t poly = 0x8bb7; // t10dif standard + + for (i = 0; i < len; i++) { + rem = rem ^ (buf[i] << 8); + for (j = 0; j < MAX_ITER; j++) { + rem = rem << 1; + rem = (rem & 0x10000) ? rem ^ poly : rem; + } + } + return rem; +} + +// crc32_ieee reference function, slow crc32 from the definition. +uint32_t crc32_ieee(uint32_t seed, const unsigned char* buf, uint64_t len) { + uint64_t rem = ~seed; + unsigned int i, j; + + uint32_t poly = 0x04C11DB7; // IEEE standard + + for (i = 0; i < len; i++) { + rem = rem ^ ((uint64_t)buf[i] << 24); + for (j = 0; j < MAX_ITER; j++) { + rem = rem << 1; + rem = (rem & 0x100000000ULL) ? rem ^ poly : rem; + } + } + return ~rem; +} +} +#endif