Consolidate tests.

CMakeLists.txt

@@ -1,7 +1,6 @@
 cmake_minimum_required (VERSION 3.11)
-project (homeobject)
+project (homeobject LANGUAGES CXX)
 
-include (cmake/Flags.cmake)
 set(CMAKE_CXX_STANDARD 20)
 
 if(EXISTS ${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
@@ -11,53 +10,49 @@ else ()
     message("The file conanbuildinfo.cmake doesn't exist, some properties will be unavailable")
 endif ()
 
-if (NOT DEFINED BUILD_COVERAGE)
-    set(BUILD_COVERAGE OFF)
-endif()
 if (NOT DEFINED CMAKE_BUILD_TYPE)
     set (CMAKE_BUILD_TYPE "Debug")
 endif()
-if (DEFINED BUILD_COVERAGE)
-  if (${BUILD_COVERAGE})
-    include (cmake/CodeCoverage.cmake)
-    APPEND_COVERAGE_COMPILER_FLAGS()
-  endif ()
-endif ()
 
-if (NOT DEFINED MEMORY_SANITIZER_ON)
-    set(MEMORY_SANITIZER_ON OFF)
-endif()
-if (${MEMORY_SANITIZER_ON})
-    include (cmake/mem_sanitizer.cmake)
-    message(STATUS "********* WARNING: Running with Memory Sanitizer ON *********")
+include (cmake/Flags.cmake)
+if (BUILD_TESTING)
+   enable_testing()
+   if ((DEFINED CODE_COVERAGE) AND (${CODE_COVERAGE}))
+      include (cmake/CodeCoverage.cmake)
+      APPEND_COVERAGE_COMPILER_FLAGS()
+   elseif ((DEFINED MEMORY_SANITIZER_ON) AND (${MEMORY_SANITIZER_ON}))
+      message(WARNING "********* Running with Memory Sanitizer *********")
+      add_flags("-fsanitize=address                 \
+                 -fsanitize=undefined               \
+                 -fsanitize-address-use-after-scope \
+                 -fno-sanitize=alignment            \
+                 -DCDS_ADDRESS_SANITIZER_ENABLED    \
+                 -fno-omit-frame-pointer            \
+                 -fno-optimize-sibling-calls        "
+                 )
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize=undefined")
+   endif()
+   find_package(GTest QUIET REQUIRED)
 endif()
 
-find_program(CCACHE_FOUND ccache)
+find_program(CCACHE_FOUND ccache QUIET)
 if (CCACHE_FOUND)
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
+   set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
+   set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
 endif ()
 
-if (BUILD_TESTING)
-enable_testing()
-endif()
-
-# add conan information 
+# add component version information
 add_flags("-DPACKAGE_NAME=${PROJECT_NAME}")
-if (DEFINED CONAN_PACKAGE_VERSION)
-  message("Package Version: [${CONAN_PACKAGE_VERSION}]")
-  add_flags("-DPACKAGE_VERSION=\\\"${CONAN_PACKAGE_VERSION}\\\"")
-else ()
-  message("Unknown Package Version")
-  add_flags("-DPACKAGE_VERSION=\\\"${CONAN_PACKAGE_VERSION}\\\"")
+if (NOT DEFINED CONAN_PACKAGE_VERSION)
+   set(CONAN_PACKAGE_VERSION "0.0.0")
 endif ()
+add_flags("-DPACKAGE_VERSION=${CONAN_PACKAGE_VERSION}")
 
 add_subdirectory(src)
 
 # build info
 string(TOUPPER "${CMAKE_BUILD_TYPE}" UC_CMAKE_BUILD_TYPE)
 message(STATUS "Build configuration:       ${CMAKE_BUILD_TYPE}")
-message(STATUS "C compiler info:           ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} ${CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN}")
 message(STATUS "C++ compiler info:         ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN}")
 message(STATUS "C flags:                   ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UC_CMAKE_BUILD_TYPE}}")
 message(STATUS "C++ flags:                 ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UC_CMAKE_BUILD_TYPE}}")

conanfile.py

@@ -73,7 +73,7 @@ def build(self):
             if self.options.sanitize:
                 definitions['MEMORY_SANITIZER_ON'] = 'ON'
             elif self.options.coverage:
-                definitions['BUILD_COVERAGE'] = 'ON'
+                definitions['CODE_COVERAGE'] = 'ON'
 
         cmake = CMake(self)
         cmake.configure(defs=definitions)

src/CMakeLists.txt

@@ -3,7 +3,6 @@ cmake_minimum_required (VERSION 3.11)
 find_package(Threads QUIET REQUIRED)
 find_package(sisl QUIET REQUIRED)
 find_package(homestore QUIET REQUIRED)
-find_package(GTest QUIET REQUIRED)
 
 link_directories(${spdk_LIB_DIRS} ${dpdk_LIB_DIRS})
 

src/lib/blob_route.hpp

@@ -15,9 +15,7 @@ struct BlobRoute {
     shard_id_t shard;
     blob_id_t blob;
     auto operator<=>(BlobRoute const&) const = default;
-    sisl::blob to_blob() const {
-        return sisl::blob{uintptr_cast(const_cast< BlobRoute* >(this)), sizeof(*this)};
-    }
+    sisl::blob to_blob() const { return sisl::blob{uintptr_cast(const_cast< BlobRoute* >(this)), sizeof(*this)}; }
 };
 #pragma pack()
 

src/lib/homestore_backend/CMakeLists.txt

@@ -25,20 +25,23 @@ target_link_libraries(pg_homestore_test
     ${COMMON_TEST_DEPS}
   )
 add_test(NAME PGHomestoreTest COMMAND pg_homestore_test -csv error)
+set_property(TEST PGHomestoreTest PROPERTY RUN_SERIAL 1)
 
 add_executable (shard_homestore_test)
 target_sources(shard_homestore_test PRIVATE $<TARGET_OBJECTS:shard_test>)
 target_link_libraries(shard_homestore_test
     homeobject_homestore
     ${COMMON_TEST_DEPS}
   )
-#add_test(NAME ShardHomestoreTest COMMAND shard_homestore_test -csv error)
+add_test(NAME ShardHomestoreTest COMMAND shard_homestore_test -csv error)
+set_property(TEST ShardHomestoreTest PROPERTY RUN_SERIAL 1)
 
 add_executable (blob_homestore_test)
 target_sources(blob_homestore_test PRIVATE $<TARGET_OBJECTS:blob_test>)
 target_link_libraries(blob_homestore_test
     homeobject_homestore
     ${COMMON_TEST_DEPS}
   )
-#add_test(NAME BlobHomestoreTest COMMAND blob_homestore_test -csv error)
+#add_test(NAME BlobHomestoreTest COMMAND blob_homestore_test -csv error --num_iters 100)
+#set_property(TEST BlobHomestoreTest PROPERTY RUN_SERIAL 1)
 endif()

src/lib/homestore_backend/hs_blob_manager.cpp

@@ -2,6 +2,7 @@
 #include "replication_message.hpp"
 #include "replication_state_machine.hpp"
 #include "lib/homeobject_impl.hpp"
+#include "lib/blob_route.hpp"
 
 SISL_LOGGING_DECL(blobmgr)
 
@@ -128,7 +129,7 @@ void HSHomeObject::on_blob_put_commit(int64_t lsn, sisl::blob const& header, sis
 
     auto msg_header = r_cast< ReplicationMessageHeader* >(header.bytes);
     if (msg_header->corrupted()) {
-        LOGERROR("replication message header is corrupted with crc error, lsn:{}", lsn);
+        LOGE("replication message header is corrupted with crc error, lsn:{}", lsn);
         if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError::CHECKSUM_MISMATCH)); }
         return;
     }
@@ -150,7 +151,7 @@ void HSHomeObject::on_blob_put_commit(int64_t lsn, sisl::blob const& header, sis
     // Write to index table with key {shard id, blob id } and value {pba}.
     auto r = add_to_index_table(index_table, blob_info);
     if (r.hasError()) {
-        LOGERROR("Failed to insert into index table for blob {} err {}", lsn, r.error());
+        LOGE("Failed to insert into index table for blob {} err {}", lsn, r.error());
         ctx->promise_.setValue(folly::makeUnexpected(r.error()));
         return;
     }
@@ -176,7 +177,7 @@ BlobManager::AsyncResult< Blob > HSHomeObject::_get_blob(ShardInfo const& shard,
 
     auto r = get_from_index_table(index_table, shard.id, blob_id);
     if (!r) {
-        LOGWARN("Blob not found in index id {} shard {}", blob_id, shard.id);
+        LOGW("Blob not found in index id {} shard {}", blob_id, shard.id);
         return folly::makeUnexpected(r.error());
     }
 
@@ -193,20 +194,19 @@ BlobManager::AsyncResult< Blob > HSHomeObject::_get_blob(ShardInfo const& shard,
         .thenValue([this, blob_id, req_len, req_offset, shard, multi_blkids,
                     iov_base](auto&& result) mutable -> BlobManager::AsyncResult< Blob > {
             if (result) {
-                LOGERROR("Failed to read blob {} shard {} err {}", blob_id, shard.id, result.value());
+                LOGE("Failed to read blob {} shard {} err {}", blob_id, shard.id, result.value());
                 return folly::makeUnexpected(BlobError::READ_FAILED);
             }
 
+            auto const b_route = BlobRoute{blob_id, shard.id};
             BlobHeader* header = (BlobHeader*)iov_base.get();
             if (!header->valid()) {
-                LOGERROR("Invalid header found for blob {} shard {}", blob_id, shard.id);
-                LOGERROR("Blob header {}", header->to_string());
+                LOGE("Invalid header found for [route={}] [header={}]", b_route, header->to_string());
                 return folly::makeUnexpected(BlobError::READ_FAILED);
             }
 
             if (header->shard_id != shard.id) {
-                LOGERROR("Invalid shard id found in header for blob {} shard {}", blob_id, shard.id);
-                LOGERROR("Blob header {}", header->to_string());
+                LOGE("Invalid shard id found in header for [route={}] [header={}]", b_route, header->to_string());
                 return folly::makeUnexpected(BlobError::READ_FAILED);
             }
 
@@ -225,15 +225,14 @@ BlobManager::AsyncResult< Blob > HSHomeObject::_get_blob(ShardInfo const& shard,
             compute_blob_payload_hash(header->hash_algorithm, blob_bytes, blob_size, user_key_bytes, user_key_size,
                                       computed_hash, BlobHeader::blob_max_hash_len);
             if (std::memcmp(computed_hash, header->hash, BlobHeader::blob_max_hash_len) != 0) {
-                LOGERROR("Hash mismatch for blob {} shard {}", blob_id, shard.id);
-                LOGERROR("Computed: {} Header: {}", hex_bytes(computed_hash, BlobHeader::blob_max_hash_len),
-                         hex_bytes(header->hash, BlobHeader::blob_max_hash_len));
+                LOGE("Hash mismatch for [route={}] [header={}] [computed={}]", b_route, header->to_string(),
+                     hex_bytes(computed_hash, BlobHeader::blob_max_hash_len));
                 return folly::makeUnexpected(BlobError::CHECKSUM_MISMATCH);
             }
 
             if (req_offset + req_len > blob_size) {
-                LOGERROR("Invalid offset length request in get blob {} offset {} len {} size {}", blob_id, req_offset,
-                         req_len, blob_size);
+                LOGE("Invalid offset length request in get blob {} offset {} len {} size {}", blob_id, req_offset,
+                     req_len, blob_size);
                 return folly::makeUnexpected(BlobError::INVALID_ARG);
             }
 
@@ -265,7 +264,7 @@ homestore::blk_alloc_hints HSHomeObject::blob_put_get_blk_alloc_hints(sisl::blob
 
     auto msg_header = r_cast< ReplicationMessageHeader* >(header.bytes);
     if (msg_header->corrupted()) {
-        LOGERROR("replication message header is corrupted with crc error shard:{}", msg_header->shard_id);
+        LOGE("replication message header is corrupted with crc error shard:{}", msg_header->shard_id);
         if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError::CHECKSUM_MISMATCH)); }
         return {};
     }
@@ -275,7 +274,7 @@ homestore::blk_alloc_hints HSHomeObject::blob_put_get_blk_alloc_hints(sisl::blob
     RELEASE_ASSERT(shard_iter != _shard_map.end(), "Couldnt find shard id");
     auto hs_shard = d_cast< HS_Shard* >((*shard_iter->second).get());
     auto chunk_id = hs_shard->sb_->chunk_id;
-    LOGINFO("Got shard id {} chunk id {}", msg_header->shard_id, chunk_id);
+    LOGI("Got shard id {} chunk id {}", msg_header->shard_id, chunk_id);
     homestore::blk_alloc_hints hints;
     hints.chunk_id_hint = chunk_id;
     return hints;

src/lib/homestore_backend/hs_homeobject.cpp

@@ -1,11 +1,14 @@
+#include "hs_homeobject.hpp"
+
+#include <spdlog/fmt/bin_to_hex.h>
+
 #include <homestore/homestore.hpp>
 #include <homestore/meta_service.hpp>
 #include <homestore/replication_service.hpp>
 #include <homestore/index_service.hpp>
 #include <iomgr/io_environment.hpp>
 
 #include <homeobject/homeobject.hpp>
-#include "hs_homeobject.hpp"
 #include "heap_chunk_selector.h"
 #include "index_kv.hpp"
 
@@ -122,13 +125,6 @@ void HSHomeObject::on_shard_meta_blk_recover_completed(bool success) {
     chunk_selector_->build_per_dev_chunk_heap(excluding_chunks);
 }
 
-std::string hex_bytes(uint8_t* bytes, size_t len) {
-    std::stringstream ss;
-    ss << std::hex;
-    for (size_t i = 0; i < len; i++) {
-        ss << std::setw(2) << std::setfill('0') << (int)bytes[i];
-    }
-    return ss.str();
-}
+std::string hex_bytes(uint8_t* bytes, size_t len) { return fmt::format("{}", spdlog::to_hex(bytes, bytes + len)); }
 
 } // namespace homeobject

src/lib/homestore_backend/hs_homeobject.hpp

@@ -4,13 +4,13 @@
 #include <mutex>
 
 #include <homestore/homestore.hpp>
+#include <homestore/index/index_table.hpp>
 #include <homestore/superblk_handler.hpp>
 #include <homestore/replication/repl_dev.h>
 
 #include "heap_chunk_selector.h"
 #include "lib/homeobject_impl.hpp"
 #include "replication_message.hpp"
-#include "index_kv.hpp"
 
 namespace homestore {
 struct meta_blk;
@@ -21,6 +21,8 @@ namespace homeobject {
 
 std::string hex_bytes(uint8_t* bytes, size_t len);
 
+class BlobRouteKey;
+class BlobRouteValue;
 using BlobIndexTable = homestore::IndexTable< BlobRouteKey, BlobRouteValue >;
 
 class HSHomeObject : public HomeObjectImpl {
@@ -126,9 +128,9 @@ class HSHomeObject : public HomeObjectImpl {
 
         bool valid() const { return magic == blob_header_magic || version <= blob_header_version; }
         std::string to_string() {
-            return fmt::format("magic={:#x} version={} algo={} hash={} shard={} blob_size={} user_size={}", magic, version,
-                               (uint8_t)hash_algorithm, hex_bytes(hash, blob_max_hash_len), shard_id, blob_size,
-                               user_key_size);
+            return fmt::format("magic={:#x} version={} algo={} hash={} shard={} blob_size={} user_size={}", magic,
+                               version, (uint8_t)hash_algorithm, hex_bytes(hash, blob_max_hash_len), shard_id,
+                               blob_size, user_key_size);
         }
     };
 #pragma pack()
@@ -207,7 +209,7 @@ class BlobIndexServiceCallbacks : public homestore::IndexServiceCallbacks {
     BlobIndexServiceCallbacks(HSHomeObject* home_object) : home_object_(home_object) {}
     std::shared_ptr< homestore::IndexTableBase >
     on_index_table_found(const homestore::superblk< homestore::index_table_sb >& sb) override {
-        LOGINFO("Recovered index table to index service");
+        LOGI("Recovered index table to index service");
         return home_object_->recover_index_table(sb);
     }
 

src/lib/homestore_backend/hs_pg_manager.cpp

@@ -63,7 +63,7 @@ PGManager::NullAsyncResult HSHomeObject::_create_pg(PGInfo&& pg_info, std::set<
             RELEASE_ASSERT(index_table_pg_map_.count(uuid_str) == 0, "duplicate index table found");
             index_table_pg_map_[uuid_str] = PgIndexTable{pg_id, index_table};
 
-            LOGINFO("Index table created for pg {} uuid {}", pg_id, uuid_str);
+            LOGI("Index table created for pg {} uuid {}", pg_id, uuid_str);
             hs_pg->index_table_ = index_table;
             // Add to index service, so that it gets cleaned up when index service is shutdown.
             homestore::hs()->index_service().add_index_table(index_table);

src/lib/homestore_backend/index_kv.cpp

@@ -66,7 +66,7 @@ HSHomeObject::get_from_index_table(shared< BlobIndexTable > index_table, shard_i
     homestore::BtreeSingleGetRequest get_req{&index_key, &index_value};
     auto status = index_table->get(get_req);
     if (status != homestore::btree_status_t::success) {
-        LOGERROR("Failed to get from index table {}", index_key.to_string());
+        LOGERROR("Failed to get from index table [route={}]", index_key);
         return folly::makeUnexpected(BlobError::UNKNOWN_BLOB);
     }
 
@@ -78,12 +78,12 @@ void HSHomeObject::print_btree_index(pg_id_t pg_id) {
     {
         std::shared_lock lock_guard(_pg_lock);
         auto iter = _pg_map.find(pg_id);
-        RELEASE_ASSERT (iter != _pg_map.end(), "Unknown PG");
+        RELEASE_ASSERT(iter != _pg_map.end(), "Unknown PG");
         index_table = static_cast< HS_PG* >(iter->second.get())->index_table_;
         RELEASE_ASSERT(index_table != nullptr, "Index table not intialized");
     }
 
-    LOGINFO("Index UUID {}", boost::uuids::to_string(index_table->uuid()));
+    LOGI("Index UUID {}", boost::uuids::to_string(index_table->uuid()));
     index_table->print_tree();
 }