Implement put blob and get blob api's.

Add index table and its recovery for each pg.
UT to do basic verification of get and put with recovery.

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeObjectConan(ConanFile):
     name = "homeobject"
-    version = "0.11.1"
+    version = "0.12.0"
     homepage = "https://github.com/eBay/HomeObject"
     description = "Blob Store built on HomeReplication"
     topics = ("ebay")

src/include/homeobject/blob_manager.hpp

@@ -9,17 +9,17 @@
 
 namespace homeobject {
 
-ENUM(BlobError, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, NOT_LEADER, UNKNOWN_SHARD, UNKNOWN_BLOB,
-     CHECKSUM_MISMATCH);
+ENUM(BlobError, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, NOT_LEADER, UNKNOWN_SHARD, UNKNOWN_BLOB, CHECKSUM_MISMATCH,
+     UNKNOWN_PG, PG_NOT_READY, READ_FAILED, INDEX_ERROR);
 
 struct Blob {
     Blob(sisl::io_blob_safe b, std::string const& u, uint64_t o) : body(std::move(b)), user_key(u), object_off(o) {}
 
     Blob clone() const;
 
     sisl::io_blob_safe body;
-    std::string user_key;
-    uint64_t object_off;
+    std::string user_key{};
+    uint64_t object_off{};
     std::optional< peer_id_t > current_leader{std::nullopt};
 };
 

src/include/homeobject/common.hpp

@@ -11,7 +11,7 @@
 
 SISL_LOGGING_DECL(homeobject);
 
-#define HOMEOBJECT_LOG_MODS grpc_server, HOMESTORE_LOG_MODS, homeobject
+#define HOMEOBJECT_LOG_MODS grpc_server, HOMESTORE_LOG_MODS, homeobject, blobmgr
 
 #ifndef Ki
 constexpr uint64_t Ki = 1024ul;

src/lib/blob_manager.cpp

@@ -4,25 +4,25 @@ namespace homeobject {
 
 std::shared_ptr< BlobManager > HomeObjectImpl::blob_manager() { return shared_from_this(); }
 
-BlobManager::AsyncResult< Blob > HomeObjectImpl::get(shard_id_t shard, blob_id_t const& blob, uint64_t off,
+BlobManager::AsyncResult< Blob > HomeObjectImpl::get(shard_id_t shard, blob_id_t const& blob_id, uint64_t off,
                                                      uint64_t len) const {
-    return _get_shard(shard).thenValue([this, blob](auto const e) -> BlobManager::Result< Blob > {
+    return _get_shard(shard).thenValue([this, blob_id, off, len](auto const e) -> BlobManager::AsyncResult< Blob > {
         if (!e) return folly::makeUnexpected(BlobError::UNKNOWN_SHARD);
-        return _get_blob(e.value(), blob);
+        return _get_blob(e.value(), blob_id, off, len);
     });
 }
 
 BlobManager::AsyncResult< blob_id_t > HomeObjectImpl::put(shard_id_t shard, Blob&& blob) {
     return _get_shard(shard).thenValue(
-        [this, blob = std::move(blob)](auto const e) mutable -> BlobManager::Result< blob_id_t > {
+        [this, blob = std::move(blob)](auto const e) mutable -> BlobManager::AsyncResult< blob_id_t > {
             if (!e) return folly::makeUnexpected(BlobError::UNKNOWN_SHARD);
             if (ShardInfo::State::SEALED == e.value().state) return folly::makeUnexpected(BlobError::INVALID_ARG);
             return _put_blob(e.value(), std::move(blob));
         });
 }
 
 BlobManager::NullAsyncResult HomeObjectImpl::del(shard_id_t shard, blob_id_t const& blob) {
-    return _get_shard(shard).thenValue([this, blob](auto const e) mutable -> BlobManager::NullResult {
+    return _get_shard(shard).thenValue([this, blob](auto const e) mutable -> BlobManager::NullAsyncResult {
         if (!e) return folly::makeUnexpected(BlobError::UNKNOWN_SHARD);
         return _del_blob(e.value(), blob);
     });

src/lib/blob_route.hpp

@@ -10,11 +10,16 @@ namespace homeobject {
 ///
 // A Key used in the IndexService (BTree). The inclusion of Shard allows BlobRoutes
 // to appear in a different Index should the Blob (Shard) be moved between Pgs.
+#pragma pack(1)
 struct BlobRoute {
     shard_id_t shard;
     blob_id_t blob;
     auto operator<=>(BlobRoute const&) const = default;
+    sisl::blob to_blob() const {
+        return sisl::blob{uintptr_cast(const_cast< BlobRoute* >(this)), sizeof(*this)};
+    }
 };
+#pragma pack()
 
 } // namespace homeobject
 

src/lib/homeobject_impl.hpp

@@ -76,9 +76,10 @@ class HomeObjectImpl : public HomeObject,
     virtual ShardManager::AsyncResult< ShardInfo > _create_shard(pg_id_t, uint64_t size_bytes) = 0;
     virtual ShardManager::Result< ShardInfo > _seal_shard(shard_id_t) = 0;
 
-    virtual BlobManager::Result< blob_id_t > _put_blob(ShardInfo const&, Blob&&) = 0;
-    virtual BlobManager::Result< Blob > _get_blob(ShardInfo const&, blob_id_t) const = 0;
-    virtual BlobManager::NullResult _del_blob(ShardInfo const&, blob_id_t) = 0;
+    virtual BlobManager::AsyncResult< blob_id_t > _put_blob(ShardInfo const&, Blob&&) = 0;
+    virtual BlobManager::AsyncResult< Blob > _get_blob(ShardInfo const&, blob_id_t, uint64_t off = 0,
+                                                       uint64_t len = 0) const = 0;
+    virtual BlobManager::NullAsyncResult _del_blob(ShardInfo const&, blob_id_t) = 0;
     ///
     folly::Future< ShardManager::Result< ShardInfo > > _get_shard(shard_id_t id) const;
     auto _defer() const { return folly::makeSemiFuture().via(folly::getGlobalCPUExecutor()); }

src/lib/homestore_backend/CMakeLists.txt

@@ -2,18 +2,19 @@ cmake_minimum_required (VERSION 3.11)
 
 add_library ("${PROJECT_NAME}_homestore")
 target_sources("${PROJECT_NAME}_homestore" PRIVATE
-		hs_homeobject.cpp
-		hs_blob_manager.cpp
-		hs_shard_manager.cpp
-		hs_pg_manager.cpp
-		heap_chunk_selector.cpp
-		replication_state_machine.cpp
+        hs_homeobject.cpp
+        hs_blob_manager.cpp
+        hs_shard_manager.cpp
+        hs_pg_manager.cpp
+        index_kv.cpp
+        heap_chunk_selector.cpp
+        replication_state_machine.cpp
         $<TARGET_OBJECTS:${PROJECT_NAME}_core>
-	)
+    )
 target_link_libraries("${PROJECT_NAME}_homestore"
             ${COMMON_DEPS}
         )
 
 if(BUILD_TESTING)
-	add_subdirectory(tests)
+    add_subdirectory(tests)
 endif()