feat: Extraction basics (#1733)

For #1596

src/data/CassandraBackend.hpp

@@ -949,7 +949,7 @@ class BasicCassandraBackend : public BackendInterface {
     {
         std::vector<Statement> statements;
         statements.reserve(data.size());
-for (auto [mptId, holder] : data)
+        for (auto [mptId, holder] : data)
             statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
 
         executor_.write(std::move(statements));

src/etl/LedgerFetcherInterface.hpp

@@ -0,0 +1,65 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+/** @file */
+#pragma once
+
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
+
+#include <cstdint>
+#include <optional>
+
+namespace etl {
+
+/**
+ * @brief An interface for LedgerFetcher
+ */
+struct LedgerFetcherInterface {
+    using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
+    using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
+
+    virtual ~LedgerFetcherInterface() = default;
+
+    /**
+     * @brief Extract data for a particular ledger from an ETL source
+     *
+     * This function continously tries to extract the specified ledger (using all available ETL sources) until the
+     * extraction succeeds, or the server shuts down.
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
+     */
+    [[nodiscard]] virtual OptionalGetLedgerResponseType
+    fetchData(uint32_t seq) = 0;
+
+    /**
+     * @brief Extract diff data for a particular ledger from an ETL source.
+     *
+     * This function continously tries to extract the specified ledger (using all available ETL sources) until the
+     * extraction succeeds, or the server shuts down.
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
+     */
+    [[nodiscard]] virtual OptionalGetLedgerResponseType
+    fetchDataAndDiff(uint32_t seq) = 0;
+};
+
+}  // namespace etl

src/etl/impl/Extractor.hpp

@@ -121,19 +121,19 @@ class Extractor {
         pipe_.get().finish(startSequence_);
     }
 
-    bool
+    [[nodiscard]] bool
     isStopping() const
     {
         return state_.get().isStopping;
     }
 
-    bool
+    [[nodiscard]] bool
     hasWriteConflict() const
     {
         return state_.get().writeConflict;
     }
 
-    bool
+    [[nodiscard]] bool
     shouldFinish(uint32_t seq) const
     {
         // Stopping conditions:

src/etl/impl/LedgerFetcher.hpp

@@ -63,7 +63,7 @@ class LedgerFetcher {
      * @param sequence sequence of the ledger to extract
      * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
      */
-    OptionalGetLedgerResponseType
+    [[nodiscard]] OptionalGetLedgerResponseType
     fetchData(uint32_t sequence)
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
@@ -83,7 +83,7 @@ class LedgerFetcher {
      * @param sequence sequence of the ledger to extract
      * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
      */
-    OptionalGetLedgerResponseType
+    [[nodiscard]] OptionalGetLedgerResponseType
     fetchDataAndDiff(uint32_t sequence)
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;

src/etlng/CMakeLists.txt

@@ -1,5 +1,5 @@
-add_library(clio_etlng INTERFACE)
+add_library(clio_etlng)
 
-# target_sources(clio_etlng PRIVATE )
+target_sources(clio_etlng PRIVATE impl/Extraction.cpp)
 
-target_link_libraries(clio_etlng INTERFACE clio_data)
+target_link_libraries(clio_etlng PUBLIC clio_data)

src/etlng/ExtractorInterface.hpp

@@ -0,0 +1,54 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+
+#include <cstdint>
+#include <optional>
+
+namespace etlng {
+
+/**
+ * @brief An interface for the Extractor
+ */
+struct ExtractorInterface {
+    virtual ~ExtractorInterface() = default;
+
+    /**
+     * @brief Extract diff data for a particular ledger
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger data diff between sequence and parent if available
+     */
+    [[nodiscard]] virtual std::optional<model::LedgerData>
+    extractLedgerWithDiff(uint32_t seq) = 0;
+
+    /**
+     * @brief Extract data for a particular ledger
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger header and transaction+metadata blobs if available
+     */
+    [[nodiscard]] virtual std::optional<model::LedgerData>
+    extractLedgerOnly(uint32_t seq) = 0;
+};
+
+}  // namespace etlng

src/etlng/Models.hpp

@@ -120,6 +120,7 @@ struct LedgerData {
     std::vector<Transaction> transactions;
     std::vector<Object> objects;
     std::optional<std::vector<BookSuccessor>> successors;
+    std::optional<std::vector<std::string>> edgeKeys;
 
     ripple::LedgerHeader header;
     std::string rawHeader;