1545 without dos and stats cleanups

.clang-format-ignore

@@ -0,0 +1,19 @@
+./test/tools/libtesteth/*
+./test/tools/fuzzTesting/*
+./test/tools/libtestutils/*
+./test/tools/jsontests/*
+./test/unittests/external-dependencies/*
+./test/unittests/libskutils/*
+./test/unittests/libdevcrypto/*
+./test/unittests/libethcore/*
+./test/unittests/libethereum/*
+./test/unittests/libweb3core/*
+./test/unittests/libweb3jsonrpc/*
+./test/unittests/libtesteth/*
+./test/unittests/libdevcore/*
+./test/unittests/libethashseal/*
+./test/unittests/mapreduce_consensus/*
+./test/unittests/libevm/*
+./test/unittests/libskale/*
+./storage_benchmark/*
+./skale-vm/*

VERSION

@@ -1 +1,2 @@
 4.0.0
+

libbatched-io/batched_db.cpp

@@ -4,11 +4,6 @@ namespace batched_io {
 
 using namespace dev::db;
 
-batched_db::~batched_db() {
-    // all batches should be either commit()'ted or revert()'ed!
-    assert( !m_batch );
-}
-
 db_operations_face* db_splitter::new_interface() {
     assert( this->m_interfaces.size() < 256 );
 

libbatched-io/batched_db.h

@@ -3,6 +3,7 @@
 
 #include "batched_io.h"
 
+#include <libdevcore/DBImpl.h>
 #include <libdevcore/LevelDB.h>
 
 #include <shared_mutex>
@@ -76,7 +77,64 @@ class batched_db : public db_face {
         m_db->forEachWithPrefix( _prefix, f );
     }
 
-    virtual ~batched_db();
+    virtual ~batched_db() = default;
+
+protected:
+    void recover() { /*nothing*/
+    }
+};
+
+
+class read_only_snap_based_batched_db : public db_face {
+private:
+    std::shared_ptr< dev::db::DBImpl > m_db;
+    std::shared_ptr< dev::db::LevelDBSnap > m_snap;
+
+public:
+    read_only_snap_based_batched_db(
+        std::shared_ptr< dev::db::DBImpl > _db, std::shared_ptr< dev::db::LevelDBSnap > _snap ) {
+        LDB_CHECK( _db );
+        LDB_CHECK( _snap );
+        m_db = _db;
+        m_snap = _snap;
+    }
+
+    bool is_open() const { return !!m_db; };
+
+    void insert( dev::db::Slice, dev::db::Slice ) override {
+        throw std::runtime_error( "Function not implemented:" + std::string( __FUNCTION__ ) );
+    }
+
+    void kill( dev::db::Slice ) override {
+        throw std::runtime_error( "Function not implemented:" + std::string( __FUNCTION__ ) );
+    }
+
+    void revert() override {
+        throw std::runtime_error( "Function not implemented:" + std::string( __FUNCTION__ ) );
+    }
+
+    void commit( const std::string& ) override {
+        throw std::runtime_error( "Function not implemented:" + std::string( __FUNCTION__ ) );
+    }
+
+    // readonly
+    std::string lookup( dev::db::Slice _key ) const override {
+        return m_db->lookup( _key, m_snap );
+    }
+
+    bool exists( dev::db::Slice _key ) const override { return m_db->exists( _key, m_snap ); }
+
+    void forEach( std::function< bool( dev::db::Slice, dev::db::Slice ) > _f ) const override {
+        static std::string emptyString;
+        return forEachWithPrefix( emptyString, _f );
+    }
+
+    void forEachWithPrefix( std::string& _prefix,
+        std::function< bool( dev::db::Slice, dev::db::Slice ) > _f ) const override {
+        m_db->forEachWithPrefix( _prefix, _f, m_snap );
+    }
+
+    virtual ~read_only_snap_based_batched_db() = default;
 
 protected:
     void recover() { /*nothing*/

libdevcore/LevelDB.cpp

@@ -20,6 +20,7 @@
 
 #include "LevelDB.h"
 #include "Assertions.h"
+#include "LevelDBSnap.h"
 #include "Log.h"
 #include <libdevcore/microprofile.h>
 
@@ -144,6 +145,7 @@ void LevelDB::openDBInstanceUnsafe() {
 
     m_db.reset( db );
     m_lastDBOpenTimeMs = getCurrentTimeMs();
+    m_dbReopenId++;
     cnote << "LEVELDB_OPENED:TIME_MS:" << m_lastDBOpenTimeMs - startTimeMs;
 }
 uint64_t LevelDB::getCurrentTimeMs() {
@@ -159,14 +161,15 @@ LevelDB::~LevelDB() {
 }
 
 std::string LevelDB::lookup( Slice _key ) const {
+    return lookup( _key, nullptr );
+}
+
+std::string LevelDB::lookup( Slice _key, const std::shared_ptr< LevelDBSnap >& _snap ) const {
     leveldb::Slice const key( _key.data(), _key.size() );
     std::string value;
 
-    leveldb::Status status;
-    {
-        SharedDBGuard readLock( *this );
-        status = m_db->Get( m_readOptions, key, &value );
-    }
+    auto status = getValue( m_readOptions, key, value, _snap );
+
     if ( status.IsNotFound() )
         return std::string();
 
@@ -175,26 +178,41 @@ std::string LevelDB::lookup( Slice _key ) const {
 }
 
 bool LevelDB::exists( Slice _key ) const {
+    return exists( _key, nullptr );
+}
+
+bool LevelDB::exists( Slice _key, const std::shared_ptr< LevelDBSnap >& _snap ) const {
     std::string value;
     leveldb::Slice const key( _key.data(), _key.size() );
-    leveldb::Status status;
-    {
-        SharedDBGuard lock( *this );
-        status = m_db->Get( m_readOptions, key, &value );
-    }
+
+    auto status = getValue( m_readOptions, key, value, _snap );
+
     if ( status.IsNotFound() )
         return false;
 
     checkStatus( status );
     return true;
 }
 
+leveldb::Status LevelDB::getValue( leveldb::ReadOptions _readOptions, const leveldb::Slice& _key,
+    std::string& _value, const std::shared_ptr< LevelDBSnap >& _snap ) const {
+    SharedDBGuard lock( *this );  // protect so db is not reopened during get call
+    if ( _snap ) {
+        // sanity check to make sure that the snap was created for this particular
+        // db handle
+        LDB_CHECK( m_dbReopenId == _snap->getParentDbReopenId() );
+        return _snap->getValue( m_db, _readOptions, _key, _value );
+    } else {
+        return m_db->Get( _readOptions, _key, &_value );
+    }
+}
+
 void LevelDB::insert( Slice _key, Slice _value ) {
     leveldb::Slice const key( _key.data(), _key.size() );
     leveldb::Slice const value( _value.data(), _value.size() );
     leveldb::Status status;
     {
-        SharedDBGuard lock( *this );
+        SharedDBGuard lock( *this );  // protect so db is not reopened during Put() call
         status = m_db->Put( m_writeOptions, key, value );
     }
     checkStatus( status );
@@ -224,7 +242,7 @@ void LevelDB::commit( std::unique_ptr< WriteBatchFace > _batch ) {
     }
     leveldb::Status status;
     {
-        SharedDBGuard lock( *this );
+        SharedDBGuard lock( *this );  // protect so db is not reopened during Write() call
         status = m_db->Write( m_writeOptions, &batchPtr->writeBatch() );
     }
     // Commit happened. This means the keys actually got deleted in LevelDB. Increment key deletes
@@ -247,17 +265,25 @@ void LevelDB::reopenDataBaseIfNeeded() {
     auto currentTimeMs = getCurrentTimeMs();
 
     if ( currentTimeMs - m_lastDBOpenTimeMs >= ( uint64_t ) m_reopenPeriodMs ) {
-        ExclusiveDBGuard lock( *this );
-        // releasing unique pointer will cause database destructor to be called that will close db
-        m_db.reset();
-        // now open db while holding the exclusive lock
-        openDBInstanceUnsafe();
+        reopen();
     }
 }
+void LevelDB::reopen() {
+    ExclusiveDBGuard lock( *this );
+    // close all current snaps by passing max lifetime as zero
+    auto aliveSnaps = m_snapManager.garbageCollectUnusedOldSnaps( m_db, m_dbReopenId, 0 );
+    LDB_CHECK( aliveSnaps == 0 );
+
+    // releasing unique pointer will cause database destructor to be called that will close db
+    LDB_CHECK( m_db );
+    m_db.reset();
+    // now open db while holding the exclusive lock
+    openDBInstanceUnsafe();
+}
 
 void LevelDB::forEach( std::function< bool( Slice, Slice ) > f ) const {
     cwarn << "Iterating over the entire LevelDB database: " << this->m_path;
-    SharedDBGuard lock( *this );
+    SharedDBGuard lock( *this );  // protect so db is not reopened during iteration
     std::unique_ptr< leveldb::Iterator > itr( m_db->NewIterator( m_readOptions ) );
     if ( itr == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
@@ -273,10 +299,22 @@ void LevelDB::forEach( std::function< bool( Slice, Slice ) > f ) const {
 }
 
 void LevelDB::forEachWithPrefix(
-    std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
-    cnote << "Iterating over the LevelDB prefix: " << _prefix;
-    SharedDBGuard lock( *this );
-    std::unique_ptr< leveldb::Iterator > itr( m_db->NewIterator( m_readOptions ) );
+    std::string& _prefix, std::function< bool( Slice, Slice ) > _f ) const {
+    forEachWithPrefix( _prefix, _f, nullptr );
+}
+
+void LevelDB::forEachWithPrefix( std::string& _prefix, std::function< bool( Slice, Slice ) > f,
+    const std::shared_ptr< LevelDBSnap >& _snap ) const {
+    SharedDBGuard lock( *this );  // protect so DB is not reopened during iteration
+
+    std::unique_ptr< leveldb::Iterator > itr;
+
+    if ( _snap ) {
+        LDB_CHECK( m_dbReopenId == _snap->getParentDbReopenId() );
+        itr = _snap->getIterator( m_db, m_readOptions );
+    } else {
+        itr.reset( m_db->NewIterator( m_readOptions ) );
+    }
     if ( itr == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
@@ -292,6 +330,18 @@ void LevelDB::forEachWithPrefix(
     }
 }
 
+void LevelDB::createBlockSnap( uint64_t _blockNumber ) {
+    SharedDBGuard lock( *this );  // protect so db is not reopened during snap creation
+    m_snapManager.addSnapForBlock( _blockNumber, m_db, m_dbReopenId );
+}
+
+std::shared_ptr< LevelDBSnap > LevelDB::getLastBlockSnap() const {
+    SharedDBGuard lock( *this );  // protect so db is not reopened when while we get snap
+    auto snap = m_snapManager.getLastBlockSnap();
+    LDB_CHECK( snap );
+    return snap;
+}
+
 h256 LevelDB::hashBase() const {
     SharedDBGuard lock( *this );
     std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );

libdevcore/LevelDB.h

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include "LevelDBSnapManager.h"
 #include "db.h"
 
 #include <leveldb/db.h>
@@ -29,7 +30,17 @@
 #include <secp256k1_sha256.h>
 #include <shared_mutex>
 
+#define LDB_CHECK( _EXPRESSION_ )                                                             \
+    if ( !( _EXPRESSION_ ) ) {                                                                \
+        auto __msg__ = std::string( "State check failed::" ) + #_EXPRESSION_ + " " +          \
+                       std::string( __FILE__ ) + ":" + std::to_string( __LINE__ );            \
+        BOOST_THROW_EXCEPTION( dev::db::DatabaseError() << dev::errinfo_comment( __msg__ ) ); \
+    }
+
 namespace dev::db {
+
+class LevelDBSnap;
+
 class LevelDB : public DatabaseFace {
 public:
     static leveldb::ReadOptions defaultReadOptions();
@@ -58,6 +69,18 @@ class LevelDB : public DatabaseFace {
     void forEachWithPrefix(
         std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const override;
 
+    // create a read only snap after blockl processing
+    void createBlockSnap( uint64_t _blockNumber );
+
+    // get block snap for the lasty block
+    std::shared_ptr< LevelDBSnap > getLastBlockSnap() const;
+
+    // perform operations with respect to a particular read only snap
+    std::string lookup( Slice _key, const std::shared_ptr< LevelDBSnap >& _snap ) const;
+    bool exists( Slice _key, const std::shared_ptr< LevelDBSnap >& _snap ) const;
+    void forEachWithPrefix( std::string& _prefix, std::function< bool( Slice, Slice ) > f,
+        const std::shared_ptr< LevelDBSnap >& _snap ) const;
+
     h256 hashBase() const override;
     h256 hashBaseWithPrefix( char _prefix ) const;
 
@@ -75,6 +98,14 @@ class LevelDB : public DatabaseFace {
 
 private:
     std::unique_ptr< leveldb::DB > m_db;
+
+    // stores and manages snap objects
+    LevelDBSnapManager m_snapManager;
+    // this is incremented each time this LevelDB instance is reopened
+    // we reopen states LevelDB every day on archive nodes to avoid
+    // meta file getting too large
+    // in other cases LevelDB is never reopened to this stays zero
+    std::atomic< uint64_t > m_dbReopenId = 0;
     leveldb::ReadOptions const m_readOptions;
     leveldb::WriteOptions const m_writeOptions;
     leveldb::Options m_options;
@@ -125,6 +156,9 @@ class LevelDB : public DatabaseFace {
     };
     void openDBInstanceUnsafe();
     void reopenDataBaseIfNeeded();
+    leveldb::Status getValue( leveldb::ReadOptions _readOptions, const leveldb::Slice& _key,
+        std::string& _value, const std::shared_ptr< LevelDBSnap >& _snap ) const;
+    void reopen();
 };
 
 }  // namespace dev::db