resolved conflicts

CMakeLists.txt

@@ -97,6 +97,7 @@ option( HUNTER_RUN_UPLOAD "Upload binaries to the cache server" ${run_upload} )
 
 include( HunterGate )
 HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.76.tar.gz" SHA1 "c7b60993e841850e2c449afd454f5d5aa4ec04e4" LOCAL )
+#HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.214.tar.gz" SHA1 "e14bc153a7f16d6a5eeec845fb0283c8fad8c358" LOCAL ) new leveldb
 
 set( CMAKE_CXX_STANDARD 17 )
 

libbatched-io/batched_db.h

@@ -3,7 +3,7 @@
 
 #include "batched_io.h"
 
-#include <libdevcore/db.h>
+#include <libdevcore/LevelDB.h>
 
 #include <shared_mutex>
 

libbatched-io/batched_rotating_db_io.h

@@ -3,7 +3,7 @@
 
 #include "batched_io.h"
 
-#include <libdevcore/db.h>
+#include <libdevcore/LevelDB.h>
 
 #include <boost/filesystem.hpp>
 

libdevcore/LevelDB.cpp

@@ -227,5 +227,9 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
     return hash;
 }
 
+// void LevelDB::doCompaction() const {
+//    m_db->CompactRange( NULL, NULL );
+//}
+
 }  // namespace db
 }  // namespace dev

libdevcore/LevelDB.h

@@ -51,6 +51,8 @@ class LevelDB : public DatabaseFace {
     h256 hashBase() const override;
     h256 hashBaseWithPrefix( char _prefix ) const;
 
+    //    void doCompaction() const;
+
 private:
     std::unique_ptr< leveldb::DB > m_db;
     leveldb::ReadOptions const m_readOptions;

libethereum/BlockChain.cpp

@@ -1364,6 +1364,14 @@ void BlockChain::clearCaches() {
     }
 }
 
+// void BlockChain::doLevelDbCompaction() const {
+//    for ( auto it = m_rotator->begin(); it != m_rotator->end(); ++it ) {
+//        dev::db::LevelDB* ldb = dynamic_cast< dev::db::LevelDB* >( it->get() );
+//        assert( ldb );
+//        ldb->doCompaction();
+//    }
+//}
+
 void BlockChain::checkConsistency() {
     DEV_WRITE_GUARDED( x_details ) { m_details.clear(); }
 

libethereum/BlockChain.h

@@ -470,6 +470,8 @@ class BlockChain {
     void open( boost::filesystem::path const& _path, bool _applyPatches, WithExisting _we );
     /// Finalise everything and close the database.
     void close();
+    //    /// compact db before snapshot
+    //    void doLevelDbCompaction() const;
 
 private:
     bool rotateDBIfNeeded( uint64_t pieceUsageBytes );
@@ -587,6 +589,7 @@ class BlockChain {
     uint64_t m_maxStorageUsage;
 
     /// The disk DBs. Thread-safe, so no need for locks.
+    //    std::shared_ptr< batched_io::rotating_db_io > m_rotator;       // for compaction
     std::shared_ptr< db::ManuallyRotatingLevelDB > m_rotating_db;  // rotate()
     std::shared_ptr< batched_io::db_face > m_db;                   // insert()/commit()
     std::unique_ptr< batched_io::db_splitter > m_db_splitter;      // new_interface()

libethereum/Client.h

@@ -309,6 +309,10 @@ class Client : public ClientBase, protected Worker {
         return chainParams().sChain.nodeGroups[imaBLSPublicKeyGroupIndex].blsPublicKey;
     }
 
+    //    void doStateDbCompaction() const { m_state.getOriginalDb()->doCompaction(); }
+
+    //    void doBlocksDbCompaction() const { m_bc.doLevelDbCompaction(); }
+
     std::pair< uint64_t, uint64_t > getBlocksDbUsage() const;
 
     std::pair< uint64_t, uint64_t > getStateDbUsage() const;

libethereum/Precompiled.cpp

@@ -546,31 +546,7 @@ ETH_REGISTER_PRECOMPILED( calculateFileHash )( bytesConstRef _in ) {
             throw std::runtime_error( "calculateFileHash() failed because file does not exist" );
         }
 
-        std::ifstream file( filePath.string() );
-        file.seekg( 0, std::ios::end );
-        size_t fileSize = file.tellg();
-        std::string fileContent( fileSize, ' ' );
-        file.seekg( 0 );
-        file.read( &fileContent[0], fileSize );
-
-        const std::string fileHashName = filePath.string() + "._hash";
-
-        std::string relativePath =
-            filePath.string().substr( filePath.string().find( "filestorage" ) );
-
-        dev::h256 filePathHash = dev::sha256( relativePath );
-
-        dev::h256 fileContentHash = dev::sha256( fileContent );
-
-        secp256k1_sha256_t ctx;
-        secp256k1_sha256_initialize( &ctx );
-        secp256k1_sha256_write( &ctx, filePathHash.data(), filePathHash.size );
-        secp256k1_sha256_write( &ctx, fileContentHash.data(), fileContentHash.size );
-
-        dev::h256 commonFileHash;
-        secp256k1_sha256_finalize( &ctx, commonFileHash.data() );
-
-        g_overlayFS->writeHashFile( fileHashName, commonFileHash );
+        g_overlayFS->calculateFileHash( filePath.string() );
 
         u256 code = 1;
         bytes response = toBigEndian( code );

libethereum/SkaleHost.cpp

@@ -484,9 +484,9 @@ ConsensusExtFace::transactions_vector SkaleHost::pendingTransactions(
     }
     if ( found_difference ) {
         clog( VerbosityError, "skale-host" ) << "Transaction order disorder detected!!";
-        clog( VerbosityError, "skale-host" ) << "<i> <old> <new>";
+        clog( VerbosityTrace, "skale-host" ) << "<i> <old> <new>";
         for ( size_t i = 0; i < txns.size(); ++i ) {
-            clog( VerbosityError, "skale-host" )
+            clog( VerbosityTrace, "skale-host" )
                 << i << " " << saved_txns[i].sha3() << " " << txns[i].sha3();
         }
     }

libskale/OverlayFS.cpp

@@ -24,6 +24,8 @@
 
 #include "OverlayFS.h"
 #include "RevertableFSPatch.h"
+#include <libdevcrypto/Hash.h>
+#include <secp256k1_sha256.h>
 #include <fstream>
 
 
@@ -110,7 +112,7 @@ bool WriteChunkOp::execute() {
         std::fstream file;
         file.open( this->path, std::ios::binary | std::ios::out | std::ios::in );
         file.seekp( static_cast< long >( this->position ) );
-        file.write( ( char* ) this->data, this->dataLength );
+        file.write( reinterpret_cast< const char* >( &this->data[0] ), this->dataLength );
         return true;
     } catch ( std::exception& ex ) {
         std::string strError = ex.what();
@@ -123,12 +125,35 @@ bool WriteChunkOp::execute() {
     return false;
 }
 
-bool WriteHashFileOp::execute() {
+bool CalculateFileHash::execute() {
     try {
+        std::ifstream file( this->path );
+        file.seekg( 0, std::ios::end );
+        size_t fileSize = file.tellg();
+        std::string fileContent( fileSize, ' ' );
+        file.seekg( 0 );
+        file.read( &fileContent[0], fileSize );
+
+        const std::string fileHashName = this->path + "._hash";
+
+        std::string relativePath = this->path.substr( this->path.find( "filestorage" ) );
+
+        dev::h256 filePathHash = dev::sha256( relativePath );
+
+        dev::h256 fileContentHash = dev::sha256( fileContent );
+
+        secp256k1_sha256_t ctx;
+        secp256k1_sha256_initialize( &ctx );
+        secp256k1_sha256_write( &ctx, filePathHash.data(), filePathHash.size );
+        secp256k1_sha256_write( &ctx, fileContentHash.data(), fileContentHash.size );
+
+        dev::h256 commonFileHash;
+        secp256k1_sha256_finalize( &ctx, commonFileHash.data() );
+
         std::fstream fileHash;
-        fileHash.open( this->path, std::ios::binary | std::ios::out );
+        fileHash.open( fileHashName, std::ios::binary | std::ios::out );
         fileHash.clear();
-        fileHash << this->commonFileHash;
+        fileHash << commonFileHash;
         fileHash.close();
         return true;
     } catch ( std::exception& ex ) {
@@ -183,8 +208,8 @@ void OverlayFS::writeChunk( const std::string& filePath, const size_t position,
         operation->execute();
 }
 
-void OverlayFS::writeHashFile( const std::string& filePath, const dev::h256& commonFileHash ) {
-    auto operation = std::make_shared< WriteHashFileOp >( filePath, commonFileHash );
+void OverlayFS::calculateFileHash( const std::string& filePath ) {
+    auto operation = std::make_shared< CalculateFileHash >( filePath );
     if ( isCacheEnabled() )
         m_cache.push_back( operation );
     else

libskale/OverlayFS.h

@@ -80,25 +80,26 @@ class WriteChunkOp : public BaseOp {
 public:
     WriteChunkOp( const std::string& _path, const size_t _position, const size_t _dataLength,
         const _byte_* _data )
-        : path( _path ), position( _position ), dataLength( _dataLength ), data( _data ) {}
+        : path( _path ),
+          position( _position ),
+          dataLength( _dataLength ),
+          data( _data, _data + dataLength ) {}
     bool execute() override;
 
 private:
     const std::string path;
     const size_t position;
     const size_t dataLength;
-    const _byte_* data;
+    std::vector< _byte_ > data;
 };
 
-class WriteHashFileOp : public BaseOp {
+class CalculateFileHash : public BaseOp {
 public:
-    WriteHashFileOp( const std::string& _path, const dev::h256 _commonFileHash )
-        : path( _path ), commonFileHash( _commonFileHash ) {}
+    CalculateFileHash( const std::string& _path ) : path( _path ) {}
     bool execute() override;
 
 private:
     const std::string path;
-    const dev::h256 commonFileHash;
 };
 
 class OverlayFS {
@@ -125,7 +126,7 @@ class OverlayFS {
         const _byte_* data );
     void deleteFile( const std::string& filePath );
     void deleteDirectory( const std::string& path );
-    void writeHashFile( const std::string& filePath, const dev::h256& commonFileHash );
+    void calculateFileHash( const std::string& filePath );
 
 private:
     std::vector< std::shared_ptr< BaseOp > > m_cache;  // vector of filestorage operations for

libskale/SnapshotHashAgent.cpp

@@ -34,10 +34,11 @@
 #include <libff/common/profiling.hpp>
 
 SnapshotHashAgent::SnapshotHashAgent( const dev::eth::ChainParams& chain_params,
-    const std::array< std::string, 4 >& common_public_key, bool requireSnapshotMajority )
+    const std::array< std::string, 4 >& common_public_key,
+    const std::string& ipToDownloadSnapshotFrom )
     : chain_params_( chain_params ),
       n_( chain_params.sChain.nodes.size() ),
-      requireSnapshotMajority_( requireSnapshotMajority ) {
+      ipToDownloadSnapshotFrom_( ipToDownloadSnapshotFrom ) {
     this->hashes_.resize( n_ );
     this->signatures_.resize( n_ );
     this->public_keys_.resize( n_ );
@@ -106,7 +107,7 @@ size_t SnapshotHashAgent::verifyAllData() const {
 bool SnapshotHashAgent::voteForHash() {
     std::map< dev::h256, size_t > map_hash;
 
-    if ( 3 * this->verifyAllData() < 2 * this->n_ + 1 ) {
+    if ( 3 * this->verifyAllData() < 2 * this->n_ + 1 && ipToDownloadSnapshotFrom_.empty() ) {
         return false;
     }
 
@@ -121,7 +122,7 @@ bool SnapshotHashAgent::voteForHash() {
     }
 
     std::map< dev::h256, size_t >::iterator it;
-    if ( requireSnapshotMajority_ ) {
+    if ( ipToDownloadSnapshotFrom_.empty() ) {
         it = std::find_if(
             map_hash.begin(), map_hash.end(), [this]( const std::pair< dev::h256, size_t > p ) {
                 return 3 * p.second > 2 * this->n_;
@@ -226,10 +227,23 @@ bool SnapshotHashAgent::voteForHash() {
             return true;
         }
     } else {
-        it = map_hash.begin();
+        size_t nodeIdx = std::distance( this->chain_params_.sChain.nodes.begin(),
+            std::find_if( this->chain_params_.sChain.nodes.begin(),
+                this->chain_params_.sChain.nodes.end(), [this]( const dev::eth::sChainNode& node ) {
+                    return node.ip == ipToDownloadSnapshotFrom_;
+                } ) );
+
+        dev::h256 requiredHashValue = this->hashes_[nodeIdx];
+        if ( requiredHashValue == dev::h256() ) {
+            throw IsNotVerified( "Hash from the required node is empty" );
+        }
+
+        it = map_hash.find( requiredHashValue );
 
         this->voted_hash_.first = ( *it ).first;
-        this->voted_hash_.second = libff::alt_bn128_G1::random_element();
+        this->voted_hash_.second = this->signatures_[nodeIdx];
+
+        this->nodes_to_download_snapshot_from_.push_back( nodeIdx );
     }
 
     return true;
@@ -257,7 +271,10 @@ std::vector< std::string > SnapshotHashAgent::getNodesToDownloadSnapshotFrom(
                     unsigned n = skaleClient.skale_getLatestSnapshotBlockNumber();
                     if ( n == 0 ) {
                         const std::lock_guard< std::mutex > lock( this->hashes_mutex );
-                        this->nodes_to_download_snapshot_from_.push_back( i );
+                        if ( ipToDownloadSnapshotFrom_.empty() )
+                            nodes_to_download_snapshot_from_.push_back( i );
+                        else if ( ipToDownloadSnapshotFrom_ == chain_params_.sChain.nodes[i].ip )
+                            nodes_to_download_snapshot_from_.push_back( i );
                         delete jsonRpcClient;
                         return;
                     }

libskale/SnapshotHashAgent.h

@@ -67,7 +67,7 @@ class SnapshotHashAgent {
 public:
     SnapshotHashAgent( const dev::eth::ChainParams& chain_params,
         const std::array< std::string, 4 >& common_public_key,
-        bool requireSnapshotMajority = true );
+        const std::string& ipToDownloadSnapshotFrom );
 
     std::vector< std::string > getNodesToDownloadSnapshotFrom( unsigned block_number );
 
@@ -78,7 +78,7 @@ class SnapshotHashAgent {
 private:
     dev::eth::ChainParams chain_params_;
     unsigned n_;
-    bool requireSnapshotMajority_;
+    std::string ipToDownloadSnapshotFrom_;
     std::shared_ptr< libBLS::Bls > bls_;
 
     std::vector< dev::h256 > hashes_;

libskale/State.h

@@ -41,6 +41,7 @@
 #include "BaseState.h"
 #include "OverlayDB.h"
 #include "OverlayFS.h"
+#include <libdevcore/DBImpl.h>
 
 
 namespace std {
@@ -393,6 +394,8 @@ class State {
     /// Check if state is empty
     bool empty() const;
 
+    //    const dev::db::DBImpl* getOriginalDb() const { return m_orig_db.get(); }
+
     void resetStorageChanges() {
         storageUsage.clear();
         currentStorageUsed_ = 0;
@@ -479,6 +482,9 @@ class State {
     std::shared_ptr< boost::shared_mutex > x_db_ptr;
     std::shared_ptr< OverlayDB > m_db_ptr;  ///< Our overlay for the state.
     std::shared_ptr< OverlayFS > m_fs_ptr;  ///< Our overlay for the file system operations.
+                                            //    // HACK
+                                            //    // TODO Implement DB-registry, remove it!
+                                            //    std::shared_ptr< dev::db::DBImpl > m_orig_db;
     std::shared_ptr< size_t > m_storedVersion;
     size_t m_currentVersion;
     mutable std::unordered_map< dev::Address, dev::eth::Account > m_cache;  ///< Our address cache.