Merge pull request #333 from skalenetwork/bug/SKALE-3135-shutdown-can…

…-be-better bug/SKALE-3135-shutdown-can-be-better
skalenetwork · Aug 28, 2020 · ba2043e · ba2043e
2 parents 9f19299 + f1dd646
commit ba2043e
Show file tree

Hide file tree

Showing 13 changed files with 320 additions and 124 deletions.
diff --git a/libconsensus b/libconsensus
diff --git a/libdevcore/Worker.h b/libdevcore/Worker.h
@@ -60,9 +60,11 @@ class Worker {
     /// Starts worker thread; causes startedWorking() to be called.
     void startWorking();
 
+public:
     /// Stop worker thread; causes call to stopWorking().
     void stopWorking();
 
+protected:
     /// Returns if worker thread is present.
     bool isWorking() const {
         Guard l( x_work );

diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h
@@ -450,10 +450,12 @@ class BlockChain {
     /// Initialise everything and ready for openning the database.
     void init( ChainParams const& _p );
     /// Open the database.
+public:
     void open( boost::filesystem::path const& _path, WithExisting _we );
     /// Finalise everything and close the database.
     void close();
 
+private:
     void rotateDBIfNeeded();
 
     ImportRoute insertBlockAndExtras( VerifiedBlockRef const& _block, bytesConstRef _receipts,

diff --git a/libethereum/BlockQueue.cpp b/libethereum/BlockQueue.cpp
@@ -49,11 +49,16 @@ BlockQueue::BlockQueue() {
     // Allow some room for other activity
     unsigned verifierThreads = 1;  // needed for JsonRpcTests (real mining)  // std::max(
                                    // thread::hardware_concurrency(), 3U ) - 2U;
-    for ( unsigned i = 0; i < verifierThreads; ++i )
+    for ( unsigned i = 0; i < verifierThreads; ++i ) {
+        if ( this->m_deleting )
+            return;
         m_verifiers.emplace_back( [=]() {
+            if ( this->m_deleting )
+                return;
             setThreadName( "blockVerifier" + toString( i ) );
             this->verifierBody();
         } );
+    }
 }
 
 BlockQueue::~BlockQueue() {

diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp
@@ -119,30 +119,56 @@ Client::Client( ChainParams const& _params, int _networkID,
 }
 
 Client::~Client() {
-    if ( m_snapshotHashComputing != nullptr )
-        m_snapshotHashComputing->join();
+    stopWorking();
+}
 
-    m_new_block_watch.uninstallAll();
-    m_new_pending_transaction_watch.uninstallAll();
+void Client::stopWorking() {
+    Worker::stopWorking();
 
     if ( m_skaleHost )
         m_skaleHost->stopWorking();  // TODO Find and document a systematic way to sart/stop all
                                      // workers
     else
         cerror << "Instance of SkaleHost was not properly created.";
 
+    if ( m_snapshotHashComputing != nullptr ) {
+        try {
+            if ( m_snapshotHashComputing->joinable() )
+                m_snapshotHashComputing->join();
+        } catch ( ... ) {
+        }
+    }
+
+    m_new_block_watch.uninstallAll();
+    m_new_pending_transaction_watch.uninstallAll();
+
     m_signalled.notify_all();  // to wake up the thread from Client::doWork()
-    stopWorking();
 
     m_tq.HandleDestruction();  // l_sergiy: destroy transaction queue earlier
     m_bq.stop();               // l_sergiy: added to stop block queue processing
 
-    terminate();
+    m_bc.close();
+    LOG( m_logger ) << cc::success( "Blockchain is closed" );
 
-    if ( !m_skaleHost || m_skaleHost->exitedForcefully() == false )
+    bool isForcefulExit =
+        ( !m_skaleHost || m_skaleHost->exitedForcefully() == false ) ? false : true;
+    if ( !isForcefulExit ) {
         delete_lock_file( m_dbPath );
+        LOG( m_logger ) << cc::success( "Deleted lock file " )
+                        << cc::p( boost::filesystem::canonical( m_dbPath ).string() +
+                                  std::string( "/skaled.lock" ) );
+    } else {
+        LOG( m_logger ) << cc::fatal( "ATTENTION:" ) << " " << cc::error( "Deleted lock file " )
+                        << cc::p( boost::filesystem::canonical( m_dbPath ).string() +
+                                  std::string( "/skaled.lock" ) )
+                        << cc::error( " after foreceful exit" );
+    }
+    LOG( m_logger ).flush();
+
+    terminate();
 }
 
+
 void Client::injectSkaleHost( std::shared_ptr< SkaleHost > _skaleHost ) {
     assert( !m_skaleHost );
 
@@ -224,7 +250,7 @@ ImportResult Client::queueBlock( bytes const& _block, bool _isSafe ) {
 }
 
 tuple< ImportRoute, bool, unsigned > Client::syncQueue( unsigned _max ) {
-    stopWorking();
+    Worker::stopWorking();
     return bc().sync( m_bq, m_state, _max );
 }
 
@@ -1006,12 +1032,12 @@ ExecutionResult Client::call( Address const& _from, u256 _value, Address _dest,
             temp.mutableState().addBalance( _from, ( u256 )( t.gas() * t.gasPrice() + t.value() ) );
         ret = temp.execute( bc().lastBlockHashes(), t, Permanence::Reverted );
     } catch ( InvalidNonce const& in ) {
-        std::cout << "exception in client call(1):"
-                  << boost::current_exception_diagnostic_information() << std::endl;
+        LOG( m_logger ) << "exception in client call(1):"
+                        << boost::current_exception_diagnostic_information() << std::endl;
         throw std::runtime_error( "call with invalid nonce" );
     } catch ( ... ) {
-        std::cout << "exception in client call(2):"
-                  << boost::current_exception_diagnostic_information() << std::endl;
+        LOG( m_logger ) << "exception in client call(2):"
+                        << boost::current_exception_diagnostic_information() << std::endl;
         throw;
     }
     return ret;

diff --git a/libethereum/Client.h b/libethereum/Client.h
@@ -90,6 +90,8 @@ class Client : public ClientBase, protected Worker {
     /// Destructor.
     virtual ~Client();
 
+    void stopWorking();
+
     void injectSkaleHost( std::shared_ptr< SkaleHost > _skaleHost = nullptr );
 
     /// Get information on this chain.

diff --git a/libethereum/SkaleHost.cpp b/libethereum/SkaleHost.cpp
@@ -660,7 +660,9 @@ void SkaleHost::stopWorking() {
     // if we could not lock from 1st attempt - then exit forcefully!
     if ( !locked ) {
         m_exitedForcefully = true;
-        clog( VerbosityWarning, "skale-host" ) << "Forcefully shutting down consensus!";
+        clog( VerbosityWarning, "skale-host" )
+            << cc::fatal( "ATTENTION:" ) << " "
+            << cc::error( "Forcefully shutting down consensus!" );
     }
 
 

diff --git a/libethereum/ValidationSchemes.cpp b/libethereum/ValidationSchemes.cpp
@@ -153,9 +153,28 @@ void validateConfigJson( js::mObject const& _obj ) {
             {"logLevel", {{js::str_type}, JsonFieldPresence::Optional}},
             {"logLevelConfig", {{js::str_type}, JsonFieldPresence::Optional}},
             {"logLevelProposal", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"aa", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"acceptors", {{js::int_type}, JsonFieldPresence::Optional}},
+            {"adminOrigins", {{js::array_type}, JsonFieldPresence::Optional}},
+            {"db-path", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"ipcpath", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"enable-debug-behavior-apis", {{js::bool_type}, JsonFieldPresence::Optional}},
+            {"unsafe-transactions", {{js::bool_type}, JsonFieldPresence::Optional}},
+            {"web3-trace", {{js::bool_type}, JsonFieldPresence::Optional}},
+            {"web3-shutdown", {{js::bool_type}, JsonFieldPresence::Optional}},
+            {"unsafe-transactions", {{js::bool_type}, JsonFieldPresence::Optional}},
+            {"max-connections", {{js::int_type}, JsonFieldPresence::Optional}},
+            {"max-http-queues", {{js::int_type}, JsonFieldPresence::Optional}},
+            {"ws-mode", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"ws-log", {{js::str_type}, JsonFieldPresence::Optional}},
             {"imaMainNet", {{js::str_type}, JsonFieldPresence::Optional}},
             {"imaMessageProxySChain", {{js::str_type}, JsonFieldPresence::Optional}},
-            {"imaMessageProxyMainNet", {{js::str_type}, JsonFieldPresence::Optional}}} );
+            {"imaMessageProxyMainNet", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"imaMessageProxySChain", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"imaMessageProxyMainNet", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"imaCallerAddressSChain", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"imaCallerAddressMainNet", {{js::str_type}, JsonFieldPresence::Optional}},
+            {"wallets", {{js::obj_type}, JsonFieldPresence::Optional}}} );
 
     std::string keyShareName = "";
     try {
@@ -207,9 +226,13 @@ void validateConfigJson( js::mObject const& _obj ) {
                 {"ip6", {{js::str_type}, JsonFieldPresence::Optional}},
                 {"basePort6", {{js::int_type}, JsonFieldPresence::Optional}},
                 {"httpRpcPort", {{js::int_type}, JsonFieldPresence::Optional}},
+                {"httpRpcPort6", {{js::int_type}, JsonFieldPresence::Optional}},
                 {"httpsRpcPort", {{js::int_type}, JsonFieldPresence::Optional}},
+                {"httpsRpcPort6", {{js::int_type}, JsonFieldPresence::Optional}},
                 {"wsRpcPort", {{js::int_type}, JsonFieldPresence::Optional}},
+                {"wsRpcPort6", {{js::int_type}, JsonFieldPresence::Optional}},
                 {"wssRpcPort", {{js::int_type}, JsonFieldPresence::Optional}},
+                {"wssRpcPort6", {{js::int_type}, JsonFieldPresence::Optional}},
                 {"schainIndex", {{js::int_type}, JsonFieldPresence::Required}},
                 {"publicKey", {{js::str_type}, JsonFieldPresence::Optional}},
                 {"blsPublicKey0", {{js::str_type}, JsonFieldPresence::Optional}},

diff --git a/libskutils/include/skutils/dispatch.h b/libskutils/include/skutils/dispatch.h
@@ -664,6 +664,7 @@ class queue : public ref_retain_release {
 
 class domain : public ref_retain_release {
 public:
+    static std::atomic_bool g_bVerboseDispatchThreadDetailsLogging;
     typedef skutils::multithreading::recursive_mutex_type mutex_type;
     typedef std::lock_guard< mutex_type > lock_type;
 
@@ -708,6 +709,7 @@ class domain : public ref_retain_release {
     //
     skutils::thread_pool thread_pool_;
     std::atomic_size_t cntRunningThreads_;
+    std::atomic_size_t cntStartTestedThreads_;
     //
     std::atomic_uint64_t decrease_accumulators_counter_, decrease_accumulators_period_;
     //

diff --git a/libskutils/include/skutils/thread_pool.h b/libskutils/include/skutils/thread_pool.h
@@ -146,6 +146,8 @@ class thread_pool {
         // wake up one thread if its waiting
         conditional_lock_.notify_one();
     }
+    void notify_one() { conditional_lock_.notify_one(); }
+    void notify_all() { conditional_lock_.notify_all(); }
 };  /// class thread_pool
 
 };  // namespace skutils
+28 −1		.github/workflows/build.yml
+4 −0		abstracttcpclient/AbstractClientAgent.cpp
+10 −3		abstracttcpserver/AbstractServerAgent.cpp
+15 −0		chains/Schain.cpp
+3 −0		monitoring/MonitoringAgent.cpp
+2 −0		network/Network.cpp
+6 −0		node/ConsensusEngine.cpp
+7 −1		node/Node.cpp
+4 −0		protocols/blockconsensus/BlockConsensusAgent.cpp