Enhancement/skaled 1425 threading rules

.github/workflows/custom_build.yml

@@ -95,18 +95,18 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=Debug
           cd deps
           ./clean.sh
           rm -f ./libwebsockets-from-git.tar.gz
-          ./build.sh PARALLEL_COUNT=$(nproc)
+          ./build.sh PARALLEL_COUNT=$(nproc) DEBUG=1
           cd ..
       - name: Configure all
         run: |
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=Debug
           mkdir -p build
           cd build
           cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE ${{ github.event.inputs.cmake_options }} ..
@@ -116,11 +116,11 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=Debug
           cd build
           make skaled -j$(nproc)
-          echo "Ensure release mode skaled does not have any debug markers"
-          strip skaled/skaled
+          #echo "Ensure release mode skaled does not have any debug markers"
+          #strip skaled/skaled
           cd ..
 
       - name: Build and publish container

.github/workflows/publish.yml

@@ -115,7 +115,7 @@ jobs:
           export CMAKE_BUILD_TYPE=Release
           cd build
           make skaled -j$(nproc)
-          echo "Ensure release mode skaled does not have any debug markers"
+          #echo "Ensure release mode skaled does not have any debug markers"
           strip skaled/skaled
           cd ..
       - name: Configure historic state build
@@ -137,7 +137,7 @@ jobs:
           export CMAKE_BUILD_TYPE=Release
           cd build-historic
           make skaled -j$(nproc)
-          echo "Ensure release mode skaled does not have any debug markers"
+          #echo "Ensure release mode skaled does not have any debug markers"
           strip skaled/skaled
           cd ..
       - name: Build and publish container

libdevcore/Common.cpp

@@ -20,6 +20,7 @@
 #include "Common.h"
 #include "Exceptions.h"
 #include "Log.h"
+#include "taskmon.h"
 
 #include <skale/buildinfo.h>
 
@@ -34,41 +35,132 @@ char const* Version = skale_get_buildinfo()->project_version;
 bytes const NullBytes;
 std::string const EmptyString;
 
-std::shared_ptr< StatusAndControl > ExitHandler::statusAndControl;
-
 bool ExitHandler::shouldExit() {
-    return skutils::signal::g_bStop;
+    return s_bStop;
 }
 int ExitHandler::getSignal() {
-    return skutils::signal::g_nStopSignal;
+    return s_nStopSignal;
 }
 
 void ExitHandler::exitHandler( int s ) {
     exitHandler( s, ec_success );
 }
 
-void ExitHandler::exitHandler( int s, ExitHandler::exit_code_t ec ) {
-    skutils::signal::g_nStopSignal = s;
+void ExitHandler::exitHandler( int nSignalNo, ExitHandler::exit_code_t ec ) {
+    std::string strMessagePrefix = ExitHandler::shouldExit() ?
+                                       cc::error( "\nStop flag was already raised on. " ) +
+                                           cc::fatal( "WILL FORCE TERMINATE." ) +
+                                           cc::error( " Caught (second) signal. " ) :
+                                       cc::error( "\nCaught (first) signal. " );
+    std::cerr << strMessagePrefix << cc::error( skutils::signal::signal2str( nSignalNo ) )
+              << "\n\n";
+    std::cerr.flush();
+
+    switch ( nSignalNo ) {
+    case SIGINT:
+    case SIGTERM:
+    case SIGHUP:
+        // exit normally
+        // just fall through
+        break;
+
+    case SIGSTOP:
+    case SIGTSTP:
+    case SIGPIPE:
+        // ignore
+        return;
+        break;
+
+    case SIGQUIT:
+        // exit immediately
+        _exit( ExitHandler::ec_termninated_by_signal );
+        break;
+
+    default:
+        // abort signals
+        std::cout << "\n" << skutils::signal::generate_stack_trace() << "\n";
+        std::cout.flush();
+
+        break;
+    }  // switch
+
+    // try to exit nicely - then abort
+    if ( !ExitHandler::shouldExit() ) {
+        static volatile bool g_bSelfKillStarted = false;
+        if ( !g_bSelfKillStarted ) {
+            g_bSelfKillStarted = true;
+
+            auto start_time = std::chrono::steady_clock::now();
+
+            std::thread( [nSignalNo, start_time]() {
+                std::cerr << ( "\n" + cc::fatal( "SELF-KILL:" ) + " " + cc::error( "Will sleep " ) +
+                               cc::size10( ExitHandler::KILL_TIMEOUT ) +
+                               cc::error( " seconds before force exit..." ) + "\n\n" );
+                std::cerr.flush();
+
+                clog( VerbosityInfo, "exit" ) << "THREADS timer started";
+
+                // while waiting, every 0.1s check whch threades exited
+                vector< string > threads;
+                for ( int i = 0; i < ExitHandler::KILL_TIMEOUT * 10; ++i ) {
+                    auto end_time = std::chrono::steady_clock::now();
+                    float seconds = std::chrono::duration< float >( end_time - start_time ).count();
+
+                    try {
+                        vector< string > new_threads = taskmon::list_names();
+                        vector< string > threads_diff = taskmon::lists_diff( threads, new_threads );
+                        threads = new_threads;
+
+                        if ( threads_diff.size() ) {
+                            cerr << seconds << " THREADS " << threads.size() << ":";
+                            for ( const string& t : threads_diff )
+                                cerr << " " << t;
+                            cerr << endl;
+                        }
+                    } catch ( ... ) {
+                        // swallow it
+                    }
+
+                    std::this_thread::sleep_for( 100ms );
+                }
+
+                std::cerr << ( "\n" + cc::fatal( "SELF-KILL:" ) + " " +
+                               cc::error( "Will force exit after sleeping " ) +
+                               cc::size10( ExitHandler::KILL_TIMEOUT ) + cc::error( " second(s)" ) +
+                               "\n\n" );
+                std::cerr.flush();
+
+                // TODO deduplicate this with main() before return
+                ExitHandler::exit_code_t ec = ExitHandler::requestedExitCode();
+                if ( ec == ExitHandler::ec_success ) {
+                    if ( nSignalNo != SIGINT && nSignalNo != SIGTERM )
+                        ec = ExitHandler::ec_failure;
+                }
+
+                _exit( ec );
+            } ).detach();
+        }  // if( ! g_bSelfKillStarted )
+    }      // if ( !skutils::signal::g_bStop )
+
+    // nice exit here:
+
+    if ( ExitHandler::shouldExit() ) {
+        std::cerr << ( "\n" + cc::fatal( "SIGNAL-HANDLER:" ) + " " +
+                       cc::error( "Will force exit now..." ) + "\n\n" );
+        _exit( 13 );
+    }
+
+    s_nStopSignal = nSignalNo;
 
     if ( ec != ec_success ) {
-        g_ec = ec;
+        s_ec = ec;
     }
 
     // indicate failure if signal is not INT or TERM!
-    if ( g_ec == ec_success && s != SIGINT && s != SIGTERM )
-        g_ec = ExitHandler::ec_failure;
-
-    if ( statusAndControl ) {
-        statusAndControl->setExitState( StatusAndControl::StartAgain, ( g_ec != ec_success ) );
-        statusAndControl->setExitState(
-            StatusAndControl::StartFromSnapshot, ( g_ec == ec_state_root_mismatch ) );
-        statusAndControl->setExitState(
-            StatusAndControl::ClearDataDir, ( g_ec == ec_state_root_mismatch ) );
-    }  // if
-
-    skutils::signal::g_bStop = true;
-    // HACK wait for loop in main to send exit call to consensus et al.
-    std::this_thread::sleep_for( chrono::milliseconds( 2000 ) );
+    if ( s_ec == ec_success && nSignalNo != SIGINT && nSignalNo != SIGTERM )
+        s_ec = ExitHandler::ec_failure;
+
+    s_bStop = true;
 }
 
 void InvariantChecker::checkInvariants(
@@ -126,6 +218,8 @@ string inUnits( bigint const& _b, strings const& _units ) {
     return ret.str();
 }
 
-volatile ExitHandler::exit_code_t ExitHandler::g_ec = ExitHandler::ec_success;
+std::atomic< ExitHandler::exit_code_t > ExitHandler::s_ec = ExitHandler::ec_success;
+std::atomic_int ExitHandler::s_nStopSignal{ 0 };
+std::atomic_bool ExitHandler::s_bStop{ false };
 
 }  // namespace dev

libdevcore/Common.h

@@ -353,12 +353,12 @@ class ExitHandler {
     static void exitHandler( int s, ExitHandler::exit_code_t ec );
     static bool shouldExit();
     static int getSignal();
-    static exit_code_t requestedExitCode() { return g_ec; }
-
-    static std::shared_ptr< StatusAndControl > statusAndControl;
+    static exit_code_t requestedExitCode() { return s_ec; }
 
 private:
-    static volatile exit_code_t g_ec;
+    static std::atomic< exit_code_t > s_ec;
+    static std::atomic_int s_nStopSignal;
+    static std::atomic_bool s_bStop;
 
     ExitHandler() = delete;
 };