From 5a45a1990e8799b564d4a15dcaae17d3309bea72 Mon Sep 17 00:00:00 2001 From: Dima Litvinov Date: Mon, 22 Jan 2024 15:52:56 +0000 Subject: [PATCH 1/9] SKALED-1693 More logs and cleaner names for functional-tests github action --- .github/workflows/functional-tests.yml | 5 +++-- .github/workflows/publish.yml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 4042e3a05..52a420c3d 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -13,7 +13,7 @@ jobs: functional-tests: - name: Functional tests + name: Functional tests for ${{ inputs.version }} runs-on: ubuntu-20.04 env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true @@ -87,11 +87,12 @@ if: ${{ always() }} continue-on-error: true with: - name: debug + name: debug-${{ inputs.version }} path: | skaled_providers !skaled_providers/**/skaled /tmp/tmp* + /tmp/*log *.log ./integration_tests/skaled/internals/third_party/skale-node-tests/btrfs !**/.env diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 372bc11a6..d29c19089 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -221,7 +221,7 @@ jobs: functional-tests: uses: ./.github/workflows/functional-tests.yml - name: Functional testing for build + name: Functional testing for orig build needs: [build] with: version: ${{ needs.build.outputs.version_orig }} @@ -229,7 +229,7 @@ jobs: functional-tests-historic: uses: ./.github/workflows/functional-tests.yml - name: Functional testing for build + name: Functional testing for historic build needs: [build] with: version: ${{ needs.build.outputs.version_historic }} From dcc9e4a18f6536836f175653665eafe04c9e6667 Mon Sep 17 00:00:00 2001 From: Dima Litvinov Date: Thu, 25 Jan 2024 15:06:20 +0000 Subject: [PATCH 2/9] SKALED-1431 Added debug_getFutureTransactions again --- libethereum/Client.h | 2 ++ libethereum/TransactionQueue.cpp | 11 +++++++++++ libethereum/TransactionQueue.h | 2 ++ libweb3jsonrpc/Debug.cpp | 7 +++++++ libweb3jsonrpc/Debug.h | 2 ++ libweb3jsonrpc/DebugFace.h | 10 ++++++++++ 6 files changed, 34 insertions(+) diff --git a/libethereum/Client.h b/libethereum/Client.h index 3bf2015c4..d62af01b1 100644 --- a/libethereum/Client.h +++ b/libethereum/Client.h @@ -127,6 +127,8 @@ class Client : public ClientBase, protected Worker { /// Retrieve pending transactions Transactions pending() const override; + Transactions debugGetFutureTransactions() const { return m_tq.debugGetFutureTransactions(); } + /// Queues a block for import. ImportResult queueBlock( bytes const& _block, bool _isSafe = false ); diff --git a/libethereum/TransactionQueue.cpp b/libethereum/TransactionQueue.cpp index bfeee388f..6a075bf84 100644 --- a/libethereum/TransactionQueue.cpp +++ b/libethereum/TransactionQueue.cpp @@ -537,3 +537,14 @@ void TransactionQueue::verifierBody() { MICROPROFILE_LEAVE(); } } + +Transactions TransactionQueue::debugGetFutureTransactions() const { + Transactions res; + ReadGuard l( m_lock ); + for ( auto addressAndMap : m_future ) { + for ( auto nonceAndTransaction : addressAndMap.second ) { + res.push_back( nonceAndTransaction.second.transaction ); + } // for nonce + } // for address + return res; +} diff --git a/libethereum/TransactionQueue.h b/libethereum/TransactionQueue.h index 63ad3ee96..d089c4004 100644 --- a/libethereum/TransactionQueue.h +++ b/libethereum/TransactionQueue.h @@ -123,6 +123,8 @@ class TransactionQueue { template < class... Args > Transactions topTransactionsSync( unsigned _limit, Args... args ); + Transactions debugGetFutureTransactions() const; + /// Get a hash set of transactions in the queue /// @returns A hash set of all transactions in the queue const h256Hash knownTransactions() const; diff --git a/libweb3jsonrpc/Debug.cpp b/libweb3jsonrpc/Debug.cpp index 0ef884262..91cc76bca 100644 --- a/libweb3jsonrpc/Debug.cpp +++ b/libweb3jsonrpc/Debug.cpp @@ -314,3 +314,10 @@ uint64_t Debug::debug_doBlocksDbCompaction() { return boost::chrono::duration_cast< boost::chrono::milliseconds >( t2 - t1 ).count(); } + +Json::Value Debug::debug_getFutureTransactions() { + auto res = toJson( m_eth.debugGetFutureTransactions() ); + for ( auto& t : res ) + t.removeMember( "data" ); + return res; +} diff --git a/libweb3jsonrpc/Debug.h b/libweb3jsonrpc/Debug.h index f5337001d..6d85b9d7d 100644 --- a/libweb3jsonrpc/Debug.h +++ b/libweb3jsonrpc/Debug.h @@ -60,6 +60,8 @@ class Debug : public DebugFace { virtual uint64_t debug_doStateDbCompaction() override; virtual uint64_t debug_doBlocksDbCompaction() override; + virtual Json::Value debug_getFutureTransactions() override; + private: eth::Client const& m_eth; SkaleDebugInterface* m_debugInterface = nullptr; diff --git a/libweb3jsonrpc/DebugFace.h b/libweb3jsonrpc/DebugFace.h index ebcc07fe2..ade7094b0 100644 --- a/libweb3jsonrpc/DebugFace.h +++ b/libweb3jsonrpc/DebugFace.h @@ -98,6 +98,10 @@ class DebugFace : public ServerInterface< DebugFace > { this->bindAndAddMethod( jsonrpc::Procedure( "debug_doBlocksDbCompaction", jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_STRING, NULL ), &dev::rpc::DebugFace::debug_doBlocksDbCompactionI ); + + this->bindAndAddMethod( jsonrpc::Procedure( "debug_getFutureTransactions", + jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_STRING, NULL ), + &dev::rpc::DebugFace::debug_getFutureTransactionsI ); } inline virtual void debug_accountRangeAtI( const Json::Value& request, Json::Value& response ) { response = this->debug_accountRangeAt( request[0u].asString(), request[1u].asInt(), @@ -179,6 +183,10 @@ class DebugFace : public ServerInterface< DebugFace > { response = this->debug_doBlocksDbCompaction(); } + virtual void debug_getFutureTransactionsI( const Json::Value&, Json::Value& response ) { + response = this->debug_getFutureTransactions(); + } + virtual Json::Value debug_accountRangeAt( const std::string& param1, int param2, const std::string& param3, int param4 ) = 0; virtual Json::Value debug_traceTransaction( @@ -206,6 +214,8 @@ class DebugFace : public ServerInterface< DebugFace > { virtual uint64_t debug_doStateDbCompaction() = 0; virtual uint64_t debug_doBlocksDbCompaction() = 0; + + virtual Json::Value debug_getFutureTransactions() = 0; }; } // namespace rpc From 0ffa23854df91b45e2a7143f154b70aece82ef8d Mon Sep 17 00:00:00 2001 From: Dima Litvinov Date: Thu, 25 Jan 2024 15:24:46 +0000 Subject: [PATCH 3/9] SKALED-1431 Tests for debug_getFutureTransactions --- .../libweb3jsonrpc/WebThreeStubClient.cpp | 10 ++++++++++ .../libweb3jsonrpc/WebThreeStubClient.h | 1 + test/unittests/libweb3jsonrpc/jsonrpc.cpp | 17 +++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/test/unittests/libweb3jsonrpc/WebThreeStubClient.cpp b/test/unittests/libweb3jsonrpc/WebThreeStubClient.cpp index acdebbf42..680bdb57e 100644 --- a/test/unittests/libweb3jsonrpc/WebThreeStubClient.cpp +++ b/test/unittests/libweb3jsonrpc/WebThreeStubClient.cpp @@ -1344,3 +1344,13 @@ Json::Value WebThreeStubClient::debug_doBlocksDbCompaction() { throw jsonrpc::JsonRpcException( jsonrpc::Errors::ERROR_CLIENT_INVALID_RESPONSE, result.toStyledString() ); } + +Json::Value WebThreeStubClient::debug_getFutureTransactions() { + Json::Value p; + Json::Value result = this->CallMethod( "debug_getFutureTransactions", p ); + if ( result.isArray() ) + return result; + else + throw jsonrpc::JsonRpcException( + jsonrpc::Errors::ERROR_CLIENT_INVALID_RESPONSE, result.toStyledString() ); +} diff --git a/test/unittests/libweb3jsonrpc/WebThreeStubClient.h b/test/unittests/libweb3jsonrpc/WebThreeStubClient.h index 5f56db895..6385a340d 100644 --- a/test/unittests/libweb3jsonrpc/WebThreeStubClient.h +++ b/test/unittests/libweb3jsonrpc/WebThreeStubClient.h @@ -160,6 +160,7 @@ class WebThreeStubClient : public jsonrpc::Client { const Json::Value& param3 ) noexcept( false ); Json::Value debug_doStateDbCompaction() noexcept( false ); Json::Value debug_doBlocksDbCompaction() noexcept( false ); + Json::Value debug_getFutureTransactions() noexcept( false ); }; #endif // JSONRPC_CPP_STUB_WEBTHREESTUBCLIENT_H_ diff --git a/test/unittests/libweb3jsonrpc/jsonrpc.cpp b/test/unittests/libweb3jsonrpc/jsonrpc.cpp index 714bb525e..041b8d6b0 100644 --- a/test/unittests/libweb3jsonrpc/jsonrpc.cpp +++ b/test/unittests/libweb3jsonrpc/jsonrpc.cpp @@ -2954,23 +2954,40 @@ BOOST_AUTO_TEST_CASE( mtm_import_future_txs ) { BOOST_REQUIRE( h1 ); BOOST_REQUIRE_EQUAL( tq->futureSize(), 1); + Json::Value call = fixture.rpcClient->debug_getFutureTransactions(); + BOOST_REQUIRE_EQUAL( call.size(), 1); + h256 h2 = fixture.client->importTransaction( tx3 ); BOOST_REQUIRE( h2 ); BOOST_REQUIRE_EQUAL( tq->futureSize(), 2); + + call = fixture.rpcClient->debug_getFutureTransactions(); + BOOST_REQUIRE_EQUAL( call.size(), 2); + BOOST_REQUIRE_EQUAL( call[0]["from"], string("0x")+txJson["from"].asString() ); + h256 h3 = fixture.client->importTransaction( tx2 ); BOOST_REQUIRE( h3 ); BOOST_REQUIRE_EQUAL( tq->futureSize(), 3); + call = fixture.rpcClient->debug_getFutureTransactions(); + BOOST_REQUIRE_EQUAL( call.size(), 3); + h256 h4 = fixture.client->importTransaction( tx1 ); BOOST_REQUIRE( h4 ); BOOST_REQUIRE_EQUAL( tq->futureSize(), 1); BOOST_REQUIRE_EQUAL( tq->status().current, 3); + call = fixture.rpcClient->debug_getFutureTransactions(); + BOOST_REQUIRE_EQUAL( call.size(), 1); + h256 h5 = fixture.client->importTransaction( tx4 ); BOOST_REQUIRE( h5 ); BOOST_REQUIRE_EQUAL( tq->futureSize(), 0); BOOST_REQUIRE_EQUAL( tq->status().current, 5); + call = fixture.rpcClient->debug_getFutureTransactions(); + BOOST_REQUIRE_EQUAL( call.size(), 0); + fixture.client->skaleHost()->pauseConsensus( false ); } From ac4f86a2dcd8247b61ccbf02eb218192539f6208 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Thu, 25 Jan 2024 18:49:07 +0000 Subject: [PATCH 4/9] Update custom_build.yml --- .github/workflows/custom_build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index aca1fac06..d7f37eed8 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -15,6 +15,14 @@ on: description: 'Additional cmake options' default: '' required: false + build_type: + description: 'Build type' + type: choice + required: true + options: + - Debug + - RelWithDebInfo + default: Debug jobs: main_job: From 5b673d7ae19fcd4de88f9720a6bacb8e247e3d5d Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Thu, 25 Jan 2024 19:02:06 +0000 Subject: [PATCH 5/9] Update custom_build.yml --- .github/workflows/custom_build.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index d7f37eed8..e1e18bfd0 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -103,18 +103,19 @@ jobs: export CC=gcc-9 export CXX=g++-9 export TARGET=all - export CMAKE_BUILD_TYPE=Debug + export CMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} + ( test $CMAKE_BUILD_TYPE = "Debug" && export DEBUG_FLAG=1 ) || export DEBUG_FLAG=0 cd deps ./clean.sh rm -f ./libwebsockets-from-git.tar.gz - ./build.sh PARALLEL_COUNT=$(nproc) DEBUG=1 + ./build.sh PARALLEL_COUNT=$(nproc) DEBUG=$DEBUG_FLAG cd .. - name: Configure all run: | export CC=gcc-9 export CXX=g++-9 export TARGET=all - export CMAKE_BUILD_TYPE=Debug + export CMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} mkdir -p build cd build cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE ${{ github.event.inputs.cmake_options }} .. @@ -124,7 +125,7 @@ jobs: export CC=gcc-9 export CXX=g++-9 export TARGET=all - export CMAKE_BUILD_TYPE=Debug + export CMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} cd build make skaled -j$(nproc) #echo "Ensure release mode skaled does not have any debug markers" From ff994728d4ecbd6d2d32e670e594c75c921eb637 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Thu, 25 Jan 2024 19:53:50 +0000 Subject: [PATCH 6/9] Update custom_build.yml --- .github/workflows/custom_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index e1e18bfd0..320a1973c 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -16,7 +16,7 @@ on: default: '' required: false build_type: - description: 'Build type' + description: 'Build type of skaled binary' type: choice required: true options: From aa56bfb8b885587de8d0c5b651a140c7cdbb2dde Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Fri, 26 Jan 2024 16:30:52 +0000 Subject: [PATCH 7/9] Update custom_build.yml --- .github/workflows/custom_build.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index 320a1973c..5872ccc7c 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -22,7 +22,7 @@ on: options: - Debug - RelWithDebInfo - default: Debug + default: RelWithDebInfo jobs: main_job: @@ -31,6 +31,7 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: true DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + BUILD_TYPE: ${{ github.event.inputs.build_type }} steps: - name: update apt run: sudo add-apt-repository ppa:ubuntu-toolchain-r/test; sudo apt-get update @@ -103,7 +104,7 @@ jobs: export CC=gcc-9 export CXX=g++-9 export TARGET=all - export CMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} + export CMAKE_BUILD_TYPE=$BUILD_TYPE ( test $CMAKE_BUILD_TYPE = "Debug" && export DEBUG_FLAG=1 ) || export DEBUG_FLAG=0 cd deps ./clean.sh @@ -115,7 +116,7 @@ jobs: export CC=gcc-9 export CXX=g++-9 export TARGET=all - export CMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} + export CMAKE_BUILD_TYPE=$BUILD_TYPE mkdir -p build cd build cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE ${{ github.event.inputs.cmake_options }} .. @@ -125,7 +126,7 @@ jobs: export CC=gcc-9 export CXX=g++-9 export TARGET=all - export CMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} + export CMAKE_BUILD_TYPE=$BUILD_TYPE cd build make skaled -j$(nproc) #echo "Ensure release mode skaled does not have any debug markers" @@ -136,7 +137,8 @@ jobs: run: | cp build/skaled/skaled scripts/skale_build/executable/ export BRANCH=${{ github.event.inputs.branch_name }} - export VERSION=${{ github.event.inputs.image_version }} + ( test $BUILD_TYPE = "Debug" && export SUFFIX=debug ) || export SUFFIX=release + export VERSION=${{ github.event.inputs.image_version }}-$SUFFIX echo "Version $VERSION" export RELEASE=true bash ./scripts/build_and_publish.sh From 17fe1c1d508be8be2fae62000e988598139b5f76 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Fri, 26 Jan 2024 18:04:00 +0000 Subject: [PATCH 8/9] Update custom_build.yml --- .github/workflows/custom_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index 5872ccc7c..9b21ae4ba 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -105,7 +105,7 @@ jobs: export CXX=g++-9 export TARGET=all export CMAKE_BUILD_TYPE=$BUILD_TYPE - ( test $CMAKE_BUILD_TYPE = "Debug" && export DEBUG_FLAG=1 ) || export DEBUG_FLAG=0 + [[ $CMAKE_BUILD_TYPE = "Debug" ]] && export DEBUG_FLAG=1 || export DEBUG_FLAG=0 cd deps ./clean.sh rm -f ./libwebsockets-from-git.tar.gz @@ -137,7 +137,7 @@ jobs: run: | cp build/skaled/skaled scripts/skale_build/executable/ export BRANCH=${{ github.event.inputs.branch_name }} - ( test $BUILD_TYPE = "Debug" && export SUFFIX=debug ) || export SUFFIX=release + [[ $BUILD_TYPE = "Debug" ]] && export SUFFIX=debug || export SUFFIX=release export VERSION=${{ github.event.inputs.image_version }}-$SUFFIX echo "Version $VERSION" export RELEASE=true From 921adbed741688b2d7c9b2e59f2c91195d2ba8b1 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Fri, 26 Jan 2024 18:08:31 +0000 Subject: [PATCH 9/9] Update custom_build.yml --- .github/workflows/custom_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index 9b21ae4ba..bac69d511 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -137,8 +137,8 @@ jobs: run: | cp build/skaled/skaled scripts/skale_build/executable/ export BRANCH=${{ github.event.inputs.branch_name }} - [[ $BUILD_TYPE = "Debug" ]] && export SUFFIX=debug || export SUFFIX=release - export VERSION=${{ github.event.inputs.image_version }}-$SUFFIX + [[ $BUILD_TYPE = "Debug" ]] && export VERSION_SUFFIX=debug || export VERSION_SUFFIX=release + export VERSION=${{ github.event.inputs.image_version }}-$VERSION_SUFFIX echo "Version $VERSION" export RELEASE=true bash ./scripts/build_and_publish.sh