Merge pull request #1551 from skalenetwork/v3.17.0

v3.17.0
skalenetwork · Jun 20, 2023 · eb00844 · eb00844
2 parents 1b3d6b8 + b1b79aa
commit eb00844
Show file tree

Hide file tree

Showing 95 changed files with 1,803 additions and 4,044 deletions.
diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml
@@ -56,24 +56,6 @@ jobs:
           which gcov-tool
           gcov-tool --version
 
-      - name: Get newest lcov
-        run: |
-          echo "Removing previous lcov version..."
-          sudo apt-get remove lcov || true
-          echo "Installing newest lcov version..."
-          rm -rf newer_lcov || true
-          mkdir newer_lcov
-          cd newer_lcov
-          git clone https://github.com/linux-test-project/lcov --recursive --recurse-submodules
-          cd lcov
-          sudo make install
-          cd ..
-          cd ..
-          echo "Checking installed lcov version..."
-          which lcov
-          lcov --version
-
-
       - name: Extract repo name
         run: echo ::set-env name=REPOSITORY_NAME::$(echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}')
         shell: bash
@@ -113,18 +95,18 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=Debug
           cd deps
           ./clean.sh
           rm -f ./libwebsockets-from-git.tar.gz
-          ./build.sh PARALLEL_COUNT=$(nproc)
+          ./build.sh PARALLEL_COUNT=$(nproc) DEBUG=1
           cd ..
       - name: Configure all
         run: |
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=Debug
           mkdir -p build
           cd build
           cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE ${{ github.event.inputs.cmake_options }} ..
@@ -134,11 +116,11 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=Debug
           cd build
           make skaled -j$(nproc)
-          echo "Ensure release mode skaled does not have any debug markers"
-          strip skaled/skaled
+          #echo "Ensure release mode skaled does not have any debug markers"
+          #strip skaled/skaled
           cd ..
 
       - name: Build and publish container
@@ -149,8 +131,8 @@ jobs:
           echo "Version $VERSION"
           export RELEASE=true
           bash ./scripts/build_and_publish.sh
-          
-      - name: Upload skaled binary as artifact 
+
+      - name: Upload skaled binary as artifact
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:

diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml
@@ -0,0 +1,98 @@
+ name: Run functional tests
+ on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      version:
+        required: true
+        type: string
+
+ jobs:
+  functional-tests:
+    name: Functional tests
+    runs-on: ubuntu-20.04
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+      SKALED_RELEASE: ${{ inputs.version }}
+      NO_ULIMIT_CHECK: 1
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        repository: skalenetwork/skale-ci-integration_tests
+        submodules: recursive
+    - name: Set up Node 
+      uses: actions/[email protected]
+      with:
+        node-version: 14
+    - name: Install packages
+      run: |
+        sudo apt-get update
+        sudo apt-get install python3-pip python3-venv jq btrfs-progs nodejs npm
+        sudo npm install -g truffle
+        sudo npm install -g yarn
+        sudo chown -R runner:runner ~/.config   # HACK
+        
+    - uses: docker-practice/[email protected]
+
+    - name: Prepare SGX keys/certs
+      run: |
+        echo "${{ secrets.sgx_key }}" > sgx_certs/sgx.key
+        chmod 600 sgx_certs/sgx.key
+        sudo mkdir /skale_node_data
+        sudo cp -r sgx_certs /skale_node_data
+        sudo chown -R runner:runner /skale_node_data/sgx_certs/*
+        wc /skale_node_data/sgx_certs/sgx.key
+    - name: Update Environment
+      run: |
+        ./update_environment.sh skaled+internals
+        ./update_environment.sh skaled+load_python
+        ./update_environment.sh skaled+load_js
+        ./update_environment.sh skaled+contractsRunningTest
+        ./update_environment.sh skaled+filestorage
+        ./update_environment.sh skaled+api
+    - name: skaled+internals+pytest
+      run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+internals+pytest
+
+    - name: skaled+filestorage+all
+      run: SKALED_PROVIDER=skaled_providers/endpoint_by_container ./run_tests.sh skaled+filestorage+all
+
+    - name: skaled+contractsRunningTest+all
+      run: SKALED_PROVIDER=skaled_providers/endpoint_by_container ./run_tests.sh skaled+contractsRunningTest+all
+
+    - name: skaled+load_python+all
+      run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+load_python+all
+
+    # - name: skaled+load_js+run_angry_cats
+    #   run: SKALED_PROVIDER=skaled_providers/endpoint_by_container ./run_tests.sh skaled+load_js+run_angry_cats
+
+    - name: skaled+internals+test_snapshot_api
+      run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+internals+test_snapshot_api
+
+    - name: skaled+internals+test_node_rotation
+      run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+internals+test_node_rotation
+
+    - name: Fix access rights
+      run: |
+        sudo chown -R runner:runner /tmp/tmp* || true
+        sudo find ./integration_tests/skaled/internals/third_party/skale-node-tests/btrfs -type d -exec btrfs property set {} ro false \; || true
+        sudo chown -R runner:runner . || true
+        for C in $(docker ps -aq); do docker logs $C>$C.log; done || true
+      if: ${{ always() }}
+
+    - uses: actions/upload-artifact@v2
+      if: ${{ always() }}
+      continue-on-error: true
+      with:
+        name: debug
+        path: |
+          skaled_providers
+          !skaled_providers/**/skaled
+          /tmp/tmp*
+          *.log
+          ./integration_tests/skaled/internals/third_party/skale-node-tests/btrfs
+          !**/.env
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -102,7 +102,6 @@ jobs:
           export CXX=g++-9
           export TARGET=all
           export CMAKE_BUILD_TYPE=Release
-          export CODE_COVERAGE=ON
           mkdir -p build
           cd build
           # -DCMAKE_C_FLAGS=-O3 -DCMAKE_CXX_FLAGS=-O3
@@ -114,10 +113,9 @@ jobs:
           export CXX=g++-9
           export TARGET=all
           export CMAKE_BUILD_TYPE=Release
-          export CODE_COVERAGE=ON
           cd build
           make skaled -j$(nproc)
-          echo "Ensure release mode skaled does not have any debug markers"
+          #echo "Ensure release mode skaled does not have any debug markers"
           strip skaled/skaled
           cd ..
       - name: Configure historic state build
@@ -126,7 +124,6 @@ jobs:
           export CXX=g++-9
           export TARGET=all
           export CMAKE_BUILD_TYPE=Release
-          export CODE_COVERAGE=ON
           mkdir -p build-historic
           cd build-historic
           # -DCMAKE_C_FLAGS=-O3 -DCMAKE_CXX_FLAGS=-O3
@@ -138,10 +135,9 @@ jobs:
           export CXX=g++-9
           export TARGET=all
           export CMAKE_BUILD_TYPE=Release
-          export CODE_COVERAGE=ON
           cd build-historic
           make skaled -j$(nproc)
-          echo "Ensure release mode skaled does not have any debug markers"
+          #echo "Ensure release mode skaled does not have any debug markers"
           strip skaled/skaled
           cd ..
       - name: Build and publish container
@@ -197,3 +193,13 @@ jobs:
           asset_path: ./build-historic/skaled/skaled
           asset_name: skaled-historic
           asset_content_type: application/octet-stream
+    outputs:
+      version: ${{ env.VERSION }}
+
+  functional-tests:
+    uses: ./.github/workflows/functional-tests.yml
+    name: Functional testing for build
+    needs: [build]
+    with:
+      version: ${{ needs.build.outputs.version }}
+    secrets: inherit
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -8,7 +8,7 @@ on:
 defaults:
   run:
     shell: bash
-jobs:
+jobs: 
   build:
     runs-on: self-hosted
     env:
@@ -162,7 +162,7 @@ jobs:
           cd build/test
           export NO_NTP_CHECK=1
           export NO_ULIMIT_CHECK=1          
-          # we specifically run each test for easier log review
+          # we specifically run each test for easier log review          
           ./testeth -t BlockchainTests -- --express && touch /tmp/BlockchainTestsPassed
           ./testeth -t TransitionTests  -- --express && touch /tmp/TransitionTestsPassed
           ./testeth -t TransactionTests -- --express && touch /tmp/TransactionTestsPassed
@@ -259,6 +259,22 @@ jobs:
           ls /tmp/HashSnapshotTestSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t HashSnapshotTestSuite -- --all --verbosity 4
           ls /tmp/ClientSnapshotsSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t ClientSnapshotsSuite -- --all --verbosity 4
           cd ..
+          
+      - name: Create lcov report
+        run: |
+          lcov --capture --directory . --output-file coverage.info
+          lcov --remove coverage.info '/usr/*' --output-file coverage.info # filter system-files
+          lcov --remove coverage.info 'deps/*' --output-file coverage.info # filter dependency files
+          lcov --remove coverage.info 'libconsensus/deps/*' --output-file coverage.info # filter dependency files
+          lcov --remove coverage.info 'libconsensus/libBLS/deps/*' --output-file coverage.info # filter dependency files
+          lcov --remove coverage.info '.hunter/*' --output-file coverage.info # filter dependency files
+           
+      - name: Upload to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.info
+
       - name: Configure all as historic
         run: |
           export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -96,8 +96,8 @@ endif()
 option( HUNTER_RUN_UPLOAD "Upload binaries to the cache server" ${run_upload} )
 
 include( HunterGate )
-HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.76.tar.gz" SHA1 "c7b60993e841850e2c449afd454f5d5aa4ec04e4" LOCAL )
-#HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.214.tar.gz" SHA1 "e14bc153a7f16d6a5eeec845fb0283c8fad8c358" LOCAL ) new leveldb
+#HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.76.tar.gz" SHA1 "c7b60993e841850e2c449afd454f5d5aa4ec04e4" LOCAL )
+HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.214.tar.gz" SHA1 "e14bc153a7f16d6a5eeec845fb0283c8fad8c358" LOCAL ) #leveldb 1.22
 
 set( CMAKE_CXX_STANDARD 17 )
 

diff --git a/CODEOWNERS b/CODEOWNERS
@@ -1,2 +1,2 @@
-* @skalenetwork/codeowners
+* @DmytroNazarenko @kladkogex
 *.md @skalenetwork/docowners
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-3.16.1
+3.17.0
diff --git a/codecov.yml b/codecov.yml
@@ -2,11 +2,6 @@ codecov:
   branch: develop
 coverage:
   range: 50...100
-  #notify:
-    #gitter:
-      #default:
-        #threshold: 0
-        #url: https://gitter.im/ethereum/cpp-ethereum-development
   status:
     patch:
       default:
@@ -15,13 +10,6 @@ coverage:
       default:
         target: auto
         threshold: 0.05
-      app:
-        target: auto
-        threshold: 0.05
-        paths: "!test/"
-      tests:
-        target: auto
-        paths: "test/"
     changes:
       default:
         enabled: no

diff --git a/deps/build.sh b/deps/build.sh
@@ -1878,6 +1878,7 @@ then
 			fi
 			echo -e "${COLOR_INFO}configuring it${COLOR_DOTS}...${COLOR_RESET}"
 			cd fmt
+                        git checkout 9158bea1e148c190aa3f9f084b82887ecb29d2f8
 			eval mkdir -p build
 			cd build
 			eval "$CMAKE" "${CMAKE_CROSSCOMPILING_OPTS}" -DCMAKE_INSTALL_PREFIX="$INSTALL_ROOT" -DCMAKE_BUILD_TYPE="$TOP_CMAKE_BUILD_TYPE" ..
@@ -2087,9 +2088,9 @@ then
 			echo -e "${COLOR_INFO}configuring it${COLOR_DOTS}...${COLOR_RESET}"
 			cd folly
 			eval mkdir -p build2
-			cd build2
+                        cd build2
 			eval "$CMAKE" "${CMAKE_CROSSCOMPILING_OPTS}" -DCMAKE_INSTALL_PREFIX="$INSTALL_ROOT" -DCMAKE_BUILD_TYPE="$TOP_CMAKE_BUILD_TYPE" \
-                -DBOOST_ROOT="$INSTALL_ROOT" -DBOOST_LIBRARYDIR="$INSTALL_ROOT/lib" -DBoost_NO_WARN_NEW_VERSIONS=1 -DBoost_DEBUG=ON \
+                                -DBOOST_ROOT="$INSTALL_ROOT" -DBOOST_LIBRARYDIR="$INSTALL_ROOT/lib" -DBoost_NO_WARN_NEW_VERSIONS=1 -DBoost_DEBUG=ON \
 				-DBUILD_SHARED_LIBS=OFF \
 				-DBUILD_TESTS=OFF -DBUILD_BROKEN_TESTS=OFF -DBUILD_HANGING_TESTS=OFF -DBUILD_SLOW_TESTS=OFF \
 				..

diff --git a/libbatched-io/batched_db.cpp b/libbatched-io/batched_db.cpp
@@ -59,4 +59,15 @@ void db_splitter::prefixed_db::forEach(
     } );
 }
 
+void db_splitter::prefixed_db::forEachWithPrefix(
+    std::string& _prefix, std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const {
+    backend->forEachWithPrefix( _prefix, [&]( dev::db::Slice _key, dev::db::Slice _val ) -> bool {
+        if ( _key[0] != this->prefix )
+            return true;
+        dev::db::Slice key_short = dev::db::Slice( _key.data() + 1, _key.size() - 1 );
+        return f( key_short, _val );
+    } );
+}
+
+
 }  // namespace batched_io
diff --git a/libbatched-io/batched_db.h b/libbatched-io/batched_db.h
@@ -18,7 +18,8 @@ class db_operations_face {
     virtual std::string lookup( dev::db::Slice _key ) const = 0;
     virtual bool exists( dev::db::Slice _key ) const = 0;
     virtual void forEach( std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const = 0;
-
+    virtual void forEachWithPrefix(
+        std::string& _prefix, std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const = 0;
     virtual ~db_operations_face() = default;
 };
 
@@ -69,6 +70,12 @@ class batched_db : public db_face {
         m_db->forEach( f );
     }
 
+    virtual void forEachWithPrefix(
+        std::string& _prefix, std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const {
+        std::lock_guard< std::mutex > foreach_lock( m_batch_mutex );
+        m_db->forEachWithPrefix( _prefix, f );
+    }
+
     virtual ~batched_db();
 
 protected:
@@ -105,6 +112,8 @@ class db_splitter {
         virtual std::string lookup( dev::db::Slice _key ) const;
         virtual bool exists( dev::db::Slice _key ) const;
         virtual void forEach( std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const;
+        virtual void forEachWithPrefix(
+            std::string& _prefix, std::function< bool( dev::db::Slice, dev::db::Slice ) > f ) const;
 
     protected:
         virtual void recover() { /* nothing */

diff --git a/libconsensus b/libconsensus