Merge pull request #1942 from skalenetwork/beta

Beta to stable

.github/workflows/custom_build.yml

@@ -25,126 +25,26 @@ on:
         default: RelWithDebInfo
 
 jobs:
-  main_job:
-    runs-on: self-hosted
-    env:
-        ACTIONS_ALLOW_UNSECURE_COMMANDS: true
-        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-        BUILD_TYPE: ${{ github.event.inputs.build_type }}
-    steps:
-      - name: update apt
-        run: |
-          sudo add-apt-repository ppa:ubuntu-toolchain-r/test || true 
-          sudo apt-get update || true
-      - name: install packages
-        run: |
-          sudo apt-get -y remove libzmq* || true
-          sudo apt-get -y install software-properties-common gcc-9 g++-9 || true
+  core_build:
+    uses: ./.github/workflows/setup-build-publish.yml
+    with:
+      image_version: ${{ github.event.inputs.image_version }}
+      branch_name: ${{ github.event.inputs.branch_name }}
+      cmake_options: ${{ github.event.inputs.cmake_options }}
+      build_type: ${{ github.event.inputs.build_type }}
+      node_type: core
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Use g++-9 and gcov-9 by default
-        run: |
-          echo "Updating all needed alternatives"
-          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 9
-          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 9
-          sudo update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-9 9
-          sudo update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-9 9
-          sudo update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-9 9
-          echo "Checking alternative for gcc"
-          which gcc
-          gcc --version
-          echo "Checking alternative for g++"
-          which g++
-          g++ --version
-          echo "Checking alternative for gcov"
-          which gcov
-          gcov --version
-          echo "Checking alternative for gcov-dump"
-          which gcov-dump
-          gcov-dump --version
-          echo "Checking alternative for gcov-tool"
-          which gcov-tool
-          gcov-tool --version
-
-      - name: Extract repo name
-        run: echo ::set-env name=REPOSITORY_NAME::$(echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}')
-        shell: bash
-        env:
-          ACTIONS_ALLOW_UNSECURE_COMMANDS: true
-      - name: Extract branch name
-        shell: bash
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: extract_branch
-      - name: checkout
-        uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.inputs.branch_name }}
-      - name: Submodule update
-        run: |
-          rm -rf ./libconsensus || true
-          ls -1
-          git submodule update --init --recursive 
-
-      - name: Prepare ccache timestamp
-        id: ccache_cache_timestamp
-        shell: cmake -P {0}
-        run: |
-          string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-          message("::set-output name=timestamp::${current_date}")
-      - name: Ccache cache files
-        uses: actions/[email protected]
-        with:
-          path: .ccache
-          key: ${ { matrix.config.name } }-ccache-${ { steps.ccache_cache_timestamp.outputs.timestamp } }
-          restore-keys: |
-            ${ { matrix.config.name } }-ccache-
-      - name: Build dependencies
-        run: |
-          export CC=gcc-9
-          export CXX=g++-9
-          export TARGET=all
-          export CMAKE_BUILD_TYPE=$BUILD_TYPE
-          [[ $CMAKE_BUILD_TYPE = "Debug" ]] && export DEBUG_FLAG=1 || export DEBUG_FLAG=0
-          cd deps
-          ./clean.sh
-          rm -f ./libwebsockets-from-git.tar.gz
-          ./build.sh PARALLEL_COUNT=$(nproc) DEBUG=$DEBUG_FLAG
-          cd ..
-      - name: Configure all
-        run: |
-          export CC=gcc-9
-          export CXX=g++-9
-          export TARGET=all
-          export CMAKE_BUILD_TYPE=$BUILD_TYPE
-          mkdir -p build
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE ${{ github.event.inputs.cmake_options }} ..
-          cd ..
-      - name: Build all
-        run: |
-          export CC=gcc-9
-          export CXX=g++-9
-          export TARGET=all
-          export CMAKE_BUILD_TYPE=$BUILD_TYPE
-          cd build
-          make skaled -j$(nproc)
-          #echo "Ensure release mode skaled does not have any debug markers"
-          #strip skaled/skaled
-          cd ..
-
-      - name: Build and publish container
-        run: |
-          cp build/skaled/skaled scripts/skale_build/executable/
-          export BRANCH=${{ github.event.inputs.branch_name }}
-          [[ $BUILD_TYPE = "Debug" ]] && export VERSION_SUFFIX=debug || export VERSION_SUFFIX=release
-          export VERSION=${{ github.event.inputs.image_version }}-$VERSION_SUFFIX
-          echo "Version $VERSION"
-          export RELEASE=true
-          bash ./scripts/build_and_publish.sh
-
-      - name: Upload skaled binary as artifact
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: skaled
-          path: ./build/skaled/skaled
+  historic_build:
+    uses: ./.github/workflows/setup-build-publish.yml
+    with:
+      image_version: ${{ github.event.inputs.image_version }}
+      branch_name: ${{ github.event.inputs.branch_name }}
+      cmake_options: ${{ github.event.inputs.cmake_options }}
+      build_type: ${{ github.event.inputs.build_type }}
+      node_type: historic
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/functional-tests.yml

@@ -24,7 +24,7 @@
       with:
         token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
         repository: skalenetwork/skale-ci-integration_tests
-        ref: v3.18.0
+        ref: master
         submodules: recursive
     - name: Set up Node 
       uses: actions/[email protected]

.github/workflows/setup-build-publish.yml

@@ -0,0 +1,168 @@
+name: 'Setup, Build and Publish'
+on:
+  workflow_call:
+    inputs:
+      branch_name:
+        type: string
+        description: 'Branch name'
+        required: true
+      image_version:
+        type: string
+        description: 'Image version for docker hub'
+        required: true
+      cmake_options:
+        type: string
+        description: 'Additional cmake options'
+        required: false
+      build_type:
+        type: string
+        description: 'Build type of skaled binary'
+        required: true
+      node_type:
+        type: string
+        description: 'Node type of skaled build'
+        required: true
+    secrets:
+      DOCKER_USERNAME:
+        required: true
+      DOCKER_PASSWORD:
+        required: true
+
+jobs:
+  main:
+    runs-on: self-hosted
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      BUILD_TYPE: ${{ inputs.build_type }}
+    steps:
+      - name: Prepare workflow variables
+        if: inputs.node_type != 'historic'
+        run: |
+          [[ $BUILD_TYPE = "Debug" ]] && export VERSION_SUFFIX=debug || export VERSION_SUFFIX=release
+          export VERSION=${{ inputs.image_version }}-$VERSION_SUFFIX
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+          export CMAKE_OPTS="${{ inputs.cmake_options }}"
+          echo "CMAKE_OPTS=$CMAKE_OPTS" >> $GITHUB_ENV
+      - name: Prepare workflow variables (historic)
+        if: inputs.node_type == 'historic'
+        run: |
+          [[ $BUILD_TYPE = "Debug" ]] && export VERSION_SUFFIX=debug || export VERSION_SUFFIX=release
+          export VERSION=${{ inputs.image_version }}-$VERSION_SUFFIX-historic
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+          export CMAKE_OPTS="-DHISTORIC_STATE=1 ${{ inputs.cmake_options }}"
+          echo "CMAKE_OPTS=$CMAKE_OPTS" >> $GITHUB_ENV
+      - name: update apt
+        run: |
+          sudo add-apt-repository ppa:ubuntu-toolchain-r/test || true 
+          sudo apt-get update || true
+      - name: install packages
+        run: |
+          sudo apt-get -y remove libzmq* || true
+          sudo apt-get -y install software-properties-common gcc-9 g++-9 || true
+  
+      - name: Use g++-9 and gcov-9 by default
+        run: |
+          echo "Updating all needed alternatives"
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 9
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 9
+          sudo update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-9 9
+          sudo update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-9 9
+          sudo update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-9 9
+          echo "Checking alternative for gcc"
+          which gcc
+          gcc --version
+          echo "Checking alternative for g++"
+          which g++
+          g++ --version
+          echo "Checking alternative for gcov"
+          which gcov
+          gcov --version
+          echo "Checking alternative for gcov-dump"
+          which gcov-dump
+          gcov-dump --version
+          echo "Checking alternative for gcov-tool"
+          which gcov-tool
+          gcov-tool --version
+  
+      - name: Extract repo name
+        run: echo ::set-env name=REPOSITORY_NAME::$(echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}')
+        shell: bash
+        env:
+          ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+      - name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ inputs.branch_name }}
+      - name: Submodule update
+        run: |
+          rm -rf ./libconsensus || true
+          ls -1
+          git submodule update --init --recursive 
+  
+      - name: Prepare ccache timestamp
+        id: ccache_cache_timestamp
+        shell: cmake -P {0}
+        run: |
+          string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
+          message("::set-output name=timestamp::${current_date}")
+      - name: Ccache cache files
+        uses: actions/[email protected]
+        with:
+          path: .ccache
+          key: ${ { matrix.config.name } }-ccache-${ { steps.ccache_cache_timestamp.outputs.timestamp } }
+          restore-keys: |
+            ${ { matrix.config.name } }-ccache-
+      - name: Build dependencies
+        run: |
+          export CC=gcc-9
+          export CXX=g++-9
+          export TARGET=all
+          export CMAKE_BUILD_TYPE=$BUILD_TYPE
+          [[ $CMAKE_BUILD_TYPE = "Debug" ]] && export DEBUG_FLAG=1 || export DEBUG_FLAG=0
+          cd deps
+          ./clean.sh
+          rm -f ./libwebsockets-from-git.tar.gz
+          ./build.sh PARALLEL_COUNT=$(nproc) DEBUG=$DEBUG_FLAG
+          cd ..
+      - name: Configure all
+        env:
+          CMAKE_OPTS: ${{ env.CMAKE_OPTS }}
+        run: |
+          export CC=gcc-9
+          export CXX=g++-9
+          export TARGET=all
+          export CMAKE_BUILD_TYPE=$BUILD_TYPE
+          mkdir -p build
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE $CMAKE_OPTS ..
+          cd ..
+      - name: Build all
+        run: |
+          export CC=gcc-9
+          export CXX=g++-9
+          export TARGET=all
+          export CMAKE_BUILD_TYPE=$BUILD_TYPE
+          cd build
+          make skaled -j$(nproc)
+          cd ..
+      - name: Build and publish container
+        env:
+          VERSION: ${{ env.VERSION }}
+        run: |
+          cp build/skaled/skaled scripts/skale_build/executable/
+          export BRANCH=${{ inputs.branch_name }}
+          export RELEASE=true
+          bash ./scripts/build_and_publish.sh
+  
+      - name: Upload skaled binary as artifact
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: skaled-${{ inputs.node_type }}
+          path: ./build/skaled/skaled

VERSION

@@ -1 +1 @@
-3.18.2
+3.19.0

libdevcore/JsonUtils.cpp

@@ -57,10 +57,12 @@ std::string dev::jsonTypeAsString( json_spirit::Value_type _type ) {
 }
 
 void dev::requireJsonFields( json_spirit::mObject const& _o, std::string const& _config,
-    std::map< std::string, JsonFieldOptions > const& _validationMap ) {
+    std::map< std::string, JsonFieldOptions > const& _validationMap,
+    std::function< bool( const std::string& ) > _callbackForUnexpected ) {
     // check for unexpected fiedls
     for ( auto const& field : _o ) {
-        if ( !_validationMap.count( field.first ) ) {
+        // _callbackForUnexpected allows to accept ertain unexpected fields
+        if ( !_validationMap.count( field.first ) && !_callbackForUnexpected( field.first ) ) {
             std::string const comment =
                 "Unexpected field '" + field.first + "' in config: " + _config;
             cerror << comment << "\n"

libdevcore/JsonUtils.h

@@ -41,6 +41,10 @@ using JsonFieldOptions = std::pair< JsonTypeSet, JsonFieldPresence >;
   @param _validationMap a map with json objects that would be checked. "objName" -> {js::str_type,
   jsonField::Required}
 */
-void requireJsonFields( json_spirit::mObject const& _o, std::string const& _configName,
-    std::map< std::string, JsonFieldOptions > const& _validationMap );
+void requireJsonFields(
+    json_spirit::mObject const& _o, std::string const& _configName,
+    std::map< std::string, JsonFieldOptions > const& _validationMap,
+    std::function< bool( const std::string& ) > _callbackForUnexpected = []( const std::string& ) {
+        return false;
+    } );
 }  // namespace dev

libdevcore/LevelDB.h

@@ -26,8 +26,8 @@
 #include <leveldb/write_batch.h>
 #include <boost/filesystem.hpp>
 
-#include "shared_mutex"
 #include <secp256k1_sha256.h>
+#include <shared_mutex>
 
 namespace dev::db {
 class LevelDB : public DatabaseFace {

libethashseal/Ethash.cpp

@@ -130,18 +130,6 @@ void Ethash::verify( Strictness _s, BlockHeader const& _bi, BlockHeader const& _
     }
 }
 
-void Ethash::verifyTransaction( ImportRequirements::value _ir, TransactionBase const& _t,
-    BlockHeader const& _header, u256 const& _startGasUsed ) const {
-    SealEngineFace::verifyTransaction( _ir, _t, _header, _startGasUsed );
-
-    if ( _ir & ImportRequirements::TransactionSignatures ) {
-        if ( _header.number() >= chainParams().EIP158ForkBlock ) {
-            uint64_t chainID = chainParams().chainID;
-            _t.checkChainId( chainID, chainParams().skaleDisableChainIdCheck );
-        }  // if
-    }
-}
-
 u256 Ethash::childGasLimit( BlockHeader const& _bi, u256 const& _gasFloorTarget ) const {
     u256 gasFloorTarget = _gasFloorTarget == Invalid256 ? 3141562 : _gasFloorTarget;
     u256 gasLimit = _bi.gasLimit();

libethashseal/Ethash.h

@@ -52,8 +52,6 @@ class Ethash : public SealEngineBase {
     StringHashMap jsInfo( BlockHeader const& _bi ) const override;
     void verify( Strictness _s, BlockHeader const& _bi, BlockHeader const& _parent,
         bytesConstRef _block ) const override;
-    void verifyTransaction( ImportRequirements::value _ir, TransactionBase const& _t,
-        BlockHeader const& _header, u256 const& _startGasUsed ) const override;
     void populateFromParent( BlockHeader& _bi, BlockHeader const& _parent ) const override;
 
     strings sealers() const override;