Merge pull request #735 from skalenetwork/merge-beta-sync-node-to-stable

Merge beta sync node to stable

.dockerignore

@@ -0,0 +1,6 @@
+tests
+helper-scripts
+dist
+build
+.github
+.gitmodules

.github/workflows/publish.yml

@@ -14,26 +14,31 @@ jobs:
   create_release:
     if: github.event.pull_request.merged
     name: Create release
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     outputs:
       upload_url: ${{ steps.create_release.outputs.upload_url }}
       version: ${{ steps.export_outputs.outputs.version }}
       branch: ${{ steps.export_outputs.outputs.branch }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+        with:
+          submodules: true
+
       - name: Checkout submodules
         run: git submodule update --init
       - name: Install ubuntu dependencies
         run: |
           sudo apt-get update
           sudo apt-get install python-setuptools
+
       - name: Set Versions
         run: |
           bash ./scripts/set_versions_ga.sh
+
       - name: Set release
         run: |
-          if [[ "$BRANCH" == "stable" ]]; then
+          if [[ "$BRANCH" == "stable" || "$BRANCH" == "sync-node" ]]; then
             export PRERELEASE=false
           else
             export PRERELEASE=true
@@ -55,6 +60,7 @@ jobs:
         run: |
           echo "::set-output name=version::$VERSION"
           echo "::set-output name=branch::$BRANCH"
+
   build_and_publish_normal:
     if: github.event.pull_request.merged
     needs: create_release
@@ -63,40 +69,57 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-18.04
+          - os: ubuntu-20.04
             asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.8
+    - name: Set up Python 3.11
       uses: actions/setup-python@v1
       with:
-        python-version: 3.8
+        python-version: 3.11
+
     - name: Install ubuntu dependencies
-      if: matrix.os == 'ubuntu-18.04'
+      if: matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get update
-    - name: Install python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e .
-        pip install -e .[dev]
-        pip install wheel
-        pip install --upgrade 'setuptools<45.0.0'
+
     - name: Checkout submodules
       run: git submodule update --init
-    - name: Build normal CLI
+
+    - name: Build normal binary
       run: |
-        bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal
-    - name: Upload normal CLI
+        mkdir ./dist
+        docker build . -t node-cli-builder
+        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal
+        ls -altr /home/ubuntu/dist/
+        docker rm -f $(docker ps -aq)
+
+    - name: Save sha512sum
+      run: |
+        sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum
+
+    - name: Upload release binary
       id: upload-release-asset
       uses: actions/upload-release-asset@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/${{ matrix.asset_name }}
+        asset_path: /home/ubuntu/dist/${{ matrix.asset_name }}
         asset_name: ${{ matrix.asset_name }}
         asset_content_type: application/octet-stream
+
+    - name: Upload release checksum
+      id: upload-release-checksum
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.create_release.outputs.upload_url }}
+        asset_path: /home/ubuntu/dist/sha512sum
+        asset_name: ${{ matrix.asset_name }}.sha512
+        asset_content_type: text/plain
+
   build_and_publish_sync:
     if: github.event.pull_request.merged
     needs: create_release
@@ -105,37 +128,53 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-18.04
-            asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64
+          - os: ubuntu-20.04
+            asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64-sync
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.8
+    - name: Set up Python 3.11
       uses: actions/setup-python@v1
       with:
-        python-version: 3.8
+        python-version: 3.11
+
     - name: Install ubuntu dependencies
-      if: matrix.os == 'ubuntu-18.04'
+      if: matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get update
-    - name: Install python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e .
-        pip install -e .[dev]
-        pip install wheel
-        pip install --upgrade 'setuptools<45.0.0'
+
     - name: Checkout submodules
       run: git submodule update --init
-    - name: Build sync CLI
+
+    - name: Build sync release binary
       run: |
-        bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync
-    - name: Upload sync CLI
-      id: upload-release-asset
+        mkdir ./dist
+        docker build . -t node-cli-builder
+        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync
+        ls -altr /home/ubuntu/dist/
+        docker rm -f $(docker ps -aq)
+
+    - name: Save sha512sum
+      run: |
+        sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum
+
+    - name: Upload release sync CLI
+      id: upload-sync-release-asset
       uses: actions/upload-release-asset@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/${{ matrix.asset_name }}-sync
-        asset_name: ${{ matrix.asset_name }}-sync
+        asset_path: /home/ubuntu/dist/${{ matrix.asset_name }}
+        asset_name: ${{ matrix.asset_name }}
         asset_content_type: application/octet-stream
+
+    - name: Upload release sync CLI checksum
+      id: upload-sync-release-checksum
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.create_release.outputs.upload_url }}
+        asset_path: /home/ubuntu/dist/sha512sum
+        asset_name: ${{ matrix.asset_name }}.sha512
+        asset_content_type: text/plain

.github/workflows/test.yml

@@ -3,35 +3,55 @@ on: [push, pull_request]
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.11]
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Checkout submodules
+      run: git submodule update --init
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install ubuntu dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install python-setuptools
+        sudo apt-get install python-setuptools iptables
+
     - name: Install python dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e .
         pip install -e .[dev]
         pip install --upgrade 'setuptools<45.0.0'
+
     - name: Lint with flake8
       run: |
         flake8 .
-    - name: Build binary
+
+    - name: Build sync binary in Ubuntu 18.04 environment
       run: |
-        bash scripts/build.sh 1.0.0 test-branch normal
-    - name: Build binary sync
+        mkdir ./dist
+        docker build . -t node-cli-builder
+        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test sync
+        docker rm -f $(docker ps -aq)
+
+    - name: Check build
+      run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync
+
+    - name: Build sync binary in Ubuntu 20.04 environment
       run: |
-        bash scripts/build.sh 1.0.0 test-branch sync
+        scripts/build.sh test test sync
+
+    - name: Check build
+      run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync
+
     - name: Run tests
-      run: |
-        bash ./scripts/run_tests.sh
+      run: bash ./scripts/run_tests.sh

.gitmodules

@@ -1,4 +1,9 @@
 [submodule "helper-scripts"]
 	path = helper-scripts
 	url = https://github.com/skalenetwork/helper-scripts.git
+    branch = develop
+
+[submodule "lvmpy"]
+	path = lvmpy
+	url = https://github.com/skalenetwork/docker-lvmpy
 	branch = develop

Dockerfile

@@ -0,0 +1,29 @@
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get install -y  \
+                       git \
+                       python3.8 \
+                       libpython3.8-dev \
+                       python3.8-venv \
+                       python3.8-distutils \
+                       python3.8-dev \
+                       build-essential \
+                       zlib1g-dev \
+                       libffi-dev \
+                       libssl-dev \
+                       swig \
+                       iptables
+
+RUN mkdir /app
+WORKDIR /app
+
+COPY . .
+
+ENV PATH=/app/buildvenv/bin:$PATH
+RUN python3.8 -m venv /app/buildvenv && \
+    pip install --upgrade pip && \
+    pip install wheel setuptools==63.2.0 && \
+    pip install -e '.[dev]' 

main.spec

@@ -1,9 +1,5 @@
 # -*- mode: python -*-
 
-# import distutils
-# if distutils.distutils_path.endswith('__init__.py'):
-#    distutils.distutils_path = os.path.dirname(distutils.distutils_path)
-
 import importlib.util
 
 libxtwrapper_path = importlib.util.find_spec('libxtwrapper').origin

node_cli/cli/__init__.py

@@ -1,4 +1,5 @@
-__version__ = '2.2.1'
+__version__ = '2.3.0'
+
 
 if __name__ == "__main__":
     print(__version__)

node_cli/cli/lvmpy.py

@@ -0,0 +1,62 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of node-cli
+#
+#   Copyright (C) 2020 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import click
+
+from node_cli.utils.helper import abort_if_false
+from node_cli.utils.texts import Texts
+from lvmpy.src.app import run as run_lvmpy
+from lvmpy.src.health import heal_service
+
+G_TEXTS = Texts()
+TEXTS = G_TEXTS['lvmpy']
+
+
+@click.group()
+def lvmpy_cli():
+    pass
+
+
+@lvmpy_cli.group('lvmpy', help=TEXTS['help'])
+def health():
+    pass
+
+
+@health.command(help=TEXTS['run']['help'])
+@click.option(
+    '--yes',
+    is_flag=True,
+    callback=abort_if_false,
+    expose_value=False,
+    prompt=TEXTS['run']['prompt']
+)
+def run():
+    run_lvmpy()
+
+
+@health.command(help=TEXTS['heal']['help'])
+@click.option(
+    '--yes',
+    is_flag=True,
+    callback=abort_if_false,
+    expose_value=False,
+    prompt=TEXTS['heal']['prompt']
+)
+def heal():
+    heal_service()

node_cli/cli/node.py

@@ -131,9 +131,15 @@ def backup_node(backup_folder_path):
 @node.command('restore', help="Restore SKALE node on another machine")
 @click.argument('backup_path')
 @click.argument('env_file')
+@click.option(
+    '--no-snapshot',
+    help='Do not restore sChains from snapshot',
+    is_flag=True,
+    hidden=True
+)
 @streamed_cmd
-def restore_node(backup_path, env_file):
-    restore(backup_path, env_file)
+def restore_node(backup_path, env_file, no_snapshot):
+    restore(backup_path, env_file, no_snapshot)
 
 
 @node.command('maintenance-on', help="Set SKALE node into maintenance mode")