diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..060cae42 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +tests +helper-scripts +dist +build +.github +.gitmodules diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f7399e36..8d761ae0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: create_release: if: github.event.pull_request.merged name: Create release - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 outputs: upload_url: ${{ steps.create_release.outputs.upload_url }} version: ${{ steps.export_outputs.outputs.version }} @@ -22,18 +22,23 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + with: + submodules: true + - name: Checkout submodules run: git submodule update --init - name: Install ubuntu dependencies run: | sudo apt-get update sudo apt-get install python-setuptools + - name: Set Versions run: | bash ./scripts/set_versions_ga.sh + - name: Set release run: | - if [[ "$BRANCH" == "stable" ]]; then + if [[ "$BRANCH" == "stable" || "$BRANCH" == "sync-node" ]]; then export PRERELEASE=false else export PRERELEASE=true @@ -55,6 +60,7 @@ jobs: run: | echo "::set-output name=version::$VERSION" echo "::set-output name=branch::$BRANCH" + build_and_publish_normal: if: github.event.pull_request.merged needs: create_release @@ -63,40 +69,57 @@ jobs: strategy: matrix: include: - - os: ubuntu-18.04 + - os: ubuntu-20.04 asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.11 + - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-18.04' + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -e .[dev] - pip install wheel - pip install --upgrade 'setuptools<45.0.0' + - name: Checkout submodules run: git submodule update --init - - name: Build normal CLI + + - name: Build normal binary run: | - bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal - - name: Upload normal CLI + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal + ls -altr /home/ubuntu/dist/ + docker rm -f $(docker ps -aq) + + - name: Save sha512sum + run: | + sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum + + - name: Upload release binary id: upload-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }} + asset_path: /home/ubuntu/dist/${{ matrix.asset_name }} asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream + + - name: Upload release checksum + id: upload-release-checksum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: /home/ubuntu/dist/sha512sum + asset_name: ${{ matrix.asset_name }}.sha512 + asset_content_type: text/plain + build_and_publish_sync: if: github.event.pull_request.merged needs: create_release @@ -105,37 +128,53 @@ jobs: strategy: matrix: include: - - os: ubuntu-18.04 - asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 + - os: ubuntu-20.04 + asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64-sync steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.11 + - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-18.04' + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -e .[dev] - pip install wheel - pip install --upgrade 'setuptools<45.0.0' + - name: Checkout submodules run: git submodule update --init - - name: Build sync CLI + + - name: Build sync release binary run: | - bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync - - name: Upload sync CLI - id: upload-release-asset + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync + ls -altr /home/ubuntu/dist/ + docker rm -f $(docker ps -aq) + + - name: Save sha512sum + run: | + sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum + + - name: Upload release sync CLI + id: upload-sync-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }}-sync - asset_name: ${{ matrix.asset_name }}-sync + asset_path: /home/ubuntu/dist/${{ matrix.asset_name }} + asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream + + - name: Upload release sync CLI checksum + id: upload-sync-release-checksum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: /home/ubuntu/dist/sha512sum + asset_name: ${{ matrix.asset_name }}.sha512 + asset_content_type: text/plain diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e362e16..943486f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,35 +3,55 @@ on: [push, pull_request] jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.8] + python-version: [3.11] steps: - uses: actions/checkout@v2 + with: + submodules: true + + - name: Checkout submodules + run: git submodule update --init + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: Install ubuntu dependencies run: | sudo apt-get update - sudo apt-get install python-setuptools + sudo apt-get install python-setuptools iptables + - name: Install python dependencies run: | python -m pip install --upgrade pip pip install -e . pip install -e .[dev] pip install --upgrade 'setuptools<45.0.0' + - name: Lint with flake8 run: | flake8 . - - name: Build binary + + - name: Build sync binary in Ubuntu 18.04 environment run: | - bash scripts/build.sh 1.0.0 test-branch normal - - name: Build binary sync + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test sync + docker rm -f $(docker ps -aq) + + - name: Check build + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync + + - name: Build sync binary in Ubuntu 20.04 environment run: | - bash scripts/build.sh 1.0.0 test-branch sync + scripts/build.sh test test sync + + - name: Check build + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync + - name: Run tests - run: | - bash ./scripts/run_tests.sh + run: bash ./scripts/run_tests.sh diff --git a/.gitmodules b/.gitmodules index d54a2086..b9ae3738 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,9 @@ [submodule "helper-scripts"] path = helper-scripts url = https://github.com/skalenetwork/helper-scripts.git + branch = develop + +[submodule "lvmpy"] + path = lvmpy + url = https://github.com/skalenetwork/docker-lvmpy branch = develop diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..943f66e6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get install -y \ + git \ + python3.8 \ + libpython3.8-dev \ + python3.8-venv \ + python3.8-distutils \ + python3.8-dev \ + build-essential \ + zlib1g-dev \ + libffi-dev \ + libssl-dev \ + swig \ + iptables + +RUN mkdir /app +WORKDIR /app + +COPY . . + +ENV PATH=/app/buildvenv/bin:$PATH +RUN python3.8 -m venv /app/buildvenv && \ + pip install --upgrade pip && \ + pip install wheel setuptools==63.2.0 && \ + pip install -e '.[dev]' diff --git a/lvmpy b/lvmpy new file mode 160000 index 00000000..8ee051bf --- /dev/null +++ b/lvmpy @@ -0,0 +1 @@ +Subproject commit 8ee051bf24aa3feecc0ef97fb5eec970eb068512 diff --git a/main.spec b/main.spec index 3c776386..a4dbe395 100644 --- a/main.spec +++ b/main.spec @@ -1,9 +1,5 @@ # -*- mode: python -*- -# import distutils -# if distutils.distutils_path.endswith('__init__.py'): -# distutils.distutils_path = os.path.dirname(distutils.distutils_path) - import importlib.util libxtwrapper_path = importlib.util.find_spec('libxtwrapper').origin diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index 5b1ba474..02a4b07a 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,5 @@ -__version__ = '2.2.1' +__version__ = '2.3.0' + if __name__ == "__main__": print(__version__) diff --git a/node_cli/cli/lvmpy.py b/node_cli/cli/lvmpy.py new file mode 100644 index 00000000..473defa8 --- /dev/null +++ b/node_cli/cli/lvmpy.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2020 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.utils.helper import abort_if_false +from node_cli.utils.texts import Texts +from lvmpy.src.app import run as run_lvmpy +from lvmpy.src.health import heal_service + +G_TEXTS = Texts() +TEXTS = G_TEXTS['lvmpy'] + + +@click.group() +def lvmpy_cli(): + pass + + +@lvmpy_cli.group('lvmpy', help=TEXTS['help']) +def health(): + pass + + +@health.command(help=TEXTS['run']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['run']['prompt'] +) +def run(): + run_lvmpy() + + +@health.command(help=TEXTS['heal']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['heal']['prompt'] +) +def heal(): + heal_service() diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 7ec32dbf..866d07e9 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -131,9 +131,15 @@ def backup_node(backup_folder_path): @node.command('restore', help="Restore SKALE node on another machine") @click.argument('backup_path') @click.argument('env_file') +@click.option( + '--no-snapshot', + help='Do not restore sChains from snapshot', + is_flag=True, + hidden=True +) @streamed_cmd -def restore_node(backup_path, env_file): - restore(backup_path, env_file) +def restore_node(backup_path, env_file, no_snapshot): + restore(backup_path, env_file, no_snapshot) @node.command('maintenance-on', help="Set SKALE node into maintenance mode") diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 81e0743d..50748b6d 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -36,6 +36,7 @@ FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SNAPSHOTS_SHARED_VOLUME = 'shared-space' SCHAINS_MNT_DIR = '/var/lib/skale/schains' +VOLUME_GROUP = 'schains' SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') SKALE_TMP_DIR = os.path.join(SKALE_DIR, '.tmp') @@ -77,6 +78,16 @@ FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml') DOCKER_LVMPY_PATH = os.path.join(SKALE_DIR, 'docker-lvmpy') +DOCKER_LVMPY_BIN_PATH = '/usr/local/bin/skale' + +LVMPY_CMD = f'{DOCKER_LVMPY_BIN_PATH} lvmpy' +LVMPY_RUN_CMD = f'{LVMPY_CMD} run --yes' +LVMPY_HEAL_CMD = f'{LVMPY_CMD} heal --yes' + +LVMPY_CRON_LOG_PATH = '/var/log/docker-lvmpy/cron.log' +LVMPY_CRON_SCHEDULE_MINUTES = 3 + +LVMPY_LOG_DIR = '/var/log/docker-lvmpy' IPTABLES_DIR = '/etc/iptables/' IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 168228e3..73b2b51d 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -199,13 +199,17 @@ def update_sync(env_filepath): @check_not_inited -def restore(backup_path, env_filepath): +def restore(backup_path, env_filepath, no_snapshot=False): env = get_node_env(env_filepath) if env is None: return save_env_params(env_filepath) env['SKALE_DIR'] = SKALE_DIR - env['BACKUP_RUN'] = 'True' # should be str + + if not no_snapshot: + logger.info('Adding BACKUP_RUN to env ...') + env['BACKUP_RUN'] = 'True' # should be str + restored_ok = restore_op(env, backup_path) if not restored_ok: error_exit( diff --git a/node_cli/main.py b/node_cli/main.py index bda53a09..fa58b4f4 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -29,6 +29,7 @@ from node_cli.cli.health import health_cli from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION, TYPE from node_cli.cli.logs import logs_cli +from node_cli.cli.lvmpy import lvmpy_cli from node_cli.cli.node import node_cli from node_cli.cli.schains import schains_cli from node_cli.cli.wallet import wallet_cli @@ -90,7 +91,8 @@ def get_sources_list(): wallet_cli, ssl_cli, exit_cli, - validate_cli + validate_cli, + lvmpy_cli ] @@ -113,6 +115,7 @@ def handle_exception(exc_type, exc_value, exc_traceback): logger.debug(f'cmd: {" ".join(str(x) for x in args)}, v.{__version__}') sources = get_sources_list() cmd_collection = click.CommandCollection(sources=sources) + try: cmd_collection() except Exception as err: diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index d5a35c30..45cdb948 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import distro import functools import logging from typing import Dict @@ -39,11 +40,10 @@ ) from node_cli.operations.volume import ( cleanup_volume_artifacts, - docker_lvmpy_update, - docker_lvmpy_install, ensure_filestorage_mapping, prepare_block_device ) +from node_cli.operations.docker_lvmpy import lvmpy_install # noqa from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables @@ -111,7 +111,7 @@ def update(env_filepath: str, env: Dict) -> None: backup_old_contracts() download_contracts(env) - docker_lvmpy_update(env) + lvmpy_install(env) generate_nginx_config() prepare_host( @@ -134,7 +134,9 @@ def update(env_filepath: str, env: Dict) -> None: update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') compose_up(env) @@ -161,13 +163,15 @@ def init(env_filepath: str, env: dict) -> bool: configure_iptables() generate_nginx_config() - docker_lvmpy_install(env) + lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') @@ -215,7 +219,9 @@ def init_sync( update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) @@ -255,7 +261,9 @@ def update_sync(env_filepath: str, env: Dict) -> bool: update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) compose_up_sync(env) @@ -292,13 +300,15 @@ def restore(env, backup_path): link_env_file() configure_iptables() - docker_lvmpy_install(env) + lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) compose_up(env) diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py new file mode 100644 index 00000000..31036c95 --- /dev/null +++ b/node_cli/operations/docker_lvmpy.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import os +import shutil + +import crontab + +from node_cli.utils.git_utils import sync_repo +from node_cli.configs import ( + DOCKER_LVMPY_PATH, + DOCKER_LVMPY_REPO_URL, + FILESTORAGE_MAPPING, + LVMPY_RUN_CMD, + LVMPY_HEAL_CMD, + LVMPY_CRON_LOG_PATH, + LVMPY_CRON_SCHEDULE_MINUTES, + SCHAINS_MNT_DIR, + VOLUME_GROUP +) +from lvmpy.src.install import setup as setup_lvmpy + +logger = logging.getLogger(__name__) + + +def update_docker_lvmpy_env(env): + env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] + env['VOLUME_GROUP'] = 'schains' + env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING + env['MNT_DIR'] = SCHAINS_MNT_DIR + env['PATH'] = os.environ.get('PATH', None) + return env + + +def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): + if not os.path.isdir(FILESTORAGE_MAPPING): + os.makedirs(FILESTORAGE_MAPPING) + + +def sync_docker_lvmpy_repo(env): + if os.path.isdir(DOCKER_LVMPY_PATH): + shutil.rmtree(DOCKER_LVMPY_PATH) + sync_repo( + DOCKER_LVMPY_REPO_URL, + DOCKER_LVMPY_PATH, + env["DOCKER_LVMPY_STREAM"] + ) + + +def lvmpy_install(env): + ensure_filestorage_mapping() + logging.info('Configuring and starting lvmpy') + setup_lvmpy( + block_device=env['DISK_MOUNTPOINT'], + volume_group=VOLUME_GROUP, + exec_start=LVMPY_RUN_CMD + ) + init_healing_cron() + logger.info('docker-lvmpy is configured and started') + + +def init_healing_cron(): + logger.info('Configuring cron job for healing lvmpy') + cron_line = f'{LVMPY_HEAL_CMD} >> {LVMPY_CRON_LOG_PATH} 2>&1' + legacy_line = f'cd /opt/docker-lvmpy && venv/bin/python -c "import health; health.heal_service()" >> {LVMPY_CRON_LOG_PATH} 2>&1' # noqa + + with crontab.CronTab(user='root') as c: + jobs = [c.command for c in c] + if legacy_line in jobs: + c.remove_all(command=legacy_line) + if cron_line not in jobs: + job = c.new( + command=cron_line + ) + job.minute.every(LVMPY_CRON_SCHEDULE_MINUTES) diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index b49d9f61..7eb50f4c 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -6,20 +6,23 @@ DEFAULT_VERSION = '1.0.0' DEFAULT_CONFIG_STREAM = '1.1.0' DEFAULT_DOCKER_LVMPY_STREAM = '1.0.0' +DEFAULT_OS_ID = 'ubuntu' +DEFAULT_OS_VERSION = '18.04' class CliMeta( namedtuple( 'Node', - ['version', 'config_stream', 'docker_lvmpy_stream'] + ['version', 'config_stream', 'docker_lvmpy_stream', 'os_id', 'os_version'] ) ): __slots__ = () def __new__(cls, version=DEFAULT_VERSION, config_stream=DEFAULT_CONFIG_STREAM, - docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM): + docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION): return super(CliMeta, cls).__new__( - cls, version, config_stream, docker_lvmpy_stream + cls, version, config_stream, docker_lvmpy_stream, os_id, os_version ) @@ -41,7 +44,8 @@ def save_meta(meta: CliMeta) -> None: def compose_default_meta() -> CliMeta: return CliMeta(version=DEFAULT_VERSION, docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, - config_stream=DEFAULT_CONFIG_STREAM) + config_stream=DEFAULT_CONFIG_STREAM, os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION) def ensure_meta(meta: CliMeta = None) -> None: @@ -51,7 +55,7 @@ def ensure_meta(meta: CliMeta = None) -> None: def update_meta(version: str, config_stream: str, - docker_lvmpy_stream: str) -> None: + docker_lvmpy_stream: str, os_id: str, os_version: str) -> None: ensure_meta() - meta = CliMeta(version, config_stream, docker_lvmpy_stream) + meta = CliMeta(version, config_stream, docker_lvmpy_stream, os_id, os_version) save_meta(meta) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 12cce17a..afbb2068 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,4 +3,9 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -HIDE_STREAM_LOG=true TEST_HOME_DIR="$PROJECT_DIR/tests/" GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ +LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \ + HIDE_STREAM_LOG=true \ + TEST_HOME_DIR="$PROJECT_DIR/tests/" \ + GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \ + DOTENV_FILEPATH='tests/test-env' \ + py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ diff --git a/setup.py b/setup.py index c68db0ca..6a05adcd 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ def find_version(*file_paths): install_requires=[ "click==8.1.3", "PyInstaller==5.6.2", + "distro==1.4.0", "docker==6.0.1", "texttable==1.6.4", "python-dateutil==2.8.2", @@ -60,15 +61,19 @@ def find_version(*file_paths): "python-dotenv==0.21.0", "terminaltables==3.1.10", "requests==2.28.1", - "GitPython==3.1.27", + "GitPython==3.1.30", "packaging==21.3", "python-debian==0.1.48", - "python-iptables==1.0.0", + "python-iptables==1.0.1", "PyYAML==6.0", "pyOpenSSL==22.0.0", "MarkupSafe==2.1.1", "cryptography==37.0.2", - "filelock==3.0.12" + "filelock==3.0.12", + 'Flask==2.2.2', + 'itsdangerous==2.0.1', + 'sh==1.14.2', + 'python-crontab==2.6.0' ], python_requires='>=3.8,<4', extras_require=extras_require, @@ -80,6 +85,6 @@ def find_version(*file_paths): 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Affero General Public License v3', 'Natural Language :: English', - 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.11' ], ) diff --git a/tests/cli/main_test.py b/tests/cli/main_test.py index 2703f0b5..5ce570ad 100644 --- a/tests/cli/main_test.py +++ b/tests/cli/main_test.py @@ -18,14 +18,13 @@ # along with this program. If not, see . -from node_cli.cli import info from node_cli.main import version from tests.helper import run_command def test_version(): result = run_command(version, []) - expected = f'SKALE Node CLI version: {info.VERSION}\n' + expected = 'SKALE Node CLI version: test\n' assert result.output == expected result = run_command(version, ['--short']) - assert result.output == f'{info.VERSION}\n' + assert result.output == 'test\n' diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 3b9fe1d0..319137ed 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -20,6 +20,7 @@ import pathlib import mock +from unittest.mock import MagicMock, patch import requests import logging @@ -302,10 +303,11 @@ def test_restore(mocked_g_config): ) backup_path = result.output.replace( 'Backup archive successfully created: ', '').replace('\n', '') - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.restore_op'), \ - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ + patch('subprocess.run', new=subprocess_run_mock), \ + patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( restore_node, [backup_path, './tests/test-env'] @@ -313,6 +315,31 @@ def test_restore(mocked_g_config): assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') == 'True' + + +def test_restore_no_snapshot(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + result = run_command( + backup_node, + ['/tmp'] + ) + backup_path = result.output.replace( + 'Backup archive successfully created: ', '').replace('\n', '') + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ + patch('subprocess.run', new=subprocess_run_mock), \ + patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + patch('node_cli.utils.decorators.is_node_inited', return_value=False): + result = run_command( + restore_node, + [backup_path, './tests/test-env', '--no-snapshot'] + ) + assert result.exit_code == 0 + assert 'Node is restored from backup\n' in result.output # noqa + + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') is None + def test_maintenance_on(): resp_mock = response_mock( diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 24318a81..fb7e4eb2 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -53,8 +53,8 @@ def test_init_sync(mocked_g_config): def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ +# with mock.patch('subprocess.run', new=subprocess_run_mock), \ + with mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ mock.patch('node_cli.operations.base.cleanup_volume_artifacts'), \ mock.patch('node_cli.operations.base.download_skale_node'), \ mock.patch('node_cli.operations.base.sync_skale_node'), \ @@ -74,13 +74,13 @@ def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( _init_sync, - ['./tests/test-env', '--archive', '--catchup'] + ['./tests/test-env', '--archive', '--catchup', '--historic-state'] ) node_options = NodeOptions() assert node_options.archive assert node_options.catchup - assert not node_options.historic_state + assert node_options.historic_state assert result.exit_code == 0 diff --git a/tests/conftest.py b/tests/conftest.py index bdb6d263..5e79dce1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,7 +43,7 @@ from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file -from tests.helper import TEST_META_V1, TEST_META_V2 +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 TEST_ENV_PARAMS = """ @@ -280,6 +280,16 @@ def meta_file_v2(): os.remove(META_FILEPATH) +@pytest.fixture +def meta_file_v3(): + with open(META_FILEPATH, 'w') as f: + json.dump(TEST_META_V3, f) + try: + yield META_FILEPATH + finally: + os.remove(META_FILEPATH) + + @pytest.fixture def ensure_meta_removed(): try: diff --git a/tests/helper.py b/tests/helper.py index 7f519f0f..832ac577 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -37,6 +37,14 @@ } +TEST_META_V3 = { + 'version': '0.1.1', + 'config_stream': 'develop', + 'docker_lvmpy_stream': '1.1.2', + 'os_id': 'ubuntu', + 'os_version': '18.04' +} + def response_mock( status_code=0, @@ -70,8 +78,7 @@ def run_command_mock(mock_call_path, response_mock, return run_command(command, params, input=input) -def subprocess_run_mock(cmd=None, shell=None, stdout=None, - stderr=None, env=None, returncode=0): +def subprocess_run_mock(*args, returncode=0, **kwargs): result = MagicMock() result.returncode = returncode result.stdout = MagicMock() diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index 95ab7658..431533db 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -7,7 +7,7 @@ ensure_meta, get_meta_info, save_meta, update_meta ) -from tests.helper import TEST_META_V1, TEST_META_V2 +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 def test_get_meta_info_v1(meta_file_v1): @@ -24,6 +24,15 @@ def test_get_meta_info_v2(meta_file_v2): assert meta.docker_lvmpy_stream == TEST_META_V2['docker_lvmpy_stream'] +def test_get_meta_info_v3(meta_file_v3): + meta = get_meta_info() + assert meta.version == TEST_META_V3['version'] + assert meta.config_stream == TEST_META_V3['config_stream'] + assert meta.docker_lvmpy_stream == TEST_META_V3['docker_lvmpy_stream'] + assert meta.os_id == TEST_META_V3['os_id'] + assert meta.os_version == TEST_META_V3['os_version'] + + def test_get_meta_info_empty(): meta = get_meta_info() assert meta is None @@ -34,6 +43,8 @@ def test_compose_default_meta(): assert meta.version == '1.0.0' assert meta.config_stream == '1.1.0' assert meta.docker_lvmpy_stream == '1.0.0' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '18.04' def test_save_meta(meta_file_v2): @@ -44,28 +55,45 @@ def test_save_meta(meta_file_v2): assert saved_json == { 'version': '1.1.2', 'config_stream': '2.2.2', - 'docker_lvmpy_stream': '1.0.0' + 'docker_lvmpy_stream': '1.0.0', + 'os_id': 'ubuntu', + 'os_version': '18.04', } -def test_update_meta(meta_file_v2): +def test_update_meta_from_v2_to_v3(meta_file_v2): old_meta = get_meta_info() update_meta(version='3.3.3', config_stream='1.1.1', - docker_lvmpy_stream='1.2.2') + docker_lvmpy_stream='1.2.2', os_id='debian', os_version='11') meta = get_meta_info() assert meta.version == '3.3.3' assert meta.config_stream == '1.1.1' assert meta.docker_lvmpy_stream == '1.2.2' + assert meta.os_id == 'debian' + assert meta.os_version == '11' assert meta != old_meta def test_update_meta_from_v1(meta_file_v1): update_meta(version='4.4.4', config_stream='beta', - docker_lvmpy_stream='1.3.3') + docker_lvmpy_stream='1.3.3', os_id='debian', os_version='11') meta = get_meta_info() assert meta.version == '4.4.4' assert meta.config_stream == 'beta' assert meta.docker_lvmpy_stream == '1.3.3' + assert meta.os_id == 'debian' + assert meta.os_version == '11' + + +def test_update_meta_from_v3(meta_file_v3): + update_meta(version='5.5.5', config_stream='stable', + docker_lvmpy_stream='1.2.3', os_id='ubuntu', os_version='20.04') + meta = get_meta_info() + assert meta.version == '5.5.5' + assert meta.config_stream == 'stable' + assert meta.docker_lvmpy_stream == '1.2.3' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '20.04' def test_ensure_meta(ensure_meta_removed): diff --git a/text.yml b/text.yml index 0617d643..08bf65e8 100644 --- a/text.yml +++ b/text.yml @@ -65,4 +65,13 @@ sync_node: help: Initialize sync SKALE node archive: Run sync node in an archive node (disable block rotation) historic_state: Enable historic state (works only in pair with --archive flag) - catchup: Add a flag to start sync node in catchup mode \ No newline at end of file + catchup: Add a flag to start sync node in catchup mode + +lvmpy: + help: Lvmpy commands + run: + help: Run lvmpy http server + prompt: Are you sure you want to run lvmpy server? + heal: + help: Run healing procedure for lvmpy server + prompt: Are you sure you want run healing procedure?