From 62db9398baf9940d716dbfa63a2434fe76977ff8 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 16 Jun 2023 14:12:12 +0000 Subject: [PATCH 01/12] Merge 2.3.0 to beta-sync-node --- .dockerignore | 6 +++ .github/workflows/publish.yml | 98 ++++++++++++++++++++++++----------- .github/workflows/test.yml | 38 ++++++++++---- .gitmodules | 5 ++ Dockerfile | 29 +++++++++++ lvmpy | 1 + main.spec | 4 -- node_cli/cli/__init__.py | 3 +- node_cli/cli/lvmpy.py | 62 ++++++++++++++++++++++ node_cli/cli/node.py | 10 +++- node_cli/configs/__init__.py | 10 ++++ node_cli/core/node.py | 8 ++- node_cli/main.py | 5 +- node_cli/operations/base.py | 22 +++++--- node_cli/utils/meta.py | 16 +++--- scripts/run_tests.sh | 7 ++- setup.py | 11 ++-- tests/cli/main_test.py | 5 +- tests/cli/node_test.py | 35 +++++++++++-- tests/conftest.py | 12 ++++- tests/helper.py | 8 +++ tests/tools_meta_test.py | 38 ++++++++++++-- text.yml | 11 +++- 23 files changed, 362 insertions(+), 82 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 160000 lvmpy create mode 100644 node_cli/cli/lvmpy.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..060cae42 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +tests +helper-scripts +dist +build +.github +.gitmodules diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f7399e36..545fe4b4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: create_release: if: github.event.pull_request.merged name: Create release - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 outputs: upload_url: ${{ steps.create_release.outputs.upload_url }} version: ${{ steps.export_outputs.outputs.version }} @@ -22,6 +22,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + with: + submodules: true + - name: Checkout submodules run: git submodule update --init - name: Install ubuntu dependencies @@ -63,40 +66,57 @@ jobs: strategy: matrix: include: - - os: ubuntu-18.04 + - os: ubuntu-20.04 asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.11 + - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-18.04' + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -e .[dev] - pip install wheel - pip install --upgrade 'setuptools<45.0.0' + - name: Checkout submodules run: git submodule update --init - - name: Build normal CLI + + - name: Build normal binary + run: | + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} + ls -altr /home/ubuntu/dist/ + docker rm -f $(docker ps -aq) + + - name: Save sha512sum run: | - bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal - - name: Upload normal CLI + sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum + + - name: Upload release binary id: upload-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }} + asset_path: /home/ubuntu/dist/${{ matrix.asset_name }} asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream + + - name: Upload release checksum + id: upload-release-checksum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: /home/ubuntu/dist/sha512sum + asset_name: ${{ matrix.asset_name }}.sha512 + asset_content_type: text/plain + build_and_publish_sync: if: github.event.pull_request.merged needs: create_release @@ -105,32 +125,37 @@ jobs: strategy: matrix: include: - - os: ubuntu-18.04 + - os: ubuntu-20.04 asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.11 + - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-18.04' + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -e .[dev] - pip install wheel - pip install --upgrade 'setuptools<45.0.0' + - name: Checkout submodules run: git submodule update --init - - name: Build sync CLI + + - name: Build normal binary run: | - bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync - - name: Upload sync CLI - id: upload-release-asset + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} + ls -altr /home/ubuntu/dist/ + docker rm -f $(docker ps -aq) + + - name: Save sha512sum + run: | + sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum + + - name: Upload release sync CLI + id: upload-sync-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -139,3 +164,14 @@ jobs: asset_path: ./dist/${{ matrix.asset_name }}-sync asset_name: ${{ matrix.asset_name }}-sync asset_content_type: application/octet-stream + + - name: Upload release sync CLI checksum + id: upload-sync-release-checksum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: /home/ubuntu/dist/sha512sum + asset_name: ${{ matrix.asset_name }}-sync.sha512 + asset_content_type: text/plain diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e362e16..b647a846 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,35 +3,55 @@ on: [push, pull_request] jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.8] + python-version: [3.11] steps: - uses: actions/checkout@v2 + with: + submodules: true + + - name: Checkout submodules + run: git submodule update --init + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: Install ubuntu dependencies run: | sudo apt-get update - sudo apt-get install python-setuptools + sudo apt-get install python-setuptools iptables + - name: Install python dependencies run: | python -m pip install --upgrade pip pip install -e . pip install -e .[dev] pip install --upgrade 'setuptools<45.0.0' + - name: Lint with flake8 run: | flake8 . - - name: Build binary + + - name: Build sync binary in Ubuntu 18.04 environment run: | - bash scripts/build.sh 1.0.0 test-branch normal - - name: Build binary sync + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test + docker rm -f $(docker ps -aq) + + - name: Check build + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + + - name: Build sync binary in Ubuntu 20.04 environment run: | - bash scripts/build.sh 1.0.0 test-branch sync + scripts/build.sh test test + + - name: Check build + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + - name: Run tests - run: | - bash ./scripts/run_tests.sh + run: bash ./scripts/run_tests.sh diff --git a/.gitmodules b/.gitmodules index d54a2086..b9ae3738 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,9 @@ [submodule "helper-scripts"] path = helper-scripts url = https://github.com/skalenetwork/helper-scripts.git + branch = develop + +[submodule "lvmpy"] + path = lvmpy + url = https://github.com/skalenetwork/docker-lvmpy branch = develop diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..67a6a42d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get install -y \ + git \ + python3.11 \ + libpython3.11-dev \ + python3.11-venv \ + python3.11-distutils \ + python3.11-dev \ + build-essential \ + zlib1g-dev \ + libssl-dev \ + libffi-dev \ + swig \ + iptables + +RUN mkdir /app +WORKDIR /app + +COPY . . + +ENV PATH=/app/buildvenv/bin:$PATH +RUN python3.11 -m venv /app/buildvenv && \ + pip install --upgrade pip && \ + pip install wheel setuptools==63.2.0 && \ + pip install -e '.[dev]' diff --git a/lvmpy b/lvmpy new file mode 160000 index 00000000..8ee051bf --- /dev/null +++ b/lvmpy @@ -0,0 +1 @@ +Subproject commit 8ee051bf24aa3feecc0ef97fb5eec970eb068512 diff --git a/main.spec b/main.spec index 3c776386..a4dbe395 100644 --- a/main.spec +++ b/main.spec @@ -1,9 +1,5 @@ # -*- mode: python -*- -# import distutils -# if distutils.distutils_path.endswith('__init__.py'): -# distutils.distutils_path = os.path.dirname(distutils.distutils_path) - import importlib.util libxtwrapper_path = importlib.util.find_spec('libxtwrapper').origin diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index 5b1ba474..02a4b07a 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,5 @@ -__version__ = '2.2.1' +__version__ = '2.3.0' + if __name__ == "__main__": print(__version__) diff --git a/node_cli/cli/lvmpy.py b/node_cli/cli/lvmpy.py new file mode 100644 index 00000000..473defa8 --- /dev/null +++ b/node_cli/cli/lvmpy.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2020 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.utils.helper import abort_if_false +from node_cli.utils.texts import Texts +from lvmpy.src.app import run as run_lvmpy +from lvmpy.src.health import heal_service + +G_TEXTS = Texts() +TEXTS = G_TEXTS['lvmpy'] + + +@click.group() +def lvmpy_cli(): + pass + + +@lvmpy_cli.group('lvmpy', help=TEXTS['help']) +def health(): + pass + + +@health.command(help=TEXTS['run']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['run']['prompt'] +) +def run(): + run_lvmpy() + + +@health.command(help=TEXTS['heal']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['heal']['prompt'] +) +def heal(): + heal_service() diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 7ec32dbf..866d07e9 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -131,9 +131,15 @@ def backup_node(backup_folder_path): @node.command('restore', help="Restore SKALE node on another machine") @click.argument('backup_path') @click.argument('env_file') +@click.option( + '--no-snapshot', + help='Do not restore sChains from snapshot', + is_flag=True, + hidden=True +) @streamed_cmd -def restore_node(backup_path, env_file): - restore(backup_path, env_file) +def restore_node(backup_path, env_file, no_snapshot): + restore(backup_path, env_file, no_snapshot) @node.command('maintenance-on', help="Set SKALE node into maintenance mode") diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 81e0743d..8534c4b0 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -77,6 +77,16 @@ FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml') DOCKER_LVMPY_PATH = os.path.join(SKALE_DIR, 'docker-lvmpy') +DOCKER_LVMPY_BIN_PATH = '/usr/local/bin/skale' + +LVMPY_CMD = f'{DOCKER_LVMPY_BIN_PATH} lvmpy' +LVMPY_RUN_CMD = f'{LVMPY_CMD} run --yes' +LVMPY_HEAL_CMD = f'{LVMPY_CMD} heal --yes' + +LVMPY_CRON_LOG_PATH = '/var/log/docker-lvmpy/cron.log' +LVMPY_CRON_SCHEDULE_MINUTES = 3 + +LVMPY_LOG_DIR = '/var/log/docker-lvmpy' IPTABLES_DIR = '/etc/iptables/' IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 168228e3..73b2b51d 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -199,13 +199,17 @@ def update_sync(env_filepath): @check_not_inited -def restore(backup_path, env_filepath): +def restore(backup_path, env_filepath, no_snapshot=False): env = get_node_env(env_filepath) if env is None: return save_env_params(env_filepath) env['SKALE_DIR'] = SKALE_DIR - env['BACKUP_RUN'] = 'True' # should be str + + if not no_snapshot: + logger.info('Adding BACKUP_RUN to env ...') + env['BACKUP_RUN'] = 'True' # should be str + restored_ok = restore_op(env, backup_path) if not restored_ok: error_exit( diff --git a/node_cli/main.py b/node_cli/main.py index bda53a09..fa58b4f4 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -29,6 +29,7 @@ from node_cli.cli.health import health_cli from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION, TYPE from node_cli.cli.logs import logs_cli +from node_cli.cli.lvmpy import lvmpy_cli from node_cli.cli.node import node_cli from node_cli.cli.schains import schains_cli from node_cli.cli.wallet import wallet_cli @@ -90,7 +91,8 @@ def get_sources_list(): wallet_cli, ssl_cli, exit_cli, - validate_cli + validate_cli, + lvmpy_cli ] @@ -113,6 +115,7 @@ def handle_exception(exc_type, exc_value, exc_traceback): logger.debug(f'cmd: {" ".join(str(x) for x in args)}, v.{__version__}') sources = get_sources_list() cmd_collection = click.CommandCollection(sources=sources) + try: cmd_collection() except Exception as err: diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index d5a35c30..cfefcc75 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import distro import functools import logging from typing import Dict @@ -39,11 +40,10 @@ ) from node_cli.operations.volume import ( cleanup_volume_artifacts, - docker_lvmpy_update, - docker_lvmpy_install, ensure_filestorage_mapping, prepare_block_device ) +from node_cli.operations.docker_lvmpy import lvmpy_install # noqa from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables @@ -111,7 +111,7 @@ def update(env_filepath: str, env: Dict) -> None: backup_old_contracts() download_contracts(env) - docker_lvmpy_update(env) + lvmpy_install(env) generate_nginx_config() prepare_host( @@ -134,7 +134,9 @@ def update(env_filepath: str, env: Dict) -> None: update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') compose_up(env) @@ -161,7 +163,7 @@ def init(env_filepath: str, env: dict) -> bool: configure_iptables() generate_nginx_config() - docker_lvmpy_install(env) + lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) update_meta( @@ -215,7 +217,9 @@ def init_sync( update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) @@ -292,13 +296,15 @@ def restore(env, backup_path): link_env_file() configure_iptables() - docker_lvmpy_install(env) + lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) compose_up(env) diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index b49d9f61..69078af2 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -6,20 +6,23 @@ DEFAULT_VERSION = '1.0.0' DEFAULT_CONFIG_STREAM = '1.1.0' DEFAULT_DOCKER_LVMPY_STREAM = '1.0.0' +DEFAULT_OS_ID = 'ubuntu' +DEFAULT_OS_VERSION = '18.04' class CliMeta( namedtuple( 'Node', - ['version', 'config_stream', 'docker_lvmpy_stream'] + ['version', 'config_stream', 'docker_lvmpy_stream', 'os_id', 'os_version'] ) ): __slots__ = () def __new__(cls, version=DEFAULT_VERSION, config_stream=DEFAULT_CONFIG_STREAM, - docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM): + docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION): return super(CliMeta, cls).__new__( - cls, version, config_stream, docker_lvmpy_stream + cls, version, config_stream, docker_lvmpy_stream, os_id, os_version ) @@ -41,7 +44,8 @@ def save_meta(meta: CliMeta) -> None: def compose_default_meta() -> CliMeta: return CliMeta(version=DEFAULT_VERSION, docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, - config_stream=DEFAULT_CONFIG_STREAM) + config_stream=DEFAULT_CONFIG_STREAM, os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION) def ensure_meta(meta: CliMeta = None) -> None: @@ -51,7 +55,7 @@ def ensure_meta(meta: CliMeta = None) -> None: def update_meta(version: str, config_stream: str, - docker_lvmpy_stream: str) -> None: + docker_lvmpy_stream: str, os_id: str, os_version: str) -> None: ensure_meta() - meta = CliMeta(version, config_stream, docker_lvmpy_stream) + meta = CliMeta(version, config_stream, docker_lvmpy_stream, os_id, os_version) save_meta(meta) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 12cce17a..afbb2068 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,4 +3,9 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -HIDE_STREAM_LOG=true TEST_HOME_DIR="$PROJECT_DIR/tests/" GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ +LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \ + HIDE_STREAM_LOG=true \ + TEST_HOME_DIR="$PROJECT_DIR/tests/" \ + GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \ + DOTENV_FILEPATH='tests/test-env' \ + py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ diff --git a/setup.py b/setup.py index c68db0ca..23211e5d 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ def find_version(*file_paths): install_requires=[ "click==8.1.3", "PyInstaller==5.6.2", + "distro==1.4.0", "docker==6.0.1", "texttable==1.6.4", "python-dateutil==2.8.2", @@ -60,15 +61,19 @@ def find_version(*file_paths): "python-dotenv==0.21.0", "terminaltables==3.1.10", "requests==2.28.1", - "GitPython==3.1.27", + "GitPython==3.1.30", "packaging==21.3", "python-debian==0.1.48", - "python-iptables==1.0.0", + "python-iptables==1.0.1", "PyYAML==6.0", "pyOpenSSL==22.0.0", "MarkupSafe==2.1.1", "cryptography==37.0.2", "filelock==3.0.12" + 'Flask==2.2.2', + 'itsdangerous==2.0.1', + 'sh==1.14.2', + 'python-crontab==2.6.0' ], python_requires='>=3.8,<4', extras_require=extras_require, @@ -80,6 +85,6 @@ def find_version(*file_paths): 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Affero General Public License v3', 'Natural Language :: English', - 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.11' ], ) diff --git a/tests/cli/main_test.py b/tests/cli/main_test.py index 2703f0b5..5ce570ad 100644 --- a/tests/cli/main_test.py +++ b/tests/cli/main_test.py @@ -18,14 +18,13 @@ # along with this program. If not, see . -from node_cli.cli import info from node_cli.main import version from tests.helper import run_command def test_version(): result = run_command(version, []) - expected = f'SKALE Node CLI version: {info.VERSION}\n' + expected = 'SKALE Node CLI version: test\n' assert result.output == expected result = run_command(version, ['--short']) - assert result.output == f'{info.VERSION}\n' + assert result.output == 'test\n' diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 3b9fe1d0..319137ed 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -20,6 +20,7 @@ import pathlib import mock +from unittest.mock import MagicMock, patch import requests import logging @@ -302,10 +303,11 @@ def test_restore(mocked_g_config): ) backup_path = result.output.replace( 'Backup archive successfully created: ', '').replace('\n', '') - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.restore_op'), \ - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ + patch('subprocess.run', new=subprocess_run_mock), \ + patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( restore_node, [backup_path, './tests/test-env'] @@ -313,6 +315,31 @@ def test_restore(mocked_g_config): assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') == 'True' + + +def test_restore_no_snapshot(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + result = run_command( + backup_node, + ['/tmp'] + ) + backup_path = result.output.replace( + 'Backup archive successfully created: ', '').replace('\n', '') + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ + patch('subprocess.run', new=subprocess_run_mock), \ + patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + patch('node_cli.utils.decorators.is_node_inited', return_value=False): + result = run_command( + restore_node, + [backup_path, './tests/test-env', '--no-snapshot'] + ) + assert result.exit_code == 0 + assert 'Node is restored from backup\n' in result.output # noqa + + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') is None + def test_maintenance_on(): resp_mock = response_mock( diff --git a/tests/conftest.py b/tests/conftest.py index bdb6d263..5e79dce1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,7 +43,7 @@ from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file -from tests.helper import TEST_META_V1, TEST_META_V2 +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 TEST_ENV_PARAMS = """ @@ -280,6 +280,16 @@ def meta_file_v2(): os.remove(META_FILEPATH) +@pytest.fixture +def meta_file_v3(): + with open(META_FILEPATH, 'w') as f: + json.dump(TEST_META_V3, f) + try: + yield META_FILEPATH + finally: + os.remove(META_FILEPATH) + + @pytest.fixture def ensure_meta_removed(): try: diff --git a/tests/helper.py b/tests/helper.py index 7f519f0f..2aca1efd 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -37,6 +37,14 @@ } +TEST_META_V3 = { + 'version': '0.1.1', + 'config_stream': 'develop', + 'docker_lvmpy_stream': '1.1.2', + 'os_id': 'ubuntu', + 'os_version': '18.04' +} + def response_mock( status_code=0, diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index 95ab7658..431533db 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -7,7 +7,7 @@ ensure_meta, get_meta_info, save_meta, update_meta ) -from tests.helper import TEST_META_V1, TEST_META_V2 +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 def test_get_meta_info_v1(meta_file_v1): @@ -24,6 +24,15 @@ def test_get_meta_info_v2(meta_file_v2): assert meta.docker_lvmpy_stream == TEST_META_V2['docker_lvmpy_stream'] +def test_get_meta_info_v3(meta_file_v3): + meta = get_meta_info() + assert meta.version == TEST_META_V3['version'] + assert meta.config_stream == TEST_META_V3['config_stream'] + assert meta.docker_lvmpy_stream == TEST_META_V3['docker_lvmpy_stream'] + assert meta.os_id == TEST_META_V3['os_id'] + assert meta.os_version == TEST_META_V3['os_version'] + + def test_get_meta_info_empty(): meta = get_meta_info() assert meta is None @@ -34,6 +43,8 @@ def test_compose_default_meta(): assert meta.version == '1.0.0' assert meta.config_stream == '1.1.0' assert meta.docker_lvmpy_stream == '1.0.0' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '18.04' def test_save_meta(meta_file_v2): @@ -44,28 +55,45 @@ def test_save_meta(meta_file_v2): assert saved_json == { 'version': '1.1.2', 'config_stream': '2.2.2', - 'docker_lvmpy_stream': '1.0.0' + 'docker_lvmpy_stream': '1.0.0', + 'os_id': 'ubuntu', + 'os_version': '18.04', } -def test_update_meta(meta_file_v2): +def test_update_meta_from_v2_to_v3(meta_file_v2): old_meta = get_meta_info() update_meta(version='3.3.3', config_stream='1.1.1', - docker_lvmpy_stream='1.2.2') + docker_lvmpy_stream='1.2.2', os_id='debian', os_version='11') meta = get_meta_info() assert meta.version == '3.3.3' assert meta.config_stream == '1.1.1' assert meta.docker_lvmpy_stream == '1.2.2' + assert meta.os_id == 'debian' + assert meta.os_version == '11' assert meta != old_meta def test_update_meta_from_v1(meta_file_v1): update_meta(version='4.4.4', config_stream='beta', - docker_lvmpy_stream='1.3.3') + docker_lvmpy_stream='1.3.3', os_id='debian', os_version='11') meta = get_meta_info() assert meta.version == '4.4.4' assert meta.config_stream == 'beta' assert meta.docker_lvmpy_stream == '1.3.3' + assert meta.os_id == 'debian' + assert meta.os_version == '11' + + +def test_update_meta_from_v3(meta_file_v3): + update_meta(version='5.5.5', config_stream='stable', + docker_lvmpy_stream='1.2.3', os_id='ubuntu', os_version='20.04') + meta = get_meta_info() + assert meta.version == '5.5.5' + assert meta.config_stream == 'stable' + assert meta.docker_lvmpy_stream == '1.2.3' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '20.04' def test_ensure_meta(ensure_meta_removed): diff --git a/text.yml b/text.yml index 0617d643..08bf65e8 100644 --- a/text.yml +++ b/text.yml @@ -65,4 +65,13 @@ sync_node: help: Initialize sync SKALE node archive: Run sync node in an archive node (disable block rotation) historic_state: Enable historic state (works only in pair with --archive flag) - catchup: Add a flag to start sync node in catchup mode \ No newline at end of file + catchup: Add a flag to start sync node in catchup mode + +lvmpy: + help: Lvmpy commands + run: + help: Run lvmpy http server + prompt: Are you sure you want to run lvmpy server? + heal: + help: Run healing procedure for lvmpy server + prompt: Are you sure you want run healing procedure? From 46348f69197799bae5ad841ba087766152bccbbc Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 16 Jun 2023 18:07:36 +0000 Subject: [PATCH 02/12] Fix missing VOLUME_GROUP config param --- lvmpy | 1 - node_cli/configs/__init__.py | 1 + node_cli/operations/docker_lvmpy.py | 92 +++++++++++++++++++++++++++++ setup.py | 2 +- 4 files changed, 94 insertions(+), 2 deletions(-) delete mode 160000 lvmpy create mode 100644 node_cli/operations/docker_lvmpy.py diff --git a/lvmpy b/lvmpy deleted file mode 160000 index 8ee051bf..00000000 --- a/lvmpy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8ee051bf24aa3feecc0ef97fb5eec970eb068512 diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 8534c4b0..50748b6d 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -36,6 +36,7 @@ FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SNAPSHOTS_SHARED_VOLUME = 'shared-space' SCHAINS_MNT_DIR = '/var/lib/skale/schains' +VOLUME_GROUP = 'schains' SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') SKALE_TMP_DIR = os.path.join(SKALE_DIR, '.tmp') diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py new file mode 100644 index 00000000..31036c95 --- /dev/null +++ b/node_cli/operations/docker_lvmpy.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import os +import shutil + +import crontab + +from node_cli.utils.git_utils import sync_repo +from node_cli.configs import ( + DOCKER_LVMPY_PATH, + DOCKER_LVMPY_REPO_URL, + FILESTORAGE_MAPPING, + LVMPY_RUN_CMD, + LVMPY_HEAL_CMD, + LVMPY_CRON_LOG_PATH, + LVMPY_CRON_SCHEDULE_MINUTES, + SCHAINS_MNT_DIR, + VOLUME_GROUP +) +from lvmpy.src.install import setup as setup_lvmpy + +logger = logging.getLogger(__name__) + + +def update_docker_lvmpy_env(env): + env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] + env['VOLUME_GROUP'] = 'schains' + env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING + env['MNT_DIR'] = SCHAINS_MNT_DIR + env['PATH'] = os.environ.get('PATH', None) + return env + + +def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): + if not os.path.isdir(FILESTORAGE_MAPPING): + os.makedirs(FILESTORAGE_MAPPING) + + +def sync_docker_lvmpy_repo(env): + if os.path.isdir(DOCKER_LVMPY_PATH): + shutil.rmtree(DOCKER_LVMPY_PATH) + sync_repo( + DOCKER_LVMPY_REPO_URL, + DOCKER_LVMPY_PATH, + env["DOCKER_LVMPY_STREAM"] + ) + + +def lvmpy_install(env): + ensure_filestorage_mapping() + logging.info('Configuring and starting lvmpy') + setup_lvmpy( + block_device=env['DISK_MOUNTPOINT'], + volume_group=VOLUME_GROUP, + exec_start=LVMPY_RUN_CMD + ) + init_healing_cron() + logger.info('docker-lvmpy is configured and started') + + +def init_healing_cron(): + logger.info('Configuring cron job for healing lvmpy') + cron_line = f'{LVMPY_HEAL_CMD} >> {LVMPY_CRON_LOG_PATH} 2>&1' + legacy_line = f'cd /opt/docker-lvmpy && venv/bin/python -c "import health; health.heal_service()" >> {LVMPY_CRON_LOG_PATH} 2>&1' # noqa + + with crontab.CronTab(user='root') as c: + jobs = [c.command for c in c] + if legacy_line in jobs: + c.remove_all(command=legacy_line) + if cron_line not in jobs: + job = c.new( + command=cron_line + ) + job.minute.every(LVMPY_CRON_SCHEDULE_MINUTES) diff --git a/setup.py b/setup.py index 23211e5d..6a05adcd 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def find_version(*file_paths): "pyOpenSSL==22.0.0", "MarkupSafe==2.1.1", "cryptography==37.0.2", - "filelock==3.0.12" + "filelock==3.0.12", 'Flask==2.2.2', 'itsdangerous==2.0.1', 'sh==1.14.2', From 411e78ff518710859ec7742720d74cdb29bbe96c Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 08:45:25 +0000 Subject: [PATCH 03/12] Rollback to python 3.8 for 18.04 compatible build --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 67a6a42d..1b8463ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,11 @@ RUN apt-get update && apt-get install -y software-properties-common RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get install -y \ git \ - python3.11 \ - libpython3.11-dev \ - python3.11-venv \ - python3.11-distutils \ - python3.11-dev \ + python3.8 \ + libpython3.8-dev \ + python3.8-venv \ + python3.8-distutils \ + python3.8-dev \ build-essential \ zlib1g-dev \ libssl-dev \ @@ -23,7 +23,7 @@ WORKDIR /app COPY . . ENV PATH=/app/buildvenv/bin:$PATH -RUN python3.11 -m venv /app/buildvenv && \ +RUN python3.8 -m venv /app/buildvenv && \ pip install --upgrade pip && \ pip install wheel setuptools==63.2.0 && \ pip install -e '.[dev]' From 76bb02f1c133fb0e12225a6b3b8d9a4cbdf09ee3 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 08:52:37 +0000 Subject: [PATCH 04/12] Add missing sync param to build script intput --- .github/workflows/publish.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 545fe4b4..b5499dc2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -142,11 +142,11 @@ jobs: - name: Checkout submodules run: git submodule update --init - - name: Build normal binary + - name: Build sync release binary run: | mkdir ./dist docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync ls -altr /home/ubuntu/dist/ docker rm -f $(docker ps -aq) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b647a846..73df9e54 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,7 @@ jobs: run: | mkdir ./dist docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test sync docker rm -f $(docker ps -aq) - name: Check build @@ -48,7 +48,7 @@ jobs: - name: Build sync binary in Ubuntu 20.04 environment run: | - scripts/build.sh test test + scripts/build.sh test test sync - name: Check build run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 From 70271e82d16ce4208d332b476fa1300ecea526e0 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 09:57:16 +0000 Subject: [PATCH 05/12] Fix asset name --- .github/workflows/publish.yml | 3 ++- .github/workflows/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b5499dc2..b2c77f9e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -58,6 +58,7 @@ jobs: run: | echo "::set-output name=version::$VERSION" echo "::set-output name=branch::$BRANCH" + build_and_publish_normal: if: github.event.pull_request.merged needs: create_release @@ -126,7 +127,7 @@ jobs: matrix: include: - os: ubuntu-20.04 - asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 + asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64-sync steps: - uses: actions/checkout@v2 - name: Set up Python 3.11 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 73df9e54..943486f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,14 +44,14 @@ jobs: docker rm -f $(docker ps -aq) - name: Check build - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync - name: Build sync binary in Ubuntu 20.04 environment run: | scripts/build.sh test test sync - name: Check build - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync - name: Run tests run: bash ./scripts/run_tests.sh From fb7f3499db02a0fb5d9e8a9fa2fb042daf2e2cb2 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 10:11:00 +0000 Subject: [PATCH 06/12] Add missing lvmpy submodule --- lvmpy | 1 + 1 file changed, 1 insertion(+) create mode 160000 lvmpy diff --git a/lvmpy b/lvmpy new file mode 160000 index 00000000..8ee051bf --- /dev/null +++ b/lvmpy @@ -0,0 +1 @@ +Subproject commit 8ee051bf24aa3feecc0ef97fb5eec970eb068512 From d24cb30f264f5f58e11274b8c234818c0876d251 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 12:12:45 +0000 Subject: [PATCH 07/12] Fix tests --- node_cli/utils/meta.py | 2 +- tests/cli/sync_node_test.py | 8 ++++---- tests/helper.py | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index 69078af2..7eb50f4c 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -55,7 +55,7 @@ def ensure_meta(meta: CliMeta = None) -> None: def update_meta(version: str, config_stream: str, - docker_lvmpy_stream: str, os_id: str, os_version: str) -> None: + docker_lvmpy_stream: str, os_id: str, os_version: str) -> None: ensure_meta() meta = CliMeta(version, config_stream, docker_lvmpy_stream, os_id, os_version) save_meta(meta) diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 24318a81..fb7e4eb2 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -53,8 +53,8 @@ def test_init_sync(mocked_g_config): def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ +# with mock.patch('subprocess.run', new=subprocess_run_mock), \ + with mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ mock.patch('node_cli.operations.base.cleanup_volume_artifacts'), \ mock.patch('node_cli.operations.base.download_skale_node'), \ mock.patch('node_cli.operations.base.sync_skale_node'), \ @@ -74,13 +74,13 @@ def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( _init_sync, - ['./tests/test-env', '--archive', '--catchup'] + ['./tests/test-env', '--archive', '--catchup', '--historic-state'] ) node_options = NodeOptions() assert node_options.archive assert node_options.catchup - assert not node_options.historic_state + assert node_options.historic_state assert result.exit_code == 0 diff --git a/tests/helper.py b/tests/helper.py index 2aca1efd..832ac577 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -78,8 +78,7 @@ def run_command_mock(mock_call_path, response_mock, return run_command(command, params, input=input) -def subprocess_run_mock(cmd=None, shell=None, stdout=None, - stderr=None, env=None, returncode=0): +def subprocess_run_mock(*args, returncode=0, **kwargs): result = MagicMock() result.returncode = returncode result.stdout = MagicMock() From a63112aa39fcf050cb6033ba9bb40b2e60e07ccc Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 20 Jun 2023 19:20:02 +0000 Subject: [PATCH 08/12] Fix sync node publish pipeline --- .github/workflows/publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b2c77f9e..7ce9a521 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -162,8 +162,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }}-sync - asset_name: ${{ matrix.asset_name }}-sync + asset_path: ./dist/${{ matrix.asset_name }} + asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream - name: Upload release sync CLI checksum @@ -174,5 +174,5 @@ jobs: with: upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: /home/ubuntu/dist/sha512sum - asset_name: ${{ matrix.asset_name }}-sync.sha512 + asset_name: ${{ matrix.asset_name }}.sha512 asset_content_type: text/plain From 819a262b5919d97a4e4ae93a3d32ff108c53f2ee Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 20 Jun 2023 19:29:57 +0000 Subject: [PATCH 09/12] Fix publish.yml home dir path --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7ce9a521..85161fa7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -88,7 +88,7 @@ jobs: run: | mkdir ./dist docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal ls -altr /home/ubuntu/dist/ docker rm -f $(docker ps -aq) @@ -162,7 +162,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }} + asset_path: /home/ubuntu/dist/${{ matrix.asset_name }} asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream From e6353544cdc47e66e06a9f20580ad5e2a51c4d84 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 7 Jul 2023 12:45:33 +0000 Subject: [PATCH 10/12] Remove prerelease from sync-node based releases --- .github/workflows/publish.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 85161fa7..8d761ae0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,12 +31,14 @@ jobs: run: | sudo apt-get update sudo apt-get install python-setuptools + - name: Set Versions run: | bash ./scripts/set_versions_ga.sh + - name: Set release run: | - if [[ "$BRANCH" == "stable" ]]; then + if [[ "$BRANCH" == "stable" || "$BRANCH" == "sync-node" ]]; then export PRERELEASE=false else export PRERELEASE=true From 7eac870bff4ddd935addb4095b122d5322c31984 Mon Sep 17 00:00:00 2001 From: badrogger Date: Sat, 8 Jul 2023 12:49:59 +0000 Subject: [PATCH 11/12] Fix update-meta invocation --- node_cli/operations/base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index cfefcc75..45cdb948 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -169,7 +169,9 @@ def init(env_filepath: str, env: dict) -> bool: update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') @@ -259,7 +261,9 @@ def update_sync(env_filepath: str, env: Dict) -> bool: update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) compose_up_sync(env) From 7eb2aaeddbf7f18f35a76587e34389f4b211a674 Mon Sep 17 00:00:00 2001 From: badrogger Date: Sat, 8 Jul 2023 12:54:05 +0000 Subject: [PATCH 12/12] Trigger build --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1b8463ed..943f66e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,8 @@ RUN apt-get install -y \ python3.8-dev \ build-essential \ zlib1g-dev \ - libssl-dev \ libffi-dev \ + libssl-dev \ swig \ iptables