From c6636efcf3d33f8f895a4157ac9cbd71e4c2b198 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 4 May 2022 16:46:50 +0100 Subject: [PATCH 01/86] SKALE-779 Full sync node init cmd --- .gitignore | 3 ++- node_cli/cli/__init__.py | 2 +- node_cli/cli/node.py | 9 ++++++- node_cli/configs/__init__.py | 1 + node_cli/configs/env.py | 22 ++++++++++----- node_cli/core/host.py | 13 ++++++--- node_cli/core/node.py | 24 ++++++++++++----- node_cli/operations/__init__.py | 1 + node_cli/operations/base.py | 48 ++++++++++++++++++++++++++++----- node_cli/operations/common.py | 7 ----- node_cli/utils/docker_utils.py | 12 ++++++++- node_cli/utils/helper.py | 5 ++-- tests/operations/common_test.py | 4 --- 13 files changed, 111 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 707381fd..db6f3fd0 100644 --- a/.gitignore +++ b/.gitignore @@ -116,4 +116,5 @@ meta.json disk_mountpoint.txt sgx_server_url.txt resource_allocation.json -conf.json \ No newline at end of file +conf.json +test-env \ No newline at end of file diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index 55f4a051..680cf87f 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.1.0' +__version__ = '2.2.0' if __name__ == "__main__": print(__version__) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 6a54a805..e982cdf1 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -24,7 +24,7 @@ from node_cli.core.node import ( configure_firewall_rules, - get_node_signature, init, restore, + get_node_signature, init, init_sync, restore, register_node as register, update, backup, set_maintenance_mode_on, set_maintenance_mode_off, @@ -122,6 +122,13 @@ def init_node(env_file): init(env_file) +@node.command('init-sync', help="Initialize sync SKALE node") +@click.argument('env_file') +@streamed_cmd +def _init_sync(env_file): + init_sync(env_file) + + @node.command('update', help='Update node from .env file') @click.option('--yes', is_flag=True, callback=abort_if_false, expose_value=False, diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index df789afb..b938f6b7 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -51,6 +51,7 @@ SGX_CERTIFICATES_DIR_NAME = 'sgx_certs' COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') +SYNC_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-sync.yml') FILESTORAGE_INFO_FILE = os.path.join( CONTAINER_CONFIG_PATH, 'filestorage_info.json') FILESTORAGE_ARTIFACTS_FILE = os.path.join( diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index 662bdc2a..038eea3e 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -9,8 +9,7 @@ ALLOWED_ENV_TYPES = ['mainnet', 'testnet', 'qanet', 'devnet'] - -base_params = { +REQUIRED_PARAMS = { 'IMA_ENDPOINT': '', 'CONTAINER_CONFIGS_STREAM': '', 'ENDPOINT': '', @@ -24,8 +23,17 @@ 'ENV_TYPE': '', } +REQUIRED_PARAMS_SYNC = { + 'CONTAINER_CONFIGS_STREAM': '', + 'ENDPOINT': '', + 'MANAGER_CONTRACTS_ABI_URL': '', + 'IMA_CONTRACTS_ABI_URL': '', + 'DISK_MOUNTPOINT': '', + 'DOCKER_LVMPY_STREAM': '', + 'ENV_TYPE': '' +} -optional_params = { +OPTIONAL_PARAMS = { 'MONITORING_CONTAINERS': '', 'TG_API_KEY': '', 'TG_CHAT_ID': '', @@ -40,15 +48,15 @@ def absent_params(params): return list(filter( - lambda key: key not in optional_params and not params[key], + lambda key: key not in OPTIONAL_PARAMS and not params[key], params) ) -def get_env_config(env_filepath: str = SKALE_DIR_ENV_FILEPATH): +def get_env_config(env_filepath: str = SKALE_DIR_ENV_FILEPATH, sync_node: bool = False): load_dotenv(dotenv_path=env_filepath) - params = base_params.copy() - params.update(optional_params) + params = REQUIRED_PARAMS_SYNC.copy() if sync_node else REQUIRED_PARAMS.copy() + params.update(OPTIONAL_PARAMS) for option_name in params: env_param = os.getenv(option_name) if env_param is not None: diff --git a/node_cli/core/host.py b/node_cli/core/host.py index f6736c4f..1b15869f 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -69,12 +69,19 @@ def get_flask_secret_key(): return key_file.read().strip() -def prepare_host(env_filepath, disk_mountpoint, sgx_server_url, env_type, - allocation=False): +def prepare_host( + env_filepath: str, + disk_mountpoint: str, + env_type: str, + sgx_server_url: str = None, + allocation: bool = False +): logger.info(f'Preparing host started, disk_mountpoint: {disk_mountpoint}') make_dirs() save_env_params(env_filepath) - save_sgx_server_url(sgx_server_url) + + if sgx_server_url: + save_sgx_server_url(sgx_server_url) if allocation: update_resource_allocation(disk_mountpoint, env_type) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 33caaf1f..a8c55879 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -51,7 +51,7 @@ from node_cli.core.resources import update_resource_allocation from node_cli.operations import ( update_op, - init_op, turn_off_op, turn_on_op, restore_op + init_op, turn_off_op, turn_on_op, restore_op, init_sync_op ) from node_cli.utils.print_formatters import ( print_failed_requirements_checks, print_node_cmd_error, print_node_info @@ -141,6 +141,18 @@ def init(env_filepath): logger.info('Init procedure finished') +def init_sync(env_filepath: str) -> None: + env = get_node_env(env_filepath, sync_node=True) + if env is None: + return + inited_ok = init_sync_op(env_filepath, env) + if not inited_ok: + error_exit( + 'Init operation failed', + exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR + ) + + @check_not_inited def restore(backup_path, env_filepath): env = get_node_env(env_filepath) @@ -162,24 +174,24 @@ def restore(backup_path, env_filepath): print('Node is restored from backup') -def get_node_env(env_filepath, inited_node=False, sync_schains=None): +def get_node_env(env_filepath, inited_node=False, sync_schains=None, sync_node=False): if env_filepath is not None: - env_params = extract_env_params(env_filepath) + env_params = extract_env_params(env_filepath, sync_node=sync_node) if env_params is None: return save_env_params(env_filepath) else: - env_params = extract_env_params(INIT_ENV_FILEPATH) + env_params = extract_env_params(INIT_ENV_FILEPATH, sync_node=sync_node) env = { 'SKALE_DIR': SKALE_DIR, 'SCHAINS_MNT_DIR': SCHAINS_MNT_DIR, 'FILESTORAGE_MAPPING': FILESTORAGE_MAPPING, **env_params } - if inited_node: + if inited_node and not sync_node: flask_secret_key = get_flask_secret_key() env['FLASK_SECRET_KEY'] = flask_secret_key - if sync_schains: + if sync_schains and not sync_node: env['BACKUP_RUN'] = 'True' return {k: v for k, v in env.items() if v != ''} diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index e511af44..faa5066f 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -20,6 +20,7 @@ from node_cli.operations.base import ( # noqa update as update_op, init as init_op, + init_sync as init_sync_op, turn_off as turn_off_op, turn_on as turn_on_op, restore as restore_op diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index aeecf770..c2da06e4 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -30,14 +30,16 @@ from node_cli.core.resources import update_resource_allocation, init_shared_space_volume from node_cli.operations.common import ( - backup_old_contracts, download_contracts, download_filestorage_artifacts, configure_filebeat, + backup_old_contracts, download_contracts, configure_filebeat, configure_flask, unpack_backup_archive ) from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables -from node_cli.utils.docker_utils import compose_rm, compose_up, remove_dynamic_containers +from node_cli.utils.docker_utils import ( + compose_rm, compose_up, remove_dynamic_containers, compose_up_sync +) from node_cli.utils.meta import update_meta from node_cli.utils.print_formatters import print_failed_requirements_checks @@ -93,15 +95,14 @@ def update(env_filepath: str, env: Dict) -> None: backup_old_contracts() download_contracts(env) - download_filestorage_artifacts() docker_lvmpy_update(env) generate_nginx_config() prepare_host( env_filepath, env['DISK_MOUNTPOINT'], - env['SGX_SERVER_URL'], env['ENV_TYPE'], + env['SGX_SERVER_URL'], allocation=True ) init_shared_space_volume(env['ENV_TYPE']) @@ -126,12 +127,11 @@ def init(env_filepath: str, env: str) -> bool: prepare_host( env_filepath, env['DISK_MOUNTPOINT'], - env['SGX_SERVER_URL'], env_type=env['ENV_TYPE'], + sgx_server_url=env['SGX_SERVER_URL'] ) link_env_file() download_contracts(env) - download_filestorage_artifacts() configure_filebeat() configure_flask() @@ -155,6 +155,42 @@ def init(env_filepath: str, env: str) -> bool: return True +def init_sync(env_filepath: str, env: str) -> bool: + download_skale_node( + env['CONTAINER_CONFIGS_STREAM'], + env.get('CONTAINER_CONFIGS_DIR') + ) + sync_skale_node() + + if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_docker() + + prepare_host( + env_filepath, + env['DISK_MOUNTPOINT'], + env_type=env['ENV_TYPE'], + ) + link_env_file() + download_contracts(env) + + generate_nginx_config() + docker_lvmpy_install(env) + + update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + env['DOCKER_LVMPY_STREAM'] + ) + update_resource_allocation( + disk_device=env['DISK_MOUNTPOINT'], + env_type=env['ENV_TYPE'] + ) + update_images(env.get('CONTAINER_CONFIGS_DIR') != '') + compose_up_sync(env) + + return True + + def turn_off(): logger.info('Turning off the node...') compose_rm() diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 5fd6bef7..5e2ec855 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -49,13 +49,6 @@ def download_contracts(env): urllib.request.urlretrieve(env['IMA_CONTRACTS_ABI_URL'], IMA_CONTRACTS_FILEPATH) -def download_filestorage_artifacts(): - logger.info('Updating filestorage artifacts') - fs_artifacts_url = read_json(FILESTORAGE_INFO_FILE)['artifacts_url'] - logger.debug(f'Downloading {fs_artifacts_url} to {FILESTORAGE_ARTIFACTS_FILE}') - urllib.request.urlretrieve(fs_artifacts_url, FILESTORAGE_ARTIFACTS_FILE) - - def configure_filebeat(): logger.info('Configuring filebeat...') copyfile(SRC_FILEBEAT_CONFIG_PATH, FILEBEAT_CONFIG_PATH) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index cccf9989..ec6ef3d5 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -29,6 +29,7 @@ from node_cli.utils.helper import run_cmd, str_to_bool from node_cli.configs import ( COMPOSE_PATH, + SYNC_COMPOSE_PATH, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTIFICATES_DIR_NAME, SKALE_DIR @@ -220,7 +221,11 @@ def get_up_compose_cmd(services): return ('docker-compose', '-f', COMPOSE_PATH, 'up', '-d', *services) -def compose_up(env): +def get_up_compose_sync_cmd(): + return ('docker-compose', '-f', SYNC_COMPOSE_PATH, 'up', '-d') + + +def compose_up(env, sync_node=False): logger.info('Running base set of containers') if 'SGX_CERTIFICATES_DIR_NAME' not in env: @@ -233,3 +238,8 @@ def compose_up(env): if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: logger.info('Running containers for Telegram notifications') run_cmd(cmd=get_up_compose_cmd(NOTIFICATION_COMPOSE_SERVICES), env=env) + + +def compose_up_sync(env) -> None: + logger.info('Running containers for sync node') + run_cmd(cmd=get_up_compose_sync_cmd(), env=env) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 6a61e06c..9d9b7523 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -125,9 +125,8 @@ def get_username(): return os.environ.get('USERNAME') or os.environ.get('USER') -def extract_env_params(env_filepath): - env_params = get_env_config(env_filepath) - +def extract_env_params(env_filepath, sync_node=False): + env_params = get_env_config(env_filepath, sync_node=sync_node) absent_params = ', '.join(absent_env_params(env_params)) if absent_params: click.echo(f"Your env file({env_filepath}) have some absent params: " diff --git a/tests/operations/common_test.py b/tests/operations/common_test.py index 8a019ffd..cee64ebc 100644 --- a/tests/operations/common_test.py +++ b/tests/operations/common_test.py @@ -16,10 +16,6 @@ def test_docker_lvmpy_update(): assert False -def test_download_filestorage_artifacts(): - assert False - - def test_update_skale_node_repo(): assert False From a8c037d7fa357f0745534faa9471ceaed0edd7ff Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 9 May 2022 16:20:36 +0100 Subject: [PATCH 02/86] SKALE-779 Add schain name to required params --- node_cli/configs/env.py | 1 + 1 file changed, 1 insertion(+) diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index 038eea3e..a4c33f0d 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -24,6 +24,7 @@ } REQUIRED_PARAMS_SYNC = { + 'SCHAIN_NAME': '', 'CONTAINER_CONFIGS_STREAM': '', 'ENDPOINT': '', 'MANAGER_CONTRACTS_ABI_URL': '', From 707c85db925f3c99cfa49612b1a262f685f7dade Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 11 May 2022 23:16:03 +0100 Subject: [PATCH 03/86] SKALE-779 Add sync node allocation --- node_cli/configs/resource_allocation.py | 1 + node_cli/core/resources.py | 5 +- tests/.skale/config/schain_allocation.yml | 100 ++++++++++++---------- 3 files changed, 60 insertions(+), 46 deletions(-) diff --git a/node_cli/configs/resource_allocation.py b/node_cli/configs/resource_allocation.py index 003d64e3..ad3056de 100644 --- a/node_cli/configs/resource_allocation.py +++ b/node_cli/configs/resource_allocation.py @@ -20,6 +20,7 @@ import os from node_cli.configs import NODE_DATA_PATH +SYNC_NODE_DIVIDER = 1 LARGE_DIVIDER = 1 MEDIUM_DIVIDER = 8 TEST_DIVIDER = 8 diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index ab27430a..9d7ce52e 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -36,7 +36,7 @@ from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, TEST_DIVIDER, SMALL_DIVIDER, MEDIUM_DIVIDER, LARGE_DIVIDER, - MEMORY_FACTOR, MAX_CPU_SHARES + SYNC_NODE_DIVIDER, MEMORY_FACTOR, MAX_CPU_SHARES ) logger = logging.getLogger(__name__) @@ -53,7 +53,8 @@ def __init__(self, value, fractional=False): 'test': value / TEST_DIVIDER, 'small': value / SMALL_DIVIDER, 'medium': value / MEDIUM_DIVIDER, - 'large': value / LARGE_DIVIDER + 'large': value / LARGE_DIVIDER, + 'sync_node': value / SYNC_NODE_DIVIDER, } if not fractional: for k in self.values: diff --git a/tests/.skale/config/schain_allocation.yml b/tests/.skale/config/schain_allocation.yml index 0aebc880..29bb7be1 100644 --- a/tests/.skale/config/schain_allocation.yml +++ b/tests/.skale/config/schain_allocation.yml @@ -6,30 +6,28 @@ devnet: large: 71039975424 medium: 8879996928 small: 554999808 + sync_node: 71039975424 test: 8879996928 test4: 8879996928 leveldb_limits: large: contract_storage: 12787195576 - db_storage: 8524797050 + db_storage: 4262398525 medium: contract_storage: 1598399446 - db_storage: 1065599631 + db_storage: 532799815 small: contract_storage: 99899965 - db_storage: 66599976 + db_storage: 33299988 + sync_node: + contract_storage: 12787195576 + db_storage: -1 test: contract_storage: 1598399446 - db_storage: 1065599631 + db_storage: 532799815 test4: contract_storage: 1598399446 - db_storage: 1065599631 - rotate_after_block: - large: 1310721 - medium: 163840 - small: 10240 - test: 163840 - test4: 163840 + db_storage: 532799815 shared_space: 8959950848 volume_limits: large: @@ -47,6 +45,11 @@ devnet: max_file_storage_bytes: 166499942 max_reserved_storage_bytes: 55499980 max_skaled_leveldb_storage_bytes: 166499942 + sync_node: + max_consensus_storage_bytes: 21311992627 + max_file_storage_bytes: 21311992627 + max_reserved_storage_bytes: 7103997542 + max_skaled_leveldb_storage_bytes: 21311992627 test: max_consensus_storage_bytes: 2663999078 max_file_storage_bytes: 2663999078 @@ -62,30 +65,28 @@ mainnet: large: 1687199940608 medium: 210899992576 small: 13181249536 + sync_node: 1687199940608 test: 210899992576 test4: 210899992576 leveldb_limits: large: contract_storage: 303695989309 - db_storage: 202463992872 + db_storage: 101231996436 medium: contract_storage: 37961998663 - db_storage: 25307999108 + db_storage: 12653999554 small: contract_storage: 2372624916 - db_storage: 1581749944 + db_storage: 790874972 + sync_node: + contract_storage: 303695989309 + db_storage: -1 test: contract_storage: 37961998663 - db_storage: 25307999108 + db_storage: 12653999554 test4: contract_storage: 37961998663 - db_storage: 25307999108 - rotate_after_block: - large: 31129628 - medium: 3891203 - small: 243200 - test: 3891203 - test4: 3891203 + db_storage: 12653999554 shared_space: 212799979520 volume_limits: large: @@ -103,6 +104,11 @@ mainnet: max_file_storage_bytes: 3954374860 max_reserved_storage_bytes: 1318124953 max_skaled_leveldb_storage_bytes: 3954374860 + sync_node: + max_consensus_storage_bytes: 506159982182 + max_file_storage_bytes: 506159982182 + max_reserved_storage_bytes: 168719994060 + max_skaled_leveldb_storage_bytes: 506159982182 test: max_consensus_storage_bytes: 63269997772 max_file_storage_bytes: 63269997772 @@ -118,30 +124,28 @@ qanet: large: 177599938560 medium: 22199992320 small: 1387499520 + sync_node: 177599938560 test: 22199992320 test4: 22199992320 leveldb_limits: large: contract_storage: 31967988940 - db_storage: 21311992627 + db_storage: 10655996313 medium: contract_storage: 3995998617 - db_storage: 2663999078 + db_storage: 1331999539 small: contract_storage: 249749913 - db_storage: 166499942 + db_storage: 83249971 + sync_node: + contract_storage: 31967988940 + db_storage: -1 test: contract_storage: 3995998617 - db_storage: 2663999078 + db_storage: 1331999539 test4: contract_storage: 3995998617 - db_storage: 2663999078 - rotate_after_block: - large: 3276803 - medium: 409600 - small: 25600 - test: 409600 - test4: 409600 + db_storage: 1331999539 shared_space: 22399942656 volume_limits: large: @@ -159,6 +163,11 @@ qanet: max_file_storage_bytes: 416249856 max_reserved_storage_bytes: 138749952 max_skaled_leveldb_storage_bytes: 416249856 + sync_node: + max_consensus_storage_bytes: 53279981568 + max_file_storage_bytes: 53279981568 + max_reserved_storage_bytes: 17759993856 + max_skaled_leveldb_storage_bytes: 53279981568 test: max_consensus_storage_bytes: 6659997696 max_file_storage_bytes: 6659997696 @@ -174,30 +183,28 @@ testnet: large: 177599938560 medium: 22199992320 small: 1387499520 + sync_node: 177599938560 test: 22199992320 test4: 22199992320 leveldb_limits: large: contract_storage: 31967988940 - db_storage: 21311992627 + db_storage: 10655996313 medium: contract_storage: 3995998617 - db_storage: 2663999078 + db_storage: 1331999539 small: contract_storage: 249749913 - db_storage: 166499942 + db_storage: 83249971 + sync_node: + contract_storage: 31967988940 + db_storage: -1 test: contract_storage: 3995998617 - db_storage: 2663999078 + db_storage: 1331999539 test4: contract_storage: 3995998617 - db_storage: 2663999078 - rotate_after_block: - large: 3276803 - medium: 409600 - small: 25600 - test: 409600 - test4: 409600 + db_storage: 1331999539 shared_space: 22399942656 volume_limits: large: @@ -215,6 +222,11 @@ testnet: max_file_storage_bytes: 416249856 max_reserved_storage_bytes: 138749952 max_skaled_leveldb_storage_bytes: 416249856 + sync_node: + max_consensus_storage_bytes: 53279981568 + max_file_storage_bytes: 53279981568 + max_reserved_storage_bytes: 17759993856 + max_skaled_leveldb_storage_bytes: 53279981568 test: max_consensus_storage_bytes: 6659997696 max_file_storage_bytes: 6659997696 From d80cde5f5532d411f0b4cbbe0e8c2e792354feff Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 13 May 2022 16:45:45 +0100 Subject: [PATCH 04/86] SKALE-779 Fix flake8. --- node_cli/configs/__init__.py | 4 ---- node_cli/operations/common.py | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index b938f6b7..941e6493 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -52,10 +52,6 @@ COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') SYNC_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-sync.yml') -FILESTORAGE_INFO_FILE = os.path.join( - CONTAINER_CONFIG_PATH, 'filestorage_info.json') -FILESTORAGE_ARTIFACTS_FILE = os.path.join( - NODE_DATA_PATH, 'filestorage_artifacts.json') ENVIRONMENT_PARAMS_FILEPATH = os.path.join( CONTAINER_CONFIG_PATH, 'environment_params.yaml') NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 5e2ec855..673f7bbf 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -29,11 +29,9 @@ from distutils.dir_util import copy_tree from node_cli.configs import ( - CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, - MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, SRC_FILEBEAT_CONFIG_PATH, G_CONF_HOME, - FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE, FILEBEAT_CONFIG_PATH, FLASK_SECRET_KEY_FILE + CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, + SRC_FILEBEAT_CONFIG_PATH, G_CONF_HOME, FILEBEAT_CONFIG_PATH, FLASK_SECRET_KEY_FILE ) -from node_cli.utils.helper import read_json logger = logging.getLogger(__name__) From 713f32fa626d6b2d00ac16724da9d9b21a74034c Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 13 May 2022 19:04:07 +0100 Subject: [PATCH 05/86] SKALE-779 Fix resource allocation tests --- node_cli/configs/__init__.py | 3 +-- node_cli/core/resources.py | 5 +---- tests/resources_test.py | 27 ++++++++++++--------------- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 941e6493..77d0310f 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -109,8 +109,7 @@ def _get_env(): SKALED_SSL_TEST_SCRIPT = os.path.join(DATAFILES_FOLDER, 'skaled-ssl-test') -ALLOCATION_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, - 'schain_allocation.yml') +ALLOCATION_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'schain_allocation.yml') REDIS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'redis-data') diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 9d7ce52e..788f2082 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -72,7 +72,6 @@ def get_resource_allocation_info(): def compose_resource_allocation_config( - disk_device: str, env_type: str, params_by_env_type: Dict = None ) -> Dict: @@ -124,9 +123,7 @@ def generate_resource_allocation_config(env_file, force=False) -> None: def update_resource_allocation(disk_device: str, env_type: str) -> None: - resource_allocation_config = compose_resource_allocation_config( - disk_device, env_type - ) + resource_allocation_config = compose_resource_allocation_config(env_type) write_json(RESOURCE_ALLOCATION_FILEPATH, resource_allocation_config) diff --git a/tests/resources_test.py b/tests/resources_test.py index 3eb04950..c3439ae4 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -14,7 +14,7 @@ from node_cli.utils.helper import write_json, safe_load_yml -SCHAIN_VOLUME_PARTS = {'large': {'max_consensus_storage_bytes': 21311992627, 'max_file_storage_bytes': 21311992627, 'max_reserved_storage_bytes': 7103997542, 'max_skaled_leveldb_storage_bytes': 21311992627}, 'medium': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'small': {'max_consensus_storage_bytes': 166499942, 'max_file_storage_bytes': 166499942, 'max_reserved_storage_bytes': 55499980, 'max_skaled_leveldb_storage_bytes': 166499942}, 'test': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'test4': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}} # noqa +SCHAIN_VOLUME_PARTS = {'large': {'max_consensus_storage_bytes': 21311992627, 'max_file_storage_bytes': 21311992627, 'max_reserved_storage_bytes': 7103997542, 'max_skaled_leveldb_storage_bytes': 21311992627}, 'medium': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'small': {'max_consensus_storage_bytes': 166499942, 'max_file_storage_bytes': 166499942, 'max_reserved_storage_bytes': 55499980, 'max_skaled_leveldb_storage_bytes': 166499942}, 'test': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'test4': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'sync_node': {'max_consensus_storage_bytes': 21311992627, 'max_file_storage_bytes': 21311992627, 'max_reserved_storage_bytes': 7103997542, 'max_skaled_leveldb_storage_bytes': 21311992627}} # noqa DEFAULT_ENV_TYPE = 'devnet' @@ -47,9 +47,7 @@ def resource_alloc_config(): def test_generate_resource_allocation_config(): disk_device = '/dev/test' with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): - resource_allocation_config = compose_resource_allocation_config( - disk_device, DEFAULT_ENV_TYPE - ) + resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) assert resource_allocation_config['schain']['cpu_shares']['test4'] == 102 assert resource_allocation_config['schain']['cpu_shares']['test'] == 102 @@ -69,7 +67,8 @@ def test_generate_resource_allocation_config(): assert resource_allocation_config['schain']['disk']['medium'] == 8879996928 assert resource_allocation_config['schain']['disk']['large'] == 71039975424 - assert resource_allocation_config['ima']['cpu_shares'] == {'large': 204, 'medium': 25, 'small': 1, 'test': 25, 'test4': 25} # noqa + assert resource_allocation_config['ima']['cpu_shares'] == { + 'large': 204, 'medium': 25, 'small': 1, 'sync_node': 204, 'test': 25, 'test4': 25} assert isinstance(resource_allocation_config['ima']['mem'], dict) assert resource_allocation_config['schain']['volume_limits'] == SCHAIN_VOLUME_PARTS @@ -103,6 +102,7 @@ def test_get_static_disk_alloc_devnet( 'large': 71039975424, 'medium': 8879996928, 'small': 554999808, + 'sync_node': 71039975424, 'test': 8879996928, 'test4': 8879996928 } @@ -162,15 +162,12 @@ def test_get_memory_alloc(params_by_env_type): def test_leveldb_limits(): disk_device = '/dev/test' with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): - resource_allocation_config = compose_resource_allocation_config( - disk_device, - DEFAULT_ENV_TYPE - ) - + resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) assert resource_allocation_config['schain']['leveldb_limits'] == { - 'large': {'contract_storage': 12787195576, 'db_storage': 8524797050}, - 'medium': {'contract_storage': 1598399446, 'db_storage': 1065599631}, - 'small': {'contract_storage': 99899965, 'db_storage': 66599976}, - 'test': {'contract_storage': 1598399446, 'db_storage': 1065599631}, - 'test4': {'contract_storage': 1598399446, 'db_storage': 1065599631} + 'large': {'contract_storage': 12787195576, 'db_storage': 4262398525}, + 'medium': {'contract_storage': 1598399446, 'db_storage': 532799815}, + 'small': {'contract_storage': 99899965, 'db_storage': 33299988}, + 'test': {'contract_storage': 1598399446, 'db_storage': 532799815}, + 'test4': {'contract_storage': 1598399446, 'db_storage': 532799815}, + 'sync_node': {'contract_storage': 12787195576, 'db_storage': -1}, } From 96b20970149e0a51c4496baf466fd41243311caa Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 13 May 2022 19:06:07 +0100 Subject: [PATCH 06/86] SKALE-779 Fix resource allocation tests --- tests/resources_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/resources_test.py b/tests/resources_test.py index c3439ae4..61f1a30c 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -45,7 +45,6 @@ def resource_alloc_config(): def test_generate_resource_allocation_config(): - disk_device = '/dev/test' with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) @@ -160,7 +159,6 @@ def test_get_memory_alloc(params_by_env_type): def test_leveldb_limits(): - disk_device = '/dev/test' with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) assert resource_allocation_config['schain']['leveldb_limits'] == { From 274d97de62f194eddb2b22a947f1eb5eaf3c6a66 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 19 May 2022 22:35:11 +0100 Subject: [PATCH 07/86] SKALE-779 Remove unused disk_mountpoint --- node_cli/core/host.py | 14 ++------------ node_cli/core/node.py | 7 ++----- node_cli/core/resources.py | 3 +-- node_cli/operations/base.py | 22 ++++------------------ tests/resources_test.py | 3 +-- 5 files changed, 10 insertions(+), 39 deletions(-) diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 1b15869f..47c17ded 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -71,19 +71,15 @@ def get_flask_secret_key(): def prepare_host( env_filepath: str, - disk_mountpoint: str, env_type: str, - sgx_server_url: str = None, allocation: bool = False ): - logger.info(f'Preparing host started, disk_mountpoint: {disk_mountpoint}') + logger.info(f'Preparing host started') make_dirs() save_env_params(env_filepath) - if sgx_server_url: - save_sgx_server_url(sgx_server_url) if allocation: - update_resource_allocation(disk_mountpoint, env_type) + update_resource_allocation(env_type) def is_node_inited(): @@ -102,12 +98,6 @@ def make_dirs(): safe_mkdir(dir_path) -def save_sgx_server_url(sgx_server_url): - logger.info(f'Saving sgx_server_url option to {SGX_SERVER_URL_FILEPATH}') - with open(SGX_SERVER_URL_FILEPATH, 'w') as f: - f.write(sgx_server_url) - - def save_env_params(env_filepath): copyfile(env_filepath, SKALE_DIR_ENV_FILEPATH) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index a8c55879..70aa95ab 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -127,7 +127,6 @@ def init(env_filepath): 'Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR ) - return logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): @@ -135,9 +134,8 @@ def init(env_filepath): 'Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR ) - return logger.info('Generating resource allocation file ...') - update_resource_allocation(env['DISK_MOUNTPOINT'], env['ENV_TYPE']) + update_resource_allocation(env['ENV_TYPE']) logger.info('Init procedure finished') @@ -167,10 +165,9 @@ def restore(backup_path, env_filepath): 'Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR ) - return time.sleep(RESTORE_SLEEP_TIMEOUT) logger.info('Generating resource allocation file ...') - update_resource_allocation(env['DISK_MOUNTPOINT'], env['ENV_TYPE']) + update_resource_allocation(env['ENV_TYPE']) print('Node is restored from backup') diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 788f2082..078c99f8 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -109,7 +109,6 @@ def generate_resource_allocation_config(env_file, force=False) -> None: logger.info('Generating resource allocation file ...') try: update_resource_allocation( - env_params['DISK_MOUNTPOINT'], env_params['ENV_TYPE'] ) except Exception as e: @@ -122,7 +121,7 @@ def generate_resource_allocation_config(env_file, force=False) -> None: ) -def update_resource_allocation(disk_device: str, env_type: str) -> None: +def update_resource_allocation(env_type: str) -> None: resource_allocation_config = compose_resource_allocation_config(env_type) write_json(RESOURCE_ALLOCATION_FILEPATH, resource_allocation_config) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index c2da06e4..106f9ab2 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -100,9 +100,7 @@ def update(env_filepath: str, env: Dict) -> None: prepare_host( env_filepath, - env['DISK_MOUNTPOINT'], env['ENV_TYPE'], - env['SGX_SERVER_URL'], allocation=True ) init_shared_space_volume(env['ENV_TYPE']) @@ -126,9 +124,7 @@ def init(env_filepath: str, env: str) -> bool: prepare_host( env_filepath, - env['DISK_MOUNTPOINT'], - env_type=env['ENV_TYPE'], - sgx_server_url=env['SGX_SERVER_URL'] + env_type=env['ENV_TYPE'] ) link_env_file() download_contracts(env) @@ -146,10 +142,7 @@ def init(env_filepath: str, env: str) -> bool: env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'] ) - update_resource_allocation( - disk_device=env['DISK_MOUNTPOINT'], - env_type=env['ENV_TYPE'] - ) + update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') compose_up(env) return True @@ -167,7 +160,6 @@ def init_sync(env_filepath: str, env: str) -> bool: prepare_host( env_filepath, - env['DISK_MOUNTPOINT'], env_type=env['ENV_TYPE'], ) link_env_file() @@ -181,10 +173,7 @@ def init_sync(env_filepath: str, env: str) -> bool: env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'] ) - update_resource_allocation( - disk_device=env['DISK_MOUNTPOINT'], - env_type=env['ENV_TYPE'] - ) + update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') compose_up_sync(env) @@ -228,10 +217,7 @@ def restore(env, backup_path): env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'] ) - update_resource_allocation( - disk_device=env['DISK_MOUNTPOINT'], - env_type=env['ENV_TYPE'] - ) + update_resource_allocation(env_type=env['ENV_TYPE']) compose_up(env) failed_checks = run_host_checks( diff --git a/tests/resources_test.py b/tests/resources_test.py index 61f1a30c..caf1c542 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -74,9 +74,8 @@ def test_generate_resource_allocation_config(): def test_update_allocation_config(resource_alloc_config): - block_device = '/dev/test' with mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE): - update_resource_allocation(block_device, DEFAULT_ENV_TYPE) + update_resource_allocation(DEFAULT_ENV_TYPE) with open(RESOURCE_ALLOCATION_FILEPATH) as jfile: assert json.load(jfile) != INITIAL_CONFIG From 7f9f0082ee26376752cc6d18c2d4d1c14b630cca Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 19 May 2022 23:33:40 +0100 Subject: [PATCH 08/86] SKALE-779 Add update-sync cmd, add tests, add sync-cli pipeline, update readme --- .github/workflows/publish.yml | 20 +++++-- README.md | 94 +++++++++++++++++++++++++++++---- node_cli/cli/node.py | 11 +--- node_cli/cli/sync_node.py | 52 ++++++++++++++++++ node_cli/core/node.py | 35 ++++++++++-- node_cli/main.py | 38 +++++++------ node_cli/operations/__init__.py | 1 + node_cli/operations/base.py | 30 +++++++++++ scripts/build.sh | 17 +++++- tests/cli/sync_node_test.py | 65 +++++++++++++++++++++++ tests/test-env | 3 +- 11 files changed, 322 insertions(+), 44 deletions(-) create mode 100644 node_cli/cli/sync_node.py create mode 100644 tests/cli/sync_node_test.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 645b1ecb..2f8ddd14 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,6 +7,7 @@ on: - develop - beta - stable + - sync-node jobs: create_release: @@ -82,10 +83,13 @@ jobs: pip install --upgrade 'setuptools<45.0.0' - name: Checkout submodules run: git submodule update --init - - name: Build library + - name: Build normal CLI run: | - bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} - - name: Upload Release Asset + bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal + - name: Build sync CLI + run: | + bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync + - name: Upload normal CLI id: upload-release-asset uses: actions/upload-release-asset@v1 env: @@ -95,3 +99,13 @@ jobs: asset_path: ./dist/${{ matrix.asset_name }} asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream + - name: Upload sync CLI + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ./dist/${{ matrix.asset_name }}-sync + asset_name: ${{ matrix.asset_name }}-sync + asset_content_type: application/octet-stream diff --git a/README.md b/README.md index 0d9e31e3..6be93c9a 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,11 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 2.7 [Logs](#logs-commands) 2.8 [Resources allocation](#resources-allocation-commands) 2.9 [Validate](#validate-commands) -3. [Exit codes](#exit-codes) -4. [Development](#development) +3. [Sync CLI usage](#sync-cli-usage) + 3.1 [Top level commands](#top-level-commands-sync) + 3.2 [Sync node commands](#sync-node-commands) +4. [Exit codes](#exit-codes) +5. [Development](#development) ## Installation @@ -34,10 +37,10 @@ Ensure that the following package is installed: **docker**, **docker-compose** ( VERSION_NUM={put the version number here} && sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m` > /usr/local/bin/skale" ``` -For versions `<1.1.0`: +For Sync node version: ```shell -VERSION_NUM=0.0.0 && sudo -E bash -c "curl -L https://skale-cli.sfo2.cdn.digitaloceanspaces.com/skale-$VERSION_NUM-`uname -s`-`uname -m` > /usr/local/bin/skale" +VERSION_NUM={put the version number here} && sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m`-sync > /usr/local/bin/skale" ``` - Apply executable permissions to the downloaded binary: @@ -104,7 +107,7 @@ skale node init [ENV_FILE] Arguments: -- `ENV_FILE` - path to .env file (required parameters are listed in the `skale init` command) +- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command) You should specify the following environment variables: @@ -117,6 +120,7 @@ You should specify the following environment variables: - `MANAGER_CONTRACTS_ABI_URL` - URL to SKALE Manager contracts ABI and addresses - `IMA_CONTRACTS_ABI_URL` - URL to IMA contracts ABI and addresses - `FILEBEAT_URL` - URL to the Filebeat log server +- `ENV_TYPE` - environement type (mainnet, testnet, etc) Optional variables: @@ -136,7 +140,7 @@ skale node restore [BACKUP_PATH] [ENV_FILE] Arguments: - `BACKUP_PATH` - path to the archive with backup data generated by `skale node backup` command -- `ENV_FILE` - path to .env file (required parameters are listed in the `skale init` command) +- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command) #### Node backup @@ -177,7 +181,7 @@ skale node update [ENV_FILEPATH] Options: -- `--yes` - remove without additional confirmation +- `--yes` - update without additional confirmation Arguments: @@ -197,7 +201,7 @@ skale node turn-off Options: - `--maintenance-on` - set SKALE node into maintenance mode before turning off -- `--yes` - remove without additional confirmation +- `--yes` - turn off without additional confirmation #### Node turn-on @@ -210,7 +214,7 @@ skale node turn-on [ENV_FILEPATH] Options: - `--maintenance-off` - turn off maintenance mode after turning on the node -- `--yes` - remove without additional confirmation +- `--yes` - turn on without additional confirmation Arguments: @@ -477,7 +481,7 @@ skale resources-allocation generate [ENV_FILE] Arguments: -- `ENV_FILE` - path to .env file (required parameters are listed in the `skale init` command) +- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command) Options: @@ -500,6 +504,76 @@ Options: - `--json` - show validation result in json format + +## Sync CLI usage + +### Top level commands sync + +#### Info + +Print build info + +```shell +skale info +``` + +#### Version + +Print version number + +```shell +skale version +``` + +Options: + +- `--short` - prints version only, without additional text. + +### Sync node commands + +> Prefix: `skale sync-node` + +#### Sync node initialization + +Initialize full sync SKALE node on current machine + +```shell +skale sync-node init [ENV_FILE] +``` + +Arguments: + +- `ENV_FILE` - path to .env file (required parameters are listed in the `skale sync-node init` command) + +You should specify the following environment variables: + +- `DISK_MOUNTPOINT` - disk mount point for storing sChains data +- `DOCKER_LVMPY_STREAM` - stream of `docker-lvmpy` to use +- `CONTAINER_CONFIGS_STREAM` - stream of `skale-node` to use +- `ENDPOINT` - RPC endpoint of the node in the network where SKALE Manager is deployed +- `MANAGER_CONTRACTS_ABI_URL` - URL to SKALE Manager contracts ABI and addresses +- `IMA_CONTRACTS_ABI_URL` - URL to IMA contracts ABI and addresses +- `SCHAIN_NAME` - name of the SKALE chain to sync +- `ENV_TYPE` - environement type (mainnet, testnet, etc) + +#### Sync node update + +Update full sync SKALE node on current machine + +```shell +skale sync-node update [ENV_FILEPATH] +``` + +Options: + +- `--yes` - update without additional confirmation + +Arguments: + +- `ENV_FILEPATH` - path to env file where parameters are defined + +> NOTE: You can just update a file with environment variables used during `skale sync-node init`. + ## Exit codes Exit codes conventions for SKALE CLI tools diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index e982cdf1..de2caa8f 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -2,7 +2,7 @@ # # This file is part of node-cli # -# Copyright (C) 2019 SKALE Labs +# Copyright (C) 2019-Present SKALE Labs # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -24,7 +24,7 @@ from node_cli.core.node import ( configure_firewall_rules, - get_node_signature, init, init_sync, restore, + get_node_signature, init, restore, register_node as register, update, backup, set_maintenance_mode_on, set_maintenance_mode_off, @@ -122,13 +122,6 @@ def init_node(env_file): init(env_file) -@node.command('init-sync', help="Initialize sync SKALE node") -@click.argument('env_file') -@streamed_cmd -def _init_sync(env_file): - init_sync(env_file) - - @node.command('update', help='Update node from .env file') @click.option('--yes', is_flag=True, callback=abort_if_false, expose_value=False, diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py new file mode 100644 index 00000000..2a1cac3d --- /dev/null +++ b/node_cli/cli/sync_node.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.core.node import init_sync, update_sync +from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd + + +TEXTS = safe_load_texts() + +@click.group() +def sync_node_cli(): + pass + + +@sync_node_cli.group(help="SKALE sync node commands") +def sync_node(): + pass + + +@sync_node.command('init', help="Initialize sync SKALE node") +@click.argument('env_file') +@streamed_cmd +def _init_sync(env_file): + init_sync(env_file) + + +@sync_node.command('update', help='Update sync node from .env file') +@click.option('--yes', is_flag=True, callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to update SKALE node software?') +@click.argument('env_file') +@streamed_cmd +def _update_sync(env_file): + update_sync(env_file) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 70aa95ab..1bbb2f7f 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -50,8 +50,7 @@ from node_cli.core.checks import run_checks as run_host_checks from node_cli.core.resources import update_resource_allocation from node_cli.operations import ( - update_op, - init_op, turn_off_op, turn_on_op, restore_op, init_sync_op + update_op, init_op, turn_off_op, turn_on_op, restore_op, init_sync_op, update_sync_op ) from node_cli.utils.print_formatters import ( print_failed_requirements_checks, print_node_cmd_error, print_node_info @@ -70,6 +69,7 @@ logger = logging.getLogger(__name__) TEXTS = Texts() +SYNC_BASE_CONTAINERS_AMOUNT = 2 BASE_CONTAINERS_AMOUNT = 5 BLUEPRINT_NAME = 'node' @@ -139,6 +139,7 @@ def init(env_filepath): logger.info('Init procedure finished') +@check_not_inited def init_sync(env_filepath: str) -> None: env = get_node_env(env_filepath, sync_node=True) if env is None: @@ -149,6 +150,31 @@ def init_sync(env_filepath: str) -> None: 'Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR ) + logger.info('Waiting for containers initialization') + time.sleep(TM_INIT_TIMEOUT) + if not is_base_containers_alive(sync_node=True): + error_exit( + 'Containers are not running', + exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR + ) + logger.info('Sync node initialized successfully') + + +@check_inited +@check_user +def update_sync(env_filepath): + logger.info('Node update started') + env = get_node_env(env_filepath, sync_node=True) + update_ok = update_sync_op(env_filepath, env) + if update_ok: + logger.info('Waiting for containers initialization') + time.sleep(TM_INIT_TIMEOUT) + alive = is_base_containers_alive(sync_node=True) + if not update_ok or not alive: + print_node_cmd_error() + return + else: + logger.info('Node update finished') @check_not_inited @@ -340,13 +366,14 @@ def turn_on(maintenance_off, sync_schains, env_file): set_maintenance_mode_off() -def is_base_containers_alive(): +def is_base_containers_alive(sync_node: bool = False): dclient = docker.from_env() containers = dclient.containers.list() skale_containers = list(filter( lambda c: c.name.startswith('skale_'), containers )) - return len(skale_containers) >= BASE_CONTAINERS_AMOUNT + containers_amount = SYNC_BASE_CONTAINERS_AMOUNT if sync_node else BASE_CONTAINERS_AMOUNT + return len(skale_containers) >= containers_amount def get_node_info(format): diff --git a/node_cli/main.py b/node_cli/main.py index 6a0abe83..493f842f 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -27,7 +27,7 @@ from node_cli.cli import __version__ from node_cli.cli.health import health_cli -from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION +from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION, TYPE from node_cli.cli.logs import logs_cli from node_cli.cli.node import node_cli from node_cli.cli.schains import schains_cli @@ -36,6 +36,8 @@ from node_cli.cli.exit import exit_cli from node_cli.cli.validate import validate_cli from node_cli.cli.resources_allocation import resources_allocation_cli +from node_cli.cli.sync_node import sync_node_cli + from node_cli.utils.helper import safe_load_texts, init_default_logger from node_cli.configs import LONG_LINE from node_cli.core.host import init_logs_dir @@ -73,6 +75,24 @@ def info(): ''')) +def get_sources_list(): + if TYPE == 'sync': + return [sync_node_cli] + else: + return [ + cli, + health_cli, + schains_cli, + logs_cli, + resources_allocation_cli, + node_cli, + wallet_cli, + ssl_cli, + exit_cli, + validate_cli + ] + + def handle_exception(exc_type, exc_value, exc_traceback): if issubclass(exc_type, KeyboardInterrupt): sys.__excepthook__(exc_type, exc_value, exc_traceback) @@ -90,20 +110,8 @@ def handle_exception(exc_type, exc_value, exc_traceback): args = sys.argv # todo: hide secret variables (passwords, private keys) logger.debug(f'cmd: {" ".join(str(x) for x in args)}, v.{__version__}') - - cmd_collection = click.CommandCollection( - sources=[ - cli, - health_cli, - schains_cli, - logs_cli, - resources_allocation_cli, - node_cli, - wallet_cli, - ssl_cli, - exit_cli, - validate_cli - ]) + sources = get_sources_list() + cmd_collection = click.CommandCollection(sources=sources) try: cmd_collection() except Exception as err: diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index faa5066f..11d3dd4d 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -21,6 +21,7 @@ update as update_op, init as init_op, init_sync as init_sync_op, + update_sync as update_sync_op, turn_off as turn_off_op, turn_on as turn_on_op, restore as restore_op diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 106f9ab2..56340d0a 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -176,7 +176,37 @@ def init_sync(env_filepath: str, env: str) -> bool: update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') compose_up_sync(env) + return True + + +def update_sync(env_filepath: str, env: Dict) -> None: + compose_rm(env) + remove_dynamic_containers() + + sync_skale_node() + if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_docker() + + backup_old_contracts() + download_contracts(env) + + docker_lvmpy_update(env) + generate_nginx_config() + + prepare_host( + env_filepath, + env['ENV_TYPE'], + allocation=True + ) + + update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + env['DOCKER_LVMPY_STREAM'] + ) + update_images(env.get('CONTAINER_CONFIGS_DIR') != '') + compose_up_sync(env) return True diff --git a/scripts/build.sh b/scripts/build.sh index 432eb7d2..c7478f1e 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,8 +4,9 @@ set -e VERSION=$1 BRANCH=$2 +TYPE=$3 -USAGE_MSG='Usage: build.sh [VERSION] [BRANCH]' +USAGE_MSG='Usage: build.sh [VERSION] [BRANCH] [TYPE]' if [ -z "$1" ] then @@ -21,6 +22,13 @@ then exit 1 fi +if [ -z "$3" ] +then + (>&2 echo 'You should provide type: normal or sync') + echo $USAGE_MSG + exit 1 +fi + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PARENT_DIR="$(dirname "$DIR")" @@ -37,8 +45,13 @@ echo "COMMIT = '$LATEST_COMMIT'" >> $DIST_INFO_FILEPATH echo "BRANCH = '$BRANCH'" >> $DIST_INFO_FILEPATH echo "OS = '$OS'" >> $DIST_INFO_FILEPATH echo "VERSION = '$VERSION'" >> $DIST_INFO_FILEPATH +echo "TYPE = '$TYPE'" >> $DIST_INFO_FILEPATH -EXECUTABLE_NAME=skale-$VERSION-$OS +if [ "$TYPE" = "sync" ]; then + EXECUTABLE_NAME=skale-$VERSION-$OS-sync +else + EXECUTABLE_NAME=skale-$VERSION-$OS +fi pyinstaller --onefile main.spec diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py new file mode 100644 index 00000000..7e75c8c1 --- /dev/null +++ b/tests/cli/sync_node_test.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2019 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pathlib + +import mock +import requests +import logging + +from node_cli.configs import SKALE_DIR, G_CONF_HOME +from node_cli.cli.sync_node import _init_sync, _update_sync +from node_cli.utils.helper import init_default_logger + +from tests.helper import ( + response_mock, run_command_mock, + run_command, subprocess_run_mock +) +from tests.resources_test import BIG_DISK_SIZE + +logger = logging.getLogger(__name__) +init_default_logger() + + +def test_init_sync(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + with mock.patch('subprocess.run', new=subprocess_run_mock), \ + mock.patch('node_cli.core.node.init_sync_op'), \ + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + result = run_command( + _init_sync, + ['./tests/test-env'] + ) + assert result.exit_code == 0 + + +def test_update_sync(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + with mock.patch('subprocess.run', new=subprocess_run_mock), \ + mock.patch('node_cli.core.node.update_sync_op'), \ + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): + result = run_command( + _update_sync, + ['./tests/test-env', '--yes'] + ) + assert result.exit_code == 0 diff --git a/tests/test-env b/tests/test-env index 0c293ba3..9afb4c68 100644 --- a/tests/test-env +++ b/tests/test-env @@ -11,4 +11,5 @@ MANAGER_CONTRACTS_ABI_URL=http://127.0.0.1 SGX_SERVER_URL=http://127.0.0.1 DISK_MOUNTPOINT=/dev/sss DOCKER_LVMPY_STREAM='master' -ENV_TYPE='devnet' \ No newline at end of file +ENV_TYPE='devnet' +SCHAIN_NAME='test' \ No newline at end of file From 4f0dfd49e6f9d8a36098ad79c47fff4f85ad65bf Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 19 May 2022 23:50:43 +0100 Subject: [PATCH 09/86] SKALE-779 Fix flake8 errors --- node_cli/cli/sync_node.py | 1 + node_cli/core/host.py | 3 +-- tests/cli/sync_node_test.py | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index 2a1cac3d..ce94c8be 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -25,6 +25,7 @@ TEXTS = safe_load_texts() + @click.group() def sync_node_cli(): pass diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 47c17ded..84151f82 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -36,8 +36,7 @@ SKALE_RUN_DIR, SKALE_TMP_DIR ) from node_cli.configs.resource_allocation import ( - RESOURCE_ALLOCATION_FILEPATH, - SGX_SERVER_URL_FILEPATH + RESOURCE_ALLOCATION_FILEPATH ) from node_cli.configs.cli_logger import LOG_DATA_PATH from node_cli.configs.env import SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 7e75c8c1..9483f7c9 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -20,15 +20,13 @@ import pathlib import mock -import requests import logging -from node_cli.configs import SKALE_DIR, G_CONF_HOME +from node_cli.configs import SKALE_DIR from node_cli.cli.sync_node import _init_sync, _update_sync from node_cli.utils.helper import init_default_logger from tests.helper import ( - response_mock, run_command_mock, run_command, subprocess_run_mock ) from tests.resources_test import BIG_DISK_SIZE From 8d4917165ba5997cc4212164d794ad975a602583 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 20 May 2022 11:33:48 +0100 Subject: [PATCH 10/86] SKALE-779 Fix sync node build --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b69634d7..851e0c31 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,10 @@ jobs: flake8 . - name: Build binary run: | - bash scripts/build.sh 1.0.0 test-branch + bash scripts/build.sh 1.0.0 test-branch normal + - name: Build binary sync + run: | + bash scripts/build.sh 1.0.0 test-branch sync - name: Run tests run: | bash ./scripts/run_tests.sh From d3e57f5221be33640f4866e31c771e68c17d0f20 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 20 May 2022 13:13:22 +0100 Subject: [PATCH 11/86] SKALE-779 Fix publish pipeline --- .github/workflows/publish.yml | 37 +++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2f8ddd14..d83f552c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -54,7 +54,7 @@ jobs: run: | echo "::set-output name=version::$VERSION" echo "::set-output name=branch::$BRANCH" - build_and_publish: + build_and_publish_normal: if: github.event.pull_request.merged needs: create_release name: Build and publish for ${{ matrix.os }} @@ -86,9 +86,6 @@ jobs: - name: Build normal CLI run: | bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal - - name: Build sync CLI - run: | - bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync - name: Upload normal CLI id: upload-release-asset uses: actions/upload-release-asset@v1 @@ -99,6 +96,38 @@ jobs: asset_path: ./dist/${{ matrix.asset_name }} asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream + build_and_publish_sync: + if: github.event.pull_request.merged + needs: create_release + name: Build and publish for ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-18.04 + asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install ubuntu dependencies + if: matrix.os == 'ubuntu-18.04' + run: | + sudo apt-get update + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install -e .[dev] + pip install wheel + pip install --upgrade 'setuptools<45.0.0' + - name: Checkout submodules + run: git submodule update --init + - name: Build sync CLI + run: | + bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync - name: Upload sync CLI id: upload-release-asset uses: actions/upload-release-asset@v1 From 88e2eb67d5a0d35505286645eb5b0fde1978308f Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 20 May 2022 17:28:45 +0100 Subject: [PATCH 12/86] SKALE-779 Fix sync-node update cmd --- node_cli/operations/base.py | 10 +++++++--- node_cli/operations/skale_node.py | 6 +++--- node_cli/utils/docker_utils.py | 19 +++++++++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 56340d0a..b55593c6 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -174,15 +174,19 @@ def init_sync(env_filepath: str, env: str) -> bool: env['DOCKER_LVMPY_STREAM'] ) update_resource_allocation(env_type=env['ENV_TYPE']) - update_images(env.get('CONTAINER_CONFIGS_DIR') != '') + update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) compose_up_sync(env) return True def update_sync(env_filepath: str, env: Dict) -> None: - compose_rm(env) + compose_rm(env, sync_node=True) remove_dynamic_containers() + download_skale_node( + env['CONTAINER_CONFIGS_STREAM'], + env.get('CONTAINER_CONFIGS_DIR') + ) sync_skale_node() if env.get('SKIP_DOCKER_CONFIG') != 'True': @@ -205,7 +209,7 @@ def update_sync(env_filepath: str, env: Dict) -> None: env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'] ) - update_images(env.get('CONTAINER_CONFIGS_DIR') != '') + update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) compose_up_sync(env) return True diff --git a/node_cli/operations/skale_node.py b/node_cli/operations/skale_node.py index 044f7a5d..b3745070 100644 --- a/node_cli/operations/skale_node.py +++ b/node_cli/operations/skale_node.py @@ -35,11 +35,11 @@ logger = logging.getLogger(__name__) -def update_images(local: bool = False) -> None: +def update_images(local: bool = False, sync_node: bool = False) -> None: if local: - compose_build() + compose_build(sync_node=sync_node) else: - compose_pull() + compose_pull(sync_node=sync_node) def download_skale_node(stream: Optional[str], src: Optional[str]) -> None: diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index ec6ef3d5..437b61f0 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -183,12 +183,13 @@ def is_volume_exists(name: str, dutils=None): return True -def compose_rm(env={}): +def compose_rm(env={}, sync_node: bool = False): logger.info('Removing compose containers') + compose_path = get_compose_path(sync_node) run_cmd( cmd=( 'docker-compose', - '-f', COMPOSE_PATH, + '-f', compose_path, 'down', '-t', str(COMPOSE_SHUTDOWN_TIMEOUT), ), @@ -197,20 +198,22 @@ def compose_rm(env={}): logger.info('Compose containers removed') -def compose_pull(): +def compose_pull(sync_node: bool = False): logger.info('Pulling compose containers') + compose_path = get_compose_path(sync_node) run_cmd( - cmd=('docker-compose', '-f', COMPOSE_PATH, 'pull'), + cmd=('docker-compose', '-f', compose_path, 'pull'), env={ 'SKALE_DIR': SKALE_DIR } ) -def compose_build(): +def compose_build(sync_node: bool = False): logger.info('Building compose containers') + compose_path = get_compose_path(sync_node) run_cmd( - cmd=('docker-compose', '-f', COMPOSE_PATH, 'build'), + cmd=('docker-compose', '-f', compose_path, 'build'), env={ 'SKALE_DIR': SKALE_DIR } @@ -225,6 +228,10 @@ def get_up_compose_sync_cmd(): return ('docker-compose', '-f', SYNC_COMPOSE_PATH, 'up', '-d') +def get_compose_path(sync_node: bool) -> str: + return SYNC_COMPOSE_PATH if sync_node else COMPOSE_PATH + + def compose_up(env, sync_node=False): logger.info('Running base set of containers') From 927f06c353bead70f87901892e1529d4e121abea Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 1 Jun 2022 20:34:12 +0100 Subject: [PATCH 13/86] SKALE-779 Add SSL CLI to sync-node-cli --- node_cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/main.py b/node_cli/main.py index 493f842f..987cb34e 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -77,7 +77,7 @@ def info(): def get_sources_list(): if TYPE == 'sync': - return [sync_node_cli] + return [sync_node_cli, ssl_cli] else: return [ cli, From 3a799ce6aed898b2c519f5bf75ac4ce76b68e04d Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 2 Jun 2022 19:24:15 +0100 Subject: [PATCH 14/86] SKALE-779 Restructure SSL commands --- node_cli/cli/ssl.py | 13 +- node_cli/configs/__init__.py | 9 - node_cli/configs/ssl.py | 33 +++ node_cli/core/host2/__init__.py | 0 node_cli/core/ssl/__init__.py | 22 ++ node_cli/core/{ssl.py => ssl/check.py} | 312 ++++++++++--------------- node_cli/core/ssl/status.py | 63 +++++ node_cli/core/ssl/upload.py | 40 ++++ node_cli/core/ssl/utils.py | 63 +++++ node_cli/utils/helper.py | 8 + setup.py | 1 + 11 files changed, 361 insertions(+), 203 deletions(-) create mode 100644 node_cli/configs/ssl.py delete mode 100644 node_cli/core/host2/__init__.py create mode 100644 node_cli/core/ssl/__init__.py rename node_cli/core/{ssl.py => ssl/check.py} (68%) create mode 100644 node_cli/core/ssl/status.py create mode 100644 node_cli/core/ssl/upload.py create mode 100644 node_cli/core/ssl/utils.py diff --git a/node_cli/cli/ssl.py b/node_cli/cli/ssl.py index 2d0bfdfa..89b81b51 100644 --- a/node_cli/cli/ssl.py +++ b/node_cli/cli/ssl.py @@ -21,9 +21,9 @@ from terminaltables import SingleTable from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.helper import get_request, safe_load_texts, error_exit -from node_cli.configs import DEFAULT_SSL_CHECK_PORT, SSL_CERT_FILEPATH, SSL_KEY_FILEPATH -from node_cli.core.ssl import check_cert, upload_cert +from node_cli.utils.helper import safe_load_texts, error_exit +from node_cli.configs.ssl import DEFAULT_SSL_CHECK_PORT, SSL_CERT_FILEPATH, SSL_KEY_FILEPATH +from node_cli.core.ssl import check_cert, upload_cert, cert_status TEXTS = safe_load_texts() @@ -42,10 +42,7 @@ def ssl(): @ssl.command(help="Status of the SSL certificates on the node") def status(): - status, payload = get_request( - blueprint=BLUEPRINT_NAME, - method='status' - ) + status, payload = cert_status() if status == 'ok': if payload.get('is_empty'): print(TEXTS['ssl']['no_cert']) @@ -79,7 +76,7 @@ def upload(key_path, cert_path, force): if status == 'ok': print(TEXTS['ssl']['uploaded']) else: - error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + error_exit(payload, exit_code=CLIExitCodes.FAILURE) @ssl.command(help="Check certificates") diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 77d0310f..1ae0f1fa 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -107,8 +107,6 @@ def _get_env(): TEXT_FILE = os.path.join(PROJECT_DIR, 'text.yml') DATAFILES_FOLDER = os.path.join(PARDIR, 'datafiles') -SKALED_SSL_TEST_SCRIPT = os.path.join(DATAFILES_FOLDER, 'skaled-ssl-test') - ALLOCATION_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'schain_allocation.yml') REDIS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'redis-data') @@ -123,11 +121,6 @@ def _get_env(): BACKUP_ARCHIVE_NAME = 'skale-node-backup' -SSL_FOLDER_PATH = os.path.join(NODE_DATA_PATH, 'ssl') -SSL_CERT_FILEPATH = os.path.join(SSL_FOLDER_PATH, 'ssl_cert') -SSL_KEY_FILEPATH = os.path.join(SSL_FOLDER_PATH, 'ssl_key') - - TM_INIT_TIMEOUT = 20 RESTORE_SLEEP_TIMEOUT = 20 @@ -136,8 +129,6 @@ def _get_env(): META_FILEPATH = os.path.join(NODE_DATA_PATH, 'meta.json') -DEFAULT_SSL_CHECK_PORT = 4536 - SKALE_NODE_REPO_URL = 'https://github.com/skalenetwork/skale-node.git' DOCKER_LVMPY_REPO_URL = 'https://github.com/skalenetwork/docker-lvmpy.git' diff --git a/node_cli/configs/ssl.py b/node_cli/configs/ssl.py new file mode 100644 index 00000000..50528421 --- /dev/null +++ b/node_cli/configs/ssl.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +from configs import NODE_DATA_PATH, DATAFILES_FOLDER + + +DEFAULT_SSL_CHECK_PORT = 4536 +SKALED_SSL_TEST_SCRIPT = os.path.join(DATAFILES_FOLDER, 'skaled-ssl-test') + +SSL_FOLDER_PATH = os.path.join(NODE_DATA_PATH, 'ssl') +SSL_CERT_FILEPATH = os.path.join(SSL_FOLDER_PATH, 'ssl_cert') +SSL_KEY_FILEPATH = os.path.join(SSL_FOLDER_PATH, 'ssl_key') + +CERTS_UPLOADED_ERR_MSG = 'SSL Certificates are already uploaded' +CERTS_INVALID_FORMAT = 'Certificates have invalid format' diff --git a/node_cli/core/host2/__init__.py b/node_cli/core/host2/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/node_cli/core/ssl/__init__.py b/node_cli/core/ssl/__init__.py new file mode 100644 index 00000000..75ec1911 --- /dev/null +++ b/node_cli/core/ssl/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from .upload import upload_cert # noqa +from .check import check_cert # noqa +from .status import cert_status # noqa diff --git a/node_cli/core/ssl.py b/node_cli/core/ssl/check.py similarity index 68% rename from node_cli/core/ssl.py rename to node_cli/core/ssl/check.py index 38bfc73d..592bade4 100644 --- a/node_cli/core/ssl.py +++ b/node_cli/core/ssl/check.py @@ -1,98 +1,129 @@ -import json -import os -import socket -import subprocess -import time -from contextlib import contextmanager +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import time +import socket import logging +from contextlib import contextmanager -from node_cli.configs import ( +from node_cli.core.ssl.utils import detached_subprocess +from node_cli.configs.ssl import ( DEFAULT_SSL_CHECK_PORT, SKALED_SSL_TEST_SCRIPT, SSL_CERT_FILEPATH, SSL_KEY_FILEPATH ) -from node_cli.utils.helper import post_request -from node_cli.utils.helper import run_cmd - -logger = logging.getLogger(__name__) -COMMUNICATION_TIMEOUT = 3 - -def read_file_bytes(path, mode='rb'): - with open(path, mode) as f: - return f - - -def load_ssl_files(key_path, cert_path): - return { - 'ssl_key': (os.path.basename(key_path), - read_file_bytes(key_path), 'application/octet-stream'), - 'ssl_cert': (os.path.basename(cert_path), - read_file_bytes(cert_path), 'application/octet-stream') - } +logger = logging.getLogger(__name__) -def run_simple_openssl_server(certfile, keyfile, port=DEFAULT_SSL_CHECK_PORT): - cmd = [ - 'openssl', 's_server', - '-cert', certfile, - '-key', keyfile, - '-WWW', - '-port', str(port), - '-verify_return_error', '-Verify', '1' - ] - run_cmd(cmd) +def check_cert( + cert_path=SSL_CERT_FILEPATH, + key_path=SSL_KEY_FILEPATH, + port=DEFAULT_SSL_CHECK_PORT, + check_type='all', + no_client=False, + no_wss=False +): + if check_type in ('all', 'openssl'): + try: + check_cert_openssl( + cert_path, key_path, + host='127.0.0.1', port=port, no_client=no_client + ) + except Exception as err: + logger.exception('Cerificate/key pair is incorrect') + return 'error', f'Certificate check failed. {err}' + if check_type in ('all', 'skaled'): + try: + check_cert_skaled( + cert_path, key_path, + host='127.0.0.1', port=port, no_wss=no_wss + ) + except Exception as err: + logger.exception('Certificate/key pair is incorrect for skaled') + return 'error', f'Skaled ssl check failed. {err}' -@contextmanager -def detached_subprocess(cmd, expose_output=False): - logger.debug(f'Starting detached subprocess: {cmd}') - p = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - encoding='utf-8' - ) - try: - yield p - finally: - p.terminate() - output = p.stdout.read() - if expose_output: - print(output) - logger.debug(f'Detached process {cmd} output:\n{output}') + return 'ok', None -class SSLHealthcheckError(Exception): - pass +def check_cert_openssl( + cert_path, + key_path, + host='127.0.0.1', + port=DEFAULT_SSL_CHECK_PORT, + no_client=False, + silent=False +): + with openssl_server( + host, port, cert_path, + key_path, silent=silent + ) as serv: + time.sleep(1) + code = serv.poll() + if code is not None: + logger.error('Healthcheck server failed to start') + raise SSLHealthcheckError('OpenSSL server was failed to start') + logger.info('Server successfully started') -def check_endpoint(host, port): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - result = sock.connect_ex((host, port)) - logger.info('Checking healthcheck endpoint ...') - if result != 0: - logger.error('Port is closed') - return False - return True + # Connect to ssl server + if not no_client: + if not check_endpoint(host, port): + raise SSLHealthcheckError( + f'Healthcheck port is closed on {host}:{port}' + ) + check_ssl_connection(host, port, silent=silent) + logger.info('Healthcheck connection passed') -def check_ssl_connection(host, port, silent=False): - logger.info(f'Connecting to public ssl endpoint {host}:{port} ...') - ssl_check_cmd = [ - 'openssl', 's_client', - '-connect', f'{host}:{port}', - '-verify_return_error', '-verify', '2' +@contextmanager +def openssl_server(host, port, cert_path, key_path, silent=False): + ssl_server_cmd = [ + 'openssl', 's_server', + '-cert', cert_path, + '-cert_chain', cert_path, + '-key', key_path, + '-WWW', + '-accept', f'{host}:{port}', + '-verify_return_error', '-verify', '1' ] + logger.info(f'Staring healthcheck server on port {port} ...') expose_output = not silent - with detached_subprocess(ssl_check_cmd, expose_output=expose_output) as dp: - time.sleep(1) - code = dp.poll() - if code is not None: - logger.error('Healthcheck connection failed') - raise SSLHealthcheckError('OpenSSL connection verification failed') + with detached_subprocess( + ssl_server_cmd, expose_output=expose_output + ) as dp: + yield dp + + +def check_cert_skaled( + cert_path, + key_path, + host='127.0.0.1', + port=DEFAULT_SSL_CHECK_PORT, + no_wss=False +): + run_skaled_https_healthcheck(cert_path, key_path, host, port) + if not no_wss: + run_skaled_wss_healthcheck(cert_path, key_path, host, port) def run_skaled_https_healthcheck( @@ -146,122 +177,31 @@ def run_skaled_wss_healthcheck( logger.info('Skaled wss check server successfully started') -def check_cert_skaled( - cert_path, - key_path, - host='127.0.0.1', - port=DEFAULT_SSL_CHECK_PORT, - no_wss=False -): - run_skaled_https_healthcheck(cert_path, key_path, host, port) - if not no_wss: - run_skaled_wss_healthcheck(cert_path, key_path, host, port) +class SSLHealthcheckError(Exception): + pass -@contextmanager -def openssl_server(host, port, cert_path, key_path, silent=False): - ssl_server_cmd = [ - 'openssl', 's_server', - '-cert', cert_path, - '-cert_chain', cert_path, - '-key', key_path, - '-WWW', - '-accept', f'{host}:{port}', - '-verify_return_error', '-verify', '1' - ] - logger.info(f'Staring healthcheck server on port {port} ...') - expose_output = not silent - with detached_subprocess( - ssl_server_cmd, expose_output=expose_output - ) as dp: - yield dp +def check_endpoint(host, port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + result = sock.connect_ex((host, port)) + logger.info('Checking healthcheck endpoint ...') + if result != 0: + logger.error('Port is closed') + return False + return True -def check_cert_openssl( - cert_path, - key_path, - host='127.0.0.1', - port=DEFAULT_SSL_CHECK_PORT, - no_client=False, - silent=False -): - with openssl_server( - host, port, cert_path, - key_path, silent=silent - ) as serv: +def check_ssl_connection(host, port, silent=False): + logger.info(f'Connecting to public ssl endpoint {host}:{port} ...') + ssl_check_cmd = [ + 'openssl', 's_client', + '-connect', f'{host}:{port}', + '-verify_return_error', '-verify', '2' + ] + expose_output = not silent + with detached_subprocess(ssl_check_cmd, expose_output=expose_output) as dp: time.sleep(1) - code = serv.poll() + code = dp.poll() if code is not None: - logger.error('Healthcheck server failed to start') - raise SSLHealthcheckError('OpenSSL server was failed to start') - - logger.info('Server successfully started') - - # Connect to ssl server - if not no_client: - if not check_endpoint(host, port): - raise SSLHealthcheckError( - f'Healthcheck port is closed on {host}:{port}' - ) - check_ssl_connection(host, port, silent=silent) - logger.info('Healthcheck connection passed') - - -def send_saving_cert_request(key_path, cert_path, force): - with open(key_path, 'rb') as key_file, open(cert_path, 'rb') as cert_file: - files_data = { - 'ssl_key': (os.path.basename(key_path), key_file, - 'application/octet-stream'), - 'ssl_cert': (os.path.basename(cert_path), cert_file, - 'application/octet-stream') - } - files_data['json'] = ( - None, json.dumps({'force': force}), - 'application/json' - ) - return post_request( - blueprint='ssl', - method='upload', - files=files_data - ) - - -def upload_cert(cert_path, key_path, force, no_client=False): - try: - check_cert_openssl( - cert_path, key_path, silent=True, no_client=no_client) - except Exception as err: - logger.exception('Certificate/key pair is incorrect') - return 'error', f'Certificate check failed. {err}' - return send_saving_cert_request(key_path, cert_path, force) - - -def check_cert( - cert_path=SSL_CERT_FILEPATH, - key_path=SSL_KEY_FILEPATH, - port=DEFAULT_SSL_CHECK_PORT, - check_type='all', - no_client=False, - no_wss=False -): - if check_type in ('all', 'openssl'): - try: - check_cert_openssl( - cert_path, key_path, - host='127.0.0.1', port=port, no_client=no_client - ) - except Exception as err: - logger.exception('Cerificate/key pair is incorrect') - return 'error', f'Certificate check failed. {err}' - - if check_type in ('all', 'skaled'): - try: - check_cert_skaled( - cert_path, key_path, - host='127.0.0.1', port=port, no_wss=no_wss - ) - except Exception as err: - logger.exception('Certificate/key pair is incorrect for skaled') - return 'error', f'Skaled ssl check failed. {err}' - - return 'ok', None + logger.error('Healthcheck connection failed') + raise SSLHealthcheckError('OpenSSL connection verification failed') diff --git a/node_cli/core/ssl/status.py b/node_cli/core/ssl/status.py new file mode 100644 index 00000000..3f6d0c56 --- /dev/null +++ b/node_cli/core/ssl/status.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +from dateutil import parser + +from OpenSSL import crypto + +from node_cli.core.ssl.utils import is_ssl_folder_empty, cert_from_file +from node_cli.configs.ssl import SSL_CERT_FILEPATH, CERTS_INVALID_FORMAT +from node_cli.utils.helper import ok_result, err_result + +logger = logging.getLogger(__name__) + + +def cert_status(): + if is_ssl_folder_empty(): + return ok_result({'is_empty': True}) + + cert = cert_from_file(SSL_CERT_FILEPATH) + status, info = get_cert_info(cert) + if status == 'error': + return status, CERTS_INVALID_FORMAT + else: + return ok_result(payload={ + 'issued_to': info['issued_to'], + 'expiration_date': info['expiration_date'] + }) + + +def get_cert_info(cert): + try: + crypto_cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert) + subject = crypto_cert.get_subject() + issued_to = subject.CN + expiration_date_raw = crypto_cert.get_notAfter() + expiration_date = parser.parse( + expiration_date_raw + ).strftime('%Y-%m-%dT%H:%M:%S') + except Exception as err: + logger.exception('Error during parsing certs') + return err_result(str(err)) + return ok_result({ + 'subject': subject, + 'issued_to': issued_to, + 'expiration_date': expiration_date + }) diff --git a/node_cli/core/ssl/upload.py b/node_cli/core/ssl/upload.py new file mode 100644 index 00000000..af9759d4 --- /dev/null +++ b/node_cli/core/ssl/upload.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +from node_cli.configs.ssl import CERTS_UPLOADED_ERR_MSG + +from node_cli.core.ssl.check import check_cert_openssl +from node_cli.core.ssl.utils import is_ssl_folder_empty, save_cert_key_pair +from node_cli.utils.helper import ok_result + +logger = logging.getLogger(__name__) + + +def upload_cert(cert_path, key_path, force, no_client=False): + try: + check_cert_openssl( + cert_path, key_path, silent=True, no_client=no_client) + except Exception as err: + logger.exception('Certificate/key pair is incorrect') + return 'error', f'Certificate check failed. {err}' + if not is_ssl_folder_empty() and not force: + return 'error', CERTS_UPLOADED_ERR_MSG + save_cert_key_pair(key_path, cert_path) + return ok_result() diff --git a/node_cli/core/ssl/utils.py b/node_cli/core/ssl/utils.py new file mode 100644 index 00000000..293f69d7 --- /dev/null +++ b/node_cli/core/ssl/utils.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import logging +import subprocess +from contextlib import contextmanager +from node_cli.configs.ssl import SSL_CERT_FILEPATH, SSL_KEY_FILEPATH, SSL_FOLDER_PATH + + +logger = logging.getLogger(__name__) + + +def save_cert_key_pair(cert, key): + with open(SSL_CERT_FILEPATH, 'wb') as cert_file: + cert_file.write(cert) + with open(SSL_KEY_FILEPATH, 'wb') as key_file: + key_file.write(key) + + +def cert_from_file(cert_filepath): + if not os.path.isfile(cert_filepath): + return None + with open(cert_filepath) as cert_file: + return cert_file.read() + + +def is_ssl_folder_empty(ssl_path=SSL_FOLDER_PATH): + return len(os.listdir(ssl_path)) == 0 + + +@contextmanager +def detached_subprocess(cmd, expose_output=False): + logger.debug(f'Starting detached subprocess: {cmd}') + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + encoding='utf-8' + ) + try: + yield p + finally: + p.terminate() + output = p.stdout.read() + if expose_output: + print(output) + logger.debug(f'Detached process {cmd} output:\n{output}') diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 9d9b7523..6e43cc61 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -333,3 +333,11 @@ def rsync_dirs(src: str, dest: str) -> None: logger.info(f'Syncing {dest} with {src}') run_cmd(['rsync', '-r', f'{src}/', dest]) run_cmd(['rsync', '-r', f'{src}/.git', dest]) + + +def ok_result(payload: dict = None): + return 'ok', payload + + +def err_result(msg: str = None): + return 'error', msg diff --git a/setup.py b/setup.py index 1301bcba..9db38ade 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,7 @@ def find_version(*file_paths): "python-debian==0.1.39", "python-iptables==1.0.0", "MarkupSafe==2.0.1", + "pyOpenSSL==19.1.0" ], python_requires='>=3.6,<4', extras_require=extras_require, From c9fedfb8f4304edfde58097b84be0d18df84f912 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 3 Jun 2022 11:41:41 +0100 Subject: [PATCH 15/86] SKALE-779 Fix configs import --- node_cli/configs/ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/configs/ssl.py b/node_cli/configs/ssl.py index 50528421..670c3746 100644 --- a/node_cli/configs/ssl.py +++ b/node_cli/configs/ssl.py @@ -19,7 +19,7 @@ import os -from configs import NODE_DATA_PATH, DATAFILES_FOLDER +from node_cli.configs import NODE_DATA_PATH, DATAFILES_FOLDER DEFAULT_SSL_CHECK_PORT = 4536 From 0e21a8d63bace82b77b4879e9567d6f62bb07db8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 3 Jun 2022 13:01:52 +0100 Subject: [PATCH 16/86] Fix SSL import --- tests/core_ssl_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core_ssl_test.py b/tests/core_ssl_test.py index 318109b2..8e2ec0b0 100644 --- a/tests/core_ssl_test.py +++ b/tests/core_ssl_test.py @@ -4,7 +4,8 @@ import mock import pytest -from node_cli.core.ssl import check_cert_openssl, SSLHealthcheckError, upload_cert +from node_cli.core.ssl import upload_cert +from node_cli.core.ssl.check import check_cert_openssl, SSLHealthcheckError from node_cli.utils.helper import run_cmd From 2926055189be2d6eef9e7750e1e08259360bb873 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 3 Jun 2022 18:50:51 +0100 Subject: [PATCH 17/86] SKLAE-779 Update ssl test --- tests/core_ssl_test.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/tests/core_ssl_test.py b/tests/core_ssl_test.py index 8e2ec0b0..f6b14135 100644 --- a/tests/core_ssl_test.py +++ b/tests/core_ssl_test.py @@ -1,13 +1,12 @@ import os import pathlib -import mock import pytest from node_cli.core.ssl import upload_cert from node_cli.core.ssl.check import check_cert_openssl, SSLHealthcheckError from node_cli.utils.helper import run_cmd - +from node_cli.configs.ssl import SSL_CERT_FILEPATH, SSL_KEY_FILEPATH HOST = '127.0.0.1' @@ -73,23 +72,13 @@ def test_verify_cert_bad_key(bad_key): check_cert_openssl(cert, key, host=HOST, no_client=True) -@mock.patch('node_cli.core.ssl.post_request') -def test_upload_cert(pr_mock, cert_key_pair): +def test_upload_cert(cert_key_pair): cert, key = cert_key_pair + + assert not os.path.isfile(SSL_KEY_FILEPATH) + assert not os.path.isfile(SSL_CERT_FILEPATH) + upload_cert(cert, key, force=False, no_client=True) - # args = pr_mock.call_args.args - # assert args[0] == 'ssl_upload' - kwargs = pr_mock.call_args.kwargs - assert kwargs['files']['ssl_cert'][1].name == cert - assert kwargs['files']['ssl_key'][1].name == key - assert kwargs['files']['json'][1] == '{"force": false}' - - upload_cert(cert, key, force=True, no_client=True) - # args = pr_mock.call_args.args - # assert args[0] == 'ssl_upload' - kwargs = pr_mock.call_args.kwargs - assert kwargs['files']['ssl_cert'][1].name == cert - assert kwargs['files']['ssl_key'][1].name == key - assert kwargs['files']['json'][1] == '{"force": true}' - - assert pr_mock.call_count == 2 + + assert os.path.isfile(SSL_KEY_FILEPATH) + assert os.path.isfile(SSL_CERT_FILEPATH) From 2724a3ea3f70a95cfa80c5b30908f77055f13826 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 3 Jun 2022 19:59:33 +0100 Subject: [PATCH 18/86] SKALE-779 Fix SSL upload --- node_cli/core/ssl/upload.py | 5 +++-- node_cli/core/ssl/utils.py | 9 ++++----- tests/conftest.py | 11 +++++++++++ tests/core_ssl_test.py | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/node_cli/core/ssl/upload.py b/node_cli/core/ssl/upload.py index af9759d4..143d49b2 100644 --- a/node_cli/core/ssl/upload.py +++ b/node_cli/core/ssl/upload.py @@ -21,7 +21,7 @@ from node_cli.configs.ssl import CERTS_UPLOADED_ERR_MSG from node_cli.core.ssl.check import check_cert_openssl -from node_cli.core.ssl.utils import is_ssl_folder_empty, save_cert_key_pair +from node_cli.core.ssl.utils import is_ssl_folder_empty, copy_cert_key_pair from node_cli.utils.helper import ok_result logger = logging.getLogger(__name__) @@ -29,6 +29,7 @@ def upload_cert(cert_path, key_path, force, no_client=False): try: + pass check_cert_openssl( cert_path, key_path, silent=True, no_client=no_client) except Exception as err: @@ -36,5 +37,5 @@ def upload_cert(cert_path, key_path, force, no_client=False): return 'error', f'Certificate check failed. {err}' if not is_ssl_folder_empty() and not force: return 'error', CERTS_UPLOADED_ERR_MSG - save_cert_key_pair(key_path, cert_path) + copy_cert_key_pair(key_path, cert_path) return ok_result() diff --git a/node_cli/core/ssl/utils.py b/node_cli/core/ssl/utils.py index 293f69d7..ffab7ea0 100644 --- a/node_cli/core/ssl/utils.py +++ b/node_cli/core/ssl/utils.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import os +import shutil import logging import subprocess from contextlib import contextmanager @@ -27,11 +28,9 @@ logger = logging.getLogger(__name__) -def save_cert_key_pair(cert, key): - with open(SSL_CERT_FILEPATH, 'wb') as cert_file: - cert_file.write(cert) - with open(SSL_KEY_FILEPATH, 'wb') as key_file: - key_file.write(key) +def copy_cert_key_pair(cert, key): + shutil.copyfile(cert, SSL_CERT_FILEPATH) + shutil.copyfile(key, SSL_KEY_FILEPATH) def cert_from_file(cert_filepath): diff --git a/tests/conftest.py b/tests/conftest.py index db56e6a9..856fdf4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,6 +34,7 @@ ) from node_cli.utils.global_config import generate_g_config_file from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH +from node_cli.configs.ssl import SSL_FOLDER_PATH TEST_ENV_PARAMS = """ @@ -185,3 +186,13 @@ def resource_alloc(): json.dump({}, alloc_file) yield RESOURCE_ALLOCATION_FILEPATH os.remove(RESOURCE_ALLOCATION_FILEPATH) + + +@pytest.fixture +def ssl_folder(): + if os.path.isdir(SSL_FOLDER_PATH): + shutil.rmtree(SSL_FOLDER_PATH) + path = pathlib.Path(SSL_FOLDER_PATH) + path.mkdir(parents=True, exist_ok=True) + yield + shutil.rmtree(SSL_FOLDER_PATH) diff --git a/tests/core_ssl_test.py b/tests/core_ssl_test.py index f6b14135..5032b6b2 100644 --- a/tests/core_ssl_test.py +++ b/tests/core_ssl_test.py @@ -72,7 +72,7 @@ def test_verify_cert_bad_key(bad_key): check_cert_openssl(cert, key, host=HOST, no_client=True) -def test_upload_cert(cert_key_pair): +def test_upload_cert(ssl_folder, cert_key_pair): cert, key = cert_key_pair assert not os.path.isfile(SSL_KEY_FILEPATH) From 21c0e19b4ad0283c0456bedcfecf8b30aff58283 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 3 Jun 2022 20:09:22 +0100 Subject: [PATCH 19/86] SKALE-779 Fix SSL upload --- tests/conftest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 856fdf4e..b65e9deb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -190,9 +190,9 @@ def resource_alloc(): @pytest.fixture def ssl_folder(): - if os.path.isdir(SSL_FOLDER_PATH): + if os.path.isdir(SSL_FOLDER_PATH): + shutil.rmtree(SSL_FOLDER_PATH) + path = pathlib.Path(SSL_FOLDER_PATH) + path.mkdir(parents=True, exist_ok=True) + yield shutil.rmtree(SSL_FOLDER_PATH) - path = pathlib.Path(SSL_FOLDER_PATH) - path.mkdir(parents=True, exist_ok=True) - yield - shutil.rmtree(SSL_FOLDER_PATH) From e1610bf39f772100bc99d37cfb4487a2358a2628 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 6 Jun 2022 14:43:53 +0100 Subject: [PATCH 20/86] SKALE-799 Fix SSL upload --- node_cli/core/ssl/upload.py | 2 +- node_cli/core/ssl/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/node_cli/core/ssl/upload.py b/node_cli/core/ssl/upload.py index 143d49b2..35dbb9b0 100644 --- a/node_cli/core/ssl/upload.py +++ b/node_cli/core/ssl/upload.py @@ -37,5 +37,5 @@ def upload_cert(cert_path, key_path, force, no_client=False): return 'error', f'Certificate check failed. {err}' if not is_ssl_folder_empty() and not force: return 'error', CERTS_UPLOADED_ERR_MSG - copy_cert_key_pair(key_path, cert_path) + copy_cert_key_pair(cert_path, key_path) return ok_result() diff --git a/node_cli/core/ssl/utils.py b/node_cli/core/ssl/utils.py index ffab7ea0..a2e71e1f 100644 --- a/node_cli/core/ssl/utils.py +++ b/node_cli/core/ssl/utils.py @@ -35,6 +35,7 @@ def copy_cert_key_pair(cert, key): def cert_from_file(cert_filepath): if not os.path.isfile(cert_filepath): + logger.warning(f'Trying to read cert that does not exist: {cert_filepath}') return None with open(cert_filepath) as cert_file: return cert_file.read() From f0adc50a47e74ff7ffbcfffe25fa10cb53661881 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 7 Jun 2022 17:23:39 +0100 Subject: [PATCH 21/86] SKALE-779 Fix minor issues --- node_cli/core/ssl/check.py | 3 ++- node_cli/core/ssl/status.py | 2 +- node_cli/core/ssl/upload.py | 10 ++++------ tests/conftest.py | 6 ++++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/node_cli/core/ssl/check.py b/node_cli/core/ssl/check.py index 592bade4..8f498d15 100644 --- a/node_cli/core/ssl/check.py +++ b/node_cli/core/ssl/check.py @@ -23,6 +23,7 @@ from contextlib import contextmanager from node_cli.core.ssl.utils import detached_subprocess +from node_cli.utils.helper import err_result from node_cli.configs.ssl import ( DEFAULT_SSL_CHECK_PORT, SKALED_SSL_TEST_SCRIPT, @@ -50,7 +51,7 @@ def check_cert( ) except Exception as err: logger.exception('Cerificate/key pair is incorrect') - return 'error', f'Certificate check failed. {err}' + return err_result(f'Certificate check failed. {err}') if check_type in ('all', 'skaled'): try: diff --git a/node_cli/core/ssl/status.py b/node_cli/core/ssl/status.py index 3f6d0c56..31ab4b9a 100644 --- a/node_cli/core/ssl/status.py +++ b/node_cli/core/ssl/status.py @@ -36,7 +36,7 @@ def cert_status(): cert = cert_from_file(SSL_CERT_FILEPATH) status, info = get_cert_info(cert) if status == 'error': - return status, CERTS_INVALID_FORMAT + return err_result(CERTS_INVALID_FORMAT) else: return ok_result(payload={ 'issued_to': info['issued_to'], diff --git a/node_cli/core/ssl/upload.py b/node_cli/core/ssl/upload.py index 35dbb9b0..903b0b73 100644 --- a/node_cli/core/ssl/upload.py +++ b/node_cli/core/ssl/upload.py @@ -22,20 +22,18 @@ from node_cli.core.ssl.check import check_cert_openssl from node_cli.core.ssl.utils import is_ssl_folder_empty, copy_cert_key_pair -from node_cli.utils.helper import ok_result +from node_cli.utils.helper import ok_result, err_result logger = logging.getLogger(__name__) def upload_cert(cert_path, key_path, force, no_client=False): try: - pass - check_cert_openssl( - cert_path, key_path, silent=True, no_client=no_client) + check_cert_openssl(cert_path, key_path, silent=True, no_client=no_client) except Exception as err: logger.exception('Certificate/key pair is incorrect') - return 'error', f'Certificate check failed. {err}' + return err_result(f'Certificate check failed. {err}') if not is_ssl_folder_empty() and not force: - return 'error', CERTS_UPLOADED_ERR_MSG + return err_result(CERTS_UPLOADED_ERR_MSG) copy_cert_key_pair(cert_path, key_path) return ok_result() diff --git a/tests/conftest.py b/tests/conftest.py index b65e9deb..07a4e5af 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -194,5 +194,7 @@ def ssl_folder(): shutil.rmtree(SSL_FOLDER_PATH) path = pathlib.Path(SSL_FOLDER_PATH) path.mkdir(parents=True, exist_ok=True) - yield - shutil.rmtree(SSL_FOLDER_PATH) + try: + yield + finally: + shutil.rmtree(SSL_FOLDER_PATH) From 25cebac1dfded1d26a0cf1b45a4c619e15b09bca Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 7 Jun 2022 18:16:19 +0100 Subject: [PATCH 22/86] Fix flake --- tests/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 07a4e5af..7e2638e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,8 +29,8 @@ import yaml from node_cli.configs import ( - ENVIRONMENT_PARAMS_FILEPATH, GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH, - REMOVED_CONTAINERS_FOLDER_PATH + ENVIRONMENT_PARAMS_FILEPATH, GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH, + REMOVED_CONTAINERS_FOLDER_PATH ) from node_cli.utils.global_config import generate_g_config_file from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH @@ -195,6 +195,6 @@ def ssl_folder(): path = pathlib.Path(SSL_FOLDER_PATH) path.mkdir(parents=True, exist_ok=True) try: - yield + yield finally: - shutil.rmtree(SSL_FOLDER_PATH) + shutil.rmtree(SSL_FOLDER_PATH) From 3bdd0eb5905b33cc4b4cda9432998806984cc744 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 8 Jun 2022 18:47:23 +0100 Subject: [PATCH 23/86] Hotfix: add version and build info commands to sync-node-cli --- node_cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/main.py b/node_cli/main.py index 987cb34e..3e9d4f38 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -77,7 +77,7 @@ def info(): def get_sources_list(): if TYPE == 'sync': - return [sync_node_cli, ssl_cli] + return [cli, sync_node_cli, ssl_cli] else: return [ cli, From 76596ff85f0457e94aa0545f673ad16d7be61d3b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 9 Jun 2022 19:26:49 +0100 Subject: [PATCH 24/86] SKALE-779 Add firwall setup for sync node --- node_cli/core/node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 1bbb2f7f..7ce9344d 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -141,6 +141,7 @@ def init(env_filepath): @check_not_inited def init_sync(env_filepath: str) -> None: + configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) if env is None: return @@ -164,6 +165,7 @@ def init_sync(env_filepath: str) -> None: @check_user def update_sync(env_filepath): logger.info('Node update started') + configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) update_ok = update_sync_op(env_filepath, env) if update_ok: From d9f4334fffe347961cd39618c67ac4c4fdf6906a Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 9 Jun 2022 19:42:18 +0100 Subject: [PATCH 25/86] Add cryptography package, update pyOpenSSL --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9db38ade..208eaba9 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,8 @@ def find_version(*file_paths): "python-debian==0.1.39", "python-iptables==1.0.0", "MarkupSafe==2.0.1", - "pyOpenSSL==19.1.0" + "pyOpenSSL==22.0.0", + "cryptography==37.0.2" ], python_requires='>=3.6,<4', extras_require=extras_require, From 1a30cc3a34cc0e086f745fe720f2b9ab49fa2d79 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 9 Jun 2022 19:45:45 +0100 Subject: [PATCH 26/86] Mock configure_firewall_rules in tests --- tests/cli/sync_node_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 9483f7c9..48a93ded 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -41,6 +41,7 @@ def test_init_sync(mocked_g_config): mock.patch('node_cli.core.node.init_sync_op'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + mock.patch('node_cli.core.node.configure_firewall_rules'), \ mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( _init_sync, @@ -55,6 +56,7 @@ def test_update_sync(mocked_g_config): mock.patch('node_cli.core.node.update_sync_op'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + mock.patch('node_cli.core.node.configure_firewall_rules'), \ mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command( _update_sync, From c50c0f1366905845915315af2ab87a17e263a1e0 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 21 Jun 2022 18:12:20 +0100 Subject: [PATCH 27/86] Hotfix: add nginx reload to ssl upload --- .gitignore | 4 ++- node_cli/configs/__init__.py | 1 + node_cli/core/nginx.py | 29 ++++++++++++++++++- node_cli/core/ssl/upload.py | 3 ++ node_cli/utils/docker_utils.py | 10 +++++-- tests/.skale/config/nginx.conf.j2 | 47 +++++++++++++++++++++++++++++++ tests/conftest.py | 33 +++++++++++++++++++++- tests/core_ssl_test.py | 13 ++++++++- 8 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 tests/.skale/config/nginx.conf.j2 diff --git a/.gitignore b/.gitignore index db6f3fd0..de02cf68 100644 --- a/.gitignore +++ b/.gitignore @@ -117,4 +117,6 @@ disk_mountpoint.txt sgx_server_url.txt resource_allocation.json conf.json -test-env \ No newline at end of file +test-env + +nginx.conf \ No newline at end of file diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 1ae0f1fa..4de87991 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -56,6 +56,7 @@ CONTAINER_CONFIG_PATH, 'environment_params.yaml') NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') NGINX_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, 'nginx.conf') +NGINX_CONTAINER_NAME = 'skale_nginx' LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' diff --git a/node_cli/core/nginx.py b/node_cli/core/nginx.py index d3d9b60f..e87b17db 100644 --- a/node_cli/core/nginx.py +++ b/node_cli/core/nginx.py @@ -1,17 +1,38 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import logging import os.path +from node_cli.utils.docker_utils import restart_nginx_container, docker_client from node_cli.configs import NODE_CERTS_PATH, NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH from node_cli.utils.helper import process_template logger = logging.getLogger(__name__) + SSL_KEY_NAME = 'ssl_key' SSL_CRT_NAME = 'ssl_cert' -def generate_nginx_config(): +def generate_nginx_config() -> None: ssl_on = check_ssl_certs() template_data = { 'ssl': ssl_on, @@ -24,3 +45,9 @@ def check_ssl_certs(): crt_path = os.path.join(NODE_CERTS_PATH, SSL_CRT_NAME) key_path = os.path.join(NODE_CERTS_PATH, SSL_KEY_NAME) return os.path.exists(crt_path) and os.path.exists(key_path) + + +def reload_nginx() -> None: + dutils = docker_client() + generate_nginx_config() + restart_nginx_container(dutils=dutils) diff --git a/node_cli/core/ssl/upload.py b/node_cli/core/ssl/upload.py index 903b0b73..5da9b20d 100644 --- a/node_cli/core/ssl/upload.py +++ b/node_cli/core/ssl/upload.py @@ -23,6 +23,8 @@ from node_cli.core.ssl.check import check_cert_openssl from node_cli.core.ssl.utils import is_ssl_folder_empty, copy_cert_key_pair from node_cli.utils.helper import ok_result, err_result +from node_cli.core.nginx import reload_nginx + logger = logging.getLogger(__name__) @@ -36,4 +38,5 @@ def upload_cert(cert_path, key_path, force, no_client=False): if not is_ssl_folder_empty() and not force: return err_result(CERTS_UPLOADED_ERR_MSG) copy_cert_key_pair(cert_path, key_path) + reload_nginx() return ok_result() diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 437b61f0..5ed77182 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -32,7 +32,8 @@ SYNC_COMPOSE_PATH, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTIFICATES_DIR_NAME, - SKALE_DIR + SKALE_DIR, + NGINX_CONTAINER_NAME ) @@ -74,7 +75,6 @@ def get_sanitized_container_name(container_info: dict) -> str: def get_containers(container_name_filter=None, _all=True) -> list: return docker_client().containers.list(all=_all) - return docker_client().containers.list(all=_all, filters={'name': container_name_filter}) def get_all_schain_containers(_all=True) -> list: @@ -250,3 +250,9 @@ def compose_up(env, sync_node=False): def compose_up_sync(env) -> None: logger.info('Running containers for sync node') run_cmd(cmd=get_up_compose_sync_cmd(), env=env) + + +def restart_nginx_container(dutils=None): + dutils = dutils or docker_client() + nginx_container = dutils.containers.get(NGINX_CONTAINER_NAME) + nginx_container.restart() diff --git a/tests/.skale/config/nginx.conf.j2 b/tests/.skale/config/nginx.conf.j2 new file mode 100644 index 00000000..dc264362 --- /dev/null +++ b/tests/.skale/config/nginx.conf.j2 @@ -0,0 +1,47 @@ +limit_req_zone $binary_remote_addr zone=one:10m rate=7r/s; + +server { + listen 3009; + + {% if ssl %} + listen 311 ssl; + ssl_certificate /ssl/ssl_cert; + ssl_certificate_key /ssl/ssl_key; + {% endif %} + + proxy_read_timeout 500s; + proxy_connect_timeout 500s; + proxy_send_timeout 500s; + + error_log /var/log/nginx/error.log warn; + client_max_body_size 20m; + + server_name localhost; + limit_req zone=one burst=10; + + location / { + include uwsgi_params; + uwsgi_read_timeout 500s; + uwsgi_socket_keepalive on; + uwsgi_pass 127.0.0.1:3010; + } +} + +server { + listen 80; + + {% if ssl %} + listen 443 ssl; + ssl_certificate /ssl/ssl_cert; + ssl_certificate_key /ssl/ssl_key; + {% endif %} + + error_log /var/log/nginx/error.log warn; + client_max_body_size 20m; + server_name localhost; + limit_req zone=one burst=50; + + location / { + root /filestorage; + } +} \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 7e2638e4..449ec58b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,11 +30,12 @@ from node_cli.configs import ( ENVIRONMENT_PARAMS_FILEPATH, GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH, - REMOVED_CONTAINERS_FOLDER_PATH + REMOVED_CONTAINERS_FOLDER_PATH, NGINX_CONTAINER_NAME ) from node_cli.utils.global_config import generate_g_config_file from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.ssl import SSL_FOLDER_PATH +from node_cli.utils.docker_utils import docker_client TEST_ENV_PARAMS = """ @@ -198,3 +199,33 @@ def ssl_folder(): yield finally: shutil.rmtree(SSL_FOLDER_PATH) + + +@pytest.fixture +def dutils(): + return docker_client() + + +@pytest.fixture +def nginx_container(dutils, ssl_folder): + c = None + try: + c = dutils.containers.run( + 'nginx:1.20.2', + name=NGINX_CONTAINER_NAME, + detach=True, + volumes={ + ssl_folder: { + 'bind': '/ssl', + 'mode': 'ro', + 'propagation': 'slave' + } + } + ) + yield c + finally: + if c is not None: + try: + c.remove(force=True) + except Exception: + pass diff --git a/tests/core_ssl_test.py b/tests/core_ssl_test.py index 5032b6b2..a7b3eaff 100644 --- a/tests/core_ssl_test.py +++ b/tests/core_ssl_test.py @@ -1,5 +1,6 @@ import os import pathlib +from docker import APIClient import pytest @@ -7,6 +8,8 @@ from node_cli.core.ssl.check import check_cert_openssl, SSLHealthcheckError from node_cli.utils.helper import run_cmd from node_cli.configs.ssl import SSL_CERT_FILEPATH, SSL_KEY_FILEPATH +from node_cli.configs import NGINX_CONTAINER_NAME + HOST = '127.0.0.1' @@ -72,9 +75,14 @@ def test_verify_cert_bad_key(bad_key): check_cert_openssl(cert, key, host=HOST, no_client=True) -def test_upload_cert(ssl_folder, cert_key_pair): +def test_upload_cert(cert_key_pair, nginx_container, dutils): cert, key = cert_key_pair + docker_api = APIClient() + nginx_container = dutils.containers.get(NGINX_CONTAINER_NAME) + stats = docker_api.inspect_container(nginx_container.id) + started_at = stats['State']['StartedAt'] + assert not os.path.isfile(SSL_KEY_FILEPATH) assert not os.path.isfile(SSL_CERT_FILEPATH) @@ -82,3 +90,6 @@ def test_upload_cert(ssl_folder, cert_key_pair): assert os.path.isfile(SSL_KEY_FILEPATH) assert os.path.isfile(SSL_CERT_FILEPATH) + + stats = docker_api.inspect_container(nginx_container.id) + assert started_at != stats['State']['StartedAt'] From 64c3bdcb5c44c13f62c283d5647505caff1f3980 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 19 Jul 2022 18:03:24 +0000 Subject: [PATCH 28/86] Create btrfs volume without lvmpy for syncnode --- node_cli/operations/base.py | 8 ++++++-- node_cli/operations/docker_lvmpy.py | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index b55593c6..4d03dd23 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -33,7 +33,11 @@ backup_old_contracts, download_contracts, configure_filebeat, configure_flask, unpack_backup_archive ) -from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install +from node_cli.operations.docker_lvmpy import ( + docker_lvmpy_update, + docker_lvmpy_install, + prepare_device +) from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables @@ -166,7 +170,7 @@ def init_sync(env_filepath: str, env: str) -> bool: download_contracts(env) generate_nginx_config() - docker_lvmpy_install(env) + prepare_device(env['DISK_MOUNTPOINT']) update_meta( VERSION, diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index 00f2deb2..3b04b602 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -23,8 +23,13 @@ from node_cli.utils.helper import run_cmd from node_cli.utils.git_utils import sync_repo -from node_cli.configs import (DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL, - FILESTORAGE_MAPPING, SCHAINS_MNT_DIR) +from node_cli.configs import ( + DOCKER_LVMPY_PATH, + DOCKER_LVMPY_REPO_URL, + FILESTORAGE_MAPPING, + SCHAINS_MNT_DIR, + SKALE_STATE_DIR +) logger = logging.getLogger(__name__) @@ -74,3 +79,10 @@ def docker_lvmpy_install(env): env=env ) logger.info('docker-lvmpy installed') + + +def prepare_device(block_device): + run_cmd(['mkfs.btrfs', block_device]) + mountpoint = os.path.join(SKALE_STATE_DIR, 'schains') + os.makedirs(mountpoint) + run_cmd(['mount', block_device, mountpoint]) From 395d9e6e64a9e56a3de2c5f19dc243ef7b255f2f Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 21 Jul 2022 16:29:29 +0000 Subject: [PATCH 29/86] Fix error message for malformed env config --- node_cli/core/node.py | 8 ++-- node_cli/operations/base.py | 10 ++--- .../operations/{docker_lvmpy.py => volume.py} | 39 +++++++++++++++++-- node_cli/utils/helper.py | 8 +++- 4 files changed, 52 insertions(+), 13 deletions(-) rename node_cli/operations/{docker_lvmpy.py => volume.py} (68%) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 7ce9344d..2794bac9 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -201,9 +201,11 @@ def restore(backup_path, env_filepath): def get_node_env(env_filepath, inited_node=False, sync_schains=None, sync_node=False): if env_filepath is not None: - env_params = extract_env_params(env_filepath, sync_node=sync_node) - if env_params is None: - return + env_params = extract_env_params( + env_filepath, + sync_node=sync_node, + raise_for_status=True + ) save_env_params(env_filepath) else: env_params = extract_env_params(INIT_ENV_FILEPATH, sync_node=sync_node) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 4d03dd23..c575907d 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -33,10 +33,10 @@ backup_old_contracts, download_contracts, configure_filebeat, configure_flask, unpack_backup_archive ) -from node_cli.operations.docker_lvmpy import ( +from node_cli.operations.volume import ( docker_lvmpy_update, docker_lvmpy_install, - prepare_device + prepare_block_device ) from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images from node_cli.core.checks import CheckType, run_checks as run_host_checks @@ -154,7 +154,7 @@ def init(env_filepath: str, env: str) -> bool: def init_sync(env_filepath: str, env: str) -> bool: download_skale_node( - env['CONTAINER_CONFIGS_STREAM'], + env.get('CONTAINER_CONFIGS_STREAM'), env.get('CONTAINER_CONFIGS_DIR') ) sync_skale_node() @@ -170,7 +170,7 @@ def init_sync(env_filepath: str, env: str) -> bool: download_contracts(env) generate_nginx_config() - prepare_device(env['DISK_MOUNTPOINT']) + prepare_block_device(env['DISK_MOUNTPOINT']) update_meta( VERSION, @@ -199,7 +199,7 @@ def update_sync(env_filepath: str, env: Dict) -> None: backup_old_contracts() download_contracts(env) - docker_lvmpy_update(env) + prepare_block_device(env['DISK_MOUNTPOINT']) generate_nginx_config() prepare_host( diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/volume.py similarity index 68% rename from node_cli/operations/docker_lvmpy.py rename to node_cli/operations/volume.py index 3b04b602..8da785d7 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/volume.py @@ -20,6 +20,7 @@ import logging import os import shutil +import tempfile from node_cli.utils.helper import run_cmd from node_cli.utils.git_utils import sync_repo @@ -34,6 +35,10 @@ logger = logging.getLogger(__name__) +class FilesystemExistsError(Exception): + pass + + def update_docker_lvmpy_env(env): env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] env['VOLUME_GROUP'] = 'schains' @@ -81,8 +86,34 @@ def docker_lvmpy_install(env): logger.info('docker-lvmpy installed') -def prepare_device(block_device): - run_cmd(['mkfs.btrfs', block_device]) - mountpoint = os.path.join(SKALE_STATE_DIR, 'schains') - os.makedirs(mountpoint) +def get_block_device_filesystem(block_device): + with tempfile.TemporaryDirectory(dir=SKALE_STATE_DIR) as tempdir: + try: + run_cmd(['mount', block_device, tempdir.path]) + r = run_cmd(['stat', '-f', '-c', '%T']) + return r.stdout.decode('utf-8') + finally: + run_cmd(['umount', block_device]) + + +def format_as_btrfs(block_device): + filesystem = get_block_device_filesystem(block_device) + logger.info('Found filesystem on the %s: %s', block_device, filesystem) + if filesystem == 'btrfs': + logger.info('Formatting %s as btrfs', block_device) + run_cmd(['mkfs.btrfs', block_device]) + else: + raise FilesystemExistsError(f'{block_device} contains {filesystem}') + + +def mount_device(block_device, mountpoint): + os.makedirs(mountpoint, exists_ok=True) + logger.info('Mounting %s as btrfs', block_device) run_cmd(['mount', block_device, mountpoint]) + + +def prepare_block_device(block_device): + mountpoint = os.path.join(SKALE_STATE_DIR, 'schains') + if not os.path.ismount(mountpoint): + format_as_btrfs(block_device) + mount_device(block_device, mountpoint) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 4a6bd7ad..17e7b10d 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -71,6 +71,10 @@ } +class InvalidEnvFileError(Exception): + pass + + def read_json(path): with open(path, encoding='utf-8') as data_file: return json.loads(data_file.read()) @@ -141,7 +145,7 @@ def get_username(): return os.environ.get('USERNAME') or os.environ.get('USER') -def extract_env_params(env_filepath, sync_node=False): +def extract_env_params(env_filepath, sync_node=False, raise_for_status=True): env_params = get_env_config(env_filepath, sync_node=sync_node) absent_params = ', '.join(absent_env_params(env_params)) if absent_params: @@ -150,6 +154,8 @@ def extract_env_params(env_filepath, sync_node=False): f"You should specify them to make sure that " f"all services are working", err=True) + if raise_for_status: + raise InvalidEnvFileError(f'Missing params: {absent_params}') return None return env_params From 08ff20aa289e66bd1734aeb68c8e408960780453 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 25 Jul 2022 17:05:39 +0000 Subject: [PATCH 30/86] Add command to resize block device --- node_cli/cli/sync_node.py | 8 +++- node_cli/core/host.py | 4 +- node_cli/core/node.py | 15 ++++++- node_cli/operations/base.py | 4 +- node_cli/operations/volume.py | 73 ++++++++++++++++++++++++++--------- tests/helper.py | 4 ++ 6 files changed, 85 insertions(+), 23 deletions(-) diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index ce94c8be..7ac480bf 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -19,7 +19,7 @@ import click -from node_cli.core.node import init_sync, update_sync +from node_cli.core.node import init_sync, resize_filesystem, update_sync from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd @@ -51,3 +51,9 @@ def _init_sync(env_file): @streamed_cmd def _update_sync(env_file): update_sync(env_file) + + +@sync_node.command(help='Update sync node from .env file') +@click.argument('block_device') +def extend_fs(block_device): + resize_filesystem(block_device) diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 84151f82..23c93518 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -33,7 +33,7 @@ SCHAINS_DATA_PATH, LOG_PATH, REMOVED_CONTAINERS_FOLDER_PATH, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH, - SKALE_RUN_DIR, SKALE_TMP_DIR + SKALE_RUN_DIR, SKALE_STATE_DIR, SKALE_TMP_DIR ) from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH @@ -92,7 +92,7 @@ def make_dirs(): REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REPORTS_PATH, REDIS_DATA_PATH, - SKALE_RUN_DIR, SKALE_TMP_DIR + SKALE_RUN_DIR, SKALE_STATE_DIR, SKALE_TMP_DIR ): safe_mkdir(dir_path) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 2794bac9..2ddcd09c 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -50,8 +50,15 @@ from node_cli.core.checks import run_checks as run_host_checks from node_cli.core.resources import update_resource_allocation from node_cli.operations import ( - update_op, init_op, turn_off_op, turn_on_op, restore_op, init_sync_op, update_sync_op + update_op, + init_op, + turn_off_op, + turn_on_op, + restore_op, + init_sync_op, + update_sync_op ) +from node_cli.operations.volume import ensure_btrfs_for_all_space from node_cli.utils.print_formatters import ( print_failed_requirements_checks, print_node_cmd_error, print_node_info ) @@ -451,3 +458,9 @@ def configure_firewall_rules() -> None: print('Configuring firewall ...') configure_iptables() print('Done') + + +def resize_filesystem(block_device: str) -> None: + print('Resizing fs') + ensure_btrfs_for_all_space(block_device) + print('Done') diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index c575907d..cfe19186 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -34,6 +34,7 @@ configure_flask, unpack_backup_archive ) from node_cli.operations.volume import ( + cleanup_volume_artifacts, docker_lvmpy_update, docker_lvmpy_install, prepare_block_device @@ -153,6 +154,7 @@ def init(env_filepath: str, env: str) -> bool: def init_sync(env_filepath: str, env: str) -> bool: + cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node( env.get('CONTAINER_CONFIGS_STREAM'), env.get('CONTAINER_CONFIGS_DIR') @@ -186,7 +188,7 @@ def init_sync(env_filepath: str, env: str) -> bool: def update_sync(env_filepath: str, env: Dict) -> None: compose_rm(env, sync_node=True) remove_dynamic_containers() - + cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node( env['CONTAINER_CONFIGS_STREAM'], env.get('CONTAINER_CONFIGS_DIR') diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index 8da785d7..fcfa1c60 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import glob import logging import os import shutil @@ -86,34 +87,70 @@ def docker_lvmpy_install(env): logger.info('docker-lvmpy installed') +def block_device_mounted(block_device): + with open('/proc/mounts') as mounts: + return any(block_device in mount for mount in mounts.readlines()) + + +def ensure_not_mounted(block_device): + logger.info('Making sure %s is not mounted', block_device) + if block_device_mounted(block_device): + run_cmd(['umount', block_device]) + + +def cleanup_static_path(filestorage_mapping=FILESTORAGE_MAPPING): + logger.info('Removing all links from filestorage mapping') + for dir_link in glob.glob(os.path.join(filestorage_mapping, '*')): + logger.debug('Unlinking %s', dir_link) + if os.path.islink(dir_link): + os.unlink(dir_link) + + +def cleanup_volume_artifacts(block_device): + ensure_not_mounted(block_device) + cleanup_static_path() + + def get_block_device_filesystem(block_device): - with tempfile.TemporaryDirectory(dir=SKALE_STATE_DIR) as tempdir: - try: - run_cmd(['mount', block_device, tempdir.path]) - r = run_cmd(['stat', '-f', '-c', '%T']) - return r.stdout.decode('utf-8') - finally: - run_cmd(['umount', block_device]) + r = run_cmd(['blkid', '-o', 'udev', block_device]) + output = r.stdout.decode('utf-8') + logger.debug('blkid output %s', output) + fs_line = next(filter(lambda s: s.startswith('ID_FS_TYPE'), output.split('\n'))) + return fs_line.split('=')[1] def format_as_btrfs(block_device): - filesystem = get_block_device_filesystem(block_device) - logger.info('Found filesystem on the %s: %s', block_device, filesystem) - if filesystem == 'btrfs': - logger.info('Formatting %s as btrfs', block_device) - run_cmd(['mkfs.btrfs', block_device]) - else: - raise FilesystemExistsError(f'{block_device} contains {filesystem}') + logger.info('Formating %s as btrfs', block_device) + run_cmd(['mkfs.btrfs', block_device, '-f']) def mount_device(block_device, mountpoint): - os.makedirs(mountpoint, exists_ok=True) + os.makedirs(mountpoint, exist_ok=True) logger.info('Mounting %s as btrfs', block_device) run_cmd(['mount', block_device, mountpoint]) def prepare_block_device(block_device): - mountpoint = os.path.join(SKALE_STATE_DIR, 'schains') - if not os.path.ismount(mountpoint): + filesystem = get_block_device_filesystem(block_device) + if filesystem == 'btrfs': + logger.info('%s already formatted as btrfs', block_device) + ensure_btrfs_for_all_space(block_device) + else: + logger.info('%s contains %s filesystem', block_device, filesystem) format_as_btrfs(block_device) - mount_device(block_device, mountpoint) + mountpoint = os.path.join(SKALE_STATE_DIR, 'schains') + mount_device(block_device, mountpoint) + + +def max_resize_btrfs(path): + run_cmd(['btrfs', 'filesystem', 'resize', 'max', path]) + + +def ensure_btrfs_for_all_space(block_device): + with tempfile.TemporaryDirectory(dir=SKALE_STATE_DIR) as mountpoint: + try: + mount_device(block_device, mountpoint) + logger.info('Resizing btrfs filesystem for %s', block_device) + max_resize_btrfs(mountpoint) + finally: + ensure_not_mounted(block_device) diff --git a/tests/helper.py b/tests/helper.py index c4b66516..c1d4ab7a 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -18,9 +18,13 @@ # along with this program. If not, see . import mock +import os + from click.testing import CliRunner from mock import Mock, MagicMock +BLOCK_DEVICE = os.getenv('BLOCK_DEVICE') + def response_mock(status_code=0, json_data=None, headers=None, raw=None): From bb77dece09030d11529e823f16274414484b3653 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 25 Jul 2022 17:50:56 +0000 Subject: [PATCH 31/86] Improve logs --- node_cli/operations/volume.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index fcfa1c60..4973f895 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -126,7 +126,7 @@ def format_as_btrfs(block_device): def mount_device(block_device, mountpoint): os.makedirs(mountpoint, exist_ok=True) - logger.info('Mounting %s as btrfs', block_device) + logger.info('Mounting %s device', block_device) run_cmd(['mount', block_device, mountpoint]) From a414a48f6a1a23255b4b391198aee056a3281e6b Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 28 Jul 2022 13:25:30 +0000 Subject: [PATCH 32/86] Handle missing filesystem --- node_cli/configs/env.py | 3 ++- node_cli/operations/base.py | 10 ++++++++-- node_cli/operations/volume.py | 12 ++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index a4c33f0d..7ee9c628 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -43,7 +43,8 @@ 'DEFAULT_GAS_LIMIT': '', 'DEFAULT_GAS_PRICE_WEI': '', 'DISABLE_IMA': '', - 'SKIP_DOCKER_CONFIG': '' + 'SKIP_DOCKER_CONFIG': '', + 'ENFORCE_BTRFS': '' } diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index cfe19186..7039468c 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -172,7 +172,10 @@ def init_sync(env_filepath: str, env: str) -> bool: download_contracts(env) generate_nginx_config() - prepare_block_device(env['DISK_MOUNTPOINT']) + prepare_block_device( + env['DISK_MOUNTPOINT'], + force=env['ENFORCE_BTRFS'] == 'True' + ) update_meta( VERSION, @@ -201,7 +204,10 @@ def update_sync(env_filepath: str, env: Dict) -> None: backup_old_contracts() download_contracts(env) - prepare_block_device(env['DISK_MOUNTPOINT']) + prepare_block_device( + env['DISK_MOUNTPOINT'], + force=env['ENFORCE_BTRFS'] == 'True' + ) generate_nginx_config() prepare_host( diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index 4973f895..f5cc2672 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -130,8 +130,16 @@ def mount_device(block_device, mountpoint): run_cmd(['mount', block_device, mountpoint]) -def prepare_block_device(block_device): - filesystem = get_block_device_filesystem(block_device) +def prepare_block_device(block_device, force=False): + filesystem = None + try: + filesystem = get_block_device_filesystem(block_device) + except Exception as e: + logger.info('Cannot get filesystem type %s', e) + logger.debug('Cannot get filesystem type', exc_info=True) + if not force: + raise + if filesystem == 'btrfs': logger.info('%s already formatted as btrfs', block_device) ensure_btrfs_for_all_space(block_device) From 25149bff5b40fa795cae6e6afe7ef04da0262080 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 18 Aug 2022 17:38:47 +0000 Subject: [PATCH 33/86] Remove redundant function --- node_cli/cli/sync_node.py | 8 +------- node_cli/core/node.py | 7 ------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index 7ac480bf..ce94c8be 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -19,7 +19,7 @@ import click -from node_cli.core.node import init_sync, resize_filesystem, update_sync +from node_cli.core.node import init_sync, update_sync from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd @@ -51,9 +51,3 @@ def _init_sync(env_file): @streamed_cmd def _update_sync(env_file): update_sync(env_file) - - -@sync_node.command(help='Update sync node from .env file') -@click.argument('block_device') -def extend_fs(block_device): - resize_filesystem(block_device) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 2ddcd09c..add0ed64 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -58,7 +58,6 @@ init_sync_op, update_sync_op ) -from node_cli.operations.volume import ensure_btrfs_for_all_space from node_cli.utils.print_formatters import ( print_failed_requirements_checks, print_node_cmd_error, print_node_info ) @@ -458,9 +457,3 @@ def configure_firewall_rules() -> None: print('Configuring firewall ...') configure_iptables() print('Done') - - -def resize_filesystem(block_device: str) -> None: - print('Resizing fs') - ensure_btrfs_for_all_space(block_device) - print('Done') From 8a37aa9529f64919af208c38e876ffcbcc8555db Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 22 Sep 2022 15:08:09 +0000 Subject: [PATCH 34/86] Publish builds for beta-sync-node --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d83f552c..efb1b493 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,6 +8,7 @@ on: - beta - stable - sync-node + - beta-sync-node jobs: create_release: From 79effcc72cf1c86f886d6bce4b6f9f37c14f7597 Mon Sep 17 00:00:00 2001 From: badrogger Date: Sat, 24 Sep 2022 17:06:59 +0000 Subject: [PATCH 35/86] Fix filestorage mapping --- node_cli/operations/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index b7700ba5..b7c6e88a 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -37,6 +37,7 @@ cleanup_volume_artifacts, docker_lvmpy_update, docker_lvmpy_install, + ensure_filestorage_mapping, prepare_block_device ) from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images @@ -182,6 +183,7 @@ def init_sync(env_filepath: str, env: str) -> bool: env_filepath, env_type=env['ENV_TYPE'], ) + ensure_filestorage_mapping() link_env_file() download_contracts(env) @@ -215,6 +217,7 @@ def update_sync(env_filepath: str, env: Dict) -> None: if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() + ensure_filestorage_mapping() backup_old_contracts() download_contracts(env) From 6adb53df9ba5fde749d722ba3e1e5060ce50a22a Mon Sep 17 00:00:00 2001 From: badrogger Date: Sun, 25 Sep 2022 09:39:04 +0000 Subject: [PATCH 36/86] Pass SKALE_STATE_DIR env to docker-compose --- node_cli/configs/__init__.py | 1 + node_cli/core/host.py | 2 +- node_cli/core/node.py | 10 ++++++---- node_cli/operations/volume.py | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 4de87991..9c1e58cb 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -32,6 +32,7 @@ G_CONF_HOME = os.getenv('TEST_HOME_DIR') or GLOBAL_CONFIG['home_dir'] SKALE_STATE_DIR = '/var/lib/skale' +SCHAIN_STATE_DIR = os.path.join('/var/lib/skale', 'schains') FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SNAPSHOTS_SHARED_VOLUME = 'shared-space' SCHAINS_MNT_DIR = '/mnt' diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 23c93518..8cfed7f6 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -73,7 +73,7 @@ def prepare_host( env_type: str, allocation: bool = False ): - logger.info(f'Preparing host started') + logger.info('Preparing host started') make_dirs() save_env_params(env_filepath) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 9f9237dd..c9baec96 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -30,15 +30,16 @@ import docker from node_cli.configs import ( + BACKUP_ARCHIVE_NAME, CONTAINER_CONFIG_PATH, FILESTORAGE_MAPPING, - SKALE_DIR, INIT_ENV_FILEPATH, - BACKUP_ARCHIVE_NAME, + LOG_PATH, RESTORE_SLEEP_TIMEOUT, SCHAINS_MNT_DIR, - TM_INIT_TIMEOUT, - LOG_PATH + SKALE_DIR, + SKALE_STATE_DIR, + TM_INIT_TIMEOUT ) from node_cli.configs.env import get_env_config from node_cli.configs.cli_logger import LOG_DATA_PATH as CLI_LOG_DATA_PATH @@ -220,6 +221,7 @@ def get_node_env(env_filepath, inited_node=False, sync_schains=None, sync_node=F 'SKALE_DIR': SKALE_DIR, 'SCHAINS_MNT_DIR': SCHAINS_MNT_DIR, 'FILESTORAGE_MAPPING': FILESTORAGE_MAPPING, + 'SKALE_LIB_PATH': SKALE_STATE_DIR, **env_params } if inited_node and not sync_node: diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index f5cc2672..d67c5a0c 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -30,6 +30,7 @@ DOCKER_LVMPY_REPO_URL, FILESTORAGE_MAPPING, SCHAINS_MNT_DIR, + SCHAIN_STATE_DIR, SKALE_STATE_DIR ) @@ -146,8 +147,7 @@ def prepare_block_device(block_device, force=False): else: logger.info('%s contains %s filesystem', block_device, filesystem) format_as_btrfs(block_device) - mountpoint = os.path.join(SKALE_STATE_DIR, 'schains') - mount_device(block_device, mountpoint) + mount_device(block_device, SCHAIN_STATE_DIR) def max_resize_btrfs(path): From 396a0681bc780a01c99b26ea3acb3f381ec8b91f Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 1 Dec 2022 20:41:25 +0000 Subject: [PATCH 37/86] Add new sync node options --- node_cli/cli/__init__.py | 2 +- node_cli/cli/sync_node.py | 32 +++++++++++--- node_cli/configs/node_options.py | 24 +++++++++++ node_cli/core/node.py | 16 ++++++- node_cli/core/node_options.py | 74 ++++++++++++++++++++++++++++++++ node_cli/operations/base.py | 23 +++++++--- node_cli/utils/helper.py | 9 +++- setup.py | 7 +-- tests/cli/sync_node_test.py | 53 ++++++++++++++++++++++- tests/conftest.py | 14 ++++++ tests/test-env | 3 +- text.yml | 13 +++++- 12 files changed, 248 insertions(+), 22 deletions(-) create mode 100644 node_cli/configs/node_options.py create mode 100644 node_cli/core/node_options.py diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index b59a9860..4fb25b25 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.2.0' +__version__ = '2.2.1' if __name__ == "__main__": diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index ce94c8be..1a9e0882 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -20,10 +20,12 @@ import click from node_cli.core.node import init_sync, update_sync -from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd +from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd, error_exit +from node_cli.utils.exit_codes import CLIExitCodes -TEXTS = safe_load_texts() +G_TEXTS = safe_load_texts() +TEXTS = G_TEXTS['sync_node'] @click.group() @@ -36,11 +38,31 @@ def sync_node(): pass -@sync_node.command('init', help="Initialize sync SKALE node") +@sync_node.command('init', help=TEXTS['init']['help']) @click.argument('env_file') +@click.option( + '--archive', + help=TEXTS['init']['archive'], + is_flag=True +) +@click.option( + '--catchup', + help=TEXTS['init']['catchup'], + is_flag=True +) +@click.option( + '--historic-state', + help=TEXTS['init']['historic_state'], + is_flag=True +) @streamed_cmd -def _init_sync(env_file): - init_sync(env_file) +def _init_sync(env_file, archive, catchup, historic_state): + if (historic_state and not archive): + error_exit( + '--historic-state can be used only is combination with --archive', + exit_code=CLIExitCodes.FAILURE + ) + init_sync(env_file, archive, catchup, historic_state) @sync_node.command('update', help='Update sync node from .env file') diff --git a/node_cli/configs/node_options.py b/node_cli/configs/node_options.py new file mode 100644 index 00000000..e2b6c894 --- /dev/null +++ b/node_cli/configs/node_options.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +from node_cli.configs import NODE_DATA_PATH + +NODE_OPTIONS_FILEPATH = os.path.join(NODE_DATA_PATH, 'node_options.json') +NODE_OPTIONS_LOCK_PATH = os.path.join(NODE_DATA_PATH, 'node_options.json.lock') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index c9baec96..e1e5b60a 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -148,12 +148,24 @@ def init(env_filepath): @check_not_inited -def init_sync(env_filepath: str) -> None: +def init_sync( + env_filepath: str, + archive: bool, + catchup: bool, + historic_state: bool +) -> None: configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) if env is None: return - inited_ok = init_sync_op(env_filepath, env) + inited_ok = init_sync_op( + env_filepath, + env, + archive, + catchup, + historic_state + + ) if not inited_ok: error_exit( 'Init operation failed', diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py new file mode 100644 index 00000000..f86e537b --- /dev/null +++ b/node_cli/core/node_options.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# +# This file is part of SKALE Admin +# +# Copyright (C) 2022 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +from filelock import FileLock + +from node_cli.utils.helper import read_json, write_json, init_file +from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH + +logger = logging.getLogger(__name__) + + +class NodeOptions: + def __init__( + self, + filepath: str = NODE_OPTIONS_FILEPATH + ): + self.filepath = filepath + self.lock_filepath = filepath + '.lock' + init_file(filepath, {}) + + def _get(self, field_name: str): + config = read_json(self.filepath) + return config.get(field_name) + + def _set(self, field_name: str, field_value) -> None: + lock = FileLock(self.lock_filepath) + with lock: + config = read_json(self.filepath) + config[field_name] = field_value + write_json(self.filepath, config) + + @property + def archive(self) -> bool: + return self._get('archive') + + @archive.setter + def archive(self, archive: bool) -> None: + return self._set('archive', archive) + + @property + def catchup(self) -> bool: + return self._get('catchup') + + @catchup.setter + def catchup(self, catchup: bool) -> None: + return self._set('catchup', catchup) + + @property + def historic_state(self) -> bool: + return self._get('historic_state') + + @historic_state.setter + def historic_state(self, historic_state: bool) -> None: + return self._set('historic_state', historic_state) + + def all(self) -> dict: + return read_json(self.filepath) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index b7c6e88a..b56c4474 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -2,7 +2,7 @@ # # This file is part of node-cli # -# Copyright (C) 2021 SKALE Labs +# Copyright (C) 2021-Present SKALE Labs # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -28,6 +28,7 @@ from node_cli.core.docker_config import configure_docker from node_cli.core.nginx import generate_nginx_config from node_cli.core.resources import update_resource_allocation, init_shared_space_volume +from node_cli.core.node_options import NodeOptions from node_cli.operations.common import ( backup_old_contracts, download_contracts, configure_filebeat, @@ -93,7 +94,7 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): @checked_host -def update(env_filepath: str, env: Dict) -> None: +def update(env_filepath: str, env: Dict) -> bool: compose_rm(env) remove_dynamic_containers() @@ -136,7 +137,7 @@ def update(env_filepath: str, env: Dict) -> None: @checked_host -def init(env_filepath: str, env: str) -> bool: +def init(env_filepath: str, env: dict) -> bool: sync_skale_node() if env.get('SKIP_DOCKER_CONFIG') != 'True': @@ -168,7 +169,13 @@ def init(env_filepath: str, env: str) -> bool: return True -def init_sync(env_filepath: str, env: str) -> bool: +def init_sync( + env_filepath: str, + env: dict, + archive: bool, + catchup: bool, + historic_state: bool +) -> bool: cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node( env.get('CONTAINER_CONFIGS_STREAM'), @@ -183,6 +190,12 @@ def init_sync(env_filepath: str, env: str) -> bool: env_filepath, env_type=env['ENV_TYPE'], ) + + node_options = NodeOptions() + node_options.archive = archive + node_options.catchup = catchup + node_options.historic_state = historic_state + ensure_filestorage_mapping() link_env_file() download_contracts(env) @@ -204,7 +217,7 @@ def init_sync(env_filepath: str, env: str) -> bool: return True -def update_sync(env_filepath: str, env: Dict) -> None: +def update_sync(env_filepath: str, env: Dict) -> bool: compose_rm(env, sync_node=True) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 17e7b10d..77fc9667 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -85,6 +85,11 @@ def write_json(path, content): json.dump(content, outfile, indent=4) +def init_file(path, content=None): + if not os.path.exists(path): + write_json(path, content) + + def run_cmd( cmd, env={}, @@ -357,9 +362,9 @@ def rsync_dirs(src: str, dest: str) -> None: run_cmd(['rsync', '-r', f'{src}/.git', dest]) -def ok_result(payload: dict = None): +def ok_result(payload: dict | None = None): return 'ok', payload -def err_result(msg: str = None): +def err_result(msg: str | None = None): return 'error', msg diff --git a/setup.py b/setup.py index 208eaba9..02fe135a 100644 --- a/setup.py +++ b/setup.py @@ -20,12 +20,12 @@ def find_version(*file_paths): extras_require = { 'linter': [ - "flake8==3.7.9", + "flake8==6.0.0", "isort>=4.2.15,<5.8.1", ], 'dev': [ "bumpversion==0.6.0", - "pytest==6.2.3", + "pytest==7.2.0", "pytest-cov==2.9.0", "twine==2.0.0", "mock==4.0.2", @@ -67,7 +67,8 @@ def find_version(*file_paths): "python-iptables==1.0.0", "MarkupSafe==2.0.1", "pyOpenSSL==22.0.0", - "cryptography==37.0.2" + "cryptography==37.0.2", + "filelock==3.0.12" ], python_requires='>=3.6,<4', extras_require=extras_require, diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 48a93ded..24318a81 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -22,9 +22,10 @@ import mock import logging -from node_cli.configs import SKALE_DIR +from node_cli.configs import SKALE_DIR, NODE_DATA_PATH from node_cli.cli.sync_node import _init_sync, _update_sync from node_cli.utils.helper import init_default_logger +from node_cli.core.node_options import NodeOptions from tests.helper import ( run_command, subprocess_run_mock @@ -50,6 +51,56 @@ def test_init_sync(mocked_g_config): assert result.exit_code == 0 +def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): + pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) + with mock.patch('subprocess.run', new=subprocess_run_mock), \ + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ + mock.patch('node_cli.operations.base.cleanup_volume_artifacts'), \ + mock.patch('node_cli.operations.base.download_skale_node'), \ + mock.patch('node_cli.operations.base.sync_skale_node'), \ + mock.patch('node_cli.operations.base.configure_docker'), \ + mock.patch('node_cli.operations.base.prepare_host'), \ + mock.patch('node_cli.operations.base.ensure_filestorage_mapping'), \ + mock.patch('node_cli.operations.base.link_env_file'), \ + mock.patch('node_cli.operations.base.download_contracts'), \ + mock.patch('node_cli.operations.base.generate_nginx_config'), \ + mock.patch('node_cli.operations.base.prepare_block_device'), \ + mock.patch('node_cli.operations.base.update_meta'), \ + mock.patch('node_cli.operations.base.update_resource_allocation'), \ + mock.patch('node_cli.operations.base.update_images'), \ + mock.patch('node_cli.operations.base.compose_up_sync'), \ + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + mock.patch('node_cli.core.node.configure_firewall_rules'), \ + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + result = run_command( + _init_sync, + ['./tests/test-env', '--archive', '--catchup'] + ) + node_options = NodeOptions() + + assert node_options.archive + assert node_options.catchup + assert not node_options.historic_state + + assert result.exit_code == 0 + + +def test_init_sync_historic_state_fail(mocked_g_config, clean_node_options): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + with mock.patch('subprocess.run', new=subprocess_run_mock), \ + mock.patch('node_cli.core.node.init_sync_op'), \ + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + mock.patch('node_cli.core.node.configure_firewall_rules'), \ + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + result = run_command( + _init_sync, + ['./tests/test-env', '--historic-state'] + ) + assert result.exit_code == 1 + assert '--historic-state can be used only' in result.output + + def test_update_sync(mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with mock.patch('subprocess.run', new=subprocess_run_mock), \ diff --git a/tests/conftest.py b/tests/conftest.py index 17bddb8e..4d00c556 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,7 @@ ENVIRONMENT_PARAMS_FILEPATH, GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH, REMOVED_CONTAINERS_FOLDER_PATH, NGINX_CONTAINER_NAME ) +from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH, NODE_OPTIONS_LOCK_PATH from node_cli.utils.global_config import generate_g_config_file from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.ssl import SSL_FOLDER_PATH @@ -189,6 +190,19 @@ def mocked_g_config(): generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) +@pytest.fixture() +def clean_node_options(): + pathlib.Path(NODE_OPTIONS_FILEPATH).unlink(missing_ok=True) + pathlib.Path(NODE_OPTIONS_LOCK_PATH).unlink(missing_ok=True) + yield + pathlib.Path(NODE_OPTIONS_FILEPATH).unlink(missing_ok=True) + pathlib.Path(NODE_OPTIONS_LOCK_PATH).unlink(missing_ok=True) + + +def clean_node_data(): + pass + + @pytest.fixture def resource_alloc(): with open(RESOURCE_ALLOCATION_FILEPATH, 'w') as alloc_file: diff --git a/tests/test-env b/tests/test-env index 9afb4c68..eb598381 100644 --- a/tests/test-env +++ b/tests/test-env @@ -12,4 +12,5 @@ SGX_SERVER_URL=http://127.0.0.1 DISK_MOUNTPOINT=/dev/sss DOCKER_LVMPY_STREAM='master' ENV_TYPE='devnet' -SCHAIN_NAME='test' \ No newline at end of file +SCHAIN_NAME='test' +ENFORCE_BTRFS=False \ No newline at end of file diff --git a/text.yml b/text.yml index 862635c7..5afed71f 100644 --- a/text.yml +++ b/text.yml @@ -30,8 +30,8 @@ node: You should run < skale node init > already_inited: Node was already inited before. cmd_failed: |- - Command failed. Please, check out < skale logs cli > - and logs from docker containers for more information + Command failed. Please, check out < skale logs cli > + and logs from docker containers for more information domain_name_changed: Domain name successfully changed wallet: successful_transfer: "Funds were successfully transferred" @@ -60,3 +60,12 @@ exit: in_progress: "Node exiting is in progress" wait_for_rotations: "Node is waiting to finish rotations" completed: "Node exiting is completed" + +sync_node: + init: + help: Initialize sync SKALE node + archive: Run sync node in an archive node (disable block rotation) + historic_state: Enable historic state (works only in pair with --archive flag) + catchup: Add a flag to start sync node in catchup mode + update: + catchup: Add a flag to start sync node in catchup mode From 6714841564a384665cf74d7cbc5c386acd2b21c8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 1 Dec 2022 20:44:32 +0000 Subject: [PATCH 38/86] Update README with new options --- README.md | 7 +++++++ text.yml | 4 +--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6be93c9a..8eb7c975 100644 --- a/README.md +++ b/README.md @@ -555,6 +555,13 @@ You should specify the following environment variables: - `IMA_CONTRACTS_ABI_URL` - URL to IMA contracts ABI and addresses - `SCHAIN_NAME` - name of the SKALE chain to sync - `ENV_TYPE` - environement type (mainnet, testnet, etc) + + +Options: + +- `--archive` - Run sync node in an archive node (disable block rotation) +- `--historic-state` - Enable historic state (works only in pair with --archive flag) +- `--catchup` - Add a flag to start sync node in catchup mode #### Sync node update diff --git a/text.yml b/text.yml index 5afed71f..b15f1113 100644 --- a/text.yml +++ b/text.yml @@ -66,6 +66,4 @@ sync_node: help: Initialize sync SKALE node archive: Run sync node in an archive node (disable block rotation) historic_state: Enable historic state (works only in pair with --archive flag) - catchup: Add a flag to start sync node in catchup mode - update: - catchup: Add a flag to start sync node in catchup mode + catchup: Add a flag to start sync node in catchup mode \ No newline at end of file From c46895acbfcff61d02532b5bbf9a7f58844cd2aa Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 1 Dec 2022 20:49:26 +0000 Subject: [PATCH 39/86] Bump python version --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 02fe135a..0a77037e 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ def find_version(*file_paths): "cryptography==37.0.2", "filelock==3.0.12" ], - python_requires='>=3.6,<4', + python_requires='>=3.7,<4', extras_require=extras_require, keywords=['skale', 'cli'], @@ -80,6 +80,6 @@ def find_version(*file_paths): 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Affero General Public License v3', 'Natural Language :: English', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ], ) From 329c40b5d72305e95f1f78d69aac94d3cdc00a43 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 1 Dec 2022 20:53:21 +0000 Subject: [PATCH 40/86] Downgrade flake8 to 5.0.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0a77037e..9383ac56 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def find_version(*file_paths): extras_require = { 'linter': [ - "flake8==6.0.0", + "flake8==5.0.4", "isort>=4.2.15,<5.8.1", ], 'dev': [ From 8e11f2a236f9c221167f617c6736537a8f6538ae Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 2 Dec 2022 16:17:42 +0000 Subject: [PATCH 41/86] Hotfix: Remove double types --- node_cli/utils/helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 77fc9667..735aff25 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -362,9 +362,9 @@ def rsync_dirs(src: str, dest: str) -> None: run_cmd(['rsync', '-r', f'{src}/.git', dest]) -def ok_result(payload: dict | None = None): +def ok_result(payload: dict = None): return 'ok', payload -def err_result(msg: str | None = None): +def err_result(msg: str = None): return 'error', msg From 65bf278411778ddd070b39d0e7d536544331258b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 2 Dec 2022 16:34:36 +0000 Subject: [PATCH 42/86] Bump python version to 3.8 --- .github/workflows/publish.yml | 8 ++++---- .github/workflows/test.yml | 2 +- setup.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index efb1b493..f7399e36 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -67,10 +67,10 @@ jobs: asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.8 - name: Install ubuntu dependencies if: matrix.os == 'ubuntu-18.04' run: | @@ -109,10 +109,10 @@ jobs: asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.8 - name: Install ubuntu dependencies if: matrix.os == 'ubuntu-18.04' run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 851e0c31..5e362e16 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python-version: [3.7] + python-version: [3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index 9383ac56..f9384ca3 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ def find_version(*file_paths): "cryptography==37.0.2", "filelock==3.0.12" ], - python_requires='>=3.7,<4', + python_requires='>=3.8,<4', extras_require=extras_require, keywords=['skale', 'cli'], @@ -80,6 +80,6 @@ def find_version(*file_paths): 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Affero General Public License v3', 'Natural Language :: English', - 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ], ) From adb6b1d1fcc6b29fe1348ed4f4806d2ee43e9955 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 2 Dec 2022 18:10:17 +0000 Subject: [PATCH 43/86] Fix licence header --- node_cli/configs/routes.py | 2 +- node_cli/core/node_options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 19e7847e..87fac8f5 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# This file is part of SKALE Admin +# This file is part of node-cli # # Copyright (C) 2020 SKALE Labs # diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index f86e537b..33deab8d 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# This file is part of SKALE Admin +# This file is part of node-cli # # Copyright (C) 2022 SKALE Labs # From f39e010e57981a7e8415df0f565fa2efb674b84f Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 2 Dec 2022 18:49:19 +0000 Subject: [PATCH 44/86] Remove sync node from resource allocation --- node_cli/configs/resource_allocation.py | 2 +- node_cli/core/resources.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/node_cli/configs/resource_allocation.py b/node_cli/configs/resource_allocation.py index ad3056de..8a51729d 100644 --- a/node_cli/configs/resource_allocation.py +++ b/node_cli/configs/resource_allocation.py @@ -20,7 +20,7 @@ import os from node_cli.configs import NODE_DATA_PATH -SYNC_NODE_DIVIDER = 1 + LARGE_DIVIDER = 1 MEDIUM_DIVIDER = 8 TEST_DIVIDER = 8 diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 078c99f8..778afcab 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -36,7 +36,7 @@ from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, TEST_DIVIDER, SMALL_DIVIDER, MEDIUM_DIVIDER, LARGE_DIVIDER, - SYNC_NODE_DIVIDER, MEMORY_FACTOR, MAX_CPU_SHARES + MEMORY_FACTOR, MAX_CPU_SHARES ) logger = logging.getLogger(__name__) @@ -53,8 +53,7 @@ def __init__(self, value, fractional=False): 'test': value / TEST_DIVIDER, 'small': value / SMALL_DIVIDER, 'medium': value / MEDIUM_DIVIDER, - 'large': value / LARGE_DIVIDER, - 'sync_node': value / SYNC_NODE_DIVIDER, + 'large': value / LARGE_DIVIDER } if not fractional: for k in self.values: From e074e1129384f96df298ab76976c85f00da31833 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 2 Dec 2022 19:27:38 +0000 Subject: [PATCH 45/86] Remove sync node from tests --- tests/resources_test.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/resources_test.py b/tests/resources_test.py index caf1c542..242522c4 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -14,7 +14,7 @@ from node_cli.utils.helper import write_json, safe_load_yml -SCHAIN_VOLUME_PARTS = {'large': {'max_consensus_storage_bytes': 21311992627, 'max_file_storage_bytes': 21311992627, 'max_reserved_storage_bytes': 7103997542, 'max_skaled_leveldb_storage_bytes': 21311992627}, 'medium': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'small': {'max_consensus_storage_bytes': 166499942, 'max_file_storage_bytes': 166499942, 'max_reserved_storage_bytes': 55499980, 'max_skaled_leveldb_storage_bytes': 166499942}, 'test': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'test4': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'sync_node': {'max_consensus_storage_bytes': 21311992627, 'max_file_storage_bytes': 21311992627, 'max_reserved_storage_bytes': 7103997542, 'max_skaled_leveldb_storage_bytes': 21311992627}} # noqa +SCHAIN_VOLUME_PARTS = {'large': {'max_consensus_storage_bytes': 21311992627, 'max_file_storage_bytes': 21311992627, 'max_reserved_storage_bytes': 7103997542, 'max_skaled_leveldb_storage_bytes': 21311992627}, 'medium': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'small': {'max_consensus_storage_bytes': 166499942, 'max_file_storage_bytes': 166499942, 'max_reserved_storage_bytes': 55499980, 'max_skaled_leveldb_storage_bytes': 166499942}, 'test': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'test4': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}} # noqa DEFAULT_ENV_TYPE = 'devnet' @@ -67,7 +67,7 @@ def test_generate_resource_allocation_config(): assert resource_allocation_config['schain']['disk']['large'] == 71039975424 assert resource_allocation_config['ima']['cpu_shares'] == { - 'large': 204, 'medium': 25, 'small': 1, 'sync_node': 204, 'test': 25, 'test4': 25} + 'large': 204, 'medium': 25, 'small': 1, 'test': 25, 'test4': 25} assert isinstance(resource_allocation_config['ima']['mem'], dict) assert resource_allocation_config['schain']['volume_limits'] == SCHAIN_VOLUME_PARTS @@ -100,7 +100,6 @@ def test_get_static_disk_alloc_devnet( 'large': 71039975424, 'medium': 8879996928, 'small': 554999808, - 'sync_node': 71039975424, 'test': 8879996928, 'test4': 8879996928 } @@ -165,6 +164,5 @@ def test_leveldb_limits(): 'medium': {'contract_storage': 1598399446, 'db_storage': 532799815}, 'small': {'contract_storage': 99899965, 'db_storage': 33299988}, 'test': {'contract_storage': 1598399446, 'db_storage': 532799815}, - 'test4': {'contract_storage': 1598399446, 'db_storage': 532799815}, - 'sync_node': {'contract_storage': 12787195576, 'db_storage': -1}, + 'test4': {'contract_storage': 1598399446, 'db_storage': 532799815} } From c8c583054b83bfa3500e6e15ff24993192b33cd0 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 2 Dec 2022 19:57:08 +0000 Subject: [PATCH 46/86] Update schain_allocation.yml in tests --- tests/.skale/config/schain_allocation.yml | 36 ----------------------- 1 file changed, 36 deletions(-) diff --git a/tests/.skale/config/schain_allocation.yml b/tests/.skale/config/schain_allocation.yml index 29bb7be1..14947b77 100644 --- a/tests/.skale/config/schain_allocation.yml +++ b/tests/.skale/config/schain_allocation.yml @@ -6,7 +6,6 @@ devnet: large: 71039975424 medium: 8879996928 small: 554999808 - sync_node: 71039975424 test: 8879996928 test4: 8879996928 leveldb_limits: @@ -19,9 +18,6 @@ devnet: small: contract_storage: 99899965 db_storage: 33299988 - sync_node: - contract_storage: 12787195576 - db_storage: -1 test: contract_storage: 1598399446 db_storage: 532799815 @@ -45,11 +41,6 @@ devnet: max_file_storage_bytes: 166499942 max_reserved_storage_bytes: 55499980 max_skaled_leveldb_storage_bytes: 166499942 - sync_node: - max_consensus_storage_bytes: 21311992627 - max_file_storage_bytes: 21311992627 - max_reserved_storage_bytes: 7103997542 - max_skaled_leveldb_storage_bytes: 21311992627 test: max_consensus_storage_bytes: 2663999078 max_file_storage_bytes: 2663999078 @@ -65,7 +56,6 @@ mainnet: large: 1687199940608 medium: 210899992576 small: 13181249536 - sync_node: 1687199940608 test: 210899992576 test4: 210899992576 leveldb_limits: @@ -78,9 +68,6 @@ mainnet: small: contract_storage: 2372624916 db_storage: 790874972 - sync_node: - contract_storage: 303695989309 - db_storage: -1 test: contract_storage: 37961998663 db_storage: 12653999554 @@ -104,11 +91,6 @@ mainnet: max_file_storage_bytes: 3954374860 max_reserved_storage_bytes: 1318124953 max_skaled_leveldb_storage_bytes: 3954374860 - sync_node: - max_consensus_storage_bytes: 506159982182 - max_file_storage_bytes: 506159982182 - max_reserved_storage_bytes: 168719994060 - max_skaled_leveldb_storage_bytes: 506159982182 test: max_consensus_storage_bytes: 63269997772 max_file_storage_bytes: 63269997772 @@ -124,7 +106,6 @@ qanet: large: 177599938560 medium: 22199992320 small: 1387499520 - sync_node: 177599938560 test: 22199992320 test4: 22199992320 leveldb_limits: @@ -137,9 +118,6 @@ qanet: small: contract_storage: 249749913 db_storage: 83249971 - sync_node: - contract_storage: 31967988940 - db_storage: -1 test: contract_storage: 3995998617 db_storage: 1331999539 @@ -163,11 +141,6 @@ qanet: max_file_storage_bytes: 416249856 max_reserved_storage_bytes: 138749952 max_skaled_leveldb_storage_bytes: 416249856 - sync_node: - max_consensus_storage_bytes: 53279981568 - max_file_storage_bytes: 53279981568 - max_reserved_storage_bytes: 17759993856 - max_skaled_leveldb_storage_bytes: 53279981568 test: max_consensus_storage_bytes: 6659997696 max_file_storage_bytes: 6659997696 @@ -183,7 +156,6 @@ testnet: large: 177599938560 medium: 22199992320 small: 1387499520 - sync_node: 177599938560 test: 22199992320 test4: 22199992320 leveldb_limits: @@ -196,9 +168,6 @@ testnet: small: contract_storage: 249749913 db_storage: 83249971 - sync_node: - contract_storage: 31967988940 - db_storage: -1 test: contract_storage: 3995998617 db_storage: 1331999539 @@ -222,11 +191,6 @@ testnet: max_file_storage_bytes: 416249856 max_reserved_storage_bytes: 138749952 max_skaled_leveldb_storage_bytes: 416249856 - sync_node: - max_consensus_storage_bytes: 53279981568 - max_file_storage_bytes: 53279981568 - max_reserved_storage_bytes: 17759993856 - max_skaled_leveldb_storage_bytes: 53279981568 test: max_consensus_storage_bytes: 6659997696 max_file_storage_bytes: 6659997696 From a9c77c28a7f552f3b012784a2885f1890bd8d14b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 6 Dec 2022 16:48:00 +0000 Subject: [PATCH 47/86] Minor changes --- node_cli/cli/sync_node.py | 2 +- node_cli/operations/base.py | 2 +- tests/conftest.py | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index 1a9e0882..5bc79050 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -57,7 +57,7 @@ def sync_node(): ) @streamed_cmd def _init_sync(env_file, archive, catchup, historic_state): - if (historic_state and not archive): + if historic_state and not archive: error_exit( '--historic-state can be used only is combination with --archive', exit_code=CLIExitCodes.FAILURE diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index b56c4474..3d2adb7c 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -94,7 +94,7 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): @checked_host -def update(env_filepath: str, env: Dict) -> bool: +def update(env_filepath: str, env: dict) -> bool: compose_rm(env) remove_dynamic_containers() diff --git a/tests/conftest.py b/tests/conftest.py index 4d00c556..210166dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -194,9 +194,11 @@ def mocked_g_config(): def clean_node_options(): pathlib.Path(NODE_OPTIONS_FILEPATH).unlink(missing_ok=True) pathlib.Path(NODE_OPTIONS_LOCK_PATH).unlink(missing_ok=True) - yield - pathlib.Path(NODE_OPTIONS_FILEPATH).unlink(missing_ok=True) - pathlib.Path(NODE_OPTIONS_LOCK_PATH).unlink(missing_ok=True) + try: + yield + finally: + pathlib.Path(NODE_OPTIONS_FILEPATH).unlink(missing_ok=True) + pathlib.Path(NODE_OPTIONS_LOCK_PATH).unlink(missing_ok=True) def clean_node_data(): From edfcbda19c53ac7add61ace4cab4fa12ac706f6a Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 6 Dec 2022 17:39:32 +0000 Subject: [PATCH 48/86] Remove lock file for node options --- node_cli/configs/node_options.py | 1 - node_cli/core/node_options.py | 10 +++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/node_cli/configs/node_options.py b/node_cli/configs/node_options.py index e2b6c894..6565b3d7 100644 --- a/node_cli/configs/node_options.py +++ b/node_cli/configs/node_options.py @@ -21,4 +21,3 @@ from node_cli.configs import NODE_DATA_PATH NODE_OPTIONS_FILEPATH = os.path.join(NODE_DATA_PATH, 'node_options.json') -NODE_OPTIONS_LOCK_PATH = os.path.join(NODE_DATA_PATH, 'node_options.json.lock') diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index 33deab8d..70573a65 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -18,7 +18,6 @@ # along with this program. If not, see . import logging -from filelock import FileLock from node_cli.utils.helper import read_json, write_json, init_file from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH @@ -32,7 +31,6 @@ def __init__( filepath: str = NODE_OPTIONS_FILEPATH ): self.filepath = filepath - self.lock_filepath = filepath + '.lock' init_file(filepath, {}) def _get(self, field_name: str): @@ -40,11 +38,9 @@ def _get(self, field_name: str): return config.get(field_name) def _set(self, field_name: str, field_value) -> None: - lock = FileLock(self.lock_filepath) - with lock: - config = read_json(self.filepath) - config[field_name] = field_value - write_json(self.filepath, config) + config = read_json(self.filepath) + config[field_name] = field_value + write_json(self.filepath, config) @property def archive(self) -> bool: From 6cd70561cdf07bf5d13fd789f61b60fe51a6114b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 6 Dec 2022 18:45:41 +0000 Subject: [PATCH 49/86] Fix tests --- tests/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 210166dd..69ef983e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,7 +32,7 @@ ENVIRONMENT_PARAMS_FILEPATH, GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH, REMOVED_CONTAINERS_FOLDER_PATH, NGINX_CONTAINER_NAME ) -from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH, NODE_OPTIONS_LOCK_PATH +from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH from node_cli.utils.global_config import generate_g_config_file from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.ssl import SSL_FOLDER_PATH @@ -193,12 +193,10 @@ def mocked_g_config(): @pytest.fixture() def clean_node_options(): pathlib.Path(NODE_OPTIONS_FILEPATH).unlink(missing_ok=True) - pathlib.Path(NODE_OPTIONS_LOCK_PATH).unlink(missing_ok=True) try: yield finally: pathlib.Path(NODE_OPTIONS_FILEPATH).unlink(missing_ok=True) - pathlib.Path(NODE_OPTIONS_LOCK_PATH).unlink(missing_ok=True) def clean_node_data(): From 21f42784048e091547c5b8952617603dbdd34532 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 8 Dec 2022 14:55:41 +0000 Subject: [PATCH 50/86] Increase schain stop timeout to 5 minutes --- node_cli/utils/docker_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 2384f846..d4ef53f2 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -39,7 +39,7 @@ logger = logging.getLogger(__name__) -SCHAIN_REMOVE_TIMEOUT = 60 +SCHAIN_REMOVE_TIMEOUT = 300 IMA_REMOVE_TIMEOUT = 20 MAIN_COMPOSE_CONTAINERS = ('skale-api', 'bounty', 'skale-admin') @@ -94,20 +94,20 @@ def remove_dynamic_containers(): def rm_all_schain_containers(): schain_containers = get_all_schain_containers() - remove_containers(schain_containers, stop_timeout=SCHAIN_REMOVE_TIMEOUT) + remove_containers(schain_containers, timeout=SCHAIN_REMOVE_TIMEOUT) def rm_all_ima_containers(): ima_containers = get_all_ima_containers() - remove_containers(ima_containers, stop_timeout=IMA_REMOVE_TIMEOUT) + remove_containers(ima_containers, timeout=IMA_REMOVE_TIMEOUT) -def remove_containers(containers, stop_timeout): +def remove_containers(containers, timeout): for container in containers: - safe_rm(container, stop_timeout=stop_timeout) + safe_rm(container, stop_timeout=timeout) -def safe_rm(container: Container, stop_timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kwargs): +def safe_rm(container: Container, timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kwargs): """ Saves docker container logs (last N lines) in the .skale/node_data/log/.removed_containers folder. Then stops and removes container with specified params. @@ -115,7 +115,7 @@ def safe_rm(container: Container, stop_timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kw container_name = container.name logger.info( f'Stopping container: {container_name}, timeout: {stop_timeout}') - container.stop(timeout=stop_timeout) + container.stop(timeout=timeout) backup_container_logs(container) logger.info(f'Removing container: {container_name}, kwargs: {kwargs}') container.remove(**kwargs) From 58352f5ed5f1d13c6efe385a3e2cb7af3f30447b Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 8 Dec 2022 14:58:49 +0000 Subject: [PATCH 51/86] Fix flake8 --- node_cli/utils/docker_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index d4ef53f2..bf5b1065 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -104,7 +104,7 @@ def rm_all_ima_containers(): def remove_containers(containers, timeout): for container in containers: - safe_rm(container, stop_timeout=timeout) + safe_rm(container, timeout=timeout) def safe_rm(container: Container, timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kwargs): @@ -114,7 +114,7 @@ def safe_rm(container: Container, timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kwargs) """ container_name = container.name logger.info( - f'Stopping container: {container_name}, timeout: {stop_timeout}') + f'Stopping container: {container_name}, timeout: {timeout}') container.stop(timeout=timeout) backup_container_logs(container) logger.info(f'Removing container: {container_name}, kwargs: {kwargs}') From ccb6fbd6953422f345304d4afcd7866d881585ba Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 19 Jan 2023 17:39:09 +0000 Subject: [PATCH 52/86] Fix filestorage endpoint --- node_cli/configs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 9c1e58cb..e156d361 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -35,7 +35,7 @@ SCHAIN_STATE_DIR = os.path.join('/var/lib/skale', 'schains') FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SNAPSHOTS_SHARED_VOLUME = 'shared-space' -SCHAINS_MNT_DIR = '/mnt' +SCHAINS_MNT_DIR = '/var/lib/skale/schains' SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') SKALE_TMP_DIR = os.path.join(SKALE_DIR, '.tmp') From b3cdad140ebfc8c207a9bf8dcef5685a74c426ae Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 19 Jan 2023 17:43:16 +0000 Subject: [PATCH 53/86] Small change to trigger build --- tests/helper.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/helper.py b/tests/helper.py index c1d4ab7a..166b0bb8 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -26,8 +26,12 @@ BLOCK_DEVICE = os.getenv('BLOCK_DEVICE') -def response_mock(status_code=0, json_data=None, - headers=None, raw=None): +def response_mock( + status_code=0, + json_data=None, + headers=None, + raw=None +): result = MagicMock() result.status_code = status_code From 2d8adcd75690581cc2fe3b3187d61f253aa2ec79 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 9 Feb 2023 19:31:44 +0000 Subject: [PATCH 54/86] Fix tests --- node_cli/cli/node.py | 35 ++--------------------------------- node_cli/cli/sync_node.py | 32 +++++++++++++++++++++++++++----- node_cli/core/node.py | 11 ++++++----- node_cli/core/node_options.py | 1 + node_cli/main.py | 1 + node_cli/operations/base.py | 13 +++++++++++-- node_cli/utils/helper.py | 32 ++++++++++++++++++++++++++++++++ setup.py | 5 ++++- tests/conftest.py | 2 ++ 9 files changed, 86 insertions(+), 46 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 141e9d3b..b4367849 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -17,9 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import ipaddress from typing import Optional -from urllib.parse import urlparse import click @@ -45,7 +43,8 @@ from node_cli.utils.helper import ( abort_if_false, safe_load_texts, - streamed_cmd + streamed_cmd, + IP_TYPE ) from node_cli.utils.meta import get_meta_info from node_cli.utils.print_formatters import print_meta_info @@ -54,36 +53,6 @@ TEXTS = safe_load_texts() -class UrlType(click.ParamType): - name = 'url' - - def convert(self, value, param, ctx): - try: - result = urlparse(value) - except ValueError: - self.fail(f'Some characters are not allowed in {value}', - param, ctx) - if not all([result.scheme, result.netloc]): - self.fail(f'Expected valid url. Got {value}', param, ctx) - return value - - -class IpType(click.ParamType): - name = 'ip' - - def convert(self, value, param, ctx): - try: - ipaddress.ip_address(value) - except ValueError: - self.fail(f'expected valid ipv4/ipv6 address. Got {value}', - param, ctx) - return value - - -URL_TYPE = UrlType() -IP_TYPE = IpType() - - @click.group() def node_cli(): pass diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index 5bc79050..8f67968d 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -17,10 +17,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional + import click from node_cli.core.node import init_sync, update_sync -from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd, error_exit +from node_cli.utils.helper import ( + abort_if_false, + safe_load_texts, + streamed_cmd, + error_exit, + IP_TYPE +) from node_cli.utils.exit_codes import CLIExitCodes @@ -55,14 +63,21 @@ def sync_node(): help=TEXTS['init']['historic_state'], is_flag=True ) +@click.option( + '--snapshot-from', + type=IP_TYPE, + default=None, + hidden=True, + help='Ip of the node from to download snapshot from' +) @streamed_cmd -def _init_sync(env_file, archive, catchup, historic_state): +def _init_sync(env_file, archive, catchup, historic_state, snapshot_from: Optional[str] = None): if historic_state and not archive: error_exit( '--historic-state can be used only is combination with --archive', exit_code=CLIExitCodes.FAILURE ) - init_sync(env_file, archive, catchup, historic_state) + init_sync(env_file, archive, catchup, historic_state, snapshot_from=snapshot_from) @sync_node.command('update', help='Update sync node from .env file') @@ -70,6 +85,13 @@ def _init_sync(env_file, archive, catchup, historic_state): expose_value=False, prompt='Are you sure you want to update SKALE node software?') @click.argument('env_file') +@click.option( + '--snapshot-from', + type=IP_TYPE, + default=None, + hidden=True, + help='Ip of the node from to download snapshot from' +) @streamed_cmd -def _update_sync(env_file): - update_sync(env_file) +def _update_sync(env_file, snapshot_from: Optional[str] = None): + update_sync(env_file, snapshot_from=snapshot_from) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 76d5e06a..fb728b5f 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -152,7 +152,8 @@ def init_sync( env_filepath: str, archive: bool, catchup: bool, - historic_state: bool + historic_state: bool, + snapshot_from: Optional[str] = None ) -> None: configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) @@ -163,8 +164,8 @@ def init_sync( env, archive, catchup, - historic_state - + historic_state, + snapshot_from=snapshot_from ) if not inited_ok: error_exit( @@ -183,11 +184,11 @@ def init_sync( @check_inited @check_user -def update_sync(env_filepath): +def update_sync(env_filepath, snapshot_from: Optional[str] = None): logger.info('Node update started') configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) - update_ok = update_sync_op(env_filepath, env) + update_ok = update_sync_op(env_filepath, env, snapshot_from=snapshot_from) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index 2ebb1e12..a1b5889e 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -67,6 +67,7 @@ def historic_state(self) -> bool: def historic_state(self, historic_state: bool) -> None: return self._set('historic_state', historic_state) + @property def snapshot_from(self) -> Optional[str]: return self._get('snapshot_from') diff --git a/node_cli/main.py b/node_cli/main.py index 3e9d4f38..bda53a09 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -86,6 +86,7 @@ def get_sources_list(): logs_cli, resources_allocation_cli, node_cli, + sync_node_cli, wallet_cli, ssl_cli, exit_cli, diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 1bfc763d..35afe41b 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -186,7 +186,8 @@ def init_sync( env: dict, archive: bool, catchup: bool, - historic_state: bool + historic_state: bool, + snapshot_from: Optional[str] = None ) -> bool: cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node( @@ -207,6 +208,7 @@ def init_sync( node_options.archive = archive node_options.catchup = catchup node_options.historic_state = historic_state + node_options.snapshot_from = snapshot_from ensure_filestorage_mapping() link_env_file() @@ -229,7 +231,11 @@ def init_sync( return True -def update_sync(env_filepath: str, env: Dict) -> bool: +def update_sync( + env_filepath: str, + env: Dict, + snapshot_from: Optional[str] = None +) -> bool: compose_rm(env, sync_node=True) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -242,6 +248,9 @@ def update_sync(env_filepath: str, env: Dict) -> bool: if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() + node_options = NodeOptions() + node_options.snapshot_from = snapshot_from + ensure_filestorage_mapping() backup_old_contracts() download_contracts(env) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 735aff25..4d76164b 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -17,10 +17,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import ipaddress import json import os import re import sys +from urllib.parse import urlparse import yaml import shutil @@ -368,3 +370,33 @@ def ok_result(payload: dict = None): def err_result(msg: str = None): return 'error', msg + + +class UrlType(click.ParamType): + name = 'url' + + def convert(self, value, param, ctx): + try: + result = urlparse(value) + except ValueError: + self.fail(f'Some characters are not allowed in {value}', + param, ctx) + if not all([result.scheme, result.netloc]): + self.fail(f'Expected valid url. Got {value}', param, ctx) + return value + + +class IpType(click.ParamType): + name = 'ip' + + def convert(self, value, param, ctx): + try: + ipaddress.ip_address(value) + except ValueError: + self.fail(f'expected valid ipv4/ipv6 address. Got {value}', + param, ctx) + return value + + +URL_TYPE = UrlType() +IP_TYPE = IpType() diff --git a/setup.py b/setup.py index 7b4c9c63..c68db0ca 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,10 @@ def find_version(*file_paths): "python-debian==0.1.48", "python-iptables==1.0.0", "PyYAML==6.0", - "MarkupSafe==2.1.1" + "pyOpenSSL==22.0.0", + "MarkupSafe==2.1.1", + "cryptography==37.0.2", + "filelock==3.0.12" ], python_requires='>=3.8,<4', extras_require=extras_require, diff --git a/tests/conftest.py b/tests/conftest.py index 8ff6397e..bdb6d263 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -218,6 +218,7 @@ def resource_alloc(): os.remove(RESOURCE_ALLOCATION_FILEPATH) +@pytest.fixture def ssl_folder(): if os.path.isdir(SSL_FOLDER_PATH): shutil.rmtree(SSL_FOLDER_PATH) @@ -259,6 +260,7 @@ def nginx_container(dutils, ssl_folder): pass +@pytest.fixture def meta_file_v1(): with open(META_FILEPATH, 'w') as f: json.dump(TEST_META_V1, f) From 976d6e68b8c1874d26154e2f3503df1e35464357 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 10 Feb 2023 16:41:20 +0000 Subject: [PATCH 55/86] Remove snapshot_from from init/update sync --- node_cli/cli/sync_node.py | 27 +++++---------------------- node_cli/core/node.py | 10 ++++------ node_cli/operations/base.py | 20 ++++---------------- 3 files changed, 13 insertions(+), 44 deletions(-) diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index 8f67968d..bca887b6 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Optional - import click from node_cli.core.node import init_sync, update_sync @@ -26,8 +24,7 @@ abort_if_false, safe_load_texts, streamed_cmd, - error_exit, - IP_TYPE + error_exit ) from node_cli.utils.exit_codes import CLIExitCodes @@ -63,21 +60,14 @@ def sync_node(): help=TEXTS['init']['historic_state'], is_flag=True ) -@click.option( - '--snapshot-from', - type=IP_TYPE, - default=None, - hidden=True, - help='Ip of the node from to download snapshot from' -) @streamed_cmd -def _init_sync(env_file, archive, catchup, historic_state, snapshot_from: Optional[str] = None): +def _init_sync(env_file, archive, catchup, historic_state): if historic_state and not archive: error_exit( '--historic-state can be used only is combination with --archive', exit_code=CLIExitCodes.FAILURE ) - init_sync(env_file, archive, catchup, historic_state, snapshot_from=snapshot_from) + init_sync(env_file, archive, catchup, historic_state) @sync_node.command('update', help='Update sync node from .env file') @@ -85,13 +75,6 @@ def _init_sync(env_file, archive, catchup, historic_state, snapshot_from: Option expose_value=False, prompt='Are you sure you want to update SKALE node software?') @click.argument('env_file') -@click.option( - '--snapshot-from', - type=IP_TYPE, - default=None, - hidden=True, - help='Ip of the node from to download snapshot from' -) @streamed_cmd -def _update_sync(env_file, snapshot_from: Optional[str] = None): - update_sync(env_file, snapshot_from=snapshot_from) +def _update_sync(env_file): + update_sync(env_file) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 696c99f7..168228e3 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -152,8 +152,7 @@ def init_sync( env_filepath: str, archive: bool, catchup: bool, - historic_state: bool, - snapshot_from: Optional[str] = None + historic_state: bool ) -> None: configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) @@ -164,8 +163,7 @@ def init_sync( env, archive, catchup, - historic_state, - snapshot_from=snapshot_from + historic_state ) if not inited_ok: error_exit( @@ -184,11 +182,11 @@ def init_sync( @check_inited @check_user -def update_sync(env_filepath, snapshot_from: Optional[str] = None): +def update_sync(env_filepath): logger.info('Node update started') configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) - update_ok = update_sync_op(env_filepath, env, snapshot_from=snapshot_from) + update_ok = update_sync_op(env_filepath, env) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 3f0772a9..d5a35c30 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -19,7 +19,7 @@ import functools import logging -from typing import Dict, Optional +from typing import Dict from node_cli.cli.info import VERSION from node_cli.configs import CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH @@ -142,7 +142,7 @@ def update(env_filepath: str, env: Dict) -> None: @checked_host -def init(env_filepath: str, env: dict, snapshot_from: Optional[str] = None) -> bool: +def init(env_filepath: str, env: dict) -> bool: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() @@ -164,9 +164,6 @@ def init(env_filepath: str, env: dict, snapshot_from: Optional[str] = None) -> b docker_lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) - node_options = NodeOptions() - node_options.snapshot_from = snapshot_from - update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], @@ -183,8 +180,7 @@ def init_sync( env: dict, archive: bool, catchup: bool, - historic_state: bool, - snapshot_from: Optional[str] = None + historic_state: bool ) -> bool: cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node( @@ -205,7 +201,6 @@ def init_sync( node_options.archive = archive node_options.catchup = catchup node_options.historic_state = historic_state - node_options.snapshot_from = snapshot_from ensure_filestorage_mapping() link_env_file() @@ -228,11 +223,7 @@ def init_sync( return True -def update_sync( - env_filepath: str, - env: Dict, - snapshot_from: Optional[str] = None -) -> bool: +def update_sync(env_filepath: str, env: Dict) -> bool: compose_rm(env, sync_node=True) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -245,9 +236,6 @@ def update_sync( if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() - node_options = NodeOptions() - node_options.snapshot_from = snapshot_from - ensure_filestorage_mapping() backup_old_contracts() download_contracts(env) From 62db9398baf9940d716dbfa63a2434fe76977ff8 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 16 Jun 2023 14:12:12 +0000 Subject: [PATCH 56/86] Merge 2.3.0 to beta-sync-node --- .dockerignore | 6 +++ .github/workflows/publish.yml | 98 ++++++++++++++++++++++++----------- .github/workflows/test.yml | 38 ++++++++++---- .gitmodules | 5 ++ Dockerfile | 29 +++++++++++ lvmpy | 1 + main.spec | 4 -- node_cli/cli/__init__.py | 3 +- node_cli/cli/lvmpy.py | 62 ++++++++++++++++++++++ node_cli/cli/node.py | 10 +++- node_cli/configs/__init__.py | 10 ++++ node_cli/core/node.py | 8 ++- node_cli/main.py | 5 +- node_cli/operations/base.py | 22 +++++--- node_cli/utils/meta.py | 16 +++--- scripts/run_tests.sh | 7 ++- setup.py | 11 ++-- tests/cli/main_test.py | 5 +- tests/cli/node_test.py | 35 +++++++++++-- tests/conftest.py | 12 ++++- tests/helper.py | 8 +++ tests/tools_meta_test.py | 38 ++++++++++++-- text.yml | 11 +++- 23 files changed, 362 insertions(+), 82 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 160000 lvmpy create mode 100644 node_cli/cli/lvmpy.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..060cae42 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +tests +helper-scripts +dist +build +.github +.gitmodules diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f7399e36..545fe4b4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: create_release: if: github.event.pull_request.merged name: Create release - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 outputs: upload_url: ${{ steps.create_release.outputs.upload_url }} version: ${{ steps.export_outputs.outputs.version }} @@ -22,6 +22,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + with: + submodules: true + - name: Checkout submodules run: git submodule update --init - name: Install ubuntu dependencies @@ -63,40 +66,57 @@ jobs: strategy: matrix: include: - - os: ubuntu-18.04 + - os: ubuntu-20.04 asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.11 + - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-18.04' + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -e .[dev] - pip install wheel - pip install --upgrade 'setuptools<45.0.0' + - name: Checkout submodules run: git submodule update --init - - name: Build normal CLI + + - name: Build normal binary + run: | + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} + ls -altr /home/ubuntu/dist/ + docker rm -f $(docker ps -aq) + + - name: Save sha512sum run: | - bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal - - name: Upload normal CLI + sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum + + - name: Upload release binary id: upload-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }} + asset_path: /home/ubuntu/dist/${{ matrix.asset_name }} asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream + + - name: Upload release checksum + id: upload-release-checksum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: /home/ubuntu/dist/sha512sum + asset_name: ${{ matrix.asset_name }}.sha512 + asset_content_type: text/plain + build_and_publish_sync: if: github.event.pull_request.merged needs: create_release @@ -105,32 +125,37 @@ jobs: strategy: matrix: include: - - os: ubuntu-18.04 + - os: ubuntu-20.04 asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.11 + - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-18.04' + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -e .[dev] - pip install wheel - pip install --upgrade 'setuptools<45.0.0' + - name: Checkout submodules run: git submodule update --init - - name: Build sync CLI + + - name: Build normal binary run: | - bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync - - name: Upload sync CLI - id: upload-release-asset + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} + ls -altr /home/ubuntu/dist/ + docker rm -f $(docker ps -aq) + + - name: Save sha512sum + run: | + sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum + + - name: Upload release sync CLI + id: upload-sync-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -139,3 +164,14 @@ jobs: asset_path: ./dist/${{ matrix.asset_name }}-sync asset_name: ${{ matrix.asset_name }}-sync asset_content_type: application/octet-stream + + - name: Upload release sync CLI checksum + id: upload-sync-release-checksum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: /home/ubuntu/dist/sha512sum + asset_name: ${{ matrix.asset_name }}-sync.sha512 + asset_content_type: text/plain diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e362e16..b647a846 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,35 +3,55 @@ on: [push, pull_request] jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.8] + python-version: [3.11] steps: - uses: actions/checkout@v2 + with: + submodules: true + + - name: Checkout submodules + run: git submodule update --init + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: Install ubuntu dependencies run: | sudo apt-get update - sudo apt-get install python-setuptools + sudo apt-get install python-setuptools iptables + - name: Install python dependencies run: | python -m pip install --upgrade pip pip install -e . pip install -e .[dev] pip install --upgrade 'setuptools<45.0.0' + - name: Lint with flake8 run: | flake8 . - - name: Build binary + + - name: Build sync binary in Ubuntu 18.04 environment run: | - bash scripts/build.sh 1.0.0 test-branch normal - - name: Build binary sync + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test + docker rm -f $(docker ps -aq) + + - name: Check build + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + + - name: Build sync binary in Ubuntu 20.04 environment run: | - bash scripts/build.sh 1.0.0 test-branch sync + scripts/build.sh test test + + - name: Check build + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + - name: Run tests - run: | - bash ./scripts/run_tests.sh + run: bash ./scripts/run_tests.sh diff --git a/.gitmodules b/.gitmodules index d54a2086..b9ae3738 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,9 @@ [submodule "helper-scripts"] path = helper-scripts url = https://github.com/skalenetwork/helper-scripts.git + branch = develop + +[submodule "lvmpy"] + path = lvmpy + url = https://github.com/skalenetwork/docker-lvmpy branch = develop diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..67a6a42d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get install -y \ + git \ + python3.11 \ + libpython3.11-dev \ + python3.11-venv \ + python3.11-distutils \ + python3.11-dev \ + build-essential \ + zlib1g-dev \ + libssl-dev \ + libffi-dev \ + swig \ + iptables + +RUN mkdir /app +WORKDIR /app + +COPY . . + +ENV PATH=/app/buildvenv/bin:$PATH +RUN python3.11 -m venv /app/buildvenv && \ + pip install --upgrade pip && \ + pip install wheel setuptools==63.2.0 && \ + pip install -e '.[dev]' diff --git a/lvmpy b/lvmpy new file mode 160000 index 00000000..8ee051bf --- /dev/null +++ b/lvmpy @@ -0,0 +1 @@ +Subproject commit 8ee051bf24aa3feecc0ef97fb5eec970eb068512 diff --git a/main.spec b/main.spec index 3c776386..a4dbe395 100644 --- a/main.spec +++ b/main.spec @@ -1,9 +1,5 @@ # -*- mode: python -*- -# import distutils -# if distutils.distutils_path.endswith('__init__.py'): -# distutils.distutils_path = os.path.dirname(distutils.distutils_path) - import importlib.util libxtwrapper_path = importlib.util.find_spec('libxtwrapper').origin diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index 5b1ba474..02a4b07a 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,5 @@ -__version__ = '2.2.1' +__version__ = '2.3.0' + if __name__ == "__main__": print(__version__) diff --git a/node_cli/cli/lvmpy.py b/node_cli/cli/lvmpy.py new file mode 100644 index 00000000..473defa8 --- /dev/null +++ b/node_cli/cli/lvmpy.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2020 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.utils.helper import abort_if_false +from node_cli.utils.texts import Texts +from lvmpy.src.app import run as run_lvmpy +from lvmpy.src.health import heal_service + +G_TEXTS = Texts() +TEXTS = G_TEXTS['lvmpy'] + + +@click.group() +def lvmpy_cli(): + pass + + +@lvmpy_cli.group('lvmpy', help=TEXTS['help']) +def health(): + pass + + +@health.command(help=TEXTS['run']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['run']['prompt'] +) +def run(): + run_lvmpy() + + +@health.command(help=TEXTS['heal']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['heal']['prompt'] +) +def heal(): + heal_service() diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 7ec32dbf..866d07e9 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -131,9 +131,15 @@ def backup_node(backup_folder_path): @node.command('restore', help="Restore SKALE node on another machine") @click.argument('backup_path') @click.argument('env_file') +@click.option( + '--no-snapshot', + help='Do not restore sChains from snapshot', + is_flag=True, + hidden=True +) @streamed_cmd -def restore_node(backup_path, env_file): - restore(backup_path, env_file) +def restore_node(backup_path, env_file, no_snapshot): + restore(backup_path, env_file, no_snapshot) @node.command('maintenance-on', help="Set SKALE node into maintenance mode") diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 81e0743d..8534c4b0 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -77,6 +77,16 @@ FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml') DOCKER_LVMPY_PATH = os.path.join(SKALE_DIR, 'docker-lvmpy') +DOCKER_LVMPY_BIN_PATH = '/usr/local/bin/skale' + +LVMPY_CMD = f'{DOCKER_LVMPY_BIN_PATH} lvmpy' +LVMPY_RUN_CMD = f'{LVMPY_CMD} run --yes' +LVMPY_HEAL_CMD = f'{LVMPY_CMD} heal --yes' + +LVMPY_CRON_LOG_PATH = '/var/log/docker-lvmpy/cron.log' +LVMPY_CRON_SCHEDULE_MINUTES = 3 + +LVMPY_LOG_DIR = '/var/log/docker-lvmpy' IPTABLES_DIR = '/etc/iptables/' IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 168228e3..73b2b51d 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -199,13 +199,17 @@ def update_sync(env_filepath): @check_not_inited -def restore(backup_path, env_filepath): +def restore(backup_path, env_filepath, no_snapshot=False): env = get_node_env(env_filepath) if env is None: return save_env_params(env_filepath) env['SKALE_DIR'] = SKALE_DIR - env['BACKUP_RUN'] = 'True' # should be str + + if not no_snapshot: + logger.info('Adding BACKUP_RUN to env ...') + env['BACKUP_RUN'] = 'True' # should be str + restored_ok = restore_op(env, backup_path) if not restored_ok: error_exit( diff --git a/node_cli/main.py b/node_cli/main.py index bda53a09..fa58b4f4 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -29,6 +29,7 @@ from node_cli.cli.health import health_cli from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION, TYPE from node_cli.cli.logs import logs_cli +from node_cli.cli.lvmpy import lvmpy_cli from node_cli.cli.node import node_cli from node_cli.cli.schains import schains_cli from node_cli.cli.wallet import wallet_cli @@ -90,7 +91,8 @@ def get_sources_list(): wallet_cli, ssl_cli, exit_cli, - validate_cli + validate_cli, + lvmpy_cli ] @@ -113,6 +115,7 @@ def handle_exception(exc_type, exc_value, exc_traceback): logger.debug(f'cmd: {" ".join(str(x) for x in args)}, v.{__version__}') sources = get_sources_list() cmd_collection = click.CommandCollection(sources=sources) + try: cmd_collection() except Exception as err: diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index d5a35c30..cfefcc75 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import distro import functools import logging from typing import Dict @@ -39,11 +40,10 @@ ) from node_cli.operations.volume import ( cleanup_volume_artifacts, - docker_lvmpy_update, - docker_lvmpy_install, ensure_filestorage_mapping, prepare_block_device ) +from node_cli.operations.docker_lvmpy import lvmpy_install # noqa from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables @@ -111,7 +111,7 @@ def update(env_filepath: str, env: Dict) -> None: backup_old_contracts() download_contracts(env) - docker_lvmpy_update(env) + lvmpy_install(env) generate_nginx_config() prepare_host( @@ -134,7 +134,9 @@ def update(env_filepath: str, env: Dict) -> None: update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') compose_up(env) @@ -161,7 +163,7 @@ def init(env_filepath: str, env: dict) -> bool: configure_iptables() generate_nginx_config() - docker_lvmpy_install(env) + lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) update_meta( @@ -215,7 +217,9 @@ def init_sync( update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) @@ -292,13 +296,15 @@ def restore(env, backup_path): link_env_file() configure_iptables() - docker_lvmpy_install(env) + lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) compose_up(env) diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index b49d9f61..69078af2 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -6,20 +6,23 @@ DEFAULT_VERSION = '1.0.0' DEFAULT_CONFIG_STREAM = '1.1.0' DEFAULT_DOCKER_LVMPY_STREAM = '1.0.0' +DEFAULT_OS_ID = 'ubuntu' +DEFAULT_OS_VERSION = '18.04' class CliMeta( namedtuple( 'Node', - ['version', 'config_stream', 'docker_lvmpy_stream'] + ['version', 'config_stream', 'docker_lvmpy_stream', 'os_id', 'os_version'] ) ): __slots__ = () def __new__(cls, version=DEFAULT_VERSION, config_stream=DEFAULT_CONFIG_STREAM, - docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM): + docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION): return super(CliMeta, cls).__new__( - cls, version, config_stream, docker_lvmpy_stream + cls, version, config_stream, docker_lvmpy_stream, os_id, os_version ) @@ -41,7 +44,8 @@ def save_meta(meta: CliMeta) -> None: def compose_default_meta() -> CliMeta: return CliMeta(version=DEFAULT_VERSION, docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, - config_stream=DEFAULT_CONFIG_STREAM) + config_stream=DEFAULT_CONFIG_STREAM, os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION) def ensure_meta(meta: CliMeta = None) -> None: @@ -51,7 +55,7 @@ def ensure_meta(meta: CliMeta = None) -> None: def update_meta(version: str, config_stream: str, - docker_lvmpy_stream: str) -> None: + docker_lvmpy_stream: str, os_id: str, os_version: str) -> None: ensure_meta() - meta = CliMeta(version, config_stream, docker_lvmpy_stream) + meta = CliMeta(version, config_stream, docker_lvmpy_stream, os_id, os_version) save_meta(meta) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 12cce17a..afbb2068 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,4 +3,9 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -HIDE_STREAM_LOG=true TEST_HOME_DIR="$PROJECT_DIR/tests/" GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ +LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \ + HIDE_STREAM_LOG=true \ + TEST_HOME_DIR="$PROJECT_DIR/tests/" \ + GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \ + DOTENV_FILEPATH='tests/test-env' \ + py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ diff --git a/setup.py b/setup.py index c68db0ca..23211e5d 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ def find_version(*file_paths): install_requires=[ "click==8.1.3", "PyInstaller==5.6.2", + "distro==1.4.0", "docker==6.0.1", "texttable==1.6.4", "python-dateutil==2.8.2", @@ -60,15 +61,19 @@ def find_version(*file_paths): "python-dotenv==0.21.0", "terminaltables==3.1.10", "requests==2.28.1", - "GitPython==3.1.27", + "GitPython==3.1.30", "packaging==21.3", "python-debian==0.1.48", - "python-iptables==1.0.0", + "python-iptables==1.0.1", "PyYAML==6.0", "pyOpenSSL==22.0.0", "MarkupSafe==2.1.1", "cryptography==37.0.2", "filelock==3.0.12" + 'Flask==2.2.2', + 'itsdangerous==2.0.1', + 'sh==1.14.2', + 'python-crontab==2.6.0' ], python_requires='>=3.8,<4', extras_require=extras_require, @@ -80,6 +85,6 @@ def find_version(*file_paths): 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Affero General Public License v3', 'Natural Language :: English', - 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.11' ], ) diff --git a/tests/cli/main_test.py b/tests/cli/main_test.py index 2703f0b5..5ce570ad 100644 --- a/tests/cli/main_test.py +++ b/tests/cli/main_test.py @@ -18,14 +18,13 @@ # along with this program. If not, see . -from node_cli.cli import info from node_cli.main import version from tests.helper import run_command def test_version(): result = run_command(version, []) - expected = f'SKALE Node CLI version: {info.VERSION}\n' + expected = 'SKALE Node CLI version: test\n' assert result.output == expected result = run_command(version, ['--short']) - assert result.output == f'{info.VERSION}\n' + assert result.output == 'test\n' diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 3b9fe1d0..319137ed 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -20,6 +20,7 @@ import pathlib import mock +from unittest.mock import MagicMock, patch import requests import logging @@ -302,10 +303,11 @@ def test_restore(mocked_g_config): ) backup_path = result.output.replace( 'Backup archive successfully created: ', '').replace('\n', '') - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.restore_op'), \ - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ + patch('subprocess.run', new=subprocess_run_mock), \ + patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( restore_node, [backup_path, './tests/test-env'] @@ -313,6 +315,31 @@ def test_restore(mocked_g_config): assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') == 'True' + + +def test_restore_no_snapshot(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + result = run_command( + backup_node, + ['/tmp'] + ) + backup_path = result.output.replace( + 'Backup archive successfully created: ', '').replace('\n', '') + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ + patch('subprocess.run', new=subprocess_run_mock), \ + patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + patch('node_cli.utils.decorators.is_node_inited', return_value=False): + result = run_command( + restore_node, + [backup_path, './tests/test-env', '--no-snapshot'] + ) + assert result.exit_code == 0 + assert 'Node is restored from backup\n' in result.output # noqa + + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') is None + def test_maintenance_on(): resp_mock = response_mock( diff --git a/tests/conftest.py b/tests/conftest.py index bdb6d263..5e79dce1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,7 +43,7 @@ from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file -from tests.helper import TEST_META_V1, TEST_META_V2 +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 TEST_ENV_PARAMS = """ @@ -280,6 +280,16 @@ def meta_file_v2(): os.remove(META_FILEPATH) +@pytest.fixture +def meta_file_v3(): + with open(META_FILEPATH, 'w') as f: + json.dump(TEST_META_V3, f) + try: + yield META_FILEPATH + finally: + os.remove(META_FILEPATH) + + @pytest.fixture def ensure_meta_removed(): try: diff --git a/tests/helper.py b/tests/helper.py index 7f519f0f..2aca1efd 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -37,6 +37,14 @@ } +TEST_META_V3 = { + 'version': '0.1.1', + 'config_stream': 'develop', + 'docker_lvmpy_stream': '1.1.2', + 'os_id': 'ubuntu', + 'os_version': '18.04' +} + def response_mock( status_code=0, diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index 95ab7658..431533db 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -7,7 +7,7 @@ ensure_meta, get_meta_info, save_meta, update_meta ) -from tests.helper import TEST_META_V1, TEST_META_V2 +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 def test_get_meta_info_v1(meta_file_v1): @@ -24,6 +24,15 @@ def test_get_meta_info_v2(meta_file_v2): assert meta.docker_lvmpy_stream == TEST_META_V2['docker_lvmpy_stream'] +def test_get_meta_info_v3(meta_file_v3): + meta = get_meta_info() + assert meta.version == TEST_META_V3['version'] + assert meta.config_stream == TEST_META_V3['config_stream'] + assert meta.docker_lvmpy_stream == TEST_META_V3['docker_lvmpy_stream'] + assert meta.os_id == TEST_META_V3['os_id'] + assert meta.os_version == TEST_META_V3['os_version'] + + def test_get_meta_info_empty(): meta = get_meta_info() assert meta is None @@ -34,6 +43,8 @@ def test_compose_default_meta(): assert meta.version == '1.0.0' assert meta.config_stream == '1.1.0' assert meta.docker_lvmpy_stream == '1.0.0' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '18.04' def test_save_meta(meta_file_v2): @@ -44,28 +55,45 @@ def test_save_meta(meta_file_v2): assert saved_json == { 'version': '1.1.2', 'config_stream': '2.2.2', - 'docker_lvmpy_stream': '1.0.0' + 'docker_lvmpy_stream': '1.0.0', + 'os_id': 'ubuntu', + 'os_version': '18.04', } -def test_update_meta(meta_file_v2): +def test_update_meta_from_v2_to_v3(meta_file_v2): old_meta = get_meta_info() update_meta(version='3.3.3', config_stream='1.1.1', - docker_lvmpy_stream='1.2.2') + docker_lvmpy_stream='1.2.2', os_id='debian', os_version='11') meta = get_meta_info() assert meta.version == '3.3.3' assert meta.config_stream == '1.1.1' assert meta.docker_lvmpy_stream == '1.2.2' + assert meta.os_id == 'debian' + assert meta.os_version == '11' assert meta != old_meta def test_update_meta_from_v1(meta_file_v1): update_meta(version='4.4.4', config_stream='beta', - docker_lvmpy_stream='1.3.3') + docker_lvmpy_stream='1.3.3', os_id='debian', os_version='11') meta = get_meta_info() assert meta.version == '4.4.4' assert meta.config_stream == 'beta' assert meta.docker_lvmpy_stream == '1.3.3' + assert meta.os_id == 'debian' + assert meta.os_version == '11' + + +def test_update_meta_from_v3(meta_file_v3): + update_meta(version='5.5.5', config_stream='stable', + docker_lvmpy_stream='1.2.3', os_id='ubuntu', os_version='20.04') + meta = get_meta_info() + assert meta.version == '5.5.5' + assert meta.config_stream == 'stable' + assert meta.docker_lvmpy_stream == '1.2.3' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '20.04' def test_ensure_meta(ensure_meta_removed): diff --git a/text.yml b/text.yml index 0617d643..08bf65e8 100644 --- a/text.yml +++ b/text.yml @@ -65,4 +65,13 @@ sync_node: help: Initialize sync SKALE node archive: Run sync node in an archive node (disable block rotation) historic_state: Enable historic state (works only in pair with --archive flag) - catchup: Add a flag to start sync node in catchup mode \ No newline at end of file + catchup: Add a flag to start sync node in catchup mode + +lvmpy: + help: Lvmpy commands + run: + help: Run lvmpy http server + prompt: Are you sure you want to run lvmpy server? + heal: + help: Run healing procedure for lvmpy server + prompt: Are you sure you want run healing procedure? From 46348f69197799bae5ad841ba087766152bccbbc Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 16 Jun 2023 18:07:36 +0000 Subject: [PATCH 57/86] Fix missing VOLUME_GROUP config param --- lvmpy | 1 - node_cli/configs/__init__.py | 1 + node_cli/operations/docker_lvmpy.py | 92 +++++++++++++++++++++++++++++ setup.py | 2 +- 4 files changed, 94 insertions(+), 2 deletions(-) delete mode 160000 lvmpy create mode 100644 node_cli/operations/docker_lvmpy.py diff --git a/lvmpy b/lvmpy deleted file mode 160000 index 8ee051bf..00000000 --- a/lvmpy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8ee051bf24aa3feecc0ef97fb5eec970eb068512 diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 8534c4b0..50748b6d 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -36,6 +36,7 @@ FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SNAPSHOTS_SHARED_VOLUME = 'shared-space' SCHAINS_MNT_DIR = '/var/lib/skale/schains' +VOLUME_GROUP = 'schains' SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') SKALE_TMP_DIR = os.path.join(SKALE_DIR, '.tmp') diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py new file mode 100644 index 00000000..31036c95 --- /dev/null +++ b/node_cli/operations/docker_lvmpy.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import os +import shutil + +import crontab + +from node_cli.utils.git_utils import sync_repo +from node_cli.configs import ( + DOCKER_LVMPY_PATH, + DOCKER_LVMPY_REPO_URL, + FILESTORAGE_MAPPING, + LVMPY_RUN_CMD, + LVMPY_HEAL_CMD, + LVMPY_CRON_LOG_PATH, + LVMPY_CRON_SCHEDULE_MINUTES, + SCHAINS_MNT_DIR, + VOLUME_GROUP +) +from lvmpy.src.install import setup as setup_lvmpy + +logger = logging.getLogger(__name__) + + +def update_docker_lvmpy_env(env): + env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] + env['VOLUME_GROUP'] = 'schains' + env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING + env['MNT_DIR'] = SCHAINS_MNT_DIR + env['PATH'] = os.environ.get('PATH', None) + return env + + +def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): + if not os.path.isdir(FILESTORAGE_MAPPING): + os.makedirs(FILESTORAGE_MAPPING) + + +def sync_docker_lvmpy_repo(env): + if os.path.isdir(DOCKER_LVMPY_PATH): + shutil.rmtree(DOCKER_LVMPY_PATH) + sync_repo( + DOCKER_LVMPY_REPO_URL, + DOCKER_LVMPY_PATH, + env["DOCKER_LVMPY_STREAM"] + ) + + +def lvmpy_install(env): + ensure_filestorage_mapping() + logging.info('Configuring and starting lvmpy') + setup_lvmpy( + block_device=env['DISK_MOUNTPOINT'], + volume_group=VOLUME_GROUP, + exec_start=LVMPY_RUN_CMD + ) + init_healing_cron() + logger.info('docker-lvmpy is configured and started') + + +def init_healing_cron(): + logger.info('Configuring cron job for healing lvmpy') + cron_line = f'{LVMPY_HEAL_CMD} >> {LVMPY_CRON_LOG_PATH} 2>&1' + legacy_line = f'cd /opt/docker-lvmpy && venv/bin/python -c "import health; health.heal_service()" >> {LVMPY_CRON_LOG_PATH} 2>&1' # noqa + + with crontab.CronTab(user='root') as c: + jobs = [c.command for c in c] + if legacy_line in jobs: + c.remove_all(command=legacy_line) + if cron_line not in jobs: + job = c.new( + command=cron_line + ) + job.minute.every(LVMPY_CRON_SCHEDULE_MINUTES) diff --git a/setup.py b/setup.py index 23211e5d..6a05adcd 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def find_version(*file_paths): "pyOpenSSL==22.0.0", "MarkupSafe==2.1.1", "cryptography==37.0.2", - "filelock==3.0.12" + "filelock==3.0.12", 'Flask==2.2.2', 'itsdangerous==2.0.1', 'sh==1.14.2', From 411e78ff518710859ec7742720d74cdb29bbe96c Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 08:45:25 +0000 Subject: [PATCH 58/86] Rollback to python 3.8 for 18.04 compatible build --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 67a6a42d..1b8463ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,11 @@ RUN apt-get update && apt-get install -y software-properties-common RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get install -y \ git \ - python3.11 \ - libpython3.11-dev \ - python3.11-venv \ - python3.11-distutils \ - python3.11-dev \ + python3.8 \ + libpython3.8-dev \ + python3.8-venv \ + python3.8-distutils \ + python3.8-dev \ build-essential \ zlib1g-dev \ libssl-dev \ @@ -23,7 +23,7 @@ WORKDIR /app COPY . . ENV PATH=/app/buildvenv/bin:$PATH -RUN python3.11 -m venv /app/buildvenv && \ +RUN python3.8 -m venv /app/buildvenv && \ pip install --upgrade pip && \ pip install wheel setuptools==63.2.0 && \ pip install -e '.[dev]' From 76bb02f1c133fb0e12225a6b3b8d9a4cbdf09ee3 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 08:52:37 +0000 Subject: [PATCH 59/86] Add missing sync param to build script intput --- .github/workflows/publish.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 545fe4b4..b5499dc2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -142,11 +142,11 @@ jobs: - name: Checkout submodules run: git submodule update --init - - name: Build normal binary + - name: Build sync release binary run: | mkdir ./dist docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync ls -altr /home/ubuntu/dist/ docker rm -f $(docker ps -aq) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b647a846..73df9e54 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,7 @@ jobs: run: | mkdir ./dist docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test sync docker rm -f $(docker ps -aq) - name: Check build @@ -48,7 +48,7 @@ jobs: - name: Build sync binary in Ubuntu 20.04 environment run: | - scripts/build.sh test test + scripts/build.sh test test sync - name: Check build run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 From 70271e82d16ce4208d332b476fa1300ecea526e0 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 09:57:16 +0000 Subject: [PATCH 60/86] Fix asset name --- .github/workflows/publish.yml | 3 ++- .github/workflows/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b5499dc2..b2c77f9e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -58,6 +58,7 @@ jobs: run: | echo "::set-output name=version::$VERSION" echo "::set-output name=branch::$BRANCH" + build_and_publish_normal: if: github.event.pull_request.merged needs: create_release @@ -126,7 +127,7 @@ jobs: matrix: include: - os: ubuntu-20.04 - asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 + asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64-sync steps: - uses: actions/checkout@v2 - name: Set up Python 3.11 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 73df9e54..943486f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,14 +44,14 @@ jobs: docker rm -f $(docker ps -aq) - name: Check build - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync - name: Build sync binary in Ubuntu 20.04 environment run: | scripts/build.sh test test sync - name: Check build - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync - name: Run tests run: bash ./scripts/run_tests.sh From fb7f3499db02a0fb5d9e8a9fa2fb042daf2e2cb2 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 10:11:00 +0000 Subject: [PATCH 61/86] Add missing lvmpy submodule --- lvmpy | 1 + 1 file changed, 1 insertion(+) create mode 160000 lvmpy diff --git a/lvmpy b/lvmpy new file mode 160000 index 00000000..8ee051bf --- /dev/null +++ b/lvmpy @@ -0,0 +1 @@ +Subproject commit 8ee051bf24aa3feecc0ef97fb5eec970eb068512 From d24cb30f264f5f58e11274b8c234818c0876d251 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Jun 2023 12:12:45 +0000 Subject: [PATCH 62/86] Fix tests --- node_cli/utils/meta.py | 2 +- tests/cli/sync_node_test.py | 8 ++++---- tests/helper.py | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index 69078af2..7eb50f4c 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -55,7 +55,7 @@ def ensure_meta(meta: CliMeta = None) -> None: def update_meta(version: str, config_stream: str, - docker_lvmpy_stream: str, os_id: str, os_version: str) -> None: + docker_lvmpy_stream: str, os_id: str, os_version: str) -> None: ensure_meta() meta = CliMeta(version, config_stream, docker_lvmpy_stream, os_id, os_version) save_meta(meta) diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 24318a81..fb7e4eb2 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -53,8 +53,8 @@ def test_init_sync(mocked_g_config): def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ +# with mock.patch('subprocess.run', new=subprocess_run_mock), \ + with mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ mock.patch('node_cli.operations.base.cleanup_volume_artifacts'), \ mock.patch('node_cli.operations.base.download_skale_node'), \ mock.patch('node_cli.operations.base.sync_skale_node'), \ @@ -74,13 +74,13 @@ def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( _init_sync, - ['./tests/test-env', '--archive', '--catchup'] + ['./tests/test-env', '--archive', '--catchup', '--historic-state'] ) node_options = NodeOptions() assert node_options.archive assert node_options.catchup - assert not node_options.historic_state + assert node_options.historic_state assert result.exit_code == 0 diff --git a/tests/helper.py b/tests/helper.py index 2aca1efd..832ac577 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -78,8 +78,7 @@ def run_command_mock(mock_call_path, response_mock, return run_command(command, params, input=input) -def subprocess_run_mock(cmd=None, shell=None, stdout=None, - stderr=None, env=None, returncode=0): +def subprocess_run_mock(*args, returncode=0, **kwargs): result = MagicMock() result.returncode = returncode result.stdout = MagicMock() From a63112aa39fcf050cb6033ba9bb40b2e60e07ccc Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 20 Jun 2023 19:20:02 +0000 Subject: [PATCH 63/86] Fix sync node publish pipeline --- .github/workflows/publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b2c77f9e..7ce9a521 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -162,8 +162,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }}-sync - asset_name: ${{ matrix.asset_name }}-sync + asset_path: ./dist/${{ matrix.asset_name }} + asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream - name: Upload release sync CLI checksum @@ -174,5 +174,5 @@ jobs: with: upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: /home/ubuntu/dist/sha512sum - asset_name: ${{ matrix.asset_name }}-sync.sha512 + asset_name: ${{ matrix.asset_name }}.sha512 asset_content_type: text/plain From 819a262b5919d97a4e4ae93a3d32ff108c53f2ee Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 20 Jun 2023 19:29:57 +0000 Subject: [PATCH 64/86] Fix publish.yml home dir path --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7ce9a521..85161fa7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -88,7 +88,7 @@ jobs: run: | mkdir ./dist docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal ls -altr /home/ubuntu/dist/ docker rm -f $(docker ps -aq) @@ -162,7 +162,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }} + asset_path: /home/ubuntu/dist/${{ matrix.asset_name }} asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream From e6353544cdc47e66e06a9f20580ad5e2a51c4d84 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 7 Jul 2023 12:45:33 +0000 Subject: [PATCH 65/86] Remove prerelease from sync-node based releases --- .github/workflows/publish.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 85161fa7..8d761ae0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,12 +31,14 @@ jobs: run: | sudo apt-get update sudo apt-get install python-setuptools + - name: Set Versions run: | bash ./scripts/set_versions_ga.sh + - name: Set release run: | - if [[ "$BRANCH" == "stable" ]]; then + if [[ "$BRANCH" == "stable" || "$BRANCH" == "sync-node" ]]; then export PRERELEASE=false else export PRERELEASE=true From 7eac870bff4ddd935addb4095b122d5322c31984 Mon Sep 17 00:00:00 2001 From: badrogger Date: Sat, 8 Jul 2023 12:49:59 +0000 Subject: [PATCH 66/86] Fix update-meta invocation --- node_cli/operations/base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index cfefcc75..45cdb948 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -169,7 +169,9 @@ def init(env_filepath: str, env: dict) -> bool: update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') @@ -259,7 +261,9 @@ def update_sync(env_filepath: str, env: Dict) -> bool: update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) compose_up_sync(env) From 7eb2aaeddbf7f18f35a76587e34389f4b211a674 Mon Sep 17 00:00:00 2001 From: badrogger Date: Sat, 8 Jul 2023 12:54:05 +0000 Subject: [PATCH 67/86] Trigger build --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1b8463ed..943f66e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,8 @@ RUN apt-get install -y \ python3.8-dev \ build-essential \ zlib1g-dev \ - libssl-dev \ libffi-dev \ + libssl-dev \ swig \ iptables From 8cd70a5b74903f5dc050cdd10e80f667e04bc52c Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 19 Sep 2023 17:10:51 +0000 Subject: [PATCH 68/86] Bump GitPython to 3.1.36 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6a05adcd..b7829263 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def find_version(*file_paths): "python-dotenv==0.21.0", "terminaltables==3.1.10", "requests==2.28.1", - "GitPython==3.1.30", + "GitPython==3.1.36", "packaging==21.3", "python-debian==0.1.48", "python-iptables==1.0.1", From a968b107392fec02d1c58e3b341f4f12df516cea Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 24 Jan 2024 18:00:07 +0000 Subject: [PATCH 69/86] Merge sync-node CLI to develop - fix flake8 --- node_cli/core/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 00203fbe..9d5c83f7 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -238,7 +238,7 @@ def get_node_env( save_env_params(env_filepath) else: env_params = extract_env_params(INIT_ENV_FILEPATH, sync_node=sync_node) - + mnt_dir = SCHAINS_MNT_DIR_SYNC if sync_node else SCHAINS_MNT_DIR_REGULAR env = { 'SKALE_DIR': SKALE_DIR, From 79214286e6cacfcd01e7506b0de2b17c44ebf1f1 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 24 Jan 2024 18:02:57 +0000 Subject: [PATCH 70/86] Update publish pipeline - remove sync-node branches --- .github/workflows/publish.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8d761ae0..b44df7d8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,8 +7,6 @@ on: - develop - beta - stable - - sync-node - - beta-sync-node jobs: create_release: @@ -38,7 +36,7 @@ jobs: - name: Set release run: | - if [[ "$BRANCH" == "stable" || "$BRANCH" == "sync-node" ]]; then + if [[ "$BRANCH" == "stable" ]]; then export PRERELEASE=false else export PRERELEASE=true From d6bbb34ec411e838a660bc1981ab2798aefe6672 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 24 Jan 2024 19:20:40 +0000 Subject: [PATCH 71/86] Fix test pipeline - update binary check --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8fcf8bff..4f48d65f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,17 +40,17 @@ jobs: run: | mkdir ./dist docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test normal docker rm -f $(docker ps -aq) - - name: Check build + - name: Check build - normal run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 - name: Build binary in Ubuntu 20.04 environment run: | scripts/build.sh test test - - name: Check build + - name: Check build - normal run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 - name: Build sync binary in Ubuntu 18.04 environment run: | @@ -59,14 +59,14 @@ jobs: docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test sync docker rm -f $(docker ps -aq) - - name: Check build + - name: Check build - sync run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync - name: Build sync binary in Ubuntu 20.04 environment run: | scripts/build.sh test test sync - - name: Check build + - name: Check build - sync run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync - name: Run tests From 1575119f0e9d45b74d7ff0d87f2fde47e0d1f5a2 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 24 Jan 2024 19:25:22 +0000 Subject: [PATCH 72/86] Update volume module - use regular sChain mountpoint --- node_cli/operations/volume.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index d67c5a0c..1a4533c3 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -29,7 +29,7 @@ DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL, FILESTORAGE_MAPPING, - SCHAINS_MNT_DIR, + SCHAINS_MNT_DIR_REGULAR, SCHAIN_STATE_DIR, SKALE_STATE_DIR ) @@ -45,7 +45,7 @@ def update_docker_lvmpy_env(env): env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] env['VOLUME_GROUP'] = 'schains' env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING - env['SCHAINS_MNT_DIR'] = SCHAINS_MNT_DIR + env['SCHAINS_MNT_DIR'] = SCHAINS_MNT_DIR_REGULAR env['PATH'] = os.environ.get('PATH', None) return env From fe6e479a965da2a230d38500516274516fb4bff3 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 25 Jan 2024 11:35:15 +0000 Subject: [PATCH 73/86] Update test and publish pipelines --- .github/workflows/publish.yml | 1 + .github/workflows/test.yml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b44df7d8..a5330ced 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,6 +7,7 @@ on: - develop - beta - stable + - v*.*.* jobs: create_release: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f48d65f..80d9a5f3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: run: | flake8 . - - name: Build binary in Ubuntu 18.04 environment + - name: Build binary in Ubuntu 18.04 environment - normal run: | mkdir ./dist docker build . -t node-cli-builder @@ -46,11 +46,11 @@ jobs: - name: Check build - normal run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 - - name: Build binary in Ubuntu 20.04 environment + - name: Build binary in Ubuntu 20.04 environment - normal run: | - scripts/build.sh test test + scripts/build.sh test test normal - - name: Check build - normal + - name: Check build - sync run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 - name: Build sync binary in Ubuntu 18.04 environment run: | From 74e1028c5ec4c93a8e34797f0bbab4d7df415ad9 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 25 Jan 2024 13:19:24 +0000 Subject: [PATCH 74/86] Update build and publish pipeline - fix mkdir --- .github/workflows/publish.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a5330ced..bd164016 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -87,7 +87,7 @@ jobs: - name: Build normal binary run: | - mkdir ./dist + mkdir -p ./dist docker build . -t node-cli-builder docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal ls -altr /home/ubuntu/dist/ @@ -146,7 +146,7 @@ jobs: - name: Build sync release binary run: | - mkdir ./dist + mkdir -p ./dist docker build . -t node-cli-builder docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync ls -altr /home/ubuntu/dist/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 80d9a5f3..0b1e24ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ jobs: - name: Build binary in Ubuntu 18.04 environment - normal run: | - mkdir ./dist + mkdir -p ./dist docker build . -t node-cli-builder docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test normal docker rm -f $(docker ps -aq) @@ -54,7 +54,7 @@ jobs: run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 - name: Build sync binary in Ubuntu 18.04 environment run: | - mkdir ./dist + mkdir -p ./dist docker build . -t node-cli-builder docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test sync docker rm -f $(docker ps -aq) From ff59242ec36e9496cb92d56b3a5ac99b9d7f1435 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 26 Jan 2024 14:00:55 +0000 Subject: [PATCH 75/86] Minor changes - remove IMA_ENDPOINT, merge compose_up_sync --- README.md | 1 - node_cli/configs/__init__.py | 1 - node_cli/configs/env.py | 1 - node_cli/core/wallet.py | 7 ------- node_cli/operations/base.py | 5 ++--- node_cli/operations/volume.py | 4 ++-- node_cli/utils/docker_utils.py | 10 +++++----- tests/cli/sync_node_test.py | 2 +- tests/conftest.py | 4 ---- 9 files changed, 10 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 8eb7c975..e178c238 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,6 @@ You should specify the following environment variables: - `DISK_MOUNTPOINT` - disk mount point for storing sChains data - `DOCKER_LVMPY_STREAM` - stream of `docker-lvmpy` to use - `CONTAINER_CONFIGS_STREAM` - stream of `skale-node` to use -- `IMA_ENDPOINT` - IMA endpoint to connect - `ENDPOINT` - RPC endpoint of the node in the network where SKALE Manager is deployed - `MANAGER_CONTRACTS_ABI_URL` - URL to SKALE Manager contracts ABI and addresses - `IMA_CONTRACTS_ABI_URL` - URL to IMA contracts ABI and addresses diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index ace188f0..1b43dab5 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -32,7 +32,6 @@ G_CONF_HOME = os.getenv('TEST_HOME_DIR') or GLOBAL_CONFIG['home_dir'] SKALE_STATE_DIR = '/var/lib/skale' -SCHAIN_STATE_DIR = os.path.join('/var/lib/skale', 'schains') FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SNAPSHOTS_SHARED_VOLUME = 'shared-space' SCHAINS_MNT_DIR_REGULAR = '/mnt' diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index 8e156533..1229091b 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -10,7 +10,6 @@ ALLOWED_ENV_TYPES = ['mainnet', 'testnet', 'qanet', 'devnet'] REQUIRED_PARAMS = { - 'IMA_ENDPOINT': '', 'CONTAINER_CONFIGS_STREAM': '', 'ENDPOINT': '', 'MANAGER_CONTRACTS_ABI_URL': '', diff --git a/node_cli/core/wallet.py b/node_cli/core/wallet.py index d7d129ec..3b11d274 100644 --- a/node_cli/core/wallet.py +++ b/node_cli/core/wallet.py @@ -19,13 +19,6 @@ import json -from node_cli.utils.print_formatters import print_wallet_info, TEXTS -from node_cli.utils.helper import error_exit, get_request, post_request, logger -from node_cli.utils.exit_codes import CLIExitCodes - - -BLUEPRINT_NAME = 'wallet' - def get_wallet_info(_format): status, payload = get_request(BLUEPRINT_NAME, 'info') diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 8ab1aef6..3715d793 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -50,7 +50,6 @@ from node_cli.utils.docker_utils import ( compose_rm, compose_up, - compose_up_sync, docker_cleanup, remove_dynamic_containers ) @@ -225,7 +224,7 @@ def init_sync( ) update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) - compose_up_sync(env) + compose_up(env, sync_node=True) return True @@ -266,7 +265,7 @@ def update_sync(env_filepath: str, env: Dict) -> bool: distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) - compose_up_sync(env) + compose_up(env, sync_node=True) return True diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index 1a4533c3..b6de918a 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -30,7 +30,7 @@ DOCKER_LVMPY_REPO_URL, FILESTORAGE_MAPPING, SCHAINS_MNT_DIR_REGULAR, - SCHAIN_STATE_DIR, + SCHAINS_MNT_DIR_SYNC, SKALE_STATE_DIR ) @@ -147,7 +147,7 @@ def prepare_block_device(block_device, force=False): else: logger.info('%s contains %s filesystem', block_device, filesystem) format_as_btrfs(block_device) - mount_device(block_device, SCHAIN_STATE_DIR) + mount_device(block_device, SCHAINS_MNT_DIR_SYNC) def max_resize_btrfs(path): diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 8c572033..784ef20d 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -233,6 +233,11 @@ def get_compose_path(sync_node: bool) -> str: def compose_up(env, sync_node=False): + if sync_node: + logger.info('Running containers for sync node') + run_cmd(cmd=get_up_compose_sync_cmd(), env=env) + return + logger.info('Running base set of containers') if 'SGX_CERTIFICATES_DIR_NAME' not in env: @@ -247,11 +252,6 @@ def compose_up(env, sync_node=False): run_cmd(cmd=get_up_compose_cmd(NOTIFICATION_COMPOSE_SERVICES), env=env) -def compose_up_sync(env) -> None: - logger.info('Running containers for sync node') - run_cmd(cmd=get_up_compose_sync_cmd(), env=env) - - def restart_nginx_container(dutils=None): dutils = dutils or docker_client() nginx_container = dutils.containers.get(NGINX_CONTAINER_NAME) diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index fb7e4eb2..3966d3c8 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -68,7 +68,7 @@ def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): mock.patch('node_cli.operations.base.update_meta'), \ mock.patch('node_cli.operations.base.update_resource_allocation'), \ mock.patch('node_cli.operations.base.update_images'), \ - mock.patch('node_cli.operations.base.compose_up_sync'), \ + mock.patch('node_cli.operations.base.compose_up'), \ mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ mock.patch('node_cli.core.node.configure_firewall_rules'), \ mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): diff --git a/tests/conftest.py b/tests/conftest.py index 5e79dce1..9504523f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -206,10 +206,6 @@ def clean_node_options(): pathlib.Path(NODE_OPTIONS_FILEPATH).unlink(missing_ok=True) -def clean_node_data(): - pass - - @pytest.fixture def resource_alloc(): with open(RESOURCE_ALLOCATION_FILEPATH, 'w') as alloc_file: From 2bb429aec8552d1a88bc3b81a8a71d859fc255d8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 26 Jan 2024 15:27:20 +0000 Subject: [PATCH 76/86] Minor fix - update wallet module --- node_cli/core/wallet.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/node_cli/core/wallet.py b/node_cli/core/wallet.py index 3b11d274..d7d129ec 100644 --- a/node_cli/core/wallet.py +++ b/node_cli/core/wallet.py @@ -19,6 +19,13 @@ import json +from node_cli.utils.print_formatters import print_wallet_info, TEXTS +from node_cli.utils.helper import error_exit, get_request, post_request, logger +from node_cli.utils.exit_codes import CLIExitCodes + + +BLUEPRINT_NAME = 'wallet' + def get_wallet_info(_format): status, payload = get_request(BLUEPRINT_NAME, 'info') From 9b115948be60ea6835704923f6ca2ad7b4144888 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 29 Jan 2024 19:34:36 +0000 Subject: [PATCH 77/86] Minor update - add DISABLE_IMA to sync node --- node_cli/core/node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 9d5c83f7..7b010a03 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -254,6 +254,8 @@ def get_node_env( env['BACKUP_RUN'] = 'True' if pull_config_for_schain: env['PULL_CONFIG_FOR_SCHAIN'] = pull_config_for_schain + if sync_node: + env['DISABLE_IMA'] = 'True' return {k: v for k, v in env.items() if v != ''} From c78155ac27984407322e0ebcd3e6f87eb9944256 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 29 Jan 2024 19:37:47 +0000 Subject: [PATCH 78/86] Minor update - remove DISABLE_IMA from optional params --- node_cli/configs/env.py | 1 - 1 file changed, 1 deletion(-) diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index 1229091b..6a5736fb 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -41,7 +41,6 @@ 'DISABLE_DRY_RUN': '', 'DEFAULT_GAS_LIMIT': '', 'DEFAULT_GAS_PRICE_WEI': '', - 'DISABLE_IMA': '', 'SKIP_DOCKER_CONFIG': '', 'ENFORCE_BTRFS': '', 'SKIP_DOCKER_CLEANUP': '' From ebb42a533a632d5b80d8f8959c3d27a9f18f70ee Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 31 Jan 2024 18:25:20 +0000 Subject: [PATCH 79/86] Remove DISABLE_IMA variable from sync node --- node_cli/core/node.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 7b010a03..9d5c83f7 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -254,8 +254,6 @@ def get_node_env( env['BACKUP_RUN'] = 'True' if pull_config_for_schain: env['PULL_CONFIG_FOR_SCHAIN'] = pull_config_for_schain - if sync_node: - env['DISABLE_IMA'] = 'True' return {k: v for k, v in env.items() if v != ''} From 0699d42cdfc29a98b861e098fa5cede44c2af939 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 7 Feb 2024 19:44:43 +0000 Subject: [PATCH 80/86] Remove CONTAINER_CONFIGS_DIR from required params --- node_cli/configs/env.py | 1 - 1 file changed, 1 deletion(-) diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index 6a5736fb..f534a2e2 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -17,7 +17,6 @@ 'FILEBEAT_HOST': '', 'DISK_MOUNTPOINT': '', 'SGX_SERVER_URL': '', - 'CONTAINER_CONFIGS_DIR': '', 'DOCKER_LVMPY_STREAM': '', 'ENV_TYPE': '', } From 8874270994641bd98b3c67a02155d3c586dc48b9 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 10 Apr 2024 18:27:33 +0000 Subject: [PATCH 81/86] Fix build --- Dockerfile | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b4bb58e..fbd5248c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,9 @@ -FROM ubuntu:20.04 +FROM python:3.11-buster ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y software-properties-common -RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get install -y \ git \ - python3.11 \ - libpython3.11-dev \ - python3.11-venv \ - python3.11-distutils \ - python3.11-dev \ build-essential \ zlib1g-dev \ libssl-dev \ From fc633c1810b8f9a75d5873eed025ba9f0a7f06f9 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 11 Apr 2024 15:00:46 +0000 Subject: [PATCH 82/86] Add optional telegraf service setup step --- node_cli/configs/__init__.py | 3 ++ node_cli/configs/env.py | 4 +++ node_cli/operations/base.py | 20 ++++++++++++ node_cli/operations/telegraf.py | 51 +++++++++++++++++++++++++++++ node_cli/utils/docker_utils.py | 10 ++++-- scripts/run_tests.sh | 2 +- tests/operations/common_test.py | 32 ------------------- tests/operations/telegraf_test.py | 53 +++++++++++++++++++++++++++++++ 8 files changed, 139 insertions(+), 36 deletions(-) create mode 100644 node_cli/operations/telegraf.py delete mode 100644 tests/operations/common_test.py create mode 100644 tests/operations/telegraf_test.py diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 1b43dab5..36f89004 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -158,3 +158,6 @@ def _get_env(): AUTOLOAD_KERNEL_MODULES_PATH = '/etc/modules' BTRFS_KERNEL_MODULE = 'btrfs' + +TELEGRAF_TEMPLATE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'telegraf.conf.j2') +TELEGRAF_CONFIG_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'telegraf.conf') diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index f534a2e2..66565a83 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -34,6 +34,10 @@ OPTIONAL_PARAMS = { 'MONITORING_CONTAINERS': '', + 'TELEGRAF': '', + 'INFLUX_TOKEN': '', + 'INFLUX_ORG': '', + 'INFLUX_BUCKET': '', 'TG_API_KEY': '', 'TG_CHAT_ID': '', 'CONTAINER_CONFIGS_DIR': '', diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 3715d793..3d991094 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -45,6 +45,7 @@ ) from node_cli.operations.docker_lvmpy import lvmpy_install # noqa from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images +from node_cli.operations.telegraf import generate_telegraf_config, get_telegraf_options from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables from node_cli.utils.docker_utils import ( @@ -138,6 +139,9 @@ def update(env_filepath: str, env: Dict) -> None: distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') + if env.get('TELEGRAF'): + options = get_telegraf_options(env) + generate_telegraf_config(options) compose_up(env) return True @@ -174,6 +178,11 @@ def init(env_filepath: str, env: dict) -> bool: ) update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') + + if env.get('TELEGRAF'): + options = get_telegraf_options(env) + generate_telegraf_config(options) + compose_up(env) return True @@ -224,6 +233,11 @@ def init_sync( ) update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) + + if env.get('TELEGRAF'): + options = get_telegraf_options(env) + generate_telegraf_config(options) + compose_up(env, sync_node=True) return True @@ -265,6 +279,11 @@ def update_sync(env_filepath: str, env: Dict) -> bool: distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) + + if env.get('TELEGRAF'): + options = get_telegraf_options(env) + generate_telegraf_config(options) + compose_up(env, sync_node=True) return True @@ -310,6 +329,7 @@ def restore(env, backup_path, config_only=False): distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) + if not config_only: compose_up(env) diff --git a/node_cli/operations/telegraf.py b/node_cli/operations/telegraf.py new file mode 100644 index 00000000..ea4565ba --- /dev/null +++ b/node_cli/operations/telegraf.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2024-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +from node_cli.configs import TELEGRAF_CONFIG_PATH, TELEGRAF_TEMPLATE_PATH +from node_cli.utils.helper import process_template + + +logger = logging.getLogger(__name__) + + +class TelegrafNotConfiguredError(Exception): + pass + + +def get_telegraf_options(env) -> dict: + options = { + 'token': env.get('INFLUX_TOKEN'), + 'org': env.get('INFLUX_ORG'), + 'bucket': env.get('INFLUX_BUCKET') + } + missing = list(filter(lambda k: not options[k], options)) + if missing: + raise TelegrafNotConfiguredError('Missing options {missing}') + return options + + +def generate_telegraf_config( + extra_options: dict, + template_path: str = TELEGRAF_TEMPLATE_PATH, + config_path: str = TELEGRAF_CONFIG_PATH +) -> None: + logger.info('Processing telegraf template') + process_template(template_path, config_path, extra_options) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 784ef20d..2656302b 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -53,7 +53,8 @@ 'watchdog', 'filebeat' ) -MONITORING_COMPOSE_SERVICES = ('node-exporter', 'advisor') +MONITORING_COMPOSE_SERVICES = ('node-exporter', 'advisor',) +TELEGRAF_SERVICES = ('telegraf',) NOTIFICATION_COMPOSE_SERVICES = ('celery',) COMPOSE_TIMEOUT = 10 @@ -162,7 +163,7 @@ def get_logs_backup_filepath(container: Container) -> str: def ensure_volume(name: str, size: int, driver='lvmpy', dutils=None): dutils = dutils or docker_client() if is_volume_exists(name, dutils=dutils): - logger.info('Volume %s already exits', name) + logger.info('Volume %s already exist', name) return logger.info('Creating volume %s, size: %d', name, size) driver_opts = {'size': str(size)} if driver == 'lvmpy' else None @@ -244,9 +245,12 @@ def compose_up(env, sync_node=False): env['SGX_CERTIFICATES_DIR_NAME'] = SGX_CERTIFICATES_DIR_NAME run_cmd(cmd=get_up_compose_cmd(BASE_COMPOSE_SERVICES), env=env) - if str_to_bool(env.get('MONITORING_CONTAINERS', '')): + if str_to_bool(env.get('MONITORING_CONTAINERS', 'False')): logger.info('Running monitoring containers') run_cmd(cmd=get_up_compose_cmd(MONITORING_COMPOSE_SERVICES), env=env) + if str_to_bool(env.get('TELEGRAF', 'False')): + logger.info('Running monitoring containers') + run_cmd(cmd=get_up_compose_cmd(TELEGRAF_SERVICES), env=env) if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: logger.info('Running containers for Telegram notifications') run_cmd(cmd=get_up_compose_cmd(NOTIFICATION_COMPOSE_SERVICES), env=env) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index afbb2068..edc7fa73 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -8,4 +8,4 @@ LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \ TEST_HOME_DIR="$PROJECT_DIR/tests/" \ GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \ DOTENV_FILEPATH='tests/test-env' \ - py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ + py.test --cov=$PROJECT_DIR/ tests/ $@ diff --git a/tests/operations/common_test.py b/tests/operations/common_test.py deleted file mode 100644 index cee64ebc..00000000 --- a/tests/operations/common_test.py +++ /dev/null @@ -1,32 +0,0 @@ - - -def test_remove_dynamic_containers(): - assert False - - -def test_backup_old_contracts(): - assert False - - -def test_download_contracts(): - assert False - - -def test_docker_lvmpy_update(): - assert False - - -def test_update_skale_node_repo(): - assert False - - -def test_update_skale_node_git(): - assert False - - -def test_update_skale_node_dev(): - assert False - - -def test_sync_skale_node_dev(): - assert False diff --git a/tests/operations/telegraf_test.py b/tests/operations/telegraf_test.py new file mode 100644 index 00000000..62f48dbe --- /dev/null +++ b/tests/operations/telegraf_test.py @@ -0,0 +1,53 @@ +import os + +import pytest + +from node_cli.operations.telegraf import ( + get_telegraf_options, + generate_telegraf_config, + TelegrafNotConfiguredError +) + + +def test_get_telegraf_options(): + env = { + 'INFLUX_TOKEN': 'token', + 'INFLUX_ORG': 'org', + 'INFLUX_BUCKET': 'bucket' + } + options = get_telegraf_options(env) + assert options == { + 'token': 'token', + 'org': 'org', + 'bucket': 'bucket' + } + env.pop('INFLUX_TOKEN') + with pytest.raises(TelegrafNotConfiguredError): + get_telegraf_options(env) + + +@pytest.fixture +def template_path(tmp_dir_path): + path = os.path.join(tmp_dir_path, 'telegraf.conf.j2') + template = """ +[[outputs.influxdb_v2]] +bucket = "{{ bucket }}" +organization = "{{ org }}" +token = "{{ token }}" +""" + with open(path, 'w') as tf: + tf.write(template) + return path + + +def test_generate_telegraf_config(tmp_dir_path, template_path): + test_config_path = os.path.join(tmp_dir_path, 'telegraf.conf') + generate_telegraf_config({ + 'token': 'token', + 'org': 'org', + 'bucket': 'bucket' + }, template_path, test_config_path) + + with open(test_config_path) as config_path: + config = config_path.read() + assert config == '\n[[outputs.influxdb_v2]]\nbucket = "bucket"\norganization = "org"\ntoken = "token"' # noqa From 5a3a4f6d57b00c1672e5293f064d2727acff91fc Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 11 Apr 2024 15:31:02 +0000 Subject: [PATCH 83/86] Add missing INFLUX_URL parameter --- node_cli/configs/env.py | 1 + node_cli/operations/telegraf.py | 3 ++- tests/operations/telegraf_test.py | 13 +++++++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index 66565a83..7b619797 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -37,6 +37,7 @@ 'TELEGRAF': '', 'INFLUX_TOKEN': '', 'INFLUX_ORG': '', + 'INFLUX_URL': '', 'INFLUX_BUCKET': '', 'TG_API_KEY': '', 'TG_CHAT_ID': '', diff --git a/node_cli/operations/telegraf.py b/node_cli/operations/telegraf.py index ea4565ba..f3d3e232 100644 --- a/node_cli/operations/telegraf.py +++ b/node_cli/operations/telegraf.py @@ -34,7 +34,8 @@ def get_telegraf_options(env) -> dict: options = { 'token': env.get('INFLUX_TOKEN'), 'org': env.get('INFLUX_ORG'), - 'bucket': env.get('INFLUX_BUCKET') + 'bucket': env.get('INFLUX_BUCKET'), + 'url': env.get('INFLUX_URL') } missing = list(filter(lambda k: not options[k], options)) if missing: diff --git a/tests/operations/telegraf_test.py b/tests/operations/telegraf_test.py index 62f48dbe..15cec227 100644 --- a/tests/operations/telegraf_test.py +++ b/tests/operations/telegraf_test.py @@ -13,13 +13,15 @@ def test_get_telegraf_options(): env = { 'INFLUX_TOKEN': 'token', 'INFLUX_ORG': 'org', - 'INFLUX_BUCKET': 'bucket' + 'INFLUX_BUCKET': 'bucket', + 'INFLUX_URL': 'http://127.0.0.1:8444' } options = get_telegraf_options(env) assert options == { 'token': 'token', 'org': 'org', - 'bucket': 'bucket' + 'bucket': 'bucket', + 'url': 'http://127.0.0.1:8444' } env.pop('INFLUX_TOKEN') with pytest.raises(TelegrafNotConfiguredError): @@ -34,6 +36,8 @@ def template_path(tmp_dir_path): bucket = "{{ bucket }}" organization = "{{ org }}" token = "{{ token }}" +urls = ["{{ url }}"] + """ with open(path, 'w') as tf: tf.write(template) @@ -45,9 +49,10 @@ def test_generate_telegraf_config(tmp_dir_path, template_path): generate_telegraf_config({ 'token': 'token', 'org': 'org', - 'bucket': 'bucket' + 'bucket': 'bucket', + 'url': 'http://127.0.0.1:8444' }, template_path, test_config_path) with open(test_config_path) as config_path: config = config_path.read() - assert config == '\n[[outputs.influxdb_v2]]\nbucket = "bucket"\norganization = "org"\ntoken = "token"' # noqa + assert config == '\n[[outputs.influxdb_v2]]\nbucket = "bucket"\norganization = "org"\ntoken = "token"\nurls = ["http://127.0.0.1:8444"]\n' # noqa From c828ee8346493c42e243503ef6e3438c613f9ad9 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 18 Apr 2024 16:51:46 +0000 Subject: [PATCH 84/86] Remove INFLUX_ORG and INFLUX_BUCKET --- node_cli/configs/env.py | 2 -- node_cli/operations/telegraf.py | 2 -- tests/operations/telegraf_test.py | 14 +++----------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index 7b619797..7b6bf116 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -36,9 +36,7 @@ 'MONITORING_CONTAINERS': '', 'TELEGRAF': '', 'INFLUX_TOKEN': '', - 'INFLUX_ORG': '', 'INFLUX_URL': '', - 'INFLUX_BUCKET': '', 'TG_API_KEY': '', 'TG_CHAT_ID': '', 'CONTAINER_CONFIGS_DIR': '', diff --git a/node_cli/operations/telegraf.py b/node_cli/operations/telegraf.py index f3d3e232..de876197 100644 --- a/node_cli/operations/telegraf.py +++ b/node_cli/operations/telegraf.py @@ -33,8 +33,6 @@ class TelegrafNotConfiguredError(Exception): def get_telegraf_options(env) -> dict: options = { 'token': env.get('INFLUX_TOKEN'), - 'org': env.get('INFLUX_ORG'), - 'bucket': env.get('INFLUX_BUCKET'), 'url': env.get('INFLUX_URL') } missing = list(filter(lambda k: not options[k], options)) diff --git a/tests/operations/telegraf_test.py b/tests/operations/telegraf_test.py index 15cec227..df41b51d 100644 --- a/tests/operations/telegraf_test.py +++ b/tests/operations/telegraf_test.py @@ -12,15 +12,11 @@ def test_get_telegraf_options(): env = { 'INFLUX_TOKEN': 'token', - 'INFLUX_ORG': 'org', - 'INFLUX_BUCKET': 'bucket', 'INFLUX_URL': 'http://127.0.0.1:8444' } options = get_telegraf_options(env) assert options == { 'token': 'token', - 'org': 'org', - 'bucket': 'bucket', 'url': 'http://127.0.0.1:8444' } env.pop('INFLUX_TOKEN') @@ -32,10 +28,8 @@ def test_get_telegraf_options(): def template_path(tmp_dir_path): path = os.path.join(tmp_dir_path, 'telegraf.conf.j2') template = """ -[[outputs.influxdb_v2]] -bucket = "{{ bucket }}" -organization = "{{ org }}" -token = "{{ token }}" +[[outputs.influxdb]] +http_headers = {"Authorization": "Bearer {{ token }}"} urls = ["{{ url }}"] """ @@ -48,11 +42,9 @@ def test_generate_telegraf_config(tmp_dir_path, template_path): test_config_path = os.path.join(tmp_dir_path, 'telegraf.conf') generate_telegraf_config({ 'token': 'token', - 'org': 'org', - 'bucket': 'bucket', 'url': 'http://127.0.0.1:8444' }, template_path, test_config_path) with open(test_config_path) as config_path: config = config_path.read() - assert config == '\n[[outputs.influxdb_v2]]\nbucket = "bucket"\norganization = "org"\ntoken = "token"\nurls = ["http://127.0.0.1:8444"]\n' # noqa + assert config == '\n[[outputs.influxdb]]\nhttp_headers = {"Authorization": "Bearer token"}\nurls = ["http://127.0.0.1:8444"]\n' # noqa From fd0af6654c70eab28747e30c0cc8cfee465789a0 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 23 Apr 2024 12:04:18 +0000 Subject: [PATCH 85/86] Leave only removing telegraf during update/turn-off --- node_cli/operations/base.py | 16 ---------- node_cli/operations/telegraf.py | 50 ------------------------------- node_cli/utils/docker_utils.py | 13 +++++--- tests/operations/__init__.py | 0 tests/operations/telegraf_test.py | 50 ------------------------------- 5 files changed, 9 insertions(+), 120 deletions(-) delete mode 100644 node_cli/operations/telegraf.py delete mode 100644 tests/operations/__init__.py delete mode 100644 tests/operations/telegraf_test.py diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 3d991094..557d1632 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -45,7 +45,6 @@ ) from node_cli.operations.docker_lvmpy import lvmpy_install # noqa from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images -from node_cli.operations.telegraf import generate_telegraf_config, get_telegraf_options from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables from node_cli.utils.docker_utils import ( @@ -139,9 +138,6 @@ def update(env_filepath: str, env: Dict) -> None: distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') - if env.get('TELEGRAF'): - options = get_telegraf_options(env) - generate_telegraf_config(options) compose_up(env) return True @@ -179,10 +175,6 @@ def init(env_filepath: str, env: dict) -> bool: update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') - if env.get('TELEGRAF'): - options = get_telegraf_options(env) - generate_telegraf_config(options) - compose_up(env) return True @@ -234,10 +226,6 @@ def init_sync( update_resource_allocation(env_type=env['ENV_TYPE']) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) - if env.get('TELEGRAF'): - options = get_telegraf_options(env) - generate_telegraf_config(options) - compose_up(env, sync_node=True) return True @@ -280,10 +268,6 @@ def update_sync(env_filepath: str, env: Dict) -> bool: ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) - if env.get('TELEGRAF'): - options = get_telegraf_options(env) - generate_telegraf_config(options) - compose_up(env, sync_node=True) return True diff --git a/node_cli/operations/telegraf.py b/node_cli/operations/telegraf.py deleted file mode 100644 index de876197..00000000 --- a/node_cli/operations/telegraf.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2024-Present SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import logging - -from node_cli.configs import TELEGRAF_CONFIG_PATH, TELEGRAF_TEMPLATE_PATH -from node_cli.utils.helper import process_template - - -logger = logging.getLogger(__name__) - - -class TelegrafNotConfiguredError(Exception): - pass - - -def get_telegraf_options(env) -> dict: - options = { - 'token': env.get('INFLUX_TOKEN'), - 'url': env.get('INFLUX_URL') - } - missing = list(filter(lambda k: not options[k], options)) - if missing: - raise TelegrafNotConfiguredError('Missing options {missing}') - return options - - -def generate_telegraf_config( - extra_options: dict, - template_path: str = TELEGRAF_TEMPLATE_PATH, - config_path: str = TELEGRAF_CONFIG_PATH -) -> None: - logger.info('Processing telegraf template') - process_template(template_path, config_path, extra_options) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 2656302b..b48a3306 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -41,6 +41,7 @@ SCHAIN_REMOVE_TIMEOUT = 300 IMA_REMOVE_TIMEOUT = 20 +TELEGRAF_REMOVE_TIMEOUT = 20 MAIN_COMPOSE_CONTAINERS = ('skale-api', 'bounty', 'skale-admin') BASE_COMPOSE_SERVICES = ( @@ -86,11 +87,13 @@ def get_all_ima_containers(_all=True) -> list: return docker_client().containers.list(all=_all, filters={'name': 'skale_ima_*'}) -def remove_dynamic_containers(): +def remove_dynamic_containers() -> None: logger.info('Removing sChains containers') rm_all_schain_containers() logger.info('Removing IMA containers') rm_all_ima_containers() + logger.info('Removing telegraf (if exists)') + remove_telegraf() def rm_all_schain_containers(): @@ -103,6 +106,11 @@ def rm_all_ima_containers(): remove_containers(ima_containers, timeout=IMA_REMOVE_TIMEOUT) +def remove_telegraf() -> None: + telegraf = docker_client().containers.list(filters={'name': 'skale_telegraf'}) + remove_containers(telegraf, timeout=TELEGRAF_REMOVE_TIMEOUT) + + def remove_containers(containers, timeout): for container in containers: safe_rm(container, timeout=timeout) @@ -248,9 +256,6 @@ def compose_up(env, sync_node=False): if str_to_bool(env.get('MONITORING_CONTAINERS', 'False')): logger.info('Running monitoring containers') run_cmd(cmd=get_up_compose_cmd(MONITORING_COMPOSE_SERVICES), env=env) - if str_to_bool(env.get('TELEGRAF', 'False')): - logger.info('Running monitoring containers') - run_cmd(cmd=get_up_compose_cmd(TELEGRAF_SERVICES), env=env) if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: logger.info('Running containers for Telegram notifications') run_cmd(cmd=get_up_compose_cmd(NOTIFICATION_COMPOSE_SERVICES), env=env) diff --git a/tests/operations/__init__.py b/tests/operations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/operations/telegraf_test.py b/tests/operations/telegraf_test.py deleted file mode 100644 index df41b51d..00000000 --- a/tests/operations/telegraf_test.py +++ /dev/null @@ -1,50 +0,0 @@ -import os - -import pytest - -from node_cli.operations.telegraf import ( - get_telegraf_options, - generate_telegraf_config, - TelegrafNotConfiguredError -) - - -def test_get_telegraf_options(): - env = { - 'INFLUX_TOKEN': 'token', - 'INFLUX_URL': 'http://127.0.0.1:8444' - } - options = get_telegraf_options(env) - assert options == { - 'token': 'token', - 'url': 'http://127.0.0.1:8444' - } - env.pop('INFLUX_TOKEN') - with pytest.raises(TelegrafNotConfiguredError): - get_telegraf_options(env) - - -@pytest.fixture -def template_path(tmp_dir_path): - path = os.path.join(tmp_dir_path, 'telegraf.conf.j2') - template = """ -[[outputs.influxdb]] -http_headers = {"Authorization": "Bearer {{ token }}"} -urls = ["{{ url }}"] - -""" - with open(path, 'w') as tf: - tf.write(template) - return path - - -def test_generate_telegraf_config(tmp_dir_path, template_path): - test_config_path = os.path.join(tmp_dir_path, 'telegraf.conf') - generate_telegraf_config({ - 'token': 'token', - 'url': 'http://127.0.0.1:8444' - }, template_path, test_config_path) - - with open(test_config_path) as config_path: - config = config_path.read() - assert config == '\n[[outputs.influxdb]]\nhttp_headers = {"Authorization": "Bearer token"}\nurls = ["http://127.0.0.1:8444"]\n' # noqa From c46f60bc6b09084a61a5938ee1bd1896f69a0f3e Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 14 May 2024 10:52:57 +0000 Subject: [PATCH 86/86] Update GitPython to 3.1.41 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e6c260e3..4cefd5da 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def find_version(*file_paths): "python-dotenv==0.21.0", "terminaltables==3.1.10", "requests==2.28.1", - "GitPython==3.1.31", + "GitPython==3.1.41", "packaging==23.0", "python-debian==0.1.49", "python-iptables==1.0.1",