Merge pull request #750 from skalenetwork/add-telegraf

Add optional telegraf service setup parameters

Dockerfile

@@ -1,15 +1,9 @@
-FROM ubuntu:20.04
+FROM python:3.11-buster
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y software-properties-common
-RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get install -y  \
                        git \
-                       python3.11 \
-                       libpython3.11-dev \
-                       python3.11-venv \
-                       python3.11-distutils \
-                       python3.11-dev \
                        build-essential \
                        zlib1g-dev \
                        libssl-dev \

node_cli/configs/__init__.py

@@ -158,3 +158,6 @@ def _get_env():
 
 AUTOLOAD_KERNEL_MODULES_PATH = '/etc/modules'
 BTRFS_KERNEL_MODULE = 'btrfs'
+
+TELEGRAF_TEMPLATE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'telegraf.conf.j2')
+TELEGRAF_CONFIG_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'telegraf.conf')

node_cli/configs/env.py

@@ -34,6 +34,9 @@
 
 OPTIONAL_PARAMS = {
     'MONITORING_CONTAINERS': '',
+    'TELEGRAF': '',
+    'INFLUX_TOKEN': '',
+    'INFLUX_URL': '',
     'TG_API_KEY': '',
     'TG_CHAT_ID': '',
     'CONTAINER_CONFIGS_DIR': '',

node_cli/operations/base.py

@@ -174,6 +174,7 @@ def init(env_filepath: str, env: dict) -> bool:
     )
     update_resource_allocation(env_type=env['ENV_TYPE'])
     update_images(env.get('CONTAINER_CONFIGS_DIR') != '')
+
     compose_up(env)
     return True
 
@@ -224,6 +225,7 @@ def init_sync(
     )
     update_resource_allocation(env_type=env['ENV_TYPE'])
     update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True)
+
     compose_up(env, sync_node=True)
     return True
 
@@ -265,6 +267,7 @@ def update_sync(env_filepath: str, env: Dict) -> bool:
         distro.version()
     )
     update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True)
+
     compose_up(env, sync_node=True)
     return True
 
@@ -310,6 +313,7 @@ def restore(env, backup_path, config_only=False):
         distro.version()
     )
     update_resource_allocation(env_type=env['ENV_TYPE'])
+
     if not config_only:
         compose_up(env)
 

node_cli/utils/docker_utils.py

@@ -41,6 +41,7 @@
 
 SCHAIN_REMOVE_TIMEOUT = 300
 IMA_REMOVE_TIMEOUT = 20
+TELEGRAF_REMOVE_TIMEOUT = 20
 
 MAIN_COMPOSE_CONTAINERS = ('skale-api', 'bounty', 'skale-admin')
 BASE_COMPOSE_SERVICES = (
@@ -53,7 +54,8 @@
     'watchdog',
     'filebeat'
 )
-MONITORING_COMPOSE_SERVICES = ('node-exporter', 'advisor')
+MONITORING_COMPOSE_SERVICES = ('node-exporter', 'advisor',)
+TELEGRAF_SERVICES = ('telegraf',)
 NOTIFICATION_COMPOSE_SERVICES = ('celery',)
 COMPOSE_TIMEOUT = 10
 
@@ -85,11 +87,13 @@ def get_all_ima_containers(_all=True) -> list:
     return docker_client().containers.list(all=_all, filters={'name': 'skale_ima_*'})
 
 
-def remove_dynamic_containers():
+def remove_dynamic_containers() -> None:
     logger.info('Removing sChains containers')
     rm_all_schain_containers()
     logger.info('Removing IMA containers')
     rm_all_ima_containers()
+    logger.info('Removing telegraf (if exists)')
+    remove_telegraf()
 
 
 def rm_all_schain_containers():
@@ -102,6 +106,11 @@ def rm_all_ima_containers():
     remove_containers(ima_containers, timeout=IMA_REMOVE_TIMEOUT)
 
 
+def remove_telegraf() -> None:
+    telegraf = docker_client().containers.list(filters={'name': 'skale_telegraf'})
+    remove_containers(telegraf, timeout=TELEGRAF_REMOVE_TIMEOUT)
+
+
 def remove_containers(containers, timeout):
     for container in containers:
         safe_rm(container, timeout=timeout)
@@ -162,7 +171,7 @@ def get_logs_backup_filepath(container: Container) -> str:
 def ensure_volume(name: str, size: int, driver='lvmpy', dutils=None):
     dutils = dutils or docker_client()
     if is_volume_exists(name, dutils=dutils):
-        logger.info('Volume %s already exits', name)
+        logger.info('Volume %s already exist', name)
         return
     logger.info('Creating volume %s, size: %d', name, size)
     driver_opts = {'size': str(size)} if driver == 'lvmpy' else None
@@ -244,7 +253,7 @@ def compose_up(env, sync_node=False):
         env['SGX_CERTIFICATES_DIR_NAME'] = SGX_CERTIFICATES_DIR_NAME
 
     run_cmd(cmd=get_up_compose_cmd(BASE_COMPOSE_SERVICES), env=env)
-    if str_to_bool(env.get('MONITORING_CONTAINERS', '')):
+    if str_to_bool(env.get('MONITORING_CONTAINERS', 'False')):
         logger.info('Running monitoring containers')
         run_cmd(cmd=get_up_compose_cmd(MONITORING_COMPOSE_SERVICES), env=env)
     if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env:

scripts/run_tests.sh

@@ -8,4 +8,4 @@ LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \
     TEST_HOME_DIR="$PROJECT_DIR/tests/" \
     GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \
     DOTENV_FILEPATH='tests/test-env' \
-    py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@
+    py.test --cov=$PROJECT_DIR/ tests/ $@

setup.py

@@ -61,7 +61,7 @@ def find_version(*file_paths):
         "python-dotenv==0.21.0",
         "terminaltables==3.1.10",
         "requests==2.28.1",
-        "GitPython==3.1.31",
+        "GitPython==3.1.41",
         "packaging==23.0",
         "python-debian==0.1.49",
         "python-iptables==1.0.1",