Add metrics collector to proxy

.gitignore

@@ -8,6 +8,7 @@ __pycache__
 
 abi.json
 chains.json
+metrics.json
 
 conf/upstreams/*.conf
 conf/chains/*.conf

docker-compose.yml

@@ -33,3 +33,20 @@ services:
       options:
         max-file: "200"
         max-size: "500m"
+  metrics:
+    environment:
+      ETH_ENDPOINT: ${ETH_ENDPOINT}
+      NETWORK_NAME: ${NETWORK_NAME}
+    image: metrics:latest
+    container_name: metrics
+    build:
+      context: ./metrics
+      dockerfile: Dockerfile
+    volumes:
+      - ./data:/data
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "5"
+        max-size: "50m"
+    restart: unless-stopped

metrics/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.12.3-bookworm
+
+RUN apt-get update
+
+RUN mkdir /usr/src/metrics /data
+WORKDIR /usr/src/metrics
+
+COPY requirements.txt ./
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+ENV PYTHONPATH="/usr/src/metrics"
+ENV COLUMNS=80
+
+CMD python /usr/src/metrics/src/main.py

metrics/requirements.txt

@@ -0,0 +1,2 @@
+web3==6.19.0
+requests==2.32.3

metrics/src/collector.py

@@ -0,0 +1,114 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of portal-metrics
+#
+#   Copyright (C) 2024 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import json
+import logging
+import asyncio
+import aiohttp
+from datetime import datetime
+from typing import Any, Dict, List, Tuple
+
+import requests
+
+from explorer import get_address_counters_url, get_chain_stats
+from gas import calc_avg_gas_price
+from config import METRICS_FILEPATH
+
+logger = logging.getLogger(__name__)
+
+
+def get_metadata_url(network_name: str):
+    # return f'https://raw.githubusercontent.com/skalenetwork/skale-network/master/metadata/{network_name}/chains.json' # noqa
+    return f'https://raw.githubusercontent.com/skalenetwork/skale-network/update-mainnet-chains-metadata/metadata/{network_name}/chains.json' # noqa
+
+
+def download_metadata(network_name: str):
+    url = get_metadata_url(network_name)
+    response = requests.get(url)
+    response.raise_for_status()
+    return response.json()
+
+
+async def get_address_counters(session, network, chain_name, address):
+    url = get_address_counters_url(network, chain_name, address)
+    async with session.get(url) as response:
+        return await response.json()
+
+
+async def get_all_address_counters(network, chain_name, addresses):
+    results = {}
+    async with aiohttp.ClientSession() as session:
+        tasks = []
+        for address in addresses:
+            tasks.append(get_address_counters(session, network, chain_name, address))
+
+        responses = await asyncio.gather(*tasks)
+
+        for address, response in zip(addresses, responses):
+            results[address] = response
+
+    return results
+
+
+async def _fetch_counters_for_app(network_name, chain_name, app_name, app_info):
+    logger.info(f'fetching counters for app {app_name}')
+    if 'contracts' in app_info:
+        counters = await get_all_address_counters(network_name, chain_name, app_info['contracts'])
+        return app_name, counters
+    return app_name, None
+
+
+async def fetch_counters_for_apps(chain_info, network_name, chain_name):
+    tasks = []
+    for app_name, app_info in chain_info['apps'].items():
+        task = _fetch_counters_for_app(network_name, chain_name, app_name, app_info)
+        tasks.append(task)
+    return await asyncio.gather(*tasks)
+
+
+def transform_to_dict(apps_counters: List[Tuple[str, Any]] | None) -> Dict[str, Any]:
+    if not apps_counters:
+        return {}
+    results = {}
+    for app_name, counters in apps_counters:
+        results[app_name] = counters
+    return results
+
+
+def collect_metrics(network_name: str):
+    metadata = download_metadata(network_name)
+    metrics = {}
+    for chain_name, chain_info in metadata.items():
+        apps_counters = None
+        chain_stats = get_chain_stats(network_name, chain_name)
+        if 'apps' in chain_info:
+            apps_counters = asyncio.run(
+                fetch_counters_for_apps(chain_info, network_name, chain_name))
+        metrics[chain_name] = {
+            'chain_stats': chain_stats,
+            'apps_counters': transform_to_dict(apps_counters),
+        }
+    data = {
+        'metrics': metrics,
+        'gas': int(calc_avg_gas_price()),
+        'last_updated': int(datetime.now().timestamp())
+    }
+    logger.info(f'Saving metrics to {METRICS_FILEPATH}')
+    with open(METRICS_FILEPATH, 'w') as f:
+        json.dump(data, f, indent=4, sort_keys=True)

metrics/src/config.py

@@ -0,0 +1,50 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of portal-metrics
+#
+#   Copyright (C) 2024 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import os
+
+ERROR_TIMEOUT = os.getenv('MONITOR_INTERVAL', 60)
+MONITOR_INTERVAL = os.getenv('MONITOR_INTERVAL', 10800)
+NETWORK_NAME = os.getenv('NETWORK_NAME', 'mainnet')
+ENDPOINT = os.environ['ETH_ENDPOINT']
+
+PROXY_ENDPOINTS = {
+    'mainnet': 'mainnet.skalenodes.com',
+    'legacy': 'legacy-proxy.skaleserver.com',
+    'regression': 'regression-proxy.skalenodes.com',
+    'testnet': 'testnet.skalenodes.com'
+}
+
+BASE_EXPLORER_URLS = {
+    'mainnet': 'explorer.mainnet.skalenodes.com',
+    'legacy': 'legacy-explorer.skaleserver.com',
+    'regression': 'regression-explorer.skalenodes.com',
+    'testnet': 'explorer.testnet.skalenodes.com'
+}
+
+STATS_API = {
+    'mainnet': 'https://stats.explorer.mainnet.skalenodes.com/v2/stats',
+}
+
+HTTPS_PREFIX = "https://"
+
+BLOCK_SAMPLING = 100
+GAS_ESTIMATION_ITERATIONS = 300
+
+METRICS_FILEPATH = os.path.join('/', 'data', 'metrics.json')

metrics/src/explorer.py

@@ -0,0 +1,46 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of portal-metrics
+#
+#   Copyright (C) 2024 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+import requests
+from typing import Any
+
+from config import BASE_EXPLORER_URLS, HTTPS_PREFIX
+
+logger = logging.getLogger(__name__)
+
+
+def _get_explorer_url(network, chain_name):
+    explorer_base_url = BASE_EXPLORER_URLS[network]
+    return HTTPS_PREFIX + chain_name + '.' + explorer_base_url
+
+
+def get_chain_stats(network: str, chain_name: str) -> Any:
+    try:
+        explorer_url = _get_explorer_url(network, chain_name)
+        response = requests.get(f'{explorer_url}/api/v2/stats')
+        return response.json()
+    except Exception as e:
+        logger.error(f'Failed to get chain stats: {e}')
+        return None
+
+
+def get_address_counters_url(network: str, chain_name: str, address: str) -> str:
+    explorer_url = _get_explorer_url(network, chain_name)
+    return f'{explorer_url}/api/v2/addresses/{address}/counters'

metrics/src/gas.py

@@ -0,0 +1,48 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of portal-metrics
+#
+#   Copyright (C) 2024 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+from web3 import Web3
+from config import ENDPOINT, GAS_ESTIMATION_ITERATIONS, BLOCK_SAMPLING
+
+logger = logging.getLogger(__name__)
+
+
+def init_w3():
+    logger.info(f'Connecting to {ENDPOINT}...')
+    return Web3(Web3.HTTPProvider(ENDPOINT))
+
+
+def calc_avg_gas_price():
+    block_num = GAS_ESTIMATION_ITERATIONS * BLOCK_SAMPLING
+    logger.info(f'Calculating average gas price for the last {block_num} blocks')
+    w3 = init_w3()
+    block_number = w3.eth.block_number
+    total_gas_used = 0
+
+    logger.info(f'Getting historic block gas prices...')
+    for index in range(GAS_ESTIMATION_ITERATIONS):
+        block_number = block_number - BLOCK_SAMPLING * index
+        block = w3.eth.get_block(block_number)
+        total_gas_used += block['baseFeePerGas']
+
+    avg_gas_price = total_gas_used / GAS_ESTIMATION_ITERATIONS
+    avg_gas_price_gwei = Web3.from_wei(avg_gas_price, 'gwei')
+    logger.info(f'avg_gas_price_gwei: {avg_gas_price_gwei}')
+    return avg_gas_price_gwei

metrics/src/logs.py

@@ -0,0 +1,53 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of portal-metrics
+#
+#   Copyright (C) 2024 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import os
+import sys
+import logging
+import logging.handlers as py_handlers
+from logging import Formatter, StreamHandler
+
+
+LOG_FORMAT = '[%(asctime)s %(levelname)s] %(name)s:%(lineno)d - %(threadName)s - %(message)s'
+LOG_FILEPATH = os.path.join(os.getcwd(), 'portal-metrics.log')
+
+LOG_FILE_SIZE_MB = 300
+LOG_FILE_SIZE_BYTES = LOG_FILE_SIZE_MB * 1000000
+LOG_BACKUP_COUNT = 3
+
+MONITOR_INTERVAL = 600
+
+
+def get_file_handler(log_filepath, log_level):
+    formatter = Formatter(LOG_FORMAT)
+    f_handler = py_handlers.RotatingFileHandler(
+        log_filepath, maxBytes=LOG_FILE_SIZE_BYTES,
+        backupCount=LOG_BACKUP_COUNT)
+    f_handler.setFormatter(formatter)
+    f_handler.setLevel(log_level)
+    return f_handler
+
+
+def init_default_logger():
+    formatter = Formatter(LOG_FORMAT)
+    f_handler = get_file_handler(LOG_FILEPATH, logging.INFO)
+    stream_handler = StreamHandler(sys.stderr)
+    stream_handler.setFormatter(formatter)
+    stream_handler.setLevel(logging.INFO)
+    logging.basicConfig(level=logging.DEBUG, handlers=[f_handler, stream_handler])