From 1769745d3d2876bf8e6fe9fdd91cec610a322082 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 29 Jun 2023 17:35:54 +0100 Subject: [PATCH 1/8] Update network-ui, add CORS for files folder --- config/nginx.conf | 1 + network-ui | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/nginx.conf b/config/nginx.conf index f1d6887..09d8b10 100644 --- a/config/nginx.conf +++ b/config/nginx.conf @@ -34,6 +34,7 @@ http { } location /files/ { + add_header Access-Control-Allow-Origin *; root /usr/share/nginx/www; } diff --git a/network-ui b/network-ui index 42672c2..a015740 160000 --- a/network-ui +++ b/network-ui @@ -1 +1 @@ -Subproject commit 42672c2b4cef285f7144cc2a704e31e3bc1373b9 +Subproject commit a01574044a0130e8e930e997359379e2c5f90cd5 From ead67806e059ae7df2623477d822fff153520822 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 30 Jun 2023 13:19:39 +0100 Subject: [PATCH 2/8] Update network-ui --- network-ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network-ui b/network-ui index a015740..d0dcfe6 160000 --- a/network-ui +++ b/network-ui @@ -1 +1 @@ -Subproject commit a01574044a0130e8e930e997359379e2c5f90cd5 +Subproject commit d0dcfe677d7039c557d2a896e73198af6445a809 From a990b0f3702b40bde9510c9a93fd3c184af0feb2 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 9 Aug 2023 18:39:06 +0100 Subject: [PATCH 3/8] Filter out of sync nodes --- proxy/config.py | 2 ++ proxy/endpoints.py | 27 ++++++++++++++++++++++++++- proxy/helper.py | 21 +++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/proxy/config.py b/proxy/config.py index 61ae0b4..5e973a6 100644 --- a/proxy/config.py +++ b/proxy/config.py @@ -53,3 +53,5 @@ NGINX_CONTAINER_NAME = 'proxy_nginx' CONTAINER_RUNNING_STATUS = 'running' + +ALLOWED_TIMESTAMP_DIFF = 300 \ No newline at end of file diff --git a/proxy/endpoints.py b/proxy/endpoints.py index 3ed4c06..51d2cce 100644 --- a/proxy/endpoints.py +++ b/proxy/endpoints.py @@ -26,10 +26,11 @@ from Crypto.Hash import keccak from proxy.node_info import get_node_info -from proxy.helper import read_json +from proxy.helper import read_json, make_rpc_call from proxy.config import ENDPOINT, SM_ABI_FILEPATH from proxy.str_formatters import arguments_list_string from proxy.schain_options import parse_schain_options +from proxy.config import ALLOWED_TIMESTAMP_DIFF logger = logging.getLogger(__name__) @@ -53,11 +54,22 @@ def __init__(self, schain_name: str, nodes: list): self._format_nodes(nodes) def _format_nodes(self, nodes): + for node in nodes: + http_endpoint = node['http_endpoint_domain'] + node['block_ts'] = get_block_ts(http_endpoint) + + max_ts = max(node['block_ts'] for node in nodes) + logger.info(f'max_ts: {max_ts}') + for node in nodes: http_endpoint = node['http_endpoint_domain'] if not url_ok(http_endpoint): logger.warning(f'{http_endpoint} is not accesible, removing from the list') continue + if not is_node_out_of_sync(node['block_ts'], max_ts): + logger.warning(f'{http_endpoint} ts: {node["block_ts"]}, max ts for chain: \ +{max_ts}, allowed timestamp diff: {ALLOWED_TIMESTAMP_DIFF}') + continue self.http_endpoints.append(http_endpoint.removeprefix(URL_PREFIXES['http'])) self.ws_endpoints.append(node['ws_endpoint_domain'].removeprefix(URL_PREFIXES['ws'])) self.fs_endpoints.append(node['domain']) @@ -80,6 +92,19 @@ def url_ok(url) -> bool: return False +def is_node_out_of_sync(ts: int, compare_ts: int) -> bool: + return abs(compare_ts - ts) < ALLOWED_TIMESTAMP_DIFF + + +def get_block_ts(http_endpoint: str) -> int: + res = make_rpc_call(http_endpoint, 'eth_getBlockByNumber', ['latest', False]) + if res and res.json(): + res_data = res.json() + latest_schain_timestamp_hex = res_data['result']['timestamp'] + return int(latest_schain_timestamp_hex, 16) + return -1 + + def schain_name_to_id(name: str) -> str: keccak_hash = keccak.new(data=name.encode("utf8"), digest_bits=256) return '0x' + keccak_hash.hexdigest() diff --git a/proxy/helper.py b/proxy/helper.py index 2f40c01..f171a4b 100644 --- a/proxy/helper.py +++ b/proxy/helper.py @@ -21,6 +21,7 @@ import json import socket import logging +import requests from logging import Formatter, StreamHandler from jinja2 import Environment @@ -66,3 +67,23 @@ def init_default_logger(): handlers.append(stream_handler) logging.basicConfig(level=logging.DEBUG, handlers=handlers) + + + +def post_request(url, json, cookies=None): + try: + return requests.post( + url, + json=json, + cookies=cookies + ) + except requests.exceptions.RequestException as err: + return None + + +def make_rpc_call(http_endpoint, method, params=None): + params = params or [] + return post_request( + http_endpoint, + json={"jsonrpc": "2.0", "method": method, "params": params, "id": 1} + ) From b12ef4c9e8f4488dcc34f2772c2aa087933958aa Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 9 Aug 2023 18:39:23 +0100 Subject: [PATCH 4/8] Filter out of sync nodes --- proxy/config.py | 2 +- proxy/helper.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/proxy/config.py b/proxy/config.py index 5e973a6..f2e0f9f 100644 --- a/proxy/config.py +++ b/proxy/config.py @@ -54,4 +54,4 @@ NGINX_CONTAINER_NAME = 'proxy_nginx' CONTAINER_RUNNING_STATUS = 'running' -ALLOWED_TIMESTAMP_DIFF = 300 \ No newline at end of file +ALLOWED_TIMESTAMP_DIFF = 300 diff --git a/proxy/helper.py b/proxy/helper.py index f171a4b..ccc3609 100644 --- a/proxy/helper.py +++ b/proxy/helper.py @@ -69,7 +69,6 @@ def init_default_logger(): logging.basicConfig(level=logging.DEBUG, handlers=handlers) - def post_request(url, json, cookies=None): try: return requests.post( @@ -77,7 +76,7 @@ def post_request(url, json, cookies=None): json=json, cookies=cookies ) - except requests.exceptions.RequestException as err: + except requests.exceptions.RequestException: return None From 485a11bd1230a8441751ac9fc7e0c07a89ecb2e8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 9 Aug 2023 19:04:41 +0100 Subject: [PATCH 5/8] Update endpoint filtering logic --- proxy/endpoints.py | 4 ++-- proxy/helper.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/proxy/endpoints.py b/proxy/endpoints.py index 51d2cce..ec2feb7 100644 --- a/proxy/endpoints.py +++ b/proxy/endpoints.py @@ -66,7 +66,7 @@ def _format_nodes(self, nodes): if not url_ok(http_endpoint): logger.warning(f'{http_endpoint} is not accesible, removing from the list') continue - if not is_node_out_of_sync(node['block_ts'], max_ts): + if is_node_out_of_sync(node['block_ts'], max_ts): logger.warning(f'{http_endpoint} ts: {node["block_ts"]}, max ts for chain: \ {max_ts}, allowed timestamp diff: {ALLOWED_TIMESTAMP_DIFF}') continue @@ -93,7 +93,7 @@ def url_ok(url) -> bool: def is_node_out_of_sync(ts: int, compare_ts: int) -> bool: - return abs(compare_ts - ts) < ALLOWED_TIMESTAMP_DIFF + return abs(compare_ts - ts) > ALLOWED_TIMESTAMP_DIFF def get_block_ts(http_endpoint: str) -> int: diff --git a/proxy/helper.py b/proxy/helper.py index ccc3609..bb21ad3 100644 --- a/proxy/helper.py +++ b/proxy/helper.py @@ -82,7 +82,9 @@ def post_request(url, json, cookies=None): def make_rpc_call(http_endpoint, method, params=None): params = params or [] - return post_request( + resp = post_request( http_endpoint, json={"jsonrpc": "2.0", "method": method, "params": params, "id": 1} ) + if resp.status_code == 200: + return resp From 036158e4bcd0f1370adbb07d76048a8dbeecf7b8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 9 Aug 2023 19:17:53 +0100 Subject: [PATCH 6/8] Fix make_rpc_call --- proxy/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/helper.py b/proxy/helper.py index bb21ad3..945150e 100644 --- a/proxy/helper.py +++ b/proxy/helper.py @@ -86,5 +86,5 @@ def make_rpc_call(http_endpoint, method, params=None): http_endpoint, json={"jsonrpc": "2.0", "method": method, "params": params, "id": 1} ) - if resp.status_code == 200: + if resp and resp.status_code == 200: return resp From 2a509755d85944cb9de956eb2a9d7d2178c0e485 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 11 Aug 2023 12:38:34 +0100 Subject: [PATCH 7/8] Increase rate to 200r/so --- config/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/nginx.conf b/config/nginx.conf index 09d8b10..588f1a2 100644 --- a/config/nginx.conf +++ b/config/nginx.conf @@ -8,7 +8,7 @@ http { log_format upstreamlog '[$time_local] $request $status $host - $remote_addr to: $upstream_addr - urt: $upstream_response_time msec: $msec req_t: $request_time ($http_referer $http_user_agent)'; access_log /var/log/nginx/access.log upstreamlog; - limit_req_zone $binary_remote_addr zone=one:10m rate=100r/s; + limit_req_zone $binary_remote_addr zone=one:10m rate=200r/s; client_max_body_size 5M; server { From e9128aeac15981faa7bb19b29a5ffc91e56399a3 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 22 Aug 2023 19:03:58 +0100 Subject: [PATCH 8/8] Update metadata --- network-ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network-ui b/network-ui index d0dcfe6..897cd28 160000 --- a/network-ui +++ b/network-ui @@ -1 +1 @@ -Subproject commit d0dcfe677d7039c557d2a896e73198af6445a809 +Subproject commit 897cd287e5093514848900fd79e8d5762dc61f0d