diff --git a/.gitignore b/.gitignore index b618432..703d661 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ local* # Python venv __pycache__ +.coverage diff --git a/requirements.txt b/requirements.txt index f5d0a73..5810f0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ prometheus-client==0.13.1 -pyyaml==6.0 +pyyaml==6.0.1 schema==0.7.5 websockets==10.4 structlog==22.1.0 diff --git a/src/collectors.py b/src/collectors.py index dd838eb..a7c63dc 100644 --- a/src/collectors.py +++ b/src/collectors.py @@ -5,9 +5,10 @@ class EvmCollector(): """A collector to fetch information about evm compatible RPC endpoints.""" - def __init__(self, url, labels, chain_id, **client_parameters): + def __init__(self, url, labels, chain_id, network_status, **client_parameters): self.labels = labels self.chain_id = chain_id + self.network_status = network_status sub_payload = { "method": 'eth_subscribe', @@ -58,9 +59,10 @@ def client_version(self): class ConfluxCollector(): """A collector to fetch information about conflux RPC endpoints.""" - def __init__(self, url, labels, chain_id, **client_parameters): + def __init__(self, url, labels, chain_id, network_status, **client_parameters): self.labels = labels self.chain_id = chain_id + self.network_status = network_status sub_payload = { "method": 'cfx_subscribe', @@ -111,9 +113,10 @@ def client_version(self): class CardanoCollector(): """A collector to fetch information about cardano RPC endpoints.""" - def __init__(self, url, labels, chain_id, **client_parameters): + def __init__(self, url, labels, chain_id, network_status, **client_parameters): self.labels = labels self.chain_id = chain_id + self.network_status = network_status self.block_height_payload = { "id": "exporter", "jsonrpc": "2.0", @@ -140,10 +143,11 @@ def latency(self): class BitcoinCollector(): """A collector to fetch information about Bitcoin RPC endpoints.""" - def __init__(self, url, labels, chain_id, **client_parameters): + def __init__(self, url, labels, chain_id, network_status, **client_parameters): self.labels = labels self.chain_id = chain_id + self.network_status = network_status self.interface = HttpsInterface(url, client_parameters.get('open_timeout'), client_parameters.get('ping_timeout')) self._logger_metadata = { @@ -209,10 +213,11 @@ def latency(self): class FilecoinCollector(): """A collector to fetch information about filecoin RPC endpoints.""" - def __init__(self, url, labels, chain_id, **client_parameters): + def __init__(self, url, labels, chain_id, network_status, **client_parameters): self.labels = labels self.chain_id = chain_id + self.network_status = network_status self.interface = HttpsInterface(url, client_parameters.get('open_timeout'), client_parameters.get('ping_timeout')) self._logger_metadata = { @@ -264,14 +269,19 @@ def latency(self): """Returns connection latency.""" return self.interface.latest_query_latency + def network_status(self): + """Returns network status.""" + return self.interface.network_status + class SolanaCollector(): """A collector to fetch information about solana RPC endpoints.""" - def __init__(self, url, labels, chain_id, **client_parameters): + def __init__(self, url, labels, chain_id, network_status, **client_parameters): self.labels = labels self.chain_id = chain_id + self.network_status = network_status self.interface = HttpsInterface(url, client_parameters.get('open_timeout'), client_parameters.get('ping_timeout')) self._logger_metadata = { @@ -321,10 +331,11 @@ def latency(self): class StarknetCollector(): """A collector to fetch information about starknet RPC endpoints.""" - def __init__(self, url, labels, chain_id, **client_parameters): + def __init__(self, url, labels, chain_id, network_status, **client_parameters): self.labels = labels self.chain_id = chain_id + self.network_status = network_status self.interface = HttpsInterface(url, client_parameters.get('open_timeout'), client_parameters.get('ping_timeout')) @@ -356,10 +367,11 @@ def latency(self): class AptosCollector(): """A collector to fetch information about Aptos endpoints.""" - def __init__(self, url, labels, chain_id, **client_parameters): + def __init__(self, url, labels, chain_id, network_status, **client_parameters): self.labels = labels self.chain_id = chain_id + self.network_status = network_status self.interface = HttpsInterface(url, client_parameters.get('open_timeout'), client_parameters.get('ping_timeout')) diff --git a/src/configuration.py b/src/configuration.py index 2b8acb2..bbb7d38 100644 --- a/src/configuration.py +++ b/src/configuration.py @@ -57,6 +57,10 @@ def _load_configuration(self): And(int), 'network_name': And(str), + 'canonical_name': + And(str), + 'network_status': + And(str), 'network_type': And(str, lambda s: s in ('Testnet', 'Mainnet')), 'collector': diff --git a/src/metrics.py b/src/metrics.py index ead4747..a4f4136 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -12,7 +12,7 @@ class MetricsLoader(): def __init__(self): self._labels = [ 'url', 'provider', 'blockchain', 'network_name', 'network_type', - 'evmChainID' + 'evmChainID', 'canonical_name' ] @property @@ -51,6 +51,14 @@ def client_version_metric(self): 'brpc_client_version', 'Client version for the particular RPC endpoint.', labels=self._labels) + + @property + def network_status_metric(self): + """Returns instantiated network status metric.""" + return InfoMetricFamily( + 'brpc_network_status', + 'Network status - live, preview, degraded', + labels=['canonical_name', 'status']) @property def total_difficulty_metric(self): @@ -126,6 +134,7 @@ def collect(self): disconnects_metric = self._metrics_loader.disconnects_metric block_height_metric = self._metrics_loader.block_height_metric client_version_metric = self._metrics_loader.client_version_metric + network_status_metric = self._metrics_loader.network_status_metric total_difficulty_metric = self._metrics_loader.total_difficulty_metric latency_metric = self._metrics_loader.latency_metric block_height_delta_metric = self._metrics_loader.block_height_delta_metric @@ -149,6 +158,7 @@ def collect(self): total_difficulty_metric, 'total_difficulty') for collector in self._collector_registry: self._write_metric(collector, latency_metric, 'latency') + self._write_metric(collector, network_status_metric, 'network_status') self.delta_compared_to_max( block_height_metric, block_height_delta_metric) self.delta_compared_to_max( @@ -161,5 +171,6 @@ def collect(self): yield client_version_metric yield total_difficulty_metric yield latency_metric + yield network_status_metric yield block_height_delta_metric yield difficulty_delta_metric diff --git a/src/registries.py b/src/registries.py index c1f0482..e2952ab 100644 --- a/src/registries.py +++ b/src/registries.py @@ -10,12 +10,14 @@ class Endpoint(): # pylint: disable=too-few-public-methods """RPC Endpoint class, to store metadata.""" def __init__( # pylint: disable=too-many-arguments - self, url, provider, blockchain, network_name, network_type, + self, url, provider, blockchain, network_name, canonical_name, network_type, network_status, chain_id, **client_parameters): self.url = url self.chain_id = chain_id + self.canonical_name = canonical_name + self.network_status = network_status self.labels = [ - url, provider, blockchain, network_name, network_type, + url, provider, blockchain, network_name, network_type, canonical_name, str(chain_id) ] self.client_parameters = client_parameters @@ -50,6 +52,8 @@ def get_endpoint_registry(self) -> list: self.blockchain, self.get_property('network_name'), self.get_property('network_type'), + self.get_property('network_status'), + self.get_property('canonical_name'), self.get_property('chain_id'), **self.client_parameters)) return endpoints_list @@ -96,5 +100,6 @@ def get_collector_registry(self) -> list: else: collectors_list.append(collector(item.url, item.labels, item.chain_id, + item.network_status, **self.client_parameters)) return collectors_list diff --git a/src/test_collectors.py b/src/test_collectors.py index 5ade458..a96572e 100644 --- a/src/test_collectors.py +++ b/src/test_collectors.py @@ -12,6 +12,7 @@ def setUp(self): self.url = "wss://test.com" self.labels = ["dummy", "labels"] self.chain_id = 123 + self.network_status = "live" self.client_params = {"param1": "dummy", "param2": "data"} self.sub_payload = { "method": 'eth_subscribe', @@ -21,7 +22,7 @@ def setUp(self): } with mock.patch('collectors.WebsocketInterface') as mocked_websocket: self.evm_collector = collectors.EvmCollector( - self.url, self.labels, self.chain_id, **self.client_params) + self.url, self.labels, self.chain_id, self.network_status, **self.client_params) self.mocked_websocket = mocked_websocket def test_websocket_interface_created(self): @@ -99,6 +100,7 @@ def setUp(self): self.url = "wss://test.com" self.labels = ["dummy", "labels"] self.chain_id = 123 + self.network_status = "live" self.client_params = {"param1": "dummy", "param2": "data"} self.sub_payload = { "method": 'cfx_subscribe', @@ -108,7 +110,7 @@ def setUp(self): } with mock.patch('collectors.WebsocketInterface') as mocked_websocket: self.conflux_collector = collectors.ConfluxCollector( - self.url, self.labels, self.chain_id, **self.client_params) + self.url, self.labels, self.chain_id, self.network_status, **self.client_params) self.mocked_websocket = mocked_websocket def test_websocket_interface_created(self): @@ -186,6 +188,7 @@ def setUp(self): self.url = "wss://test.com" self.labels = ["dummy", "labels"] self.chain_id = 123 + self.network_status = "live" self.client_params = {"param1": "dummy", "param2": "data"} self.block_height_payload = { "id": "exporter", @@ -194,7 +197,7 @@ def setUp(self): } with mock.patch('collectors.WebsocketInterface') as mocked_websocket: self.cardano_collector = collectors.CardanoCollector( - self.url, self.labels, self.chain_id, **self.client_params) + self.url, self.labels, self.chain_id, self.network_status, **self.client_params) self.mocked_websocket = mocked_websocket def test_websocket_interface_created(self): @@ -242,6 +245,7 @@ def setUp(self): self.url = "wss://test.com" self.labels = ["dummy", "labels"] self.chain_id = 123 + self.network_status = "live" self.open_timeout = 8 self.ping_timeout = 9 self.client_params = { @@ -259,7 +263,7 @@ def setUp(self): } with mock.patch('collectors.HttpsInterface') as mocked_connection: self.bitcoin_collector = collectors.BitcoinCollector( - self.url, self.labels, self.chain_id, **self.client_params) + self.url, self.labels, self.chain_id, self.network_status, **self.client_params) self.mocked_connection = mocked_connection def test_logger_metadata(self): @@ -384,6 +388,7 @@ def setUp(self): self.url = "wss://test.com" self.labels = ["dummy", "labels"] self.chain_id = 123 + self.network_status = "live" self.open_timeout = 8 self.ping_timeout = 9 self.client_params = { @@ -400,7 +405,7 @@ def setUp(self): } with mock.patch('collectors.HttpsInterface') as mocked_connection: self.filecoin_collector = collectors.FilecoinCollector( - self.url, self.labels, self.chain_id, **self.client_params) + self.url, self.labels, self.chain_id, self.network_status, **self.client_params) self.mocked_connection = mocked_connection def test_logger_metadata(self): @@ -498,6 +503,7 @@ def setUp(self): self.url = "wss://test.com" self.labels = ["dummy", "labels"] self.chain_id = 123 + self.network_status = "live" self.open_timeout = 8 self.ping_timeout = 9 self.client_params = { @@ -514,7 +520,7 @@ def setUp(self): } with mock.patch('collectors.HttpsInterface') as mocked_connection: self.solana_collector = collectors.SolanaCollector( - self.url, self.labels, self.chain_id, **self.client_params) + self.url, self.labels, self.chain_id, self.network_status, **self.client_params) self.mocked_connection = mocked_connection def test_logger_metadata(self): @@ -598,6 +604,7 @@ def setUp(self): self.url = "wss://test.com" self.labels = ["dummy", "labels"] self.chain_id = 123 + self.network_status = "live" self.open_timeout = 8 self.ping_timeout = 9 self.client_params = { @@ -609,7 +616,7 @@ def setUp(self): } with mock.patch('collectors.HttpsInterface') as mocked_connection: self.starknet_collector = collectors.StarknetCollector( - self.url, self.labels, self.chain_id, **self.client_params) + self.url, self.labels, self.chain_id, self.network_status, **self.client_params) self.mocked_connection = mocked_connection def test_https_interface_created(self): @@ -658,13 +665,14 @@ def setUp(self): self.url = "https://test.com" self.labels = ["dummy", "labels"] self.chain_id = 123 + self.network_status = "live" self.open_timeout = 8 self.ping_timeout = 9 self.client_params = { "open_timeout": self.open_timeout, "ping_timeout": self.ping_timeout} with mock.patch('collectors.HttpsInterface') as mocked_connection: self.aptos_collector = collectors.AptosCollector( - self.url, self.labels, self.chain_id, **self.client_params) + self.url, self.labels, self.chain_id, self.network_status, **self.client_params) self.mocked_connection = mocked_connection def test_logger_metadata(self): diff --git a/src/test_configuration.py b/src/test_configuration.py index 407d349..a7b2cf2 100644 --- a/src/test_configuration.py +++ b/src/test_configuration.py @@ -44,10 +44,14 @@ def test_configuration_attribute(self): expected_configuration = { "blockchain": "TestChain", + "canonical_name": + "canonical_name", "chain_id": 1234, "network_name": "TestNetwork", + "network_status": + "live", "network_type": "Mainnet", "collector": diff --git a/src/test_metrics.py b/src/test_metrics.py index 7a6220a..e31e0b8 100644 --- a/src/test_metrics.py +++ b/src/test_metrics.py @@ -13,7 +13,7 @@ class TestMetricsLoader(TestCase): def setUp(self): self.metrics_loader = MetricsLoader() self.labels = [ - 'url', 'provider', 'blockchain', 'network_name', 'network_type', 'evmChainID' + 'url', 'provider', 'blockchain', 'network_name', 'network_type', 'evmChainID', "canonical_name" ] def test_labels(self): @@ -90,6 +90,15 @@ def test_client_version_metric_returns_gauge(self): """Tests the client_version_metric property returns a gauge""" self.assertEqual(InfoMetricFamily, type( self.metrics_loader.client_version_metric)) + + def test_network_status_metri (self): + """Tests the network_status_metric property calls InfoMetric with the correct args""" + with mock.patch('metrics.InfoMetricFamily') as info_mock: + self.metrics_loader.network_status_metric # pylint: disable=pointless-statement + info_mock.assert_called_once_with( + 'brpc_network_status', + 'Network status - live, preview, degraded', + labels=['canonical_name', 'status']) # this metric is chain-global hence just two labels def test_total_difficulty_metric(self): """Tests the total_difficulty_metric property calls GaugeMetric with the correct args""" @@ -170,6 +179,7 @@ def test_collect_yields_correct_metrics(self): self.mocked_loader.return_value.disconnects_metric, self.mocked_loader.return_value.block_height_metric, self.mocked_loader.return_value.client_version_metric, + self.mocked_loader.return_value.network_status_metric, self.mocked_loader.return_value.total_difficulty_metric, self.mocked_loader.return_value.latency_metric, self.mocked_loader.return_value.block_height_delta_metric, @@ -182,14 +192,14 @@ def test_collect_yields_correct_metrics(self): def test_collect_number_of_yields(self): """Tests that the collect method yields the expected number of values""" results = self.prom_collector.collect() - self.assertEqual(9, len(list(results))) + self.assertEqual(10, len(list(results))) def test_get_thread_count(self): """Tests get thread count returns the expected number of threads based on number of metrics and collectors""" thread_count = self.prom_collector.get_thread_count() - # Total of 9 metrics times 2 items in our mocked pool should give 18 - self.assertEqual(18, thread_count) + # Total of 10 metrics times 2 items in our mocked pool should give 18 + self.assertEqual(20, thread_count) def test_collect_thread_max_workers(self): """Tests the max workers is correct for the collect threads""" diff --git a/src/test_registries.py b/src/test_registries.py index a728962..78d6346 100644 --- a/src/test_registries.py +++ b/src/test_registries.py @@ -17,10 +17,12 @@ def setUp(self): self.blockchain = "test_chain" self.network_name = "test_network" self.network_type = "ETH" + self.network_status = "live" + self.canonical_name = "canonical_name" self.chain_id = 123 self.client_params = {"dummy": "data"} self.endpoint = Endpoint(self.url, self.provider, self.blockchain, - self.network_name, self.network_type, + self.network_name, self.canonical_name, self.network_type, self.network_status, self.chain_id, **self.client_params) def test_url_attribute(self): @@ -34,7 +36,7 @@ def test_chain_id_attribute(self): def test_labels_attribute(self): """Tests the labels attribute is set correctly""" labels = [self.url, self.provider, self.blockchain, - self.network_name, self.network_type, str(self.chain_id)] + self.network_name, self.network_type, self.canonical_name, str(self.chain_id)] self.assertEqual(labels, self.endpoint.labels) @@ -213,6 +215,6 @@ def helper_test_collector_registry(test_collector_registry, mock_collector): all(isinstance(col, mock.Mock) for col in collector_list)) calls = [] for item in test_collector_registry.collector_registry.get_endpoint_registry: - calls.append(mock.call(item.url, item.labels, item.chain_id, + calls.append(mock.call(item.url, item.labels, item.chain_id, item.network_status, **test_collector_registry.collector_registry.client_parameters)) mock_collector.assert_has_calls(calls, False) diff --git a/src/tests/fixtures/configuration.yaml b/src/tests/fixtures/configuration.yaml index 53734a2..af67243 100644 --- a/src/tests/fixtures/configuration.yaml +++ b/src/tests/fixtures/configuration.yaml @@ -1,7 +1,9 @@ blockchain: "TestChain" chain_id: 1234 network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "evm" endpoints: - url: wss://test1.com diff --git a/src/tests/fixtures/configuration_aptos.yaml b/src/tests/fixtures/configuration_aptos.yaml index 6daf57b..6fd56af 100644 --- a/src/tests/fixtures/configuration_aptos.yaml +++ b/src/tests/fixtures/configuration_aptos.yaml @@ -1,7 +1,9 @@ blockchain: "Aptos" chain_id: 1234 network_name: "Testnet" +canonical_name: canonical_name network_type: "Testnet" +network_status: live collector: "aptos" endpoints: - url: https://test1.com diff --git a/src/tests/fixtures/configuration_bitcoin.yaml b/src/tests/fixtures/configuration_bitcoin.yaml index 3e390cc..d7a7cb0 100644 --- a/src/tests/fixtures/configuration_bitcoin.yaml +++ b/src/tests/fixtures/configuration_bitcoin.yaml @@ -1,7 +1,9 @@ blockchain: "Bitcoin" chain_id: 1234 network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "bitcoin" endpoints: - url: wss://test1.com diff --git a/src/tests/fixtures/configuration_cardano.yaml b/src/tests/fixtures/configuration_cardano.yaml index f536d37..97604d2 100644 --- a/src/tests/fixtures/configuration_cardano.yaml +++ b/src/tests/fixtures/configuration_cardano.yaml @@ -1,7 +1,9 @@ blockchain: "cardano" chain_id: 1234 network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "cardano" endpoints: - url: wss://test1.com diff --git a/src/tests/fixtures/configuration_conflux.yaml b/src/tests/fixtures/configuration_conflux.yaml index e410731..462c444 100644 --- a/src/tests/fixtures/configuration_conflux.yaml +++ b/src/tests/fixtures/configuration_conflux.yaml @@ -1,7 +1,9 @@ blockchain: "conflux" chain_id: 1234 network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "conflux" endpoints: - url: wss://test1.com diff --git a/src/tests/fixtures/configuration_conn_params.yaml b/src/tests/fixtures/configuration_conn_params.yaml index e3569af..eda2701 100644 --- a/src/tests/fixtures/configuration_conn_params.yaml +++ b/src/tests/fixtures/configuration_conn_params.yaml @@ -1,7 +1,9 @@ blockchain: "TestChain" chain_id: 1234 network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "evm" connection_parameters: open_timeout: 1 diff --git a/src/tests/fixtures/configuration_evm.yaml b/src/tests/fixtures/configuration_evm.yaml index 80f3ac7..ccfba36 100644 --- a/src/tests/fixtures/configuration_evm.yaml +++ b/src/tests/fixtures/configuration_evm.yaml @@ -2,6 +2,8 @@ blockchain: "other" chain_id: 1234 network_name: "TestNetwork" network_type: "Mainnet" +network_status: live +canonical_name: canonical_name collector: "evm" endpoints: - url: wss://test1.com diff --git a/src/tests/fixtures/configuration_filecoin.yaml b/src/tests/fixtures/configuration_filecoin.yaml index f051de5..0e5daa3 100644 --- a/src/tests/fixtures/configuration_filecoin.yaml +++ b/src/tests/fixtures/configuration_filecoin.yaml @@ -1,7 +1,9 @@ blockchain: "filecoin" chain_id: 1234 network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "filecoin" endpoints: - url: wss://test1.com diff --git a/src/tests/fixtures/configuration_invalid.yaml b/src/tests/fixtures/configuration_invalid.yaml index 10bfd0c..ab523b7 100644 --- a/src/tests/fixtures/configuration_invalid.yaml +++ b/src/tests/fixtures/configuration_invalid.yaml @@ -1,7 +1,9 @@ blockchain: "TestChain" chain_id: '1234' # str instead of int network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "evm" endpoints: - url: wss://test1.com diff --git a/src/tests/fixtures/configuration_solana.yaml b/src/tests/fixtures/configuration_solana.yaml index 3f9e1a4..f084fc9 100644 --- a/src/tests/fixtures/configuration_solana.yaml +++ b/src/tests/fixtures/configuration_solana.yaml @@ -1,7 +1,9 @@ blockchain: "solana" chain_id: 1234 network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "solana" endpoints: - url: wss://test1.com diff --git a/src/tests/fixtures/configuration_starknet.yaml b/src/tests/fixtures/configuration_starknet.yaml index 6ac2d45..b3911e4 100644 --- a/src/tests/fixtures/configuration_starknet.yaml +++ b/src/tests/fixtures/configuration_starknet.yaml @@ -1,7 +1,9 @@ blockchain: "starknet" chain_id: 1234 network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "starknet" endpoints: - url: wss://test1.com diff --git a/src/tests/fixtures/configuration_tron.yaml b/src/tests/fixtures/configuration_tron.yaml index aa4f380..a48829f 100644 --- a/src/tests/fixtures/configuration_tron.yaml +++ b/src/tests/fixtures/configuration_tron.yaml @@ -1,7 +1,9 @@ blockchain: "Tron" chain_id: 1234 network_name: "Testnet" +canonical_name: canonical_name network_type: "Testnet" +network_status: live collector: "tron" endpoints: - url: https://test1.com diff --git a/src/tests/fixtures/configuration_unsupported_blockchain.yaml b/src/tests/fixtures/configuration_unsupported_blockchain.yaml index 4a157a7..b6e7f88 100644 --- a/src/tests/fixtures/configuration_unsupported_blockchain.yaml +++ b/src/tests/fixtures/configuration_unsupported_blockchain.yaml @@ -1,7 +1,9 @@ blockchain: "bitcoin" chain_id: 1234 network_name: "TestNetwork" +canonical_name: canonical_name network_type: "Mainnet" +network_status: live collector: "cardano" endpoints: - url: wss://test1.com