Merge pull request #121 from lidofinance/feature/steth-pool-metrics

Feature/steth pool metrics

README.md

@@ -174,6 +174,17 @@ Consists previous and current frame variables, which are used by the Oracle to d
 | **prevTransientValidators**       <br> *gauge* |                                                                  | every SLEEP seconds                       |                                                                                                                                                                        |
 | **prevTransientBalance**          <br> *gauge* |                                                                  | every SLEEP seconds                       |                                                                                                                                                                        |
 
+### 4. stETH oracle state
+Consists pool and oracle stETH peg.
+
+| name                                     | description                                                      | frequency                                 | goal                                                                                                                                                                   |
+| ---------------------------------------- | -----------------------------------------------------------------|-------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **stethOraclePrice**              <br> *gauge* | | every SLEEP seconds | |
+| **stethPoolPrice**                <br> *gauge* | | every SLEEP seconds | |
+| **stethPoolEthBalance**           <br> *gauge* | | every SLEEP seconds | |
+| **stethPoolStethBalance**         <br> *gauge* | | every SLEEP seconds | |
+| **beaconNodeTimeoutCount**         <br> *gauge* | | every SLEEP seconds | |
+
 # License
 
 2020 Lido <[email protected]>

app/beacon.py

@@ -12,6 +12,9 @@
 import requests
 from requests.compat import urljoin
 
+from requests.exceptions import ConnectTimeout
+from exceptions import BeaconConnectionTimeoutException
+
 
 def get_beacon(provider, slots_per_epoch):
     version = requests.get(urljoin(provider, 'eth/v1/node/version')).text
@@ -22,6 +25,14 @@ def get_beacon(provider, slots_per_epoch):
         return Prysm(provider, slots_per_epoch)
     raise ValueError('Unknown beacon')
 
+def proxy_connect_timeout_exception(func):
+    def inner(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except ConnectTimeout as exc:
+            raise BeaconConnectionTimeoutException() from exc
+    return inner
+
 
 class Lighthouse:
     api_version = 'eth/v1/node/version'
@@ -37,12 +48,16 @@ def __init__(self, url, slots_per_epoch):
         self.slots_per_epoch = slots_per_epoch
         self.version = requests.get(urljoin(url, self.api_version)).json()
 
+
+    @proxy_connect_timeout_exception
     def get_finalized_epoch(self):
         return int(requests.get(urljoin(self.url, self.api_beacon_head_finality_checkpoints)).json()['data']['finalized']['epoch'])
 
+    @proxy_connect_timeout_exception
     def get_genesis(self):
         return int(requests.get(urljoin(self.url, self.api_genesis)).json()['data']['genesis_time'])
 
+    @proxy_connect_timeout_exception
     def get_actual_slot(self):
         actual_slots = {}
         response = requests.get(urljoin(self.url, self.api_beacon_head_actual)).json()
@@ -51,13 +66,15 @@ def get_actual_slot(self):
         actual_slots['finalized_slot'] = int(response['data']['header']['message']['slot'])
         return actual_slots
 
+    @proxy_connect_timeout_exception
     def _convert_key_list_to_str_arr(self, key_list):
         pubkeys = []
         for key in key_list:
             pubkeys.append('0x' + binascii.hexlify(key).decode())
 
         return pubkeys
 
+    @proxy_connect_timeout_exception
     def get_balances(self, slot, key_list):
         pubkeys = self._convert_key_list_to_str_arr(key_list)
 
@@ -104,23 +121,27 @@ def __init__(self, url, slots_per_epoch):
         self.slots_per_epoch = slots_per_epoch
         self.version = requests.get(urljoin(url, self.api_version)).json()
 
+    @proxy_connect_timeout_exception
     def get_finalized_epoch(self):
         finalized_epoch = int(requests.get(urljoin(self.url, self.api_beacon_head)).json()['finalizedEpoch'])
         return finalized_epoch
 
+    @proxy_connect_timeout_exception
     def get_genesis(self):
         genesis_time = requests.get(urljoin(self.url, self.api_genesis)).json()['genesisTime']
         genesis_time = datetime.datetime.strptime(genesis_time, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)
         genesis_time = int(genesis_time.timestamp())
         return genesis_time
 
+    @proxy_connect_timeout_exception
     def get_actual_slot(self):
         actual_slots = {}
         response = requests.get(urljoin(self.url, self.api_beacon_head)).json()
         actual_slots['actual_slot'] = int(response['headSlot'])
         actual_slots['finalized_slot'] = int(response['finalizedSlot'])
         return actual_slots
 
+    @proxy_connect_timeout_exception
     def get_balances(self, slot, key_list):
         params = {}
         pubkeys = []

app/exceptions.py

@@ -0,0 +1,2 @@
+class BeaconConnectionTimeoutException(Exception):
+    pass

app/oracle.py

@@ -7,11 +7,15 @@
 import os
 import datetime
 import time
+import sys
+
+from exceptions import BeaconConnectionTimeoutException
 
 from prometheus_client import start_http_server
 from web3 import Web3, WebsocketProvider, HTTPProvider
 from web3.exceptions import SolidityError, CannotHandleRequest
 
+
 from beacon import get_beacon
 from contracts import get_total_supply
 from log import init_log
@@ -271,7 +275,13 @@ def main():
                 continue
             else:
                 raise
-
+        except BeaconConnectionTimeoutException as exc:
+            logging.exception(exc)
+            if ( run_as_daemon ):
+                metrics_exporter_state.beaconNodeTimeoutCount.inc()
+                continue
+            else:
+                raise
 
 def run_once():
     update_beacon_data()
@@ -363,12 +373,15 @@ def update_steth_price_oracle_data():
     try:
         block_number = w3.eth.block_number - steth_price_oracle_block_number_shift
 
+
         oracle_price = steth_price_oracle.functions.stethPrice().call()
         pool_price = steth_curve_pool.functions.get_dy(1, 0, 10 ** 18).call(block_identifier=block_number)
         percentage_diff = 100 * abs(1 - oracle_price / pool_price)
         logging.info(
             f'StETH stats: (pool price - {pool_price / 1e18:.6f}, oracle price - {oracle_price / 1e18:.6f}, difference - {percentage_diff:.2f}%)'
         )
+
+        metrics_exporter_state.set_steth_pool_metrics(oracle_price, pool_price)
 
         proof_params = steth_price_oracle.functions.getProofParams().call()
 
@@ -393,11 +406,12 @@ def update_steth_price_oracle_data():
         )
 
         w3.eth.call(tx)
+        logging.info('Calling tx locally succeeded.')
         sign_and_send_tx(tx)
     except SolidityError as sl:
         logging.error(f'Tx call failed : {sl}')
     except Exception as exc:
-        logging.error(f'Unexpected exception. {type(exc)}')
+        logging.exception(f'Unexpected exception. {type(exc)}')
 
 
 def sleep():

app/prometheus_metrics.py

@@ -61,6 +61,11 @@ def __init__(self):
         # self.resourceSharedMemorySize = Gauge('resourceSharedMemorySize', 'resourceSharedMemorySize')
         # self.resourceUnsharedMemorySize = Gauge('resourceUnsharedMemorySize', 'resourceUnsharedMemorySize')
 
+        self.stethOraclePrice = Gauge('stethOraclePrice', 'stethOraclePrice')
+        self.stethPoolPrice = Gauge('stethPoolPrice', 'stethPoolPrice')
+
+        self.beaconNodeTimeoutCount = Gauge('beaconNodeTimeoutCount', 'beaconNodeTimeoutCount')
+
     def set_current_pool_metrics(self, metrics: PoolMetrics):
         self.currentEthV1BlockNumber.set(metrics.blockNumber)
         self.currentEpoch.set(metrics.epoch)
@@ -90,6 +95,10 @@ def set_prev_pool_metrics(self, metrics: PoolMetrics):
         self.prevTransientValidators.set(metrics.getTransientValidators())
         self.prevTransientBalance.set(metrics.getTransientBalance())
 
+    def set_steth_pool_metrics(self, oraclePrice, poolPrice):
+        self.stethOraclePrice.set(oraclePrice)
+        self.stethPoolPrice.set(poolPrice)
+
 
 metrics_exporter_state = MetricsExporterState()
 

assets/StableSwapPool.json

@@ -9,5 +9,12 @@
     ],
     "stateMutability": "view",
     "type": "function"
+  },
+  {
+    "name": "balances",
+    "outputs": [{ "type": "uint256", "name": "" }],
+    "inputs": [{ "type": "uint256", "name": "i" }],
+    "stateMutability": "view",
+    "type": "function"
   }
 ]

tests/test_configurations.py

@@ -139,8 +139,9 @@ def test_with_priv_key_with_daemon_no_sleep():
         #   'gas': 42, 'to': '0xcD3db5ca818a645359e09543Cc0e5b7bB9593229',
         #   'data': '0x62eeb732000000000000000000000000000000000000000000000000000000000000047400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
         # }
-        tx_line = lines[-4]
+        tx_line = lines[-14]
         sleep_line = lines[-1]
+        print(tx_line, sleep_line)
         assert 'TX successful' in tx_line
         assert 'We are in DAEMON mode. Sleep' in sleep_line
         sleep = float(sleep_line.split('Sleep')[-1].split('s and continue')[0])
@@ -179,7 +180,7 @@ def test_with_priv_key_with_daemon_with_sleep():
         #   'gas': 42, 'to': '0xcD3db5ca818a645359e09543Cc0e5b7bB9593229',
         #   'data': '0x62eeb732000000000000000000000000000000000000000000000000000000000000047400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
         # }
-        tx_line = lines[-4]
+        tx_line = lines[-14]
         sleep_line = lines[-1]
         assert 'TX successful' in tx_line
         assert 'We are in DAEMON mode. Sleep' in sleep_line