Merge pull request #314 from skalenetwork/beta

2.1 Stable

.github/workflows/issue_check.yml

@@ -0,0 +1,19 @@
+name: Get linked issues
+on:
+  pull_request:
+    types: [ edited, synchronize, opened, reopened ]
+
+jobs:
+  check-linked-issues:
+    name: Check if pull request has linked issues
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get issues
+        id: get-issues
+        uses: mondeja/pr-linked-issues-action@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: PR has not linked issues
+        if: join(steps.get-issues.outputs.issues) == ''
+        run: 
+          exit 1

.github/workflows/test.yml

@@ -12,7 +12,6 @@ jobs:
       ETH_PRIVATE_KEY: ${{ secrets.ETH_PRIVATE_KEY }}
       ENDPOINT: ${{ secrets.ENDPOINT }}
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-      MANAGER_TAG: "1.8.2-develop.44"
       SGX_URL: ""
 
     steps:

VERSION

@@ -1 +1 @@
-2.1.3
+2.1.4

requirements.txt

@@ -1,2 +1 @@
-redis==3.5.3
-skale.py==5.6.dev6
+skale.py==5.8dev4

scripts/run-test-containers.sh

@@ -67,7 +67,7 @@ create_skale_dir
 create_redis_dir
 build tm hnode
 
-if [ -n ${SGX_URL} ]; then
+if [ -z ${SGX_URL} ]; then
     run_containers tm hnode redis
 else
     run_containers sgx tm hnode redis

tests/attempt/v2_test.py

@@ -1,6 +1,6 @@
 import pytest
 from mock import Mock
-from skale.utils.account_tools import send_ether
+from skale.utils.account_tools import send_eth
 
 from transaction_manager.attempt_manager.base import NoCurrentAttemptError
 from transaction_manager.attempt_manager.v2 import AttemptManagerV2
@@ -28,7 +28,7 @@ def create_attempt(nonce=1, index=2, gas_price=10 ** 9, wait_time=30):
 @pytest.fixture
 def account(w3, wallet):
     acc = w3.eth.account.create()
-    send_ether(w3, wallet, acc.address, TEST_ETH_VALUE)
+    send_eth(w3, wallet, acc.address, TEST_ETH_VALUE)
     return acc.address, acc.key.hex()
 
 

tests/conftest.py

@@ -1,7 +1,7 @@
 import pytest
 import redis
 
-from skale.utils.account_tools import send_ether
+from skale.utils.account_tools import send_eth
 from skale.wallets import RedisWalletAdapter, SgxWallet, Web3Wallet
 
 from transaction_manager.attempt_manager import (
@@ -77,7 +77,7 @@ def wallet(w3, w3wallet):
     else:
         return w3wallet
     if isinstance(w, SgxWallet):
-        send_ether(w3, w3wallet, w.address, ETH_AMOUNT_FOR_TESTS)
+        send_eth(w3, w3wallet, w.address, ETH_AMOUNT_FOR_TESTS)
     return w
 
 

tests/eth_test.py

@@ -86,3 +86,10 @@ def test_eth_tx(w3wallet, w3, eth):
     status = eth.wait_for_receipt(h)
     assert status == 1
     assert eth.get_nonce(addr) == second_nonce + 1
+    assert eth.get_tx_block(h) == eth.get_receipt(h)['blockNumber']
+
+
+def test_eth_wait_for_blocks(eth, w3):
+    eth.wait_for_blocks(amount=1, max_time=1)
+    cblock = w3.eth.block_number
+    eth.wait_for_blocks(amount=5, max_time=0, start_block=cblock - 10)

tests/processor_test.py

@@ -7,6 +7,7 @@
 from transaction_manager.structures import TxStatus
 
 from tests.utils.contracts import get_tester_abi
+from tests.utils.timing import in_time
 
 DEFAULT_GAS = 20000
 
@@ -104,25 +105,27 @@ def test_send(proc, w3, rdp, eth, tpool, wallet):
     tx.chain_id = eth.chain_id
     tx.nonce = 0
     proc.attempt_manager.make(tx)
-    tx.gas = 22000
-    tx.gas_price = 10 ** 9
 
     proc.eth.send_tx = mock.Mock(
-        side_effect=ValueError({
-                'code': -32000,
-                'message': 'replacement transaction underpriced'
-            })
+        side_effect=ValueError('unknown error')
     )
     with pytest.raises(SendingError):
         proc.send(tx)
+    # Test that attempt was not saved if it was neither sent or replaced
+    assert proc.attempt_manager.storage.get() is None
     assert tx.tx_hash is None
     assert tx.hashes == []
 
     proc.eth.send_tx = mock.Mock(
-        side_effect=ValueError('unknown error')
+        side_effect=ValueError({
+                'code': -32000,
+                'message': 'replacement transaction underpriced'
+            })
     )
     with pytest.raises(SendingError):
         proc.send(tx)
+    # Test that attempt was saved if it was replaced
+    assert proc.attempt_manager.storage.get().fee == tx.fee
     assert tx.tx_hash is None
     assert tx.hashes == []
 
@@ -133,6 +136,8 @@ def test_send(proc, w3, rdp, eth, tpool, wallet):
 
     proc.eth.send_tx = mock.Mock(return_value='0x213812903813123')
     proc.send(tx)
+    # Test that attempt was saved if it was sent
+    assert proc.attempt_manager.storage.get().fee == tx.fee
     assert tx.tx_hash == '0x213812903813123'
     assert tx.hashes == ['0x12323213213321321', '0x213812903813123']
 
@@ -183,3 +188,16 @@ def test_aquire_estimate_gas_revert(proc, w3, rdp, tpool, wallet):
     assert tx.status == TxStatus.DROPPED
 
     assert tx.tx_id.encode('utf-8') not in tpool.to_list()
+
+
+def test_confirm(proc, w3, rdp, tpool, wallet):
+    tx = push_tx(w3, rdp, tpool, wallet)
+    proc.attempt_manager.make(tx)
+    proc.send(tx)
+    proc.wait(tx, max_time=proc.attempt_manager.current.wait_time)
+    # Make sure it is confirmed after reasonable number of seconds
+    with in_time(8):
+        proc.confirm(tx)
+    # Make sure next time it is confirmed instantly
+    with in_time(0.1):
+        proc.confirm(tx)

tests/transaction_test.py

@@ -126,6 +126,11 @@ def test_tx_from_bytes():
     tx = Tx.from_bytes(tx_id, missing_field_hash)
     assert tx.hashes == []
 
+    with_type = b'{"attempts": 0, "type": "0x0", "status": "PROPOSED", "chainId": null, "data": {"test": 1}, "from": null, "gas": 22000, "gasPrice": 1000000000, "hashes": [], "nonce": 3, "score": 1, "sent_ts": null, "to": "0x1", "tx_hash": null, "value": 1}'  # noqa
+    tx = Tx.from_bytes(tx_id, with_type)
+    expected = b'{"attempts": 0, "chainId": null, "data": {"test": 1}, "from": null, "gas": 22000, "gasPrice": 1000000000, "hashes": [], "maxFeePerGas": null, "maxPriorityFeePerGas": null, "multiplier": 1.2, "nonce": 3, "score": 1, "sent_ts": null, "status": "PROPOSED", "to": "0x1", "tx_hash": null, "tx_id": "tx-1232321332132131331321", "value": 1}'  # noqa
+    tx.to_bytes() == expected
+
 
 def test_is_sent_by_ima():
     tx = Tx(

transaction_manager/config.py

@@ -34,6 +34,7 @@
 NODE_DATA_PATH = '/skale_node_data'
 
 # General
+RESTART_TIMEOUT: int = 3
 BASE_WAITING_TIME: int = 25
 CONFIRMATION_BLOCKS: int = 6
 MAX_RESUBMIT_AMOUNT: int = 10
@@ -43,6 +44,7 @@
 DISABLE_GAS_ESTIMATION = False
 TXRECORD_EXPIRATION = 24 * 60 * 60  # 1 day
 DEFAULT_ID_LEN = 19
+DEFAULT_GAS_LIMIT: int = 1000000
 IMA_ID_SUFFIX = 'js'
 
 # V1

transaction_manager/eth.py

@@ -30,6 +30,7 @@
 from .config import (
     AVG_GAS_PRICE_INC_PERCENT,
     CONFIRMATION_BLOCKS,
+    DEFAULT_GAS_LIMIT,
     DISABLE_GAS_ESTIMATION,
     GAS_MULTIPLIER,
     MAX_WAITING_TIME,
@@ -147,7 +148,7 @@ def calculate_gas(self, tx: Tx) -> int:
         multiplier = tx.multiplier
         multiplier = multiplier or GAS_MULTIPLIER
         if DISABLE_GAS_ESTIMATION:
-            return int(etx['gas'] * multiplier)
+            return int(etx.get('gas', DEFAULT_GAS_LIMIT) * multiplier)
 
         logger.info('Estimating gas for %s', etx)
 
@@ -192,32 +193,14 @@ def get_nonce(self, address: str) -> int:
         checksum_addres = self.w3.toChecksumAddress(address)
         return self.w3.eth.get_transaction_count(checksum_addres)
 
-    def get_status(
-        self,
-        tx_hash: str,
-        raise_err: bool = False
-    ) -> int:
-        try:
-            casted_hash = cast(HexStr, tx_hash)
-            receipt = self.w3.eth.get_transaction_receipt(casted_hash)
-        except TransactionNotFound as e:
-            if raise_err:
-                raise e
-            else:
-                return -1
-        logger.debug(f'Receipt for {tx_hash}: {receipt}')
-        rstatus = receipt.get('status', -1)
-        if rstatus < 0:
-            logger.error('Receipt has no "status" field')
-            return rstatus
-        return rstatus
-
     def wait_for_blocks(
         self,
         amount: int = CONFIRMATION_BLOCKS,
-        max_time: int = MAX_WAITING_TIME
+        max_time: int = MAX_WAITING_TIME,
+        start_block: Optional[int] = None
     ) -> None:
-        current_block = start_block = self.w3.eth.block_number
+        current_block = self.w3.eth.block_number
+        start_block = start_block or current_block
         current_ts = start_ts = time.time()
         while current_block - start_block < amount and \
                 current_ts - start_ts < max_time:
@@ -235,13 +218,39 @@ def wait_for_receipt(
         max_time: int = MAX_WAITING_TIME,
     ) -> int:
         start_ts = time.time()
-        rstatus = None
-        while rstatus is None and time.time() - start_ts < max_time:
-            try:
-                rstatus = self.get_status(tx_hash, raise_err=True)
-            except TransactionNotFound:
+        rstatus = -1
+        while rstatus == -1 and time.time() - start_ts < max_time:
+            rstatus = self.get_status(tx_hash)
+            if rstatus == -1:
                 time.sleep(1)
 
-        if rstatus is None:
+        if rstatus == -1:
             raise ReceiptTimeoutError(f'No receipt after {max_time}')
         return rstatus
+
+    def get_receipt(self, tx_hash: str) -> Optional[Dict]:
+        casted_hash = cast(HexStr, tx_hash)
+        receipt = None
+        try:
+            casted_hash = cast(HexStr, tx_hash)
+            receipt = self.w3.eth.get_transaction_receipt(casted_hash)
+        except TransactionNotFound:
+            pass
+        return cast(Dict, receipt)
+
+    def get_tx_block(self, tx_hash: str) -> int:
+        receipt = self.get_receipt(tx_hash)
+        if receipt is None:
+            return -1
+        return cast(int, receipt.get('blockNumber'))
+
+    def get_status(self, tx_hash: str) -> int:
+        receipt = self.get_receipt(tx_hash)
+        logger.debug('Receipt for %s: %s', tx_hash, receipt)
+        if receipt is None:
+            return -1
+        rstatus = receipt.get('status', -1)
+        if rstatus < 0:
+            logger.error('Receipt has no "status" field')
+            return rstatus
+        return rstatus

transaction_manager/log.py

@@ -17,16 +17,16 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import hashlib
 import logging
 import os
 import re
 import sys
 from logging import Formatter, Handler, StreamHandler
 from logging.handlers import RotatingFileHandler
 from typing import List
+from urllib.parse import urlparse
 
-from .config import NODE_DATA_PATH
+from .config import ENDPOINT, NODE_DATA_PATH, SGX_URL
 
 
 LOG_FOLDER = os.path.join(NODE_DATA_PATH, 'log')
@@ -41,39 +41,44 @@
 LOG_FORMAT = '%(asctime)s [%(levelname)s] [%(module)s:%(lineno)d] %(message)s'  # noqa
 
 
-HIDING_PATTERNS = [
-    r'NEK\:\w+',
-    r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',  # noqa
-    r'ws[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',  # noqa
-    r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b'  # noqa
-]
+def compose_hiding_patterns():
+    sgx_ip = urlparse(SGX_URL).hostname
+    eth_ip = urlparse(ENDPOINT).hostname
+    return {
+        rf'{sgx_ip}': '[SGX_IP]',
+        rf'{eth_ip}': '[ETH_IP]',
+        r'NEK\:\w+': '[SGX_KEY]'
+    }
 
 
 class HidingFormatter(Formatter):
-    def __init__(self, base_formatter: Formatter, patterns: List[str]) -> None:
-        self.base_formatter: Formatter = base_formatter
-        self._patterns: List[str] = patterns
-
-    @classmethod
-    def convert_match_to_sha3(cls, match) -> str:
-        return hashlib.sha3_256(match.group(0).encode('utf-8')).digest().hex()
-
-    def format(self, record):
-        msg = self.base_formatter.format(record)
-        for pattern in self._patterns:
-            pat = re.compile(pattern)
-            msg = pat.sub(self.convert_match_to_sha3, msg)
+    def __init__(self, log_format: str, patterns: dict) -> None:
+        super().__init__(log_format)
+        self._patterns: dict = patterns
+
+    def _filter_sensitive(self, msg) -> str:
+        for match, replacement in self._patterns.items():
+            pat = re.compile(match)
+            msg = pat.sub(replacement, msg)
         return msg
 
-    def __getattr__(self, attr):
-        return getattr(self.base_formatter, attr)
+    def format(self, record) -> str:
+        msg = super().format(record)
+        return self._filter_sensitive(msg)
+
+    def formatException(self, exc_info) -> str:
+        msg = super().formatException(exc_info)
+        return self._filter_sensitive(msg)
+
+    def formatStack(self, stack_info) -> str:
+        msg = super().formatStack(stack_info)
+        return self._filter_sensitive(msg)
 
 
 def init_logger() -> None:
     handlers: List[Handler] = []
-
-    base_formatter = Formatter(LOG_FORMAT)
-    formatter = HidingFormatter(base_formatter, HIDING_PATTERNS)
+    hiding_patterns = compose_hiding_patterns()
+    formatter = HidingFormatter(LOG_FORMAT, hiding_patterns)
 
     f_handler = RotatingFileHandler(
         TM_LOG_PATH,
@@ -98,4 +103,4 @@ def init_logger() -> None:
     f_handler_debug.setLevel(logging.DEBUG)
     handlers.append(f_handler_debug)
 
-    logging.basicConfig(level=logging.DEBUG, handlers=handlers)
+    logging.basicConfig(level=logging.INFO, handlers=handlers)