diff --git a/.github/workflows/issue_check.yml b/.github/workflows/issue_check.yml
new file mode 100644
index 00000000..ee5ec6b2
--- /dev/null
+++ b/.github/workflows/issue_check.yml
@@ -0,0 +1,19 @@
+name: Get linked issues
+on:
+ pull_request:
+ types: [ edited, synchronize, opened, reopened ]
+
+jobs:
+ check-linked-issues:
+ name: Check if pull request has linked issues
+ runs-on: ubuntu-latest
+ steps:
+ - name: Get issues
+ id: get-issues
+ uses: mondeja/pr-linked-issues-action@v2
+ env:
+ GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+ - name: PR has not linked issues
+ if: join(steps.get-issues.outputs.issues) == ''
+ run:
+ exit 1
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index fb8f205d..ab19862b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,7 +12,6 @@ jobs:
ETH_PRIVATE_KEY: ${{ secrets.ETH_PRIVATE_KEY }}
ENDPOINT: ${{ secrets.ENDPOINT }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- MANAGER_TAG: "1.8.2-develop.44"
SGX_URL: ""
steps:
diff --git a/VERSION b/VERSION
index ac2cdeba..7d2ed7c7 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1.3
+2.1.4
diff --git a/helper-scripts b/helper-scripts
index 5cd6ed42..22428990 160000
--- a/helper-scripts
+++ b/helper-scripts
@@ -1 +1 @@
-Subproject commit 5cd6ed42c44ddf33126a4613cbc381c231401910
+Subproject commit 2242899037c120ce43ec87d9b80f2a223c619331
diff --git a/requirements.txt b/requirements.txt
index 2f5aefcc..05c9b8ee 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1 @@
-redis==3.5.3
-skale.py==5.6.dev6
+skale.py==5.8dev4
diff --git a/scripts/run-test-containers.sh b/scripts/run-test-containers.sh
index 8c0202f5..4d83988d 100755
--- a/scripts/run-test-containers.sh
+++ b/scripts/run-test-containers.sh
@@ -67,7 +67,7 @@ create_skale_dir
create_redis_dir
build tm hnode
-if [ -n ${SGX_URL} ]; then
+if [ -z ${SGX_URL} ]; then
run_containers tm hnode redis
else
run_containers sgx tm hnode redis
diff --git a/tests/attempt/v2_test.py b/tests/attempt/v2_test.py
index 28b7d28b..bcadbc01 100644
--- a/tests/attempt/v2_test.py
+++ b/tests/attempt/v2_test.py
@@ -1,6 +1,6 @@
import pytest
from mock import Mock
-from skale.utils.account_tools import send_ether
+from skale.utils.account_tools import send_eth
from transaction_manager.attempt_manager.base import NoCurrentAttemptError
from transaction_manager.attempt_manager.v2 import AttemptManagerV2
@@ -28,7 +28,7 @@ def create_attempt(nonce=1, index=2, gas_price=10 ** 9, wait_time=30):
@pytest.fixture
def account(w3, wallet):
acc = w3.eth.account.create()
- send_ether(w3, wallet, acc.address, TEST_ETH_VALUE)
+ send_eth(w3, wallet, acc.address, TEST_ETH_VALUE)
return acc.address, acc.key.hex()
diff --git a/tests/conftest.py b/tests/conftest.py
index 6b64979d..2c599187 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,7 +1,7 @@
import pytest
import redis
-from skale.utils.account_tools import send_ether
+from skale.utils.account_tools import send_eth
from skale.wallets import RedisWalletAdapter, SgxWallet, Web3Wallet
from transaction_manager.attempt_manager import (
@@ -77,7 +77,7 @@ def wallet(w3, w3wallet):
else:
return w3wallet
if isinstance(w, SgxWallet):
- send_ether(w3, w3wallet, w.address, ETH_AMOUNT_FOR_TESTS)
+ send_eth(w3, w3wallet, w.address, ETH_AMOUNT_FOR_TESTS)
return w
diff --git a/tests/eth_test.py b/tests/eth_test.py
index 1a565e85..b7d46032 100644
--- a/tests/eth_test.py
+++ b/tests/eth_test.py
@@ -86,3 +86,10 @@ def test_eth_tx(w3wallet, w3, eth):
status = eth.wait_for_receipt(h)
assert status == 1
assert eth.get_nonce(addr) == second_nonce + 1
+ assert eth.get_tx_block(h) == eth.get_receipt(h)['blockNumber']
+
+
+def test_eth_wait_for_blocks(eth, w3):
+ eth.wait_for_blocks(amount=1, max_time=1)
+ cblock = w3.eth.block_number
+ eth.wait_for_blocks(amount=5, max_time=0, start_block=cblock - 10)
diff --git a/tests/processor_test.py b/tests/processor_test.py
index 0f7e08ab..af6f48cd 100644
--- a/tests/processor_test.py
+++ b/tests/processor_test.py
@@ -7,6 +7,7 @@
from transaction_manager.structures import TxStatus
from tests.utils.contracts import get_tester_abi
+from tests.utils.timing import in_time
DEFAULT_GAS = 20000
@@ -104,25 +105,27 @@ def test_send(proc, w3, rdp, eth, tpool, wallet):
tx.chain_id = eth.chain_id
tx.nonce = 0
proc.attempt_manager.make(tx)
- tx.gas = 22000
- tx.gas_price = 10 ** 9
proc.eth.send_tx = mock.Mock(
- side_effect=ValueError({
- 'code': -32000,
- 'message': 'replacement transaction underpriced'
- })
+ side_effect=ValueError('unknown error')
)
with pytest.raises(SendingError):
proc.send(tx)
+ # Test that attempt was not saved if it was neither sent or replaced
+ assert proc.attempt_manager.storage.get() is None
assert tx.tx_hash is None
assert tx.hashes == []
proc.eth.send_tx = mock.Mock(
- side_effect=ValueError('unknown error')
+ side_effect=ValueError({
+ 'code': -32000,
+ 'message': 'replacement transaction underpriced'
+ })
)
with pytest.raises(SendingError):
proc.send(tx)
+ # Test that attempt was saved if it was replaced
+ assert proc.attempt_manager.storage.get().fee == tx.fee
assert tx.tx_hash is None
assert tx.hashes == []
@@ -133,6 +136,8 @@ def test_send(proc, w3, rdp, eth, tpool, wallet):
proc.eth.send_tx = mock.Mock(return_value='0x213812903813123')
proc.send(tx)
+ # Test that attempt was saved if it was sent
+ assert proc.attempt_manager.storage.get().fee == tx.fee
assert tx.tx_hash == '0x213812903813123'
assert tx.hashes == ['0x12323213213321321', '0x213812903813123']
@@ -183,3 +188,16 @@ def test_aquire_estimate_gas_revert(proc, w3, rdp, tpool, wallet):
assert tx.status == TxStatus.DROPPED
assert tx.tx_id.encode('utf-8') not in tpool.to_list()
+
+
+def test_confirm(proc, w3, rdp, tpool, wallet):
+ tx = push_tx(w3, rdp, tpool, wallet)
+ proc.attempt_manager.make(tx)
+ proc.send(tx)
+ proc.wait(tx, max_time=proc.attempt_manager.current.wait_time)
+ # Make sure it is confirmed after reasonable number of seconds
+ with in_time(8):
+ proc.confirm(tx)
+ # Make sure next time it is confirmed instantly
+ with in_time(0.1):
+ proc.confirm(tx)
diff --git a/tests/transaction_test.py b/tests/transaction_test.py
index 86845950..d8a67b49 100644
--- a/tests/transaction_test.py
+++ b/tests/transaction_test.py
@@ -126,6 +126,11 @@ def test_tx_from_bytes():
tx = Tx.from_bytes(tx_id, missing_field_hash)
assert tx.hashes == []
+ with_type = b'{"attempts": 0, "type": "0x0", "status": "PROPOSED", "chainId": null, "data": {"test": 1}, "from": null, "gas": 22000, "gasPrice": 1000000000, "hashes": [], "nonce": 3, "score": 1, "sent_ts": null, "to": "0x1", "tx_hash": null, "value": 1}' # noqa
+ tx = Tx.from_bytes(tx_id, with_type)
+ expected = b'{"attempts": 0, "chainId": null, "data": {"test": 1}, "from": null, "gas": 22000, "gasPrice": 1000000000, "hashes": [], "maxFeePerGas": null, "maxPriorityFeePerGas": null, "multiplier": 1.2, "nonce": 3, "score": 1, "sent_ts": null, "status": "PROPOSED", "to": "0x1", "tx_hash": null, "tx_id": "tx-1232321332132131331321", "value": 1}' # noqa
+ tx.to_bytes() == expected
+
def test_is_sent_by_ima():
tx = Tx(
diff --git a/transaction_manager/config.py b/transaction_manager/config.py
index 8521fad1..5e52ce8d 100644
--- a/transaction_manager/config.py
+++ b/transaction_manager/config.py
@@ -34,6 +34,7 @@
NODE_DATA_PATH = '/skale_node_data'
# General
+RESTART_TIMEOUT: int = 3
BASE_WAITING_TIME: int = 25
CONFIRMATION_BLOCKS: int = 6
MAX_RESUBMIT_AMOUNT: int = 10
@@ -43,6 +44,7 @@
DISABLE_GAS_ESTIMATION = False
TXRECORD_EXPIRATION = 24 * 60 * 60 # 1 day
DEFAULT_ID_LEN = 19
+DEFAULT_GAS_LIMIT: int = 1000000
IMA_ID_SUFFIX = 'js'
# V1
diff --git a/transaction_manager/eth.py b/transaction_manager/eth.py
index 5769797c..b7d1e0eb 100644
--- a/transaction_manager/eth.py
+++ b/transaction_manager/eth.py
@@ -30,6 +30,7 @@
from .config import (
AVG_GAS_PRICE_INC_PERCENT,
CONFIRMATION_BLOCKS,
+ DEFAULT_GAS_LIMIT,
DISABLE_GAS_ESTIMATION,
GAS_MULTIPLIER,
MAX_WAITING_TIME,
@@ -147,7 +148,7 @@ def calculate_gas(self, tx: Tx) -> int:
multiplier = tx.multiplier
multiplier = multiplier or GAS_MULTIPLIER
if DISABLE_GAS_ESTIMATION:
- return int(etx['gas'] * multiplier)
+ return int(etx.get('gas', DEFAULT_GAS_LIMIT) * multiplier)
logger.info('Estimating gas for %s', etx)
@@ -192,32 +193,14 @@ def get_nonce(self, address: str) -> int:
checksum_addres = self.w3.toChecksumAddress(address)
return self.w3.eth.get_transaction_count(checksum_addres)
- def get_status(
- self,
- tx_hash: str,
- raise_err: bool = False
- ) -> int:
- try:
- casted_hash = cast(HexStr, tx_hash)
- receipt = self.w3.eth.get_transaction_receipt(casted_hash)
- except TransactionNotFound as e:
- if raise_err:
- raise e
- else:
- return -1
- logger.debug(f'Receipt for {tx_hash}: {receipt}')
- rstatus = receipt.get('status', -1)
- if rstatus < 0:
- logger.error('Receipt has no "status" field')
- return rstatus
- return rstatus
-
def wait_for_blocks(
self,
amount: int = CONFIRMATION_BLOCKS,
- max_time: int = MAX_WAITING_TIME
+ max_time: int = MAX_WAITING_TIME,
+ start_block: Optional[int] = None
) -> None:
- current_block = start_block = self.w3.eth.block_number
+ current_block = self.w3.eth.block_number
+ start_block = start_block or current_block
current_ts = start_ts = time.time()
while current_block - start_block < amount and \
current_ts - start_ts < max_time:
@@ -235,13 +218,39 @@ def wait_for_receipt(
max_time: int = MAX_WAITING_TIME,
) -> int:
start_ts = time.time()
- rstatus = None
- while rstatus is None and time.time() - start_ts < max_time:
- try:
- rstatus = self.get_status(tx_hash, raise_err=True)
- except TransactionNotFound:
+ rstatus = -1
+ while rstatus == -1 and time.time() - start_ts < max_time:
+ rstatus = self.get_status(tx_hash)
+ if rstatus == -1:
time.sleep(1)
- if rstatus is None:
+ if rstatus == -1:
raise ReceiptTimeoutError(f'No receipt after {max_time}')
return rstatus
+
+ def get_receipt(self, tx_hash: str) -> Optional[Dict]:
+ casted_hash = cast(HexStr, tx_hash)
+ receipt = None
+ try:
+ casted_hash = cast(HexStr, tx_hash)
+ receipt = self.w3.eth.get_transaction_receipt(casted_hash)
+ except TransactionNotFound:
+ pass
+ return cast(Dict, receipt)
+
+ def get_tx_block(self, tx_hash: str) -> int:
+ receipt = self.get_receipt(tx_hash)
+ if receipt is None:
+ return -1
+ return cast(int, receipt.get('blockNumber'))
+
+ def get_status(self, tx_hash: str) -> int:
+ receipt = self.get_receipt(tx_hash)
+ logger.debug('Receipt for %s: %s', tx_hash, receipt)
+ if receipt is None:
+ return -1
+ rstatus = receipt.get('status', -1)
+ if rstatus < 0:
+ logger.error('Receipt has no "status" field')
+ return rstatus
+ return rstatus
diff --git a/transaction_manager/log.py b/transaction_manager/log.py
index 387037db..d315d968 100644
--- a/transaction_manager/log.py
+++ b/transaction_manager/log.py
@@ -17,7 +17,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
-import hashlib
import logging
import os
import re
@@ -25,8 +24,9 @@
from logging import Formatter, Handler, StreamHandler
from logging.handlers import RotatingFileHandler
from typing import List
+from urllib.parse import urlparse
-from .config import NODE_DATA_PATH
+from .config import ENDPOINT, NODE_DATA_PATH, SGX_URL
LOG_FOLDER = os.path.join(NODE_DATA_PATH, 'log')
@@ -41,39 +41,44 @@
LOG_FORMAT = '%(asctime)s [%(levelname)s] [%(module)s:%(lineno)d] %(message)s' # noqa
-HIDING_PATTERNS = [
- r'NEK\:\w+',
- r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', # noqa
- r'ws[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', # noqa
- r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b' # noqa
-]
+def compose_hiding_patterns():
+ sgx_ip = urlparse(SGX_URL).hostname
+ eth_ip = urlparse(ENDPOINT).hostname
+ return {
+ rf'{sgx_ip}': '[SGX_IP]',
+ rf'{eth_ip}': '[ETH_IP]',
+ r'NEK\:\w+': '[SGX_KEY]'
+ }
class HidingFormatter(Formatter):
- def __init__(self, base_formatter: Formatter, patterns: List[str]) -> None:
- self.base_formatter: Formatter = base_formatter
- self._patterns: List[str] = patterns
-
- @classmethod
- def convert_match_to_sha3(cls, match) -> str:
- return hashlib.sha3_256(match.group(0).encode('utf-8')).digest().hex()
-
- def format(self, record):
- msg = self.base_formatter.format(record)
- for pattern in self._patterns:
- pat = re.compile(pattern)
- msg = pat.sub(self.convert_match_to_sha3, msg)
+ def __init__(self, log_format: str, patterns: dict) -> None:
+ super().__init__(log_format)
+ self._patterns: dict = patterns
+
+ def _filter_sensitive(self, msg) -> str:
+ for match, replacement in self._patterns.items():
+ pat = re.compile(match)
+ msg = pat.sub(replacement, msg)
return msg
- def __getattr__(self, attr):
- return getattr(self.base_formatter, attr)
+ def format(self, record) -> str:
+ msg = super().format(record)
+ return self._filter_sensitive(msg)
+
+ def formatException(self, exc_info) -> str:
+ msg = super().formatException(exc_info)
+ return self._filter_sensitive(msg)
+
+ def formatStack(self, stack_info) -> str:
+ msg = super().formatStack(stack_info)
+ return self._filter_sensitive(msg)
def init_logger() -> None:
handlers: List[Handler] = []
-
- base_formatter = Formatter(LOG_FORMAT)
- formatter = HidingFormatter(base_formatter, HIDING_PATTERNS)
+ hiding_patterns = compose_hiding_patterns()
+ formatter = HidingFormatter(LOG_FORMAT, hiding_patterns)
f_handler = RotatingFileHandler(
TM_LOG_PATH,
@@ -98,4 +103,4 @@ def init_logger() -> None:
f_handler_debug.setLevel(logging.DEBUG)
handlers.append(f_handler_debug)
- logging.basicConfig(level=logging.DEBUG, handlers=handlers)
+ logging.basicConfig(level=logging.INFO, handlers=handlers)
diff --git a/transaction_manager/main.py b/transaction_manager/main.py
index 066557aa..58699b91 100644
--- a/transaction_manager/main.py
+++ b/transaction_manager/main.py
@@ -18,6 +18,7 @@
# along with this program. If not, see .
import logging
+import time
from . import config
from .attempt_manager import AttemptManagerV2, RedisAttemptStorage
@@ -31,10 +32,8 @@
logger = logging.getLogger(__name__)
-def main() -> None:
- init_logger()
+def run_proc():
eth = Eth()
- logger.info('Starting. Config:\n%s', config_string(vars(config)))
pool = TxPool()
wallet = init_wallet()
attempt_manager = AttemptManagerV2(
@@ -47,5 +46,16 @@ def main() -> None:
proc.run()
+def main() -> None:
+ init_logger()
+ while True:
+ try:
+ logger.info('Running processor. Config:\n%s', config_string(vars(config)))
+ run_proc()
+ except Exception:
+ logger.exception('TM failed. Sleeping for %ds', config.RESTART_TIMEOUT)
+ time.sleep(config.RESTART_TIMEOUT)
+
+
if __name__ == '__main__':
main()
diff --git a/transaction_manager/processor.py b/transaction_manager/processor.py
index 836eabba..8ae2f2a1 100644
--- a/transaction_manager/processor.py
+++ b/transaction_manager/processor.py
@@ -67,6 +67,7 @@ def __init__(
def send(self, tx: Tx) -> None:
tx_hash, err = None, None
retry = 0
+ replaced = False
while tx_hash is None and retry < UNDERPRICED_RETRIES:
logger.info('Signing tx %s, retry %d', tx.tx_id, retry)
etx = self.eth.convert_tx(tx)
@@ -80,10 +81,14 @@ def send(self, tx: Tx) -> None:
if is_replacement_underpriced(err):
logger.info('Replacement fee is too low. Increasing')
self.attempt_manager.replace(tx, replace_attempt=retry)
+ replaced = True
retry += 1
else:
break
+ if tx_hash is not None or replaced:
+ self.attempt_manager.save()
+
if tx_hash is None:
tx.status = TxStatus.UNSENT
raise SendingError(err)
@@ -125,7 +130,11 @@ def confirm(self, tx: Tx) -> None:
'Tx %s: confirming within %d blocks',
tx.tx_id, CONFIRMATION_BLOCKS
)
- self.eth.wait_for_blocks(amount=CONFIRMATION_BLOCKS)
+ start_block = self.eth.get_tx_block(tx.tx_hash) # type: ignore
+ self.eth.wait_for_blocks(
+ amount=CONFIRMATION_BLOCKS,
+ start_block=start_block
+ )
h, r = self.get_exec_data(tx)
if h is None or r not in (0, 1):
tx.status = TxStatus.UNCONFIRMED
diff --git a/transaction_manager/structures.py b/transaction_manager/structures.py
index 9fbfffc6..3750be3d 100644
--- a/transaction_manager/structures.py
+++ b/transaction_manager/structures.py
@@ -181,6 +181,7 @@ def from_bytes(cls, tx_id: bytes, tx_bytes: bytes) -> 'Tx':
raw_tx['fee'] = cls._extract_fee(raw_tx)
raw_tx['hashes'] = raw_tx.get('hashes') or []
+ raw_tx.pop('type', None)
try:
tx = Tx(**raw_tx)
except TypeError: