From 3ae08bc34b35fea7236f4f1b3aaf8ae60622d886 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 28 Jul 2022 10:06:44 +0000 Subject: [PATCH 01/23] Save attempt when no more underpriced retries --- tests/processor_test.py | 16 +++++++++++++++- transaction_manager/processor.py | 5 +++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/processor_test.py b/tests/processor_test.py index 0f7e08a..f465826 100644 --- a/tests/processor_test.py +++ b/tests/processor_test.py @@ -105,7 +105,8 @@ def test_send(proc, w3, rdp, eth, tpool, wallet): tx.nonce = 0 proc.attempt_manager.make(tx) tx.gas = 22000 - tx.gas_price = 10 ** 9 + tx.fee.max_fee_per_gas = 10 ** 9 + tx.fee.max_priority_fee_per_gas = 10 ** 8 proc.eth.send_tx = mock.Mock( side_effect=ValueError({ @@ -115,9 +116,16 @@ def test_send(proc, w3, rdp, eth, tpool, wallet): ) with pytest.raises(SendingError): proc.send(tx) + assert tx.tx_hash is None assert tx.hashes == [] + # Test that attempt was saved + proc.attempt_manager.fetch() + after_replacement = proc.attempt_manager.current + assert after_replacement.fee.max_fee_per_gas > 10 ** 9 + assert after_replacement.fee.max_priority_fee_per_gas > 10 ** 8 + proc.eth.send_tx = mock.Mock( side_effect=ValueError('unknown error') ) @@ -126,6 +134,12 @@ def test_send(proc, w3, rdp, eth, tpool, wallet): assert tx.tx_hash is None assert tx.hashes == [] + # Test that attempt was NOT saved + proc.attempt_manager.fetch() + after_failure = proc.attempt_manager.current + assert after_failure.fee.max_fee_per_gas == after_replacement.fee.max_fee_per_gas + assert after_failure.fee.max_priority_fee_per_gas == after_replacement.fee.max_priority_fee_per_gas # noqa + proc.eth.send_tx = mock.Mock(return_value='0x12323213213321321') proc.send(tx) assert tx.tx_hash == '0x12323213213321321' diff --git a/transaction_manager/processor.py b/transaction_manager/processor.py index 836eabb..6881145 100644 --- a/transaction_manager/processor.py +++ b/transaction_manager/processor.py @@ -67,6 +67,7 @@ def __init__( def send(self, tx: Tx) -> None: tx_hash, err = None, None retry = 0 + replaced = False while tx_hash is None and retry < UNDERPRICED_RETRIES: logger.info('Signing tx %s, retry %d', tx.tx_id, retry) etx = self.eth.convert_tx(tx) @@ -80,10 +81,14 @@ def send(self, tx: Tx) -> None: if is_replacement_underpriced(err): logger.info('Replacement fee is too low. Increasing') self.attempt_manager.replace(tx, replace_attempt=retry) + replaced = True retry += 1 else: break + if tx_hash is not None or replaced: + self.attempt_manager.save() + if tx_hash is None: tx.status = TxStatus.UNSENT raise SendingError(err) From 1bec7405f68e8f68d1a4cc6543459135944a65d1 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 28 Jul 2022 10:12:41 +0000 Subject: [PATCH 02/23] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index eca07e4..ac2cdeb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.2 +2.1.3 From a0f7ddd6001e950e94238edcc99e754843759906 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 6 Sep 2022 15:07:04 +0000 Subject: [PATCH 03/23] Hide only SGX_URL and ENDPOINT --- transaction_manager/config.py | 1 + transaction_manager/log.py | 44 ++++++++++++++++++++++++----------- transaction_manager/main.py | 16 ++++++++++--- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/transaction_manager/config.py b/transaction_manager/config.py index 8521fad..21cdb60 100644 --- a/transaction_manager/config.py +++ b/transaction_manager/config.py @@ -34,6 +34,7 @@ NODE_DATA_PATH = '/skale_node_data' # General +RESTART_TIMEOUT: int = 3 BASE_WAITING_TIME: int = 25 CONFIRMATION_BLOCKS: int = 6 MAX_RESUBMIT_AMOUNT: int = 10 diff --git a/transaction_manager/log.py b/transaction_manager/log.py index ac286f2..2d3c621 100644 --- a/transaction_manager/log.py +++ b/transaction_manager/log.py @@ -25,8 +25,9 @@ from logging import Formatter, Handler, StreamHandler from logging.handlers import RotatingFileHandler from typing import List +from urllib.parse import urlparse -from .config import NODE_DATA_PATH +from .config import ENDPOINT, NODE_DATA_PATH, SGX_URL LOG_FOLDER = os.path.join(NODE_DATA_PATH, 'log') @@ -41,28 +42,42 @@ LOG_FORMAT = '%(asctime)s [%(levelname)s] [%(module)s:%(lineno)d] %(message)s' # noqa -HIDING_PATTERNS = [ - r'NEK\:\w+', - r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', # noqa - r'ws[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' # noqa -] +def compose_hiding_patterns(): + sgx_ip = urlparse(SGX_URL).hostname + eth_ip = urlparse(ENDPOINT).hostname + return { + rf'{sgx_ip}': '[SGX_IP]', + rf'{eth_ip}': '[ETH_IP]', + r'NEK\:\w+': '[SGX_KEY]' + } class HidingFormatter(Formatter): - def __init__(self, base_formatter: Formatter, patterns: List[str]) -> None: + def __init__(self, base_formatter: Formatter, patterns: dict) -> None: self.base_formatter: Formatter = base_formatter - self._patterns: List[str] = patterns + self._patterns: dict = patterns @classmethod def convert_match_to_sha3(cls, match) -> str: return hashlib.sha3_256(match.group(0).encode('utf-8')).digest().hex() + def _filter_sensitive(self, msg): + for match, replacement in self._patterns.items(): + pat = re.compile(match) + msg = pat.sub(replacement, msg) + return msg + def format(self, record): msg = self.base_formatter.format(record) - for pattern in self._patterns: - pat = re.compile(pattern) - msg = pat.sub(self.convert_match_to_sha3, msg) - return msg + return self._filter_sensitive(msg) + + def formatException(self, exc_info): + msg = self.base_formatter.formatException(exc_info) + return self._filter_sensitive(msg) + + def formatStack(self, stack_info): + msg = self.base_formatter.formatStack(stack_info) + return self._filter_sensitive(msg) def __getattr__(self, attr): return getattr(self.base_formatter, attr) @@ -72,7 +87,8 @@ def init_logger() -> None: handlers: List[Handler] = [] base_formatter = Formatter(LOG_FORMAT) - formatter = HidingFormatter(base_formatter, HIDING_PATTERNS) + hiding_patterns = compose_hiding_patterns() + formatter = HidingFormatter(base_formatter, hiding_patterns) f_handler = RotatingFileHandler( TM_LOG_PATH, @@ -97,4 +113,4 @@ def init_logger() -> None: f_handler_debug.setLevel(logging.DEBUG) handlers.append(f_handler_debug) - logging.basicConfig(level=logging.DEBUG, handlers=handlers) + logging.basicConfig(level=logging.INFO, handlers=handlers) diff --git a/transaction_manager/main.py b/transaction_manager/main.py index 066557a..58699b9 100644 --- a/transaction_manager/main.py +++ b/transaction_manager/main.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import logging +import time from . import config from .attempt_manager import AttemptManagerV2, RedisAttemptStorage @@ -31,10 +32,8 @@ logger = logging.getLogger(__name__) -def main() -> None: - init_logger() +def run_proc(): eth = Eth() - logger.info('Starting. Config:\n%s', config_string(vars(config))) pool = TxPool() wallet = init_wallet() attempt_manager = AttemptManagerV2( @@ -47,5 +46,16 @@ def main() -> None: proc.run() +def main() -> None: + init_logger() + while True: + try: + logger.info('Running processor. Config:\n%s', config_string(vars(config))) + run_proc() + except Exception: + logger.exception('TM failed. Sleeping for %ds', config.RESTART_TIMEOUT) + time.sleep(config.RESTART_TIMEOUT) + + if __name__ == '__main__': main() From ff59891dcde38dd43deceb3917f1dede379e7d4a Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 6 Sep 2022 15:16:00 +0000 Subject: [PATCH 04/23] Refactor formatter --- transaction_manager/log.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/transaction_manager/log.py b/transaction_manager/log.py index 2d3c621..0fadd80 100644 --- a/transaction_manager/log.py +++ b/transaction_manager/log.py @@ -53,30 +53,26 @@ def compose_hiding_patterns(): class HidingFormatter(Formatter): - def __init__(self, base_formatter: Formatter, patterns: dict) -> None: - self.base_formatter: Formatter = base_formatter + def __init__(self, log_format: str, patterns: dict) -> None: + super().__init__(log_format) self._patterns: dict = patterns - @classmethod - def convert_match_to_sha3(cls, match) -> str: - return hashlib.sha3_256(match.group(0).encode('utf-8')).digest().hex() - - def _filter_sensitive(self, msg): + def _filter_sensitive(self, msg) -> str: for match, replacement in self._patterns.items(): pat = re.compile(match) msg = pat.sub(replacement, msg) return msg - def format(self, record): - msg = self.base_formatter.format(record) + def format(self, record) -> str: + msg = super().format(record) return self._filter_sensitive(msg) - def formatException(self, exc_info): - msg = self.base_formatter.formatException(exc_info) + def formatException(self, exc_info) -> str: + msg = super().formatException(exc_info) return self._filter_sensitive(msg) - def formatStack(self, stack_info): - msg = self.base_formatter.formatStack(stack_info) + def formatStack(self, stack_info) -> str: + msg = super().formatStack(stack_info) return self._filter_sensitive(msg) def __getattr__(self, attr): @@ -85,10 +81,8 @@ def __getattr__(self, attr): def init_logger() -> None: handlers: List[Handler] = [] - - base_formatter = Formatter(LOG_FORMAT) hiding_patterns = compose_hiding_patterns() - formatter = HidingFormatter(base_formatter, hiding_patterns) + formatter = HidingFormatter(LOG_FORMAT, hiding_patterns) f_handler = RotatingFileHandler( TM_LOG_PATH, From 6e5f146aeb2ef1b7c38f340d88963452d340389f Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 9 Sep 2022 10:37:21 +0000 Subject: [PATCH 05/23] Fix flake8 . --- transaction_manager/log.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/transaction_manager/log.py b/transaction_manager/log.py index 0fadd80..d315d96 100644 --- a/transaction_manager/log.py +++ b/transaction_manager/log.py @@ -17,7 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import hashlib import logging import os import re @@ -75,9 +74,6 @@ def formatStack(self, stack_info) -> str: msg = super().formatStack(stack_info) return self._filter_sensitive(msg) - def __getattr__(self, attr): - return getattr(self.base_formatter, attr) - def init_logger() -> None: handlers: List[Handler] = [] From 57f50d8c453d985d16fd4267ac568489ffef04f2 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 16 Sep 2022 14:45:15 +0000 Subject: [PATCH 06/23] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index eca07e4..7d2ed7c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.2 +2.1.4 From 7c2d5aa6b9cb803bc08383a38b16a11cf2ba2dc0 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Sep 2022 16:41:55 +0000 Subject: [PATCH 07/23] Improve test_send test --- tests/processor_test.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/tests/processor_test.py b/tests/processor_test.py index f465826..aa3c446 100644 --- a/tests/processor_test.py +++ b/tests/processor_test.py @@ -104,42 +104,30 @@ def test_send(proc, w3, rdp, eth, tpool, wallet): tx.chain_id = eth.chain_id tx.nonce = 0 proc.attempt_manager.make(tx) - tx.gas = 22000 - tx.fee.max_fee_per_gas = 10 ** 9 - tx.fee.max_priority_fee_per_gas = 10 ** 8 proc.eth.send_tx = mock.Mock( - side_effect=ValueError({ - 'code': -32000, - 'message': 'replacement transaction underpriced' - }) + side_effect=ValueError('unknown error') ) with pytest.raises(SendingError): proc.send(tx) - + # Test that attempt was not saved if it was neither sent or replaced + assert proc.attempt_manager.storage.get() is None assert tx.tx_hash is None assert tx.hashes == [] - # Test that attempt was saved - proc.attempt_manager.fetch() - after_replacement = proc.attempt_manager.current - assert after_replacement.fee.max_fee_per_gas > 10 ** 9 - assert after_replacement.fee.max_priority_fee_per_gas > 10 ** 8 - proc.eth.send_tx = mock.Mock( - side_effect=ValueError('unknown error') + side_effect=ValueError({ + 'code': -32000, + 'message': 'replacement transaction underpriced' + }) ) with pytest.raises(SendingError): proc.send(tx) + # Test that attempt was saved if it was replaced + assert proc.attempt_manager.storage.get().fee == tx.fee assert tx.tx_hash is None assert tx.hashes == [] - # Test that attempt was NOT saved - proc.attempt_manager.fetch() - after_failure = proc.attempt_manager.current - assert after_failure.fee.max_fee_per_gas == after_replacement.fee.max_fee_per_gas - assert after_failure.fee.max_priority_fee_per_gas == after_replacement.fee.max_priority_fee_per_gas # noqa - proc.eth.send_tx = mock.Mock(return_value='0x12323213213321321') proc.send(tx) assert tx.tx_hash == '0x12323213213321321' @@ -147,6 +135,8 @@ def test_send(proc, w3, rdp, eth, tpool, wallet): proc.eth.send_tx = mock.Mock(return_value='0x213812903813123') proc.send(tx) + # Test that attempt was saved if it was sent + assert proc.attempt_manager.storage.get().fee == tx.fee assert tx.tx_hash == '0x213812903813123' assert tx.hashes == ['0x12323213213321321', '0x213812903813123'] From 01675db3fa4de305e40ad2324d4eeeed1689651e Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Sep 2022 18:14:28 +0000 Subject: [PATCH 08/23] Add get_tx_block --- tests/eth_test.py | 1 + transaction_manager/eth.py | 64 +++++++++++++++++++++----------------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/tests/eth_test.py b/tests/eth_test.py index 1a565e8..ca098d2 100644 --- a/tests/eth_test.py +++ b/tests/eth_test.py @@ -86,3 +86,4 @@ def test_eth_tx(w3wallet, w3, eth): status = eth.wait_for_receipt(h) assert status == 1 assert eth.get_nonce(addr) == second_nonce + 1 + assert eth.get_tx_block(h) == eth.get_receipt(h)['blockNumber'] diff --git a/transaction_manager/eth.py b/transaction_manager/eth.py index 5769797..53d5395 100644 --- a/transaction_manager/eth.py +++ b/transaction_manager/eth.py @@ -192,32 +192,14 @@ def get_nonce(self, address: str) -> int: checksum_addres = self.w3.toChecksumAddress(address) return self.w3.eth.get_transaction_count(checksum_addres) - def get_status( - self, - tx_hash: str, - raise_err: bool = False - ) -> int: - try: - casted_hash = cast(HexStr, tx_hash) - receipt = self.w3.eth.get_transaction_receipt(casted_hash) - except TransactionNotFound as e: - if raise_err: - raise e - else: - return -1 - logger.debug(f'Receipt for {tx_hash}: {receipt}') - rstatus = receipt.get('status', -1) - if rstatus < 0: - logger.error('Receipt has no "status" field') - return rstatus - return rstatus - def wait_for_blocks( self, amount: int = CONFIRMATION_BLOCKS, - max_time: int = MAX_WAITING_TIME + max_time: int = MAX_WAITING_TIME, + start_block: Optional[int] = None ) -> None: - current_block = start_block = self.w3.eth.block_number + current_block = self.w3.eth.block_number + start_block = start_block or current_block current_ts = start_ts = time.time() while current_block - start_block < amount and \ current_ts - start_ts < max_time: @@ -235,13 +217,39 @@ def wait_for_receipt( max_time: int = MAX_WAITING_TIME, ) -> int: start_ts = time.time() - rstatus = None - while rstatus is None and time.time() - start_ts < max_time: - try: - rstatus = self.get_status(tx_hash, raise_err=True) - except TransactionNotFound: + rstatus = -1 + while rstatus == -1 and time.time() - start_ts < max_time: + rstatus = self.get_status(tx_hash) + if rstatus == -1: time.sleep(1) - if rstatus is None: + if rstatus == -1: raise ReceiptTimeoutError(f'No receipt after {max_time}') return rstatus + + def get_receipt(self, tx_hash: str) -> Optional[Dict]: + casted_hash = cast(HexStr, tx_hash) + receipt = None + try: + casted_hash = cast(HexStr, tx_hash) + receipt = self.w3.eth.get_transaction_receipt(casted_hash) + except TransactionNotFound: + pass + return cast(Dict, receipt) + + def get_tx_block(self, tx_hash: str) -> int: + receipt = self.get_receipt(tx_hash) + if receipt is None: + return -1 + return cast(int, receipt.get('blockNumber')) + + def get_status(self, tx_hash: str) -> int: + receipt = self.get_receipt(tx_hash) + logger.debug('Receipt for %s: %s', tx_hash, receipt) + if receipt is None: + return -1 + rstatus = receipt.get('status', -1) + if rstatus < 0: + logger.error('Receipt has no "status" field') + return rstatus + return rstatus From 9f12e09185400120bb7351f09c4cce26f40f8a8b Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Sep 2022 18:33:53 +0000 Subject: [PATCH 09/23] Add ability to add start block to wait from --- tests/eth_test.py | 6 ++++++ transaction_manager/processor.py | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/eth_test.py b/tests/eth_test.py index ca098d2..b7d4603 100644 --- a/tests/eth_test.py +++ b/tests/eth_test.py @@ -87,3 +87,9 @@ def test_eth_tx(w3wallet, w3, eth): assert status == 1 assert eth.get_nonce(addr) == second_nonce + 1 assert eth.get_tx_block(h) == eth.get_receipt(h)['blockNumber'] + + +def test_eth_wait_for_blocks(eth, w3): + eth.wait_for_blocks(amount=1, max_time=1) + cblock = w3.eth.block_number + eth.wait_for_blocks(amount=5, max_time=0, start_block=cblock - 10) diff --git a/transaction_manager/processor.py b/transaction_manager/processor.py index 6881145..8ae2f2a 100644 --- a/transaction_manager/processor.py +++ b/transaction_manager/processor.py @@ -130,7 +130,11 @@ def confirm(self, tx: Tx) -> None: 'Tx %s: confirming within %d blocks', tx.tx_id, CONFIRMATION_BLOCKS ) - self.eth.wait_for_blocks(amount=CONFIRMATION_BLOCKS) + start_block = self.eth.get_tx_block(tx.tx_hash) # type: ignore + self.eth.wait_for_blocks( + amount=CONFIRMATION_BLOCKS, + start_block=start_block + ) h, r = self.get_exec_data(tx) if h is None or r not in (0, 1): tx.status = TxStatus.UNCONFIRMED From bd47dc61fc2949dc2885be4e8e074abb9f94fe0c Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Sep 2022 18:48:20 +0000 Subject: [PATCH 10/23] Add processor confirm test --- tests/processor_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/processor_test.py b/tests/processor_test.py index aa3c446..af6f48c 100644 --- a/tests/processor_test.py +++ b/tests/processor_test.py @@ -7,6 +7,7 @@ from transaction_manager.structures import TxStatus from tests.utils.contracts import get_tester_abi +from tests.utils.timing import in_time DEFAULT_GAS = 20000 @@ -187,3 +188,16 @@ def test_aquire_estimate_gas_revert(proc, w3, rdp, tpool, wallet): assert tx.status == TxStatus.DROPPED assert tx.tx_id.encode('utf-8') not in tpool.to_list() + + +def test_confirm(proc, w3, rdp, tpool, wallet): + tx = push_tx(w3, rdp, tpool, wallet) + proc.attempt_manager.make(tx) + proc.send(tx) + proc.wait(tx, max_time=proc.attempt_manager.current.wait_time) + # Make sure it is confirmed after reasonable number of seconds + with in_time(8): + proc.confirm(tx) + # Make sure next time it is confirmed instantly + with in_time(0.1): + proc.confirm(tx) From ff16f1868f80b206e28e2799f14cb50125dc98bf Mon Sep 17 00:00:00 2001 From: badrogger Date: Sun, 2 Oct 2022 21:46:00 +0000 Subject: [PATCH 11/23] Set default gas limit if dry run is skipped --- transaction_manager/config.py | 1 + transaction_manager/eth.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/transaction_manager/config.py b/transaction_manager/config.py index 21cdb60..f0dac37 100644 --- a/transaction_manager/config.py +++ b/transaction_manager/config.py @@ -44,6 +44,7 @@ DISABLE_GAS_ESTIMATION = False TXRECORD_EXPIRATION = 24 * 60 * 60 # 1 day DEFAULT_ID_LEN = 19 +DEFAULT_GAS_LIMIT = 1000000 IMA_ID_SUFFIX = 'js' # V1 diff --git a/transaction_manager/eth.py b/transaction_manager/eth.py index 53d5395..b7d1e0e 100644 --- a/transaction_manager/eth.py +++ b/transaction_manager/eth.py @@ -30,6 +30,7 @@ from .config import ( AVG_GAS_PRICE_INC_PERCENT, CONFIRMATION_BLOCKS, + DEFAULT_GAS_LIMIT, DISABLE_GAS_ESTIMATION, GAS_MULTIPLIER, MAX_WAITING_TIME, @@ -147,7 +148,7 @@ def calculate_gas(self, tx: Tx) -> int: multiplier = tx.multiplier multiplier = multiplier or GAS_MULTIPLIER if DISABLE_GAS_ESTIMATION: - return int(etx['gas'] * multiplier) + return int(etx.get('gas', DEFAULT_GAS_LIMIT) * multiplier) logger.info('Estimating gas for %s', etx) From f2c726e4df4eb6dd282eece5afee7307909a8ba7 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 24 Oct 2022 19:42:25 +0000 Subject: [PATCH 12/23] Update dependencies --- helper-scripts | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helper-scripts b/helper-scripts index 5cd6ed4..2242899 160000 --- a/helper-scripts +++ b/helper-scripts @@ -1 +1 @@ -Subproject commit 5cd6ed42c44ddf33126a4613cbc381c231401910 +Subproject commit 2242899037c120ce43ec87d9b80f2a223c619331 diff --git a/requirements.txt b/requirements.txt index 2f5aefc..c809661 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -redis==3.5.3 +redis==4.3.4 skale.py==5.6.dev6 From f0e6608fccd0dbcd3bb5de888dc9a8e5fc419cbb Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 25 Oct 2022 09:46:29 +0000 Subject: [PATCH 13/23] Fix tests --- requirements.txt | 3 +-- scripts/run-test-containers.sh | 2 +- tests/attempt/v2_test.py | 4 ++-- tests/conftest.py | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index c809661..96c6271 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -redis==4.3.4 -skale.py==5.6.dev6 +skale.py==5.8dev3 diff --git a/scripts/run-test-containers.sh b/scripts/run-test-containers.sh index 8c0202f..4d83988 100755 --- a/scripts/run-test-containers.sh +++ b/scripts/run-test-containers.sh @@ -67,7 +67,7 @@ create_skale_dir create_redis_dir build tm hnode -if [ -n ${SGX_URL} ]; then +if [ -z ${SGX_URL} ]; then run_containers tm hnode redis else run_containers sgx tm hnode redis diff --git a/tests/attempt/v2_test.py b/tests/attempt/v2_test.py index 28b7d28..bcadbc0 100644 --- a/tests/attempt/v2_test.py +++ b/tests/attempt/v2_test.py @@ -1,6 +1,6 @@ import pytest from mock import Mock -from skale.utils.account_tools import send_ether +from skale.utils.account_tools import send_eth from transaction_manager.attempt_manager.base import NoCurrentAttemptError from transaction_manager.attempt_manager.v2 import AttemptManagerV2 @@ -28,7 +28,7 @@ def create_attempt(nonce=1, index=2, gas_price=10 ** 9, wait_time=30): @pytest.fixture def account(w3, wallet): acc = w3.eth.account.create() - send_ether(w3, wallet, acc.address, TEST_ETH_VALUE) + send_eth(w3, wallet, acc.address, TEST_ETH_VALUE) return acc.address, acc.key.hex() diff --git a/tests/conftest.py b/tests/conftest.py index 6b64979..2c59918 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ import pytest import redis -from skale.utils.account_tools import send_ether +from skale.utils.account_tools import send_eth from skale.wallets import RedisWalletAdapter, SgxWallet, Web3Wallet from transaction_manager.attempt_manager import ( @@ -77,7 +77,7 @@ def wallet(w3, w3wallet): else: return w3wallet if isinstance(w, SgxWallet): - send_ether(w3, w3wallet, w.address, ETH_AMOUNT_FOR_TESTS) + send_eth(w3, w3wallet, w.address, ETH_AMOUNT_FOR_TESTS) return w From 37fc70eb4d9c400546898d14d2c19247d95c4248 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 25 Oct 2022 16:18:48 +0000 Subject: [PATCH 14/23] Update to skale.py 5.8dev4 (fixed redis wallet) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 96c6271..05c9b8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -skale.py==5.8dev3 +skale.py==5.8dev4 From f3b8003cca773d7dd00027a8b4d8d95b3f97eba7 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 26 Oct 2022 17:59:30 +0000 Subject: [PATCH 15/23] Increase default gas limit --- transaction_manager/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction_manager/config.py b/transaction_manager/config.py index f0dac37..1133d2e 100644 --- a/transaction_manager/config.py +++ b/transaction_manager/config.py @@ -44,7 +44,7 @@ DISABLE_GAS_ESTIMATION = False TXRECORD_EXPIRATION = 24 * 60 * 60 # 1 day DEFAULT_ID_LEN = 19 -DEFAULT_GAS_LIMIT = 1000000 +DEFAULT_GAS_LIMIT = 1500000 IMA_ID_SUFFIX = 'js' # V1 From 45ccc2f3353133760b763086594af05b3c2ce916 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 26 Oct 2022 18:32:39 +0000 Subject: [PATCH 16/23] Increase default gas limit to 2800000 --- transaction_manager/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction_manager/config.py b/transaction_manager/config.py index 1133d2e..51eae6a 100644 --- a/transaction_manager/config.py +++ b/transaction_manager/config.py @@ -44,7 +44,7 @@ DISABLE_GAS_ESTIMATION = False TXRECORD_EXPIRATION = 24 * 60 * 60 # 1 day DEFAULT_ID_LEN = 19 -DEFAULT_GAS_LIMIT = 1500000 +DEFAULT_GAS_LIMIT: int = 2800000 IMA_ID_SUFFIX = 'js' # V1 From 607b6facdefdd5b0e88511454698d84e54b5eed9 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 21 Nov 2022 13:26:41 +0000 Subject: [PATCH 17/23] Update MANAGER_TAG in GA --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb8f205..038a17a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: ETH_PRIVATE_KEY: ${{ secrets.ETH_PRIVATE_KEY }} ENDPOINT: ${{ secrets.ENDPOINT }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - MANAGER_TAG: "1.8.2-develop.44" + MANAGER_TAG: "1.9.3-beta.0" SGX_URL: "" steps: From 5ae8d94976bbc6c9e3f55fb42a114fe8503a774f Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 21 Nov 2022 13:46:58 +0000 Subject: [PATCH 18/23] Remove MANAGER_TAG completely --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 038a17a..ab19862 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,6 @@ jobs: ETH_PRIVATE_KEY: ${{ secrets.ETH_PRIVATE_KEY }} ENDPOINT: ${{ secrets.ENDPOINT }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - MANAGER_TAG: "1.9.3-beta.0" SGX_URL: "" steps: From 3e0f2661797efcde9b2acef204ebbacb5aabd2bc Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Mon, 21 Nov 2022 23:08:16 +0200 Subject: [PATCH 19/23] Create issue_check.yml --- .github/workflows/issue_check.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/issue_check.yml diff --git a/.github/workflows/issue_check.yml b/.github/workflows/issue_check.yml new file mode 100644 index 0000000..ee5ec6b --- /dev/null +++ b/.github/workflows/issue_check.yml @@ -0,0 +1,19 @@ +name: Get linked issues +on: + pull_request: + types: [ edited, synchronize, opened, reopened ] + +jobs: + check-linked-issues: + name: Check if pull request has linked issues + runs-on: ubuntu-latest + steps: + - name: Get issues + id: get-issues + uses: mondeja/pr-linked-issues-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + - name: PR has not linked issues + if: join(steps.get-issues.outputs.issues) == '' + run: + exit 1 From df25080ff537e26fd3869fff734bb62753f19470 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 23 Nov 2022 17:50:04 +0000 Subject: [PATCH 20/23] Reduce default gas limit to 100000 --- transaction_manager/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction_manager/config.py b/transaction_manager/config.py index 51eae6a..9e83fc8 100644 --- a/transaction_manager/config.py +++ b/transaction_manager/config.py @@ -44,7 +44,7 @@ DISABLE_GAS_ESTIMATION = False TXRECORD_EXPIRATION = 24 * 60 * 60 # 1 day DEFAULT_ID_LEN = 19 -DEFAULT_GAS_LIMIT: int = 2800000 +DEFAULT_GAS_LIMIT: int = 100000 IMA_ID_SUFFIX = 'js' # V1 From 2944f0eee7d5bb72a21e04af5a46900c7bf96ef2 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 23 Nov 2022 17:52:09 +0000 Subject: [PATCH 21/23] Increase to 1000000 --- transaction_manager/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction_manager/config.py b/transaction_manager/config.py index 9e83fc8..5e52ce8 100644 --- a/transaction_manager/config.py +++ b/transaction_manager/config.py @@ -44,7 +44,7 @@ DISABLE_GAS_ESTIMATION = False TXRECORD_EXPIRATION = 24 * 60 * 60 # 1 day DEFAULT_ID_LEN = 19 -DEFAULT_GAS_LIMIT: int = 100000 +DEFAULT_GAS_LIMIT: int = 1000000 IMA_ID_SUFFIX = 'js' # V1 From 5449e0b186c542943debebf932df2e851df9f35e Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Dec 2022 19:08:49 +0000 Subject: [PATCH 22/23] Handle type field properly --- transaction_manager/structures.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/transaction_manager/structures.py b/transaction_manager/structures.py index 9fbfffc..da32887 100644 --- a/transaction_manager/structures.py +++ b/transaction_manager/structures.py @@ -65,6 +65,7 @@ class Tx: score: int to: str fee: Fee + type_: Optional[int] = None hashes: List = field(default_factory=list) attempts: int = 0 value: int = 0 @@ -82,7 +83,8 @@ class Tx: 'gasPrice': 'gas_price', 'maxFeePerGas': 'max_fee_per_gas', 'maxPriorityFeePerGas': 'max_priority_fee_per_gas', - 'from': 'source' + 'from': 'source', + 'type': 'type_' } def __post_init__(self): From 6e18861f5a6acfbe233710e193783528d1e883b7 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 19 Dec 2022 19:32:41 +0000 Subject: [PATCH 23/23] Exclude type from raw tx --- tests/transaction_test.py | 5 +++++ transaction_manager/structures.py | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/transaction_test.py b/tests/transaction_test.py index 8684595..d8a67b4 100644 --- a/tests/transaction_test.py +++ b/tests/transaction_test.py @@ -126,6 +126,11 @@ def test_tx_from_bytes(): tx = Tx.from_bytes(tx_id, missing_field_hash) assert tx.hashes == [] + with_type = b'{"attempts": 0, "type": "0x0", "status": "PROPOSED", "chainId": null, "data": {"test": 1}, "from": null, "gas": 22000, "gasPrice": 1000000000, "hashes": [], "nonce": 3, "score": 1, "sent_ts": null, "to": "0x1", "tx_hash": null, "value": 1}' # noqa + tx = Tx.from_bytes(tx_id, with_type) + expected = b'{"attempts": 0, "chainId": null, "data": {"test": 1}, "from": null, "gas": 22000, "gasPrice": 1000000000, "hashes": [], "maxFeePerGas": null, "maxPriorityFeePerGas": null, "multiplier": 1.2, "nonce": 3, "score": 1, "sent_ts": null, "status": "PROPOSED", "to": "0x1", "tx_hash": null, "tx_id": "tx-1232321332132131331321", "value": 1}' # noqa + tx.to_bytes() == expected + def test_is_sent_by_ima(): tx = Tx( diff --git a/transaction_manager/structures.py b/transaction_manager/structures.py index da32887..3750be3 100644 --- a/transaction_manager/structures.py +++ b/transaction_manager/structures.py @@ -65,7 +65,6 @@ class Tx: score: int to: str fee: Fee - type_: Optional[int] = None hashes: List = field(default_factory=list) attempts: int = 0 value: int = 0 @@ -83,8 +82,7 @@ class Tx: 'gasPrice': 'gas_price', 'maxFeePerGas': 'max_fee_per_gas', 'maxPriorityFeePerGas': 'max_priority_fee_per_gas', - 'from': 'source', - 'type': 'type_' + 'from': 'source' } def __post_init__(self): @@ -183,6 +181,7 @@ def from_bytes(cls, tx_id: bytes, tx_bytes: bytes) -> 'Tx': raw_tx['fee'] = cls._extract_fee(raw_tx) raw_tx['hashes'] = raw_tx.get('hashes') or [] + raw_tx.pop('type', None) try: tx = Tx(**raw_tx) except TypeError: