From 5232cf7ed710be9c5a39ba69965c51403018564d Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Mon, 6 May 2024 16:49:36 -0400 Subject: [PATCH 01/13] Using multiprocesses for key creation and keystore export --- staking_deposit/credentials.py | 51 ++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/staking_deposit/credentials.py b/staking_deposit/credentials.py index 1f59ceee..36eaa891 100644 --- a/staking_deposit/credentials.py +++ b/staking_deposit/credentials.py @@ -4,6 +4,7 @@ import time import json from typing import Dict, List, Optional, Any, Sequence +import concurrent.futures from eth_typing import Address, HexAddress from eth_utils import to_canonical_address @@ -224,6 +225,15 @@ def save_exit_transaction(self, validator_index: int, epoch: int, folder: str) - return export_exit_transaction_json(folder=folder, signed_exit=signed_voluntary_exit) +def credential_builder(kwargs: Dict[str, Any]) -> Credential: + return Credential(**kwargs) + + +def keystore_exporter(kwargs: Dict[str, Any]) -> str: + credential: Credential = kwargs.pop('credential') + return credential.save_signing_keystore(**kwargs) + + class CredentialList: """ A collection of multiple Credentials, one for each validator. @@ -246,17 +256,40 @@ def from_mnemonic(cls, f"The number of keys ({num_keys}) doesn't equal to the corresponding deposit amounts ({len(amounts)})." ) key_indices = range(start_index, start_index + num_keys) - with click.progressbar(key_indices, label=load_text(['msg_key_creation']), - show_percent=False, show_pos=True) as indices: - return cls([Credential(mnemonic=mnemonic, mnemonic_password=mnemonic_password, - index=index, amount=amounts[index - start_index], chain_setting=chain_setting, - hex_eth1_withdrawal_address=hex_eth1_withdrawal_address) - for index in indices]) + + return_list: List[Credential] = [] + with click.progressbar(length=num_keys, label=load_text(['msg_key_creation']), + show_percent=False, show_pos=True) as bar: + kwargs = [{ + 'mnemonic': mnemonic, + 'mnemonic_password': mnemonic_password, + 'index': index, + 'amount': amounts[index - start_index], + 'chain_setting': chain_setting, + 'hex_eth1_withdrawal_address': hex_eth1_withdrawal_address, + } for index in key_indices] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for index, credential in zip(key_indices, executor.map(credential_builder, kwargs)): + return_list.append(credential) + bar.update(1) + return cls(return_list) def export_keystores(self, password: str, folder: str) -> List[str]: - with click.progressbar(self.credentials, label=load_text(['msg_keystore_creation']), - show_percent=False, show_pos=True) as credentials: - return [credential.save_signing_keystore(password=password, folder=folder) for credential in credentials] + return_list: List[str] = [] + with click.progressbar(length=len(self.credentials), label=load_text(['msg_keystore_creation']), + show_percent=False, show_pos=True) as bar: + kwargs = [{ + 'credential': credential, + 'password': password, + 'folder': folder, + } for credential in self.credentials] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for credential, keystore in zip(self.credentials, executor.map(keystore_exporter, kwargs)): + return_list.append(keystore) + bar.update(1) + return return_list def export_deposit_data_json(self, folder: str) -> str: with click.progressbar(self.credentials, label=load_text(['msg_depositdata_creation']), From 90e75935de431899c54985e1faed0d977523fb91 Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Tue, 7 May 2024 10:57:45 -0400 Subject: [PATCH 02/13] Using multiprocess for more steps in the key creation process --- staking_deposit/credentials.py | 50 ++++++++++++++++++++++------- staking_deposit/utils/validation.py | 27 +++++++++++++--- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/staking_deposit/credentials.py b/staking_deposit/credentials.py index 36eaa891..43128fb6 100644 --- a/staking_deposit/credentials.py +++ b/staking_deposit/credentials.py @@ -3,8 +3,8 @@ from enum import Enum import time import json -from typing import Dict, List, Optional, Any, Sequence import concurrent.futures +from typing import Dict, List, Optional, Any, Sequence from eth_typing import Address, HexAddress from eth_utils import to_canonical_address @@ -225,15 +225,24 @@ def save_exit_transaction(self, validator_index: int, epoch: int, folder: str) - return export_exit_transaction_json(folder=folder, signed_exit=signed_voluntary_exit) -def credential_builder(kwargs: Dict[str, Any]) -> Credential: +def _credential_builder(kwargs: Dict[str, Any]) -> Credential: return Credential(**kwargs) -def keystore_exporter(kwargs: Dict[str, Any]) -> str: +def _keystore_exporter(kwargs: Dict[str, Any]) -> str: credential: Credential = kwargs.pop('credential') return credential.save_signing_keystore(**kwargs) +def _deposit_data_builder(credential: Credential) -> Dict[str, bytes]: + return credential.deposit_datum_dict + + +def _keystore_verifier(kwargs: Dict[str, Any]) -> bool: + credential: Credential = kwargs.pop('credential') + return credential.verify_keystore(**kwargs) + + class CredentialList: """ A collection of multiple Credentials, one for each validator. @@ -270,7 +279,7 @@ def from_mnemonic(cls, } for index in key_indices] with concurrent.futures.ProcessPoolExecutor() as executor: - for index, credential in zip(key_indices, executor.map(credential_builder, kwargs)): + for credential in executor.map(_credential_builder, kwargs): return_list.append(credential) bar.update(1) return cls(return_list) @@ -286,15 +295,21 @@ def export_keystores(self, password: str, folder: str) -> List[str]: } for credential in self.credentials] with concurrent.futures.ProcessPoolExecutor() as executor: - for credential, keystore in zip(self.credentials, executor.map(keystore_exporter, kwargs)): + for keystore in executor.map(_keystore_exporter, kwargs): return_list.append(keystore) bar.update(1) return return_list def export_deposit_data_json(self, folder: str) -> str: - with click.progressbar(self.credentials, label=load_text(['msg_depositdata_creation']), - show_percent=False, show_pos=True) as credentials: - deposit_data = [cred.deposit_datum_dict for cred in credentials] + deposit_data = [] + with click.progressbar(length=len(self.credentials), label=load_text(['msg_depositdata_creation']), + show_percent=False, show_pos=True) as bar: + + with concurrent.futures.ProcessPoolExecutor() as executor: + for datum_dict in executor.map(_deposit_data_builder, self.credentials): + deposit_data.append(datum_dict) + bar.update(1) + filefolder = os.path.join(folder, 'deposit_data-%i.json' % time.time()) with open(filefolder, 'w') as f: json.dump(deposit_data, f, default=lambda x: x.hex()) @@ -303,11 +318,22 @@ def export_deposit_data_json(self, folder: str) -> str: return filefolder def verify_keystores(self, keystore_filefolders: List[str], password: str) -> bool: - with click.progressbar(zip(self.credentials, keystore_filefolders), + valid = True + with click.progressbar(length=len(self.credentials), label=load_text(['msg_keystore_verification']), - length=len(self.credentials), show_percent=False, show_pos=True) as items: - return all(credential.verify_keystore(keystore_filefolder=filefolder, password=password) - for credential, filefolder in items) + show_percent=False, show_pos=True) as bar: + kwargs = [{ + 'credential': credential, + 'keystore_filefolder': fileholder, + 'password': password, + } for credential, fileholder in zip(self.credentials, keystore_filefolders)] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for valid_keystore in executor.map(_keystore_verifier, kwargs): + valid &= valid_keystore + bar.update(1) + + return valid def export_bls_to_execution_change_json(self, folder: str, validator_indices: Sequence[int]) -> str: with click.progressbar(self.credentials, label=load_text(['msg_bls_to_execution_change_creation']), diff --git a/staking_deposit/utils/validation.py b/staking_deposit/utils/validation.py index 0cd7c1bf..22cad36a 100644 --- a/staking_deposit/utils/validation.py +++ b/staking_deposit/utils/validation.py @@ -1,6 +1,7 @@ import click import json import re +import concurrent.futures from typing import Any, Dict, Sequence from eth_typing import ( @@ -42,16 +43,34 @@ # Deposit # +def _deposit_validator(kwargs: Dict[str, Any]) -> bool: + deposit: Dict[str, Any] = kwargs.pop('deposit') + credential: Credential = kwargs.pop('credential') + return validate_deposit(deposit, credential) + + def verify_deposit_data_json(filefolder: str, credentials: Sequence[Credential]) -> bool: """ Validate every deposit found in the deposit-data JSON file folder. """ + valid = True + deposit_json = [] with open(filefolder, 'r', encoding='utf-8') as f: deposit_json = json.load(f) - with click.progressbar(deposit_json, label=load_text(['msg_deposit_verification']), - show_percent=False, show_pos=True) as deposits: - return all([validate_deposit(deposit, credential) for deposit, credential in zip(deposits, credentials)]) - return False + + with click.progressbar(length=len(deposit_json), label=load_text(['msg_deposit_verification']), + show_percent=False, show_pos=True) as bar: + kwargs = [{ + 'credential': credential, + 'deposit': deposit, + } for deposit, credential in zip(deposit_json, credentials)] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for valid_deposit in executor.map(_deposit_validator, kwargs): + valid &= valid_deposit + bar.update(1) + + return valid def validate_deposit(deposit_data_dict: Dict[str, Any], credential: Credential) -> bool: From 96dae2c2467f9c687a90e3fa9b0da2cc58b4e457 Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Wed, 8 May 2024 14:09:55 -0400 Subject: [PATCH 03/13] Using multiprocess for the steps in the generate-bls-to-execution-change command steps --- .../cli/generate_bls_to_execution_change.py | 34 ++++++++++++++--- staking_deposit/credentials.py | 34 ++++++++++++----- .../cli/generate_bls_to_execution_change.json | 1 + staking_deposit/utils/validation.py | 38 ++++++++++++------- 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/staking_deposit/cli/generate_bls_to_execution_change.py b/staking_deposit/cli/generate_bls_to_execution_change.py index fa497305..bd38f046 100644 --- a/staking_deposit/cli/generate_bls_to_execution_change.py +++ b/staking_deposit/cli/generate_bls_to_execution_change.py @@ -1,15 +1,19 @@ import os import click import json +import concurrent.futures from typing import ( Any, Sequence, + Dict, + Optional ) from eth_typing import HexAddress from staking_deposit.credentials import ( CredentialList, + Credential ) from staking_deposit.utils.validation import ( validate_bls_withdrawal_credentials_list, @@ -48,6 +52,17 @@ def get_password(text: str) -> str: return click.prompt(text, hide_input=True, show_default=False, type=str) +def _validate_credentials_match(kwargs: Dict[str, Any]) -> Optional[ValidationError]: + credential: Credential = kwargs.pop('credential') + bls_withdrawal_credentials: bytes = kwargs.pop('bls_withdrawal_credentials') + + try: + validate_bls_withdrawal_credentials_matching(bls_withdrawal_credentials, credential) + except ValidationError as e: + return e + return None + + FUNC_NAME = 'generate_bls_to_execution_change' @@ -177,12 +192,19 @@ def generate_bls_to_execution_change( ) # Check if the given old bls_withdrawal_credentials is as same as the mnemonic generated - for i, credential in enumerate(credentials.credentials): - try: - validate_bls_withdrawal_credentials_matching(bls_withdrawal_credentials_list[i], credential) - except ValidationError as e: - click.echo('\n[Error] ' + str(e)) - return + with click.progressbar(length=len(credentials.credentials), label=load_text(['msg_credentials_verification']), + show_percent=False, show_pos=True) as bar: + executor_kwargs = [{ + 'credential': credential, + 'bls_withdrawal_credentials': bls_withdrawal_credentials_list[i], + } for i, credential in enumerate(credentials.credentials)] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for e in executor.map(_validate_credentials_match, executor_kwargs): + bar.update(1) + if e is not None: + click.echo('\n[Error] ' + str(e)) + return btec_file = credentials.export_bls_to_execution_change_json(bls_to_execution_changes_folder, validator_indices) diff --git a/staking_deposit/credentials.py b/staking_deposit/credentials.py index 43128fb6..821d513d 100644 --- a/staking_deposit/credentials.py +++ b/staking_deposit/credentials.py @@ -243,6 +243,11 @@ def _keystore_verifier(kwargs: Dict[str, Any]) -> bool: return credential.verify_keystore(**kwargs) +def _bls_to_execution_change_builder(kwargs: Dict[str, Any]) -> Dict[str, bytes]: + credential: Credential = kwargs.pop('credential') + return credential.get_bls_to_execution_change_dict(**kwargs) + + class CredentialList: """ A collection of multiple Credentials, one for each validator. @@ -269,7 +274,7 @@ def from_mnemonic(cls, return_list: List[Credential] = [] with click.progressbar(length=num_keys, label=load_text(['msg_key_creation']), show_percent=False, show_pos=True) as bar: - kwargs = [{ + executor_kwargs = [{ 'mnemonic': mnemonic, 'mnemonic_password': mnemonic_password, 'index': index, @@ -279,7 +284,7 @@ def from_mnemonic(cls, } for index in key_indices] with concurrent.futures.ProcessPoolExecutor() as executor: - for credential in executor.map(_credential_builder, kwargs): + for credential in executor.map(_credential_builder, executor_kwargs): return_list.append(credential) bar.update(1) return cls(return_list) @@ -288,14 +293,14 @@ def export_keystores(self, password: str, folder: str) -> List[str]: return_list: List[str] = [] with click.progressbar(length=len(self.credentials), label=load_text(['msg_keystore_creation']), show_percent=False, show_pos=True) as bar: - kwargs = [{ + executor_kwargs = [{ 'credential': credential, 'password': password, 'folder': folder, } for credential in self.credentials] with concurrent.futures.ProcessPoolExecutor() as executor: - for keystore in executor.map(_keystore_exporter, kwargs): + for keystore in executor.map(_keystore_exporter, executor_kwargs): return_list.append(keystore) bar.update(1) return return_list @@ -322,24 +327,33 @@ def verify_keystores(self, keystore_filefolders: List[str], password: str) -> bo with click.progressbar(length=len(self.credentials), label=load_text(['msg_keystore_verification']), show_percent=False, show_pos=True) as bar: - kwargs = [{ + executor_kwargs = [{ 'credential': credential, 'keystore_filefolder': fileholder, 'password': password, } for credential, fileholder in zip(self.credentials, keystore_filefolders)] with concurrent.futures.ProcessPoolExecutor() as executor: - for valid_keystore in executor.map(_keystore_verifier, kwargs): + for valid_keystore in executor.map(_keystore_verifier, executor_kwargs): valid &= valid_keystore bar.update(1) return valid def export_bls_to_execution_change_json(self, folder: str, validator_indices: Sequence[int]) -> str: - with click.progressbar(self.credentials, label=load_text(['msg_bls_to_execution_change_creation']), - show_percent=False, show_pos=True) as credentials: - bls_to_execution_changes = [cred.get_bls_to_execution_change_dict(validator_indices[i]) - for i, cred in enumerate(credentials)] + bls_to_execution_changes = [] + with click.progressbar(length=len(self.credentials), label=load_text(['msg_bls_to_execution_change_creation']), + show_percent=False, show_pos=True) as bar: + + executor_kwargs = [{ + 'credential': credential, + 'validator_index': validator_indices[i], + } for i, credential in enumerate(self.credentials)] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for bls_to_execution_change in executor.map(_bls_to_execution_change_builder, executor_kwargs): + bls_to_execution_changes.append(bls_to_execution_change) + bar.update(1) filefolder = os.path.join(folder, 'bls_to_execution_change-%i.json' % time.time()) with open(filefolder, 'w') as f: diff --git a/staking_deposit/intl/en/cli/generate_bls_to_execution_change.json b/staking_deposit/intl/en/cli/generate_bls_to_execution_change.json index cca5adfd..da05c016 100644 --- a/staking_deposit/intl/en/cli/generate_bls_to_execution_change.json +++ b/staking_deposit/intl/en/cli/generate_bls_to_execution_change.json @@ -34,6 +34,7 @@ "help": "The folder path for the keystore(s). Pointing to `./bls_to_execution_changes` by default." }, "msg_key_creation": "Creating your SignedBLSToExecutionChange.", + "msg_credentials_verification": "Verifying your withdrawal credentials.", "msg_creation_success": "\nSuccess!\nYour SignedBLSToExecutionChange JSON file can be found at: ", "msg_pause": "\n\nPress any key.", "err_verify_btec": "Failed to verify the bls_to_execution_change JSON files." diff --git a/staking_deposit/utils/validation.py b/staking_deposit/utils/validation.py index 22cad36a..e72d983e 100644 --- a/staking_deposit/utils/validation.py +++ b/staking_deposit/utils/validation.py @@ -60,13 +60,13 @@ def verify_deposit_data_json(filefolder: str, credentials: Sequence[Credential]) with click.progressbar(length=len(deposit_json), label=load_text(['msg_deposit_verification']), show_percent=False, show_pos=True) as bar: - kwargs = [{ + executor_kwargs = [{ 'credential': credential, 'deposit': deposit, } for deposit, credential in zip(deposit_json, credentials)] with concurrent.futures.ProcessPoolExecutor() as executor: - for valid_deposit in executor.map(_deposit_validator, kwargs): + for valid_deposit in executor.map(_deposit_validator, executor_kwargs): valid &= valid_deposit bar.update(1) @@ -168,6 +168,9 @@ def validate_eth1_withdrawal_address(cts: click.Context, param: Any, address: st # BLSToExecutionChange # +def _bls_to_execution_change_validator(kwargs: Dict[str, Any]) -> bool: + return validate_bls_to_execution_change(**kwargs) + def verify_bls_to_execution_change_json(filefolder: str, credentials: Sequence[Credential], @@ -178,19 +181,28 @@ def verify_bls_to_execution_change_json(filefolder: str, """ Validate every BLSToExecutionChange found in the bls_to_execution_change JSON file folder. """ + btec_json = [] with open(filefolder, 'r', encoding='utf-8') as f: btec_json = json.load(f) - with click.progressbar(btec_json, label=load_text(['msg_bls_to_execution_change_verification']), - show_percent=False, show_pos=True) as btecs: - return all([ - validate_bls_to_execution_change( - btec, credential, - input_validator_index=input_validator_index, - input_execution_address=input_execution_address, - chain_setting=chain_setting) - for btec, credential, input_validator_index in zip(btecs, credentials, input_validator_indices) - ]) - return False + + valid = True + with click.progressbar(length=len(btec_json), label=load_text(['msg_bls_to_execution_change_verification']), + show_percent=False, show_pos=True) as bar: + + executor_kwargs = [{ + 'btec_dict': btec, + 'credential': credential, + 'input_validator_index': input_validator_index, + 'input_execution_address': input_execution_address, + 'chain_setting': chain_setting, + } for btec, credential, input_validator_index in zip(btec_json, credentials, input_validator_indices)] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for valid_bls_change in executor.map(_bls_to_execution_change_validator, executor_kwargs): + valid &= valid_bls_change + bar.update(1) + + return valid def validate_bls_to_execution_change(btec_dict: Dict[str, Any], From fcf477ac60370b090bac27762bb6e433dfeb24b4 Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Wed, 8 May 2024 14:24:55 -0400 Subject: [PATCH 04/13] Better naming local variables --- staking_deposit/credentials.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/staking_deposit/credentials.py b/staking_deposit/credentials.py index 821d513d..eab83f16 100644 --- a/staking_deposit/credentials.py +++ b/staking_deposit/credentials.py @@ -271,7 +271,7 @@ def from_mnemonic(cls, ) key_indices = range(start_index, start_index + num_keys) - return_list: List[Credential] = [] + credentials: List[Credential] = [] with click.progressbar(length=num_keys, label=load_text(['msg_key_creation']), show_percent=False, show_pos=True) as bar: executor_kwargs = [{ @@ -285,12 +285,12 @@ def from_mnemonic(cls, with concurrent.futures.ProcessPoolExecutor() as executor: for credential in executor.map(_credential_builder, executor_kwargs): - return_list.append(credential) + credentials.append(credential) bar.update(1) - return cls(return_list) + return cls(credentials) def export_keystores(self, password: str, folder: str) -> List[str]: - return_list: List[str] = [] + filefolders: List[str] = [] with click.progressbar(length=len(self.credentials), label=load_text(['msg_keystore_creation']), show_percent=False, show_pos=True) as bar: executor_kwargs = [{ @@ -300,10 +300,10 @@ def export_keystores(self, password: str, folder: str) -> List[str]: } for credential in self.credentials] with concurrent.futures.ProcessPoolExecutor() as executor: - for keystore in executor.map(_keystore_exporter, executor_kwargs): - return_list.append(keystore) + for filefolder in executor.map(_keystore_exporter, executor_kwargs): + filefolders.append(filefolder) bar.update(1) - return return_list + return filefolders def export_deposit_data_json(self, folder: str) -> str: deposit_data = [] From 79b2f41569a4eb86de3daf951194479160c4b907 Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Thu, 9 May 2024 11:15:38 -0400 Subject: [PATCH 05/13] Using multiprocess for the steps exit-transaction-mnemonic command --- .../cli/exit_transaction_mnemonic.py | 102 ++++++++++++------ staking_deposit/credentials.py | 6 +- .../en/cli/exit_transaction_mnemonic.json | 1 + staking_deposit/utils/validation.py | 12 +-- 4 files changed, 78 insertions(+), 43 deletions(-) diff --git a/staking_deposit/cli/exit_transaction_mnemonic.py b/staking_deposit/cli/exit_transaction_mnemonic.py index 3a7f8376..8d933da0 100644 --- a/staking_deposit/cli/exit_transaction_mnemonic.py +++ b/staking_deposit/cli/exit_transaction_mnemonic.py @@ -1,7 +1,8 @@ import click import os +import concurrent.futures -from typing import Any, Sequence +from typing import Any, Sequence, Dict from staking_deposit.cli.existing_mnemonic import load_mnemonic_arguments_decorator from staking_deposit.credentials import Credential from staking_deposit.exceptions import ValidationError @@ -23,6 +24,22 @@ from staking_deposit.utils.validation import validate_int_range, validate_validator_indices, verify_signed_exit_json +def _credential_builder(kwargs: Dict[str, Any]) -> Credential: + return Credential(**kwargs) + + +def _exit_exporter(kwargs: Dict[str, Any]) -> str: + credential: Credential = kwargs.pop('credential') + return credential.save_exit_transaction(**kwargs) + + +def _exit_verifier(kwargs: Dict[str, Any]) -> bool: + credential: Credential = kwargs.pop('credential') + kwargs['pubkey'] = credential.signing_pk.hex() + kwargs['chain_settings'] = credential.chain_setting + return verify_signed_exit_json(**kwargs) + + FUNC_NAME = 'exit_transaction_mnemonic' @@ -94,39 +111,56 @@ def exit_transaction_mnemonic( key_indices = range(validator_start_index, validator_start_index + num_keys) # We are not using CredentialList because from_mnemonic assumes key generation flow - credentials = [ - Credential( - mnemonic=mnemonic, - mnemonic_password=mnemonic_password, - index=key_index, - amount=0, # Unneeded for this purpose - chain_setting=chain_settings, - hex_eth1_withdrawal_address=None - ) for key_index in key_indices - ] - - with click.progressbar(zip(credentials, validator_indices), - label=load_text(['msg_exit_transaction_creation']), - show_percent=False, - length=num_keys, - show_pos=True) as items: - transaction_filefolders = [ - credential.save_exit_transaction(validator_index=validator_index, epoch=epoch, folder=folder) - for credential, validator_index in items - ] - - with click.progressbar(zip(transaction_filefolders, credentials), - label=load_text(['msg_verify_exit_transaction']), - show_percent=False, - length=num_keys, - show_pos=True) as items: - if not all( - verify_signed_exit_json(file_folder=file, - pubkey=credential.signing_pk.hex(), - chain_settings=credential.chain_setting) - for file, credential in items - ): - raise ValidationError(load_text(['err_verify_exit_transactions'])) + credentials = [] + with click.progressbar(length=num_keys, label=load_text(['msg_key_creation']), + show_percent=False, show_pos=True) as bar: + + executor_kwargs = [{ + 'mnemonic': mnemonic, + 'mnemonic_password': mnemonic_password, + 'index': index, + 'amount': 0, + 'chain_setting': chain_settings, + 'hex_eth1_withdrawal_address': None, + } for index in key_indices] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for credential in executor.map(_credential_builder, executor_kwargs): + credentials.append(credential) + bar.update(1) + + transaction_filefolders = [] + with click.progressbar(length=num_keys, label=load_text(['msg_exit_transaction_creation']), + show_percent=False, show_pos=True) as bar: + + executor_kwargs = [{ + 'credential': credential, + 'validator_index': validator_index, + 'epoch': epoch, + 'folder': folder, + } for credential, validator_index in zip(credentials, validator_indices)] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for filefolder in executor.map(_exit_exporter, executor_kwargs): + transaction_filefolders.append(filefolder) + bar.update(1) + + all_valid_exits = True + with click.progressbar(length=num_keys, label=load_text(['msg_verify_exit_transaction']), + show_percent=False, show_pos=True) as bar: + + executor_kwargs = [{ + 'file_folder': file, + 'credential': credential, + } for file, credential in zip(transaction_filefolders, credentials)] + + with concurrent.futures.ProcessPoolExecutor() as executor: + for valid_exit in executor.map(_exit_verifier, executor_kwargs): + all_valid_exits &= valid_exit + bar.update(1) + + if not all_valid_exits: + raise ValidationError(load_text(['err_verify_exit_transactions'])) click.echo(load_text(['msg_creation_success']) + folder) click.pause(load_text(['msg_pause'])) diff --git a/staking_deposit/credentials.py b/staking_deposit/credentials.py index eab83f16..ef9de5d7 100644 --- a/staking_deposit/credentials.py +++ b/staking_deposit/credentials.py @@ -323,7 +323,7 @@ def export_deposit_data_json(self, folder: str) -> str: return filefolder def verify_keystores(self, keystore_filefolders: List[str], password: str) -> bool: - valid = True + all_valid_keystores = True with click.progressbar(length=len(self.credentials), label=load_text(['msg_keystore_verification']), show_percent=False, show_pos=True) as bar: @@ -335,10 +335,10 @@ def verify_keystores(self, keystore_filefolders: List[str], password: str) -> bo with concurrent.futures.ProcessPoolExecutor() as executor: for valid_keystore in executor.map(_keystore_verifier, executor_kwargs): - valid &= valid_keystore + all_valid_keystores &= valid_keystore bar.update(1) - return valid + return all_valid_keystores def export_bls_to_execution_change_json(self, folder: str, validator_indices: Sequence[int]) -> str: bls_to_execution_changes = [] diff --git a/staking_deposit/intl/en/cli/exit_transaction_mnemonic.json b/staking_deposit/intl/en/cli/exit_transaction_mnemonic.json index 722bfa69..b0e8510c 100644 --- a/staking_deposit/intl/en/cli/exit_transaction_mnemonic.json +++ b/staking_deposit/intl/en/cli/exit_transaction_mnemonic.json @@ -21,6 +21,7 @@ "arg_exit_transaction_mnemonic_output_folder": { "help": "The folder path where the exit transactions will be saved to. Pointing to `./exit_transactions` by default." }, + "msg_key_creation": "Creating your keys:\t", "msg_exit_transaction_creation": "Creating your exit transactions:\t", "msg_verify_exit_transaction": "Verifying your exit transactions:\t", "err_verify_exit_transactions": "\nThere was a problem verifying your exit transactions.\nPlease try again", diff --git a/staking_deposit/utils/validation.py b/staking_deposit/utils/validation.py index e72d983e..ddcf920f 100644 --- a/staking_deposit/utils/validation.py +++ b/staking_deposit/utils/validation.py @@ -53,7 +53,7 @@ def verify_deposit_data_json(filefolder: str, credentials: Sequence[Credential]) """ Validate every deposit found in the deposit-data JSON file folder. """ - valid = True + all_valid_deposits = True deposit_json = [] with open(filefolder, 'r', encoding='utf-8') as f: deposit_json = json.load(f) @@ -67,10 +67,10 @@ def verify_deposit_data_json(filefolder: str, credentials: Sequence[Credential]) with concurrent.futures.ProcessPoolExecutor() as executor: for valid_deposit in executor.map(_deposit_validator, executor_kwargs): - valid &= valid_deposit + all_valid_deposits &= valid_deposit bar.update(1) - return valid + return all_valid_deposits def validate_deposit(deposit_data_dict: Dict[str, Any], credential: Credential) -> bool: @@ -185,7 +185,7 @@ def verify_bls_to_execution_change_json(filefolder: str, with open(filefolder, 'r', encoding='utf-8') as f: btec_json = json.load(f) - valid = True + all_valid_bls_changes = True with click.progressbar(length=len(btec_json), label=load_text(['msg_bls_to_execution_change_verification']), show_percent=False, show_pos=True) as bar: @@ -199,10 +199,10 @@ def verify_bls_to_execution_change_json(filefolder: str, with concurrent.futures.ProcessPoolExecutor() as executor: for valid_bls_change in executor.map(_bls_to_execution_change_validator, executor_kwargs): - valid &= valid_bls_change + all_valid_bls_changes &= valid_bls_change bar.update(1) - return valid + return all_valid_bls_changes def validate_bls_to_execution_change(btec_dict: Dict[str, Any], From 8fd698e443054768db2c79729e963f6f3cbb84c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Roy?= <303593+remyroy@users.noreply.github.com> Date: Thu, 9 May 2024 11:35:22 -0400 Subject: [PATCH 06/13] Adding freeze support for Windows bundle issue --- staking_deposit/deposit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/staking_deposit/deposit.py b/staking_deposit/deposit.py index d45eaa3f..b58f0dc1 100644 --- a/staking_deposit/deposit.py +++ b/staking_deposit/deposit.py @@ -1,5 +1,6 @@ import click import sys +from multiprocessing import freeze_support from staking_deposit.cli.existing_mnemonic import existing_mnemonic from staking_deposit.cli.exit_transaction_keystore import exit_transaction_keystore @@ -63,6 +64,7 @@ def cli(ctx: click.Context, language: str, non_interactive: bool) -> None: def run() -> None: + freeze_support() # Needed when running under Windows in a frozen bundle check_python_version() print('\n***Using the tool on an offline and secure device is highly recommended to keep your mnemonic safe.***\n') cli() From 6e3cb3c2ccd48315ace887d28f47a5bc9ca50873 Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Thu, 9 May 2024 11:56:51 -0400 Subject: [PATCH 07/13] Fix linter issue --- staking_deposit/deposit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staking_deposit/deposit.py b/staking_deposit/deposit.py index 4ffc755a..226e5d91 100644 --- a/staking_deposit/deposit.py +++ b/staking_deposit/deposit.py @@ -92,7 +92,7 @@ def cli(ctx: click.Context, language: str, non_interactive: bool, ignore_connect def run() -> None: - freeze_support() # Needed when running under Windows in a frozen bundle + freeze_support() # Needed when running under Windows in a frozen bundle check_python_version() cli() From 548940d428f794e61a2aee9b79cd95e852f9806a Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Thu, 23 May 2024 11:01:56 -0400 Subject: [PATCH 08/13] Fix for CliRunner not supporting concurrency --- .../test_exit_transaction_mnemonic.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/test_cli/test_exit_transaction_mnemonic.py b/tests/test_cli/test_exit_transaction_mnemonic.py index cd783e6f..1e5bf9cc 100644 --- a/tests/test_cli/test_exit_transaction_mnemonic.py +++ b/tests/test_cli/test_exit_transaction_mnemonic.py @@ -1,3 +1,4 @@ +import asyncio import os import pytest @@ -9,7 +10,7 @@ from tests.test_cli.helpers import clean_exit_transaction_folder, read_json_file, verify_file_permission -def test_exit_transaction_menmonic() -> None: +def test_exit_transaction_mnemonic() -> None: # Prepare folder my_folder_path = os.path.join(os.getcwd(), 'TESTING_TEMP_FOLDER') clean_exit_transaction_folder(my_folder_path) @@ -54,30 +55,43 @@ def test_exit_transaction_menmonic() -> None: clean_exit_transaction_folder(my_folder_path) -def test_exit_transaction_menmonic_multiple() -> None: +@pytest.mark.asyncio +async def test_exit_transaction_mnemonic_multiple() -> None: # Prepare folder my_folder_path = os.path.join(os.getcwd(), 'TESTING_TEMP_FOLDER') clean_exit_transaction_folder(my_folder_path) if not os.path.exists(my_folder_path): os.mkdir(my_folder_path) - runner = CliRunner() - inputs = [] - data = '\n'.join(inputs) - arguments = [ + if os.name == 'nt': # Windows + run_script_cmd = 'sh deposit.sh' + else: # Mac or Linux + run_script_cmd = './deposit.sh' + + install_cmd = run_script_cmd + ' install' + proc = await asyncio.create_subprocess_shell( + install_cmd, + ) + await proc.wait() + + cmd_args = [ + run_script_cmd, '--language', 'english', '--non_interactive', 'exit-transaction-mnemonic', '--output_folder', my_folder_path, '--chain', 'mainnet', - '--mnemonic', 'aban aban aban aban aban aban aban aban aban aban aban abou', + '--mnemonic', '"aban aban aban aban aban aban aban aban aban aban aban abou"', '--validator_start_index', '0', - '--validator_indices', '0 1 2 3', + '--validator_indices', '0,1,2,3', '--epoch', '1234', ] - result = runner.invoke(cli, arguments, input=data) + proc = await asyncio.create_subprocess_shell( + ' '.join(cmd_args), + ) + await proc.wait() - assert result.exit_code == 0 + assert proc.returncode == 0 # Check files exit_transaction_folder_path = os.path.join(my_folder_path, DEFAULT_EXIT_TRANSACTION_FOLDER_NAME) From ab10f1122386f55628eefd1ea40bf2e9bfadf5a3 Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Mon, 27 May 2024 11:11:24 -0400 Subject: [PATCH 09/13] Add a another newline to make the error clear on the CLI --- staking_deposit/cli/generate_bls_to_execution_change.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staking_deposit/cli/generate_bls_to_execution_change.py b/staking_deposit/cli/generate_bls_to_execution_change.py index bd38f046..ae3ab9d6 100644 --- a/staking_deposit/cli/generate_bls_to_execution_change.py +++ b/staking_deposit/cli/generate_bls_to_execution_change.py @@ -203,7 +203,7 @@ def generate_bls_to_execution_change( for e in executor.map(_validate_credentials_match, executor_kwargs): bar.update(1) if e is not None: - click.echo('\n[Error] ' + str(e)) + click.echo('\n\n[Error] ' + str(e)) return btec_file = credentials.export_bls_to_execution_change_json(bls_to_execution_changes_folder, validator_indices) From bc30f2cceeba42cf2f8ad955705d27fb3d9a7513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Roy?= <303593+remyroy@users.noreply.github.com> Date: Mon, 27 May 2024 11:12:54 -0400 Subject: [PATCH 10/13] Early return on failed exit verifier Co-authored-by: valefar-on-discord <124839138+valefar-on-discord@users.noreply.github.com> --- staking_deposit/cli/exit_transaction_mnemonic.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/staking_deposit/cli/exit_transaction_mnemonic.py b/staking_deposit/cli/exit_transaction_mnemonic.py index 8d933da0..3fb36c6f 100644 --- a/staking_deposit/cli/exit_transaction_mnemonic.py +++ b/staking_deposit/cli/exit_transaction_mnemonic.py @@ -145,7 +145,6 @@ def exit_transaction_mnemonic( transaction_filefolders.append(filefolder) bar.update(1) - all_valid_exits = True with click.progressbar(length=num_keys, label=load_text(['msg_verify_exit_transaction']), show_percent=False, show_pos=True) as bar: @@ -156,11 +155,9 @@ def exit_transaction_mnemonic( with concurrent.futures.ProcessPoolExecutor() as executor: for valid_exit in executor.map(_exit_verifier, executor_kwargs): - all_valid_exits &= valid_exit bar.update(1) - - if not all_valid_exits: - raise ValidationError(load_text(['err_verify_exit_transactions'])) + if not valid_exit: + raise ValidationError(load_text(['err_verify_exit_transactions'])) click.echo(load_text(['msg_creation_success']) + folder) click.pause(load_text(['msg_pause'])) From b59b0601cd159506106bf0fd60f9e942b3f30144 Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Mon, 27 May 2024 11:33:16 -0400 Subject: [PATCH 11/13] Simplify concurrent deposit validation --- staking_deposit/utils/validation.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/staking_deposit/utils/validation.py b/staking_deposit/utils/validation.py index ddcf920f..fc542c84 100644 --- a/staking_deposit/utils/validation.py +++ b/staking_deposit/utils/validation.py @@ -43,11 +43,6 @@ # Deposit # -def _deposit_validator(kwargs: Dict[str, Any]) -> bool: - deposit: Dict[str, Any] = kwargs.pop('deposit') - credential: Credential = kwargs.pop('credential') - return validate_deposit(deposit, credential) - def verify_deposit_data_json(filefolder: str, credentials: Sequence[Credential]) -> bool: """ @@ -60,13 +55,9 @@ def verify_deposit_data_json(filefolder: str, credentials: Sequence[Credential]) with click.progressbar(length=len(deposit_json), label=load_text(['msg_deposit_verification']), show_percent=False, show_pos=True) as bar: - executor_kwargs = [{ - 'credential': credential, - 'deposit': deposit, - } for deposit, credential in zip(deposit_json, credentials)] with concurrent.futures.ProcessPoolExecutor() as executor: - for valid_deposit in executor.map(_deposit_validator, executor_kwargs): + for valid_deposit in executor.map(validate_deposit, deposit_json, credentials): all_valid_deposits &= valid_deposit bar.update(1) From c35380e9d0dd8fe6ce1198e27c717e7a124c2a2e Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Tue, 28 May 2024 12:27:41 -0400 Subject: [PATCH 12/13] Fix a potential race in creating the ouput folder during exit-transaction-mnemonic command --- staking_deposit/cli/exit_transaction_keystore.py | 2 ++ staking_deposit/cli/exit_transaction_mnemonic.py | 3 +++ staking_deposit/exit_transaction.py | 3 --- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/staking_deposit/cli/exit_transaction_keystore.py b/staking_deposit/cli/exit_transaction_keystore.py index 029ac2f2..1216f94d 100644 --- a/staking_deposit/cli/exit_transaction_keystore.py +++ b/staking_deposit/cli/exit_transaction_keystore.py @@ -114,6 +114,8 @@ def exit_transaction_keystore( ) folder = os.path.join(output_folder, DEFAULT_EXIT_TRANSACTION_FOLDER_NAME) + if not os.path.exists(folder): + os.mkdir(folder) click.echo(load_text(['msg_exit_transaction_creation'])) saved_folder = export_exit_transaction_json(folder=folder, signed_exit=signed_exit) diff --git a/staking_deposit/cli/exit_transaction_mnemonic.py b/staking_deposit/cli/exit_transaction_mnemonic.py index 3fb36c6f..56a081f8 100644 --- a/staking_deposit/cli/exit_transaction_mnemonic.py +++ b/staking_deposit/cli/exit_transaction_mnemonic.py @@ -129,6 +129,9 @@ def exit_transaction_mnemonic( credentials.append(credential) bar.update(1) + if not os.path.exists(folder): + os.mkdir(folder) + transaction_filefolders = [] with click.progressbar(length=num_keys, label=load_text(['msg_exit_transaction_creation']), show_percent=False, show_pos=True) as bar: diff --git a/staking_deposit/exit_transaction.py b/staking_deposit/exit_transaction.py index c73376fd..608409a3 100644 --- a/staking_deposit/exit_transaction.py +++ b/staking_deposit/exit_transaction.py @@ -48,9 +48,6 @@ def export_exit_transaction_json(folder: str, signed_exit: SignedVoluntaryExit) signed_exit_json.update({'message': message}) signed_exit_json.update({'signature': '0x' + signed_exit.signature.hex()}) # type: ignore[attr-defined] - if not os.path.exists(folder): - os.mkdir(folder) - filefolder = os.path.join( folder, 'signed_exit_transaction-%s-%i.json' % ( From 9704f760831537e68b4aaf523d2f7a507eb89591 Mon Sep 17 00:00:00 2001 From: Remy Roy <303593+remyroy@users.noreply.github.com> Date: Tue, 28 May 2024 12:53:20 -0400 Subject: [PATCH 13/13] Make sure to wait for the process to exit to prevent some races --- test_binary_btec_script.py | 2 ++ test_binary_deposit_script.py | 2 ++ test_btec_script.py | 2 ++ test_deposit_script.py | 2 ++ tests/test_cli/test_existing_mnemonic.py | 2 ++ tests/test_cli/test_new_mnemonic.py | 4 ++++ 6 files changed, 14 insertions(+) diff --git a/test_binary_btec_script.py b/test_binary_btec_script.py index 6fb92d32..8850eca0 100755 --- a/test_binary_btec_script.py +++ b/test_binary_btec_script.py @@ -61,6 +61,8 @@ async def main(argv): assert len(seed_phrase) > 0 + await proc.wait() + # Check files validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path)) diff --git a/test_binary_deposit_script.py b/test_binary_deposit_script.py index e041d0aa..b5757750 100755 --- a/test_binary_deposit_script.py +++ b/test_binary_deposit_script.py @@ -57,6 +57,8 @@ async def main(argv): assert len(seed_phrase) > 0 + await proc.wait() + # Check files validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path)) diff --git a/test_btec_script.py b/test_btec_script.py index 6e6e22b7..11d21816 100755 --- a/test_btec_script.py +++ b/test_btec_script.py @@ -66,6 +66,8 @@ async def main(): assert len(seed_phrase) > 0 + await proc.wait() + # Check files validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path)) diff --git a/test_deposit_script.py b/test_deposit_script.py index 66317aa5..43dfdeee 100755 --- a/test_deposit_script.py +++ b/test_deposit_script.py @@ -62,6 +62,8 @@ async def main(): assert len(seed_phrase) > 0 + await proc.wait() + # Check files validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path)) diff --git a/tests/test_cli/test_existing_mnemonic.py b/tests/test_cli/test_existing_mnemonic.py index 5b253c8f..64977635 100644 --- a/tests/test_cli/test_existing_mnemonic.py +++ b/tests/test_cli/test_existing_mnemonic.py @@ -281,6 +281,7 @@ async def test_script() -> None: ' '.join(cmd_args), ) await proc.wait() + # Check files validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path)) @@ -329,6 +330,7 @@ async def test_script_abbreviated_mnemonic() -> None: ' '.join(cmd_args), ) await proc.wait() + # Check files validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path)) diff --git a/tests/test_cli/test_new_mnemonic.py b/tests/test_cli/test_new_mnemonic.py index cdbb65fa..8a955ff0 100644 --- a/tests/test_cli/test_new_mnemonic.py +++ b/tests/test_cli/test_new_mnemonic.py @@ -409,6 +409,8 @@ async def test_script_bls_withdrawal() -> None: assert len(seed_phrase) > 0 + await proc.wait() + # Check files validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path)) @@ -495,6 +497,8 @@ async def test_script_abbreviated_mnemonic() -> None: assert len(seed_phrase) > 0 + await proc.wait() + # Check files validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path))