Merge #6071: backport: Merge bitcoin#22619, 22593

ad840ec Merge bitcoin#22593: remove `hex_str_to_bytes` helper (Kittywhiskers Van Gogh)
cbc4c63 Merge bitcoin#22619: test: refactor: use consistent bytes <-> hex-string conversion in functional test framework (MarcoFalke)

Pull request description:

  bitcoin backports

ACKs for top commit:
  knst:
    utACK ad840ec
  kwvg:
    utACK ad840ec

Tree-SHA512: 48b4137683daab01a1bf51493d082ec359f80be7a9930b3423476e9ac5f4e73035d0f64d0f8e9b6b0c61b3e06efb648f9cc6bd620088c8cb5f27830157440adb

test/functional/feature_addressindex.py

@@ -7,8 +7,6 @@
 # Test addressindex generation and fetching
 #
 
-import binascii
-
 from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.test_node import ErrorMatch
@@ -132,7 +130,7 @@ def run_test(self):
 
         # Check that outputs with the same address will only return one txid
         self.log.info("Testing for txid uniqueness...")
-        addressHash = binascii.unhexlify("FE30B718DCF0BF8A2A686BF1820C073F8B2C3B37")
+        addressHash = bytes.fromhex("FE30B718DCF0BF8A2A686BF1820C073F8B2C3B37")
         scriptPubKey = CScript([OP_HASH160, addressHash, OP_EQUAL])
         unspent = self.nodes[0].listunspent()
         tx = CTransaction()
@@ -159,7 +157,7 @@ def run_test(self):
         self.log.info("Testing balances after spending...")
         privkey2 = "cU4zhap7nPJAWeMFu4j6jLrfPmqakDAzy8zn8Fhb3oEevdm4e5Lc"
         address2 = "yeMpGzMj3rhtnz48XsfpB8itPHhHtgxLc3"
-        addressHash2 = binascii.unhexlify("C5E4FB9171C22409809A3E8047A29C83886E325D")
+        addressHash2 = bytes.fromhex("C5E4FB9171C22409809A3E8047A29C83886E325D")
         scriptPubKey2 = CScript([OP_DUP, OP_HASH160, addressHash2, OP_EQUALVERIFY, OP_CHECKSIG])
         self.nodes[0].importprivkey(privkey2)
 
@@ -254,7 +252,7 @@ def run_test(self):
 
         privKey3 = "cRyrMvvqi1dmpiCmjmmATqjAwo6Wu7QTjKu1ABMYW5aFG4VXW99K"
         address3 = "yWB15aAdpeKuSaQHFVJpBDPbNSLZJSnDLA"
-        addressHash3 = binascii.unhexlify("6C186B3A308A77C779A9BB71C3B5A7EC28232A13")
+        addressHash3 = bytes.fromhex("6C186B3A308A77C779A9BB71C3B5A7EC28232A13")
         scriptPubKey3 = CScript([OP_DUP, OP_HASH160, addressHash3, OP_EQUALVERIFY, OP_CHECKSIG])
         # address4 = "2N8oFVB2vThAKury4vnLquW2zVjsYjjAkYQ"
         scriptPubKey4 = CScript([OP_HASH160, addressHash3, OP_EQUAL])
@@ -320,7 +318,7 @@ def run_test(self):
         # sending and receiving to the same address
         privkey1 = "cMvZn1pVWntTEcsK36ZteGQXRAcZ8CoTbMXF1QasxBLdnTwyVQCc"
         address1 = "yM9Eed1bxjy7tYxD3yZDHxjcVT48WdRoB1"
-        address1hash = binascii.unhexlify("0909C84A817651502E020AAD0FBCAE5F656E7D8A")
+        address1hash = bytes.fromhex("0909C84A817651502E020AAD0FBCAE5F656E7D8A")
         address1script = CScript([OP_DUP, OP_HASH160, address1hash, OP_EQUALVERIFY, OP_CHECKSIG])
 
         self.nodes[0].sendtoaddress(address1, 10)

test/functional/feature_asset_locks.py

@@ -41,7 +41,6 @@
     assert_equal,
     assert_greater_than,
     assert_greater_than_or_equal,
-    hex_str_to_bytes,
 )
 from test_framework.wallet_util import bytes_to_wif
 
@@ -170,7 +169,7 @@ def create_and_check_block(self, txes, expected_error = None):
 
         cbb = create_coinbase(height, dip4_activated=True, v20_activated=True)
         gbt = node_wallet.getblocktemplate()
-        cbb.vExtraPayload = hex_str_to_bytes(gbt["coinbase_payload"])
+        cbb.vExtraPayload = bytes.fromhex(gbt["coinbase_payload"])
         cbb.rehash()
         block = create_block(tip, cbb, block_time, version=4)
         # Add quorum commitments from block template

test/functional/feature_dbcrash.py

@@ -41,7 +41,6 @@
 from test_framework.util import (
     assert_equal,
     create_confirmed_utxos,
-    hex_str_to_bytes,
 )
 
 
@@ -204,7 +203,7 @@ def generate_small_transactions(self, node, count, utxo_list):
                 continue
 
             for _ in range(3):
-                tx.vout.append(CTxOut(output_amount, hex_str_to_bytes(utxo['scriptPubKey'])))
+                tx.vout.append(CTxOut(output_amount, bytes.fromhex(utxo['scriptPubKey'])))
 
             # Sign and send the transaction to get into the mempool
             tx_signed_hex = node.signrawtransactionwithwallet(tx.serialize().hex())['hex']

test/functional/feature_llmq_chainlocks.py

@@ -15,7 +15,7 @@
 
 from test_framework.messages import CBlock, CCbTx
 from test_framework.test_framework import DashTestFramework
-from test_framework.util import assert_equal, assert_raises_rpc_error, force_finish_mnsync, hex_str_to_bytes, softfork_active
+from test_framework.util import assert_equal, assert_raises_rpc_error, force_finish_mnsync, softfork_active
 
 
 class LLMQChainLocksTest(DashTestFramework):
@@ -307,7 +307,7 @@ def test_bestCLHeightDiff(self, mn_rr_active):
         tip0_hash = self.nodes[0].generate(1)[0]
         block_hex = self.nodes[0].getblock(tip0_hash, 0)
         mal_block = CBlock()
-        mal_block.deserialize(BytesIO(hex_str_to_bytes(block_hex)))
+        mal_block.deserialize(BytesIO(bytes.fromhex(block_hex)))
         cbtx = CCbTx()
         cbtx.deserialize(BytesIO(mal_block.vtx[0].vExtraPayload))
         assert_equal(cbtx.bestCLHeightDiff, 0)
@@ -322,7 +322,7 @@ def test_bestCLHeightDiff(self, mn_rr_active):
         assert_equal(self.nodes[1].getbestblockhash(), tip1_hash)
 
         # Update the sig too and it should pass now
-        cbtx.bestCLSignature = hex_str_to_bytes(self.nodes[1].getblock(tip1_hash, 2)["tx"][0]["cbTx"]["bestCLSignature"])
+        cbtx.bestCLSignature = bytes.fromhex(self.nodes[1].getblock(tip1_hash, 2)["tx"][0]["cbTx"]["bestCLSignature"])
         mal_block.vtx[0].vExtraPayload = cbtx.serialize()
         mal_block.vtx[0].rehash()
         mal_block.hashMerkleRoot = mal_block.calc_merkle_root()
@@ -347,7 +347,7 @@ def test_bestCLHeightDiff(self, mn_rr_active):
 
         # Update the sig too and it should pass now when mn_rr is not active and fail otherwise
         old_blockhash = self.nodes[1].getblockhash(self.nodes[1].getblockcount() - 1)
-        cbtx.bestCLSignature = hex_str_to_bytes(self.nodes[1].getblock(old_blockhash, 2)["tx"][0]["cbTx"]["bestCLSignature"])
+        cbtx.bestCLSignature = bytes.fromhex(self.nodes[1].getblock(old_blockhash, 2)["tx"][0]["cbTx"]["bestCLSignature"])
         mal_block.vtx[0].vExtraPayload = cbtx.serialize()
         mal_block.vtx[0].rehash()
         mal_block.hashMerkleRoot = mal_block.calc_merkle_root()

test/functional/feature_llmq_is_cl_conflicts.py

@@ -16,7 +16,7 @@
 from test_framework.messages import CInv, hash256, msg_clsig, msg_inv, ser_string, tx_from_hex, uint256_from_str
 from test_framework.p2p import P2PInterface
 from test_framework.test_framework import DashTestFramework
-from test_framework.util import assert_equal, assert_raises_rpc_error, hex_str_to_bytes
+from test_framework.util import assert_equal, assert_raises_rpc_error
 
 
 class TestP2PConn(P2PInterface):
@@ -90,7 +90,7 @@ def test_chainlock_overrides_islock(self, test_block_conflict, mine_confllicting
         rawtx2_obj = tx_from_hex(rawtx2)
 
         rawtx1_txid = self.nodes[0].sendrawtransaction(rawtx1)
-        rawtx2_txid = hash256(hex_str_to_bytes(rawtx2))[::-1].hex()
+        rawtx2_txid = hash256(bytes.fromhex(rawtx2))[::-1].hex()
 
         # Create a chained TX on top of tx1
         inputs = []
@@ -203,8 +203,8 @@ def test_chainlock_overrides_islock_overrides_nonchainlock(self):
         rawtx1 = self.create_raw_tx(self.nodes[0], self.nodes[0], 1, 1, 100)['hex']
         rawtx2 = self.create_raw_tx(self.nodes[0], self.nodes[0], 1, 1, 100)['hex']
 
-        rawtx1_txid = hash256(hex_str_to_bytes(rawtx1))[::-1].hex()
-        rawtx2_txid = hash256(hex_str_to_bytes(rawtx2))[::-1].hex()
+        rawtx1_txid = hash256(bytes.fromhex(rawtx1))[::-1].hex()
+        rawtx2_txid = hash256(bytes.fromhex(rawtx2))[::-1].hex()
 
         # Create an ISLOCK but don't broadcast it yet
         isdlock = self.create_isdlock(rawtx2)
@@ -278,7 +278,7 @@ def create_chainlock(self, height, block):
         message_hash = block.hash
 
         recSig = self.get_recovered_sig(request_id, message_hash)
-        clsig = msg_clsig(height, block.sha256, hex_str_to_bytes(recSig['sig']))
+        clsig = msg_clsig(height, block.sha256, bytes.fromhex(recSig['sig']))
         return clsig
 
 if __name__ == '__main__':

test/functional/feature_llmq_signing.py

@@ -13,7 +13,7 @@
 from test_framework.messages import CSigShare, msg_qsigshare, uint256_to_string
 from test_framework.p2p import P2PInterface
 from test_framework.test_framework import DashTestFramework
-from test_framework.util import assert_equal, assert_raises_rpc_error, force_finish_mnsync, hex_str_to_bytes, wait_until_helper
+from test_framework.util import assert_equal, assert_raises_rpc_error, force_finish_mnsync, wait_until_helper
 
 
 class LLMQSigningTest(DashTestFramework):
@@ -89,7 +89,7 @@ def assert_sigs_nochange(hasrecsigs, isconflicting1, isconflicting2, timeout):
             sig_share.quorumMember = int(sig_share_rpc_1["quorumMember"])
             sig_share.id = int(sig_share_rpc_1["id"], 16)
             sig_share.msgHash = int(sig_share_rpc_1["msgHash"], 16)
-            sig_share.sigShare = hex_str_to_bytes(sig_share_rpc_1["signature"])
+            sig_share.sigShare = bytes.fromhex(sig_share_rpc_1["signature"])
             for mn in self.mninfo:
                 assert mn.node.getconnectioncount() == self.llmq_size
             # Get the current recovery member of the quorum

test/functional/feature_spentindex.py

@@ -7,7 +7,6 @@
 # Test addressindex generation and fetching
 #
 
-import binascii
 from decimal import Decimal
 
 from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut
@@ -64,7 +63,7 @@ def run_test(self):
         self.log.info("Testing spent index...")
 
         privkey = "cU4zhap7nPJAWeMFu4j6jLrfPmqakDAzy8zn8Fhb3oEevdm4e5Lc"
-        addressHash = binascii.unhexlify("C5E4FB9171C22409809A3E8047A29C83886E325D")
+        addressHash = bytes.fromhex("C5E4FB9171C22409809A3E8047A29C83886E325D")
         scriptPubKey = CScript([OP_DUP, OP_HASH160, addressHash, OP_EQUALVERIFY, OP_CHECKSIG])
         unspent = self.nodes[0].listunspent()
         tx = CTransaction()
@@ -103,7 +102,7 @@ def run_test(self):
 
         # Check that verbose raw transaction includes address values and input values
         address2 = "yeMpGzMj3rhtnz48XsfpB8itPHhHtgxLc3"
-        addressHash2 = binascii.unhexlify("C5E4FB9171C22409809A3E8047A29C83886E325D")
+        addressHash2 = bytes.fromhex("C5E4FB9171C22409809A3E8047A29C83886E325D")
         scriptPubKey2 = CScript([OP_DUP, OP_HASH160, addressHash2, OP_EQUALVERIFY, OP_CHECKSIG])
         tx2 = CTransaction()
         tx2.vin = [CTxIn(COutPoint(int(txid, 16), 0))]

test/functional/feature_txindex.py

@@ -7,8 +7,6 @@
 # Test txindex generation and fetching
 #
 
-import binascii
-
 from test_framework.messages import COutPoint, CTransaction, CTxIn, CTxOut
 from test_framework.script import CScript, OP_CHECKSIG, OP_DUP, OP_EQUALVERIFY, OP_HASH160
 from test_framework.test_framework import BitcoinTestFramework
@@ -48,7 +46,7 @@ def run_test(self):
 
         self.log.info("Testing transaction index...")
 
-        addressHash = binascii.unhexlify("C5E4FB9171C22409809A3E8047A29C83886E325D")
+        addressHash = bytes.fromhex("C5E4FB9171C22409809A3E8047A29C83886E325D")
         scriptPubKey = CScript([OP_DUP, OP_HASH160, addressHash, OP_EQUALVERIFY, OP_CHECKSIG])
         unspent = self.nodes[0].listunspent()
         tx = CTransaction()

test/functional/interface_rest.py

@@ -4,7 +4,6 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the REST API."""
 
-import binascii
 from decimal import Decimal
 from enum import Enum
 from io import BytesIO
@@ -19,7 +18,6 @@
     assert_equal,
     assert_greater_than,
     assert_greater_than_or_equal,
-    hex_str_to_bytes,
 )
 
 from test_framework.messages import BLOCK_HEADER_SIZE
@@ -157,7 +155,7 @@ def run_test(self):
 
         bin_request = b'\x01\x02'
         for txid, n in [spending, spent]:
-            bin_request += hex_str_to_bytes(txid)
+            bin_request += bytes.fromhex(txid)
             bin_request += pack("i", n)
 
         bin_response = self.test_rest_request("/getutxos", http_method='POST', req_type=ReqType.BIN, body=bin_request, ret_type=RetType.BYTES)
@@ -246,13 +244,13 @@ def run_test(self):
         response_hex = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ)
         assert_greater_than(int(response_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2)
         response_hex_bytes = response_hex.read().strip(b'\n')
-        assert_equal(binascii.hexlify(response_bytes), response_hex_bytes)
+        assert_equal(response_bytes.hex().encode(), response_hex_bytes)
 
         # Compare with hex block header
         response_header_hex = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ)
         assert_greater_than(int(response_header_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2)
         response_header_hex_bytes = response_header_hex.read(BLOCK_HEADER_SIZE*2)
-        assert_equal(binascii.hexlify(response_bytes[:BLOCK_HEADER_SIZE]), response_header_hex_bytes)
+        assert_equal(response_bytes[:BLOCK_HEADER_SIZE].hex().encode(), response_header_hex_bytes)
 
         # Check json format
         block_json_obj = self.test_rest_request("/block/{}".format(bb_hash))

test/functional/rpc_addresses_deprecation.py

@@ -10,7 +10,6 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
-    hex_str_to_bytes
 )
 
 
@@ -36,8 +35,8 @@ def test_addresses_deprecation(self):
 
         # This transaction is derived from test/util/data/txcreatemultisig1.json
         tx = CTransaction()
-        tx.deserialize(BytesIO(hex_str_to_bytes(signed)))
-        tx.vout[0].scriptPubKey = hex_str_to_bytes("522102a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff39721021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d2102df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb48553ae")
+        tx.deserialize(BytesIO(bytes.fromhex(signed)))
+        tx.vout[0].scriptPubKey = bytes.fromhex("522102a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff39721021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d2102df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb48553ae")
         tx_signed = node.signrawtransactionwithwallet(tx.serialize().hex())['hex']
         txid = node.sendrawtransaction(hexstring=tx_signed, maxfeerate=0)
 

test/functional/rpc_createmultisig.py

@@ -3,7 +3,6 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test multisig RPCs"""
-import binascii
 import decimal
 import itertools
 import json
@@ -64,9 +63,9 @@ def run_test(self):
 
         # decompress pk2
         pk_obj = ECPubKey()
-        pk_obj.set(binascii.unhexlify(pk2))
+        pk_obj.set(bytes.fromhex(pk2))
         pk_obj.compressed = False
-        pk2 = binascii.hexlify(pk_obj.get_bytes()).decode()
+        pk2 = pk_obj.get_bytes().hex()
 
         node0.createwallet(wallet_name='wmulti0', disable_private_keys=True)
         wmulti0 = node0.get_wallet_rpc('wmulti0')

test/functional/rpc_decodescript.py

@@ -10,7 +10,6 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
-    hex_str_to_bytes,
 )
 
 
@@ -154,23 +153,23 @@ def decoderawtransaction_asm_sighashtype(self):
         signature_2_sighash_decoded = der_signature + '[NONE|ANYONECANPAY]'
 
         # 1) P2PK scriptSig
-        txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature)
+        txSave.vin[0].scriptSig = bytes.fromhex(push_signature)
         rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex())
         assert_equal(signature_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm'])
 
         # make sure that the sighash decodes come out correctly for a more complex / lesser used case.
-        txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature_2)
+        txSave.vin[0].scriptSig = bytes.fromhex(push_signature_2)
         rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex())
         assert_equal(signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm'])
 
         # 2) multisig scriptSig
-        txSave.vin[0].scriptSig = hex_str_to_bytes('00' + push_signature + push_signature_2)
+        txSave.vin[0].scriptSig = bytes.fromhex('00' + push_signature + push_signature_2)
         rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex())
         assert_equal('0 ' + signature_sighash_decoded + ' ' + signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm'])
 
         # 3) test a scriptSig that contains more than push operations.
         # in fact, it contains an OP_RETURN with data specially crafted to cause improper decode if the code does not catch it.
-        txSave.vin[0].scriptSig = hex_str_to_bytes('6a143011020701010101010101020601010101010101')
+        txSave.vin[0].scriptSig = bytes.fromhex('6a143011020701010101010101020601010101010101')
         rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex())
         assert_equal('OP_RETURN 3011020701010101010101020601010101010101', rpc_result['vin'][0]['scriptSig']['asm'])
 

test/functional/rpc_mnauth.py

@@ -6,7 +6,7 @@
 from test_framework.messages import hash256
 from test_framework.p2p import P2PInterface
 from test_framework.test_framework import DashTestFramework
-from test_framework.util import assert_equal, assert_raises_rpc_error, hex_str_to_bytes
+from test_framework.util import assert_equal, assert_raises_rpc_error
 
 '''
 rpc_mnauth.py
@@ -39,7 +39,7 @@ def run_test(self):
         assert "verified_proregtx_hash" in peerinfo
         assert "verified_pubkey_hash" in peerinfo
         assert_equal(peerinfo["verified_proregtx_hash"], protx_hash)
-        assert_equal(peerinfo["verified_pubkey_hash"], hash256(hex_str_to_bytes(public_key))[::-1].hex())
+        assert_equal(peerinfo["verified_pubkey_hash"], hash256(bytes.fromhex(public_key))[::-1].hex())
         # Test some error cases
         null_hash = "0000000000000000000000000000000000000000000000000000000000000000"
         assert_raises_rpc_error(-8, "proTxHash invalid", masternode.node.mnauth,

test/functional/test_framework/address.py

@@ -12,7 +12,7 @@
 import unittest
 
 from .script import hash160, hash256, CScript
-from .util import assert_equal, hex_str_to_bytes
+from .util import assert_equal
 
 # Note unlike in bitcoin, this address isn't bech32 since we don't (at this time) support bech32.
 ADDRESS_BCRT1_UNSPENDABLE = 'yVg3NBUHNEhgDceqwVUjsZHreC5PBHnUo9'
@@ -27,7 +27,7 @@ def byte_to_base58(b, version):
     result = ''
     str = b.hex()
     str = chr(version).encode('latin-1').hex() + str
-    checksum = hash256(hex_str_to_bytes(str)).hex()
+    checksum = hash256(bytes.fromhex(str)).hex()
     str += checksum[:8]
     value = int('0x' + str, 0)
     while value > 0:
@@ -89,14 +89,14 @@ def script_to_p2sh(script, main=False):
 
 def check_key(key):
     if (type(key) is str):
-        key = hex_str_to_bytes(key)  # Assuming this is hex string
+        key = bytes.fromhex(key)  # Assuming this is hex string
     if (type(key) is bytes and (len(key) == 33 or len(key) == 65)):
         return key
     assert False
 
 def check_script(script):
     if (type(script) is str):
-        script = hex_str_to_bytes(script)  # Assuming this is hex string
+        script = bytes.fromhex(script)  # Assuming this is hex string
     if (type(script) is bytes or type(script) is CScript):
         return script
     assert False