Add a --descriptors option to various tests

Adds a --descriptors option globally to the test framework. This will
make the test create and use descriptor wallets. However some tests may
not work with this.

Some tests are modified to work with --descriptors and run with that
option in test_runer:
* wallet_basic.py
* wallet_encryption.py
* wallet_keypool.py <---- wallet_keypool_hd.py actually
* wallet_keypool_topup.py
* wallet_labels.py
* wallet_avoidreuse.py

test/functional/rpc_createmultisig.py

@@ -4,13 +4,14 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test multisig RPCs"""
 
+from test_framework.authproxy import JSONRPCException
 from test_framework.descriptors import descsum_create, drop_origins
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_raises_rpc_error,
     assert_equal,
 )
-from test_framework.key import ECPubKey
+from test_framework.key import ECPubKey, ECKey, bytes_to_wif
 
 import binascii
 import decimal
@@ -28,10 +29,14 @@ def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
 
     def get_keys(self):
+        self.pub = []
+        self.priv = []
         node0, node1, node2 = self.nodes
-        add = [node1.getnewaddress() for _ in range(self.nkeys)]
-        self.pub = [node1.getaddressinfo(a)["pubkey"] for a in add]
-        self.priv = [node1.dumpprivkey(a) for a in add]
+        for _ in range(self.nkeys):
+            k = ECKey()
+            k.generate()
+            self.pub.append(k.get_pubkey().get_bytes().hex())
+            self.priv.append(bytes_to_wif(k.get_bytes(), k.is_compressed))
         self.final = node2.getnewaddress()
 
     def run_test(self):
@@ -63,11 +68,14 @@ def run_test(self):
         pk_obj.compressed = False
         pk2 = binascii.hexlify(pk_obj.get_bytes()).decode()
 
+        node0.createwallet(wallet_name='wmulti0', disable_private_keys=True)
+        wmulti0 = node0.get_wallet_rpc('wmulti0')
+
         # Check all permutations of keys because order matters apparently
         for keys in itertools.permutations([pk0, pk1, pk2]):
             # Results should be the same as this legacy one
             legacy_addr = node0.createmultisig(2, keys)['address']
-            assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '')['address'])
+            assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '')['address'])
 
         self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors')
         with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f:
@@ -82,6 +90,8 @@ def run_test(self):
             assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address'])
 
     def check_addmultisigaddress_errors(self):
+        if self.options.descriptors:
+            return
         self.log.info('Check that addmultisigaddress fails when the private keys are missing')
         addresses = [self.nodes[1].getnewaddress() for _ in range(2)]
         assert_raises_rpc_error(-5, 'no full public key for address', lambda: self.nodes[0].addmultisigaddress(nrequired=1, keys=addresses))
@@ -109,6 +119,16 @@ def checkbalances(self):
 
     def do_multisig(self):
         node0, node1, node2 = self.nodes
+        if 'wmulti' not in node1.listwallets():
+            try:
+                node1.loadwallet('wmulti')
+            except JSONRPCException as e:
+                path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti")
+                if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']:
+                    node1.createwallet(wallet_name='wmulti', disable_private_keys=True)
+                else:
+                    raise
+        wmulti = node1.get_wallet_rpc('wmulti')
 
         # Construct the expected descriptor
         desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub))
@@ -121,7 +141,7 @@ def do_multisig(self):
         assert_equal(desc, msig['descriptor'])
 
         # compare against addmultisigaddress
-        msigw = node1.addmultisigaddress(self.nsigs, self.pub, None)
+        msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None)
         maddw = msigw["address"]
         mredeemw = msigw["redeemScript"]
         assert_equal(desc, drop_origins(msigw['descriptor']))
@@ -168,6 +188,8 @@ def do_multisig(self):
         txinfo = node0.getrawtransaction(tx, True, blk)
         self.log.info("n/m=%d/%d size=%d" % (self.nsigs, self.nkeys, txinfo["size"]))
 
+        wmulti.unloadwallet()
+
 
 if __name__ == '__main__':
     RpcCreateMultiSigTest().main()

test/functional/rpc_psbt.py

@@ -46,7 +46,7 @@ def run_test(self):
         final_tx = self.nodes[0].finalizepsbt(signed_tx)['hex']
         self.nodes[0].sendrawtransaction(final_tx)
 
-        # Create p2sh and p2pkh addresses
+        # Get pubkeys
         pubkey0 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress())['pubkey']
         pubkey1 = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['pubkey']
         pubkey2 = self.nodes[2].getaddressinfo(self.nodes[2].getnewaddress())['pubkey']
@@ -204,7 +204,7 @@ def run_test(self):
 
         # Signer tests
         for i, signer in enumerate(signers):
-            self.nodes[2].createwallet("wallet{}".format(i))
+            self.nodes[2].createwallet(wallet_name="wallet{}".format(i))
             wrpc = self.nodes[2].get_wallet_rpc("wallet{}".format(i))
             for key in signer['privkeys']:
                 wrpc.importprivkey(key)

test/functional/test_framework/key.py

@@ -8,6 +8,8 @@
 anything but tests."""
 import random
 
+from .address import byte_to_base58
+
 from .util import modinv
 
 def jacobi_symbol(n, k):
@@ -369,3 +371,14 @@ def sign_ecdsa(self, msg, low_s=True):
         rb = r.to_bytes((r.bit_length() + 8) // 8, 'big')
         sb = s.to_bytes((s.bit_length() + 8) // 8, 'big')
         return b'\x30' + bytes([4 + len(rb) + len(sb), 2, len(rb)]) + rb + bytes([2, len(sb)]) + sb
+
+def bytes_to_wif(b, compressed=True):
+    if compressed:
+        b += b'\x01'
+    return byte_to_base58(b, 239)
+
+def generate_wif_key():
+    # Makes a WIF privkey for imports
+    k = ECKey()
+    k.generate()
+    return bytes_to_wif(k.get_bytes(), k.is_compressed)

test/functional/test_framework/test_framework.py

@@ -211,6 +211,8 @@ def parse_args(self):
         parser.add_argument("--randomseed", type=int,
                             help="set a random seed for deterministically reproducing a previous test run")
         parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts')
+        parser.add_argument("--descriptors", default=False, action="store_true",
+                            help="Run test using a descriptor wallet")
 
         self.add_options(parser)
         self.options = parser.parse_args()
@@ -415,7 +417,7 @@ def setup_network(self):
 
     def setup_nodes(self):
         """Override this method to customize test node setup"""
-        extra_args = None
+        extra_args = [[]] * self.num_nodes
         if hasattr(self, "extra_args"):
             extra_args = self.extra_args
         self.add_nodes(self.num_nodes, extra_args)
@@ -491,6 +493,7 @@ def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None):
                 use_cli=self.options.usecli,
                 start_perf=self.options.perf,
                 use_valgrind=self.options.valgrind,
+                descriptors=self.options.descriptors,
             ))
 
     def add_dynamically_node(self, extra_args=None, *, rpchost=None, binary=None):
@@ -831,6 +834,7 @@ def _initialize_chain(self):
                     mocktime=self.mocktime,
                     coverage_dir=None,
                     cwd=self.options.tmpdir,
+                    descriptors=self.options.descriptors,
                 ))
             self.start_node(CACHE_NODE_ID)
             cache_node = self.nodes[CACHE_NODE_ID]

test/functional/test_framework/test_node.py

@@ -64,7 +64,7 @@ class TestNode():
     To make things easier for the test writer, any unrecognised messages will
     be dispatched to the RPC connection."""
 
-    def __init__(self, i, datadir, extra_args_from_options, *, chain, rpchost, timewait, timeout_factor, bitcoind, bitcoin_cli, mocktime, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False):
+    def __init__(self, i, datadir, extra_args_from_options, *, chain, rpchost, timewait, timeout_factor, bitcoind, bitcoin_cli, mocktime, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, descriptors=False):
         """
         Kwargs:
             start_perf (bool): If True, begin profiling the node with `perf` as soon as
@@ -83,6 +83,7 @@ def __init__(self, i, datadir, extra_args_from_options, *, chain, rpchost, timew
         self.coverage_dir = coverage_dir
         self.cwd = cwd
         self.mocktime = mocktime
+        self.descriptors = descriptors
         if extra_conf is not None:
             append_config(datadir, extra_conf)
         # Most callers will just need to add extra args to the standard list below.
@@ -185,10 +186,10 @@ def __del__(self):
     def __getattr__(self, name):
         """Dispatches any unrecognised messages to the RPC connection or a CLI instance."""
         if self.use_cli:
-            return getattr(RPCOverloadWrapper(self.cli, True), name)
+            return getattr(RPCOverloadWrapper(self.cli, True, self.descriptors), name)
         else:
             assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection")
-            return getattr(RPCOverloadWrapper(self.rpc), name)
+            return getattr(RPCOverloadWrapper(self.rpc, descriptors=self.descriptors), name)
 
     def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs):
         """Start the node."""
@@ -311,11 +312,11 @@ def generate(self, nblocks, maxtries=1000000):
 
     def get_wallet_rpc(self, wallet_name):
         if self.use_cli:
-            return RPCOverloadWrapper(self.cli("-rpcwallet={}".format(wallet_name)), True)
+            return RPCOverloadWrapper(self.cli("-rpcwallet={}".format(wallet_name)), True, self.descriptors)
         else:
             assert self.rpc_connected and self.rpc, self._node_msg("RPC not connected")
             wallet_path = "wallet/{}".format(urllib.parse.quote(wallet_name))
-            return RPCOverloadWrapper(self.rpc / wallet_path)
+            return RPCOverloadWrapper(self.rpc / wallet_path, descriptors=self.descriptors)
 
     def version_is_at_least(self, ver):
         return self.version is None or self.version >= ver
@@ -661,13 +662,30 @@ def send_cli(self, command=None, *args, **kwargs):
             return cli_stdout.rstrip("\n")
 
 class RPCOverloadWrapper():
-    def __init__(self, rpc, cli=False):
+    def __init__(self, rpc, cli=False, descriptors=False):
         self.rpc = rpc
         self.is_cli = cli
+        self.descriptors = descriptors
 
     def __getattr__(self, name):
         return getattr(self.rpc, name)
 
+    def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase=None, avoid_reuse=None, descriptors=None, load_on_startup=None):
+        if self.is_cli:
+            if disable_private_keys is None:
+                disable_private_keys = 'null'
+            if blank is None:
+                blank = 'null'
+            if passphrase is None:
+                passphrase = ''
+            if avoid_reuse is None:
+                avoid_reuse = 'null'
+            if load_on_startup is None:
+                load_on_startup = 'null'
+        if descriptors is None:
+            descriptors = self.descriptors
+        return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors, load_on_startup)
+
     def importprivkey(self, privkey, label=None, rescan=None):
         wallet_info = self.getwalletinfo()
         if self.is_cli:

test/functional/test_runner.py

@@ -87,6 +87,7 @@
     'feature_dip3_deterministicmns.py', # NOTE: needs dash_hash to pass
     'feature_llmq_data_recovery.py',
     'wallet_hd.py',
+    'wallet_hd.py --descriptors',
     'wallet_backup.py',
     # vv Tests less than 5m vv
     'mining_getblocktemplate_longpoll.py', # FIXME: "socket.error: [Errno 54] Connection reset by peer" on my Mac, same as https://github.com/bitcoin/bitcoin/issues/6651
@@ -99,7 +100,9 @@
     # vv Tests less than 2m vv
     'p2p_instantsend.py',
     'wallet_basic.py',
+    'wallet_basic.py --descriptors',
     'wallet_labels.py',
+    'wallet_labels.py --descriptors',
     'p2p_timeouts.py',
     'feature_bip68_sequence.py',
     'mempool_updatefromblock.py',
@@ -136,6 +139,7 @@
     # vv Tests less than 30s vv
     'rpc_quorum.py',
     'wallet_keypool_topup.py',
+    'wallet_keypool_topup.py --descriptors',
     'feature_fee_estimation.py',
     'interface_zmq_dash.py',
     'interface_zmq.py',
@@ -149,6 +153,7 @@
     'interface_rest.py',
     'mempool_spend_coinbase.py',
     'wallet_avoidreuse.py',
+    'wallet_avoidreuse.py --descriptors',
     'mempool_reorg.py',
     'mempool_persist.py',
     'wallet_multiwallet.py',
@@ -160,6 +165,7 @@
     'interface_http.py',
     'interface_rpc.py',
     'rpc_psbt.py',
+    #'rpc_psbt.py --descriptors', # TODO: enable rpc_psbt.py --descriptors that fails currently
     'rpc_users.py',
     'rpc_whitelist.py',
     'feature_proxy.py',
@@ -180,6 +186,7 @@
     'rpc_net.py',
     'wallet_keypool.py',
     'wallet_keypool_hd.py',
+    'wallet_keypool_hd.py --descriptors',
     'wallet_descriptor.py',
     'p2p_mempool.py',
     'p2p_filter.py',
@@ -202,6 +209,7 @@
     'mempool_packages.py',
     'mempool_package_onemore.py',
     'rpc_createmultisig.py',
+    'rpc_createmultisig.py --descriptors',
     'feature_versionbits_warning.py',
     'rpc_preciousblock.py',
     'wallet_importprunedfunds.py',
@@ -230,6 +238,7 @@
     'feature_sporks.py',
     'rpc_getblockstats.py',
     'wallet_encryption.py',
+    'wallet_encryption.py --descriptors',
     'wallet_upgradetohd.py',
     'feature_dersig.py',
     'feature_cltv.py',

test/functional/wallet_avoidreuse.py

@@ -85,6 +85,9 @@ def run_test(self):
         self.test_persistence()
         self.test_immutable()
 
+        # TODO: somehow pre-mined blocks doesn't appear in `listunspent` for descriptor wallets; listunspent() returns []
+        # need to mine forcely some extra blocks for nodes[1] to have coins there
+        self.nodes[1].generate(10)
         self.nodes[0].generate(110)
         self.sync_all()
         self.test_change_remains_change(self.nodes[1])
@@ -132,7 +135,7 @@ def test_immutable(self):
         tempwallet = ".wallet_avoidreuse.py_test_immutable_wallet.dat"
 
         # Create a wallet with disable_private_keys set; this should work
-        self.nodes[1].createwallet(tempwallet, True)
+        self.nodes[1].createwallet(wallet_name=tempwallet, disable_private_keys=True)
         w = self.nodes[1].get_wallet_rpc(tempwallet)
 
         # Attempt to unset the disable_private_keys flag; this should not work
@@ -248,31 +251,32 @@ def test_sending_from_reused_address_fails(self):
         # getbalances should show no used, 5 btc trusted
         assert_balances(self.nodes[1], mine={"used": 0, "trusted": 5})
 
-        self.nodes[0].sendtoaddress(fundaddr, 10)
-        self.nodes[0].generate(1)
-        self.sync_all()
+        if not self.options.descriptors:
+            self.nodes[0].sendtoaddress(fundaddr, 10)
+            self.nodes[0].generate(1)
+            self.sync_all()
 
-        # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10)
-        assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10)
-        # getbalances should show 10 used, 5 btc trusted
-        assert_balances(self.nodes[1], mine={"used": 10, "trusted": 5})
+            # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10)
+            assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10)
+            # getbalances should show 10 used, 5 btc trusted
+            assert_balances(self.nodes[1], mine={"used": 10, "trusted": 5})
 
-        # node 1 should now have a balance of 5 (no dirty) or 15 (including dirty)
-        assert_approx(self.nodes[1].getbalance(), 5, 0.001)
-        assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 15, 0.001)
+            # node 1 should now have a balance of 5 (no dirty) or 15 (including dirty)
+            assert_approx(self.nodes[1].getbalance(), 5, 0.001)
+            assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 15, 0.001)
 
-        assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[1].sendtoaddress, retaddr, 10)
+            assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[1].sendtoaddress, retaddr, 10)
 
-        self.nodes[1].sendtoaddress(retaddr, 4)
+            self.nodes[1].sendtoaddress(retaddr, 4)
 
-        # listunspent should show 2 total outputs (1, 10 btc), one unused (1), one reused (10)
-        assert_unspent(self.nodes[1], total_count=2, total_sum=11, reused_count=1, reused_sum=10)
-        # getbalances should show 10 used, 1 btc trusted
-        assert_balances(self.nodes[1], mine={"used": 10, "trusted": 1})
+            # listunspent should show 2 total outputs (1, 10 btc), one unused (1), one reused (10)
+            assert_unspent(self.nodes[1], total_count=2, total_sum=11, reused_count=1, reused_sum=10)
+            # getbalances should show 10 used, 1 btc trusted
+            assert_balances(self.nodes[1], mine={"used": 10, "trusted": 1})
 
-        # node 1 should now have about 1 btc left (no dirty) and 11 (including dirty)
-        assert_approx(self.nodes[1].getbalance(), 1, 0.001)
-        assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 11, 0.001)
+            # node 1 should now have about 1 btc left (no dirty) and 11 (including dirty)
+            assert_approx(self.nodes[1].getbalance(), 1, 0.001)
+            assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 11, 0.001)
 
     def test_getbalances_used(self):
         '''