From 5c049780c8b310428cf72fb304bf0c1071742785 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sat, 27 Jun 2020 13:13:51 +0300 Subject: [PATCH 001/457] test: Add test for erase orphan tx from peer --- test/functional/p2p_invalid_tx.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 8fef2d173f786..037a4249cc768 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -60,7 +60,6 @@ def run_test(self): block.solve() # Save the coinbase for later block1 = block - tip = block.sha256 node.p2ps[0].send_blocks_and_test([block], node, success=True) self.log.info("Mature the block.") @@ -157,6 +156,7 @@ def run_test(self): with node.assert_debug_log(['orphanage overflow, removed 1 tx']): node.p2ps[0].send_txs_and_test(orphan_tx_pool, node, success=False) + self.log.info('Test orphan with rejected parents') rejected_parent = CTransaction() rejected_parent.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_2_invalid.sha256, 0))) rejected_parent.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) @@ -164,6 +164,10 @@ def run_test(self): with node.assert_debug_log(['not keeping orphan with rejected parents {}'.format(rejected_parent.hash)]): node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False) + self.log.info('Test that a peer disconnection causes erase its transactions from the orphan pool') + with node.assert_debug_log(['Erased 100 orphan tx from peer=25']): + self.reconnect_p2p(num_connections=1) + if __name__ == '__main__': InvalidTxRequestTest().main() From fa45bb21193ae0c220cfc224d5e3ea0e7f3ec988 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sat, 27 Jun 2020 13:15:55 +0300 Subject: [PATCH 002/457] test: Add test for erase orphan tx included by block --- test/functional/p2p_invalid_tx.py | 33 +++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 037a4249cc768..0eb099b1d5ab2 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -92,24 +92,24 @@ def run_test(self): SCRIPT_PUB_KEY_OP_TRUE = b'\x51\x75' * 15 + b'\x51' tx_withhold = CTransaction() tx_withhold.vin.append(CTxIn(outpoint=COutPoint(block1.vtx[0].sha256, 0))) - tx_withhold.vout.append(CTxOut(nValue=50 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_withhold.vout = [CTxOut(nValue=25 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 2 tx_withhold.calc_sha256() # Our first orphan tx with some outputs to create further orphan txs tx_orphan_1 = CTransaction() tx_orphan_1.vin.append(CTxIn(outpoint=COutPoint(tx_withhold.sha256, 0))) - tx_orphan_1.vout = [CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 3 + tx_orphan_1.vout = [CTxOut(nValue=8 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 3 tx_orphan_1.calc_sha256() # A valid transaction with low fee tx_orphan_2_no_fee = CTransaction() tx_orphan_2_no_fee.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 0))) - tx_orphan_2_no_fee.vout.append(CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_2_no_fee.vout.append(CTxOut(nValue=8 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) # A valid transaction with sufficient fee tx_orphan_2_valid = CTransaction() tx_orphan_2_valid.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 1))) - tx_orphan_2_valid.vout.append(CTxOut(nValue=10 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_2_valid.vout.append(CTxOut(nValue=8 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) tx_orphan_2_valid.calc_sha256() # An invalid transaction with negative fee @@ -168,6 +168,31 @@ def run_test(self): with node.assert_debug_log(['Erased 100 orphan tx from peer=25']): self.reconnect_p2p(num_connections=1) + self.log.info('Test that a transaction in the orphan pool is included in a new tip block causes erase this transaction from the orphan pool') + tx_withhold_until_block_A = CTransaction() + tx_withhold_until_block_A.vin.append(CTxIn(outpoint=COutPoint(tx_withhold.sha256, 1))) + tx_withhold_until_block_A.vout = [CTxOut(nValue=12 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 2 + tx_withhold_until_block_A.calc_sha256() + + tx_orphan_include_by_block_A = CTransaction() + tx_orphan_include_by_block_A.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_A.sha256, 0))) + tx_orphan_include_by_block_A.vout.append(CTxOut(nValue=12 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_include_by_block_A.calc_sha256() + + self.log.info('Send the orphan ... ') + node.p2ps[0].send_txs_and_test([tx_orphan_include_by_block_A], node, success=False) + + tip = int(node.getbestblockhash(), 16) + height = node.getblockcount() + 1 + block_A = create_block(tip, create_coinbase(height)) + block_A.vtx.extend([tx_withhold, tx_withhold_until_block_A, tx_orphan_include_by_block_A]) + block_A.hashMerkleRoot = block_A.calc_merkle_root() + block_A.solve() + + self.log.info('Send the block that includes the previous orphan ... ') + with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): + node.p2ps[0].send_blocks_and_test([block_A], node, success=True) + if __name__ == '__main__': InvalidTxRequestTest().main() From c0a5fceee9858afd24fe0bf655b7b30728e96e78 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sat, 27 Jun 2020 13:21:12 +0300 Subject: [PATCH 003/457] test: Add test for erase orphan tx conflicted by block --- test/functional/p2p_invalid_tx.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 0eb099b1d5ab2..54d4f966d34c0 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -193,6 +193,35 @@ def run_test(self): with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): node.p2ps[0].send_blocks_and_test([block_A], node, success=True) + self.log.info('Test that a transaction in the orphan pool conflicts with a new tip block causes erase this transaction from the orphan pool') + tx_withhold_until_block_B = CTransaction() + tx_withhold_until_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_A.sha256, 1))) + tx_withhold_until_block_B.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_withhold_until_block_B.calc_sha256() + + tx_orphan_include_by_block_B = CTransaction() + tx_orphan_include_by_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_B.sha256, 0))) + tx_orphan_include_by_block_B.vout.append(CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_include_by_block_B.calc_sha256() + + tx_orphan_conflict_by_block_B = CTransaction() + tx_orphan_conflict_by_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_B.sha256, 0))) + tx_orphan_conflict_by_block_B.vout.append(CTxOut(nValue=9 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_conflict_by_block_B.calc_sha256() + self.log.info('Send the orphan ... ') + node.p2ps[0].send_txs_and_test([tx_orphan_conflict_by_block_B], node, success=False) + + tip = int(node.getbestblockhash(), 16) + height = node.getblockcount() + 1 + block_B = create_block(tip, create_coinbase(height)) + block_B.vtx.extend([tx_withhold_until_block_B, tx_orphan_include_by_block_B]) + block_B.hashMerkleRoot = block_B.calc_merkle_root() + block_B.solve() + + self.log.info('Send the block that includes a transaction which conflicts with the previous orphan ... ') + with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): + node.p2ps[0].send_blocks_and_test([block_B], node, success=True) + if __name__ == '__main__': InvalidTxRequestTest().main() From 817326a828d6148dc63d9ef08f641b9c0c522411 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Fri, 29 Mar 2019 17:10:11 -0400 Subject: [PATCH 004/457] wallet: avoid rescans if under the snapshot Refuse to load a wallet if it requires a rescan lower than the height of an unvalidated snapshot we're running -- in more general terms, if we don't have data for the blocks. --- src/interfaces/chain.h | 3 +++ src/node/interfaces.cpp | 5 +++++ src/wallet/wallet.cpp | 17 ++++++++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 4f5105a5c123e..07acf880c6428 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -286,6 +286,9 @@ class Chain //! to be prepared to handle this by ignoring notifications about unknown //! removed transactions and already added new transactions. virtual void requestMempoolTransactions(Notifications& notifications) = 0; + + //! Return true if an assumed-valid chain is in use. + virtual bool hasAssumedValidChain() = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 1a48957f0fdfd..855db7b0ec2b3 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -723,6 +723,11 @@ class ChainImpl : public Chain notifications.transactionAddedToMempool(entry.GetSharedTx(), 0 /* mempool_sequence */); } } + bool hasAssumedValidChain() override + { + return Assert(m_node.chainman)->IsSnapshotActive(); + } + NodeContext& m_node; }; } // namespace diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 3fcb086d2de46..6726ea8e4d4f7 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2903,20 +2903,31 @@ bool CWallet::AttachChain(const std::shared_ptr& walletInstance, interf if (tip_height && *tip_height != rescan_height) { - if (chain.havePruned()) { + // Technically we could execute the code below in any case, but performing the + // `while` loop below can make startup very slow, so only check blocks on disk + // if necessary. + if (chain.havePruned() || chain.hasAssumedValidChain()) { int block_height = *tip_height; while (block_height > 0 && chain.haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - // We can't rescan beyond non-pruned blocks, stop and throw an error. + // We can't rescan beyond blocks we don't have data for, stop and throw an error. // This might happen if a user uses an old wallet within a pruned node // or if they ran -disablewallet for a longer time, then decided to re-enable // Exit early and print an error. + // It also may happen if an assumed-valid chain is in use and therefore not + // all block data is available. // If a block is pruned after this check, we will load the wallet, // but fail the rescan with a generic error. - error = _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); + + error = chain.hasAssumedValidChain() ? + _( + "Assumed-valid: last wallet synchronisation goes beyond " + "available block data. You need to wait for the background " + "validation chain to download more blocks.") : + _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); return false; } } From fa8671018766b2f0e18c94cff3ab2a67c6b3a41d Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 14 Mar 2022 16:52:55 +0100 Subject: [PATCH 005/457] Clarify that CheckSequenceLocksAtTip is a validation function --- src/validation.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validation.h b/src/validation.h index 965ed4225eb3b..2199de3197568 100644 --- a/src/validation.h +++ b/src/validation.h @@ -273,7 +273,7 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx const Package& txns, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/* Transaction policy functions */ +/* Mempool validation helper functions */ /** * Check if transaction will be final in the next block to be created. From f59959e3818692c5b3c2dfa51c14e515085e940f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Barbosa?= Date: Fri, 25 Mar 2022 22:41:21 +0000 Subject: [PATCH 006/457] wallet: Prevent wallet unload on GetWalletForJSONRPCRequest Don't extend shared ownership of all wallets to GetWalletForJSONRPCRequest scope. --- src/wallet/rpc/util.cpp | 9 ++++----- src/wallet/wallet.cpp | 7 +++++++ src/wallet/wallet.h | 1 + 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 59683c5fd8c4e..f788d0d2d33f2 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -64,12 +64,11 @@ std::shared_ptr GetWalletForJSONRPCRequest(const JSONRPCRequest& reques return pwallet; } - std::vector> wallets = GetWallets(context); - if (wallets.size() == 1) { - return wallets[0]; - } + size_t count{0}; + auto wallet = GetDefaultWallet(context, count); + if (wallet) return wallet; - if (wallets.empty()) { + if (count == 0) { throw JSONRPCError( RPC_WALLET_NOT_FOUND, "No wallet is loaded. Load a wallet using loadwallet or create a new one with createwallet. (Note: A default wallet is no longer automatically created)"); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index be64b4cdbbc06..96b9a69d45565 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -151,6 +151,13 @@ std::vector> GetWallets(WalletContext& context) return context.wallets; } +std::shared_ptr GetDefaultWallet(WalletContext& context, size_t& count) +{ + LOCK(context.wallets_mutex); + count = context.wallets.size(); + return count == 1 ? context.wallets[0] : nullptr; +} + std::shared_ptr GetWallet(WalletContext& context, const std::string& name) { LOCK(context.wallets_mutex); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 0490d321abe64..686e10229407a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -62,6 +62,7 @@ bool AddWallet(WalletContext& context, const std::shared_ptr& wallet); bool RemoveWallet(WalletContext& context, const std::shared_ptr& wallet, std::optional load_on_start, std::vector& warnings); bool RemoveWallet(WalletContext& context, const std::shared_ptr& wallet, std::optional load_on_start); std::vector> GetWallets(WalletContext& context); +std::shared_ptr GetDefaultWallet(WalletContext& context, size_t& count); std::shared_ptr GetWallet(WalletContext& context, const std::string& name); std::shared_ptr LoadWallet(WalletContext& context, const std::string& name, std::optional load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector& warnings); std::shared_ptr CreateWallet(WalletContext& context, const std::string& name, std::optional load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector& warnings); From 395767e9f15b7a1b5203da68f1fbe3df281ae906 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 6 Apr 2022 18:11:27 -0400 Subject: [PATCH 007/457] Add test case mimicking issue 24765 --- test/functional/feature_taproot.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index c3925dbb00865..daa7ac1221ce8 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -1143,6 +1143,12 @@ def predict_sigops_ratio(n, dummy_size): tap = taproot_construct(pubs[0], scripts) add_spender(spenders, "alwaysvalid/notsuccessx", tap=tap, leaf="op_success", inputs=[], standard=False, failure={"leaf": "normal"}) # err_msg differs based on opcode + # == Test case for https://github.com/bitcoin/bitcoin/issues/24765 == + + zero_fn = lambda h: bytes([0 for _ in range(32)]) + tap = taproot_construct(pubs[0], [("leaf", CScript([pubs[1], OP_CHECKSIG, pubs[1], OP_CHECKSIGADD, OP_2, OP_EQUAL])), zero_fn]) + add_spender(spenders, "case24765", tap=tap, leaf="leaf", inputs=[getter("sign"), getter("sign")], key=secs[1], no_fail=True) + # == Legacy tests == # Also add a few legacy spends into the mix, so that transactions which combine taproot and pre-taproot spends get tested too. From 1276090705060fcc97072481c2383bbaaa556194 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 25 Mar 2022 21:36:22 +0100 Subject: [PATCH 008/457] util, refactor: Use GetPathArg to read "-conf" value Also "includeconf" values been normalized. --- src/bitcoin-cli.cpp | 2 +- src/init/common.cpp | 2 +- src/qt/guiutil.cpp | 2 +- src/util/system.cpp | 14 +++++++------- src/util/system.h | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index dea46693bc4d7..8f7b836a7c2f8 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -801,7 +801,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co if (failedToGetAuthCookie) { throw std::runtime_error(strprintf( "Could not locate RPC credentials. No authentication cookie could be found, and RPC password is not set. See -rpcpassword and -stdinrpcpass. Configuration file: (%s)", - fs::PathToString(GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME))))); + fs::PathToString(GetConfigFile(gArgs.GetPathArg("-conf", BITCOIN_CONF_FILENAME))))); } else { throw std::runtime_error("Authorization failed: Incorrect rpcuser or rpcpassword"); } diff --git a/src/init/common.cpp b/src/init/common.cpp index 688471b35da75..3d46857f3cad3 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -137,7 +137,7 @@ bool StartLogging(const ArgsManager& args) LogPrintf("Using data directory %s\n", fs::PathToString(gArgs.GetDataDirNet())); // Only log conf file usage message if conf file actually exists. - fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); + fs::path config_file_path = GetConfigFile(args.GetPathArg("-conf", BITCOIN_CONF_FILENAME)); if (fs::exists(config_file_path)) { LogPrintf("Config file: %s\n", fs::PathToString(config_file_path)); } else if (args.IsArgSet("-conf")) { diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 6fb5fce5b309a..e70ca3053d4f3 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -431,7 +431,7 @@ void openDebugLogfile() bool openBitcoinConf() { - fs::path pathConfig = GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)); + fs::path pathConfig = GetConfigFile(gArgs.GetPathArg("-conf", BITCOIN_CONF_FILENAME)); /* Create the file */ std::ofstream configFile{pathConfig, std::ios_base::app}; diff --git a/src/util/system.cpp b/src/util/system.cpp index a7e66defcdf6d..5acc9a8e5894f 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -826,9 +826,9 @@ bool CheckDataDirOption() return datadir.empty() || fs::is_directory(fs::absolute(datadir)); } -fs::path GetConfigFile(const std::string& confPath) +fs::path GetConfigFile(const fs::path& configuration_file_path) { - return AbsPathForConfigVal(fs::PathFromString(confPath), false); + return AbsPathForConfigVal(configuration_file_path, /*net_specific=*/false); } static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector>& options, std::list& sections) @@ -912,17 +912,17 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) m_config_sections.clear(); } - const std::string confPath = GetArg("-conf", BITCOIN_CONF_FILENAME); - std::ifstream stream{GetConfigFile(confPath)}; + const fs::path conf_path = GetPathArg("-conf", BITCOIN_CONF_FILENAME); + std::ifstream stream{GetConfigFile(conf_path)}; // not ok to have a config file specified that cannot be opened if (IsArgSet("-conf") && !stream.good()) { - error = strprintf("specified config file \"%s\" could not be opened.", confPath); + error = strprintf("specified config file \"%s\" could not be opened.", fs::PathToString(conf_path)); return false; } // ok to not have a config file if (stream.good()) { - if (!ReadConfigStream(stream, confPath, error, ignore_invalid_keys)) { + if (!ReadConfigStream(stream, fs::PathToString(conf_path), error, ignore_invalid_keys)) { return false; } // `-includeconf` cannot be included in the command line arguments except @@ -960,7 +960,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) const size_t default_includes = add_includes({}); for (const std::string& conf_file_name : conf_file_names) { - std::ifstream conf_file_stream{GetConfigFile(conf_file_name)}; + std::ifstream conf_file_stream{GetConfigFile(fs::PathFromString(conf_file_name))}; if (conf_file_stream.good()) { if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; diff --git a/src/util/system.h b/src/util/system.h index a66b597d41231..796205fbfc2c3 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -98,7 +98,7 @@ bool TryCreateDirectories(const fs::path& p); fs::path GetDefaultDataDir(); // Return true if -datadir option points to a valid directory or is not specified. bool CheckDataDirOption(); -fs::path GetConfigFile(const std::string& confPath); +fs::path GetConfigFile(const fs::path& configuration_file_path); #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif From 138c668e2b4d64279ddefbe07c1d9b7c3d3c537c Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 25 Mar 2022 21:37:28 +0100 Subject: [PATCH 009/457] util, refactor: Use GetPathArg to read "-rpccookiefile" value --- src/rpc/request.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index 95a7c25b9393e..c2f1b15d9cd01 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -66,16 +66,16 @@ UniValue JSONRPCError(int code, const std::string& message) */ static const std::string COOKIEAUTH_USER = "__cookie__"; /** Default name for auth cookie file */ -static const std::string COOKIEAUTH_FILE = ".cookie"; +static const char* const COOKIEAUTH_FILE = ".cookie"; /** Get name of RPC authentication cookie file */ static fs::path GetAuthCookieFile(bool temp=false) { - std::string arg = gArgs.GetArg("-rpccookiefile", COOKIEAUTH_FILE); + fs::path arg = gArgs.GetPathArg("-rpccookiefile", COOKIEAUTH_FILE); if (temp) { arg += ".tmp"; } - return AbsPathForConfigVal(fs::PathFromString(arg)); + return AbsPathForConfigVal(arg); } bool GenerateAuthCookie(std::string *cookie_out) From b01f336708019f8c8274ea701d3446e4123e7af2 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 25 Mar 2022 21:42:58 +0100 Subject: [PATCH 010/457] util, refactor: Drop explicit conversion to fs::path Removes unhelpful noise/verbosity. See: https://github.com/bitcoin/bitcoin/pull/24306#discussion_r809363741 --- src/util/system.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/system.cpp b/src/util/system.cpp index 5acc9a8e5894f..bfb1a79b400d7 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -528,7 +528,7 @@ bool ArgsManager::InitSettings(std::string& error) bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const { - fs::path settings = GetPathArg("-settings", fs::path{BITCOIN_SETTINGS_FILENAME}); + fs::path settings = GetPathArg("-settings", BITCOIN_SETTINGS_FILENAME); if (settings.empty()) { return false; } From 734b9669ff7b2f5e2820993443a6f868f6b0b20a Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Fri, 22 Apr 2022 11:09:45 +0200 Subject: [PATCH 011/457] test: add getblockfrompeer coverage of invalid inputs --- test/functional/rpc_getblockfrompeer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index b65322d920b06..cc9ccf306628d 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -49,14 +49,17 @@ def run_test(self): assert_equal(len(peers), 1) peer_0_peer_1_id = peers[0]["id"] - self.log.info("Arguments must be sensible") - assert_raises_rpc_error(-8, "hash must be of length 64 (not 4, for '1234')", self.nodes[0].getblockfrompeer, "1234", 0) + self.log.info("Arguments must be valid") + assert_raises_rpc_error(-8, "hash must be of length 64 (not 4, for '1234')", self.nodes[0].getblockfrompeer, "1234", peer_0_peer_1_id) + assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].getblockfrompeer, 1234, peer_0_peer_1_id) + assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].getblockfrompeer, short_tip, "0") self.log.info("We must already have the header") assert_raises_rpc_error(-1, "Block header missing", self.nodes[0].getblockfrompeer, "00" * 32, 0) self.log.info("Non-existent peer generates error") - assert_raises_rpc_error(-1, "Peer does not exist", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id + 1) + for peer_id in [-1, peer_0_peer_1_id + 1]: + assert_raises_rpc_error(-1, "Peer does not exist", self.nodes[0].getblockfrompeer, short_tip, peer_id) self.log.info("Successful fetch") result = self.nodes[0].getblockfrompeer(short_tip, peer_0_peer_1_id) From 2c3ee4c347838ecadb17a011932dffc077e46630 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Fri, 12 Nov 2021 16:09:54 -0500 Subject: [PATCH 012/457] gui: Load Base64 PSBT string from file Some .psbt files may have the PSBT as a base64 string instead of in binary. We should be able to load those files. --- src/qt/walletframe.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index dc4e25a02b230..abeb6a827baca 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -215,6 +215,14 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) } std::ifstream in{filename.toLocal8Bit().data(), std::ios::binary}; data.assign(std::istream_iterator{in}, {}); + + // Some psbt files may be base64 strings in the file rather than binary data + std::string b64_str{data.begin(), data.end()}; + b64_str.erase(b64_str.find_last_not_of(" \t\n\r\f\v") + 1); // Trim trailing whitespace + auto b64_dec = DecodeBase64(b64_str); + if (b64_dec.has_value()) { + data = b64_dec.value(); + } } std::string error; From b0a53d50d9142bed51a8372eeb848816bfa94da8 Mon Sep 17 00:00:00 2001 From: Patrick Strateman Date: Sun, 14 Jun 2020 20:14:34 -0400 Subject: [PATCH 013/457] Make sanity check in GCSFilter constructor optional BlockFilterIndex will perform the cheaper check of verifying the filter hash when reading the filter from disk. --- src/blockfilter.cpp | 8 +++++--- src/blockfilter.h | 6 +++--- src/index/blockfilterindex.cpp | 13 +++++++++---- src/index/blockfilterindex.h | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 63a9ba498f522..1ad687214358a 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -47,7 +47,7 @@ GCSFilter::GCSFilter(const Params& params) : m_params(params), m_N(0), m_F(0), m_encoded{0} {} -GCSFilter::GCSFilter(const Params& params, std::vector encoded_filter) +GCSFilter::GCSFilter(const Params& params, std::vector encoded_filter, bool skip_decode_check) : m_params(params), m_encoded(std::move(encoded_filter)) { SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded}; @@ -59,6 +59,8 @@ GCSFilter::GCSFilter(const Params& params, std::vector encoded_fi } m_F = static_cast(m_N) * static_cast(m_params.m_M); + if (skip_decode_check) return; + // Verify that the encoded filter contains exactly N elements. If it has too much or too little // data, a std::ios_base::failure exception will be raised. BitStreamReader bitreader{stream}; @@ -219,14 +221,14 @@ static GCSFilter::ElementSet BasicFilterElements(const CBlock& block, } BlockFilter::BlockFilter(BlockFilterType filter_type, const uint256& block_hash, - std::vector filter) + std::vector filter, bool skip_decode_check) : m_filter_type(filter_type), m_block_hash(block_hash) { GCSFilter::Params params; if (!BuildParams(params)) { throw std::invalid_argument("unknown filter_type"); } - m_filter = GCSFilter(params, std::move(filter)); + m_filter = GCSFilter(params, std::move(filter), skip_decode_check); } BlockFilter::BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo) diff --git a/src/blockfilter.h b/src/blockfilter.h index 96cefbf3b2f9f..d6a51e95c24e2 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -59,7 +59,7 @@ class GCSFilter explicit GCSFilter(const Params& params = Params()); /** Reconstructs an already-created filter from an encoding. */ - GCSFilter(const Params& params, std::vector encoded_filter); + GCSFilter(const Params& params, std::vector encoded_filter, bool skip_decode_check); /** Builds a new filter from the params and set of elements. */ GCSFilter(const Params& params, const ElementSet& elements); @@ -122,7 +122,7 @@ class BlockFilter //! Reconstruct a BlockFilter from parts. BlockFilter(BlockFilterType filter_type, const uint256& block_hash, - std::vector filter); + std::vector filter, bool skip_decode_check); //! Construct a new BlockFilter of the specified type from a block. BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo); @@ -164,7 +164,7 @@ class BlockFilter if (!BuildParams(params)) { throw std::ios_base::failure("unknown filter_type"); } - m_filter = GCSFilter(params, std::move(encoded_filter)); + m_filter = GCSFilter(params, std::move(encoded_filter), /*skip_decode_check=*/false); } }; diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 4f99eddfd77af..a8e1860481249 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -143,18 +144,22 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch) return BaseIndex::CommitInternal(batch); } -bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const +bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const { CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { return false; } + // Check that the hash of the encoded_filter matches the one stored in the db. uint256 block_hash; std::vector encoded_filter; try { filein >> block_hash >> encoded_filter; - filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter)); + uint256 result; + CHash256().Write(encoded_filter).Finalize(result); + if (result != hash) return error("Checksum mismatch in filter decode."); + filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter), /*skip_decode_check=*/true); } catch (const std::exception& e) { return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what()); @@ -381,7 +386,7 @@ bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& return false; } - return ReadFilterFromDisk(entry.pos, filter_out); + return ReadFilterFromDisk(entry.pos, entry.hash, filter_out); } bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) @@ -425,7 +430,7 @@ bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* st filters_out.resize(entries.size()); auto filter_pos_it = filters_out.begin(); for (const auto& entry : entries) { - if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) { + if (!ReadFilterFromDisk(entry.pos, entry.hash, *filter_pos_it)) { return false; } ++filter_pos_it; diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index a049019c020fe..09c3c78c6310c 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -31,7 +31,7 @@ class BlockFilterIndex final : public BaseIndex FlatFilePos m_next_filter_pos; std::unique_ptr m_filter_fileseq; - bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const; + bool ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const; size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter); Mutex m_cs_headers_cache; From 299023c1d9962628d158fac0306f8531506a0123 Mon Sep 17 00:00:00 2001 From: Patrick Strateman Date: Mon, 15 Jun 2020 12:30:22 -0400 Subject: [PATCH 014/457] Add GCSFilterDecode and GCSBlockFilterGetHash benchmarks. All of the benchmarks are standardized on the BASIC filter parameters so we can compare between all the benchmarks. All the GCS benchmarks are renamed to have "GCS" as the prefix. --- src/bench/gcs_filter.cpp | 57 +++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index 607e4392b7a1a..c96e9b7f31162 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -5,7 +5,7 @@ #include #include -static void ConstructGCSFilter(benchmark::Bench& bench) +static const GCSFilter::ElementSet GenerateGCSTestElements() { GCSFilter::ElementSet elements; for (int i = 0; i < 10000; ++i) { @@ -15,29 +15,56 @@ static void ConstructGCSFilter(benchmark::Bench& bench) elements.insert(std::move(element)); } + return elements; +} + +static void GCSBlockFilterGetHash(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + BlockFilter block_filter(BlockFilterType::BASIC, {}, filter.GetEncoded(), /*skip_decode_check=*/false); + + bench.run([&] { + block_filter.GetHash(); + }); +} + +static void GCSFilterConstruct(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + uint64_t siphash_k0 = 0; bench.batch(elements.size()).unit("elem").run([&] { - GCSFilter filter({siphash_k0, 0, 20, 1 << 20}, elements); + GCSFilter filter({siphash_k0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); siphash_k0++; }); } -static void MatchGCSFilter(benchmark::Bench& bench) +static void GCSFilterDecode(benchmark::Bench& bench) { - GCSFilter::ElementSet elements; - for (int i = 0; i < 10000; ++i) { - GCSFilter::Element element(32); - element[0] = static_cast(i); - element[1] = static_cast(i >> 8); - elements.insert(std::move(element)); - } - GCSFilter filter({0, 0, 20, 1 << 20}, elements); + auto elements = GenerateGCSTestElements(); - bench.unit("elem").run([&] { - filter.Match(GCSFilter::Element()); + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + auto encoded = filter.GetEncoded(); + + bench.run([&] { + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/false); }); } -BENCHMARK(ConstructGCSFilter); -BENCHMARK(MatchGCSFilter); +static void GCSFilterMatch(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + + bench.run([&] { + filter.Match(GCSFilter::Element()); + }); +} +BENCHMARK(GCSBlockFilterGetHash); +BENCHMARK(GCSFilterConstruct); +BENCHMARK(GCSFilterDecode); +BENCHMARK(GCSFilterMatch); From aee9a8140b3a58b744766f9e89572f1d953a808b Mon Sep 17 00:00:00 2001 From: Patrick Strateman Date: Mon, 15 Jun 2020 19:41:18 -0400 Subject: [PATCH 015/457] Add GCSFilterDecodeSkipCheck benchmark This benchmark allows us to compare the differences between doing the sanity check for corruption via GolombRiceDecode() vs checking the hash of the encoded block filter. --- src/bench/gcs_filter.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index c96e9b7f31162..0b43652453035 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -54,6 +54,18 @@ static void GCSFilterDecode(benchmark::Bench& bench) }); } +static void GCSFilterDecodeSkipCheck(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + auto encoded = filter.GetEncoded(); + + bench.run([&] { + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/true); + }); +} + static void GCSFilterMatch(benchmark::Bench& bench) { auto elements = GenerateGCSTestElements(); @@ -67,4 +79,5 @@ static void GCSFilterMatch(benchmark::Bench& bench) BENCHMARK(GCSBlockFilterGetHash); BENCHMARK(GCSFilterConstruct); BENCHMARK(GCSFilterDecode); +BENCHMARK(GCSFilterDecodeSkipCheck); BENCHMARK(GCSFilterMatch); From e734228d8585c0870c71ce8ba8c037f8cf8b249a Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Sun, 22 May 2022 14:17:15 +0900 Subject: [PATCH 016/457] Update GCSFilter benchmarks Element count used in the GCSFilter benchmarks are increased to 100,000 from 10,000. Testing the benchmarks with different element counts showed that a filter with 100,000 elements resulted in the same ns/op. This this a desirable thing to have as it allows us to reason about how long a single filter element takes to process, letting us easily calculate how long a filter with N elements (where N > 100,000) would take to process. GCSFilterConstruct benchmark is now called without batch. This makes intra-bench results more intuitive as all benchmarks are in ns/op instead of a custom unit. There are no downsides to this change as testing showed that there is no observable difference in error rates in the benchmarks when calling without batch. --- src/bench/gcs_filter.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index 0b43652453035..80babb213b3e0 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -8,7 +8,12 @@ static const GCSFilter::ElementSet GenerateGCSTestElements() { GCSFilter::ElementSet elements; - for (int i = 0; i < 10000; ++i) { + + // Testing the benchmarks with different number of elements show that a filter + // with at least 100,000 elements results in benchmarks that have the same + // ns/op. This makes it easy to reason about how long (in nanoseconds) a single + // filter element takes to process. + for (int i = 0; i < 100000; ++i) { GCSFilter::Element element(32); element[0] = static_cast(i); element[1] = static_cast(i >> 8); @@ -35,7 +40,7 @@ static void GCSFilterConstruct(benchmark::Bench& bench) auto elements = GenerateGCSTestElements(); uint64_t siphash_k0 = 0; - bench.batch(elements.size()).unit("elem").run([&] { + bench.run([&]{ GCSFilter filter({siphash_k0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); siphash_k0++; From 292b1a3e9c98b9ba74b28d149df8554d4ad8e5c0 Mon Sep 17 00:00:00 2001 From: amadeuszpawlik Date: Sat, 28 May 2022 20:40:51 +0200 Subject: [PATCH 017/457] GetExternalSigner(): fail if multiple signers are found If there are multiple external signers, `GetExternalSigner()` will just pick the first one in the list. If the user has two or more hardware wallets connected at the same time, he might not notice this. This PR adds a check and fails with suitable message. --- src/qt/walletcontroller.cpp | 4 +++ .../external_signer_scriptpubkeyman.cpp | 3 +- test/functional/mocks/invalid_signer.py | 2 +- test/functional/mocks/multi_signers.py | 30 +++++++++++++++++++ test/functional/mocks/signer.py | 2 +- test/functional/rpc_signer.py | 5 +--- test/functional/wallet_signer.py | 15 ++++++++++ 7 files changed, 54 insertions(+), 7 deletions(-) create mode 100755 test/functional/mocks/multi_signers.py diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index d27ddf1aba52b..fae4c7cdf1311 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -293,6 +293,10 @@ void CreateWalletActivity::create() } catch (const std::runtime_error& e) { QMessageBox::critical(nullptr, tr("Can't list signers"), e.what()); } + if (signers.size() > 1) { + QMessageBox::critical(nullptr, tr("Too many external signers found"), QString::fromStdString("More than one external signer found. Please connect only one at a time.")); + signers.clear(); + } m_create_wallet_dialog->setSigners(signers); m_create_wallet_dialog->setWindowModality(Qt::ApplicationModal); diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index 9d5f58b784c6a..76de51ac3e0d2 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -45,7 +45,8 @@ ExternalSigner ExternalSignerScriptPubKeyMan::GetExternalSigner() { std::vector signers; ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); if (signers.empty()) throw std::runtime_error(std::string(__func__) + ": No external signers found"); - // TODO: add fingerprint argument in case of multiple signers + // TODO: add fingerprint argument instead of failing in case of multiple signers. + if (signers.size() > 1) throw std::runtime_error(std::string(__func__) + ": More than one external signer found. Please connect only one at a time."); return signers[0]; } diff --git a/test/functional/mocks/invalid_signer.py b/test/functional/mocks/invalid_signer.py index e30cc9e20b053..14f9fed72e888 100755 --- a/test/functional/mocks/invalid_signer.py +++ b/test/functional/mocks/invalid_signer.py @@ -18,7 +18,7 @@ def perform_pre_checks(): sys.exit(int(mock_result[0])) def enumerate(args): - sys.stdout.write(json.dumps([{"fingerprint": "b3c19bfc", "type": "trezor", "model": "trezor_t"}, {"fingerprint": "00000002"}])) + sys.stdout.write(json.dumps([{"fingerprint": "b3c19bfc", "type": "trezor", "model": "trezor_t"}])) def getdescriptors(args): xpub_pkh = "xpub6CRhJvXV8x2AKWvqi1ZSMFU6cbxzQiYrv3dxSUXCawjMJ1JzpqVsveH4way1yCmJm29KzH1zrVZmVwes4Qo6oXVE1HFn4fdiKrYJngqFFc6" diff --git a/test/functional/mocks/multi_signers.py b/test/functional/mocks/multi_signers.py new file mode 100755 index 0000000000000..88f93e23dea0a --- /dev/null +++ b/test/functional/mocks/multi_signers.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import argparse +import json +import sys + +def enumerate(args): + sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}, + {"fingerprint": "00000002", "type": "trezor", "model": "trezor_one"}])) + +parser = argparse.ArgumentParser(prog='./multi_signers.py', description='External multi-signer mock') + +subparsers = parser.add_subparsers(description='Commands', dest='command') +subparsers.required = True + +parser_enumerate = subparsers.add_parser('enumerate', help='list available signers') +parser_enumerate.set_defaults(func=enumerate) + + +if not sys.stdin.isatty(): + buffer = sys.stdin.read() + if buffer and buffer.rstrip() != "": + sys.argv.extend(buffer.rstrip().split(" ")) + +args = parser.parse_args() + +args.func(args) diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py index b732b26a53d07..c5a8f7b1e9234 100755 --- a/test/functional/mocks/signer.py +++ b/test/functional/mocks/signer.py @@ -18,7 +18,7 @@ def perform_pre_checks(): sys.exit(int(mock_result[0])) def enumerate(args): - sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}, {"fingerprint": "00000002"}])) + sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}])) def getdescriptors(args): xpub = "tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B" diff --git a/test/functional/rpc_signer.py b/test/functional/rpc_signer.py index f1107197c53a4..de17b2b929228 100755 --- a/test/functional/rpc_signer.py +++ b/test/functional/rpc_signer.py @@ -77,10 +77,7 @@ def run_test(self): ) self.clear_mock_result(self.nodes[1]) - result = self.nodes[1].enumeratesigners() - assert_equal(len(result['signers']), 2) - assert_equal(result['signers'][0]["fingerprint"], "00000001") - assert_equal(result['signers'][0]["name"], "trezor_t") + assert_equal({'fingerprint': '00000001', 'name': 'trezor_t'} in self.nodes[1].enumeratesigners()['signers'], True) if __name__ == '__main__': RPCSignerTest().main() diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 8e4e1f5d36b67..5609ac9bf5845 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -32,6 +32,13 @@ def mock_invalid_signer_path(self): else: return path + def mock_multi_signers_path(self): + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mocks', 'multi_signers.py') + if platform.system() == "Windows": + return "py " + path + else: + return path + def set_test_params(self): self.num_nodes = 2 # The experimental syscall sandbox feature (-sandbox) is not compatible with -signer (which @@ -58,6 +65,8 @@ def run_test(self): self.test_valid_signer() self.restart_node(1, [f"-signer={self.mock_invalid_signer_path()}", "-keypool=10"]) self.test_invalid_signer() + self.restart_node(1, [f"-signer={self.mock_multi_signers_path()}", "-keypool=10"]) + self.test_multiple_signers() def test_valid_signer(self): self.log.debug(f"-signer={self.mock_signer_path()}") @@ -212,5 +221,11 @@ def test_invalid_signer(self): self.log.info('Test invalid external signer') assert_raises_rpc_error(-1, "Invalid descriptor", self.nodes[1].createwallet, wallet_name='hww_invalid', disable_private_keys=True, descriptors=True, external_signer=True) + def test_multiple_signers(self): + self.log.debug(f"-signer={self.mock_multi_signers_path()}") + self.log.info('Test multiple external signers') + + assert_raises_rpc_error(-1, "GetExternalSigner: More than one external signer found", self.nodes[1].createwallet, wallet_name='multi_hww', disable_private_keys=True, descriptors=True, external_signer=True) + if __name__ == '__main__': WalletSignerTest().main() From 192eb1e61c3c43baec7f32c498ab0ce0656a58f7 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 10:45:08 -0300 Subject: [PATCH 018/457] refactor: getAddress don't access m_address_book, use FindAddressEntry function --- src/wallet/interfaces.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index e1203817e0de9..5cfcf16e16424 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -191,18 +191,16 @@ class WalletImpl : public Wallet std::string* purpose) override { LOCK(m_wallet->cs_wallet); - auto it = m_wallet->m_address_book.find(dest); - if (it == m_wallet->m_address_book.end() || it->second.IsChange()) { - return false; - } + const auto& entry = m_wallet->FindAddressBookEntry(dest, /*allow_change=*/false); + if (!entry) return false; // addr not found if (name) { - *name = it->second.GetLabel(); + *name = entry->GetLabel(); } if (is_mine) { *is_mine = m_wallet->IsMine(dest); } if (purpose) { - *purpose = it->second.purpose; + *purpose = entry->purpose; } return true; } From bda8ebe608e6572eaaf40cd28dab6954241c9b0d Mon Sep 17 00:00:00 2001 From: furszy Date: Thu, 16 Jun 2022 10:30:04 -0300 Subject: [PATCH 019/457] wallet: don't read db every time that a new WalletBatch is created Better to perform the action only one time (during 'LoadWallet'). Where the value is being used. --- src/wallet/bdb.cpp | 6 ------ src/wallet/walletdb.cpp | 8 +++----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index f8230f7a1d6ce..dbd768a758252 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -315,12 +315,6 @@ BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const bool read_only, b env = database.env.get(); pdb = database.m_db.get(); strFile = fs::PathToString(database.m_filename); - if (!Exists(std::string("version"))) { - bool fTmp = fReadOnly; - fReadOnly = false; - Write(std::string("version"), CLIENT_VERSION); - fReadOnly = fTmp; - } } void BerkeleyDatabase::Open() diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 79e0a330b7975..2cedd62930ac9 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -885,10 +885,8 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Last client version to open this wallet, was previously the file version number int last_client = CLIENT_VERSION; - m_batch->Read(DBKeys::VERSION, last_client); - - int wallet_version = pwallet->GetVersion(); - pwallet->WalletLogPrintf("Wallet File Version = %d\n", wallet_version > 0 ? wallet_version : last_client); + bool has_last_client = m_batch->Read(DBKeys::VERSION, last_client); + pwallet->WalletLogPrintf("Wallet file version = %d, last client version = %d\n", pwallet->GetVersion(), last_client); pwallet->WalletLogPrintf("Keys: %u plaintext, %u encrypted, %u w/ metadata, %u total. Unknown wallet records: %u\n", wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); @@ -909,7 +907,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (wss.fIsEncrypted && (last_client == 40000 || last_client == 50000)) return DBErrors::NEED_REWRITE; - if (last_client < CLIENT_VERSION) // Update + if (!has_last_client || last_client < CLIENT_VERSION) // Update m_batch->Write(DBKeys::VERSION, CLIENT_VERSION); if (wss.fAnyUnordered) From c318211ddd48d44dd81dded553afeee3bc41c89e Mon Sep 17 00:00:00 2001 From: furszy Date: Thu, 16 Jun 2022 15:30:37 -0300 Subject: [PATCH 020/457] walletdb: fix last client version update The value was only being updated launching releases with higher version numbers and not if the user launched a previous release. Co-authored-by: MacroFake --- src/wallet/walletdb.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 2cedd62930ac9..dedbc4ec49a5f 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -883,7 +883,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (result != DBErrors::LOAD_OK) return result; - // Last client version to open this wallet, was previously the file version number + // Last client version to open this wallet int last_client = CLIENT_VERSION; bool has_last_client = m_batch->Read(DBKeys::VERSION, last_client); pwallet->WalletLogPrintf("Wallet file version = %d, last client version = %d\n", pwallet->GetVersion(), last_client); @@ -907,7 +907,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (wss.fIsEncrypted && (last_client == 40000 || last_client == 50000)) return DBErrors::NEED_REWRITE; - if (!has_last_client || last_client < CLIENT_VERSION) // Update + if (!has_last_client || last_client != CLIENT_VERSION) // Update m_batch->Write(DBKeys::VERSION, CLIENT_VERSION); if (wss.fAnyUnordered) From 09649bc95d5f2855a54a8cf02e65215a3b333c92 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:06:22 -0300 Subject: [PATCH 021/457] refactor: implement general 'ListAddrBookAddresses' for addressbook destinations lookup --- src/wallet/rpc/coins.cpp | 6 +++--- src/wallet/wallet.cpp | 15 +++++++-------- src/wallet/wallet.h | 13 ++++++++++++- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 2649fa586c102..6050ad7b4ba21 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -18,10 +18,10 @@ namespace wallet { static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { - std::set addresses; + std::vector addresses; if (by_label) { // Get the set of addresses assigned to label - addresses = wallet.GetLabelAddresses(LabelFromValue(params[0])); + addresses = wallet.ListAddrBookAddresses(CWallet::AddrBookFilter{LabelFromValue(params[0])}); if (addresses.empty()) throw JSONRPCError(RPC_WALLET_ERROR, "Label not found in wallet"); } else { // Get the address @@ -29,7 +29,7 @@ static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool b if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); } - addresses.insert(dest); + addresses.emplace_back(dest); } // Filter by own scripts only diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 910562e66901a..8ca8ef0a198c7 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2348,17 +2348,16 @@ void CWallet::MarkDestinationsDirty(const std::set& destinations } } -std::set CWallet::GetLabelAddresses(const std::string& label) const +std::vector CWallet::ListAddrBookAddresses(const std::optional& _filter) const { AssertLockHeld(cs_wallet); - std::set result; - for (const std::pair& item : m_address_book) - { - if (item.second.IsChange()) continue; - const CTxDestination& address = item.first; + std::vector result; + AddrBookFilter filter = _filter ? *_filter : AddrBookFilter(); + for (const std::pair& item : m_address_book) { + if (filter.ignore_change && item.second.IsChange()) continue; const std::string& strName = item.second.GetLabel(); - if (strName == label) - result.insert(address); + if (filter.m_op_label && *filter.m_op_label != strName) continue; + result.emplace_back(item.first); } return result; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 7da601c3b7fd3..970d3e2e757d9 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -635,7 +635,18 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati std::optional GetOldestKeyPoolTime() const; - std::set GetLabelAddresses(const std::string& label) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + // Filter struct for 'ListAddrBookAddresses' + struct AddrBookFilter { + // Fetch addresses with the provided label + std::optional m_op_label{std::nullopt}; + // Don't include change addresses by default + bool ignore_change{true}; + }; + + /** + * Filter and retrieve destinations stored in the addressbook + */ + std::vector ListAddrBookAddresses(const std::optional& filter) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Marks all outputs in each one of the destinations dirty, so their cache is From 032842ae4196aaed5ea3567ea01a61ed75ab2edd Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:35:14 -0300 Subject: [PATCH 022/457] wallet: implement ForEachAddrBookEntry method --- src/wallet/wallet.cpp | 23 +++++++++++++++++------ src/wallet/wallet.h | 7 +++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 8ca8ef0a198c7..f7eb0bbc03f7e 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2348,17 +2348,28 @@ void CWallet::MarkDestinationsDirty(const std::set& destinations } } +void CWallet::ForEachAddrBookEntry(const ListAddrBookFunc& func) const +{ + AssertLockHeld(cs_wallet); + for (const std::pair& item : m_address_book) { + const auto& entry = item.second; + func(item.first, entry.GetLabel(), entry.purpose, entry.IsChange()); + } +} + std::vector CWallet::ListAddrBookAddresses(const std::optional& _filter) const { AssertLockHeld(cs_wallet); std::vector result; AddrBookFilter filter = _filter ? *_filter : AddrBookFilter(); - for (const std::pair& item : m_address_book) { - if (filter.ignore_change && item.second.IsChange()) continue; - const std::string& strName = item.second.GetLabel(); - if (filter.m_op_label && *filter.m_op_label != strName) continue; - result.emplace_back(item.first); - } + ForEachAddrBookEntry([&result, &filter](const CTxDestination& dest, const std::string& label, const std::string& purpose, bool is_change) { + // Filter by change + if (filter.ignore_change && is_change) return; + // Filter by label + if (filter.m_op_label && *filter.m_op_label != label) return; + // All good + result.emplace_back(dest); + }); return result; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 970d3e2e757d9..3775f325ba40c 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -648,6 +648,13 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati */ std::vector ListAddrBookAddresses(const std::optional& filter) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** + * Walk-through the address book entries. + * Stops when the provided 'ListAddrBookFunc' returns false. + */ + using ListAddrBookFunc = std::function; + void ForEachAddrBookEntry(const ListAddrBookFunc& func) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** * Marks all outputs in each one of the destinations dirty, so their cache is * reset and does not return outdated information. From 2b48642499016cb357e4bcec32481cd50361194e Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:38:19 -0300 Subject: [PATCH 023/457] refactor: use ForEachAddrBookEntry in interfaces::getAddresses --- src/interfaces/wallet.h | 2 +- src/wallet/interfaces.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index f26ac866dcf95..b3cb0ae3875d8 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -114,7 +114,7 @@ class Wallet std::string* purpose) = 0; //! Get wallet address list. - virtual std::vector getAddresses() = 0; + virtual std::vector getAddresses() const = 0; //! Get receive requests. virtual std::vector getAddressReceiveRequests() = 0; diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 5cfcf16e16424..823deed71ba91 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -204,14 +204,14 @@ class WalletImpl : public Wallet } return true; } - std::vector getAddresses() override + std::vector getAddresses() const override { LOCK(m_wallet->cs_wallet); std::vector result; - for (const auto& item : m_wallet->m_address_book) { - if (item.second.IsChange()) continue; - result.emplace_back(item.first, m_wallet->IsMine(item.first), item.second.GetLabel(), item.second.purpose); - } + m_wallet->ForEachAddrBookEntry([&](const CTxDestination& dest, const std::string& label, const std::string& purpose, bool is_change) EXCLUSIVE_LOCKS_REQUIRED(m_wallet->cs_wallet) { + if (is_change) return; + result.emplace_back(dest, m_wallet->IsMine(dest), label, purpose); + }); return result; } std::vector getAddressReceiveRequests() override { From 83e42c4b94e376a19d3eb0a2379769b8b8ac5fc8 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:39:25 -0300 Subject: [PATCH 024/457] refactor: use 'ForEachAddrBookEntry' in RPC 'getaddressesbylabel' --- src/wallet/rpc/addresses.cpp | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index f25ad595282b0..55dd2f935c477 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -637,17 +637,6 @@ RPCHelpMan getaddressinfo() }; } -/** Convert CAddressBookData to JSON record. */ -static UniValue AddressBookDataToJSON(const CAddressBookData& data, const bool verbose) -{ - UniValue ret(UniValue::VOBJ); - if (verbose) { - ret.pushKV("name", data.GetLabel()); - } - ret.pushKV("purpose", data.purpose); - return ret; -} - RPCHelpMan getaddressesbylabel() { return RPCHelpMan{"getaddressesbylabel", @@ -680,10 +669,10 @@ RPCHelpMan getaddressesbylabel() // Find all addresses that have the given label UniValue ret(UniValue::VOBJ); std::set addresses; - for (const std::pair& item : pwallet->m_address_book) { - if (item.second.IsChange()) continue; - if (item.second.GetLabel() == label) { - std::string address = EncodeDestination(item.first); + pwallet->ForEachAddrBookEntry([&](const CTxDestination& _dest, const std::string& _label, const std::string& _purpose, bool _is_change) { + if (_is_change) return; + if (_label == label) { + std::string address = EncodeDestination(_dest); // CWallet::m_address_book is not expected to contain duplicate // address strings, but build a separate set as a precaution just in // case it does. @@ -693,9 +682,11 @@ RPCHelpMan getaddressesbylabel() // and since duplicate addresses are unexpected (checked with // std::set in O(log(N))), UniValue::__pushKV is used instead, // which currently is O(1). - ret.__pushKV(address, AddressBookDataToJSON(item.second, false)); + UniValue value(UniValue::VOBJ); + value.pushKV("purpose", _purpose); + ret.__pushKV(address, value); } - } + }); if (ret.empty()) { throw JSONRPCError(RPC_WALLET_INVALID_LABEL_NAME, std::string("No addresses with label " + label)); From fa9f2ab8fd53075d2a3ec93ddac4908e73525c46 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 11:46:14 -0300 Subject: [PATCH 025/457] refactor: RPC 'listlabels', encapsulate 'CWallet::ListAddrBookLabels' functionality Mainly to not access 'm_address_book' externally. --- src/wallet/rpc/addresses.cpp | 8 +------- src/wallet/wallet.cpp | 14 ++++++++++++++ src/wallet/wallet.h | 5 +++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 55dd2f935c477..da4cc44ee68c7 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -733,13 +733,7 @@ RPCHelpMan listlabels() } // Add to a set to sort by label name, then insert into Univalue array - std::set label_set; - for (const std::pair& entry : pwallet->m_address_book) { - if (entry.second.IsChange()) continue; - if (purpose.empty() || entry.second.purpose == purpose) { - label_set.insert(entry.second.GetLabel()); - } - } + std::set label_set = pwallet->ListAddrBookLabels(purpose); UniValue ret(UniValue::VARR); for (const std::string& name : label_set) { diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index f7eb0bbc03f7e..3c211fb3482cc 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2373,6 +2373,20 @@ std::vector CWallet::ListAddrBookAddresses(const std::optional CWallet::ListAddrBookLabels(const std::string& purpose) const +{ + AssertLockHeld(cs_wallet); + std::set label_set; + ForEachAddrBookEntry([&](const CTxDestination& _dest, const std::string& _label, + const std::string& _purpose, bool _is_change) { + if (_is_change) return; + if (purpose.empty() || _purpose == purpose) { + label_set.insert(_label); + } + }); + return label_set; +} + bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool internal, bilingual_str& error) { m_spk_man = pwallet->GetScriptPubKeyMan(type, internal); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 3775f325ba40c..8bc1189bec15b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -648,6 +648,11 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati */ std::vector ListAddrBookAddresses(const std::optional& filter) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** + * Retrieve all the known labels in the address book + */ + std::set ListAddrBookLabels(const std::string& purpose) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** * Walk-through the address book entries. * Stops when the provided 'ListAddrBookFunc' returns false. From b459fc122feace9e9a738c48aab21961cf15dddc Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 12:00:33 -0300 Subject: [PATCH 026/457] refactor: RPC 'ListReceived', encapsulate m_address_book access --- src/wallet/rpc/transactions.cpp | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index fae9bf3ea5a0e..5d9b0965bcaac 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -138,26 +138,12 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons UniValue ret(UniValue::VARR); std::map label_tally; - // Create m_address_book iterator - // If we aren't filtering, go from begin() to end() - auto start = wallet.m_address_book.begin(); - auto end = wallet.m_address_book.end(); - // If we are filtering, find() the applicable entry - if (has_filtered_address) { - start = wallet.m_address_book.find(filtered_address); - if (start != end) { - end = std::next(start); - } - } + const auto& func = [&](const CTxDestination& address, const std::string& label, const std::string& purpose, bool is_change) { + if (is_change) return; // no change addresses - for (auto item_it = start; item_it != end; ++item_it) - { - if (item_it->second.IsChange()) continue; - const CTxDestination& address = item_it->first; - const std::string& label = item_it->second.GetLabel(); auto it = mapTally.find(address); if (it == mapTally.end() && !fIncludeEmpty) - continue; + return; CAmount nAmount = 0; int nConf = std::numeric_limits::max(); @@ -196,6 +182,14 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons obj.pushKV("txids", transactions); ret.push_back(obj); } + }; + + if (has_filtered_address) { + const auto& entry = wallet.FindAddressBookEntry(filtered_address, /*allow_change=*/false); + if (entry) func(filtered_address, entry->GetLabel(), entry->purpose, /*is_change=*/false); + } else { + // No filtered addr, walk-through the addressbook entry + wallet.ForEachAddrBookEntry(func); } if (by_label) From 324f00a6420bbd64c67c264e50632e6fa36ae732 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 11 Jun 2022 12:05:54 -0300 Subject: [PATCH 027/457] refactor: 'ListReceived' use optional for filtered address Plus remove open bracket jump line --- src/wallet/rpc/transactions.cpp | 45 ++++++++++++--------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index 5d9b0965bcaac..6e42c0736d1f3 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -85,14 +85,12 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons filter |= ISMINE_WATCH_ONLY; } - bool has_filtered_address = false; - CTxDestination filtered_address = CNoDestination(); + std::optional filtered_address{std::nullopt}; if (!by_label && !params[3].isNull() && !params[3].get_str().empty()) { if (!IsValidDestinationString(params[3].get_str())) { throw JSONRPCError(RPC_WALLET_ERROR, "address_filter parameter was invalid"); } filtered_address = DecodeDestination(params[3].get_str()); - has_filtered_address = true; } // Tally @@ -106,23 +104,21 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons // Coinbase with less than 1 confirmation is no longer in the main chain if ((wtx.IsCoinBase() && (nDepth < 1)) - || (wallet.IsTxImmatureCoinBase(wtx) && !include_immature_coinbase)) - { + || (wallet.IsTxImmatureCoinBase(wtx) && !include_immature_coinbase)) { continue; } - for (const CTxOut& txout : wtx.tx->vout) - { + for (const CTxOut& txout : wtx.tx->vout) { CTxDestination address; if (!ExtractDestination(txout.scriptPubKey, address)) continue; - if (has_filtered_address && !(filtered_address == address)) { + if (filtered_address && !(filtered_address == address)) { continue; } isminefilter mine = wallet.IsMine(address); - if(!(mine & filter)) + if (!(mine & filter)) continue; tallyitem& item = mapTally[address]; @@ -148,34 +144,27 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons CAmount nAmount = 0; int nConf = std::numeric_limits::max(); bool fIsWatchonly = false; - if (it != mapTally.end()) - { + if (it != mapTally.end()) { nAmount = (*it).second.nAmount; nConf = (*it).second.nConf; fIsWatchonly = (*it).second.fIsWatchonly; } - if (by_label) - { + if (by_label) { tallyitem& _item = label_tally[label]; _item.nAmount += nAmount; _item.nConf = std::min(_item.nConf, nConf); _item.fIsWatchonly = fIsWatchonly; - } - else - { + } else { UniValue obj(UniValue::VOBJ); - if(fIsWatchonly) - obj.pushKV("involvesWatchonly", true); + if (fIsWatchonly) obj.pushKV("involvesWatchonly", true); obj.pushKV("address", EncodeDestination(address)); obj.pushKV("amount", ValueFromAmount(nAmount)); obj.pushKV("confirmations", (nConf == std::numeric_limits::max() ? 0 : nConf)); obj.pushKV("label", label); UniValue transactions(UniValue::VARR); - if (it != mapTally.end()) - { - for (const uint256& _item : (*it).second.txids) - { + if (it != mapTally.end()) { + for (const uint256& _item : (*it).second.txids) { transactions.push_back(_item.GetHex()); } } @@ -184,18 +173,16 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons } }; - if (has_filtered_address) { - const auto& entry = wallet.FindAddressBookEntry(filtered_address, /*allow_change=*/false); - if (entry) func(filtered_address, entry->GetLabel(), entry->purpose, /*is_change=*/false); + if (filtered_address) { + const auto& entry = wallet.FindAddressBookEntry(*filtered_address, /*allow_change=*/false); + if (entry) func(*filtered_address, entry->GetLabel(), entry->purpose, /*is_change=*/false); } else { // No filtered addr, walk-through the addressbook entry wallet.ForEachAddrBookEntry(func); } - if (by_label) - { - for (const auto& entry : label_tally) - { + if (by_label) { + for (const auto& entry : label_tally) { CAmount nAmount = entry.second.nAmount; int nConf = entry.second.nConf; UniValue obj(UniValue::VOBJ); From d69045e291e32e02d105d1b5ff1c8b86db0ae69e Mon Sep 17 00:00:00 2001 From: furszy Date: Wed, 22 Jun 2022 12:46:43 -0300 Subject: [PATCH 028/457] test: add coverage for 'listreceivedbyaddress' no change addrs return --- test/functional/wallet_listreceivedby.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index db1d8eb54aec9..7ae3a40eaf39c 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -57,6 +57,11 @@ def run_test(self): {"address": empty_addr}, {"address": empty_addr, "label": "", "amount": 0, "confirmations": 0, "txids": []}) + # No returned addy should be a change addr + for node in self.nodes: + for addr_obj in node.listreceivedbyaddress(): + assert_equal(node.getaddressinfo(addr_obj["address"])["ischange"], False) + # Test Address filtering # Only on addr expected = {"address": addr, "label": "", "amount": Decimal("0.1"), "confirmations": 10, "txids": [txid, ]} From fa076515b07ac4b10b2134e323bf4f56be5996a8 Mon Sep 17 00:00:00 2001 From: glozow Date: Tue, 20 Jul 2021 11:45:52 +0100 Subject: [PATCH 029/457] [rpc] add new submitpackage RPC It could be unsafe/confusing to create an actual mainnet interface while package relay doesn't exist. However, a regtest-only interface allows wallet/application devs to test current package policies. --- src/rpc/client.cpp | 1 + src/rpc/mempool.cpp | 146 ++++++++++++++++++++++++++++++++++++++++++ src/test/fuzz/rpc.cpp | 1 + src/util/error.cpp | 2 + src/util/error.h | 1 + 5 files changed, 151 insertions(+) diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index ae0d0112babf8..9be3ab7df046c 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -110,6 +110,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendrawtransaction", 1, "maxfeerate" }, { "testmempoolaccept", 0, "rawtxs" }, { "testmempoolaccept", 1, "maxfeerate" }, + { "submitpackage", 0, "package" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, { "fundrawtransaction", 2, "iswitness" }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 01c75bfda3e4e..dbba00fd9d78d 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -5,6 +5,7 @@ #include +#include #include #include #include @@ -729,6 +730,150 @@ static RPCHelpMan savemempool() }; } +static RPCHelpMan submitpackage() +{ + return RPCHelpMan{"submitpackage", + "Submit a package of raw transactions (serialized, hex-encoded) to local node (-regtest only).\n" + "The package will be validated according to consensus and mempool policy rules. If all transactions pass, they will be accepted to mempool.\n" + "This RPC is experimental and the interface may be unstable. Refer to doc/policy/packages.md for documentation on package policies.\n" + "Warning: until package relay is in use, successful submission does not mean the transaction will propagate to other nodes on the network.\n" + "Currently, each transaction is broadcasted individually after submission, which means they must meet other nodes' feerate requirements alone.\n" + , + { + {"package", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of raw transactions.", + { + {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, + }, + }, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::OBJ_DYN, "tx-results", "transaction results keyed by wtxid", + { + {RPCResult::Type::OBJ, "wtxid", "transaction wtxid", { + {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + {RPCResult::Type::STR_HEX, "other-wtxid", /*optional=*/true, "The wtxid of a different transaction with the same txid but different witness found in the mempool. This means the submitted transaction was ignored."}, + {RPCResult::Type::NUM, "vsize", "Virtual transaction size as defined in BIP 141."}, + {RPCResult::Type::OBJ, "fees", "Transaction fees", { + {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, + }}, + }} + }}, + {RPCResult::Type::STR_AMOUNT, "package-feerate", /*optional=*/true, "package feerate used for feerate checks in " + CURRENCY_UNIT + " per KvB. Excludes transactions which were deduplicated or accepted individually."}, + {RPCResult::Type::ARR, "replaced-transactions", /*optional=*/true, "List of txids of replaced transactions", + { + {RPCResult::Type::STR_HEX, "", "The transaction id"}, + }}, + }, + }, + RPCExamples{ + HelpExampleCli("testmempoolaccept", "[rawtx1, rawtx2]") + + HelpExampleCli("submitpackage", "[rawtx1, rawtx2]") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + if (!Params().IsMockableChain()) { + throw std::runtime_error("submitpackage is for regression testing (-regtest mode) only"); + } + RPCTypeCheck(request.params, { + UniValue::VARR, + }); + const UniValue raw_transactions = request.params[0].get_array(); + if (raw_transactions.size() < 1 || raw_transactions.size() > MAX_PACKAGE_COUNT) { + throw JSONRPCError(RPC_INVALID_PARAMETER, + "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); + } + + std::vector txns; + txns.reserve(raw_transactions.size()); + for (const auto& rawtx : raw_transactions.getValues()) { + CMutableTransaction mtx; + if (!DecodeHexTx(mtx, rawtx.get_str())) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, + "TX decode failed: " + rawtx.get_str() + " Make sure the tx has at least one input."); + } + txns.emplace_back(MakeTransactionRef(std::move(mtx))); + } + + NodeContext& node = EnsureAnyNodeContext(request.context); + CTxMemPool& mempool = EnsureMemPool(node); + CChainState& chainstate = EnsureChainman(node).ActiveChainstate(); + const auto package_result = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/ false)); + + // First catch any errors. + switch(package_result.m_state.GetResult()) { + case PackageValidationResult::PCKG_RESULT_UNSET: break; + case PackageValidationResult::PCKG_POLICY: + { + throw JSONRPCTransactionError(TransactionError::INVALID_PACKAGE, + package_result.m_state.GetRejectReason()); + } + case PackageValidationResult::PCKG_MEMPOOL_ERROR: + { + throw JSONRPCTransactionError(TransactionError::MEMPOOL_ERROR, + package_result.m_state.GetRejectReason()); + } + case PackageValidationResult::PCKG_TX: + { + for (const auto& tx : txns) { + auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); + if (it != package_result.m_tx_results.end() && it->second.m_state.IsInvalid()) { + throw JSONRPCTransactionError(TransactionError::MEMPOOL_REJECTED, + strprintf("%s failed: %s", tx->GetHash().ToString(), it->second.m_state.GetRejectReason())); + } + } + // If a PCKG_TX error was returned, there must have been an invalid transaction. + NONFATAL_UNREACHABLE(); + } + } + for (const auto& tx : txns) { + size_t num_submitted{0}; + std::string err_string; + const auto err = BroadcastTransaction(node, tx, err_string, 0, true, true); + if (err != TransactionError::OK) { + throw JSONRPCTransactionError(err, + strprintf("transaction broadcast failed: %s (all transactions were submitted, %d transactions were broadcast successfully)", + err_string, num_submitted)); + } + } + UniValue rpc_result{UniValue::VOBJ}; + UniValue tx_result_map{UniValue::VOBJ}; + std::set replaced_txids; + for (const auto& tx : txns) { + auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); + CHECK_NONFATAL(it != package_result.m_tx_results.end()); + UniValue result_inner{UniValue::VOBJ}; + result_inner.pushKV("txid", tx->GetHash().GetHex()); + if (it->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS) { + result_inner.pushKV("other-wtxid", it->second.m_other_wtxid.value().GetHex()); + } + if (it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || + it->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY) { + result_inner.pushKV("vsize", int64_t{it->second.m_vsize.value()}); + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(it->second.m_base_fees.value())); + result_inner.pushKV("fees", fees); + if (it->second.m_replaced_transactions.has_value()) { + for (const auto& ptx : it->second.m_replaced_transactions.value()) { + replaced_txids.insert(ptx->GetHash()); + } + } + } + tx_result_map.pushKV(tx->GetWitnessHash().GetHex(), result_inner); + } + rpc_result.pushKV("tx-results", tx_result_map); + if (package_result.m_package_feerate.has_value()) { + rpc_result.pushKV("package-feerate", ValueFromAmount(package_result.m_package_feerate.value().GetFeePerK())); + } + UniValue replaced_list(UniValue::VARR); + for (const uint256& hash : replaced_txids) replaced_list.push_back(hash.ToString()); + rpc_result.pushKV("replaced-transactions", replaced_list); + return rpc_result; + }, + }; +} + void RegisterMempoolRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ @@ -741,6 +886,7 @@ void RegisterMempoolRPCCommands(CRPCTable& t) {"blockchain", &getmempoolinfo}, {"blockchain", &getrawmempool}, {"blockchain", &savemempool}, + {"hidden", &submitpackage}, }; for (const auto& c : commands) { t.appendCommand(c.name, &c); diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index e4e83c3f32e6f..26913a41d2759 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -159,6 +159,7 @@ const std::vector RPC_COMMANDS_SAFE_FOR_FUZZING{ "signrawtransactionwithkey", "submitblock", "submitheader", + "submitpackage", "syncwithvalidationinterfacequeue", "testmempoolaccept", "uptime", diff --git a/src/util/error.cpp b/src/util/error.cpp index af8cbd0353a16..22a59642790ce 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -35,6 +35,8 @@ bilingual_str TransactionErrorString(const TransactionError err) return Untranslated("External signer not found"); case TransactionError::EXTERNAL_SIGNER_FAILED: return Untranslated("External signer failed to sign"); + case TransactionError::INVALID_PACKAGE: + return Untranslated("Transaction rejected due to invalid package"); // no default case, so the compiler can warn about missing cases } assert(false); diff --git a/src/util/error.h b/src/util/error.h index 4cc35eb1fdd8b..0429de651a363 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -32,6 +32,7 @@ enum class TransactionError { MAX_FEE_EXCEEDED, EXTERNAL_SIGNER_NOT_FOUND, EXTERNAL_SIGNER_FAILED, + INVALID_PACKAGE, }; bilingual_str TransactionErrorString(const TransactionError error); From e866f0d0666664885d4c15c79bf59cc59975887a Mon Sep 17 00:00:00 2001 From: glozow Date: Tue, 20 Jul 2021 11:49:43 +0100 Subject: [PATCH 030/457] [functional test] submitrawpackage RPC --- test/functional/rpc_packages.py | 133 +++++++++++++++++++++++++++++++- test/functional/test_runner.py | 2 +- 2 files changed, 130 insertions(+), 5 deletions(-) diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 63533affd0397..f68784928716d 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -15,16 +15,20 @@ CTxInWitness, tx_from_hex, ) +from test_framework.p2p import P2PTxInvStore from test_framework.script import ( CScript, OP_TRUE, ) from test_framework.util import ( assert_equal, + assert_fee_amount, + assert_raises_rpc_error, ) from test_framework.wallet import ( create_child_with_parents, create_raw_chain, + DEFAULT_FEE, make_chain, ) @@ -51,7 +55,7 @@ def run_test(self): self.address = node.get_deterministic_priv_key().address self.coins = [] # The last 100 coinbase transactions are premature - for b in self.generatetoaddress(node, 200, self.address)[:100]: + for b in self.generatetoaddress(node, 220, self.address)[:-100]: coinbase = node.getblock(blockhash=b, verbosity=2)["tx"][0] self.coins.append({ "txid": coinbase["txid"], @@ -82,7 +86,7 @@ def run_test(self): self.test_multiple_parents() self.test_conflicting() self.test_rbf() - + self.test_submitpackage() def test_independent(self): self.log.info("Test multiple independent transactions in a package") @@ -132,8 +136,7 @@ def test_independent(self): def test_chain(self): node = self.nodes[0] - first_coin = self.coins.pop() - (chain_hex, chain_txns) = create_raw_chain(node, first_coin, self.address, self.privkeys) + (chain_hex, chain_txns) = create_raw_chain(node, self.coins.pop(), self.address, self.privkeys) self.log.info("Check that testmempoolaccept requires packages to be sorted by dependency") assert_equal(node.testmempoolaccept(rawtxs=chain_hex[::-1]), [{"txid": tx.rehash(), "wtxid": tx.getwtxid(), "package-error": "package-not-sorted"} for tx in chain_txns[::-1]]) @@ -306,5 +309,127 @@ def test_rbf(self): }] self.assert_testres_equal(self.independent_txns_hex + [signed_replacement_tx["hex"]], testres_rbf_package) + def assert_equal_package_results(self, node, testmempoolaccept_result, submitpackage_result): + """Assert that a successful submitpackage result is consistent with testmempoolaccept + results and getmempoolentry info. Note that the result structs are different and, due to + policy differences between testmempoolaccept and submitpackage (i.e. package feerate), + some information may be different. + """ + for testres_tx in testmempoolaccept_result: + # Grab this result from the submitpackage_result + submitres_tx = submitpackage_result["tx-results"][testres_tx["wtxid"]] + assert_equal(submitres_tx["txid"], testres_tx["txid"]) + # No "allowed" if the tx was already in the mempool + if "allowed" in testres_tx and testres_tx["allowed"]: + assert_equal(submitres_tx["vsize"], testres_tx["vsize"]) + assert_equal(submitres_tx["fees"]["base"], testres_tx["fees"]["base"]) + entry_info = node.getmempoolentry(submitres_tx["txid"]) + assert_equal(submitres_tx["vsize"], entry_info["vsize"]) + assert_equal(submitres_tx["fees"]["base"], entry_info["fees"]["base"]) + + def test_submit_child_with_parents(self, num_parents, partial_submit): + node = self.nodes[0] + peer = node.add_p2p_connection(P2PTxInvStore()) + # Test a package with num_parents parents and 1 child transaction. + package_hex = [] + package_txns = [] + values = [] + scripts = [] + for _ in range(num_parents): + parent_coin = self.coins.pop() + value = parent_coin["amount"] + (tx, txhex, value, spk) = make_chain(node, self.address, self.privkeys, parent_coin["txid"], value) + package_hex.append(txhex) + package_txns.append(tx) + values.append(value) + scripts.append(spk) + if partial_submit and random.choice([True, False]): + node.sendrawtransaction(txhex) + child_hex = create_child_with_parents(node, self.address, self.privkeys, package_txns, values, scripts) + package_hex.append(child_hex) + package_txns.append(tx_from_hex(child_hex)) + + testmempoolaccept_result = node.testmempoolaccept(rawtxs=package_hex) + submitpackage_result = node.submitpackage(package=package_hex) + + # Check that each result is present, with the correct size and fees + for i in range(num_parents + 1): + tx = package_txns[i] + wtxid = tx.getwtxid() + assert wtxid in submitpackage_result["tx-results"] + tx_result = submitpackage_result["tx-results"][wtxid] + assert_equal(tx_result, { + "txid": tx.rehash(), + "vsize": tx.get_vsize(), + "fees": { + "base": DEFAULT_FEE, + } + }) + + # submitpackage result should be consistent with testmempoolaccept and getmempoolentry + self.assert_equal_package_results(node, testmempoolaccept_result, submitpackage_result) + + # Package feerate is calculated for the remaining transactions after deduplication and + # individual submission. If only 0 or 1 transaction is left, e.g. because all transactions + # had high-feerates or were already in the mempool, no package feerate is provided. + # In this case, since all of the parents have high fees, each is accepted individually. + assert "package-feerate" not in submitpackage_result + + # The node should announce each transaction. No guarantees for propagation. + peer.wait_for_broadcast([tx.getwtxid() for tx in package_txns]) + self.generate(node, 1) + + + def test_submit_cpfp(self): + node = self.nodes[0] + peer = node.add_p2p_connection(P2PTxInvStore()) + + # 2 parent 1 child CPFP. First parent pays high fees, second parent pays 0 fees and is + # fee-bumped by the child. + coin_rich = self.coins.pop() + coin_poor = self.coins.pop() + tx_rich, hex_rich, value_rich, spk_rich = make_chain(node, self.address, self.privkeys, coin_rich["txid"], coin_rich["amount"]) + tx_poor, hex_poor, value_poor, spk_poor = make_chain(node, self.address, self.privkeys, coin_poor["txid"], coin_poor["amount"], fee=0) + package_txns = [tx_rich, tx_poor] + hex_child = create_child_with_parents(node, self.address, self.privkeys, package_txns, [value_rich, value_poor], [spk_rich, spk_poor]) + tx_child = tx_from_hex(hex_child) + package_txns.append(tx_child) + + submitpackage_result = node.submitpackage([hex_rich, hex_poor, hex_child]) + + rich_parent_result = submitpackage_result["tx-results"][tx_rich.getwtxid()] + poor_parent_result = submitpackage_result["tx-results"][tx_poor.getwtxid()] + child_result = submitpackage_result["tx-results"][tx_child.getwtxid()] + assert_equal(rich_parent_result["fees"]["base"], DEFAULT_FEE) + assert_equal(poor_parent_result["fees"]["base"], 0) + assert_equal(child_result["fees"]["base"], DEFAULT_FEE) + # Package feerate is calculated for the remaining transactions after deduplication and + # individual submission. Since this package had a 0-fee parent, package feerate must have + # been used and returned. + assert "package-feerate" in submitpackage_result + assert_fee_amount(DEFAULT_FEE, rich_parent_result["vsize"] + child_result["vsize"], submitpackage_result["package-feerate"]) + + # The node will broadcast each transaction, still abiding by its peer's fee filter + peer.wait_for_broadcast([tx.getwtxid() for tx in package_txns]) + self.generate(node, 1) + + + def test_submitpackage(self): + node = self.nodes[0] + + self.log.info("Submitpackage valid packages with 1 child and some number of parents") + for num_parents in [1, 2, 24]: + self.test_submit_child_with_parents(num_parents, False) + self.test_submit_child_with_parents(num_parents, True) + + self.log.info("Submitpackage valid packages with CPFP") + self.test_submit_cpfp() + + self.log.info("Submitpackage only allows packages of 1 child with its parents") + # Chain of 3 transactions has too many generations + chain_hex, _ = create_raw_chain(node, self.coins.pop(), self.address, self.privkeys, 3) + assert_raises_rpc_error(-25, "not-child-with-parents", node.submitpackage, chain_hex) + + if __name__ == "__main__": RPCPackagesTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 40e08c3f1f6d8..c388c2cca313a 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -130,6 +130,7 @@ 'wallet_address_types.py --descriptors', 'feature_bip68_sequence.py', 'p2p_feefilter.py', + 'rpc_packages.py', 'feature_reindex.py', 'feature_abortnode.py', # vv Tests less than 30s vv @@ -230,7 +231,6 @@ 'mempool_packages.py', 'mempool_package_onemore.py', 'rpc_createmultisig.py', - 'rpc_packages.py', 'mempool_package_limits.py', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', From a89ddfbe22b6db5beda678c9493e08fec6144122 Mon Sep 17 00:00:00 2001 From: w0xlt <94266259+w0xlt@users.noreply.github.com> Date: Fri, 29 Apr 2022 19:10:57 -0300 Subject: [PATCH 031/457] wallet: Save wallet scan progress Currently, the wallet scan progress is not saved. If it is interrupted, it will be necessary to start from scratch on the next load. With this change, progress is saved every 60 seconds. Co-authored-by: furszy Co-authored-by: Jon Atack Co-authored-by: Ryan Ofsky --- src/interfaces/chain.h | 4 ++++ src/node/interfaces.cpp | 17 ++++++++++++----- src/qt/test/wallettests.cpp | 2 +- src/wallet/rpc/transactions.cpp | 2 +- src/wallet/test/util.cpp | 2 +- src/wallet/test/wallet_tests.cpp | 8 ++++---- src/wallet/wallet.cpp | 22 +++++++++++++++++----- src/wallet/wallet.h | 2 +- 8 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index ddfb4bda95b11..df9e55874f38f 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -111,6 +111,10 @@ class Chain //! Get locator for the current chain tip. virtual CBlockLocator getTipLocator() = 0; + //! Return a locator that refers to a block in the active chain. + //! If specified block is not in the active chain, return locator for the latest ancestor that is in the chain. + virtual CBlockLocator getActiveChainLocator(const uint256& block_hash) = 0; + //! Return height of the highest block on chain in common with the locator, //! which will either be the original block used to create the locator, //! or one of its ancestors. diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 4810ae1f68cbd..1930d0a0dbfda 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -516,20 +516,27 @@ class ChainImpl : public Chain } bool haveBlockOnDisk(int height) override { - LOCK(cs_main); + LOCK(::cs_main); const CChain& active = Assert(m_node.chainman)->ActiveChain(); CBlockIndex* block = active[height]; return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; } CBlockLocator getTipLocator() override { - LOCK(cs_main); + LOCK(::cs_main); const CChain& active = Assert(m_node.chainman)->ActiveChain(); return active.GetLocator(); } + CBlockLocator getActiveChainLocator(const uint256& block_hash) override + { + LOCK(::cs_main); + const CBlockIndex* index = chainman().m_blockman.LookupBlockIndex(block_hash); + if (!index) return {}; + return chainman().ActiveChain().GetLocator(index); + } std::optional findLocatorFork(const CBlockLocator& locator) override { - LOCK(cs_main); + LOCK(::cs_main); const CChainState& active = Assert(m_node.chainman)->ActiveChainstate(); if (const CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { return fork->nHeight; @@ -585,7 +592,7 @@ class ChainImpl : public Chain void findCoins(std::map& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override { - LOCK(cs_main); + LOCK(::cs_main); return GuessVerificationProgress(chainman().GetParams().TxData(), chainman().m_blockman.LookupBlockIndex(block_hash)); } bool hasBlocks(const uint256& block_hash, int min_height, std::optional max_height) override @@ -684,7 +691,7 @@ class ChainImpl : public Chain CFeeRate relayDustFee() override { return ::dustRelayFee; } bool havePruned() override { - LOCK(cs_main); + LOCK(::cs_main); return m_node.chainman->m_blockman.m_have_pruned; } bool isReadyToBroadcast() override { return !node::fImporting && !node::fReindex && !isInitialBlockDownload(); } diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index bc06f0f23be44..3740aa9801930 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -173,7 +173,7 @@ void TestGUI(interfaces::Node& node) { WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, 0 /* block height */, {} /* max height */, reserver, true /* fUpdate */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/true, /*save_progress=*/false); QCOMPARE(result.status, CWallet::ScanResult::SUCCESS); QCOMPARE(result.last_scanned_block, node.context()->chainman->ActiveChain().Tip()->GetBlockHash()); QVERIFY(result.last_failed_block.IsNull()); diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index fae9bf3ea5a0e..f4e46a5982eb6 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -908,7 +908,7 @@ RPCHelpMan rescanblockchain() } CWallet::ScanResult result = - pwallet->ScanForWalletTransactions(start_block, start_height, stop_height, reserver, true /* fUpdate */); + pwallet->ScanForWalletTransactions(start_block, start_height, stop_height, reserver, /*fUpdate=*/true, /*save_progress=*/false); switch (result.status) { case CWallet::ScanResult::SUCCESS: break; diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index aa3121511d85e..ab72721f9d305 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -38,7 +38,7 @@ std::unique_ptr CreateSyncedWallet(interfaces::Chain& chain, CChain& cc } WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(cchain.Genesis()->GetBlockHash(), 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(cchain.Genesis()->GetBlockHash(), /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK_EQUAL(result.last_scanned_block, cchain.Tip()->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, cchain.Height()); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 70863f5464ab8..4fbce7fc47f7c 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -112,7 +112,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/{}, /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK(result.last_scanned_block.IsNull()); @@ -132,7 +132,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -161,7 +161,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, oldTip->GetBlockHash()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -188,7 +188,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, newTip->GetBlockHash()); BOOST_CHECK(result.last_scanned_block.IsNull()); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 6c333c709b7d3..56fa3eb1b0021 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1674,7 +1674,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r if (start) { // TODO: this should take into account failure by ScanResult::USER_ABORT - ScanResult result = ScanForWalletTransactions(start_block, start_height, {} /* max_height */, reserver, update); + ScanResult result = ScanForWalletTransactions(start_block, start_height, /*max_height=*/{}, reserver, /*fUpdate=*/update, /*save_progress=*/false); if (result.status == ScanResult::FAILURE) { int64_t time_max; CHECK_NONFATAL(chain().findBlock(result.last_failed_block, FoundBlock().maxTime(time_max))); @@ -1705,10 +1705,10 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r * the main chain after to the addition of any new keys you want to detect * transactions for. */ -CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate) +CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress) { using Clock = std::chrono::steady_clock; - constexpr auto LOG_INTERVAL{60s}; + constexpr auto INTERVAL_TIME{60s}; auto current_time{Clock::now()}; auto start_time{Clock::now()}; @@ -1737,7 +1737,9 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc if (block_height % 100 == 0 && progress_end - progress_begin > 0.0) { ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } - if (Clock::now() >= current_time + LOG_INTERVAL) { + + bool next_interval = Clock::now() >= current_time + INTERVAL_TIME; + if (next_interval) { current_time = Clock::now(); WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } @@ -1768,6 +1770,16 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; result.last_scanned_height = block_height; + + if (save_progress && next_interval) { + CBlockLocator loc = m_chain->getActiveChainLocator(block_hash); + + if (!loc.IsNull()) { + WalletLogPrintf("Saving scan progress %d.\n", block_height); + WalletBatch batch(GetDatabase()); + batch.WriteBestBlock(loc); + } + } } else { // could not scan block, keep scanning but record this block as the most recent failure result.last_failed_block = block_hash; @@ -3026,7 +3038,7 @@ bool CWallet::AttachChain(const std::shared_ptr& walletInstance, interf { WalletRescanReserver reserver(*walletInstance); - if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(chain.getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { + if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(chain.getBlockHash(rescan_height), rescan_height, /*max_height=*/{}, reserver, /*fUpdate=*/true, /*save_progress=*/true).status)) { error = _("Failed to rescan the wallet during initialization"); return false; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 7da601c3b7fd3..206a4417a7e3a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -528,7 +528,7 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati //! USER_ABORT. uint256 last_failed_block; }; - ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate); + ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress); void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override; void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); From 230a2f4cc3fab9f66b6c24ba809ddbea77755cb7 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 8 Jun 2022 23:05:16 -0300 Subject: [PATCH 032/457] wallet test: Add unit test for wallet scan save_progress option --- src/wallet/test/wallet_tests.cpp | 19 +++++++++++++++++-- src/wallet/wallet.cpp | 11 +++++------ src/wallet/wallet.h | 7 +++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 4fbce7fc47f7c..3483f05292b04 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -123,7 +123,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -131,13 +131,28 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); + std::chrono::steady_clock::time_point fake_time; + reserver.setNow([&] { fake_time += 60s; return fake_time; }); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); + + { + CBlockLocator locator; + BOOST_CHECK(!WalletBatch{wallet.GetDatabase()}.ReadBestBlock(locator)); + BOOST_CHECK(locator.IsNull()); + } + + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/true); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, newTip->nHeight); BOOST_CHECK_EQUAL(GetBalance(wallet).m_mine_immature, 100 * COIN); + + { + CBlockLocator locator; + BOOST_CHECK(WalletBatch{wallet.GetDatabase()}.ReadBestBlock(locator)); + BOOST_CHECK(!locator.IsNull()); + } } // Prune the older block file. diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 56fa3eb1b0021..91106cde83229 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1707,10 +1707,9 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r */ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress) { - using Clock = std::chrono::steady_clock; constexpr auto INTERVAL_TIME{60s}; - auto current_time{Clock::now()}; - auto start_time{Clock::now()}; + auto current_time{reserver.now()}; + auto start_time{reserver.now()}; assert(reserver.isReserved()); @@ -1738,9 +1737,9 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } - bool next_interval = Clock::now() >= current_time + INTERVAL_TIME; + bool next_interval = reserver.now() >= current_time + INTERVAL_TIME; if (next_interval) { - current_time = Clock::now(); + current_time = reserver.now(); WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } @@ -1817,7 +1816,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else { - auto duration_milliseconds = std::chrono::duration_cast(Clock::now() - start_time); + auto duration_milliseconds = std::chrono::duration_cast(reserver.now() - start_time); WalletLogPrintf("Rescan completed in %15dms\n", duration_milliseconds.count()); } return result; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 206a4417a7e3a..5be2ad166a187 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -896,8 +896,11 @@ void MaybeResendWalletTxs(WalletContext& context); class WalletRescanReserver { private: + using Clock = std::chrono::steady_clock; + using NowFn = std::function; CWallet& m_wallet; bool m_could_reserve; + NowFn m_now; public: explicit WalletRescanReserver(CWallet& w) : m_wallet(w), m_could_reserve(false) {} @@ -918,6 +921,10 @@ class WalletRescanReserver return (m_could_reserve && m_wallet.fScanningWallet); } + Clock::time_point now() const { return m_now ? m_now() : Clock::now(); }; + + void setNow(NowFn now) { m_now = std::move(now); } + ~WalletRescanReserver() { if (m_could_reserve) { From fa8a1c06961f4b1826696e0db8dce81dce627721 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Thu, 23 Jun 2022 21:48:43 +0200 Subject: [PATCH 033/457] rpc: Fix Univalue push_backV OOM in listtransactions --- src/univalue/include/univalue.h | 10 ++++++++++ src/wallet/rpc/transactions.cpp | 12 ++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 22be0311e835e..7f9a6aaffd989 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -84,6 +84,8 @@ class UniValue { bool push_back(const UniValue& val); bool push_backV(const std::vector& vec); + template + bool push_backV(It first, It last); void __pushKV(const std::string& key, const UniValue& val); bool pushKV(const std::string& key, const UniValue& val); @@ -137,6 +139,14 @@ class UniValue { friend const UniValue& find_value( const UniValue& obj, const std::string& name); }; +template +bool UniValue::push_backV(It first, It last) +{ + if (typ != VARR) return false; + values.insert(values.end(), first, last); + return true; +} + enum jtokentype { JTOK_ERR = -1, JTOK_NONE = 0, // eof diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index fae9bf3ea5a0e..cb303d1cf6a76 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -329,11 +329,12 @@ static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) * @param wtx The wallet transaction. * @param nMinDepth The minimum confirmation depth. * @param fLong Whether to include the JSON version of the transaction. - * @param ret The UniValue into which the result is stored. + * @param ret The vector into which the result is stored. * @param filter_ismine The "is mine" filter flags. * @param filter_label Optional label string to filter incoming transactions. */ -static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +template +static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nMinDepth, bool fLong, Vec& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { CAmount nFee; std::list listReceived; @@ -519,8 +520,7 @@ RPCHelpMan listtransactions() if (nFrom < 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative from"); - UniValue ret(UniValue::VARR); - + std::vector ret; { LOCK(pwallet->cs_wallet); @@ -542,9 +542,9 @@ RPCHelpMan listtransactions() if ((nFrom + nCount) > (int)ret.size()) nCount = ret.size() - nFrom; - const std::vector& txs = ret.getValues(); + auto txs_rev_it{std::make_move_iterator(ret.rend())}; UniValue result{UniValue::VARR}; - result.push_backV({ txs.rend() - nFrom - nCount, txs.rend() - nFrom }); // Return oldest to newest + result.push_backV(txs_rev_it - nFrom - nCount, txs_rev_it - nFrom); // Return oldest to newest return result; }, }; From eec23dad1ec471641dcc74f6679e5c0eda44da94 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Fri, 24 Jun 2022 17:56:53 +0530 Subject: [PATCH 034/457] test: remove wallet dependency from feature_nulldummy.py This test can now be run even with the Bitcoin Core wallet disabled. --- test/functional/feature_nulldummy.py | 65 ++++++++++++++++++---------- test/functional/test_runner.py | 3 +- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index 7a84098a839c1..9bfb79057ea57 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -19,9 +19,11 @@ NORMAL_GBT_REQUEST_PARAMS, add_witness_commitment, create_block, - create_transaction, ) -from test_framework.messages import CTransaction +from test_framework.messages import ( + CTransaction, + tx_from_hex, +) from test_framework.script import ( OP_0, OP_TRUE, @@ -31,6 +33,9 @@ assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import getnewdestination +from test_framework.key import ECKey +from test_framework.wallet_util import bytes_to_wif NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)" @@ -55,22 +60,26 @@ def set_test_params(self): '-par=1', # Use only one script thread to get the exact reject reason for testing ]] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def create_transaction(self, *, txid, input_details=None, addr, amount, privkey): + input = {"txid": txid, "vout": 0} + output = {addr: amount} + rawtx = self.nodes[0].createrawtransaction([input], output) + # Details only needed for scripthash or witness spends + input = None if not input_details else [{**input, **input_details}] + signedtx = self.nodes[0].signrawtransactionwithkey(rawtx, [privkey], input) + return tx_from_hex(signedtx["hex"]) def run_test(self): - self.nodes[0].createwallet(wallet_name='wmulti', disable_private_keys=True) - wmulti = self.nodes[0].get_wallet_rpc('wmulti') - w0 = self.nodes[0].get_wallet_rpc(self.default_wallet_name) - self.address = w0.getnewaddress() - self.pubkey = w0.getaddressinfo(self.address)['pubkey'] - self.ms_address = wmulti.addmultisigaddress(1, [self.pubkey])['address'] - self.wit_address = w0.getnewaddress(address_type='p2sh-segwit') - self.wit_ms_address = wmulti.addmultisigaddress(1, [self.pubkey], '', 'p2sh-segwit')['address'] - if not self.options.descriptors: - # Legacy wallets need to import these so that they are watched by the wallet. This is unnecessary (and does not need to be tested) for descriptor wallets - wmulti.importaddress(self.ms_address) - wmulti.importaddress(self.wit_ms_address) + eckey = ECKey() + eckey.generate() + self.privkey = bytes_to_wif(eckey.get_bytes()) + self.pubkey = eckey.get_pubkey().get_bytes().hex() + cms = self.nodes[0].createmultisig(1, [self.pubkey]) + wms = self.nodes[0].createmultisig(1, [self.pubkey], 'p2sh-segwit') + self.ms_address = cms["address"] + ms_unlock_details = {"scriptPubKey": self.nodes[0].validateaddress(self.ms_address)["scriptPubKey"], + "redeemScript": cms["redeemScript"]} + self.wit_ms_address = wms['address'] self.coinbase_blocks = self.generate(self.nodes[0], 2) # block height = 2 coinbase_txid = [] @@ -82,16 +91,23 @@ def run_test(self): self.lastblocktime = int(time.time()) + self.lastblockheight self.log.info(f"Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [{COINBASE_MATURITY + 3}]") - test1txs = [create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, amount=49)] + test1txs = [self.create_transaction(txid=coinbase_txid[0], addr=self.ms_address, amount=49, + privkey=self.nodes[0].get_deterministic_priv_key().key)] txid1 = self.nodes[0].sendrawtransaction(test1txs[0].serialize_with_witness().hex(), 0) - test1txs.append(create_transaction(self.nodes[0], txid1, self.ms_address, amount=48)) + test1txs.append(self.create_transaction(txid=txid1, input_details=ms_unlock_details, + addr=self.ms_address, amount=48, + privkey=self.privkey)) txid2 = self.nodes[0].sendrawtransaction(test1txs[1].serialize_with_witness().hex(), 0) - test1txs.append(create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, amount=49)) + test1txs.append(self.create_transaction(txid=coinbase_txid[1], + addr=self.wit_ms_address, amount=49, + privkey=self.nodes[0].get_deterministic_priv_key().key)) txid3 = self.nodes[0].sendrawtransaction(test1txs[2].serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], test1txs, accept=True) self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") - test2tx = create_transaction(self.nodes[0], txid2, self.ms_address, amount=47) + test2tx = self.create_transaction(txid=txid2, input_details=ms_unlock_details, + addr=self.ms_address, amount=47, + privkey=self.privkey) invalidate_nulldummy_tx(test2tx) assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test2tx.serialize_with_witness().hex(), 0) @@ -99,14 +115,19 @@ def run_test(self): self.block_submit(self.nodes[0], [test2tx], accept=True) self.log.info("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation") - test4tx = create_transaction(self.nodes[0], test2tx.hash, self.address, amount=46) + test4tx = self.create_transaction(txid=test2tx.hash, input_details=ms_unlock_details, + addr=getnewdestination()[2], amount=46, + privkey=self.privkey) test6txs = [CTransaction(test4tx)] invalidate_nulldummy_tx(test4tx) assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test4tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test4tx], accept=False) self.log.info("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") - test5tx = create_transaction(self.nodes[0], txid3, self.wit_address, amount=48) + test5tx = self.create_transaction(txid=txid3, input_details={"scriptPubKey": test1txs[2].vout[0].scriptPubKey.hex(), + "amount": 49, "witnessScript": wms["redeemScript"]}, + addr=getnewdestination(address_type='p2sh-segwit')[2], amount=48, + privkey=self.privkey) test6txs.append(CTransaction(test5tx)) test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01' assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test5tx.serialize_with_witness().hex(), 0) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6a44f9d21d413..997b062cae06d 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -244,8 +244,7 @@ 'rpc_generate.py', 'wallet_balance.py --legacy-wallet', 'wallet_balance.py --descriptors', - 'feature_nulldummy.py --legacy-wallet', - 'feature_nulldummy.py --descriptors', + 'feature_nulldummy.py', 'mempool_accept.py', 'mempool_expiry.py', 'wallet_import_rescan.py --legacy-wallet', From 50ba6697f33b44e475ed65137f7ff0444f6c4ca9 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Fri, 24 Jun 2022 18:04:48 +0530 Subject: [PATCH 035/457] remove unused functions the functions `create_transaction()` and `create_raw_transaction()` were no longer used hence removed. --- test/functional/test_framework/blocktools.py | 24 -------------------- 1 file changed, 24 deletions(-) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 40fcbf7761752..574ea10356a10 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -162,30 +162,6 @@ def create_tx_with_script(prevtx, n, script_sig=b"", *, amount, script_pub_key=C tx.calc_sha256() return tx -def create_transaction(node, txid, to_address, *, amount): - """ Return signed transaction spending the first output of the - input txid. Note that the node must have a wallet that can - sign for the output that is being spent. - """ - raw_tx = create_raw_transaction(node, txid, to_address, amount=amount) - tx = tx_from_hex(raw_tx) - return tx - -def create_raw_transaction(node, txid, to_address, *, amount): - """ Return raw signed transaction spending the first output of the - input txid. Note that the node must have a wallet that can sign - for the output that is being spent. - """ - psbt = node.createpsbt(inputs=[{"txid": txid, "vout": 0}], outputs={to_address: amount}) - for _ in range(2): - for w in node.listwallets(): - wrpc = node.get_wallet_rpc(w) - signed_psbt = wrpc.walletprocesspsbt(psbt) - psbt = signed_psbt['psbt'] - final_psbt = node.finalizepsbt(psbt) - assert_equal(final_psbt["complete"], True) - return final_psbt['hex'] - def get_legacy_sigopcount_block(block, accurate=True): count = 0 for tx in block.vtx: From 2ef5294a5bb68ceb3797d2638567a172cc21699f Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Fri, 22 Apr 2022 11:11:39 +0200 Subject: [PATCH 036/457] rpc: add RPCTypeCheck for getblockfrompeer inputs --- src/rpc/blockchain.cpp | 5 +++++ test/functional/rpc_getblockfrompeer.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f46e5e9feff3b..659e9dcd6a14f 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -439,6 +439,11 @@ static RPCHelpMan getblockfrompeer() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + RPCTypeCheck(request.params, { + UniValue::VSTR, // blockhash + UniValue::VNUM, // peer_id + }); + const NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); PeerManager& peerman = EnsurePeerman(node); diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index cc9ccf306628d..a379132db4e42 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -51,8 +51,8 @@ def run_test(self): self.log.info("Arguments must be valid") assert_raises_rpc_error(-8, "hash must be of length 64 (not 4, for '1234')", self.nodes[0].getblockfrompeer, "1234", peer_0_peer_1_id) - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].getblockfrompeer, 1234, peer_0_peer_1_id) - assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].getblockfrompeer, short_tip, "0") + assert_raises_rpc_error(-3, "Expected type string, got number", self.nodes[0].getblockfrompeer, 1234, peer_0_peer_1_id) + assert_raises_rpc_error(-3, "Expected type number, got string", self.nodes[0].getblockfrompeer, short_tip, "0") self.log.info("We must already have the header") assert_raises_rpc_error(-1, "Block header missing", self.nodes[0].getblockfrompeer, "00" * 32, 0) From e7a3f698b5f3f7dde8632c4911abd4e5bc1f06cb Mon Sep 17 00:00:00 2001 From: w0xlt <94266259+w0xlt@users.noreply.github.com> Date: Thu, 16 Dec 2021 18:22:09 -0300 Subject: [PATCH 037/457] gui: Add Wallet Restore in the GUI Co-authored-by: Shashwat Vangani <85434418+shaavan@users.noreply.github.com> Co-authored-by: furszy --- src/qt/bitcoingui.cpp | 33 +++++++++++++++++++++++++++- src/qt/bitcoingui.h | 1 + src/qt/walletcontroller.cpp | 43 +++++++++++++++++++++++++++++++++++++ src/qt/walletcontroller.h | 20 +++++++++++++++++ 4 files changed, 96 insertions(+), 1 deletion(-) diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index bfcdf6f3164e9..0f05ef2a3bb56 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -348,6 +349,12 @@ void BitcoinGUI::createActions() m_create_wallet_action->setEnabled(false); m_create_wallet_action->setStatusTip(tr("Create a new wallet")); + //: Name of the menu item that restores wallet from a backup file. + m_restore_wallet_action = new QAction(tr("Restore Wallet…"), this); + m_restore_wallet_action->setEnabled(false); + //: Status tip for Restore Wallet menu item + m_restore_wallet_action->setStatusTip(tr("Restore a wallet from a backup file")); + m_close_all_wallets_action = new QAction(tr("Close All Wallets…"), this); m_close_all_wallets_action->setStatusTip(tr("Close all wallets")); @@ -412,6 +419,27 @@ void BitcoinGUI::createActions() action->setEnabled(false); } }); + connect(m_restore_wallet_action, &QAction::triggered, [this] { + //: Name of the wallet data file format. + QString name_data_file = tr("Wallet Data"); + + //: The title for Restore Wallet File Windows + QString title_windows = tr("Load Wallet Backup"); + + QString backup_file = GUIUtil::getOpenFileName(this, title_windows, QString(), name_data_file + QLatin1String(" (*.dat)"), nullptr); + if (backup_file.isEmpty()) return; + + bool wallet_name_ok; + //: Title of the Restore Wallet input dialog (where the wallet name is entered) + QString wallet_name = QInputDialog::getText(this, tr("Restore Name"), tr("Wallet Name:"), QLineEdit::Normal, "", &wallet_name_ok); + if (!wallet_name_ok || wallet_name.isEmpty()) return; + + auto activity = new RestoreWalletActivity(m_wallet_controller, this); + connect(activity, &RestoreWalletActivity::restored, this, &BitcoinGUI::setCurrentWallet, Qt::QueuedConnection); + + auto backup_file_path = fs::PathFromString(backup_file.toStdString()); + activity->restore(backup_file_path, wallet_name.toStdString()); + }); connect(m_close_wallet_action, &QAction::triggered, [this] { m_wallet_controller->closeWallet(walletFrame->currentWalletModel(), this); }); @@ -450,8 +478,10 @@ void BitcoinGUI::createMenuBar() file->addAction(m_close_wallet_action); file->addAction(m_close_all_wallets_action); file->addSeparator(); - file->addAction(openAction); file->addAction(backupWalletAction); + file->addAction(m_restore_wallet_action); + file->addSeparator(); + file->addAction(openAction); file->addAction(signMessageAction); file->addAction(verifyMessageAction); file->addAction(m_load_psbt_action); @@ -642,6 +672,7 @@ void BitcoinGUI::setWalletController(WalletController* wallet_controller) m_create_wallet_action->setEnabled(true); m_open_wallet_action->setEnabled(true); m_open_wallet_action->setMenu(m_open_wallet_menu); + m_restore_wallet_action->setEnabled(true); GUIUtil::ExceptionSafeConnect(wallet_controller, &WalletController::walletAdded, this, &BitcoinGUI::addWallet); connect(wallet_controller, &WalletController::walletRemoved, this, &BitcoinGUI::removeWallet); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 6272d5d947fc8..b2e13245e1d15 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -157,6 +157,7 @@ class BitcoinGUI : public QMainWindow QAction* m_create_wallet_action{nullptr}; QAction* m_open_wallet_action{nullptr}; QMenu* m_open_wallet_menu{nullptr}; + QAction* m_restore_wallet_action{nullptr}; QAction* m_close_wallet_action{nullptr}; QAction* m_close_all_wallets_action{nullptr}; QAction* m_wallet_selector_label_action = nullptr; diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index d27ddf1aba52b..11140c5da973f 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -373,3 +373,46 @@ void LoadWalletsActivity::load() QTimer::singleShot(0, this, [this] { Q_EMIT finished(); }); }); } + +RestoreWalletActivity::RestoreWalletActivity(WalletController* wallet_controller, QWidget* parent_widget) + : WalletControllerActivity(wallet_controller, parent_widget) +{ +} + +void RestoreWalletActivity::restore(const fs::path& backup_file, const std::string& wallet_name) +{ + QString name = QString::fromStdString(wallet_name); + + showProgressDialog( + //: Title of progress window which is displayed when wallets are being restored. + tr("Restore Wallet"), + /*: Descriptive text of the restore wallets progress window which indicates to + the user that wallets are currently being restored.*/ + tr("Restoring Wallet %1…").arg(name.toHtmlEscaped())); + + QTimer::singleShot(0, worker(), [this, backup_file, wallet_name] { + std::unique_ptr wallet = node().walletLoader().restoreWallet(backup_file, wallet_name, m_error_message, m_warning_message); + + if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(std::move(wallet)); + + QTimer::singleShot(0, this, &RestoreWalletActivity::finish); + }); +} + +void RestoreWalletActivity::finish() +{ + if (!m_error_message.empty()) { + //: Title of message box which is displayed when the wallet could not be restored. + QMessageBox::critical(m_parent_widget, tr("Restore wallet failed"), QString::fromStdString(m_error_message.translated)); + } else if (!m_warning_message.empty()) { + //: Title of message box which is displayed when the wallet is restored with some warning. + QMessageBox::warning(m_parent_widget, tr("Restore wallet warning"), QString::fromStdString(Join(m_warning_message, Untranslated("\n")).translated)); + } else { + //: Title of message box which is displayed when the wallet is successfully restored. + QMessageBox::information(m_parent_widget, tr("Restore wallet message"), QString::fromStdString(Untranslated("Wallet restored successfully \n").translated)); + } + + if (m_wallet_model) Q_EMIT restored(m_wallet_model); + + Q_EMIT finished(); +} diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index 24f85c258c805..fcd65756c67c7 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -33,6 +33,10 @@ class Node; class Wallet; } // namespace interfaces +namespace fs { +class path; +} + class AskPassphraseDialog; class CreateWalletActivity; class CreateWalletDialog; @@ -155,4 +159,20 @@ class LoadWalletsActivity : public WalletControllerActivity void load(); }; +class RestoreWalletActivity : public WalletControllerActivity +{ + Q_OBJECT + +public: + RestoreWalletActivity(WalletController* wallet_controller, QWidget* parent_widget); + + void restore(const fs::path& backup_file, const std::string& wallet_name); + +Q_SIGNALS: + void restored(WalletModel* wallet_model); + +private: + void finish(); +}; + #endif // BITCOIN_QT_WALLETCONTROLLER_H From a94659c84ee10ac5915eb5a6b654435183d88521 Mon Sep 17 00:00:00 2001 From: S3RK <1466284+S3RK@users.noreply.github.com> Date: Mon, 27 Jun 2022 09:09:09 +0200 Subject: [PATCH 038/457] wallet: replace GetTxSpendSize with CalculateMaximumSignedInputSize --- src/bench/coin_selection.cpp | 3 ++- src/wallet/spend.cpp | 12 ++++-------- src/wallet/spend.h | 3 --- src/wallet/test/coinselector_tests.cpp | 3 ++- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index b2958bcc9f0f9..f8914e762ce93 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -58,7 +58,8 @@ static void CoinSelection(benchmark::Bench& bench) // Create coins std::vector coins; for (const auto& wtx : wtxs) { - coins.emplace_back(COutPoint(wtx->GetHash(), 0), wtx->tx->vout.at(0), /*depth=*/6 * 24, GetTxSpendSize(wallet, *wtx, 0), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); + const auto txout = wtx->tx->vout.at(0); + coins.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, false), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); } const CoinEligibilityFilter filter_standard(1, 6, 0); diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 5799a9ff2ab1e..5d9cecc3984e1 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -27,11 +27,6 @@ using interfaces::FoundBlock; namespace wallet { static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; -int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig) -{ - return CalculateMaximumSignedInputSize(wtx.tx->vout[out], &wallet, use_max_sig); -} - int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) { CMutableTransaction txn; @@ -198,7 +193,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, // Filter by spendable outputs only if (!spendable && only_spendable) continue; - int input_bytes = GetTxSpendSize(wallet, wtx, i, (coinControl && coinControl->fAllowWatchOnly)); + int input_bytes = CalculateMaximumSignedInputSize(output, provider.get(), coinControl && coinControl->fAllowWatchOnly); result.coins.emplace_back(outpoint, output, nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me, feerate); result.total_amount += output.nValue; @@ -289,8 +284,9 @@ std::map> ListCoins(const CWallet& wallet) ) { CTxDestination address; if (ExtractDestination(FindNonChangeParentOutput(wallet, *wtx.tx, output.n).scriptPubKey, address)) { + const auto out = wtx.tx->vout.at(output.n); result[address].emplace_back( - COutPoint(wtx.GetHash(), output.n), wtx.tx->vout.at(output.n), depth, GetTxSpendSize(wallet, wtx, output.n), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); + COutPoint(wtx.GetHash(), output.n), out, depth, CalculateMaximumSignedInputSize(out, &wallet, false), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); } } } @@ -447,7 +443,7 @@ std::optional SelectCoins(const CWallet& wallet, const std::vec if (ptr_wtx->tx->vout.size() <= outpoint.n) { return std::nullopt; } - input_bytes = GetTxSpendSize(wallet, *ptr_wtx, outpoint.n, false); + input_bytes = CalculateMaximumSignedInputSize(ptr_wtx->tx->vout[outpoint.n], &wallet, false); txout = ptr_wtx->tx->vout.at(outpoint.n); } else { // The input is external. We did not find the tx in mapWallet. diff --git a/src/wallet/spend.h b/src/wallet/spend.h index cba42d6fae5c3..a7d91eb9c417a 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -16,9 +16,6 @@ namespace wallet { /** Get the marginal bytes if spending the specified output from this transaction. * use_max_sig indicates whether to use the maximum sized, 72 byte signature when calculating the * size of the input spend. This should only be set when watch-only outputs are allowed */ -int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig = false); - -//Get the marginal bytes of spending the specified output int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index d6f47e99548af..0664b64244bbd 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -86,7 +86,8 @@ static void add_coin(std::vector& coins, CWallet& wallet, const CAmount auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateInactive{})); assert(ret.second); CWalletTx& wtx = (*ret.first).second; - coins.emplace_back(COutPoint(wtx.GetHash(), nInput), wtx.tx->vout.at(nInput), nAge, GetTxSpendSize(wallet, wtx, nInput), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); + const auto txout = wtx.tx->vout.at(nInput); + coins.emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, false), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); } /** Check if SelectionResult a is equivalent to SelectionResult b. From d54c5c8b1b1a38b5b38e6878aea0fa8d6c1ad7e9 Mon Sep 17 00:00:00 2001 From: S3RK <1466284+S3RK@users.noreply.github.com> Date: Mon, 27 Jun 2022 09:11:09 +0200 Subject: [PATCH 039/457] wallet: use CCoinControl to estimate signature size --- src/bench/coin_selection.cpp | 2 +- src/wallet/spend.cpp | 18 +++++++++--------- src/wallet/spend.h | 14 +++++--------- src/wallet/test/coinselector_tests.cpp | 4 ++-- src/wallet/wallet.cpp | 14 +++++++------- src/wallet/wallet.h | 2 +- 6 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index f8914e762ce93..eaefb9b63a884 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -59,7 +59,7 @@ static void CoinSelection(benchmark::Bench& bench) std::vector coins; for (const auto& wtx : wtxs) { const auto txout = wtx->tx->vout.at(0); - coins.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, false), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); + coins.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); } const CoinEligibilityFilter filter_standard(1, 6, 0); diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 5d9cecc3984e1..583c021b8f9fe 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -27,20 +27,20 @@ using interfaces::FoundBlock; namespace wallet { static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; -int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) +int CalculateMaximumSignedInputSize(const CTxOut& txout, const COutPoint outpoint, const SigningProvider* provider, const CCoinControl* coin_control) { CMutableTransaction txn; - txn.vin.push_back(CTxIn(COutPoint())); - if (!provider || !DummySignInput(*provider, txn.vin[0], txout, use_max_sig)) { + txn.vin.push_back(CTxIn(outpoint)); + if (!provider || !DummySignInput(*provider, txn.vin[0], txout, coin_control)) { return -1; } return GetVirtualTransactionInputSize(txn.vin[0]); } -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, bool use_max_sig) +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, const CCoinControl* coin_control) { const std::unique_ptr provider = wallet->GetSolvingProvider(txout.scriptPubKey); - return CalculateMaximumSignedInputSize(txout, provider.get(), use_max_sig); + return CalculateMaximumSignedInputSize(txout, COutPoint(), provider.get(), coin_control); } // txouts needs to be in the order of tx.vin @@ -193,7 +193,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, // Filter by spendable outputs only if (!spendable && only_spendable) continue; - int input_bytes = CalculateMaximumSignedInputSize(output, provider.get(), coinControl && coinControl->fAllowWatchOnly); + int input_bytes = CalculateMaximumSignedInputSize(output, COutPoint(), provider.get(), coinControl); result.coins.emplace_back(outpoint, output, nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me, feerate); result.total_amount += output.nValue; @@ -286,7 +286,7 @@ std::map> ListCoins(const CWallet& wallet) if (ExtractDestination(FindNonChangeParentOutput(wallet, *wtx.tx, output.n).scriptPubKey, address)) { const auto out = wtx.tx->vout.at(output.n); result[address].emplace_back( - COutPoint(wtx.GetHash(), output.n), out, depth, CalculateMaximumSignedInputSize(out, &wallet, false), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); + COutPoint(wtx.GetHash(), output.n), out, depth, CalculateMaximumSignedInputSize(out, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); } } } @@ -443,14 +443,14 @@ std::optional SelectCoins(const CWallet& wallet, const std::vec if (ptr_wtx->tx->vout.size() <= outpoint.n) { return std::nullopt; } - input_bytes = CalculateMaximumSignedInputSize(ptr_wtx->tx->vout[outpoint.n], &wallet, false); txout = ptr_wtx->tx->vout.at(outpoint.n); + input_bytes = CalculateMaximumSignedInputSize(txout, &wallet, &coin_control); } else { // The input is external. We did not find the tx in mapWallet. if (!coin_control.GetExternalOutput(outpoint, txout)) { return std::nullopt; } - input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /*use_max_sig=*/true); + input_bytes = CalculateMaximumSignedInputSize(txout, outpoint, &coin_control.m_external_provider, &coin_control); } // If available, override calculated size with coin control specified size if (coin_control.HasInputWeight(outpoint)) { diff --git a/src/wallet/spend.h b/src/wallet/spend.h index a7d91eb9c417a..21f0095e77588 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -14,20 +14,16 @@ namespace wallet { /** Get the marginal bytes if spending the specified output from this transaction. - * use_max_sig indicates whether to use the maximum sized, 72 byte signature when calculating the - * size of the input spend. This should only be set when watch-only outputs are allowed */ -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); -int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); - + * Use CoinControl to determine whether to expect signature grinding when calculating the size of the input spend. */ +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, const CCoinControl* coin_control = nullptr); +int CalculateMaximumSignedInputSize(const CTxOut& txout, const COutPoint outpoint, const SigningProvider* pwallet, const CCoinControl* coin_control = nullptr); struct TxSize { int64_t vsize{-1}; int64_t weight{-1}; }; -/** Calculate the size of the transaction assuming all signatures are max size -* Use DummySignatureCreator, which inserts 71 byte signatures everywhere. -* NOTE: this requires that all inputs must be in mapWallet (eg the tx should -* be AllInputsMine). */ +/** Calculate the size of the transaction using CoinControl to determine + * whether to expect signature grinding when calculating the size of the input spend. */ TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector& txouts, const CCoinControl* coin_control = nullptr); TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const CCoinControl* coin_control = nullptr) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 0664b64244bbd..e85bf58d4e3a2 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -86,8 +86,8 @@ static void add_coin(std::vector& coins, CWallet& wallet, const CAmount auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateInactive{})); assert(ret.second); CWalletTx& wtx = (*ret.first).second; - const auto txout = wtx.tx->vout.at(nInput); - coins.emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, false), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); + const auto& txout = wtx.tx->vout.at(nInput); + coins.emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); } /** Check if SelectionResult a is equivalent to SelectionResult b. diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index d0b093bbb7c91..b90ef98850e8d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1504,13 +1504,16 @@ bool CWallet::AddWalletFlags(uint64_t flags) } // Helper for producing a max-sized low-S low-R signature (eg 71 bytes) -// or a max-sized low-S signature (e.g. 72 bytes) if use_max_sig is true -bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig) +// or a max-sized low-S signature (e.g. 72 bytes) depending on coin_control +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, const CCoinControl* coin_control) { // Fill in dummy signatures for fee calculation. const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; + // Use max sig if watch only inputs were used or if this particular input is an external input + // to ensure a sufficient fee is attained for the requested feerate. + const bool use_max_sig = coin_control && (coin_control->fAllowWatchOnly || coin_control->IsExternalSelected(tx_in.prevout)); if (!ProduceSignature(provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { return false; } @@ -1577,12 +1580,9 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector nIn++; continue; } - // Use max sig if watch only inputs were used or if this particular input is an external input - // to ensure a sufficient fee is attained for the requested feerate. - const bool use_max_sig = coin_control && (coin_control->fAllowWatchOnly || coin_control->IsExternalSelected(txin.prevout)); const std::unique_ptr provider = GetSolvingProvider(txout.scriptPubKey); - if (!provider || !DummySignInput(*provider, txin, txout, use_max_sig)) { - if (!coin_control || !DummySignInput(coin_control->m_external_provider, txin, txout, use_max_sig)) { + if (!provider || !DummySignInput(*provider, txin, txout, coin_control)) { + if (!coin_control || !DummySignInput(coin_control->m_external_provider, txin, txout, coin_control)) { return false; } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index bf42b3da9ec51..1cf91d6ef665b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -932,7 +932,7 @@ bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); //! Remove wallet name from persistent configuration so it will not be loaded on startup. bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); -bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig); +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, const CCoinControl* coin_control = nullptr); bool FillInputToWeight(CTxIn& txin, int64_t target_weight); } // namespace wallet From 1d4d711de2d9beb20339da92dea298850da354b1 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 17 Mar 2022 16:03:57 -0400 Subject: [PATCH 040/457] guix: Map all guix store prefixes to /usr Without ffile-prefix-map, the debug symbols will contain paths for the guix store which will include the hashes of each package. However, the hash for the same package will differ when on different architectures. In order to be reproducible regardless of the architecture used to build the package, map all guix store prefixes to something fixed, e.g. /usr. --- contrib/guix/libexec/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index ad3129184c9aa..28cad05013527 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -223,6 +223,7 @@ CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disab # CFLAGS HOST_CFLAGS="-O2 -g" +HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) case "$HOST" in *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; *mingw*) HOST_CFLAGS+=" -fno-ident" ;; From 54faac968971131161f7d6c8def01af1aff4c6b6 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Fri, 18 Mar 2022 10:50:35 -0400 Subject: [PATCH 041/457] guix: Remove guix store paths from glibc Without ffile-prefix-map, the debug symbols will contain paths for the guix store which will include the hashes of each package. However, the hash for the same package will differ when on different architectures. In order to be reproducible regardless of the architecture used to build the package, map all guix store prefixes to something fixed, e.g. /usr. We might be able to drop this in favour of using --with-nonshared-cflags when we being using newer versions of glibc. --- contrib/guix/manifest.scm | 6 +++-- .../guix/patches/glibc-2.24-guix-prefix.patch | 25 +++++++++++++++++++ .../guix/patches/glibc-2.27-guix-prefix.patch | 25 +++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 contrib/guix/patches/glibc-2.24-guix-prefix.patch create mode 100644 contrib/guix/patches/glibc-2.27-guix-prefix.patch diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 36d8dddab6153..afc9ae8d26721 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -549,7 +549,8 @@ inspecting signatures in Mach-O binaries.") "glibc-versioned-locpath.patch" "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" "glibc-2.24-no-build-time-cxx-header-run.patch" - "glibc-2.24-fcommon.patch")))))) + "glibc-2.24-fcommon.patch" + "glibc-2.24-guix-prefix.patch")))))) (define-public glibc-2.27/bitcoin-patched (package @@ -566,7 +567,8 @@ inspecting signatures in Mach-O binaries.") "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca")) (patches (search-our-patches "glibc-ldd-x86_64.patch" "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" - "glibc-2.27-dont-redefine-nss-database.patch")))))) + "glibc-2.27-dont-redefine-nss-database.patch" + "glibc-2.27-guix-prefix.patch")))))) (packages->manifest (append diff --git a/contrib/guix/patches/glibc-2.24-guix-prefix.patch b/contrib/guix/patches/glibc-2.24-guix-prefix.patch new file mode 100644 index 0000000000000..cba2f59a8dcdb --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-guix-prefix.patch @@ -0,0 +1,25 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +We might be able to drop this in favour of using --with-nonshared-cflags +when we being using newer versions of glibc. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -950,6 +950,10 @@ object-suffixes-for-libc += .oS + # shared objects. We don't want to use CFLAGS-os because users may, for + # example, make that processor-specific. + CFLAGS-.oS = $(CFLAGS-.o) $(PIC-ccflag) ++ ++# Map Guix store paths to /usr ++CFLAGS-.oS += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -fdebug-prefix-map={}=/usr" \;` ++ + CPPFLAGS-.oS = $(CPPFLAGS-.o) -DPIC -DLIBC_NONSHARED=1 + libtype.oS = lib%_nonshared.a + endif +-- +2.35.1 + diff --git a/contrib/guix/patches/glibc-2.27-guix-prefix.patch b/contrib/guix/patches/glibc-2.27-guix-prefix.patch new file mode 100644 index 0000000000000..cdb3971f7a049 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-guix-prefix.patch @@ -0,0 +1,25 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +We might be able to drop this in favour of using --with-nonshared-cflags +when we being using newer versions of glibc. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -992,6 +992,10 @@ object-suffixes := + CPPFLAGS-.o = $(pic-default) + # libc.a must be compiled with -fPIE/-fpie for static PIE. + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) ++ ++# Map Guix store paths to /usr ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -fdebug-prefix-map={}=/usr" \;` ++ + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) +-- +2.35.1 + From 7f2450871b3ea0b4d02d56bd2ca365fcc25cf90e Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:02:55 -0500 Subject: [PATCH 042/457] Move handling of unconnecting headers into own function --- src/net_processing.cpp | 91 +++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 28 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index a55618072124a..6965695ad92a8 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -560,6 +560,11 @@ class PeerManagerImpl final : public PeerManager const std::vector& headers, bool via_compact_block) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + /** Various helpers for headers processing, invoked by ProcessHeadersMessage() */ + /** Deal with state tracking and headers sync for peers that send the + * occasional non-connecting header (this can happen due to BIP 130 headers + * announcements for blocks interacting with the 2hr (MAX_FUTURE_BLOCK_TIME) rule). */ + void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector& headers); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2194,6 +2199,48 @@ void PeerManagerImpl::SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlo m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); } +/** + * Special handling for unconnecting headers that might be part of a block + * announcement. + * + * We'll send a getheaders message in response to try to connect the chain. + * + * The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + * don't connect before given DoS points. + * + * Once a headers message is received that is valid and does connect, + * nUnconnectingHeaders gets reset back to 0. + */ +void PeerManagerImpl::HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, + const std::vector& headers) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + + nodestate->nUnconnectingHeaders++; + + // Try to fill in the missing headers. + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + m_chainman.m_best_header->nHeight, + pfrom.GetId(), nodestate->nUnconnectingHeaders); + + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); + + // The peer may just be broken, so periodically assign DoS points if this + // condition persists. + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(peer, 20, strprintf("%d non-connecting headers", nodestate->nUnconnectingHeaders)); + } +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2208,36 +2255,24 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, bool received_new_header = false; const CBlockIndex *pindexLast = nullptr; - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom.GetId()); - // If this looks like it could be a block announcement (nCount <= - // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that - // don't connect: - // - Send a getheaders message in response to try to connect the chain. - // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that - // don't connect before giving DoS points - // - Once a headers message is received that is valid and does connect, - // nUnconnectingHeaders gets reset back to 0. - if (!m_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) && nCount <= MAX_BLOCKS_TO_ANNOUNCE) { - nodestate->nUnconnectingHeaders++; - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - m_chainman.m_best_header->nHeight, - pfrom.GetId(), nodestate->nUnconnectingHeaders); - // Set hashLastUnknownBlock for this peer, so that if we - // eventually get the headers - even from a different peer - - // we can use this peer to download. - UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); - - if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(peer, 20, strprintf("%d non-connecting headers", nodestate->nUnconnectingHeaders)); - } - return; + // Do these headers connect to something in our block index? + bool headers_connect_blockindex{WITH_LOCK(::cs_main, return m_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) != nullptr)}; + + if (!headers_connect_blockindex) { + if (nCount <= MAX_BLOCKS_TO_ANNOUNCE) { + // If this looks like it could be a BIP 130 block announcement, use + // special logic for handling headers that don't connect, as this + // could be benign. + HandleFewUnconnectingHeaders(pfrom, peer, headers); + } else { + Misbehaving(peer, 10, "invalid header received"); } + return; + } + + { + LOCK(cs_main); uint256 hashLastBlock; for (const CBlockHeader& header : headers) { From 9492e93bf9f4a841bf43ca4b593871c0863d5b63 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:11:30 -0500 Subject: [PATCH 043/457] Add helper function for checking header continuity --- src/net_processing.cpp | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6965695ad92a8..75e76d28cc13a 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -565,6 +565,8 @@ class PeerManagerImpl final : public PeerManager * occasional non-connecting header (this can happen due to BIP 130 headers * announcements for blocks interacting with the 2hr (MAX_FUTURE_BLOCK_TIME) rule). */ void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector& headers); + /** Return true if the headers connect to each other, false otherwise */ + bool CheckHeadersAreContinuous(const std::vector& headers) const; void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2241,6 +2243,18 @@ void PeerManagerImpl::HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, } } +bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector& headers) const +{ + uint256 hashLastBlock; + for (const CBlockHeader& header : headers) { + if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { + return false; + } + hashLastBlock = header.GetHash(); + } + return true; +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2271,21 +2285,18 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } + // At this point, the headers connect to something in our block index. + if (!CheckHeadersAreContinuous(headers)) { + Misbehaving(peer, 20, "non-continuous headers sequence"); + return; + } + { LOCK(cs_main); - uint256 hashLastBlock; - for (const CBlockHeader& header : headers) { - if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(peer, 20, "non-continuous headers sequence"); - return; - } - hashLastBlock = header.GetHash(); - } - // If we don't have the last header, then they'll have given us // something new (if these headers are valid). - if (!m_chainman.m_blockman.LookupBlockIndex(hashLastBlock)) { + if (!m_chainman.m_blockman.LookupBlockIndex(headers.back().GetHash())) { received_new_header = true; } } From bf8ea6df75749c27f753b562c4724b3f8d263ad4 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:15:20 -0500 Subject: [PATCH 044/457] Move additional headers fetching to own function Also moves the call to happen directly after validation of a headers message (rather than mixed in with other state updates for the peer), and removes an incorrect comment in favor of one that explains why headers sync must continue from the last header a peer has sent. --- src/net_processing.cpp | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 75e76d28cc13a..795851c4910e6 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -567,6 +567,8 @@ class PeerManagerImpl final : public PeerManager void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector& headers); /** Return true if the headers connect to each other, false otherwise */ bool CheckHeadersAreContinuous(const std::vector& headers) const; + /** Request further headers from this peer from a given block header */ + void FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2255,6 +2257,25 @@ bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector& return true; } +/* + * Continue fetching headers from a given point. + * pindexLast should be the last header we learned from a peer in their prior + * headers message. + * + * This is used for headers sync with a peer; even if pindexLast is an ancestor + * of a known chain (such as our tip) we don't yet know where the peer's chain + * might fork from what we know, so we continue exactly from where the peer + * left off. + */ +void PeerManagerImpl::FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", + pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2309,6 +2330,12 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, } } + // Consider fetching more headers. + if (nCount == MAX_HEADERS_RESULTS) { + // Headers message had its maximum size; the peer may have more headers. + FetchMoreHeaders(pfrom, pindexLast, peer); + } + { LOCK(cs_main); CNodeState *nodestate = State(pfrom.GetId()); @@ -2328,15 +2355,6 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, nodestate->m_last_block_announcement = GetTime(); } - if (nCount == MAX_HEADERS_RESULTS) { - // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of m_chainman.ActiveChain().Tip or m_chainman.m_best_header, continue - // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", - pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); - } - // If this set of headers is valid and ends in a block with at least as // much work as our tip, download as much as possible. if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { From 29c45185223441943ab610e62937a118c7c3a5b2 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:33:32 -0500 Subject: [PATCH 045/457] Move headers-direct-fetch logic into own function --- src/net_processing.cpp | 126 +++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 54 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 795851c4910e6..189556e03e609 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -569,6 +569,8 @@ class PeerManagerImpl final : public PeerManager bool CheckHeadersAreContinuous(const std::vector& headers) const; /** Request further headers from this peer from a given block header */ void FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer); + /** Potentially fetch blocks from this peer upon receipt of new headers tip */ + void HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2276,6 +2278,73 @@ void PeerManagerImpl::FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLa m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); } +/* + * Given a new headers tip ending in pindexLast, potentially request blocks towards that tip. + * We require that the given tip have at least as much work as our tip, and for + * our current tip to be "close to synced" (see CanDirectFetch()). + */ +void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + + if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { + + std::vector vToFetch; + const CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !IsBlockRequested(pindexWalk->GetBlockHash()) && + (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || State(pfrom.GetId())->fHaveWitness)) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!m_chainman.ActiveChain().Contains(pindexWalk)) { + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + std::vector vGetData; + // Download as much as possible, from earliest to latest. + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + uint32_t nFetchFlags = GetFetchFlags(pfrom); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + BlockRequested(pfrom.GetId(), *pindex); + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom.GetId()); + } + if (vGetData.size() > 1) { + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (!m_ignore_incoming_txs && + nodestate->m_provides_cmpctblocks && + vGetData.size() == 1 && + mapBlocksInFlight.size() == 1 && + pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + } + } + } +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2355,60 +2424,9 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, nodestate->m_last_block_announcement = GetTime(); } - // If this set of headers is valid and ends in a block with at least as - // much work as our tip, download as much as possible. - if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { - std::vector vToFetch; - const CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. - while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !IsBlockRequested(pindexWalk->GetBlockHash()) && - (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || State(pfrom.GetId())->fHaveWitness)) { - // We don't have this block, and it's not yet in flight. - vToFetch.push_back(pindexWalk); - } - pindexWalk = pindexWalk->pprev; - } - // If pindexWalk still isn't on our main chain, we're looking at a - // very large reorg at a time we think we're close to caught up to - // the main chain -- this shouldn't really happen. Bail out on the - // direct fetch and rely on parallel download instead. - if (!m_chainman.ActiveChain().Contains(pindexWalk)) { - LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); - } else { - std::vector vGetData; - // Download as much as possible, from earliest to latest. - for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - // Can't download any more from this peer - break; - } - uint32_t nFetchFlags = GetFetchFlags(pfrom); - vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - BlockRequested(pfrom.GetId(), *pindex); - LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom.GetId()); - } - if (vGetData.size() > 1) { - LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); - } - if (vGetData.size() > 0) { - if (!m_ignore_incoming_txs && - nodestate->m_provides_cmpctblocks && - vGetData.size() == 1 && - mapBlocksInFlight.size() == 1 && - pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // In any case, we want to download using a compact block, not a regular one - vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); - } - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - } - } - } + // Consider immediately downloading blocks. + HeadersDirectFetchBlocks(pfrom, pindexLast); + // If we're in IBD, we want outbound peers that will serve us a useful // chain. Disconnect peers that are on chains with insufficient work. if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { From 2b341db731793844f12944363186edea23eabdeb Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:36:33 -0500 Subject: [PATCH 046/457] Move headers direct fetch to end of ProcessHeadersMessage --- src/net_processing.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 189556e03e609..2be5d5f50d286 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2424,9 +2424,6 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, nodestate->m_last_block_announcement = GetTime(); } - // Consider immediately downloading blocks. - HeadersDirectFetchBlocks(pfrom, pindexLast); - // If we're in IBD, we want outbound peers that will serve us a useful // chain. Disconnect peers that are on chains with insufficient work. if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { @@ -2462,6 +2459,9 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, } } + // Consider immediately downloading blocks. + HeadersDirectFetchBlocks(pfrom, pindexLast); + return; } From 6d95cd3e7444ebaaabb64a76783ea3551530f1d7 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:40:33 -0500 Subject: [PATCH 047/457] Move peer state updates from headers message into separate function --- src/net_processing.cpp | 117 ++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 53 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2be5d5f50d286..5f19f8b8ec599 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -571,6 +571,8 @@ class PeerManagerImpl final : public PeerManager void FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer); /** Potentially fetch blocks from this peer upon receipt of new headers tip */ void HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast); + /** Update peer state based on received headers message */ + void UpdatePeerStateForReceivedHeaders(CNode& pfrom, const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2345,6 +2347,67 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* } } +/** + * Given receipt of headers from a peer ending in pindexLast, along with + * whether that header was new and whether the headers message was full, + * update the state we keep for the peer. + */ +void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, + const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers) +{ + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom.GetId(), nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); + + // From here, pindexBestKnownBlock should be guaranteed to be non-null, + // because it is set in UpdateBlockAvailability. Some nullptr checks + // are still present, however, as belt-and-suspenders. + + if (received_new_header && pindexLast->nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + + // If we're in IBD, we want outbound peers that will serve us a useful + // chain. Disconnect peers that are on chains with insufficient work. + if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && !may_have_more_headers) { + // If the peer has no more headers to give us, then we know we have + // their tip. + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + // This peer has too little work on their headers chain to help + // us sync -- disconnect if it is an outbound disconnection + // candidate. + // Note: We compare their tip to nMinimumChainWork (rather than + // m_chainman.ActiveChain().Tip()) because we won't start block download + // until we have a headers chain that has at least + // nMinimumChainWork, even if a peer has a chain past our tip, + // as an anti-DoS measure. + if (pfrom.IsOutboundOrBlockRelayConn()) { + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); + pfrom.fDisconnect = true; + } + } + } + + // If this is an outbound full-relay peer, check to see if we should protect + // it from the bad/lagging chain logic. + // Note that outbound block-relay peers are excluded from this protection, and + // thus always subject to eviction under the bad/lagging chain logic. + // See ChainSyncTimeoutState. + if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { + if (m_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= m_chainman.ActiveChain().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); + nodestate->m_chain_sync.m_protect = true; + ++m_outbound_peers_with_protect_from_disconnect; + } + } +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector& headers, bool via_compact_block) @@ -2405,59 +2468,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, FetchMoreHeaders(pfrom, pindexLast, peer); } - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom.GetId()); - if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom.GetId(), nodestate->nUnconnectingHeaders); - } - nodestate->nUnconnectingHeaders = 0; - - assert(pindexLast); - UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); - - // From here, pindexBestKnownBlock should be guaranteed to be non-null, - // because it is set in UpdateBlockAvailability. Some nullptr checks - // are still present, however, as belt-and-suspenders. - - if (received_new_header && pindexLast->nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { - nodestate->m_last_block_announcement = GetTime(); - } - - // If we're in IBD, we want outbound peers that will serve us a useful - // chain. Disconnect peers that are on chains with insufficient work. - if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { - // When nCount < MAX_HEADERS_RESULTS, we know we have no more - // headers to fetch from this peer. - if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { - // This peer has too little work on their headers chain to help - // us sync -- disconnect if it is an outbound disconnection - // candidate. - // Note: We compare their tip to nMinimumChainWork (rather than - // m_chainman.ActiveChain().Tip()) because we won't start block download - // until we have a headers chain that has at least - // nMinimumChainWork, even if a peer has a chain past our tip, - // as an anti-DoS measure. - if (pfrom.IsOutboundOrBlockRelayConn()) { - LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); - pfrom.fDisconnect = true; - } - } - } - - // If this is an outbound full-relay peer, check to see if we should protect - // it from the bad/lagging chain logic. - // Note that outbound block-relay peers are excluded from this protection, and - // thus always subject to eviction under the bad/lagging chain logic. - // See ChainSyncTimeoutState. - if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { - if (m_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= m_chainman.ActiveChain().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { - LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); - nodestate->m_chain_sync.m_protect = true; - ++m_outbound_peers_with_protect_from_disconnect; - } - } - } + UpdatePeerStateForReceivedHeaders(pfrom, pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); // Consider immediately downloading blocks. HeadersDirectFetchBlocks(pfrom, pindexLast); From ffe87db247b19ffb8bfba329c5dd0be39ef5a53f Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 8 Feb 2022 17:43:38 -0500 Subject: [PATCH 048/457] Cleanup received_new_header calculation to use WITH_LOCK --- src/net_processing.cpp | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 5f19f8b8ec599..975ee16615d9e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2420,7 +2420,6 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } - bool received_new_header = false; const CBlockIndex *pindexLast = nullptr; // Do these headers connect to something in our block index? @@ -2444,15 +2443,9 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } - { - LOCK(cs_main); - - // If we don't have the last header, then they'll have given us - // something new (if these headers are valid). - if (!m_chainman.m_blockman.LookupBlockIndex(headers.back().GetHash())) { - received_new_header = true; - } - } + // If we don't have the last header, then this peer will have given us + // something new (if these headers are valid). + bool received_new_header{WITH_LOCK(::cs_main, return m_chainman.m_blockman.LookupBlockIndex(headers.back().GetHash()) == nullptr)}; BlockValidationState state; if (!m_chainman.ProcessNewBlockHeaders(headers, state, &pindexLast)) { From abf5d16c24cb08b0451bdbd4d1de63a12930e8f5 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Thu, 3 Mar 2022 14:48:38 -0500 Subject: [PATCH 049/457] Don't send getheaders message when another request is outstanding Change getheaders messages so that we wait up to 2 minutes for a response to a prior getheaders message before issuing a new one. Also change the handling of the getheaders message sent in response to a block INV, so that we no longer use the hashstop variable (including the hash stop will just mean that if our peer's headers chain is longer, then we won't learn it, so there's no benefit to using hashstop). Also, now respond to a getheaders during IBD with an empty headers message (rather than nothing) -- this better conforms to the intent of the new logic that it's better to not ignore a peer's getheaders message, even if you have nothing to give. This also avoids a lot of functional tests breaking. p2p_segwit.py is modified to use this same strategy, as the test logic (of expecting a getheaders after a block inv) would otherwise be broken. --- src/net_processing.cpp | 110 +++++++++++++++--------- test/functional/feature_minchainwork.py | 2 +- test/functional/p2p_segwit.py | 4 + 3 files changed, 72 insertions(+), 44 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 975ee16615d9e..fe1b06b18cdff 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -61,6 +61,8 @@ static constexpr auto UNCONDITIONAL_RELAY_DELAY = 2min; * Timeout = base + per_header * (expected number of headers) */ static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_BASE = 15min; static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1ms; +/** How long to wait for a peer to respond to a getheaders request */ +static constexpr auto HEADERS_RESPONSE_TIME{2min}; /** Protect at least this many outbound peers from disconnection due to slow/ * behind headers chain. */ @@ -355,6 +357,9 @@ struct Peer { /** Work queue of items requested by this peer **/ std::deque m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); + /** Time of the last getheaders message to this peer */ + std::atomic m_last_getheaders_timestamp{0s}; + Peer(NodeId id) : m_id{id} {} @@ -501,7 +506,7 @@ class PeerManagerImpl final : public PeerManager private: /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ - void ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ void EvictExtraOutboundPeers(std::chrono::seconds now) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -567,9 +572,12 @@ class PeerManagerImpl final : public PeerManager void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector& headers); /** Return true if the headers connect to each other, false otherwise */ bool CheckHeadersAreContinuous(const std::vector& headers) const; - /** Request further headers from this peer from a given block header */ - void FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer); - /** Potentially fetch blocks from this peer upon receipt of new headers tip */ + /** Request further headers from this peer with a given locator. + * We don't issue a getheaders message if we have a recent one outstanding. + * This returns true if a getheaders is actually sent, and false otherwise. + */ + bool MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer); + /** Potentially fetch blocks from this peer upon receipt of a new headers tip */ void HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast); /** Update peer state based on received headers message */ void UpdatePeerStateForReceivedHeaders(CNode& pfrom, const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers); @@ -2228,15 +2236,14 @@ void PeerManagerImpl::HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, CNodeState *nodestate = State(pfrom.GetId()); nodestate->nUnconnectingHeaders++; - // Try to fill in the missing headers. - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), peer)) { + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), m_chainman.m_best_header->nHeight, pfrom.GetId(), nodestate->nUnconnectingHeaders); - + } // Set hashLastUnknownBlock for this peer, so that if we // eventually get the headers - even from a different peer - // we can use this peer to download. @@ -2261,23 +2268,19 @@ bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector& return true; } -/* - * Continue fetching headers from a given point. - * pindexLast should be the last header we learned from a peer in their prior - * headers message. - * - * This is used for headers sync with a peer; even if pindexLast is an ancestor - * of a known chain (such as our tip) we don't yet know where the peer's chain - * might fork from what we know, so we continue exactly from where the peer - * left off. - */ -void PeerManagerImpl::FetchMoreHeaders(CNode& pfrom, const CBlockIndex *pindexLast, const Peer& peer) +bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer) { const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", - pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); + const auto current_time = GetTime(); + // Only allow a new getheaders message to go out if we don't have a recent + // one already in-flight + if (peer.m_last_getheaders_timestamp.load() < current_time - HEADERS_RESPONSE_TIME) { + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, locator, uint256())); + peer.m_last_getheaders_timestamp = current_time; + return true; + } + return false; } /* @@ -2458,7 +2461,10 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, // Consider fetching more headers. if (nCount == MAX_HEADERS_RESULTS) { // Headers message had its maximum size; the peer may have more headers. - FetchMoreHeaders(pfrom, pindexLast, peer); + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(pindexLast), peer)) { + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", + pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); + } } UpdatePeerStateForReceivedHeaders(pfrom, pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); @@ -3228,8 +3234,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (best_block != nullptr) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *best_block)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", m_chainman.m_best_header->nHeight, best_block->ToString(), pfrom.GetId()); + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *peer)) { + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", + m_chainman.m_best_header->nHeight, best_block->ToString(), + pfrom.GetId()); + } } return; @@ -3402,7 +3411,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // others. if (m_chainman.ActiveTip() == nullptr || (m_chainman.ActiveTip()->nChainWork < nMinimumChainWork && !pfrom.HasPermission(NetPermissionFlags::Download))) { - LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work\n", pfrom.GetId()); + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work; sending empty response\n", pfrom.GetId()); + // Just respond with an empty headers message, to tell the peer to + // go away but not treat us as unresponsive. + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, std::vector())); return; } @@ -3683,8 +3695,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!m_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers - if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); + if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *peer); + } return; } @@ -3958,6 +3971,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Assume that this is in response to any outstanding getheaders + // request we may have sent, and clear out the time of our last request + peer->m_last_getheaders_timestamp = 0s; + std::vector headers; // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. @@ -4386,7 +4403,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic& interrupt return fMoreWork; } -void PeerManagerImpl::ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) +void PeerManagerImpl::ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seconds time_in_seconds) { AssertLockHeld(cs_main); @@ -4424,10 +4441,15 @@ void PeerManagerImpl::ConsiderEviction(CNode& pto, std::chrono::seconds time_in_ pto.fDisconnect = true; } else { assert(state.m_chain_sync.m_work_header); + // Here, we assume that the getheaders message goes out, + // because it'll either go out or be skipped because of a + // getheaders in-flight already, in which case the peer should + // still respond to us with a sufficiently high work chain tip. + MaybeSendGetHeaders(pto, + m_chainman.ActiveChain().GetLocator(state.m_chain_sync.m_work_header->pprev), + peer); LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); - m_connman.PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); state.m_chain_sync.m_sent_getheaders = true; - constexpr auto HEADERS_RESPONSE_TIME{2min}; // Bump the timeout to allow a response, which could clear the timeout // (if the response shows the peer has synced), reset the timeout (if // the peer syncs to the required work but not to our tip), or result @@ -4835,15 +4857,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && sync_blocks_and_headers_from_peer) || m_chainman.m_best_header->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { - state.fSyncStarted = true; - state.m_headers_sync_timeout = current_time + HEADERS_DOWNLOAD_TIMEOUT_BASE + - ( - // Convert HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER to microseconds before scaling - // to maintain precision - std::chrono::microseconds{HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER} * - (GetAdjustedTime() - m_chainman.m_best_header->GetBlockTime()) / consensusParams.nPowTargetSpacing - ); - nSyncStarted++; const CBlockIndex* pindexStart = m_chainman.m_best_header; /* If possible, start at the block preceding the currently best known header. This ensures that we always get a @@ -4854,8 +4867,19 @@ bool PeerManagerImpl::SendMessages(CNode* pto) got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), peer->m_starting_height); - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexStart), uint256())); + if (MaybeSendGetHeaders(*pto, m_chainman.ActiveChain().GetLocator(pindexStart), *peer)) { + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), peer->m_starting_height); + + state.fSyncStarted = true; + state.m_headers_sync_timeout = current_time + HEADERS_DOWNLOAD_TIMEOUT_BASE + + ( + // Convert HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER to microseconds before scaling + // to maintain precision + std::chrono::microseconds{HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER} * + (GetAdjustedTime() - m_chainman.m_best_header->GetBlockTime()) / consensusParams.nPowTargetSpacing + ); + nSyncStarted++; + } } } @@ -5201,7 +5225,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Check that outbound peers have reasonable chains // GetTime() is used by this anti-DoS logic so we can test this using mocktime - ConsiderEviction(*pto, GetTime()); + ConsiderEviction(*pto, *peer, GetTime()); // // Message: getdata (blocks) diff --git a/test/functional/feature_minchainwork.py b/test/functional/feature_minchainwork.py index fa10855a987b4..9d0ad5ef9d01a 100755 --- a/test/functional/feature_minchainwork.py +++ b/test/functional/feature_minchainwork.py @@ -82,7 +82,7 @@ def run_test(self): msg.hashstop = 0 peer.send_and_ping(msg) time.sleep(5) - assert "headers" not in peer.last_message + assert ("headers" not in peer.last_message or len(peer.last_message["headers"].headers) == 0) self.log.info("Generating one more block") self.generate(self.nodes[0], 1) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 952f1e5cc5a2a..eedb7e6fa1f9d 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -371,6 +371,10 @@ def test_block_relay(self): block1 = self.build_next_block() block1.solve() + # Send an empty headers message, to clear out any prior getheaders + # messages that our peer may be waiting for us on. + self.test_node.send_message(msg_headers()) + self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block1, True) From 27c8056885b05bd25f540dbf6ede89230d43c7b9 Mon Sep 17 00:00:00 2001 From: Martin Zumsande Date: Fri, 24 Jun 2022 18:05:32 -0400 Subject: [PATCH 050/457] rpc: Disallow gettxoutsetinfo queries for a specific block with use_index=false by returning an RPC error where previously a NonFatalError would be thrown. --- src/rpc/blockchain.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 9766a237c79ac..0dbc0a358a4cc 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -845,7 +845,7 @@ static RPCHelpMan gettxoutsetinfo() "Note this call may take some time if you are not using coinstatsindex.\n", { {"hash_type", RPCArg::Type::STR, RPCArg::Default{"hash_serialized_2"}, "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, - {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "The block hash or height of the target height (only available with coinstatsindex).", "", {"", "string or numeric"}}, + {"hash_or_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"the current best block"}, "The block hash or height of the target height (only available with coinstatsindex).", "", {"", "string or numeric"}}, {"use_index", RPCArg::Type::BOOL, RPCArg::Default{true}, "Use coinstatsindex, if available."}, }, RPCResult{ @@ -881,6 +881,7 @@ static RPCHelpMan gettxoutsetinfo() HelpExampleCli("gettxoutsetinfo", R"("none")") + HelpExampleCli("gettxoutsetinfo", R"("none" 1000)") + HelpExampleCli("gettxoutsetinfo", R"("none" '"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"')") + + HelpExampleCli("-named gettxoutsetinfo", R"(hash_type='muhash' use_index='false')") + HelpExampleRpc("gettxoutsetinfo", "") + HelpExampleRpc("gettxoutsetinfo", R"("none")") + HelpExampleRpc("gettxoutsetinfo", R"("none", 1000)") + @@ -917,6 +918,9 @@ static RPCHelpMan gettxoutsetinfo() throw JSONRPCError(RPC_INVALID_PARAMETER, "hash_serialized_2 hash type cannot be queried for a specific block"); } + if (!index_requested) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot set use_index to false when querying for a specific block"); + } pindex = ParseHashOrHeight(request.params[1], chainman); } From 6666803c897e4ad27b45cb74e3a9aa74a335f1bf Mon Sep 17 00:00:00 2001 From: MacroFake Date: Mon, 6 Jun 2022 17:11:03 +0200 Subject: [PATCH 051/457] streams: Add AutoFile without ser-type and ser-version The moved parts can be reviewed with "--color-moved=dimmed-zebra". The one-char changes can be reviewed with "--word-diff-regex=.". --- src/streams.h | 69 ++++++++++++++++++------------ test/functional/feature_addrman.py | 2 +- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/streams.h b/src/streams.h index 96b7696f72c56..f14d347380ac6 100644 --- a/src/streams.h +++ b/src/streams.h @@ -465,35 +465,28 @@ class BitStreamWriter }; - /** Non-refcounted RAII wrapper for FILE* * * Will automatically close the file when it goes out of scope if not null. * If you're returning the file pointer, return file.release(). * If you need to close the file early, use file.fclose() instead of fclose(file). */ -class CAutoFile +class AutoFile { -private: - const int nType; - const int nVersion; - +protected: FILE* file; public: - CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) - { - file = filenew; - } + explicit AutoFile(FILE* filenew) : file{filenew} {} - ~CAutoFile() + ~AutoFile() { fclose(); } // Disallow copies - CAutoFile(const CAutoFile&) = delete; - CAutoFile& operator=(const CAutoFile&) = delete; + AutoFile(const AutoFile&) = delete; + AutoFile& operator=(const AutoFile&) = delete; void fclose() { @@ -504,14 +497,14 @@ class CAutoFile } /** Get wrapped FILE* with transfer of ownership. - * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller + * @note This will invalidate the AutoFile object, and makes it the responsibility of the caller * of this function to clean up the returned FILE*. */ FILE* release() { FILE* ret = file; file = nullptr; return ret; } /** Get wrapped FILE* without transfer of ownership. * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the - * CAutoFile outlives use of the passed pointer. + * AutoFile outlives use of the passed pointer. */ FILE* Get() const { return file; } @@ -522,40 +515,62 @@ class CAutoFile // // Stream subset // - int GetType() const { return nType; } - int GetVersion() const { return nVersion; } - void read(Span dst) { - if (!file) - throw std::ios_base::failure("CAutoFile::read: file handle is nullptr"); + if (!file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); if (fread(dst.data(), 1, dst.size(), file) != dst.size()) { - throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed"); + throw std::ios_base::failure(feof(file) ? "AutoFile::read: end of file" : "AutoFile::read: fread failed"); } } void ignore(size_t nSize) { - if (!file) - throw std::ios_base::failure("CAutoFile::ignore: file handle is nullptr"); + if (!file) throw std::ios_base::failure("AutoFile::ignore: file handle is nullptr"); unsigned char data[4096]; while (nSize > 0) { size_t nNow = std::min(nSize, sizeof(data)); if (fread(data, 1, nNow, file) != nNow) - throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed"); + throw std::ios_base::failure(feof(file) ? "AutoFile::ignore: end of file" : "AutoFile::read: fread failed"); nSize -= nNow; } } void write(Span src) { - if (!file) - throw std::ios_base::failure("CAutoFile::write: file handle is nullptr"); + if (!file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr"); if (fwrite(src.data(), 1, src.size(), file) != src.size()) { - throw std::ios_base::failure("CAutoFile::write: write failed"); + throw std::ios_base::failure("AutoFile::write: write failed"); } } + template + AutoFile& operator<<(const T& obj) + { + if (!file) throw std::ios_base::failure("AutoFile::operator<<: file handle is nullptr"); + ::Serialize(*this, obj); + return *this; + } + + template + AutoFile& operator>>(T&& obj) + { + if (!file) throw std::ios_base::failure("AutoFile::operator>>: file handle is nullptr"); + ::Unserialize(*this, obj); + return *this; + } +}; + +class CAutoFile : public AutoFile +{ +private: + const int nType; + const int nVersion; + +public: + CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : AutoFile{filenew}, nType(nTypeIn), nVersion(nVersionIn) {} + int GetType() const { return nType; } + int GetVersion() const { return nVersion; } + template CAutoFile& operator<<(const T& obj) { diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 5e49d0214a2f8..63abf0d9f8f9d 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -95,7 +95,7 @@ def run_test(self): with open(peers_dat, "wb") as f: f.write(serialize_addrman()[:-1]) self.nodes[0].assert_start_raises_init_error( - expected_msg=init_error("CAutoFile::read: end of file.*"), + expected_msg=init_error("AutoFile::read: end of file.*"), match=ErrorMatch.FULL_REGEX, ) From facc2fa7b8a218a0df6a19772e1641ea68dda2e3 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Mon, 6 Jun 2022 17:22:59 +0200 Subject: [PATCH 052/457] Use AutoFile where possible --- src/index/blockfilterindex.cpp | 8 ++++---- src/net.cpp | 2 +- src/policy/fees.cpp | 16 ++++++++-------- src/policy/fees.h | 6 +++--- src/rpc/blockchain.cpp | 4 ++-- src/rpc/blockchain.h | 2 +- src/test/flatfile_tests.cpp | 12 ++++++------ src/test/fuzz/autofile.cpp | 4 +--- src/test/fuzz/policy_estimator.cpp | 2 +- src/test/fuzz/policy_estimator_io.cpp | 2 +- src/test/fuzz/util.h | 7 +++---- src/test/fuzz/utxo_snapshot.cpp | 4 ++-- src/test/util/chainstate.h | 6 +++--- src/test/validation_chainstatemanager_tests.cpp | 10 +++++----- src/util/asmap.cpp | 2 +- src/validation.cpp | 4 ++-- src/validation.h | 4 ++-- 17 files changed, 46 insertions(+), 49 deletions(-) diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index c92b8c7e195f3..bc44a90c0ea52 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -131,7 +131,7 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch) const FlatFilePos& pos = m_next_filter_pos; // Flush current filter file to disk. - CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + AutoFile file{m_filter_fileseq->Open(pos)}; if (file.IsNull()) { return error("%s: Failed to open filter file %d", __func__, pos.nFile); } @@ -145,7 +145,7 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch) bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const { - CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION); + AutoFile filein{m_filter_fileseq->Open(pos, true)}; if (filein.IsNull()) { return false; } @@ -173,7 +173,7 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& // If writing the filter would overflow the file, flush and move to the next one. if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) { - CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + AutoFile last_file{m_filter_fileseq->Open(pos)}; if (last_file.IsNull()) { LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); return 0; @@ -199,7 +199,7 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& return 0; } - CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + AutoFile fileout{m_filter_fileseq->Open(pos)}; if (fileout.IsNull()) { LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); return 0; diff --git a/src/net.cpp b/src/net.cpp index 7f4e571c8dc58..a91abae944f15 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -3054,7 +3054,7 @@ void CaptureMessageToFile(const CAddress& addr, fs::create_directories(base_path); fs::path path = base_path / (is_incoming ? "msgs_recv.dat" : "msgs_sent.dat"); - CAutoFile f(fsbridge::fopen(path, "ab"), SER_DISK, CLIENT_VERSION); + AutoFile f{fsbridge::fopen(path, "ab")}; ser_writedata64(f, now.count()); f.write(MakeByteSpan(msg_type)); diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 27a6ab221fec0..2b940be07ed07 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -161,13 +161,13 @@ class TxConfirmStats unsigned int GetMaxConfirms() const { return scale * confAvg.size(); } /** Write state of estimation data to a file*/ - void Write(CAutoFile& fileout) const; + void Write(AutoFile& fileout) const; /** * Read saved state of estimation data from a file and replace all internal data structures and * variables with this state. */ - void Read(CAutoFile& filein, int nFileVersion, size_t numBuckets); + void Read(AutoFile& filein, int nFileVersion, size_t numBuckets); }; @@ -390,7 +390,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, return median; } -void TxConfirmStats::Write(CAutoFile& fileout) const +void TxConfirmStats::Write(AutoFile& fileout) const { fileout << Using(decay); fileout << scale; @@ -400,7 +400,7 @@ void TxConfirmStats::Write(CAutoFile& fileout) const fileout << Using>>(failAvg); } -void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) +void TxConfirmStats::Read(AutoFile& filein, int nFileVersion, size_t numBuckets) { // Read data file and do some very basic sanity checking // buckets and bucketMap are not updated yet, so don't access them @@ -546,7 +546,7 @@ CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath longStats = std::unique_ptr(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); // If the fee estimation file is present, read recorded estimations - CAutoFile est_file(fsbridge::fopen(m_estimation_filepath, "rb"), SER_DISK, CLIENT_VERSION); + AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "rb")}; if (est_file.IsNull() || !Read(est_file)) { LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); } @@ -904,13 +904,13 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation void CBlockPolicyEstimator::Flush() { FlushUnconfirmed(); - CAutoFile est_file(fsbridge::fopen(m_estimation_filepath, "wb"), SER_DISK, CLIENT_VERSION); + AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "wb")}; if (est_file.IsNull() || !Write(est_file)) { LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); } } -bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const +bool CBlockPolicyEstimator::Write(AutoFile& fileout) const { try { LOCK(m_cs_fee_estimator); @@ -935,7 +935,7 @@ bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const return true; } -bool CBlockPolicyEstimator::Read(CAutoFile& filein) +bool CBlockPolicyEstimator::Read(AutoFile& filein) { try { LOCK(m_cs_fee_estimator); diff --git a/src/policy/fees.h b/src/policy/fees.h index 9ee5c2938a212..e4628bf85334c 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -20,7 +20,7 @@ #include #include -class CAutoFile; +class AutoFile; class CTxMemPoolEntry; class TxConfirmStats; @@ -220,11 +220,11 @@ class CBlockPolicyEstimator EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Write estimation data to a file */ - bool Write(CAutoFile& fileout) const + bool Write(AutoFile& fileout) const EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Read estimation data from a file */ - bool Read(CAutoFile& filein) + bool Read(AutoFile& filein) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index af9458206edc6..c1b4e863b5adf 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2293,7 +2293,7 @@ static RPCHelpMan dumptxoutset() } FILE* file{fsbridge::fopen(temppath, "wb")}; - CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + AutoFile afile{file}; if (afile.IsNull()) { throw JSONRPCError( RPC_INVALID_PARAMETER, @@ -2314,7 +2314,7 @@ static RPCHelpMan dumptxoutset() UniValue CreateUTXOSnapshot( NodeContext& node, CChainState& chainstate, - CAutoFile& afile, + AutoFile& afile, const fs::path& path, const fs::path& temppath) { diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 5fbd9d5fd373d..a332fd4892773 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -55,7 +55,7 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], UniValue CreateUTXOSnapshot( node::NodeContext& node, CChainState& chainstate, - CAutoFile& afile, + AutoFile& afile, const fs::path& path, const fs::path& tmppath); diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp index d54d6b6471f88..605faa08e411f 100644 --- a/src/test/flatfile_tests.cpp +++ b/src/test/flatfile_tests.cpp @@ -41,26 +41,26 @@ BOOST_AUTO_TEST_CASE(flatfile_open) // Write first line to file. { - CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos1))}; file << LIMITED_STRING(line1, 256); } // Attempt to append to file opened in read-only mode. { - CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos2), true)}; BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure); } // Append second line to file. { - CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos2))}; file << LIMITED_STRING(line2, 256); } // Read text from file in read-only mode. { std::string text; - CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos1), true)}; file >> LIMITED_STRING(text, 256); BOOST_CHECK_EQUAL(text, line1); @@ -72,7 +72,7 @@ BOOST_AUTO_TEST_CASE(flatfile_open) // Read text from file with position offset. { std::string text; - CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos2))}; file >> LIMITED_STRING(text, 256); BOOST_CHECK_EQUAL(text, line2); @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE(flatfile_open) // Ensure another file in the sequence has no data. { std::string text; - CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(1, pos2))}; BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure); } } diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index 3b410930ed876..1a8957d0905f3 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -18,7 +18,7 @@ FUZZ_TARGET(autofile) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - CAutoFile auto_file = fuzzed_auto_file_provider.open(); + AutoFile auto_file{fuzzed_auto_file_provider.open()}; LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, @@ -53,8 +53,6 @@ FUZZ_TARGET(autofile) }); } (void)auto_file.Get(); - (void)auto_file.GetType(); - (void)auto_file.GetVersion(); (void)auto_file.IsNull(); if (fuzzed_data_provider.ConsumeBool()) { FILE* f = auto_file.release(); diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 58c19a91cb7f0..637ba503c6851 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -76,7 +76,7 @@ FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) } { FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - CAutoFile fuzzed_auto_file = fuzzed_auto_file_provider.open(); + AutoFile fuzzed_auto_file{fuzzed_auto_file_provider.open()}; block_policy_estimator.Write(fuzzed_auto_file); block_policy_estimator.Read(fuzzed_auto_file); } diff --git a/src/test/fuzz/policy_estimator_io.cpp b/src/test/fuzz/policy_estimator_io.cpp index 77402c260a8b5..436873c95544c 100644 --- a/src/test/fuzz/policy_estimator_io.cpp +++ b/src/test/fuzz/policy_estimator_io.cpp @@ -26,7 +26,7 @@ FUZZ_TARGET_INIT(policy_estimator_io, initialize_policy_estimator_io) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - CAutoFile fuzzed_auto_file = fuzzed_auto_file_provider.open(); + AutoFile fuzzed_auto_file{fuzzed_auto_file_provider.open()}; // Re-using block_policy_estimator across runs to avoid costly creation of CBlockPolicyEstimator object. static CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args)}; if (block_policy_estimator.Read(fuzzed_auto_file)) { diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 4b89ad9bdc4b5..f6a4e5570f7a8 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -361,17 +361,16 @@ class FuzzedFileProvider class FuzzedAutoFileProvider { - FuzzedDataProvider& m_fuzzed_data_provider; FuzzedFileProvider m_fuzzed_file_provider; public: - FuzzedAutoFileProvider(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_data_provider{fuzzed_data_provider}, m_fuzzed_file_provider{fuzzed_data_provider} + FuzzedAutoFileProvider(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_file_provider{fuzzed_data_provider} { } - CAutoFile open() + AutoFile open() { - return {m_fuzzed_file_provider.open(), m_fuzzed_data_provider.ConsumeIntegral(), m_fuzzed_data_provider.ConsumeIntegral()}; + return AutoFile{m_fuzzed_file_provider.open()}; } }; diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index 33496a457e12d..0b596492bef5b 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -39,13 +39,13 @@ FUZZ_TARGET_INIT(utxo_snapshot, initialize_chain) Assert(!chainman.SnapshotBlockhash()); { - CAutoFile outfile{fsbridge::fopen(snapshot_path, "wb"), SER_DISK, CLIENT_VERSION}; + AutoFile outfile{fsbridge::fopen(snapshot_path, "wb")}; const auto file_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; outfile << Span{file_data}; } const auto ActivateFuzzedSnapshot{[&] { - CAutoFile infile{fsbridge::fopen(snapshot_path, "rb"), SER_DISK, CLIENT_VERSION}; + AutoFile infile{fsbridge::fopen(snapshot_path, "rb")}; SnapshotMetadata metadata; try { infile >> metadata; diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 5ac504c24faa4..13e0e684b87a7 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -16,7 +16,7 @@ #include -const auto NoMalleation = [](CAutoFile& file, node::SnapshotMetadata& meta){}; +const auto NoMalleation = [](AutoFile& file, node::SnapshotMetadata& meta){}; /** * Create and activate a UTXO snapshot, optionally providing a function to @@ -32,7 +32,7 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); fs::path snapshot_path = root / fs::u8path(tfm::format("test_snapshot.%d.dat", height)); FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; - CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; + AutoFile auto_outfile{outfile}; UniValue result = CreateUTXOSnapshot( node, node.chainman->ActiveChainstate(), auto_outfile, snapshot_path, snapshot_path); @@ -42,7 +42,7 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma // Read the written snapshot in and then activate it. // FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; - CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; + AutoFile auto_infile{infile}; node::SnapshotMetadata metadata; auto_infile >> metadata; diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 6dc522b42145c..14de96ff41b32 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -193,7 +193,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) // Should not load malleated snapshots BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // A UTXO is missing but count is correct metadata.m_coins_count -= 1; @@ -204,22 +204,22 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) auto_infile >> coin; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Coins count is larger than coins in file metadata.m_coins_count += 1; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Coins count is smaller than coins in file metadata.m_coins_count -= 1; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Wrong hash metadata.m_base_blockhash = uint256::ZERO; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Wrong hash metadata.m_base_blockhash = uint256::ONE; })); diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index ceb8379c1c102..b1aa598453071 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -195,7 +195,7 @@ std::vector DecodeAsmap(fs::path path) { std::vector bits; FILE *filestr = fsbridge::fopen(path, "rb"); - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + AutoFile file{filestr}; if (file.IsNull()) { LogPrintf("Failed to open asmap file from disk\n"); return bits; diff --git a/src/validation.cpp b/src/validation.cpp index 6b21d33871360..36a9083117a5f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4864,7 +4864,7 @@ const AssumeutxoData* ExpectedAssumeutxo( } bool ChainstateManager::ActivateSnapshot( - CAutoFile& coins_file, + AutoFile& coins_file, const SnapshotMetadata& metadata, bool in_memory) { @@ -4959,7 +4959,7 @@ static void FlushSnapshotToDisk(CCoinsViewCache& coins_cache, bool snapshot_load bool ChainstateManager::PopulateAndValidateSnapshot( CChainState& snapshot_chainstate, - CAutoFile& coins_file, + AutoFile& coins_file, const SnapshotMetadata& metadata) { // It's okay to release cs_main before we're done using `coins_cache` because we know diff --git a/src/validation.h b/src/validation.h index 0e27e117fa5a5..cce268ee8ea9a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -820,7 +820,7 @@ class ChainstateManager //! Internal helper for ActivateSnapshot(). [[nodiscard]] bool PopulateAndValidateSnapshot( CChainState& snapshot_chainstate, - CAutoFile& coins_file, + AutoFile& coins_file, const node::SnapshotMetadata& metadata); /** @@ -909,7 +909,7 @@ class ChainstateManager //! - Move the new chainstate to `m_snapshot_chainstate` and make it our //! ChainstateActive(). [[nodiscard]] bool ActivateSnapshot( - CAutoFile& coins_file, const node::SnapshotMetadata& metadata, bool in_memory); + AutoFile& coins_file, const node::SnapshotMetadata& metadata, bool in_memory); //! The most-work chain. CChainState& ActiveChainstate() const; From f319287d819cc97b5422248aacf447ca190e20f6 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Wed, 29 Jun 2022 17:28:33 +0200 Subject: [PATCH 053/457] test: assert serialized txouts size of `gen_return_txouts` helper This assures that changing the internals of the helper function still leads to the expected outcome sizewise (preparation for the next commit). --- test/functional/test_framework/util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 6a588275eac9c..eab94c32d9127 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -499,7 +499,8 @@ def chain_transaction(node, parent_txids, vouts, value, fee, num_outputs): # Create large OP_RETURN txouts that can be appended to a transaction -# to make it large (helper for constructing large transactions). +# to make it large (helper for constructing large transactions). The +# total serialized size of the txouts is about 66k vbytes. def gen_return_txouts(): # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create # So we have big transactions (and therefore can't fit very many into each block) @@ -515,6 +516,7 @@ def gen_return_txouts(): txout.scriptPubKey = bytes.fromhex(script_pubkey) for _ in range(128): txouts.append(txout) + assert_equal(sum([len(txout.serialize()) for txout in txouts]), 67456) return txouts From b1ba3ed155e89fb6e30237cba8bc8028c7906830 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Wed, 29 Jun 2022 17:42:51 +0200 Subject: [PATCH 054/457] test: let `gen_return_txouts` create a single large OP_RETURN output Transactions with more than one datacarrier (OP_RETURN) output are never considered standard, i.e. this change is necessary in order to to get rid of the `acceptnonstdtxn` option for some tests. --- test/functional/test_framework/util.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index eab94c32d9127..1ee23f7574f82 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -502,20 +502,9 @@ def chain_transaction(node, parent_txids, vouts, value, fee, num_outputs): # to make it large (helper for constructing large transactions). The # total serialized size of the txouts is about 66k vbytes. def gen_return_txouts(): - # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create - # So we have big transactions (and therefore can't fit very many into each block) - # create one script_pubkey - script_pubkey = "6a4d0200" # OP_RETURN OP_PUSH2 512 bytes - for _ in range(512): - script_pubkey = script_pubkey + "01" - # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change - txouts = [] from .messages import CTxOut - txout = CTxOut() - txout.nValue = 0 - txout.scriptPubKey = bytes.fromhex(script_pubkey) - for _ in range(128): - txouts.append(txout) + from .script import CScript, OP_RETURN + txouts = [CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN, b'\x01'*67437]))] assert_equal(sum([len(txout.serialize()) for txout in txouts]), 67456) return txouts From 475aae846e71e355e8f70c0a1857339b1124bcc0 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Wed, 29 Jun 2022 17:51:39 +0200 Subject: [PATCH 055/457] test: pass `datacarriersize` option for tests using large outputs (instead of `acceptnonstdtxn`) By specifying the `datacarriersize` option instead of the more generic `acceptnonstdtxn`, we can be more specific about what part of the transaction is non-standard and can be sure that all other aspects follow the standard policy. --- test/functional/feature_maxuploadtarget.py | 2 +- test/functional/mempool_limit.py | 2 +- test/functional/mining_prioritisetransaction.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 0b9d651226a6d..3ea412002a843 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -46,7 +46,7 @@ def set_test_params(self): self.num_nodes = 1 self.extra_args = [[ "-maxuploadtarget=800M", - "-acceptnonstdtxn=1", + "-datacarriersize=100000", ]] self.supports_cli = False diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index e92f73304bef6..7080662b490e2 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -23,7 +23,7 @@ def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 self.extra_args = [[ - "-acceptnonstdtxn=1", + "-datacarriersize=100000", "-maxmempool=5", "-spendzeroconfchange=0", ]] diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index fb3974c1d5de0..64e66ac30a4bc 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -26,7 +26,7 @@ def set_test_params(self): self.num_nodes = 1 self.extra_args = [[ "-printpriority=1", - "-acceptnonstdtxn=1", + "-datacarriersize=100000", ]] * self.num_nodes self.supports_cli = False From d5c141f221d67aaeee989da0db0ac5383d7562d3 Mon Sep 17 00:00:00 2001 From: Jarol Rodriguez Date: Wed, 29 Jun 2022 20:02:02 -0400 Subject: [PATCH 056/457] qt: apply translator comments to reset options confirmation dialog Follow-up to #617. This applies translator strings to the reset options confirmation dialog and also refactors the way we pass the strings to the dialog in order to allow the comments to be applied. Because the strings were being concatenated, we can not apply translator comments to all of the relevant strings. What we want to do instead is have a variable in which the translatable strings are appended to using the QString append function. This satisfies the Qt translator engine and the comments are then properly applied within the `extracomment` field in the translation file. --- src/qt/optionsdialog.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 462b923d61d9d..42c8b0776326c 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -286,11 +286,19 @@ void OptionsDialog::on_resetButton_clicked() { if (model) { // confirmation dialog + /*: Text explaining that the settings changed will not come into effect + until the client is restarted. */ + QString reset_dialog_text = tr("Client restart required to activate changes.") + "

"; + /*: Text explaining to the user that the client's current settings + will be backed up at a specific location. %1 is a stand-in + argument for the backup location's path. */ + reset_dialog_text.append(tr("Current settings will be backed up at \"%1\".").arg(m_client_model->dataDir()) + "

"); + /*: Text asking the user to confirm if they would like to proceed + with a client shutdown. */ + reset_dialog_text.append(tr("Client will be shut down. Do you want to proceed?")); + //: Window title text of pop-up window shown when the user has chosen to reset options. QMessageBox::StandardButton btnRetVal = QMessageBox::question(this, tr("Confirm options reset"), - tr("Client restart required to activate changes.") + "

" + - tr("Current settings will be backed up at \"%1\".").arg(m_client_model->dataDir()) + "

" + - tr("Client will be shut down. Do you want to proceed?"), - QMessageBox::Yes | QMessageBox::Cancel, QMessageBox::Cancel); + reset_dialog_text, QMessageBox::Yes | QMessageBox::Cancel, QMessageBox::Cancel); if (btnRetVal == QMessageBox::Cancel) return; From 140d942634f9f1bba191aafa948df57812c0f3fe Mon Sep 17 00:00:00 2001 From: S3RK <1466284+S3RK@users.noreply.github.com> Date: Tue, 5 Apr 2022 08:23:49 +0200 Subject: [PATCH 057/457] wallet: don't add change fee to target if subtracting fees from output --- src/wallet/spend.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 81055a5a1a87d..1d22d0993e7e7 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -397,10 +397,13 @@ std::optional AttemptSelection(const CWallet& wallet, const CAm // The knapsack solver has some legacy behavior where it will spend dust outputs. We retain this behavior, so don't filter for positive only here. std::vector all_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, false /* positive_only */); + CAmount target_with_change = nTargetValue; // While nTargetValue includes the transaction fees for non-input things, it does not include the fee for creating a change output. - // So we need to include that for KnapsackSolver as well, as we are expecting to create a change output. - if (auto knapsack_result{KnapsackSolver(all_groups, nTargetValue + coin_selection_params.m_change_fee, - coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { + // So we need to include that for KnapsackSolver and SRD as well, as we are expecting to create a change output. + if (!coin_selection_params.m_subtract_fee_outputs) { + target_with_change += coin_selection_params.m_change_fee; + } + if (auto knapsack_result{KnapsackSolver(all_groups, target_with_change, coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { knapsack_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*knapsack_result); } @@ -409,7 +412,7 @@ std::optional AttemptSelection(const CWallet& wallet, const CAm // barely meets the target. Just use the lower bound change target instead of the randomly // generated one, since SRD will result in a random change amount anyway; avoid making the // target needlessly large. - const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + CHANGE_LOWER; + const CAmount srd_target = target_with_change + CHANGE_LOWER; if (auto srd_result{SelectCoinsSRD(positive_groups, srd_target, coin_selection_params.rng_fast)}) { srd_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*srd_result); From 748a10e896b84f084f573472428b76d49c3c9795 Mon Sep 17 00:00:00 2001 From: /dev/fd0 Date: Thu, 30 Jun 2022 03:38:27 +0530 Subject: [PATCH 058/457] rephrase error for invalid timeout --- src/init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/init.cpp b/src/init.cpp index ee90abcc9321d..d844e9b1699f3 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -966,7 +966,7 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb peer_connect_timeout = args.GetIntArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); if (peer_connect_timeout <= 0) { - return InitError(Untranslated("peertimeout cannot be configured with a negative value.")); + return InitError(Untranslated("peertimeout must be a positive integer.")); } if (args.IsArgSet("-minrelaytxfee")) { From 103c0d9f7e084c94ba7d83a44e784ab0b4a6d8e4 Mon Sep 17 00:00:00 2001 From: fanquake Date: Thu, 30 Jun 2022 11:20:29 +0100 Subject: [PATCH 059/457] guix: use elfesteem 2eb1e5384ff7a220fd1afacd4a0170acff54fe56 Our patch has been merged upstream, see https://github.com/LRGH/elfesteem/pull/3 --- contrib/guix/manifest.scm | 5 ++--- .../patches/elfsteem-value-error-python-39.patch | 13 ------------- 2 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 contrib/guix/patches/elfsteem-value-error-python-39.patch diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index df51d659e8ad1..b05a13fd0f354 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -253,7 +253,7 @@ thus should be able to compile on most platforms where these exist.") (license license:gpl3+))) ; license is with openssl exception (define-public python-elfesteem - (let ((commit "87bbd79ab7e361004c98cc8601d4e5f029fd8bd5")) + (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) (package (name "python-elfesteem") (version (git-version "0.1" "1" commit)) @@ -266,8 +266,7 @@ thus should be able to compile on most platforms where these exist.") (file-name (git-file-name name commit)) (sha256 (base32 - "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")) - (patches (search-our-patches "elfsteem-value-error-python-39.patch")))) + "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) (build-system python-build-system) ;; There are no tests, but attempting to run python setup.py test leads to ;; PYTHONPATH problems, just disable the test diff --git a/contrib/guix/patches/elfsteem-value-error-python-39.patch b/contrib/guix/patches/elfsteem-value-error-python-39.patch deleted file mode 100644 index 21e1228afd83c..0000000000000 --- a/contrib/guix/patches/elfsteem-value-error-python-39.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/examples/otool.py b/examples/otool.py -index 2b8efc0..d797b2e 100755 ---- a/examples/otool.py -+++ b/examples/otool.py -@@ -342,7 +342,7 @@ if __name__ == '__main__': - try: - e = macho_init.MACHO(raw, - parseSymbols = False) -- except ValueError, err: -+ except ValueError as err: - print("%s:" %file) - print(" %s" % err) - continue From d22bd543cc666e7fdb6fdd0eca5f7514f516b393 Mon Sep 17 00:00:00 2001 From: brunoerg Date: Wed, 29 Jun 2022 17:03:13 -0300 Subject: [PATCH 060/457] test: passing a non-positive integer value to `-peertimeout` should throw an error --- test/functional/p2p_timeouts.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index f0abbc7d8ba34..15a879ae3ca0e 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -94,6 +94,11 @@ def run_test(self): no_version_node.wait_for_disconnect(timeout=1) no_send_node.wait_for_disconnect(timeout=1) + self.stop_nodes(0) + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: peertimeout must be a positive integer.', + extra_args=['-peertimeout=0'], + ) if __name__ == '__main__': TimeoutsTest().main() From 0ee43d13e993a59fe1ba509f617dd0e01e1068e2 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Thu, 30 Jun 2022 19:12:01 +0530 Subject: [PATCH 061/457] test: refactor rpc_signrawtransaction.py rpc_signrawtransaction.py is split into rpc_signrawtransactionwithkey.py and wallet_signrawtransactionwithwallet.py. rpc_signrawtransactionwithkey.py can be run with the wallet disabled. --- .../rpc_signrawtransactionwithkey.py | 140 ++++++++++++++++++ test/functional/test_runner.py | 5 +- ...=> wallet_signrawtransactionwithwallet.py} | 100 +------------ 3 files changed, 146 insertions(+), 99 deletions(-) create mode 100755 test/functional/rpc_signrawtransactionwithkey.py rename test/functional/{rpc_signrawtransaction.py => wallet_signrawtransactionwithwallet.py} (71%) diff --git a/test/functional/rpc_signrawtransactionwithkey.py b/test/functional/rpc_signrawtransactionwithkey.py new file mode 100755 index 0000000000000..0da5a99fdb609 --- /dev/null +++ b/test/functional/rpc_signrawtransactionwithkey.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test transaction signing using the signrawtransactionwithkey RPC.""" + +from test_framework.blocktools import ( + COINBASE_MATURITY, +) +from test_framework.address import ( + script_to_p2sh, +) +from test_framework.key import ECKey +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + find_vout_for_address, +) +from test_framework.script_util import ( + key_to_p2pk_script, + key_to_p2pkh_script, + script_to_p2sh_p2wsh_script, + script_to_p2wsh_script, +) +from test_framework.wallet_util import ( + bytes_to_wif, +) + +from decimal import ( + Decimal, +) +from test_framework.wallet import ( + getnewdestination, +) + + +class SignRawTransactionWithKeyTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + + def send_to_address(self, addr, amount): + input = {"txid": self.nodes[0].getblock(self.block_hash[self.blk_idx])["tx"][0], "vout": 0} + output = {addr: amount} + self.blk_idx += 1 + rawtx = self.nodes[0].createrawtransaction([input], output) + txid = self.nodes[0].sendrawtransaction(self.nodes[0].signrawtransactionwithkey(rawtx, [self.nodes[0].get_deterministic_priv_key().key])["hex"], 0) + return txid + + def successful_signing_test(self): + """Create and sign a valid raw transaction with one input. + + Expected results: + + 1) The transaction has a complete set of signatures + 2) No script verification error occurred""" + self.log.info("Test valid raw transaction with one input") + privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N', 'cVKpPfVKSJxKqVpE9awvXNWuLHCa5j5tiE7K6zbUSptFpTEtiFrA'] + + inputs = [ + # Valid pay-to-pubkey scripts + {'txid': '9b907ef1e3c26fc71fe4a4b3580bc75264112f95050014157059c736f0202e71', 'vout': 0, + 'scriptPubKey': '76a91460baa0f494b38ce3c940dea67f3804dc52d1fb9488ac'}, + {'txid': '83a4f6a6b73660e13ee6cb3c6063fa3759c50c9b7521d0536022961898f4fb02', 'vout': 0, + 'scriptPubKey': '76a914669b857c03a5ed269d5d85a1ffac9ed5d663072788ac'}, + ] + + outputs = {'mpLQjfK79b7CCV4VMJWEWAj5Mpx8Up5zxB': 0.1} + + rawTx = self.nodes[0].createrawtransaction(inputs, outputs) + rawTxSigned = self.nodes[0].signrawtransactionwithkey(rawTx, privKeys, inputs) + + # 1) The transaction has a complete set of signatures + assert rawTxSigned['complete'] + + # 2) No script verification error occurred + assert 'errors' not in rawTxSigned + + def witness_script_test(self): + self.log.info("Test signing transaction to P2SH-P2WSH addresses without wallet") + # Create a new P2SH-P2WSH 1-of-1 multisig address: + eckey = ECKey() + eckey.generate() + embedded_privkey = bytes_to_wif(eckey.get_bytes()) + embedded_pubkey = eckey.get_pubkey().get_bytes().hex() + p2sh_p2wsh_address = self.nodes[1].createmultisig(1, [embedded_pubkey], "p2sh-segwit") + # send transaction to P2SH-P2WSH 1-of-1 multisig address + self.block_hash = self.generate(self.nodes[0], COINBASE_MATURITY + 1) + self.blk_idx = 0 + self.send_to_address(p2sh_p2wsh_address["address"], 49.999) + self.generate(self.nodes[0], 1) + # Get the UTXO info from scantxoutset + unspent_output = self.nodes[1].scantxoutset('start', [p2sh_p2wsh_address['descriptor']])['unspents'][0] + spk = script_to_p2sh_p2wsh_script(p2sh_p2wsh_address['redeemScript']).hex() + unspent_output['witnessScript'] = p2sh_p2wsh_address['redeemScript'] + unspent_output['redeemScript'] = script_to_p2wsh_script(unspent_output['witnessScript']).hex() + assert_equal(spk, unspent_output['scriptPubKey']) + # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys + spending_tx = self.nodes[0].createrawtransaction([unspent_output], {getnewdestination()[2]: Decimal("49.998")}) + spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output]) + # Check the signing completed successfully + assert 'complete' in spending_tx_signed + assert_equal(spending_tx_signed['complete'], True) + + # Now test with P2PKH and P2PK scripts as the witnessScript + for tx_type in ['P2PKH', 'P2PK']: # these tests are order-independent + self.verify_txn_with_witness_script(tx_type) + + def verify_txn_with_witness_script(self, tx_type): + self.log.info("Test with a {} script as the witnessScript".format(tx_type)) + eckey = ECKey() + eckey.generate() + embedded_privkey = bytes_to_wif(eckey.get_bytes()) + embedded_pubkey = eckey.get_pubkey().get_bytes().hex() + witness_script = { + 'P2PKH': key_to_p2pkh_script(embedded_pubkey).hex(), + 'P2PK': key_to_p2pk_script(embedded_pubkey).hex() + }.get(tx_type, "Invalid tx_type") + redeem_script = script_to_p2wsh_script(witness_script).hex() + addr = script_to_p2sh(redeem_script) + script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] + # Fund that address + txid = self.send_to_address(addr, 10) + vout = find_vout_for_address(self.nodes[0], txid, addr) + self.generate(self.nodes[0], 1) + # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys + spending_tx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], {getnewdestination()[2]: Decimal("9.999")}) + spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [{'txid': txid, 'vout': vout, 'scriptPubKey': script_pub_key, 'redeemScript': redeem_script, 'witnessScript': witness_script, 'amount': 10}]) + # Check the signing completed successfully + assert 'complete' in spending_tx_signed + assert_equal(spending_tx_signed['complete'], True) + self.nodes[0].sendrawtransaction(spending_tx_signed['hex']) + + def run_test(self): + self.successful_signing_test() + self.witness_script_test() + + +if __name__ == '__main__': + SignRawTransactionWithKeyTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6a44f9d21d413..07d0016277215 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -181,8 +181,9 @@ 'rpc_whitelist.py', 'feature_proxy.py', 'feature_syscall_sandbox.py', - 'rpc_signrawtransaction.py --legacy-wallet', - 'rpc_signrawtransaction.py --descriptors', + 'wallet_signrawtransactionwithwallet.py --legacy-wallet', + 'wallet_signrawtransactionwithwallet.py --descriptors', + 'rpc_signrawtransactionwithkey.py', 'rpc_rawtransaction.py --legacy-wallet', 'wallet_groups.py --legacy-wallet', 'wallet_transactiontime_rescan.py --descriptors', diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/wallet_signrawtransactionwithwallet.py similarity index 71% rename from test/functional/rpc_signrawtransaction.py rename to test/functional/wallet_signrawtransactionwithwallet.py index 8da2cfa72bf50..6b30386b7e604 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/wallet_signrawtransactionwithwallet.py @@ -2,16 +2,14 @@ # Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test transaction signing using the signrawtransaction* RPCs.""" +"""Test transaction signing using the signrawtransactionwithwallet RPC.""" from test_framework.blocktools import ( COINBASE_MATURITY, ) from test_framework.address import ( - script_to_p2sh, script_to_p2wsh, ) -from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -29,20 +27,13 @@ OP_DROP, OP_TRUE, ) -from test_framework.script_util import ( - key_to_p2pk_script, - key_to_p2pkh_script, - script_to_p2sh_p2wsh_script, - script_to_p2wsh_script, -) -from test_framework.wallet_util import bytes_to_wif from decimal import ( Decimal, getcontext, ) -class SignRawTransactionsTest(BitcoinTestFramework): +class SignRawTransactionWithWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 @@ -50,35 +41,6 @@ def set_test_params(self): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def successful_signing_test(self): - """Create and sign a valid raw transaction with one input. - - Expected results: - - 1) The transaction has a complete set of signatures - 2) No script verification error occurred""" - self.log.info("Test valid raw transaction with one input") - privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N', 'cVKpPfVKSJxKqVpE9awvXNWuLHCa5j5tiE7K6zbUSptFpTEtiFrA'] - - inputs = [ - # Valid pay-to-pubkey scripts - {'txid': '9b907ef1e3c26fc71fe4a4b3580bc75264112f95050014157059c736f0202e71', 'vout': 0, - 'scriptPubKey': '76a91460baa0f494b38ce3c940dea67f3804dc52d1fb9488ac'}, - {'txid': '83a4f6a6b73660e13ee6cb3c6063fa3759c50c9b7521d0536022961898f4fb02', 'vout': 0, - 'scriptPubKey': '76a914669b857c03a5ed269d5d85a1ffac9ed5d663072788ac'}, - ] - - outputs = {'mpLQjfK79b7CCV4VMJWEWAj5Mpx8Up5zxB': 0.1} - - rawTx = self.nodes[0].createrawtransaction(inputs, outputs) - rawTxSigned = self.nodes[0].signrawtransactionwithkey(rawTx, privKeys, inputs) - - # 1) The transaction has a complete set of signatures - assert rawTxSigned['complete'] - - # 2) No script verification error occurred - assert 'errors' not in rawTxSigned - def test_with_lock_outputs(self): self.log.info("Test correct error reporting when trying to sign a locked output") self.nodes[0].encryptwallet("password") @@ -191,60 +153,6 @@ def test_fully_signed_tx(self): assert_equal(signedtx["hex"], signedtx2["hex"]) self.nodes[0].walletlock() - def witness_script_test(self): - self.log.info("Test signing transaction to P2SH-P2WSH addresses without wallet") - # Create a new P2SH-P2WSH 1-of-1 multisig address: - eckey = ECKey() - eckey.generate() - embedded_privkey = bytes_to_wif(eckey.get_bytes()) - embedded_pubkey = eckey.get_pubkey().get_bytes().hex() - p2sh_p2wsh_address = self.nodes[1].createmultisig(1, [embedded_pubkey], "p2sh-segwit") - # send transaction to P2SH-P2WSH 1-of-1 multisig address - self.generate(self.nodes[0], COINBASE_MATURITY + 1) - self.nodes[0].sendtoaddress(p2sh_p2wsh_address["address"], 49.999) - self.generate(self.nodes[0], 1) - # Get the UTXO info from scantxoutset - unspent_output = self.nodes[1].scantxoutset('start', [p2sh_p2wsh_address['descriptor']])['unspents'][0] - spk = script_to_p2sh_p2wsh_script(p2sh_p2wsh_address['redeemScript']).hex() - unspent_output['witnessScript'] = p2sh_p2wsh_address['redeemScript'] - unspent_output['redeemScript'] = script_to_p2wsh_script(unspent_output['witnessScript']).hex() - assert_equal(spk, unspent_output['scriptPubKey']) - # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys - spending_tx = self.nodes[0].createrawtransaction([unspent_output], {self.nodes[1].get_wallet_rpc(self.default_wallet_name).getnewaddress(): Decimal("49.998")}) - spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output]) - # Check the signing completed successfully - assert 'complete' in spending_tx_signed - assert_equal(spending_tx_signed['complete'], True) - - # Now test with P2PKH and P2PK scripts as the witnessScript - for tx_type in ['P2PKH', 'P2PK']: # these tests are order-independent - self.verify_txn_with_witness_script(tx_type) - - def verify_txn_with_witness_script(self, tx_type): - self.log.info("Test with a {} script as the witnessScript".format(tx_type)) - eckey = ECKey() - eckey.generate() - embedded_privkey = bytes_to_wif(eckey.get_bytes()) - embedded_pubkey = eckey.get_pubkey().get_bytes().hex() - witness_script = { - 'P2PKH': key_to_p2pkh_script(embedded_pubkey).hex(), - 'P2PK': key_to_p2pk_script(embedded_pubkey).hex() - }.get(tx_type, "Invalid tx_type") - redeem_script = script_to_p2wsh_script(witness_script).hex() - addr = script_to_p2sh(redeem_script) - script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] - # Fund that address - txid = self.nodes[0].sendtoaddress(addr, 10) - vout = find_vout_for_address(self.nodes[0], txid, addr) - self.generate(self.nodes[0], 1) - # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys - spending_tx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): Decimal("9.999")}) - spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [{'txid': txid, 'vout': vout, 'scriptPubKey': script_pub_key, 'redeemScript': redeem_script, 'witnessScript': witness_script, 'amount': 10}]) - # Check the signing completed successfully - assert 'complete' in spending_tx_signed - assert_equal(spending_tx_signed['complete'], True) - self.nodes[0].sendrawtransaction(spending_tx_signed['hex']) - def OP_1NEGATE_test(self): self.log.info("Test OP_1NEGATE (0x4f) satisfies BIP62 minimal push standardness rule") hex_str = ( @@ -385,9 +293,7 @@ def test_signing_with_missing_prevtx_info(self): ]) def run_test(self): - self.successful_signing_test() self.script_verification_error_test() - self.witness_script_test() self.OP_1NEGATE_test() self.test_with_lock_outputs() self.test_fully_signed_tx() @@ -397,4 +303,4 @@ def run_test(self): if __name__ == '__main__': - SignRawTransactionsTest().main() + SignRawTransactionWithWalletTest().main() From 76fb300b63c5c0d01d768510ec69d894820432fa Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 30 Jun 2022 11:08:44 -0400 Subject: [PATCH 062/457] psbt: Check Taproot tree depth and leaf versions Since TaprootBuilder has assertions for the depth and leaf versions, the PSBT decoder should check these values before calling TaprootBuilder::Add so that the assertions are not triggered on malformed taproot trees. --- src/psbt.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/psbt.h b/src/psbt.h index a143a99988e05..c390bb67d3813 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -866,6 +866,12 @@ struct PSBTOutput s_tree >> depth; s_tree >> leaf_ver; s_tree >> script; + if (depth > TAPROOT_CONTROL_MAX_NODE_COUNT) { + throw std::ios_base::failure("Output Taproot tree has as leaf greater than Taproot maximum depth"); + } + if ((leaf_ver & ~TAPROOT_LEAF_MASK) != 0) { + throw std::ios_base::failure("Output Taproot tree has a leaf with an invalid leaf version"); + } m_tap_tree->Add((int)depth, script, (int)leaf_ver, true /* track */); } if (!m_tap_tree->IsComplete()) { From 18f5355f3ad8a5513c99c5b0ca14266b604362cc Mon Sep 17 00:00:00 2001 From: Igor Bubelov Date: Fri, 1 Jul 2022 14:34:25 +0700 Subject: [PATCH 063/457] Remove outdated comment --- src/Makefile.am | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 488ff0e273522..bc0982f74dc17 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -854,8 +854,7 @@ endif # TODO: libbitcoinkernel is a work in progress consensus engine library, as more # and more modules are decoupled from the consensus engine, this list will -# shrink to only those which are absolutely necessary. For example, things -# like index/*.cpp will be removed. +# shrink to only those which are absolutely necessary. libbitcoinkernel_la_SOURCES = \ kernel/bitcoinkernel.cpp \ arith_uint256.cpp \ From 2222842ae73f85494797b14753bc18446e4817a2 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:07:55 +0200 Subject: [PATCH 064/457] test: Allow absolute fee in MiniWallet create_self_transfer --- test/functional/test_framework/wallet.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 68d5dfa880081..8b5689c09e064 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -197,7 +197,7 @@ def get_utxos(self, *, mark_as_spent=True): return utxos def send_self_transfer(self, *, from_node, **kwargs): - """Create and send a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + """Call create_self_transfer and send the transaction.""" tx = self.create_self_transfer(**kwargs) self.sendrawtransaction(from_node=from_node, tx_hex=tx['hex']) return tx @@ -272,16 +272,18 @@ def create_self_transfer_multi( "tx": tx, } - def create_self_transfer(self, *, fee_rate=Decimal("0.003"), utxo_to_spend=None, locktime=0, sequence=0): - """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), fee=Decimal("0"), utxo_to_spend=None, locktime=0, sequence=0): + """Create and return a tx with the specified fee. If fee is 0, use fee_rate, where the resulting fee may be exact or at most one satoshi higher than needed.""" utxo_to_spend = utxo_to_spend or self.get_utxo() + assert fee_rate >= 0 + assert fee >= 0 if self._mode in (MiniWalletMode.RAW_OP_TRUE, MiniWalletMode.ADDRESS_OP_TRUE): vsize = Decimal(104) # anyone-can-spend elif self._mode == MiniWalletMode.RAW_P2PK: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False - send_value = utxo_to_spend["value"] - (fee_rate * vsize / 1000) + send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) assert send_value > 0 tx = CTransaction() From fac3800d2c962420303ab5c61b2f02f35b9f693a Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:09:56 +0200 Subject: [PATCH 065/457] test: Allow amount_per_output in MiniWallet create_self_transfer_multi --- test/functional/test_framework/wallet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 8b5689c09e064..6c83a40347989 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -232,12 +232,14 @@ def create_self_transfer_multi( *, utxos_to_spend: Optional[List[dict]] = None, num_outputs=1, + amount_per_output=0, sequence=0, fee_per_output=1000, ): """ Create and return a transaction that spends the given UTXOs and creates a - certain number of outputs with equal amounts. + certain number of outputs with equal amounts. The output amounts can be + set by amount_per_output or automatically calculated with a fee_per_output. """ utxos_to_spend = utxos_to_spend or [self.get_utxo()] # create simple tx template (1 input, 1 output) @@ -258,7 +260,7 @@ def create_self_transfer_multi( inputs_value_total = sum([int(COIN * utxo['value']) for utxo in utxos_to_spend]) outputs_value_total = inputs_value_total - fee_per_output * num_outputs for o in tx.vout: - o.nValue = outputs_value_total // num_outputs + o.nValue = amount_per_output or (outputs_value_total // num_outputs) txid = tx.rehash() return { "new_utxos": [self._create_utxo( From fa2924582726ee00b392bd0b5591e7d9770e1b90 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:27:59 +0200 Subject: [PATCH 066/457] test: Allow setting sequence per input in MiniWallet create_self_transfer_multi Previously it was only possible to set the same sequence in all inputs --- test/functional/test_framework/wallet.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 6c83a40347989..7ab54618c8b41 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -242,13 +242,17 @@ def create_self_transfer_multi( set by amount_per_output or automatically calculated with a fee_per_output. """ utxos_to_spend = utxos_to_spend or [self.get_utxo()] + sequence = [sequence] * len(utxos_to_spend) if type(sequence) is int else sequence + assert_equal(len(utxos_to_spend), len(sequence)) # create simple tx template (1 input, 1 output) tx = self.create_self_transfer( fee_rate=0, - utxo_to_spend=utxos_to_spend[0], sequence=sequence)["tx"] + utxo_to_spend=utxos_to_spend[0])["tx"] # duplicate inputs, witnesses and outputs tx.vin = [deepcopy(tx.vin[0]) for _ in range(len(utxos_to_spend))] + for txin, seq in zip(tx.vin, sequence): + txin.nSequence = seq tx.wit.vtxinwit = [deepcopy(tx.wit.vtxinwit[0]) for _ in range(len(utxos_to_spend))] tx.vout = [deepcopy(tx.vout[0]) for _ in range(num_outputs)] From fa5059b7dfba6d95c14a12f21d02b63fe29b3f2b Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:28:33 +0200 Subject: [PATCH 067/457] test: Make the scriptPubKey of MiniWallet created txs mutable This makes individual bytes of the scriptPubKey mutable, previously it could only be re-assigned as a whole. --- test/functional/test_framework/wallet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 7ab54618c8b41..216462778148b 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -294,7 +294,7 @@ def create_self_transfer(self, *, fee_rate=Decimal("0.003"), fee=Decimal("0"), u tx = CTransaction() tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=sequence)] - tx.vout = [CTxOut(int(COIN * send_value), self._scriptPubKey)] + tx.vout = [CTxOut(int(COIN * send_value), bytearray(self._scriptPubKey))] tx.nLockTime = locktime if self._mode == MiniWalletMode.RAW_P2PK: self.sign_tx(tx) From 99f4785cad94657dcf349d00fdd6f1d44cac9bb0 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Tue, 28 Jun 2022 10:53:02 -0400 Subject: [PATCH 068/457] Replace GetTime() with NodeClock in MaybeSendGetHeaders() --- src/net_processing.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index fe1b06b18cdff..8cdc08c865a14 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -358,7 +358,7 @@ struct Peer { std::deque m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); /** Time of the last getheaders message to this peer */ - std::atomic m_last_getheaders_timestamp{0s}; + std::atomic m_last_getheaders_timestamp{NodeSeconds{}}; Peer(NodeId id) : m_id{id} @@ -2272,10 +2272,11 @@ bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& loc { const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - const auto current_time = GetTime(); + const auto current_time = NodeClock::now(); + // Only allow a new getheaders message to go out if we don't have a recent // one already in-flight - if (peer.m_last_getheaders_timestamp.load() < current_time - HEADERS_RESPONSE_TIME) { + if (current_time - peer.m_last_getheaders_timestamp.load() > HEADERS_RESPONSE_TIME) { m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, locator, uint256())); peer.m_last_getheaders_timestamp = current_time; return true; @@ -3973,7 +3974,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Assume that this is in response to any outstanding getheaders // request we may have sent, and clear out the time of our last request - peer->m_last_getheaders_timestamp = 0s; + peer->m_last_getheaders_timestamp.store(NodeSeconds{}); std::vector headers; From ebe106a754d2da567702e60185142d9e381bf1cd Mon Sep 17 00:00:00 2001 From: glozow Date: Fri, 1 Jul 2022 14:09:14 +0100 Subject: [PATCH 069/457] add glozow to trusted-keys --- contrib/verify-commits/trusted-keys | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index 2f8a21009aebc..046589a583d28 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -4,3 +4,4 @@ B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B E777299FC265DD04793070EB944D35F9AC3DB76A D1DBF2C4B96F2DEBF4C16654410108112E7EA81F 152812300785C96444D3334D17565732E08E5E41 +6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C From bc13ec888cdc2791f79eeb6eb76b9134d670043e Mon Sep 17 00:00:00 2001 From: w0xlt <94266259+w0xlt@users.noreply.github.com> Date: Fri, 1 Jul 2022 10:16:42 -0300 Subject: [PATCH 070/457] doc: Add a release note about the "restore wallet" menu item --- doc/release-notes/release-notes-471.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/release-notes/release-notes-471.md diff --git a/doc/release-notes/release-notes-471.md b/doc/release-notes/release-notes-471.md new file mode 100644 index 0000000000000..7cebedd8b30be --- /dev/null +++ b/doc/release-notes/release-notes-471.md @@ -0,0 +1,4 @@ +GUI changes +-------- + +- A new menu item to restore a wallet from a backup file has been added (#471). \ No newline at end of file From eac1099e0071bfe11fe664a8a490eee6bbc80c47 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Fri, 1 Jul 2022 19:27:34 +0530 Subject: [PATCH 071/457] test: remove wallet dependency from mempool_updatefromblock.py This functional test can now be run with the wallet disabled. --- test/functional/mempool_updatefromblock.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 51de582ce0f11..f97c2223a6877 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -12,6 +12,9 @@ from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +from test_framework.address import key_to_p2pkh +from test_framework.wallet_util import bytes_to_wif +from test_framework.key import ECKey class MempoolUpdateFromBlockTest(BitcoinTestFramework): @@ -19,8 +22,13 @@ def set_test_params(self): self.num_nodes = 1 self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000', '-limitancestorcount=100']] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def get_new_address(self): + key = ECKey() + key.generate() + pubkey = key.get_pubkey().get_bytes() + address = key_to_p2pkh(pubkey) + self.priv_keys.append(bytes_to_wif(key.get_bytes())) + return address def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', end_address='', fee=Decimal(0.00100000)): """Create an acyclic tournament (a type of directed graph) of transactions and use it for testing. @@ -38,11 +46,12 @@ def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', e More details: https://en.wikipedia.org/wiki/Tournament_(graph_theory) """ + self.priv_keys = [self.nodes[0].get_deterministic_priv_key().key] if not start_input_txid: start_input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0] if not end_address: - end_address = self.nodes[0].getnewaddress() + end_address = self.get_new_address() first_block_hash = '' tx_id = [] @@ -74,7 +83,7 @@ def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', e output_value = ((inputs_value - fee) / Decimal(n_outputs)).quantize(Decimal('0.00000001')) outputs = {} for _ in range(n_outputs): - outputs[self.nodes[0].getnewaddress()] = output_value + outputs[self.get_new_address()] = output_value else: output_value = (inputs_value - fee).quantize(Decimal('0.00000001')) outputs = {end_address: output_value} @@ -84,7 +93,7 @@ def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', e # Create a new transaction. unsigned_raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) - signed_raw_tx = self.nodes[0].signrawtransactionwithwallet(unsigned_raw_tx) + signed_raw_tx = self.nodes[0].signrawtransactionwithkey(unsigned_raw_tx, self.priv_keys) tx_id.append(self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])) tx_size.append(self.nodes[0].getmempoolentry(tx_id[-1])['vsize']) From 220a5a2841172a07d6d7849596316f0e0933e272 Mon Sep 17 00:00:00 2001 From: 0xb10c <0xb10c@gmail.com> Date: Fri, 1 Jul 2022 13:08:49 +0200 Subject: [PATCH 072/457] test: hook into PID in tracing tests This makes sure to NOT hook into other bitcoind binaries run in paralell in the test framework. We only want to trace the intended binary. In interface_usdt_utxocache.py: While testing the utxocache flush with pruning, bitcoind is restarted and we need to hook into the new PID again. --- test/functional/interface_usdt_net.py | 2 +- test/functional/interface_usdt_utxocache.py | 15 ++++++++++++--- test/functional/interface_usdt_validation.py | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/test/functional/interface_usdt_net.py b/test/functional/interface_usdt_net.py index 9522cd8c59588..2235da702b2db 100755 --- a/test/functional/interface_usdt_net.py +++ b/test/functional/interface_usdt_net.py @@ -109,7 +109,7 @@ def __repr__(self): self.log.info( "hook into the net:inbound_message and net:outbound_message tracepoints") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="net:inbound_message", fn_name="trace_inbound_message") ctx.enable_probe(probe="net:outbound_message", diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index f48ff9699d8c1..61587197efba3 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -173,7 +173,7 @@ def test_uncache(self): invalid_tx.vin[0].prevout.hash = int(block_1_coinbase_txid, 16) self.log.info("hooking into the utxocache:uncache tracepoint") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="utxocache:uncache", fn_name="trace_utxocache_uncache") bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0) @@ -238,7 +238,7 @@ def test_add_spent(self): self.log.info( "hook into the utxocache:add and utxocache:spent tracepoints") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="utxocache:add", fn_name="trace_utxocache_add") ctx.enable_probe(probe="utxocache:spent", fn_name="trace_utxocache_spent") @@ -334,7 +334,7 @@ def test_flush(self): self.log.info("test the utxocache:flush tracepoint API") self.log.info("hook into the utxocache:flush tracepoint") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="utxocache:flush", fn_name="trace_utxocache_flush") bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) @@ -373,6 +373,7 @@ def handle_utxocache_flush(_, data, __): self.stop_node(0) bpf.perf_buffer_poll(timeout=200) + bpf.cleanup() self.log.info("check that we don't expect additional flushes") assert_equal(0, len(expected_flushes)) @@ -381,6 +382,14 @@ def handle_utxocache_flush(_, data, __): self.log.info("restart the node with -prune") self.start_node(0, ["-fastprune=1", "-prune=1"]) + self.log.info("test the utxocache:flush tracepoint API with pruning") + self.log.info("hook into the utxocache:flush tracepoint") + ctx = USDT(pid=self.nodes[0].process.pid) + ctx.enable_probe(probe="utxocache:flush", + fn_name="trace_utxocache_flush") + bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) + bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) + BLOCKS_TO_MINE = 350 self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") self.generate(self.wallet, BLOCKS_TO_MINE) diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py index d11809273bbd5..fb1a91b91410d 100755 --- a/test/functional/interface_usdt_validation.py +++ b/test/functional/interface_usdt_validation.py @@ -94,7 +94,7 @@ def __repr__(self): expected_blocks = list() self.log.info("hook into the validation:block_connected tracepoint") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="validation:block_connected", fn_name="trace_block_connected") bpf = BPF(text=validation_blockconnected_program, From dba6f8234217565957e37516a0ea655f1180d99c Mon Sep 17 00:00:00 2001 From: 0xb10c <0xb10c@gmail.com> Date: Fri, 1 Jul 2022 13:17:30 +0200 Subject: [PATCH 073/457] test: adopt USDT utxocache interface tests The USDT interface exposes process internals via the tracepoints. This means, the USDT interface tests somewhat awardly depend on these internals. If internals change, the tests have to adopt to that change. Previously, the USDT interface tests weren't run in the CI so changes could break the USDT interface tests without being noticed (e.g. https://github.com/bitcoin/bitcoin/pull/25486). In fa13375aa3fcb4fd5b9e0d4c69ac31cf66c3209a a 'self.rescan_utxos()' call was added in the 'generate()' function of the test framework. 'rescan_utxos()' causes the UTXO cache to be flushed. In the USDT interface tests for the 'utxocache:flush' trancepoint, 'generate()' is used. As the utxo cache is now flushed more often, the number of flushes the tests expectes need to be adopted. Also, the utxo cache has now a different size when being flushed. The utxocache tracepoint is tested by shutting the node down and pruning blocks, to test the 'for_prune' argument. Changes: - A list 'expected_flushes' is now used which contains 'mode', 'for_prune', and 'size' for each expected flush. - When a flush happens, the expected-flush is removed from the list. This list is checked to be empty (unchanged). - Previously, shutting down caused these two flushes: UTXOCacheFlush(duration=*, mode=ALWAYS, size=104, memory=*, for_prune=False) UTXOCacheFlush(duration=*, mode=ALWAYS, size=0, memory=*, for_prune=False) now it causes these flushes: UTXOCacheFlush(duration=*, mode=ALWAYS, size=2, memory=*, for_prune=False) UTXOCacheFlush(duration=*, mode=ALWAYS, size=0, memory=*, for_prune=False) The 104 UTXOs flushed previously were mainly coinbase UTXOs generated in previous tests and the test setup. These are now already flushed. - In the 'for_prune' test we previously hooked into the tracepoint before mining blocks. This changed to only get notified about the tracepoint being triggered for the prune. Here, the utxo cache is empty already as it has just been flushed in 'generate()'. old: UTXOCacheFlush(duration=*, mode=NONE, size=350, memory=*, for_prune=True) new: UTXOCacheFlush(duration=*, mode=NONE, size=0, memory=*, for_prune=True) --- test/functional/interface_usdt_utxocache.py | 33 +++++++++------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index 61587197efba3..2280de14790c1 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -345,16 +345,17 @@ def test_flush(self): # that the handle_* functions succeeded. EXPECTED_HANDLE_FLUSH_SUCCESS = 3 handle_flush_succeeds = 0 - possible_cache_sizes = set() - expected_flushes = [] + expected_flushes = list() def handle_utxocache_flush(_, data, __): nonlocal handle_flush_succeeds event = ctypes.cast(data, ctypes.POINTER(UTXOCacheFlush)).contents self.log.info(f"handle_utxocache_flush(): {event}") - expected = expected_flushes.pop(0) - assert_equal(expected["mode"], FLUSHMODE_NAME[event.mode]) - possible_cache_sizes.remove(event.size) # fails if size not in set + expected_flushes.remove({ + "mode": FLUSHMODE_NAME[event.mode], + "for_prune": event.for_prune, + "size": event.size + }) # sanity checks only assert(event.memory > 0) assert(event.duration > 0) @@ -363,13 +364,12 @@ def handle_utxocache_flush(_, data, __): bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) self.log.info("stop the node to flush the UTXO cache") - UTXOS_IN_CACHE = 104 # might need to be changed if the eariler tests are modified + UTXOS_IN_CACHE = 2 # might need to be changed if the eariler tests are modified # A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE # UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the # second flush, however it can happen that the order changes. - possible_cache_sizes = {UTXOS_IN_CACHE, 0} - flush_for_shutdown = {"mode": "ALWAYS", "for_prune": False} - expected_flushes.extend([flush_for_shutdown, flush_for_shutdown]) + expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": UTXOS_IN_CACHE}) + expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": 0}) self.stop_node(0) bpf.perf_buffer_poll(timeout=200) @@ -377,11 +377,14 @@ def handle_utxocache_flush(_, data, __): self.log.info("check that we don't expect additional flushes") assert_equal(0, len(expected_flushes)) - assert_equal(0, len(possible_cache_sizes)) self.log.info("restart the node with -prune") self.start_node(0, ["-fastprune=1", "-prune=1"]) + BLOCKS_TO_MINE = 350 + self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") + self.generate(self.wallet, BLOCKS_TO_MINE) + self.log.info("test the utxocache:flush tracepoint API with pruning") self.log.info("hook into the utxocache:flush tracepoint") ctx = USDT(pid=self.nodes[0].process.pid) @@ -390,15 +393,8 @@ def handle_utxocache_flush(_, data, __): bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) - BLOCKS_TO_MINE = 350 - self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") - self.generate(self.wallet, BLOCKS_TO_MINE) - # we added BLOCKS_TO_MINE coinbase UTXOs to the cache - possible_cache_sizes = {BLOCKS_TO_MINE} - expected_flushes.append( - {"mode": "NONE", "for_prune": True, "size_fn": lambda x: x == BLOCKS_TO_MINE}) - self.log.info(f"prune blockchain to trigger a flush for pruning") + expected_flushes.append({"mode": "NONE", "for_prune": True, "size": 0}) self.nodes[0].pruneblockchain(315) bpf.perf_buffer_poll(timeout=500) @@ -407,7 +403,6 @@ def handle_utxocache_flush(_, data, __): self.log.info( f"check that we don't expect additional flushes and that the handle_* function succeeded") assert_equal(0, len(expected_flushes)) - assert_equal(0, len(possible_cache_sizes)) assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds) From 1770be72d5eb47cae565d4ac86de5bc16f94fdd6 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Sun, 3 Jul 2022 16:06:56 +0200 Subject: [PATCH 074/457] test: pass `dustrelayfee=0` option for tests using dust (instead of `acceptnonstdtxn=1`) By specifying the `dustrelayfee=0` option instead of the more generic `acceptnonstdtxn=1`, we can be more specific about what part of the transaction is non-standard and can be sure that all other aspects follow the standard policy. --- test/functional/feature_dbcrash.py | 8 +++++--- test/functional/wallet_basic.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 62e9bec66321e..f606f26e70619 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -62,8 +62,8 @@ def set_test_params(self): self.node2_args = ["-dbcrashratio=24", "-dbcache=16"] + self.base_args # Node3 is a normal node with default args, except will mine full blocks - # and non-standard txs (e.g. txs with "dust" outputs) - self.node3_args = ["-blockmaxweight=4000000", "-acceptnonstdtxn"] + # and txs with "dust" outputs + self.node3_args = ["-blockmaxweight=4000000", "-dustrelayfee=0"] self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args] def setup_network(self): @@ -211,7 +211,9 @@ def run_test(self): self.crashed_on_restart = 0 # Track count of crashes during recovery # Start by creating a lot of utxos on node3 - utxo_list = self.wallet.send_self_transfer_multi(from_node=self.nodes[3], num_outputs=5000)['new_utxos'] + utxo_list = [] + for _ in range(5): + utxo_list.extend(self.wallet.send_self_transfer_multi(from_node=self.nodes[3], num_outputs=1000)['new_utxos']) self.generate(self.nodes[3], 1, sync_fun=self.no_op) assert_equal(len(self.nodes[3].getrawmempool()), 0) self.log.info(f"Prepped {len(utxo_list)} utxo entries") diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index a6c93ba5f9d72..f66fab19ac372 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -25,7 +25,7 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.extra_args = [[ - "-acceptnonstdtxn=1", "-walletrejectlongchains=0" + "-dustrelayfee=0", "-walletrejectlongchains=0" ]] * self.num_nodes self.setup_clean_chain = True self.supports_cli = False From cccf691c24f9cbc4aedd1b36c1d9ba173910ceca Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Sun, 3 Jul 2022 20:07:44 +0200 Subject: [PATCH 075/457] contrib: dedup `get_witness_script` helper in signet miner --- contrib/signet/miner | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/contrib/signet/miner b/contrib/signet/miner index b366b98e2d8d9..61415cb2dd021 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -21,8 +21,8 @@ PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__fi PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional")) sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL) -from test_framework.blocktools import WITNESS_COMMITMENT_HEADER, script_BIP34_coinbase_height # noqa: E402 -from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, hash256, ser_compact_size, ser_string, ser_uint256, tx_from_hex, uint256_from_str # noqa: E402 +from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height # noqa: E402 +from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, ser_compact_size, ser_string, ser_uint256, tx_from_hex # noqa: E402 from test_framework.script import CScriptOp # noqa: E402 logging.basicConfig( @@ -123,10 +123,6 @@ def create_coinbase(height, value, spk): cb.vout = [CTxOut(value, spk)] return cb -def get_witness_script(witness_root, witness_nonce): - commitment = uint256_from_str(hash256(ser_uint256(witness_root) + ser_uint256(witness_nonce))) - return b"\x6a" + CScriptOp.encode_op_pushdata(WITNESS_COMMITMENT_HEADER + ser_uint256(commitment)) - def signet_txs(block, challenge): # assumes signet solution has not been added yet so does not need # to be removed @@ -222,7 +218,7 @@ def generate_psbt(tmpl, reward_spk, *, blocktime=None): cbwit = CTxInWitness() cbwit.scriptWitness.stack = [ser_uint256(witnonce)] block.vtx[0].wit.vtxinwit = [cbwit] - block.vtx[0].vout.append(CTxOut(0, get_witness_script(witroot, witnonce))) + block.vtx[0].vout.append(CTxOut(0, bytes(get_witness_script(witroot, witnonce)))) signme, spendme = signet_txs(block, signet_spk_bin) @@ -627,5 +623,3 @@ def main(): if __name__ == "__main__": main() - - From 236239bd40ae1175537fc932df5af27902326329 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 1 May 2022 14:28:14 +0200 Subject: [PATCH 076/457] wallet: Rescan mempool for transactions as well --- src/wallet/test/wallet_tests.cpp | 8 ++++---- src/wallet/wallet.cpp | 7 ++++++- test/functional/wallet_importdescriptors.py | 4 +++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 70863f5464ab8..0ba39b52d53c8 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -55,9 +55,6 @@ static const std::shared_ptr TestLoadWallet(WalletContext& context) auto database = MakeWalletDatabase("", options, status, error); auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); NotifyWalletLoaded(context, wallet); - if (context.chain) { - wallet->postInitProcess(); - } return wallet; } @@ -765,6 +762,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // being blocked wallet = TestLoadWallet(context); BOOST_CHECK(rescan_completed); + // AddToWallet events for block_tx and mempool_tx BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); @@ -777,6 +775,8 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // transactionAddedToMempool events are processed promise.set_value(); SyncWithValidationInterfaceQueue(); + // AddToWallet events for block_tx and mempool_tx events are counted a + // second time as the notificaiton queue is processed BOOST_CHECK_EQUAL(addtx_count, 4); @@ -800,7 +800,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); }); wallet = TestLoadWallet(context); - BOOST_CHECK_EQUAL(addtx_count, 4); + BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 910562e66901a..e3aebd4f81746 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1687,7 +1687,8 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r /** * Scan the block chain (starting in start_block) for transactions * from or to us. If fUpdate is true, found transactions that already - * exist in the wallet will be updated. + * exist in the wallet will be updated. If max_height is not set, the + * mempool will be scanned as well. * * @param[in] start_block Scan starting block. If block is not on the active * chain, the scan will return SUCCESS immediately. @@ -1797,6 +1798,10 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc } } } + if (!max_height) { + WalletLogPrintf("Scanning current mempool transactions.\n"); + WITH_LOCK(cs_wallet, chain().requestMempoolTransactions(*this)); + } ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 100); // hide progress dialog in GUI if (block_height && fAbortRescan) { WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", block_height, progress_current); diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index ac74ff248440c..b74a041539e8b 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -480,7 +480,9 @@ def run_test(self): addr = wmulti_pub.getnewaddress('', 'bech32') assert_equal(addr, 'bcrt1qp8s25ckjl7gr6x2q3dx3tn2pytwp05upkjztk6ey857tt50r5aeqn6mvr9') # Derived at m/84'/0'/0'/1 change_addr = wmulti_pub.getrawchangeaddress('bech32') - assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') + assert_equal(change_addr, 'bcrt1qzxl0qz2t88kljdnkzg4n4gapr6kte26390gttrg79x66nt4p04fssj53nl') + assert(send_txid in self.nodes[0].getrawmempool(True)) + assert(send_txid in (x['txid'] for x in wmulti_pub.listunspent(0))) assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 999) # generate some utxos for next tests From 3abdbbb90a4a8f2041fec37506268e66a0b3eb31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Barbosa?= Date: Tue, 12 May 2020 23:33:43 +0100 Subject: [PATCH 077/457] rpc, wallet: Document and test mempool scan after importaddress co-authored-by: Fabian Jahr --- src/wallet/rpc/backup.cpp | 4 +++- test/functional/wallet_balance.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index aa9f23c886635..0b73d444f85d6 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -201,6 +201,8 @@ RPCHelpMan importaddress() "\nAdds an address or script (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported address exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "If you have the full public key, you should call importpubkey instead of this.\n" "Hint: use importmulti to import more than one address.\n" "\nNote: If you import a non-standard raw script in hex form, outputs sending to it will be treated\n" @@ -209,7 +211,7 @@ RPCHelpMan importaddress() { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The Bitcoin address (or hex-encoded script)"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, {"p2sh", RPCArg::Type::BOOL, RPCArg::Default{false}, "Add the P2SH version of the script as well"}, }, RPCResult{RPCResult::Type::NONE, "", ""}, diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 0c93821e7f472..94b74c0696d44 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -273,6 +273,22 @@ def test_balances(*, fee_node_1=0): self.generatetoaddress(self.nodes[1], 1, ADDRESS_WATCHONLY) assert_equal(self.nodes[0].getbalance(minconf=0), total_amount + 1) # The reorg recovered our fee of 1 coin + if not self.options.descriptors: + self.log.info('Check if mempool is taken into account after import*') + address = self.nodes[0].getnewaddress() + privkey = self.nodes[0].dumpprivkey(address) + self.nodes[0].sendtoaddress(address, 0.1) + self.nodes[0].unloadwallet('') + # check importaddress on fresh wallet + self.nodes[0].createwallet('w1', False, True) + self.nodes[0].importaddress(address) + assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], 0) + assert_equal(self.nodes[0].getbalances()['watchonly']['untrusted_pending'], Decimal('0.1')) + self.nodes[0].importprivkey(privkey) + assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('0.1')) + assert_equal(self.nodes[0].getbalances()['watchonly']['untrusted_pending'], 0) + self.nodes[0].unloadwallet('w1') + if __name__ == '__main__': WalletTest().main() From 6d3db52e667474b6c0c2e4eeb9fb5b3ba4063205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Barbosa?= Date: Tue, 12 May 2020 23:34:04 +0100 Subject: [PATCH 078/457] rpc, wallet: Document and test mempool scan after importprivkey co-authored-by: Fabian Jahr --- src/wallet/rpc/backup.cpp | 4 +++- test/functional/wallet_balance.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 0b73d444f85d6..386c0e90508ad 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -100,11 +100,13 @@ RPCHelpMan importprivkey() "Hint: use importmulti to import more than one private key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported key exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key (see dumpprivkey)"}, {"label", RPCArg::Type::STR, RPCArg::DefaultHint{"current label if address exists, otherwise \"\""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 94b74c0696d44..d49bca6855781 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -288,6 +288,10 @@ def test_balances(*, fee_node_1=0): assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('0.1')) assert_equal(self.nodes[0].getbalances()['watchonly']['untrusted_pending'], 0) self.nodes[0].unloadwallet('w1') + # check importprivkey on fresh wallet + self.nodes[0].createwallet('w2', False, True) + self.nodes[0].importprivkey(privkey) + assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('0.1')) if __name__ == '__main__': From e6d3ef85867545a5a66a211e35e818e8a1b166fa Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 1 May 2022 19:45:32 +0200 Subject: [PATCH 079/457] rpc, wallet: Document mempool scan after importpubkey --- src/wallet/rpc/backup.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 386c0e90508ad..0f058e45edf05 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -404,11 +404,13 @@ RPCHelpMan importpubkey() "Hint: use importmulti to import more than one public key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported pubkey exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"pubkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The hex-encoded public key"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ From 0e396d1ba701c9ac6280a98bf37f53352167e724 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 1 May 2022 19:53:09 +0200 Subject: [PATCH 080/457] rpc, wallet: Document mempool scan after importmulti --- src/wallet/rpc/backup.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 0f058e45edf05..e8a39239eb763 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -1255,6 +1255,8 @@ RPCHelpMan importmulti() "Conversely, if all the private keys are provided and the address/script is spendable, the watchonly option must be set to false, or a warning will be returned.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", @@ -1296,7 +1298,7 @@ RPCHelpMan importmulti() "\"requests\""}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Stating if should rescan the blockchain after all imports"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, "\"options\""}, }, From 833ce76df712932c19e99737e87b5569e2bca34b Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 1 May 2022 20:00:10 +0200 Subject: [PATCH 081/457] rpc, wallet: Document mempool rescan after importdescriptor, importwallet --- src/wallet/rpc/backup.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index e8a39239eb763..26c2cccb2f6de 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -489,7 +489,7 @@ RPCHelpMan importwallet() { return RPCHelpMan{"importwallet", "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n" - "Note: Use \"getwalletinfo\" to query the scanning progress.\n", + "Note: Blockchain and Mempool will be rescanned after a successful import. Use \"getwalletinfo\" to query the scanning progress.\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet file"}, }, @@ -1600,7 +1600,7 @@ RPCHelpMan importdescriptors() " Use the string \"now\" to substitute the current synced blockchain time.\n" " \"now\" can be specified to bypass scanning, for outputs which are known to never have been used, and\n" " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest timestamp\n" - " of all descriptors being imported will be scanned.", + "of all descriptors being imported will be scanned as well as the mempool.", /*oneline_description=*/"", {"timestamp | \"now\"", "integer / string"} }, {"internal", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether matching outputs should be treated as not incoming payments (e.g. change)"}, From 1be796418934ae7370cb0ed501877db59e738106 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Wed, 1 Jun 2022 01:11:50 +0200 Subject: [PATCH 082/457] test, wallet: Add mempool rescan test for import RPCs --- test/functional/wallet_import_rescan.py | 57 ++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index d9acc8cea5b4b..085ad51c79916 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -87,6 +87,7 @@ def check(self, txid=None, amount=None, confirmation_height=None): assert_equal(len(txs), self.expected_txs) addresses = self.node.listreceivedbyaddress(minconf=0, include_watchonly=True, address_filter=self.address['address']) + if self.expected_txs: assert_equal(len(addresses[0]["txids"]), self.expected_txs) @@ -98,13 +99,18 @@ def check(self, txid=None, amount=None, confirmation_height=None): assert_equal(tx["category"], "receive") assert_equal(tx["label"], self.label) assert_equal(tx["txid"], txid) - assert_equal(tx["confirmations"], 1 + current_height - confirmation_height) - assert "trusted" not in tx + + # If no confirmation height is given, the tx is still in the + # mempool. + confirmations = (1 + current_height - confirmation_height) if confirmation_height else 0 + assert_equal(tx["confirmations"], confirmations) + if confirmations: + assert "trusted" not in tx address, = [ad for ad in addresses if txid in ad["txids"]] assert_equal(address["address"], self.address["address"]) assert_equal(address["amount"], self.expected_balance) - assert_equal(address["confirmations"], 1 + current_height - confirmation_height) + assert_equal(address["confirmations"], confirmations) # Verify the transaction is correctly marked watchonly depending on # whether the transaction pays to an imported public key or # imported private key. The test setup ensures that transaction @@ -162,11 +168,12 @@ def setup_network(self): self.import_deterministic_coinbase_privkeys() self.stop_nodes() - self.start_nodes() + self.start_nodes(extra_args=[["-whitelist=noban@127.0.0.1"]] * self.num_nodes) for i in range(1, self.num_nodes): self.connect_nodes(i, 0) def run_test(self): + # Create one transaction on node 0 with a unique amount for # each possible type of wallet import RPC. for i, variant in enumerate(IMPORT_VARIANTS): @@ -207,7 +214,7 @@ def run_test(self): variant.check() # Create new transactions sending to each address. - for i, variant in enumerate(IMPORT_VARIANTS): + for variant in IMPORT_VARIANTS: variant.sent_amount = get_rand_amount() variant.sent_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.sent_amount) self.generate(self.nodes[0], 1) # Generate one block for each send @@ -223,6 +230,46 @@ def run_test(self): variant.expected_txs += 1 variant.check(variant.sent_txid, variant.sent_amount, variant.confirmation_height) + self.log.info('Test that the mempool is rescanned as well if the rescan parameter is set to true') + + # The late timestamp and pruned variants are not necessary when testing mempool rescan + mempool_variants = [variant for variant in IMPORT_VARIANTS if variant.rescan != Rescan.late_timestamp and not variant.prune] + # No further blocks are mined so the timestamp will stay the same + timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] + + # Create one transaction on node 0 with a unique amount for + # each possible type of wallet import RPC. + for i, variant in enumerate(mempool_variants): + variant.label = "mempool label {} {}".format(i, variant) + variant.address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress( + label=variant.label, + address_type=variant.address_type.value, + )) + variant.key = self.nodes[1].dumpprivkey(variant.address["address"]) + variant.initial_amount = get_rand_amount() + variant.initial_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.initial_amount) + variant.confirmation_height = 0 + variant.timestamp = timestamp + + assert_equal(len(self.nodes[0].getrawmempool()), len(mempool_variants)) + self.sync_mempools() + + # For each variation of wallet key import, invoke the import RPC and + # check the results from getbalance and listtransactions. + for variant in mempool_variants: + self.log.info('Run import for mempool variant {}'.format(variant)) + expect_rescan = variant.rescan == Rescan.yes + variant.node = self.nodes[2 + IMPORT_NODES.index(ImportNode(variant.prune, expect_rescan))] + variant.do_import(variant.timestamp) + if expect_rescan: + variant.expected_balance = variant.initial_amount + variant.expected_txs = 1 + variant.check(variant.initial_txid, variant.initial_amount) + else: + variant.expected_balance = 0 + variant.expected_txs = 0 + variant.check() + if __name__ == "__main__": ImportRescanTest().main() From 42aa5d5b6269d27af525d5001907558442e96023 Mon Sep 17 00:00:00 2001 From: dergoegge Date: Thu, 26 May 2022 15:40:21 +0200 Subject: [PATCH 083/457] [net] Add NoBan status to NodeEvictionCandidate --- src/net.cpp | 14 ++++++++++++-- src/net.h | 1 + src/test/fuzz/node_eviction.cpp | 1 + src/test/util/net.cpp | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 7f4e571c8dc58..91a1b05b81685 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -948,6 +948,15 @@ static void EraseLastKElements( elements.erase(std::remove_if(elements.end() - eraseSize, elements.end(), predicate), elements.end()); } +void ProtectNoBanConnections(std::vector& eviction_candidates) +{ + eviction_candidates.erase(std::remove_if(eviction_candidates.begin(), eviction_candidates.end(), + [](NodeEvictionCandidate const& n) { + return n.m_noban; + }), + eviction_candidates.end()); +} + void ProtectEvictionCandidatesByRatio(std::vector& eviction_candidates) { // Protect the half of the remaining nodes which have been connected the longest. @@ -1025,6 +1034,8 @@ void ProtectEvictionCandidatesByRatio(std::vector& evicti { // Protect connections with certain characteristics + ProtectNoBanConnections(vEvictionCandidates); + // Deterministically select 4 peers to protect by netgroup. // An attacker cannot predict which netgroups will be protected EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); @@ -1096,8 +1107,6 @@ bool CConnman::AttemptToEvictConnection() LOCK(m_nodes_mutex); for (const CNode* node : m_nodes) { - if (node->HasPermission(NetPermissionFlags::NoBan)) - continue; if (!node->IsInboundConn()) continue; if (node->fDisconnect) @@ -1115,6 +1124,7 @@ bool CConnman::AttemptToEvictConnection() Desig(prefer_evict) node->m_prefer_evict, Desig(m_is_local) node->addr.IsLocal(), Desig(m_network) node->ConnectedThroughNetwork(), + Desig(m_noban) node->HasPermission(NetPermissionFlags::NoBan), }; vEvictionCandidates.push_back(candidate); } diff --git a/src/net.h b/src/net.h index bf169649c0b8c..e46933d972ea3 100644 --- a/src/net.h +++ b/src/net.h @@ -1261,6 +1261,7 @@ struct NodeEvictionCandidate bool prefer_evict; bool m_is_local; Network m_network; + bool m_noban; }; /** diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index 6a363f00f702b..d7721f1bf6739 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -32,6 +32,7 @@ FUZZ_TARGET(node_eviction) /*prefer_evict=*/fuzzed_data_provider.ConsumeBool(), /*m_is_local=*/fuzzed_data_provider.ConsumeBool(), /*m_network=*/fuzzed_data_provider.PickValueInArray(ALL_NETWORKS), + /*m_noban=*/fuzzed_data_provider.ConsumeBool(), }); } // Make a copy since eviction_candidates may be in some valid but otherwise diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 62b770753a037..26b3acc677eff 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -58,6 +58,7 @@ std::vector GetRandomNodeEvictionCandidates(int n_candida /*prefer_evict=*/random_context.randbool(), /*m_is_local=*/random_context.randbool(), /*m_network=*/ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], + /*m_noban=*/false, }); } return candidates; From a3c27070396ab8c2941c437e8099547e8fc9c110 Mon Sep 17 00:00:00 2001 From: dergoegge Date: Thu, 26 May 2022 15:49:10 +0200 Subject: [PATCH 084/457] [net] Add connection type to NodeEvictionCandidate --- src/net.cpp | 14 ++++++++++++-- src/net.h | 1 + src/test/fuzz/node_eviction.cpp | 1 + src/test/util/net.cpp | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 91a1b05b81685..46b8975eba974 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -957,6 +957,15 @@ void ProtectNoBanConnections(std::vector& eviction_candid eviction_candidates.end()); } +void ProtectOutboundConnections(std::vector& eviction_candidates) +{ + eviction_candidates.erase(std::remove_if(eviction_candidates.begin(), eviction_candidates.end(), + [](NodeEvictionCandidate const& n) { + return n.m_conn_type != ConnectionType::INBOUND; + }), + eviction_candidates.end()); +} + void ProtectEvictionCandidatesByRatio(std::vector& eviction_candidates) { // Protect the half of the remaining nodes which have been connected the longest. @@ -1036,6 +1045,8 @@ void ProtectEvictionCandidatesByRatio(std::vector& evicti ProtectNoBanConnections(vEvictionCandidates); + ProtectOutboundConnections(vEvictionCandidates); + // Deterministically select 4 peers to protect by netgroup. // An attacker cannot predict which netgroups will be protected EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); @@ -1107,8 +1118,6 @@ bool CConnman::AttemptToEvictConnection() LOCK(m_nodes_mutex); for (const CNode* node : m_nodes) { - if (!node->IsInboundConn()) - continue; if (node->fDisconnect) continue; NodeEvictionCandidate candidate{ @@ -1125,6 +1134,7 @@ bool CConnman::AttemptToEvictConnection() Desig(m_is_local) node->addr.IsLocal(), Desig(m_network) node->ConnectedThroughNetwork(), Desig(m_noban) node->HasPermission(NetPermissionFlags::NoBan), + Desig(m_conn_type) node->m_conn_type, }; vEvictionCandidates.push_back(candidate); } diff --git a/src/net.h b/src/net.h index e46933d972ea3..c76c446dbac94 100644 --- a/src/net.h +++ b/src/net.h @@ -1262,6 +1262,7 @@ struct NodeEvictionCandidate bool m_is_local; Network m_network; bool m_noban; + ConnectionType m_conn_type; }; /** diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index d7721f1bf6739..e27b2545807e9 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -33,6 +33,7 @@ FUZZ_TARGET(node_eviction) /*m_is_local=*/fuzzed_data_provider.ConsumeBool(), /*m_network=*/fuzzed_data_provider.PickValueInArray(ALL_NETWORKS), /*m_noban=*/fuzzed_data_provider.ConsumeBool(), + /*m_conn_type=*/fuzzed_data_provider.PickValueInArray(ALL_CONNECTION_TYPES), }); } // Make a copy since eviction_candidates may be in some valid but otherwise diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 26b3acc677eff..bbcee6a5c82d5 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -59,6 +59,7 @@ std::vector GetRandomNodeEvictionCandidates(int n_candida /*m_is_local=*/random_context.randbool(), /*m_network=*/ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], /*m_noban=*/false, + /*m_conn_type=*/ConnectionType::INBOUND, }); } return candidates; From 31346a31962ab06d49cb45aee8acd92d8806538b Mon Sep 17 00:00:00 2001 From: sogoagain Date: Sun, 3 Jul 2022 00:33:03 +0900 Subject: [PATCH 085/457] [ci] apply cache size limit and print ccache statistics in "ARM64 Android APK" --- ci/test/06_script_a.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 218f5eeb63216..13693a2ecfb6b 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -11,16 +11,19 @@ if [ -z "$NO_WERROR" ]; then BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" fi +CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" +PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" + if [ -n "$ANDROID_TOOLS_URL" ]; then CI_EXEC make distclean || true CI_EXEC ./autogen.sh CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" + CI_EXEC "${PRINT_CCACHE_STATISTICS}" exit 0 fi BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" -CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" if [ -n "$CONFIG_SHELL" ]; then CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" @@ -57,6 +60,6 @@ fi CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) -CI_EXEC "ccache --version | head -n 1 && ccache --show-stats" +CI_EXEC "${PRINT_CCACHE_STATISTICS}" CI_EXEC du -sh "${DEPENDS_DIR}"/*/ CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" From da8f62de2c5561e091ef8073d6950c033f41aabf Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:39:20 -0300 Subject: [PATCH 086/457] wallet: remove always true 'fUseCache' from CachedTxGetImmatureCredit --- src/wallet/receive.cpp | 4 ++-- src/wallet/receive.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 8de4017371a34..8512ee9b2c40c 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -164,12 +164,12 @@ CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx) return wtx.nChangeCached; } -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) { AssertLockHeld(wallet.cs_wallet); if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); + return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_SPENDABLE); } return 0; diff --git a/src/wallet/receive.h b/src/wallet/receive.h index 1caef293f2729..0283bdcfca024 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -29,7 +29,7 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism //! filter decides which addresses will count towards the debit CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter); CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx); -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache = true) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); From 47b1012677821ce2939e10ba462fbe53ffff17df Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:39:53 -0300 Subject: [PATCH 087/457] wallet: remove always true 'fUseCache' from CachedTxGetImmatureWatchOnlyCredit --- src/wallet/receive.cpp | 4 ++-- src/wallet/receive.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 8512ee9b2c40c..e9d59d8f0fe02 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -175,12 +175,12 @@ CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) return 0; } -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache) +CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) { AssertLockHeld(wallet.cs_wallet); if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); + return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_WATCH_ONLY); } return 0; diff --git a/src/wallet/receive.h b/src/wallet/receive.h index 0283bdcfca024..e365d49edcd4b 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -31,7 +31,7 @@ CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const ismi CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx); CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache = true) +CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); From 4f0ca9bff6299353f595fe168dce720a96a91c41 Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:40:32 -0300 Subject: [PATCH 088/457] wallet: remove always false 'recalculate' arg from GetCachableAmount --- src/wallet/receive.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index e9d59d8f0fe02..5bee00baa6979 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -111,10 +111,10 @@ CAmount TxGetChange(const CWallet& wallet, const CTransaction& tx) return nChange; } -static CAmount GetCachableAmount(const CWallet& wallet, const CWalletTx& wtx, CWalletTx::AmountType type, const isminefilter& filter, bool recalculate = false) +static CAmount GetCachableAmount(const CWallet& wallet, const CWalletTx& wtx, CWalletTx::AmountType type, const isminefilter& filter) { auto& amount = wtx.m_amounts[type]; - if (recalculate || !amount.m_cached[filter]) { + if (!amount.m_cached[filter]) { amount.Set(filter, type == CWalletTx::DEBIT ? wallet.GetDebit(*wtx.tx, filter) : TxGetCredit(wallet, *wtx.tx, filter)); wtx.m_is_cache_empty = false; } From 04c6423f7b250ae1e51bb5cd159913e97494fb0e Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:42:35 -0300 Subject: [PATCH 089/457] wallet: remove always true 'fUseCache' arg from CachedTxGetAvailableCredit --- src/wallet/receive.cpp | 8 ++++---- src/wallet/receive.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 5bee00baa6979..243c5908ea42d 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -186,7 +186,7 @@ CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletT return 0; } -CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache, const isminefilter& filter) +CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) { AssertLockHeld(wallet.cs_wallet); @@ -197,7 +197,7 @@ CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, if (wallet.IsTxImmatureCoinBase(wtx)) return 0; - if (fUseCache && allow_cache && wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_cached[filter]) { + if (allow_cache && wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_cached[filter]) { return wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_value[filter]; } @@ -332,8 +332,8 @@ Balance GetBalance(const CWallet& wallet, const int min_depth, bool avoid_reuse) const CWalletTx& wtx = entry.second; const bool is_trusted{CachedTxIsTrusted(wallet, wtx, trusted_parents)}; const int tx_depth{wallet.GetTxDepthInMainChain(wtx)}; - const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_SPENDABLE | reuse_filter)}; - const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_WATCH_ONLY | reuse_filter)}; + const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, ISMINE_SPENDABLE | reuse_filter)}; + const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, ISMINE_WATCH_ONLY | reuse_filter)}; if (is_trusted && tx_depth >= min_depth) { ret.m_mine_trusted += tx_credit_mine; ret.m_watchonly_trusted += tx_credit_watchonly; diff --git a/src/wallet/receive.h b/src/wallet/receive.h index e365d49edcd4b..ec8d0d1baadce 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -33,7 +33,7 @@ CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); -CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) +CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter = ISMINE_SPENDABLE) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); struct COutputEntry { From 0cb177263c36118094b7cd3b8f94741c0471ff62 Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:47:05 -0300 Subject: [PATCH 090/457] wallet: unify CachedTxGetImmatureCredit and CachedTxGetImmatureWatchOnlyCredit --- src/wallet/receive.cpp | 19 ++++--------------- src/wallet/receive.h | 4 +--- src/wallet/test/wallet_tests.cpp | 4 ++-- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 243c5908ea42d..ec990339a3844 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -164,23 +164,12 @@ CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx) return wtx.nChangeCached; } -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) { AssertLockHeld(wallet.cs_wallet); if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_SPENDABLE); - } - - return 0; -} - -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) -{ - AssertLockHeld(wallet.cs_wallet); - - if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_WATCH_ONLY); + return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, filter); } return 0; @@ -342,8 +331,8 @@ Balance GetBalance(const CWallet& wallet, const int min_depth, bool avoid_reuse) ret.m_mine_untrusted_pending += tx_credit_mine; ret.m_watchonly_untrusted_pending += tx_credit_watchonly; } - ret.m_mine_immature += CachedTxGetImmatureCredit(wallet, wtx); - ret.m_watchonly_immature += CachedTxGetImmatureWatchOnlyCredit(wallet, wtx); + ret.m_mine_immature += CachedTxGetImmatureCredit(wallet, wtx, ISMINE_SPENDABLE); + ret.m_watchonly_immature += CachedTxGetImmatureCredit(wallet, wtx, ISMINE_WATCH_ONLY); } } return ret; diff --git a/src/wallet/receive.h b/src/wallet/receive.h index ec8d0d1baadce..41a70b089adf6 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -29,9 +29,7 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism //! filter decides which addresses will count towards the debit CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter); CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx); -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx) - EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter = ISMINE_SPENDABLE) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 0c03b2f4a2a6a..27aa3f02e9f1e 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -350,13 +350,13 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) // Call GetImmatureCredit() once before adding the key to the wallet to // cache the current immature credit amount, which is 0. - BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 0); + BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx, ISMINE_SPENDABLE), 0); // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); AddKey(wallet, coinbaseKey); - BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 50*COIN); + BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx, ISMINE_SPENDABLE), 50*COIN); } static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) From bf310b0e8ce82d52bacceeb47c9f5dbb26885f7e Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 4 Jul 2022 19:53:04 -0300 Subject: [PATCH 091/457] wallet: clean InputIsMine code, use GetWalletTx --- src/wallet/receive.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index ec990339a3844..1f76704b93eaf 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -9,15 +9,12 @@ #include namespace wallet { -isminetype InputIsMine(const CWallet& wallet, const CTxIn &txin) +isminetype InputIsMine(const CWallet& wallet, const CTxIn& txin) { AssertLockHeld(wallet.cs_wallet); - std::map::const_iterator mi = wallet.mapWallet.find(txin.prevout.hash); - if (mi != wallet.mapWallet.end()) - { - const CWalletTx& prev = (*mi).second; - if (txin.prevout.n < prev.tx->vout.size()) - return wallet.IsMine(prev.tx->vout[txin.prevout.n]); + const CWalletTx* prev = wallet.GetWalletTx(txin.prevout.hash); + if (prev && txin.prevout.n < prev->tx->vout.size()) { + return wallet.IsMine(prev->tx->vout[txin.prevout.n]); } return ISMINE_NO; } From 757216e31cac7dcd45e11b2a2c6148420b3b99da Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Tue, 5 Jul 2022 15:40:52 +0200 Subject: [PATCH 092/457] wallet: don't iter twice when getting the cached debit/credit amount Instead of calling GetCachableAmount twice, which will result in iterating through all the transaction txins/txouts and calling GetDebit/GetCredit (which lock cs_wallet), just merge the filters and do it once. --- src/wallet/receive.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 8de4017371a34..d3303c0b1ffa4 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -130,12 +130,10 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism return 0; CAmount credit = 0; - if (filter & ISMINE_SPENDABLE) { + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { // GetBalance can assume transactions in mapWallet won't change - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_WATCH_ONLY); + credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, get_amount_filter); } return credit; } @@ -146,11 +144,9 @@ CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const ismi return 0; CAmount debit = 0; - if (filter & ISMINE_SPENDABLE) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_WATCH_ONLY); + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { + debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, get_amount_filter); } return debit; } From f1c16ed733f416a3bea878f1c21621dbd93b267c Mon Sep 17 00:00:00 2001 From: Jarol Rodriguez Date: Tue, 5 Jul 2022 14:09:36 -0400 Subject: [PATCH 093/457] doc: remove note on arm cross-compilation from build-unix.md No reason to have this here with outdated information. We already point users to the depends readme, the doc cross builders should be pointed to , within this doc. --- doc/build-unix.md | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/doc/build-unix.md b/doc/build-unix.md index bcfa25dc765b4..874015707a25d 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -288,26 +288,3 @@ This example lists the steps necessary to setup and build a command line only di ./src/bitcoind If you intend to work with legacy Berkeley DB wallets, see [Berkeley DB](#berkeley-db) section. - -ARM Cross-compilation -------------------- -These steps can be performed on, for example, an Ubuntu VM. The depends system -will also work on other Linux distributions, however the commands for -installing the toolchain will be different. - -Make sure you install the build requirements mentioned above. -Then, install the toolchain and curl: - - sudo apt-get install g++-arm-linux-gnueabihf curl - -To build executables for ARM: - - cd depends - make HOST=arm-linux-gnueabihf NO_QT=1 - cd .. - ./autogen.sh - CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-reduce-exports LDFLAGS=-static-libstdc++ - make - - -For further documentation on the depends system see [README.md](../depends/README.md) in the depends directory. From fad690ba0a62dfd97ef22711b2344909fcd1fb73 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Fri, 1 Jul 2022 12:44:24 +0200 Subject: [PATCH 094/457] test: Remove -acceptnonstdtxn=1 from feature_rbf.py --- test/functional/feature_rbf.py | 388 ++++++++++++++++----------------- 1 file changed, 186 insertions(+), 202 deletions(-) diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 91dc222babb7d..40ad2137d4716 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -4,28 +4,18 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" -from copy import deepcopy from decimal import Decimal from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, COIN, - COutPoint, - CTransaction, - CTxIn, - CTxOut, SEQUENCE_FINAL, ) -from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) -from test_framework.script_util import ( - DUMMY_P2WPKH_SCRIPT, - DUMMY_2_P2WPKH_SCRIPT, -) from test_framework.wallet import MiniWallet from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE @@ -35,7 +25,6 @@ def set_test_params(self): self.num_nodes = 2 self.extra_args = [ [ - "-acceptnonstdtxn=1", "-maxorphantx=1000", "-limitancestorcount=50", "-limitancestorsize=101", @@ -96,15 +85,14 @@ def run_test(self): self.log.info("Passed") - def make_utxo(self, node, amount, confirmed=True, scriptPubKey=DUMMY_P2WPKH_SCRIPT): + def make_utxo(self, node, amount, *, confirmed=True, scriptPubKey=None): """Create a txout with a given amount and scriptPubKey - confirmed - txouts created will be confirmed in the blockchain; + confirmed - txout created will be confirmed in the blockchain; unconfirmed otherwise. """ - txid, n = self.wallet.send_to(from_node=node, scriptPubKey=scriptPubKey, amount=amount) + txid, n = self.wallet.send_to(from_node=node, scriptPubKey=scriptPubKey or self.wallet.get_scriptPubKey(), amount=amount) - # If requested, ensure txouts are confirmed. if confirmed: mempool_size = len(node.getrawmempool()) while mempool_size > 0: @@ -115,30 +103,24 @@ def make_utxo(self, node, amount, confirmed=True, scriptPubKey=DUMMY_P2WPKH_SCRI assert new_size < mempool_size mempool_size = new_size - return COutPoint(int(txid, 16), n) + return self.wallet.get_utxo(txid=txid, vout=n) def test_simple_doublespend(self): """Simple doublespend""" # we use MiniWallet to create a transaction template with inputs correctly set, # and modify the output (amount, scriptPubKey) according to our needs - tx_template = self.wallet.create_self_transfer()['tx'] - - tx1a = deepcopy(tx_template) - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) + tx = self.wallet.create_self_transfer()["tx"] + tx1a_txid = self.nodes[0].sendrawtransaction(tx.serialize().hex()) # Should fail because we haven't changed the fee - tx1b = deepcopy(tx_template) - tx1b.vout = [CTxOut(1 * COIN, DUMMY_2_P2WPKH_SCRIPT)] - tx1b_hex = tx1b.serialize().hex() + tx.vout[0].scriptPubKey[-1] ^= 1 # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx.serialize().hex(), 0) # Extra 0.1 BTC fee - tx1b.vout[0].nValue -= int(0.1 * COIN) - tx1b_hex = tx1b.serialize().hex() + tx.vout[0].nValue -= int(0.1 * COIN) + tx1b_hex = tx.serialize().hex() # Works when enabled tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) @@ -160,28 +142,28 @@ def test_doublespend_chain(self): chain_txids = [] while remaining_value > 1 * COIN: remaining_value -= int(0.1 * COIN) - tx = CTransaction() - tx.vin = [CTxIn(prevout, nSequence=0)] - tx.vout = [CTxOut(remaining_value, CScript([1, OP_DROP] * 15 + [1]))] - tx_hex = tx.serialize().hex() - txid = self.nodes[0].sendrawtransaction(tx_hex, 0) - chain_txids.append(txid) - prevout = COutPoint(int(txid, 16), 0) + prevout = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=prevout, + sequence=0, + fee=Decimal("0.1"), + )["new_utxo"] + chain_txids.append(prevout["txid"]) # Whether the double-spend is allowed is evaluated by including all # child fees - 4 BTC - so this attempt is rejected. - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - 3 * COIN, DUMMY_P2WPKH_SCRIPT)] + dbl_tx = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("3"), + )["tx"] dbl_tx_hex = dbl_tx.serialize().hex() # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # Accepted with sufficient fee - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(int(0.1 * COIN), DUMMY_P2WPKH_SCRIPT)] + dbl_tx.vout[0].nValue = int(0.1 * COIN) dbl_tx_hex = dbl_tx.serialize().hex() self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) @@ -205,22 +187,19 @@ def branch(prevout, initial_value, max_txs, tree_width=5, fee=0.00001 * COIN, _t if txout_value < fee: return - vout = [CTxOut(txout_value, CScript([i+1])) - for i in range(tree_width)] - tx = CTransaction() - tx.vin = [CTxIn(prevout, nSequence=0)] - tx.vout = vout - tx_hex = tx.serialize().hex() + tx = self.wallet.send_self_transfer_multi( + utxos_to_spend=[prevout], + from_node=self.nodes[0], + sequence=0, + num_outputs=tree_width, + amount_per_output=txout_value, + ) - assert len(tx.serialize()) < 100000 - txid = self.nodes[0].sendrawtransaction(tx_hex, 0) - yield tx + yield tx["txid"] _total_txs[0] += 1 - txid = int(txid, 16) - - for i, txout in enumerate(tx.vout): - for x in branch(COutPoint(txid, i), txout_value, + for utxo in tx["new_utxos"]: + for x in branch(utxo, txout_value, max_txs, tree_width=tree_width, fee=fee, _total_txs=_total_txs): @@ -232,25 +211,26 @@ def branch(prevout, initial_value, max_txs, tree_width=5, fee=0.00001 * COIN, _t assert_equal(len(tree_txs), n) # Attempt double-spend, will fail because too little fee paid - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - fee * n, DUMMY_P2WPKH_SCRIPT)] - dbl_tx_hex = dbl_tx.serialize().hex() + dbl_tx_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=(Decimal(fee) / COIN) * n, + )["hex"] # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # 0.1 BTC fee is enough - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - fee * n - int(0.1 * COIN), DUMMY_P2WPKH_SCRIPT)] - dbl_tx_hex = dbl_tx.serialize().hex() + dbl_tx_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=(Decimal(fee) / COIN) * n + Decimal("0.1"), + )["hex"] self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) mempool = self.nodes[0].getrawmempool() - for tx in tree_txs: - tx.rehash() - assert tx.hash not in mempool + for txid in tree_txs: + assert txid not in mempool # Try again, but with more total transactions than the "max txs # double-spent at once" anti-DoS limit. @@ -260,33 +240,36 @@ def branch(prevout, initial_value, max_txs, tree_width=5, fee=0.00001 * COIN, _t tree_txs = list(branch(tx0_outpoint, initial_nValue, n, fee=fee)) assert_equal(len(tree_txs), n) - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - 2 * fee * n, DUMMY_P2WPKH_SCRIPT)] - dbl_tx_hex = dbl_tx.serialize().hex() + dbl_tx_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=2 * (Decimal(fee) / COIN) * n, + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) - for tx in tree_txs: - tx.rehash() - self.nodes[0].getrawtransaction(tx.hash) + for txid in tree_txs: + self.nodes[0].getrawtransaction(txid) def test_replacement_feeperkb(self): """Replacement requires fee-per-KB to be higher""" tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - self.nodes[0].sendrawtransaction(tx1a_hex, 0) + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("0.1"), + ) # Higher fee, but the fee per KB is much lower, so the replacement is # rejected. - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.001 * COIN), CScript([b'a' * 999000]))] - tx1b_hex = tx1b.serialize().hex() + tx1b_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[tx0_outpoint], + sequence=0, + num_outputs=100, + amount_per_output=1000, + )["hex"] # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) @@ -296,37 +279,36 @@ def test_spends_of_conflicting_outputs(self): utxo1 = self.make_utxo(self.nodes[0], int(1.2 * COIN)) utxo2 = self.make_utxo(self.nodes[0], 3 * COIN) - tx1a = CTransaction() - tx1a.vin = [CTxIn(utxo1, nSequence=0)] - tx1a.vout = [CTxOut(int(1.1 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) - - tx1a_txid = int(tx1a_txid, 16) + tx1a_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=utxo1, + sequence=0, + fee=Decimal("0.1"), + )["new_utxo"] # Direct spend an output of the transaction we're replacing. - tx2 = CTransaction() - tx2.vin = [CTxIn(utxo1, nSequence=0), CTxIn(utxo2, nSequence=0)] - tx2.vin.append(CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)) - tx2.vout = tx1a.vout - tx2_hex = tx2.serialize().hex() + tx2_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[utxo1, utxo2, tx1a_utxo], + sequence=0, + amount_per_output=int(COIN * tx1a_utxo["value"]), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) # Spend tx1a's output to test the indirect case. - tx1b = CTransaction() - tx1b.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)] - tx1b.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1b_hex = tx1b.serialize().hex() - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) - tx1b_txid = int(tx1b_txid, 16) + tx1b_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1a_utxo, + sequence=0, + fee=Decimal("0.1"), + )["new_utxo"] - tx2 = CTransaction() - tx2.vin = [CTxIn(utxo1, nSequence=0), CTxIn(utxo2, nSequence=0), - CTxIn(COutPoint(tx1b_txid, 0))] - tx2.vout = tx1a.vout - tx2_hex = tx2.serialize().hex() + tx2_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[utxo1, utxo2, tx1b_utxo], + sequence=0, + amount_per_output=int(COIN * tx1a_utxo["value"]), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) @@ -334,18 +316,20 @@ def test_spends_of_conflicting_outputs(self): def test_new_unconfirmed_inputs(self): """Replacements that add new unconfirmed inputs are rejected""" confirmed_utxo = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - unconfirmed_utxo = self.make_utxo(self.nodes[0], int(0.1 * COIN), False) + unconfirmed_utxo = self.make_utxo(self.nodes[0], int(0.1 * COIN), confirmed=False) - tx1 = CTransaction() - tx1.vin = [CTxIn(confirmed_utxo)] - tx1.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1_hex = tx1.serialize().hex() - self.nodes[0].sendrawtransaction(tx1_hex, 0) + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=confirmed_utxo, + sequence=0, + fee=Decimal("0.1"), + ) - tx2 = CTransaction() - tx2.vin = [CTxIn(confirmed_utxo), CTxIn(unconfirmed_utxo)] - tx2.vout = tx1.vout - tx2_hex = tx2.serialize().hex() + tx2_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[confirmed_utxo, unconfirmed_utxo], + sequence=0, + amount_per_output=1 * COIN, + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, 0) @@ -361,45 +345,39 @@ def test_too_many_replacements(self): fee = int(0.0001 * COIN) split_value = int((initial_nValue - fee) / (MAX_REPLACEMENT_LIMIT + 1)) - outputs = [] - for _ in range(MAX_REPLACEMENT_LIMIT + 1): - outputs.append(CTxOut(split_value, CScript([1]))) - - splitting_tx = CTransaction() - splitting_tx.vin = [CTxIn(utxo, nSequence=0)] - splitting_tx.vout = outputs - splitting_tx_hex = splitting_tx.serialize().hex() - - txid = self.nodes[0].sendrawtransaction(splitting_tx_hex, 0) - txid = int(txid, 16) + splitting_tx_utxos = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[utxo], + sequence=0, + num_outputs=MAX_REPLACEMENT_LIMIT + 1, + amount_per_output=split_value, + )["new_utxos"] # Now spend each of those outputs individually - for i in range(MAX_REPLACEMENT_LIMIT + 1): - tx_i = CTransaction() - tx_i.vin = [CTxIn(COutPoint(txid, i), nSequence=0)] - tx_i.vout = [CTxOut(split_value - fee, DUMMY_P2WPKH_SCRIPT)] - tx_i_hex = tx_i.serialize().hex() - self.nodes[0].sendrawtransaction(tx_i_hex, 0) + for utxo in splitting_tx_utxos: + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=utxo, + sequence=0, + fee=Decimal(fee) / COIN, + ) # Now create doublespend of the whole lot; should fail. # Need a big enough fee to cover all spending transactions and have # a higher fee rate double_spend_value = (split_value - 100 * fee) * (MAX_REPLACEMENT_LIMIT + 1) - inputs = [] - for i in range(MAX_REPLACEMENT_LIMIT + 1): - inputs.append(CTxIn(COutPoint(txid, i), nSequence=0)) - double_tx = CTransaction() - double_tx.vin = inputs - double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] + double_tx = self.wallet.create_self_transfer_multi( + utxos_to_spend=splitting_tx_utxos, + sequence=0, + amount_per_output=double_spend_value, + )["tx"] double_tx_hex = double_tx.serialize().hex() # This will raise an exception assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, 0) # If we remove an input, it should pass - double_tx = CTransaction() - double_tx.vin = inputs[0:-1] - double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] + double_tx.vin.pop() double_tx_hex = double_tx.serialize().hex() self.nodes[0].sendrawtransaction(double_tx_hex, 0) @@ -494,20 +472,22 @@ def test_opt_in(self): tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) # Create a non-opting in transaction - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=SEQUENCE_FINAL)] - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) + tx1a_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx0_outpoint, + sequence=SEQUENCE_FINAL, + fee=Decimal("0.1"), + )["new_utxo"] # This transaction isn't shown as replaceable - assert_equal(self.nodes[0].getmempoolentry(tx1a_txid)['bip125-replaceable'], False) + assert_equal(self.nodes[0].getmempoolentry(tx1a_utxo["txid"])['bip125-replaceable'], False) # Shouldn't be able to double-spend - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.9 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx1b_hex = tx1b.serialize().hex() + tx1b_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("0.2"), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, 0) @@ -515,17 +495,19 @@ def test_opt_in(self): tx1_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) # Create a different non-opting in transaction - tx2a = CTransaction() - tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0xfffffffe)] - tx2a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx2a_hex = tx2a.serialize().hex() - tx2a_txid = self.nodes[0].sendrawtransaction(tx2a_hex, 0) + tx2a_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1_outpoint, + sequence=0xfffffffe, + fee=Decimal("0.1"), + )["new_utxo"] # Still shouldn't be able to double-spend - tx2b = CTransaction() - tx2b.vin = [CTxIn(tx1_outpoint, nSequence=0)] - tx2b.vout = [CTxOut(int(0.9 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx2b_hex = tx2b.serialize().hex() + tx2b_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx1_outpoint, + sequence=0, + fee=Decimal("0.2"), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, 0) @@ -534,34 +516,31 @@ def test_opt_in(self): # opt-in on one of the inputs # Transaction should be replaceable on either input - tx1a_txid = int(tx1a_txid, 16) - tx2a_txid = int(tx2a_txid, 16) - - tx3a = CTransaction() - tx3a.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=SEQUENCE_FINAL), - CTxIn(COutPoint(tx2a_txid, 0), nSequence=0xfffffffd)] - tx3a.vout = [CTxOut(int(0.9 * COIN), CScript([b'c'])), CTxOut(int(0.9 * COIN), CScript([b'd']))] - tx3a_hex = tx3a.serialize().hex() - - tx3a_txid = self.nodes[0].sendrawtransaction(tx3a_hex, 0) + tx3a_txid = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[tx1a_utxo, tx2a_utxo], + sequence=[SEQUENCE_FINAL, 0xfffffffd], + fee_per_output=int(0.1 * COIN), + )["txid"] # This transaction is shown as replaceable assert_equal(self.nodes[0].getmempoolentry(tx3a_txid)['bip125-replaceable'], True) - tx3b = CTransaction() - tx3b.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)] - tx3b.vout = [CTxOut(int(0.5 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx3b_hex = tx3b.serialize().hex() - - tx3c = CTransaction() - tx3c.vin = [CTxIn(COutPoint(tx2a_txid, 0), nSequence=0)] - tx3c.vout = [CTxOut(int(0.5 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx3c_hex = tx3c.serialize().hex() + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1a_utxo, + sequence=0, + fee=Decimal("0.4"), + ) - self.nodes[0].sendrawtransaction(tx3b_hex, 0) # If tx3b was accepted, tx3c won't look like a replacement, # but make sure it is accepted anyway - self.nodes[0].sendrawtransaction(tx3c_hex, 0) + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx2a_utxo, + sequence=0, + fee=Decimal("0.4"), + ) def test_prioritised_transactions(self): # Ensure that fee deltas used via prioritisetransaction are @@ -570,17 +549,20 @@ def test_prioritised_transactions(self): # 1. Check that feeperkb uses modified fees tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) + tx1a_txid = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("0.1"), + )["txid"] # Higher fee, but the actual fee per KB is much lower. - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.001 * COIN), CScript([b'a' * 740000]))] - tx1b_hex = tx1b.serialize().hex() + tx1b_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[tx0_outpoint], + sequence=0, + num_outputs=100, + amount_per_output=int(0.00001 * COIN), + )["hex"] # Verify tx1b cannot replace tx1a. assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) @@ -596,27 +578,29 @@ def test_prioritised_transactions(self): # 2. Check that absolute fee checks use modified fee. tx1_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - tx2a = CTransaction() - tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0)] - tx2a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx2a_hex = tx2a.serialize().hex() - self.nodes[0].sendrawtransaction(tx2a_hex, 0) + # tx2a + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1_outpoint, + sequence=0, + fee=Decimal("0.1"), + ) # Lower fee, but we'll prioritise it - tx2b = CTransaction() - tx2b.vin = [CTxIn(tx1_outpoint, nSequence=0)] - tx2b.vout = [CTxOut(int(1.01 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx2b.rehash() - tx2b_hex = tx2b.serialize().hex() + tx2b = self.wallet.create_self_transfer( + utxo_to_spend=tx1_outpoint, + sequence=0, + fee=Decimal("0.09"), + ) # Verify tx2b cannot replace tx2a. - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, 0) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b["hex"], 0) # Now prioritise tx2b to have a higher modified fee - self.nodes[0].prioritisetransaction(txid=tx2b.hash, fee_delta=int(0.1 * COIN)) + self.nodes[0].prioritisetransaction(txid=tx2b["txid"], fee_delta=int(0.1 * COIN)) # tx2b should now be accepted - tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, 0) + tx2b_txid = self.nodes[0].sendrawtransaction(tx2b["hex"], 0) assert tx2b_txid in self.nodes[0].getrawmempool() From 8d869a7bb55a181358031575211810416aadda70 Mon Sep 17 00:00:00 2001 From: glozow Date: Wed, 6 Jul 2022 11:15:39 +0100 Subject: [PATCH 095/457] add glozow builder key --- contrib/builder-keys/keys.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index c70069b440b03..f8377cce33254 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -19,6 +19,7 @@ BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) 6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls) 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) +6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C Gloria Zhao (glozow) D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) 2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob) From 6eb0909cb7d5883a258f76ad6cf2c989fc6f892f Mon Sep 17 00:00:00 2001 From: chinggg <24590067+chinggg@users.noreply.github.com> Date: Wed, 22 Jun 2022 20:44:43 +0800 Subject: [PATCH 096/457] fuzz: add low-level target for txorphanage --- ci/test/06_script_b.sh | 1 + src/Makefile.test.include | 1 + src/test/fuzz/txorphan.cpp | 143 +++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 src/test/fuzz/txorphan.cpp diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index bdb68e0f6f443..32f0ea5e42575 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -46,6 +46,7 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/policy/settings.cpp"\ " src/rpc/fees.cpp"\ " src/rpc/signmessage.cpp"\ + " src/test/fuzz/txorphan.cpp"\ " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" fi diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 098feacb3d79b..b806b62d5bbc8 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -325,6 +325,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/tx_in.cpp \ test/fuzz/tx_out.cpp \ test/fuzz/tx_pool.cpp \ + test/fuzz/txorphan.cpp \ test/fuzz/txrequest.cpp \ test/fuzz/utxo_snapshot.cpp \ test/fuzz/validation_load_mempool.cpp \ diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp new file mode 100644 index 0000000000000..d318baa6a22c6 --- /dev/null +++ b/src/test/fuzz/txorphan.cpp @@ -0,0 +1,143 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include