From 6df927fc6008bb7cb131c6923cfcf809c505b25c Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:51:31 +0000 Subject: [PATCH 01/11] chore: exclude underscore placeholder from shadowing linter warnings Bitcoin uses underscore in util/translation.h for translatable strings but underscores are also a placeholder used in multiple languages, incl. C++. The next commit will be introducing backports, one of them will be placing util/translation.h into validation.h, which is a header used by Dash-specific code. This causes a conflict between the normal usage of underscore as a placeholder and Bitcoin's usage of it as a function name, which is reported by the Dash-specific linter. We need to exclude shadowing warnings from the linter to account for this. --- test/lint/lint-cppcheck-dash.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/lint/lint-cppcheck-dash.sh b/test/lint/lint-cppcheck-dash.sh index 04bcdde741d2d..3c6b9ddd9384f 100755 --- a/test/lint/lint-cppcheck-dash.sh +++ b/test/lint/lint-cppcheck-dash.sh @@ -21,7 +21,7 @@ ENABLED_CHECKS=( "Variable '.*' is assigned a value that is never used." "Unused variable" "The function '.*' overrides a function in a base class but is not marked with a 'override' specifier." -# Enabale to catch all warnings +# Enable to catch all warnings ".*" ) @@ -34,13 +34,14 @@ IGNORED_WARNINGS=( "src/rpc/.*cpp:.*: note: Function pointer used here." "src/masternode/sync.cpp:.*: warning: Variable 'pnode' can be declared as pointer to const \[constVariableReference\]" "src/wallet/bip39.cpp.*: warning: The scope of the variable 'ssCurrentWord' can be reduced. \[variableScope\]" - + "src/.*:.*: warning: Local variable '_' shadows outer function \[shadowFunction\]" "src/stacktraces.cpp:.*: .*: Parameter 'info' can be declared as pointer to const" "src/stacktraces.cpp:.*: note: You might need to cast the function pointer here" "[note|warning]: Return value 'state.Invalid(.*)' is always false" "note: Calling function 'Invalid' returns 0" + "note: Shadow variable" # General catchall, for some reason any value named 'hash' is viewed as never used. "Variable 'hash' is assigned a value that is never used." From d69ca833df133899db267a6f6708d554fce81abe Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Fri, 28 Jun 2024 12:52:27 +0000 Subject: [PATCH 02/11] merge bitcoin#21727: Move more stuff to blockstorage --- src/index/txindex.cpp | 1 + src/init.cpp | 41 ---- src/llmq/instantsend.cpp | 1 + src/node/blockstorage.cpp | 329 ++++++++++++++++++++++++++++++- src/node/blockstorage.h | 51 +++++ src/rpc/index_util.cpp | 3 +- src/validation.cpp | 286 ++------------------------- src/validation.h | 41 +--- src/wallet/test/wallet_tests.cpp | 1 + 9 files changed, 398 insertions(+), 356 deletions(-) diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 7b405436f844d..0e19602c7261b 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/src/init.cpp b/src/init.cpp index 005cca71c5fa8..9eb0883ea5700 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -831,47 +831,6 @@ static void BlockNotifyGenesisWait(const CBlockIndex* pBlockIndex) } } -// If we're using -prune with -reindex, then delete block files that will be ignored by the -// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile -// is missing, do the same here to delete any later block files after a gap. Also delete all -// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile -// is in sync with what's actually on disk by the time we start downloading, so that pruning -// works correctly. -static void CleanupBlockRevFiles() -{ - std::map mapBlockFiles; - - // Glob all blk?????.dat and rev?????.dat files from the blocks directory. - // Remove the rev files immediately and insert the blk file paths into an - // ordered map keyed by block file index. - LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - fs::path blocksdir = gArgs.GetBlocksDirPath(); - for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { - if (fs::is_regular_file(*it) && - it->path().filename().string().length() == 12 && - it->path().filename().string().substr(8,4) == ".dat") - { - if (it->path().filename().string().substr(0,3) == "blk") - mapBlockFiles[it->path().filename().string().substr(3,5)] = it->path(); - else if (it->path().filename().string().substr(0,3) == "rev") - remove(it->path()); - } - } - - // Remove all block files that aren't part of a contiguous set starting at - // zero by walking the ordered map (keys are block file indices) by - // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) - // start removing block files. - int nContigCounter = 0; - for (const std::pair& item : mapBlockFiles) { - if (LocaleIndependentAtoi(item.first) == nContigCounter) { - nContigCounter++; - continue; - } - remove(item.second); - } -} - #if HAVE_SYSTEM static void StartupNotify(const ArgsManager& args) { diff --git a/src/llmq/instantsend.cpp b/src/llmq/instantsend.cpp index 5197384bee594..652285c222f24 100644 --- a/src/llmq/instantsend.cpp +++ b/src/llmq/instantsend.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index d271f93ccac07..58a275a4e927f 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -6,21 +6,317 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include #include +#include #include #include #include -// From validation. TODO move here -bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false); +std::atomic_bool fImporting(false); +std::atomic_bool fReindex(false); +bool fHavePruned = false; +bool fPruneMode = false; +uint64_t nPruneTarget = 0; + +bool fAddressIndex = DEFAULT_ADDRESSINDEX; +bool fTimestampIndex = DEFAULT_TIMESTAMPINDEX; +bool fSpentIndex = DEFAULT_SPENTINDEX; + +// TODO make namespace { +RecursiveMutex cs_LastBlockFile; +std::vector vinfoBlockFile; +int nLastBlockFile = 0; +/** Global flag to indicate we should check to see if there are +* block/undo files that should be deleted. Set on startup +* or if we allocate more file space when we're in prune mode +*/ +bool fCheckForPruning = false; + +/** Dirty block index entries. */ +std::set setDirtyBlockIndex; + +/** Dirty block file entries. */ +std::set setDirtyFileInfo; +// } // namespace + +static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); + +bool IsBlockPruned(const CBlockIndex* pblockindex) +{ + return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); +} + +// If we're using -prune with -reindex, then delete block files that will be ignored by the +// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile +// is missing, do the same here to delete any later block files after a gap. Also delete all +// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile +// is in sync with what's actually on disk by the time we start downloading, so that pruning +// works correctly. +void CleanupBlockRevFiles() +{ + std::map mapBlockFiles; + + // Glob all blk?????.dat and rev?????.dat files from the blocks directory. + // Remove the rev files immediately and insert the blk file paths into an + // ordered map keyed by block file index. + LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); + fs::path blocksdir = gArgs.GetBlocksDirPath(); + for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { + if (fs::is_regular_file(*it) && + it->path().filename().string().length() == 12 && + it->path().filename().string().substr(8,4) == ".dat") + { + if (it->path().filename().string().substr(0, 3) == "blk") { + mapBlockFiles[it->path().filename().string().substr(3, 5)] = it->path(); + } else if (it->path().filename().string().substr(0, 3) == "rev") { + remove(it->path()); + } + } + } + + // Remove all block files that aren't part of a contiguous set starting at + // zero by walking the ordered map (keys are block file indices) by + // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) + // start removing block files. + int nContigCounter = 0; + for (const std::pair& item : mapBlockFiles) { + if (LocaleIndependentAtoi(item.first) == nContigCounter) { + nContigCounter++; + continue; + } + remove(item.second); + } +} + +std::string CBlockFileInfo::ToString() const +{ + return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); +} + +CBlockFileInfo* GetBlockFileInfo(size_t n) +{ + LOCK(cs_LastBlockFile); + + return &vinfoBlockFile.at(n); +} + +static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +{ + // Open history file to append + CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) { + return error("%s: OpenUndoFile failed", __func__); + } + + // Write index header + unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); + fileout << messageStart << nSize; + + // Write undo data + long fileOutPos = ftell(fileout.Get()); + if (fileOutPos < 0) { + return error("%s: ftell failed", __func__); + } + pos.nPos = (unsigned int)fileOutPos; + fileout << blockundo; + + // calculate & write checksum + CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + hasher << hashBlock; + hasher << blockundo; + fileout << hasher.GetHash(); + + return true; +} + +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) +{ + FlatFilePos pos = pindex->GetUndoPos(); + if (pos.IsNull()) { + return error("%s: no undo data available", __func__); + } + + // Open history file to read + CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) { + return error("%s: OpenUndoFile failed", __func__); + } + + // Read block + uint256 hashChecksum; + CHashVerifier verifier(&filein); // We need a CHashVerifier as reserializing may lose data + try { + verifier << pindex->pprev->GetBlockHash(); + verifier >> blockundo; + filein >> hashChecksum; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + + // Verify checksum + if (hashChecksum != verifier.GetHash()) { + return error("%s: Checksum mismatch", __func__); + } + + return true; +} + +static void FlushUndoFile(int block_file, bool finalize = false) +{ + FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); + if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { + AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); + } +} + +void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) +{ + LOCK(cs_LastBlockFile); + FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { + AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); + } + // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, + // e.g. during IBD or a sync after a node going offline + if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); +} + +uint64_t CalculateCurrentUsage() +{ + LOCK(cs_LastBlockFile); + + uint64_t retval = 0; + for (const CBlockFileInfo& file : vinfoBlockFile) { + retval += file.nSize + file.nUndoSize; + } + return retval; +} + +void UnlinkPrunedFiles(const std::set& setFilesToPrune) +{ + for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { + FlatFilePos pos(*it, 0); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); + LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + } +} + +static FlatFileSeq BlockFileSeq() +{ + return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); +} + +static FlatFileSeq UndoFileSeq() +{ + return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); +} + +FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) +{ + return BlockFileSeq().Open(pos, fReadOnly); +} + +/** Open an undo file (rev?????.dat) */ +static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) +{ + return UndoFileSeq().Open(pos, fReadOnly); +} + +fs::path GetBlockPosFilename(const FlatFilePos& pos) +{ + return BlockFileSeq().FileName(pos); +} + +// TODO move to blockstorage +bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) +{ + LOCK(cs_LastBlockFile); + + unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } + + bool finalize_undo = false; + if (!fKnown) { + while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { + // when the undo file is keeping up with the block file, we want to flush it explicitly + // when it is lagging behind (more blocks arrive than are being connected), we let the + // undo block write case handle it + finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); + nFile++; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } + } + pos.nFile = nFile; + pos.nPos = vinfoBlockFile[nFile].nSize; + } + + if ((int)nFile != nLastBlockFile) { + if (!fKnown) { + LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + } + FlushBlockFile(!fKnown, finalize_undo); + nLastBlockFile = nFile; + } + + vinfoBlockFile[nFile].AddBlock(nHeight, nTime); + if (fKnown) { + vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); + } else { + vinfoBlockFile[nFile].nSize += nAddSize; + } + + if (!fKnown) { + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is too low!", _("Disk space is too low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; + } + } + + setDirtyFileInfo.insert(nFile); + return true; +} + +static bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) +{ + pos.nFile = nFile; + + LOCK(cs_LastBlockFile); + + pos.nPos = vinfoBlockFile[nFile].nUndoSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; + setDirtyFileInfo.insert(nFile); + + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; + } + + return true; +} static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { @@ -45,6 +341,35 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa return true; } +bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +{ + // Write undo information to disk + if (pindex->GetUndoPos().IsNull()) { + FlatFilePos _pos; + if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { + return error("ConnectBlock(): FindUndoPos failed"); + } + if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) { + return AbortNode(state, "Failed to write undo data"); + } + // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) + // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height + // in the block file info as below; note that this does not catch the case where the undo writes are keeping up + // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in + // the FindBlockPos function + if (_pos.nFile < nLastBlockFile && static_cast(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { + FlushUndoFile(_pos.nFile, true); + } + + // update nUndoPos in block index + pindex->nUndoPos = _pos.nPos; + pindex->nStatus |= BLOCK_HAVE_UNDO; + setDirtyBlockIndex.insert(pindex); + } + + return true; +} + bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index bcb07175e8c39..378a733dbb080 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -13,7 +13,9 @@ class CActiveMasternodeManager; class ArgsManager; +class BlockValidationState; class CBlock; +class CBlockFileInfo; class CBlockIndex; class CBlockUndo; class CChain; @@ -26,13 +28,62 @@ namespace Consensus { struct Params; } +static constexpr bool DEFAULT_ADDRESSINDEX{false}; +static constexpr bool DEFAULT_SPENTINDEX{false}; static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; +static constexpr bool DEFAULT_TIMESTAMPINDEX{false}; + +/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ +static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ +static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** The maximum size of a blk?????.dat file (since 0.8) */ +static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB + +extern std::atomic_bool fImporting; +extern std::atomic_bool fReindex; +/** Pruning-related variables and constants */ +/** True if any block files have ever been pruned. */ +extern bool fHavePruned; +/** True if we're running in -prune mode. */ +extern bool fPruneMode; +/** Number of MiB of block files that we're trying to stay below. */ +extern uint64_t nPruneTarget; + +/** True if we're running in -addressindex mode. */ +extern bool fAddressIndex; +/** True if we're running in -timestampindex mode. */ +extern bool fTimestampIndex; +/** True if we're running in -spentindex mode. */ +extern bool fSpentIndex; + +//! Check whether the block associated with this index entry is pruned or not. +bool IsBlockPruned(const CBlockIndex* pblockindex); + +void CleanupBlockRevFiles(); + +/** Open a block file (blk?????.dat) */ +FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false); +/** Translation to a filesystem path */ +fs::path GetBlockPosFilename(const FlatFilePos& pos); + +/** Get block file info entry for one block file */ +CBlockFileInfo* GetBlockFileInfo(size_t n); + +/** Calculate the amount of disk space the block & undo files currently use */ +uint64_t CalculateCurrentUsage(); + +/** + * Actually unlink the specified files + */ +void UnlinkPrunedFiles(const std::set& setFilesToPrune); /** Functions for disk access for blocks */ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); +bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams); FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); diff --git a/src/rpc/index_util.cpp b/src/rpc/index_util.cpp index 4e362af88cd9a..113071cb8af93 100644 --- a/src/rpc/index_util.cpp +++ b/src/rpc/index_util.cpp @@ -5,9 +5,10 @@ #include +#include +#include #include #include -#include bool GetAddressIndex(CBlockTreeDB& block_tree_db, const uint160& addressHash, const AddressType type, std::vector& addressIndex, diff --git a/src/validation.cpp b/src/validation.cpp index 15be0df400ea4..e9bc694f5cea2 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -77,10 +77,6 @@ /** Maximum kilobytes for transactions to store for processing during reorg */ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; -/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ -static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB -/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ -static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** Time to wait between writing blocks/block index to disk. */ static constexpr std::chrono::hours DATABASE_WRITE_INTERVAL{1}; /** Time to wait between flushing chainstate to disk. */ @@ -132,17 +128,9 @@ Mutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; bool g_parallel_script_checks{false}; -std::atomic_bool fImporting(false); -std::atomic_bool fReindex(false); -bool fAddressIndex = DEFAULT_ADDRESSINDEX; -bool fTimestampIndex = DEFAULT_TIMESTAMPINDEX; -bool fSpentIndex = DEFAULT_SPENTINDEX; -bool fHavePruned = false; -bool fPruneMode = false; bool fRequireStandard = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; -uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; // TODO: drop this global variable. Used by net.cpp module only @@ -156,22 +144,17 @@ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); // Internal stuff namespace { CBlockIndex* pindexBestInvalid = nullptr; - - RecursiveMutex cs_LastBlockFile; - std::vector vinfoBlockFile; - int nLastBlockFile = 0; - /** Global flag to indicate we should check to see if there are - * block/undo files that should be deleted. Set on startup - * or if we allocate more file space when we're in prune mode - */ - bool fCheckForPruning = false; - - /** Dirty block index entries. */ - std::set setDirtyBlockIndex; - - /** Dirty block file entries. */ - std::set setDirtyFileInfo; -} // anon namespace +} // namespace + +// Internal stuff from blockstorage ... +extern RecursiveMutex cs_LastBlockFile; +extern std::vector vinfoBlockFile; +extern int nLastBlockFile; +extern bool fCheckForPruning; +extern std::set setDirtyBlockIndex; +extern std::set setDirtyFileInfo; +void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); +// ... TODO move fully to blockstorage CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const { @@ -199,9 +182,6 @@ CBlockIndex* BlockManager::FindForkInGlobalIndex(const CChain& chain, const CBlo std::unique_ptr pblocktree; bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector *pvChecks = nullptr); -static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); -static FlatFileSeq BlockFileSeq(); -static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags) { @@ -1557,65 +1537,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const C return true; } -static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) -{ - // Open history file to append - CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); - if (fileout.IsNull()) - return error("%s: OpenUndoFile failed", __func__); - - // Write index header - unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); - fileout << messageStart << nSize; - - // Write undo data - long fileOutPos = ftell(fileout.Get()); - if (fileOutPos < 0) - return error("%s: ftell failed", __func__); - pos.nPos = (unsigned int)fileOutPos; - fileout << blockundo; - - // calculate & write checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - fileout << hasher.GetHash(); - - return true; -} - -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) -{ - FlatFilePos pos = pindex->GetUndoPos(); - if (pos.IsNull()) { - return error("%s: no undo data available", __func__); - } - - // Open history file to read - CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); - if (filein.IsNull()) - return error("%s: OpenUndoFile failed", __func__); - - // Read block - uint256 hashChecksum; - CHashVerifier verifier(&filein); // We need a CHashVerifier as reserializing may lose data - try { - verifier << pindex->pprev->GetBlockHash(); - verifier >> blockundo; - filein >> hashChecksum; - } - catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - - // Verify checksum - if (hashChecksum != verifier.GetHash()) - return error("%s: Checksum mismatch", __func__); - - return true; -} - -static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str()) +bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage) { AbortNode(strMessage, userMessage); return state.Error(strMessage); @@ -1819,55 +1741,6 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -static void FlushUndoFile(int block_file, bool finalize = false) -{ - FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); - if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { - AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); - } -} - -static void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) -{ - LOCK(cs_LastBlockFile); - FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); - if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { - AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); - } - // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, - // e.g. during IBD or a sync after a node going offline - if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); -} - -static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); - -static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) -{ - // Write undo information to disk - if (pindex->GetUndoPos().IsNull()) { - FlatFilePos _pos; - if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) - return error("ConnectBlock(): FindUndoPos failed"); - if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) - return AbortNode(state, "Failed to write undo data"); - // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) - // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height - // in the block file info as below; note that this does not catch the case where the undo writes are keeping up - // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in - // the FindBlockPos function - if (_pos.nFile < nLastBlockFile && static_cast(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { - FlushUndoFile(_pos.nFile, true); - } - - // update nUndoPos in block index - pindex->nUndoPos = _pos.nPos; - pindex->nStatus |= BLOCK_HAVE_UNDO; - setDirtyBlockIndex.insert(pindex); - } - - return true; -} - static CCheckQueue scriptcheckqueue(128); void StartScriptCheckWorkerThreads(int threads_num) @@ -3624,83 +3497,6 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -// TODO move to blockstorage -bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) -{ - LOCK(cs_LastBlockFile); - - unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); - } - - bool finalize_undo = false; - if (!fKnown) { - while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { - // when the undo file is keeping up with the block file, we want to flush it explicitly - // when it is lagging behind (more blocks arrive than are being connected), we let the - // undo block write case handle it - finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); - nFile++; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); - } - } - pos.nFile = nFile; - pos.nPos = vinfoBlockFile[nFile].nSize; - } - - if ((int)nFile != nLastBlockFile) { - if (!fKnown) { - LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); - } - FlushBlockFile(!fKnown, finalize_undo); - nLastBlockFile = nFile; - } - - vinfoBlockFile[nFile].AddBlock(nHeight, nTime); - if (fKnown) - vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); - else - vinfoBlockFile[nFile].nSize += nAddSize; - - if (!fKnown) { - bool out_of_space; - size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - return AbortNode("Disk space is too low!", _("Disk space is too low!")); - } - if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; - } - } - - setDirtyFileInfo.insert(nFile); - return true; -} - -static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) -{ - pos.nFile = nFile; - - LOCK(cs_LastBlockFile); - - pos.nPos = vinfoBlockFile[nFile].nUndoSize; - vinfoBlockFile[nFile].nUndoSize += nAddSize; - setDirtyFileInfo.insert(nFile); - - bool out_of_space; - size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); - } - if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; - } - - return true; -} - static bool CheckBlockHeader(const CBlockHeader& block, const uint256& hash, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true) { // Check proof of work matches claimed amount @@ -4272,18 +4068,6 @@ bool TestBlockValidity(BlockValidationState& state, * BLOCK PRUNING CODE */ -/* Calculate the amount of disk space the block & undo files currently use */ -uint64_t CalculateCurrentUsage() -{ - LOCK(cs_LastBlockFile); - - uint64_t retval = 0; - for (const CBlockFileInfo &file : vinfoBlockFile) { - retval += file.nSize + file.nUndoSize; - } - return retval; -} - void BlockManager::PruneOneBlockFile(const int fileNumber) { AssertLockHeld(cs_main); @@ -4318,17 +4102,6 @@ void BlockManager::PruneOneBlockFile(const int fileNumber) setDirtyFileInfo.insert(fileNumber); } - -void UnlinkPrunedFiles(const std::set& setFilesToPrune) -{ - for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - FlatFilePos pos(*it, 0); - fs::remove(BlockFileSeq().FileName(pos)); - fs::remove(UndoFileSeq().FileName(pos)); - LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); - } -} - void BlockManager::FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight, int chain_tip_height) { assert(fPruneMode && nManualPruneHeight > 0); @@ -4420,30 +4193,6 @@ void BlockManager::FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPr nLastBlockWeCanPrune, count); } -static FlatFileSeq BlockFileSeq() -{ - return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); -} - -static FlatFileSeq UndoFileSeq() -{ - return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); -} - -FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { - return BlockFileSeq().Open(pos, fReadOnly); -} - -/** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { - return UndoFileSeq().Open(pos, fReadOnly); -} - -fs::path GetBlockPosFilename(const FlatFilePos &pos) -{ - return BlockFileSeq().FileName(pos); -} - CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); @@ -5398,17 +5147,6 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) return ret; } -std::string CBlockFileInfo::ToString() const { - return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); -} - -CBlockFileInfo* GetBlockFileInfo(size_t n) -{ - LOCK(cs_LastBlockFile); - - return &vinfoBlockFile.at(n); -} - static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) diff --git a/src/validation.h b/src/validation.h index 9b2541fd933a5..6cead8b7a9918 100644 --- a/src/validation.h +++ b/src/validation.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -78,8 +79,6 @@ static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; -/** The maximum size of a blk?????.dat file (since 0.8) */ -static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** Maximum number of dedicated script-checking threads allowed */ static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ @@ -93,9 +92,6 @@ static const int64_t DEFAULT_MAX_TIP_AGE = 6 * 60 * 60; // ~144 blocks behind -> static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = true; static constexpr bool DEFAULT_COINSTATSINDEX{false}; -static const bool DEFAULT_ADDRESSINDEX = false; -static const bool DEFAULT_TIMESTAMPINDEX = false; -static const bool DEFAULT_SPENTINDEX = false; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; /** Default for -persistmempool */ static const bool DEFAULT_PERSIST_MEMPOOL = true; @@ -132,11 +128,6 @@ typedef std::unordered_multimap PrevBlockMap extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; extern uint256 g_best_block; -extern std::atomic_bool fImporting; -extern std::atomic_bool fReindex; -extern bool fAddressIndex; -extern bool fTimestampIndex; -extern bool fSpentIndex; /** Whether there are dedicated script-checking threads running. * False indicates all script checking is done on the main threadMessageHandler thread. */ @@ -163,20 +154,9 @@ extern arith_uint256 nMinimumChainWork; /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; -/** Pruning-related variables and constants */ -/** True if any block files have ever been pruned. */ -extern bool fHavePruned; -/** True if we're running in -prune mode. */ -extern bool fPruneMode; -/** Number of MiB of block files that we're trying to stay below. */ -extern uint64_t nPruneTarget; /** Documentation for argument 'checklevel'. */ extern const std::vector CHECKLEVEL_DOC; -/** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); -/** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Unload database information */ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman); /** Run instances of script checking worker threads */ @@ -199,17 +179,11 @@ CAmount GetSuperblockSubsidyInner(int nPrevBits, int nPrevHeight, const Consensu CAmount GetBlockSubsidy(const CBlockIndex* const pindex, const Consensus::Params& consensusParams); CAmount GetMasternodePayment(int nHeight, CAmount blockValue, bool fV20Active); +bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str{}); + /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pindex); -/** Calculate the amount of disk space the block & undo files currently use */ -uint64_t CalculateCurrentUsage(); - -/** - * Actually unlink the specified files - */ -void UnlinkPrunedFiles(const std::set& setFilesToPrune); - /** Prune block files up to a given height */ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeight); @@ -1133,9 +1107,6 @@ extern std::unique_ptr pblocktree; */ bool GetBlockHash(const CChain& active_chain, uint256& hashRet, int nBlockHeight = -1); -/** Get block file info entry for one block file */ -CBlockFileInfo* GetBlockFileInfo(size_t n); - using FopenFn = std::function; /** Dump the mempool to disk. */ @@ -1144,12 +1115,6 @@ bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function = fsbri /** Load the mempool from disk. */ bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function = fsbridge::fopen); -//! Check whether the block associated with this index entry is pruned or not. -inline bool IsBlockPruned(const CBlockIndex* pblockindex) -{ - return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); -} - /** * Return the expected assumeutxo value for a given height, if one exists. * diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index c6fc807db18fb..8d024d3f7500c 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include From 472caa048a9d5ab4acc37aa86915a97a21483138 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 26 Jun 2024 18:25:14 +0000 Subject: [PATCH 03/11] merge bitcoin#22371: Move pblocktree global to BlockManager --- src/index/txindex.cpp | 2 +- src/init.cpp | 2 +- src/node/blockstorage.cpp | 2 +- src/rpc/blockchain.cpp | 3 +- src/rpc/misc.cpp | 24 ++++++--- src/rpc/rawtransaction.cpp | 4 +- src/test/util/setup_common.cpp | 4 +- src/test/validation_chainstate_tests.cpp | 1 + src/validation.cpp | 66 ++++++++++++------------ src/validation.h | 11 ++-- 10 files changed, 62 insertions(+), 57 deletions(-) diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 0e19602c7261b..d2b72ccfbbb8a 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -204,7 +204,7 @@ bool TxIndex::Init() // Attempt to migrate txindex from the old database to the new one. Even if // chain_tip is null, the node could be reindexing and we still want to // delete txindex records in the old database. - if (!m_db->MigrateData(*pblocktree, m_chainstate->m_chain.GetLocator())) { + if (!m_db->MigrateData(*m_chainstate->m_blockman.m_block_tree_db, m_chainstate->m_chain.GetLocator())) { return false; } diff --git a/src/init.cpp b/src/init.cpp index 9eb0883ea5700..8e02350cafb1c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -338,7 +338,6 @@ void PrepareShutdown(NodeContext& node) chainstate->ResetCoinsViews(); } } - pblocktree.reset(); node.chain_helper.reset(); if (node.mnhf_manager) { node.mnhf_manager->DisconnectManagers(); @@ -1951,6 +1950,7 @@ bool AppInitMain(const CoreContext& context, NodeContext& node, interfaces::Bloc UnloadBlockIndex(node.mempool.get(), chainman); + auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 58a275a4e927f..2b2f21ca6b6fa 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -472,7 +472,7 @@ void ThreadImport(ChainstateManager& chainman, CDeterministicMNManager& dmnman, } nFile++; } - pblocktree->WriteReindexing(false); + WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false)); fReindex = false; LogPrintf("Reindexing finished\n"); // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked): diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 524736d560d1e..73cae2a2258cd 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -835,7 +835,8 @@ static RPCHelpMan getblockhashes() unsigned int low = request.params[1].get_int(); std::vector blockHashes; - if (LOCK(::cs_main); !GetTimestampIndex(*pblocktree, high, low, blockHashes)) { + ChainstateManager& chainman = EnsureAnyChainman(request.context); + if (LOCK(::cs_main); !GetTimestampIndex(*chainman.m_blockman.m_block_tree_db, high, low, blockHashes)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for block hashes"); } diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 377acc2679443..31eefff0c0c51 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -813,10 +813,11 @@ static RPCHelpMan getaddressutxos() std::vector unspentOutputs; + ChainstateManager& chainman = EnsureAnyChainman(request.context); { LOCK(::cs_main); for (const auto& address : addresses) { - if (!GetAddressUnspentIndex(*pblocktree, address.first, address.second, unspentOutputs, + if (!GetAddressUnspentIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, unspentOutputs, /* height_sort = */ true)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } @@ -900,15 +901,20 @@ static RPCHelpMan getaddressdeltas() std::vector addressIndex; + ChainstateManager& chainman = EnsureAnyChainman(request.context); { LOCK(::cs_main); for (const auto& address : addresses) { if (start > 0 && end > 0) { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex, start, end)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, + addressIndex, start, end)) + { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } else { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, + addressIndex)) + { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } @@ -978,7 +984,7 @@ static RPCHelpMan getaddressbalance() { LOCK(::cs_main); for (const auto& address : addresses) { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, addressIndex)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } @@ -1056,15 +1062,18 @@ static RPCHelpMan getaddresstxids() std::vector addressIndex; + ChainstateManager& chainman = EnsureAnyChainman(request.context); { LOCK(::cs_main); for (const auto& address : addresses) { if (start > 0 && end > 0) { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex, start, end)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, + addressIndex, start, end)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } else { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, + addressIndex)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } @@ -1137,8 +1146,9 @@ static RPCHelpMan getspentinfo() CSpentIndexKey key(txid, outputIndex); CSpentIndexValue value; + ChainstateManager& chainman = EnsureAnyChainman(request.context); CTxMemPool& mempool = EnsureAnyMemPool(request.context); - if (LOCK(::cs_main); !GetSpentIndex(*pblocktree, mempool, key, value)) { + if (LOCK(::cs_main); !GetSpentIndex(*chainman.m_blockman.m_block_tree_db, mempool, key, value)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to get spent info"); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index da0010acc6ab6..2ccf51b5824bb 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -75,7 +75,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, CTxMemPool& mempo if (!tx.IsCoinBase()) { CSpentIndexValue spentInfo; CSpentIndexKey spentKey(txin.prevout.hash, txin.prevout.n); - if (GetSpentIndex(*pblocktree, mempool, spentKey, spentInfo)) { + if (GetSpentIndex(*active_chainstate.m_blockman.m_block_tree_db, mempool, spentKey, spentInfo)) { txSpentInfo.mSpentInfo.emplace(spentKey, spentInfo); } } @@ -83,7 +83,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, CTxMemPool& mempo for (unsigned int i = 0; i < tx.vout.size(); i++) { CSpentIndexValue spentInfo; CSpentIndexKey spentKey(txid, i); - if (GetSpentIndex(*pblocktree, mempool, spentKey, spentInfo)) { + if (GetSpentIndex(*active_chainstate.m_blockman.m_block_tree_db, mempool, spentKey, spentInfo)) { txSpentInfo.mSpentInfo.emplace(spentKey, spentInfo); } } diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 85281ef164c50..cc7a942d8b0c7 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -224,12 +224,11 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve m_node.scheduler->m_service_thread = std::thread(util::TraceThread, "scheduler", [&] { m_node.scheduler->serviceQueue(); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); - pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - m_node.fee_estimator = std::make_unique(); m_node.mempool = std::make_unique(m_node.fee_estimator.get(), 1); m_node.chainman = std::make_unique(); + m_node.chainman->m_blockman.m_block_tree_db = std::make_unique(1 << 20, true); m_node.connman = std::make_unique(0x1337, 0x1337, *m_node.addrman); // Deterministic randomness for tests. @@ -264,7 +263,6 @@ ChainTestingSetup::~ChainTestingSetup() m_node.scheduler.reset(); m_node.chainman->Reset(); m_node.chainman.reset(); - pblocktree.reset(); } TestingSetup::TestingSetup(const std::string& chainName, const std::vector& extra_args) diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index c853747e5a190..746733c75ddd8 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -25,6 +25,7 @@ BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) { ChainstateManager manager; + WITH_LOCK(::cs_main, manager.m_blockman.m_block_tree_db = std::make_unique(1 << 20, true)); CTxMemPool mempool; //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given view. diff --git a/src/validation.cpp b/src/validation.cpp index e9bc694f5cea2..78309491d9443 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -179,8 +179,6 @@ CBlockIndex* BlockManager::FindForkInGlobalIndex(const CChain& chain, const CBlo return chain.Genesis(); } -std::unique_ptr pblocktree; - bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector *pvChecks = nullptr); bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags) @@ -1700,25 +1698,25 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI if (fSpentIndex) { - if (!pblocktree->UpdateSpentIndex(spentIndex)) { + if (!m_blockman.m_block_tree_db->UpdateSpentIndex(spentIndex)) { AbortNode("Failed to delete spent index"); return DISCONNECT_FAILED; } } if (fAddressIndex) { - if (!pblocktree->EraseAddressIndex(addressIndex)) { + if (!m_blockman.m_block_tree_db->EraseAddressIndex(addressIndex)) { AbortNode("Failed to delete address index"); return DISCONNECT_FAILED; } - if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + if (!m_blockman.m_block_tree_db->UpdateAddressUnspentIndex(addressUnspentIndex)) { AbortNode("Failed to write address unspent index"); return DISCONNECT_FAILED; } } if (fTimestampIndex) { - if (!pblocktree->EraseTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) { + if (!m_blockman.m_block_tree_db->EraseTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) { AbortNode("Failed to delete timestamp index"); return DISCONNECT_FAILED; } @@ -2270,21 +2268,21 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, int64_t nTime6 = GetTimeMicros(); if (fAddressIndex) { - if (!pblocktree->WriteAddressIndex(addressIndex)) { + if (!m_blockman.m_block_tree_db->WriteAddressIndex(addressIndex)) { return AbortNode(state, "Failed to write address index"); } - if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + if (!m_blockman.m_block_tree_db->UpdateAddressUnspentIndex(addressUnspentIndex)) { return AbortNode(state, "Failed to write address unspent index"); } } if (fSpentIndex) - if (!pblocktree->UpdateSpentIndex(spentIndex)) + if (!m_blockman.m_block_tree_db->UpdateSpentIndex(spentIndex)) return AbortNode(state, "Failed to write transaction index"); if (fTimestampIndex) - if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + if (!m_blockman.m_block_tree_db->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) return AbortNode(state, "Failed to write timestamp index"); int64_t nTime7 = GetTimeMicros(); nTimeIndexWrite += nTime7 - nTime6; @@ -2387,7 +2385,7 @@ bool CChainState::FlushStateToDisk( if (!setFilesToPrune.empty()) { fFlushForPrune = true; if (!fHavePruned) { - pblocktree->WriteFlag("prunedblockfiles", true); + m_blockman.m_block_tree_db->WriteFlag("prunedblockfiles", true); fHavePruned = true; } } @@ -2442,7 +2440,7 @@ bool CChainState::FlushStateToDisk( vBlocks.push_back(*it); setDirtyBlockIndex.erase(it++); } - if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { + if (!m_blockman.m_block_tree_db->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { return AbortNode(state, "Failed to write to block index database"); } } @@ -4215,11 +4213,11 @@ CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash) bool BlockManager::LoadBlockIndex( const Consensus::Params& consensus_params, - CBlockTreeDB& blocktree, std::set& block_index_candidates) { - if (!blocktree.LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) + if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { return false; + } // Calculate nChainWork std::vector > vSortedByHeight; @@ -4285,25 +4283,25 @@ void BlockManager::Unload() { m_prev_block_index.clear(); } -bool CChainState::LoadBlockIndexDB() +bool BlockManager::LoadBlockIndexDB(std::set& setBlockIndexCandidates) { - if (!m_blockman.LoadBlockIndex( - m_params.GetConsensus(), *pblocktree, + if (!LoadBlockIndex( + ::Params().GetConsensus(), setBlockIndexCandidates)) { return false; } // Load block file info - pblocktree->ReadLastBlockFile(nLastBlockFile); + m_block_tree_db->ReadLastBlockFile(nLastBlockFile); vinfoBlockFile.resize(nLastBlockFile + 1); LogPrintf("%s: last block file = %i\n", __func__, nLastBlockFile); for (int nFile = 0; nFile <= nLastBlockFile; nFile++) { - pblocktree->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); + m_block_tree_db->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); } LogPrintf("%s: last block file info: %s\n", __func__, vinfoBlockFile[nLastBlockFile].ToString()); for (int nFile = nLastBlockFile + 1; true; nFile++) { CBlockFileInfo info; - if (pblocktree->ReadBlockFileInfo(nFile, info)) { + if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) { vinfoBlockFile.push_back(info); } else { break; @@ -4313,7 +4311,7 @@ bool CChainState::LoadBlockIndexDB() // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set setBlkDataFiles; - for (const std::pair& item : m_blockman.m_block_index) { + for (const std::pair& item : m_block_index) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { setBlkDataFiles.insert(pindex->nFile); @@ -4328,25 +4326,25 @@ bool CChainState::LoadBlockIndexDB() } // Check whether we have ever pruned block & undo files - pblocktree->ReadFlag("prunedblockfiles", fHavePruned); + m_block_tree_db->ReadFlag("prunedblockfiles", fHavePruned); if (fHavePruned) LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); // Check whether we need to continue reindexing bool fReindexing = false; - pblocktree->ReadReindexing(fReindexing); + m_block_tree_db->ReadReindexing(fReindexing); if(fReindexing) fReindex = true; // Check whether we have an address index - pblocktree->ReadFlag("addressindex", fAddressIndex); + m_block_tree_db->ReadFlag("addressindex", fAddressIndex); LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); // Check whether we have a timestamp index - pblocktree->ReadFlag("timestampindex", fTimestampIndex); + m_block_tree_db->ReadFlag("timestampindex", fTimestampIndex); LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); // Check whether we have a spent index - pblocktree->ReadFlag("spentindex", fSpentIndex); + m_block_tree_db->ReadFlag("spentindex", fSpentIndex); LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); return true; @@ -4596,22 +4594,22 @@ bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& i } if (fAddressIndex) { - if (!pblocktree->WriteAddressIndex(addressIndex)) { + if (!m_blockman.m_block_tree_db->WriteAddressIndex(addressIndex)) { return error("RollforwardBlock(DASH): Failed to write address index"); } - if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + if (!m_blockman.m_block_tree_db->UpdateAddressUnspentIndex(addressUnspentIndex)) { return error("RollforwardBlock(DASH): Failed to write address unspent index"); } } if (fSpentIndex) { - if (!pblocktree->UpdateSpentIndex(spentIndex)) + if (!m_blockman.m_block_tree_db->UpdateSpentIndex(spentIndex)) return error("RollforwardBlock(DASH): Failed to write transaction index"); } if (fTimestampIndex) { - if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + if (!m_blockman.m_block_tree_db->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) return error("RollforwardBlock(DASH): Failed to write timestamp index"); } @@ -4726,7 +4724,7 @@ bool ChainstateManager::LoadBlockIndex() // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = ActiveChainstate().LoadBlockIndexDB(); + bool ret = m_blockman.LoadBlockIndexDB(ActiveChainstate().setBlockIndexCandidates); if (!ret) return false; needs_init = m_blockman.m_block_index.empty(); } @@ -4742,15 +4740,15 @@ bool ChainstateManager::LoadBlockIndex() // Use the provided setting for -addressindex in the new database fAddressIndex = gArgs.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); - pblocktree->WriteFlag("addressindex", fAddressIndex); + m_blockman.m_block_tree_db->WriteFlag("addressindex", fAddressIndex); // Use the provided setting for -timestampindex in the new database fTimestampIndex = gArgs.GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); - pblocktree->WriteFlag("timestampindex", fTimestampIndex); + m_blockman.m_block_tree_db->WriteFlag("timestampindex", fTimestampIndex); // Use the provided setting for -spentindex in the new database fSpentIndex = gArgs.GetBoolArg("-spentindex", DEFAULT_SPENTINDEX); - pblocktree->WriteFlag("spentindex", fSpentIndex); + m_blockman.m_block_tree_db->WriteFlag("spentindex", fSpentIndex); } return true; } diff --git a/src/validation.h b/src/validation.h index 6cead8b7a9918..6c5b2974cd157 100644 --- a/src/validation.h +++ b/src/validation.h @@ -468,6 +468,10 @@ class BlockManager */ std::multimap m_blocks_unlinked; + std::unique_ptr m_block_tree_db GUARDED_BY(::cs_main); + + bool LoadBlockIndexDB(std::set& setBlockIndexCandidates) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** * Load the blocktree off disk and into memory. Populate certain metadata * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral @@ -478,7 +482,6 @@ class BlockManager */ bool LoadBlockIndex( const Consensus::Params& consensus_params, - CBlockTreeDB& blocktree, std::set& block_index_candidates) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Clear all data members. */ @@ -838,8 +841,6 @@ class CChainState void InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(cs_main); void ConflictingChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - //! Indirection necessary to make lock annotations work with an optional mempool. RecursiveMutex* MempoolMutex() const LOCK_RETURNED(m_mempool->cs) { @@ -1097,10 +1098,6 @@ class ChainstateManager } }; -/** Global variable that points to the active block tree (protected by cs_main) */ -extern std::unique_ptr pblocktree; - - /** * Return true if hash can be found in active_chain at nBlockHeight height. * Fills hashRet with found hash, if no nBlockHeight is specified - active_chain.Height() is used. From a08f2f48bf4bb6e39e94a0e6a0c1a8ef0ab18558 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Thu, 27 Jun 2024 20:34:51 +0000 Subject: [PATCH 04/11] merge bitcoin#21526: UpdateTip/CheckBlockIndex assumeutxo support --- src/Makefile.test_util.include | 1 + src/chain.h | 22 ++- src/test/evo_deterministicmns_tests.cpp | 18 +-- src/test/util/chainstate.h | 54 +++++++ src/test/util/setup_common.cpp | 32 +++-- src/test/util/setup_common.h | 18 ++- src/test/validation_chainstate_tests.cpp | 76 ++++++++++ .../validation_chainstatemanager_tests.cpp | 66 ++------- src/test/validation_flush_tests.cpp | 5 +- src/validation.cpp | 136 +++++++++++++----- src/validation.h | 33 ++--- 11 files changed, 326 insertions(+), 135 deletions(-) create mode 100644 src/test/util/chainstate.h diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 1a805a1dd1f50..ee349bbc6f248 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -9,6 +9,7 @@ EXTRA_LIBRARIES += \ TEST_UTIL_H = \ test/util/blockfilter.h \ + test/util/chainstate.h \ test/util/index.h \ test/util/logging.h \ test/util/mining.h \ diff --git a/src/chain.h b/src/chain.h index c1d122234ea6e..b3170a8347f7e 100644 --- a/src/chain.h +++ b/src/chain.h @@ -124,6 +124,14 @@ enum BlockStatus: uint32_t { BLOCK_FAILED_MASK = BLOCK_FAILED_VALID | BLOCK_FAILED_CHILD, BLOCK_CONFLICT_CHAINLOCK = 128, //!< conflicts with chainlock system + + /** + * If set, this indicates that the block index entry is assumed-valid. + * Certain diagnostics will be skipped in e.g. CheckBlockIndex(). + * It almost certainly means that the block's full validation is pending + * on a background chainstate. See `doc/assumeutxo.md`. + */ + BLOCK_ASSUMED_VALID = 256, }; /** The block chain is a tree shaped structure starting with the @@ -291,14 +299,24 @@ class CBlockIndex return ((nStatus & BLOCK_VALID_MASK) >= nUpTo); } + //! @returns true if the block is assumed-valid; this means it is queued to be + //! validated by a background chainstate. + bool IsAssumedValid() const { return nStatus & BLOCK_ASSUMED_VALID; } + //! Raise the validity level of this block index entry. //! Returns true if the validity was changed. bool RaiseValidity(enum BlockStatus nUpTo) { assert(!(nUpTo & ~BLOCK_VALID_MASK)); // Only validity flags allowed. - if (nStatus & BLOCK_FAILED_MASK) - return false; + if (nStatus & BLOCK_FAILED_MASK) return false; + if ((nStatus & BLOCK_VALID_MASK) < nUpTo) { + // If this block had been marked assumed-valid and we're raising + // its validity to a certain point, there is no longer an assumption. + if (nStatus & BLOCK_ASSUMED_VALID && nUpTo >= BLOCK_VALID_SCRIPTS) { + nStatus &= ~BLOCK_ASSUMED_VALID; + } + nStatus = (nStatus & ~BLOCK_VALID_MASK) | nUpTo; return true; } diff --git a/src/test/evo_deterministicmns_tests.cpp b/src/test/evo_deterministicmns_tests.cpp index 48b73ebf99abc..caedcca2d9295 100644 --- a/src/test/evo_deterministicmns_tests.cpp +++ b/src/test/evo_deterministicmns_tests.cpp @@ -251,7 +251,7 @@ void FuncDIP3Activation(TestChainSetup& setup) int nHeight = chainman.ActiveChain().Height(); // We start one block before DIP3 activation, so mining a block with a DIP3 transaction should fail - auto block = std::make_shared(setup.CreateBlock(txns, setup.coinbaseKey)); + auto block = std::make_shared(setup.CreateBlock(txns, setup.coinbaseKey, chainman.ActiveChainstate())); chainman.ProcessNewBlock(Params(), block, true, nullptr); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight); BOOST_ASSERT(block->GetHash() != chainman.ActiveChain().Tip()->GetBlockHash()); @@ -261,7 +261,7 @@ void FuncDIP3Activation(TestChainSetup& setup) setup.CreateAndProcessBlock({}, setup.coinbaseKey); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 1); // Mining a block with a DIP3 transaction should succeed now - block = std::make_shared(setup.CreateBlock(txns, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock(txns, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 2); @@ -288,7 +288,7 @@ void FuncV19Activation(TestChainSetup& setup) int nHeight = chainman.ActiveChain().Height(); - auto block = std::make_shared(setup.CreateBlock({tx_reg}, setup.coinbaseKey)); + auto block = std::make_shared(setup.CreateBlock({tx_reg}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); BOOST_ASSERT(!DeploymentActiveAfter(chainman.ActiveChain().Tip(), Params().GetConsensus(), Consensus::DEPLOYMENT_V19)); ++nHeight; @@ -306,7 +306,7 @@ void FuncV19Activation(TestChainSetup& setup) operator_key_new.MakeNewKey(); auto tx_upreg = CreateProUpRegTx(chainman.ActiveChain(), *(setup.m_node.mempool), utxos, tx_reg_hash, owner_key, operator_key_new.GetPublicKey(), owner_key.GetPubKey().GetID(), collateralScript, setup.coinbaseKey); - block = std::make_shared(setup.CreateBlock({tx_upreg}, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock({tx_upreg}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); BOOST_ASSERT(!DeploymentActiveAfter(chainman.ActiveChain().Tip(), Params().GetConsensus(), Consensus::DEPLOYMENT_V19)); ++nHeight; @@ -326,7 +326,7 @@ void FuncV19Activation(TestChainSetup& setup) FillableSigningProvider signing_provider; signing_provider.AddKeyPubKey(collateral_key, collateral_key.GetPubKey()); BOOST_ASSERT(SignSignature(signing_provider, CTransaction(tx_reg), tx_spend, 0, SIGHASH_ALL)); - block = std::make_shared(setup.CreateBlock({tx_spend}, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock({tx_spend}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); BOOST_ASSERT(!DeploymentActiveAfter(chainman.ActiveChain().Tip(), Params().GetConsensus(), Consensus::DEPLOYMENT_V19)); ++nHeight; @@ -614,7 +614,7 @@ void FuncTestMempoolReorg(TestChainSetup& setup) FundTransaction(chainman.ActiveChain(), tx_collateral, utxos, scriptCollateral, dmn_types::Regular.collat_amount, setup.coinbaseKey); SignTransaction(*(setup.m_node.mempool), tx_collateral, setup.coinbaseKey); - auto block = std::make_shared(setup.CreateBlock({tx_collateral}, setup.coinbaseKey)); + auto block = std::make_shared(setup.CreateBlock({tx_collateral}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); setup.m_node.dmnman->UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 1); @@ -756,7 +756,7 @@ void FuncVerifyDB(TestChainSetup& setup) FundTransaction(chainman.ActiveChain(), tx_collateral, utxos, scriptCollateral, dmn_types::Regular.collat_amount, setup.coinbaseKey); SignTransaction(*(setup.m_node.mempool), tx_collateral, setup.coinbaseKey); - auto block = std::make_shared(setup.CreateBlock({tx_collateral}, setup.coinbaseKey)); + auto block = std::make_shared(setup.CreateBlock({tx_collateral}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 1); @@ -788,7 +788,7 @@ void FuncVerifyDB(TestChainSetup& setup) auto tx_reg_hash = tx_reg.GetHash(); - block = std::make_shared(setup.CreateBlock({tx_reg}, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock({tx_reg}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 2); @@ -800,7 +800,7 @@ void FuncVerifyDB(TestChainSetup& setup) collateral_utxos.emplace(payload.collateralOutpoint, std::make_pair(1, 1000)); auto proUpRevTx = CreateProUpRevTx(chainman.ActiveChain(), *(setup.m_node.mempool), collateral_utxos, tx_reg_hash, operatorKey, collateralKey); - block = std::make_shared(setup.CreateBlock({proUpRevTx}, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock({proUpRevTx}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 3); diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h new file mode 100644 index 0000000000000..81ea4c38f5744 --- /dev/null +++ b/src/test/util/chainstate.h @@ -0,0 +1,54 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#ifndef BITCOIN_TEST_UTIL_CHAINSTATE_H +#define BITCOIN_TEST_UTIL_CHAINSTATE_H + +#include +#include +#include +#include +#include +#include + +#include + +#include + +const auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; + +/** + * Create and activate a UTXO snapshot, optionally providing a function to + * malleate the snapshot. + */ +template +static bool +CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) +{ + // Write out a snapshot to the test's tempdir. + // + int height; + WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); + fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); + FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; + CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; + + UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); + BOOST_TEST_MESSAGE( + "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); + + // Read the written snapshot in and then activate it. + // + FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; + CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; + SnapshotMetadata metadata; + auto_infile >> metadata; + + malleation(auto_infile, metadata); + + return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); +} + + +#endif // BITCOIN_TEST_UTIL_CHAINSTATE_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index cc7a942d8b0c7..5219909818eae 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -373,10 +373,17 @@ void TestChainSetup::mineBlocks(int num_blocks) } } -CBlock TestChainSetup::CreateAndProcessBlock(const std::vector& txns, const CScript& scriptPubKey) +CBlock TestChainSetup::CreateAndProcessBlock( + const std::vector& txns, + const CScript& scriptPubKey, + CChainState* chainstate) { + if (!chainstate) { + chainstate = &Assert(m_node.chainman)->ActiveChainstate(); + } + const CChainParams& chainparams = Params(); - auto block = CreateBlock(txns, scriptPubKey); + auto block = this->CreateBlock(txns, scriptPubKey, *chainstate); std::shared_ptr shared_pblock = std::make_shared(block); Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr); @@ -384,17 +391,23 @@ CBlock TestChainSetup::CreateAndProcessBlock(const std::vector& txns, const CKey& scriptKey) +CBlock TestChainSetup::CreateAndProcessBlock( + const std::vector& txns, + const CKey& scriptKey, + CChainState* chainstate) { CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - return CreateAndProcessBlock(txns, scriptPubKey); + return CreateAndProcessBlock(txns, scriptPubKey, chainstate); } -CBlock TestChainSetup::CreateBlock(const std::vector& txns, const CScript& scriptPubKey) +CBlock TestChainSetup::CreateBlock( + const std::vector& txns, + const CScript& scriptPubKey, + CChainState& chainstate) { const CChainParams& chainparams = Params(); CTxMemPool empty_pool; - CBlock block = BlockAssembler(m_node.chainman->ActiveChainstate(), m_node, empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; + CBlock block = BlockAssembler(chainstate, m_node, empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; std::vector llmqCommitments; for (const auto& tx : block.vtx) { @@ -443,10 +456,13 @@ CBlock TestChainSetup::CreateBlock(const std::vector& txns, return result; } -CBlock TestChainSetup::CreateBlock(const std::vector& txns, const CKey& scriptKey) +CBlock TestChainSetup::CreateBlock( + const std::vector& txns, + const CKey& scriptKey, + CChainState& chainstate) { CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - return CreateBlock(txns, scriptPubKey); + return CreateBlock(txns, scriptPubKey, chainstate); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index d3795b5b865a5..7b623355bf2ca 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -127,15 +127,25 @@ struct TestChainSetup : public RegTestingSetup /** * Create a new block with just given transactions, coinbase paying to * scriptPubKey, and try to add it to the current chain. + * If no chainstate is specified, default to the active. */ CBlock CreateAndProcessBlock(const std::vector& txns, - const CScript& scriptPubKey); + const CScript& scriptPubKey, + CChainState* chainstate = nullptr); CBlock CreateAndProcessBlock(const std::vector& txns, - const CKey& scriptKey); + const CKey& scriptKey, + CChainState* chainstate = nullptr); + + /** + * Create a new block with just given transactions, coinbase paying to + * scriptPubKey. + */ CBlock CreateBlock(const std::vector& txns, - const CScript& scriptPubKey); + const CScript& scriptPubKey, + CChainState& chainstate); CBlock CreateBlock(const std::vector& txns, - const CKey& scriptKey); + const CKey& scriptKey, + CChainState& chainstate); //! Mine a series of new blocks on the active chain. void mineBlocks(int num_blocks); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 746733c75ddd8..dc32d93add72d 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // +#include #include #include #include @@ -10,6 +11,8 @@ #include #include #include +#include +#include #include #include #include @@ -78,4 +81,77 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) WITH_LOCK(::cs_main, manager.Unload()); } +//! Test UpdateTip behavior for both active and background chainstates. +//! +//! When run on the background chainstate, UpdateTip should do a subset +//! of what it does for the active chainstate. +BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) +{ + ChainstateManager& chainman = *Assert(m_node.chainman); + uint256 curr_tip = ::g_best_block; + + // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can + // be found. + mineBlocks(10); + + // After adding some blocks to the tip, best block should have changed. + BOOST_CHECK(::g_best_block != curr_tip); + + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + + // Ensure our active chain is the snapshot chainstate. + BOOST_CHECK(chainman.IsSnapshotActive()); + + curr_tip = ::g_best_block; + + // Mine a new block on top of the activated snapshot chainstate. + mineBlocks(1); // Defined in TestChain100Setup. + + // After adding some blocks to the snapshot tip, best block should have changed. + BOOST_CHECK(::g_best_block != curr_tip); + + curr_tip = ::g_best_block; + + CChainState* background_cs; + + BOOST_CHECK_EQUAL(chainman.GetAll().size(), 2); + for (CChainState* cs : chainman.GetAll()) { + if (cs != &chainman.ActiveChainstate()) { + background_cs = cs; + } + } + BOOST_CHECK(background_cs); + + // Create a block to append to the validation chain. + std::vector noTxns; + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, *background_cs); + auto pblock = std::make_shared(validation_block); + BlockValidationState state; + CBlockIndex* pindex = nullptr; + const CChainParams& chainparams = Params(); + bool newblock = false; + + // TODO: much of this is inlined from ProcessNewBlock(); just reuse PNB() + // once it is changed to support multiple chainstates. + { + LOCK(::cs_main); + bool checked = CheckBlock(*pblock, state, chainparams.GetConsensus()); + BOOST_CHECK(checked); + bool accepted = background_cs->AcceptBlock( + pblock, state, &pindex, true, nullptr, &newblock); + BOOST_CHECK(accepted); + } + // UpdateTip is called here + bool block_added = background_cs->ActivateBestChain(state, pblock); + + // Ensure tip is as expected + BOOST_CHECK_EQUAL(background_cs->m_chain.Tip()->GetBlockHash(), validation_block.GetHash()); + + // g_best_block should be unchanged after adding a block to the background + // validation chain. + BOOST_CHECK(block_added); + BOOST_CHECK_EQUAL(curr_tip, ::g_best_block); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index e4d6c5d62d97b..461bb0b58e5b9 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -14,13 +14,13 @@ #include #include #include +#include #include #include #include #include #include -#include #include @@ -53,7 +53,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); - BOOST_CHECK(!manager.IsBackgroundIBD(&c1)); auto all = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); @@ -66,9 +65,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto exp_tip = c1.m_chain.Tip(); BOOST_CHECK_EQUAL(active_tip, exp_tip); - auto& validated_cs = manager.ValidatedChainstate(); - BOOST_CHECK_EQUAL(&validated_cs, &c1); - BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); DashTestSetupClose(m_node); @@ -96,8 +92,8 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK(manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); - BOOST_CHECK(manager.IsBackgroundIBD(&c1)); - BOOST_CHECK(!manager.IsBackgroundIBD(&c2)); + BOOST_CHECK_EQUAL(&c2, &manager.ActiveChainstate()); + BOOST_CHECK(&c1 != &manager.ActiveChainstate()); auto all2 = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all2.begin(), all2.end(), chainstates.begin(), chainstates.end()); @@ -114,16 +110,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // CCoinsViewCache instances. BOOST_CHECK(exp_tip != exp_tip2); - auto& validated_cs2 = manager.ValidatedChainstate(); - BOOST_CHECK_EQUAL(&validated_cs2, &c1); - - auto& validated_chain = manager.ValidatedChain(); - BOOST_CHECK_EQUAL(&validated_chain, &c1.m_chain); - - auto validated_tip = manager.ValidatedTip(); - exp_tip = c1.m_chain.Tip(); - BOOST_CHECK_EQUAL(validated_tip, exp_tip); - // Let scheduler events finish running to avoid accessing memory that is going to be unloaded SyncWithValidationInterfaceQueue(); @@ -185,36 +171,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } -auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; - -template -static bool -CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) -{ - // Write out a snapshot to the test's tempdir. - // - int height; - WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); - fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); - FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; - CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; - - UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); - BOOST_TEST_MESSAGE( - "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); - - // Read the written snapshot in and then activate it. - // - FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; - CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; - SnapshotMetadata metadata; - auto_infile >> metadata; - - malleation(auto_infile, metadata); - - return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); -} - //! Test basic snapshot activation. BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) { @@ -338,27 +294,27 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) { LOCK(::cs_main); size_t coins_in_active{0}; - size_t coins_in_ibd{0}; - size_t coins_missing_ibd{0}; + size_t coins_in_background{0}; + size_t coins_missing_from_background{0}; for (CChainState* chainstate : chainman.GetAll()) { BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); CCoinsViewCache& coinscache = chainstate->CoinsTip(); - bool is_ibd = chainman.IsBackgroundIBD(chainstate); + bool is_background = chainstate != &chainman.ActiveChainstate(); for (CTransactionRef& txn : m_coinbase_txns) { COutPoint op{txn->GetHash(), 0}; if (coinscache.HaveCoin(op)) { - (is_ibd ? coins_in_ibd : coins_in_active)++; - } else if (is_ibd) { - coins_missing_ibd++; + (is_background ? coins_in_background : coins_in_active)++; + } else if (is_background) { + coins_missing_from_background++; } } } BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); - BOOST_CHECK_EQUAL(coins_in_ibd, initial_total_coins); - BOOST_CHECK_EQUAL(coins_missing_ibd, new_coins); + BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); + BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); } // Snapshot should refuse to load after one has already loaded. diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index 55ac348e728ad..1d33fc261159f 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -13,7 +13,7 @@ #include -BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, BasicTestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, ChainTestingSetup) //! Test utilities for detecting when we need to flush the coins cache based //! on estimated memory usage. @@ -24,7 +24,8 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) { CTxMemPool mempool; BlockManager blockman{}; - CChainState chainstate(&mempool, blockman, *m_node.evodb, m_node.chain_helper, llmq::chainLocksHandler, llmq::quorumInstantSendManager); + CChainState chainstate(&mempool, blockman, *Assert(m_node.chainman), *m_node.evodb, m_node.chain_helper, + llmq::chainLocksHandler, llmq::quorumInstantSendManager); chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); diff --git a/src/validation.cpp b/src/validation.cpp index 78309491d9443..f065d669f5d02 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1224,6 +1224,7 @@ void CoinsViews::InitCache() CChainState::CChainState(CTxMemPool* mempool, BlockManager& blockman, + ChainstateManager& chainman, CEvoDB& evoDb, const std::unique_ptr& chain_helper, const std::unique_ptr& clhandler, @@ -1236,6 +1237,7 @@ CChainState::CChainState(CTxMemPool* mempool, m_isman(isman), m_evoDb(evoDb), m_blockman(blockman), + m_chainman(chainman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( @@ -2525,8 +2527,44 @@ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) res += warn; } +static void UpdateTipLog( + const CCoinsViewCache& coins_tip, + const CBlockIndex* tip, + const CChainParams& params, + const CEvoDB& evo_db, + const std::string& func_name, + const std::string& prefix, + const std::string& warning_messages) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +{ + + AssertLockHeld(::cs_main); + LogPrintf("%s%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo) evodb_cache=%.1fMiB%s\n", + prefix, func_name, + tip->GetBlockHash().ToString(), tip->nHeight, tip->nVersion, + log(tip->nChainWork.getdouble()) / log(2.0), (unsigned long)tip->nChainTx, + FormatISO8601DateTime(tip->GetBlockTime()), + GuessVerificationProgress(params.TxData(), tip), + coins_tip.DynamicMemoryUsage() * (1.0 / (1 << 20)), + coins_tip.GetCacheSize(), + evo_db.GetMemoryUsage() * (1.0 / (1 << 20)), + !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages) : ""); +} + void CChainState::UpdateTip(const CBlockIndex* pindexNew) { + const auto& coins_tip = this->CoinsTip(); + + // The remainder of the function isn't relevant if we are not acting on + // the active chainstate, so return if need be. + if (this != &m_chainman.ActiveChainstate()) { + // Only log every so often so that we don't bury log messages at the tip. + constexpr int BACKGROUND_LOG_INTERVAL = 2000; + if (pindexNew->nHeight % BACKGROUND_LOG_INTERVAL == 0) { + UpdateTipLog(coins_tip, pindexNew, m_params, m_evoDb, __func__, "[background validation] ", ""); + } + return; + } + // New best block if (m_mempool) { m_mempool->AddTransactionsUpdated(1); @@ -2555,13 +2593,7 @@ void CChainState::UpdateTip(const CBlockIndex* pindexNew) } } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo) evodb_cache=%.1fMiB%s\n", __func__, - pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, - log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, - FormatISO8601DateTime(pindexNew->GetBlockTime()), - GuessVerificationProgress(m_params.TxData(), pindexNew), this->CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), this->CoinsTip().GetCacheSize(), - m_evoDb.GetMemoryUsage() * (1.0 / (1<<20)), - !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages.original) : ""); + UpdateTipLog(coins_tip, pindexNew, m_params, m_evoDb, __func__, "", warning_messages.original); } /** Disconnect m_chain's tip. @@ -4257,7 +4289,9 @@ bool BlockManager::LoadBlockIndex( pindex->nStatus |= BLOCK_FAILED_CHILD; setDirtyBlockIndex.insert(pindex); } - if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr)) { + if (pindex->IsAssumedValid() || + (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && + (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { block_index_candidates.insert(pindex); } if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) @@ -4961,12 +4995,33 @@ void CChainState::CheckBlockIndex() nNodes++; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; if (pindexFirstConflicing == nullptr && pindex->nStatus & BLOCK_CONFLICT_CHAINLOCK) pindexFirstConflicing = pindex; - if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + // Assumed-valid index entries will not have data since we haven't downloaded the + // full block yet. + if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA) && !pindex->IsAssumedValid()) { + pindexFirstMissing = pindex; + } if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotTransactionsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotChainValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotScriptsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) pindexFirstNotScriptsValid = pindex; + + if (pindex->pprev != nullptr && !pindex->IsAssumedValid()) { + // Skip validity flag checks for BLOCK_ASSUMED_VALID index entries, since these + // *_VALID_MASK flags will not be present for index entries we are temporarily assuming + // valid. + if (pindexFirstNotTransactionsValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) { + pindexFirstNotTransactionsValid = pindex; + } + + if (pindexFirstNotChainValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) { + pindexFirstNotChainValid = pindex; + } + + if (pindexFirstNotScriptsValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) { + pindexFirstNotScriptsValid = pindex; + } + } // Begin: actual consistency checks. if (pindex->pprev == nullptr) { @@ -4977,7 +5032,9 @@ void CChainState::CheckBlockIndex() if (!pindex->HaveTxsDownloaded()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. - if (!fHavePruned) { + // Unless these indexes are assumed valid and pending block download on a + // background chainstate. + if (!fHavePruned && !pindex->IsAssumedValid()) { // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); assert(pindexFirstMissing == pindexFirstNeverProcessed); @@ -4986,7 +5043,16 @@ void CChainState::CheckBlockIndex() if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); } if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); - assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + if (pindex->IsAssumedValid()) { + // Assumed-valid blocks should have some nTx value. + assert(pindex->nTx > 0); + // Assumed-valid blocks should connect to the main chain. + assert((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE); + } else { + // Otherwise there should only be an nTx value if we have + // actually seen a block's transactions. + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + } // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to HaveTxsDownloaded(). assert((pindexFirstNeverProcessed == nullptr) == pindex->HaveTxsDownloaded()); assert((pindexFirstNotTransactionsValid == nullptr) == pindex->HaveTxsDownloaded()); @@ -5007,11 +5073,17 @@ void CChainState::CheckBlockIndex() } if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { if (pindexFirstInvalid == nullptr && pindexFirstConflicing == nullptr) { + const bool is_active = this == &m_chainman.ActiveChainstate(); + // If this block sorts at least as good as the current tip and // is valid and we have all data for its parents, it must be in // setBlockIndexCandidates. m_chain.Tip() must also be there // even if some data has been pruned. - if (pindexFirstMissing == nullptr || pindex == m_chain.Tip()) { + // + // Don't perform this check for the background chainstate since + // its setBlockIndexCandidates shouldn't have some entries (i.e. those past the + // snapshot block) which do exist in the block index for the active chainstate. + if (is_active && (pindexFirstMissing == nullptr || pindex == m_chain.Tip())) { assert(setBlockIndexCandidates.count(pindex)); } // If some parent is missing, then it could be that this block was in @@ -5353,7 +5425,7 @@ CChainState& ChainstateManager::InitializeChainstate(CTxMemPool* mempool, throw std::logic_error("should not be overwriting a chainstate"); } - to_modify.reset(new CChainState(mempool, m_blockman, evoDb, chain_helper, clhandler, isman, snapshot_blockhash)); + to_modify.reset(new CChainState(mempool, m_blockman, *this, evoDb, chain_helper, clhandler, isman, snapshot_blockhash)); // Snapshot chainstates and initial IBD chaintates always become active. if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { @@ -5423,7 +5495,7 @@ bool ChainstateManager::ActivateSnapshot( } auto snapshot_chainstate = WITH_LOCK(::cs_main, return std::make_unique( - /* mempool */ nullptr, m_blockman, + /* mempool */ nullptr, m_blockman, *this, this->ActiveChainstate().m_evoDb, this->ActiveChainstate().m_chain_helper, this->ActiveChainstate().m_clhandler, @@ -5632,6 +5704,20 @@ bool ChainstateManager::PopulateAndValidateSnapshot( } // Fake nChainTx so that GuessVerificationProgress reports accurately index->nChainTx = index->pprev ? index->pprev->nChainTx + index->nTx : 1; + + // Mark unvalidated block index entries beneath the snapshot base block as assumed-valid. + if (!index->IsValid(BLOCK_VALID_SCRIPTS)) { + // This flag will be removed once the block is fully validated by a + // background chainstate. + index->nStatus |= BLOCK_ASSUMED_VALID; + } + + setDirtyBlockIndex.insert(index); + // Changes to the block index will be flushed to disk after this call + // returns in `ActivateSnapshot()`, when `MaybeRebalanceCaches()` is + // called, since we've added a snapshot chainstate and therefore will + // have to downsize the IBD chainstate, which will result in a call to + // `FlushStateToDisk(ALWAYS)`. } assert(index); @@ -5656,22 +5742,6 @@ bool ChainstateManager::IsSnapshotActive() const return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } -CChainState& ChainstateManager::ValidatedChainstate() const -{ - LOCK(::cs_main); - if (m_snapshot_chainstate && IsSnapshotValidated()) { - return *m_snapshot_chainstate.get(); - } - assert(m_ibd_chainstate); - return *m_ibd_chainstate.get(); -} - -bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const -{ - LOCK(::cs_main); - return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); -} - void ChainstateManager::Unload() { for (CChainState* chainstate : this->GetAll()) { diff --git a/src/validation.h b/src/validation.h index 6c5b2974cd157..4b4e6726abe58 100644 --- a/src/validation.h +++ b/src/validation.h @@ -127,6 +127,7 @@ typedef std::unordered_map BlockMap; typedef std::unordered_multimap PrevBlockMap; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; +/** Used to notify getblocktemplate RPC of new tips. */ extern uint256 g_best_block; /** Whether there are dedicated script-checking threads running. * False indicates all script checking is done on the main threadMessageHandler thread. @@ -630,8 +631,14 @@ class CChainState //! CChainState instances. BlockManager& m_blockman; + //! The chainstate manager that owns this chainstate. The reference is + //! necessary so that this instance can check whether it is the active + //! chainstate within deeply nested method calls. + ChainstateManager& m_chainman; + explicit CChainState(CTxMemPool* mempool, BlockManager& blockman, + ChainstateManager& chainman, CEvoDB& evoDb, const std::unique_ptr& chain_helper, const std::unique_ptr& clhandler, @@ -672,9 +679,10 @@ class CChainState const std::optional m_from_snapshot_blockhash; /** - * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and - * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be - * missing the data for the block. + * The set of all CBlockIndex entries with either BLOCK_VALID_TRANSACTIONS (for + * itself and all ancestors) *or* BLOCK_ASSUMED_VALID (if using background + * chainstates) and as good as our current tip or better. Entries may be failed, + * though, and pruning nodes may be missing the data for the block. */ std::set setBlockIndexCandidates; @@ -897,12 +905,6 @@ class CChainState * *Background IBD chainstate*: an IBD chainstate for which the * IBD process is happening in the background while use of the * active (snapshot) chainstate allows the rest of the system to function. - * - * *Validated chainstate*: the most-work chainstate which has been validated - * locally via initial block download. This will be the snapshot chainstate - * if a snapshot was loaded and all blocks up to the snapshot starting point - * have been downloaded and validated (via background validation), otherwise - * it will be the IBD chainstate. */ class ChainstateManager { @@ -1029,19 +1031,6 @@ class ChainstateManager //! Is there a snapshot in use and has it been fully validated? bool IsSnapshotValidated() const { return m_snapshot_validated; } - //! @returns true if this chainstate is being used to validate an active - //! snapshot in the background. - bool IsBackgroundIBD(CChainState* chainstate) const; - - //! Return the most-work chainstate that has been fully validated. - //! - //! During background validation of a snapshot, this is the IBD chain. After - //! background validation has completed, this is the snapshot chain. - CChainState& ValidatedChainstate() const; - - CChain& ValidatedChain() const { return ValidatedChainstate().m_chain; } - CBlockIndex* ValidatedTip() const { return ValidatedChain().Tip(); } - /** * Process an incoming block. This only returns after the best known valid * block is made active. Note that it does not, however, guarantee that the From b402fd57fac0148d331fa8f28948822a8efeb922 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:45:00 +0000 Subject: [PATCH 05/11] merge bitcoin#23174: have LoadBlockIndex account for snapshot use --- .../validation_chainstatemanager_tests.cpp | 80 ++++++++++++++++++ src/validation.cpp | 81 ++++++++++++++++--- src/validation.h | 13 +-- 3 files changed, 157 insertions(+), 17 deletions(-) diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 461bb0b58e5b9..ba35dd3e3ad7a 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -249,6 +249,9 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) *chainman.ActiveChainstate().m_from_snapshot_blockhash, *chainman.SnapshotBlockhash()); + // Ensure that the genesis block was not marked assumed-valid. + BOOST_CHECK(!chainman.ActiveChain().Genesis()->IsAssumedValid()); + const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); const CBlockIndex* tip = chainman.ActiveTip(); @@ -326,4 +329,81 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) loaded_snapshot_blockhash); } +//! Test LoadBlockIndex behavior when multiple chainstates are in use. +//! +//! - First, verfiy that setBlockIndexCandidates is as expected when using a single, +//! fully-validating chainstate. +//! +//! - Then mark a region of the chain BLOCK_ASSUMED_VALID and introduce a second chainstate +//! that will tolerate assumed-valid blocks. Run LoadBlockIndex() and ensure that the first +//! chainstate only contains fully validated blocks and the other chainstate contains all blocks, +//! even those assumed-valid. +//! +BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) +{ + ChainstateManager& chainman = *Assert(m_node.chainman); + CTxMemPool& mempool = *m_node.mempool; + CChainState& cs1 = chainman.ActiveChainstate(); + + int num_indexes{0}; + int num_assumed_valid{0}; + const int expected_assumed_valid{20}; + const int last_assumed_valid_idx{40}; + const int assumed_valid_start_idx = last_assumed_valid_idx - expected_assumed_valid; + + CBlockIndex* validated_tip{nullptr}; + CBlockIndex* assumed_tip{chainman.ActiveChain().Tip()}; + + auto reload_all_block_indexes = [&]() { + for (CChainState* cs : chainman.GetAll()) { + LOCK(::cs_main); + cs->UnloadBlockIndex(); + BOOST_CHECK(cs->setBlockIndexCandidates.empty()); + } + + WITH_LOCK(::cs_main, chainman.LoadBlockIndex()); + }; + + // Ensure that without any assumed-valid BlockIndex entries, all entries are considered + // tip candidates. + reload_all_block_indexes(); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), cs1.m_chain.Height() + 1); + + // Mark some region of the chain assumed-valid. + for (int i = 0; i <= cs1.m_chain.Height(); ++i) { + auto index = cs1.m_chain[i]; + + if (i < last_assumed_valid_idx && i >= assumed_valid_start_idx) { + index->nStatus = BlockStatus::BLOCK_VALID_TREE | BlockStatus::BLOCK_ASSUMED_VALID; + } + + ++num_indexes; + if (index->IsAssumedValid()) ++num_assumed_valid; + + // Note the last fully-validated block as the expected validated tip. + if (i == (assumed_valid_start_idx - 1)) { + validated_tip = index; + BOOST_CHECK(!index->IsAssumedValid()); + } + } + + BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid); + + CChainState& cs2 = WITH_LOCK(::cs_main, + return chainman.InitializeChainstate(&mempool, *m_node.evodb, m_node.chain_helper, llmq::chainLocksHandler, llmq::quorumInstantSendManager, GetRandHash())); + + reload_all_block_indexes(); + + // The fully validated chain only has candidates up to the start of the assumed-valid + // blocks. + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(validated_tip), 1); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(assumed_tip), 0); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), assumed_valid_start_idx); + + // The assumed-valid tolerant chain has all blocks as candidates. + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(validated_tip), 1); + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(assumed_tip), 1); + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/validation.cpp b/src/validation.cpp index f065d669f5d02..4c0ffe3162ab5 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -67,6 +67,7 @@ #include +#include #include #include #include @@ -4245,7 +4246,7 @@ CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash) bool BlockManager::LoadBlockIndex( const Consensus::Params& consensus_params, - std::set& block_index_candidates) + ChainstateManager& chainman) { if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { return false; @@ -4265,17 +4266,41 @@ bool BlockManager::LoadBlockIndex( } } sort(vSortedByHeight.begin(), vSortedByHeight.end()); + + // Find start of assumed-valid region. + int first_assumed_valid_height = std::numeric_limits::max(); + + for (const auto& [height, block] : vSortedByHeight) { + if (block->IsAssumedValid()) { + auto chainstates = chainman.GetAll(); + + // If we encounter an assumed-valid block index entry, ensure that we have + // one chainstate that tolerates assumed-valid entries and another that does + // not (i.e. the background validation chainstate), since assumed-valid + // entries should always be pending validation by a fully-validated chainstate. + auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; + assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); + assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); + + first_assumed_valid_height = height; + break; + } + } + for (const std::pair& item : vSortedByHeight) { if (ShutdownRequested()) return false; CBlockIndex* pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); - // We can link the chain of blocks for which we've received transactions at some point. + + // We can link the chain of blocks for which we've received transactions at some point, or + // blocks that are assumed-valid on the basis of snapshot load (see + // PopulateAndValidateSnapshot()). // Pruned nodes may have deleted the block. if (pindex->nTx > 0) { if (pindex->pprev) { - if (pindex->pprev->HaveTxsDownloaded()) { + if (pindex->pprev->nChainTx > 0) { pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; } else { pindex->nChainTx = 0; @@ -4292,7 +4317,36 @@ bool BlockManager::LoadBlockIndex( if (pindex->IsAssumedValid() || (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { - block_index_candidates.insert(pindex); + + // Fill each chainstate's block candidate set. Only add assumed-valid + // blocks to the tip candidate set if the chainstate is allowed to rely on + // assumed-valid blocks. + // + // If all setBlockIndexCandidates contained the assumed-valid blocks, the + // background chainstate's ActivateBestChain() call would add assumed-valid + // blocks to the chain (based on how FindMostWorkChain() works). Obviously + // we don't want this since the purpose of the background validation chain + // is to validate assued-valid blocks. + // + // Note: This is considering all blocks whose height is greater or equal to + // the first assumed-valid block to be assumed-valid blocks, and excluding + // them from the background chainstate's setBlockIndexCandidates set. This + // does mean that some blocks which are not technically assumed-valid + // (later blocks on a fork beginning before the first assumed-valid block) + // might not get added to the the background chainstate, but this is ok, + // because they will still be attached to the active chainstate if they + // actually contain more work. + // + // Instad of this height-based approach, an earlier attempt was made at + // detecting "holistically" whether the block index under consideration + // relied on an assumed-valid ancestor, but this proved to be too slow to + // be practical. + for (CChainState* chainstate : chainman.GetAll()) { + if (chainstate->reliesOnAssumedValid() || + pindex->nHeight < first_assumed_valid_height) { + chainstate->setBlockIndexCandidates.insert(pindex); + } + } } if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) pindexBestInvalid = pindex; @@ -4317,11 +4371,9 @@ void BlockManager::Unload() { m_prev_block_index.clear(); } -bool BlockManager::LoadBlockIndexDB(std::set& setBlockIndexCandidates) +bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) { - if (!LoadBlockIndex( - ::Params().GetConsensus(), - setBlockIndexCandidates)) { + if (!LoadBlockIndex(::Params().GetConsensus(), chainman)) { return false; } @@ -4758,7 +4810,7 @@ bool ChainstateManager::LoadBlockIndex() // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = m_blockman.LoadBlockIndexDB(ActiveChainstate().setBlockIndexCandidates); + bool ret = m_blockman.LoadBlockIndexDB(*this); if (!ret) return false; needs_init = m_blockman.m_block_index.empty(); } @@ -5694,7 +5746,14 @@ bool ChainstateManager::PopulateAndValidateSnapshot( // Fake various pieces of CBlockIndex state: CBlockIndex* index = nullptr; - for (int i = 0; i <= snapshot_chainstate.m_chain.Height(); ++i) { + + // Don't make any modifications to the genesis block. + // This is especially important because we don't want to erroneously + // apply BLOCK_ASSUMED_VALID to genesis, which would happen if we didn't skip + // it here (since it apparently isn't BLOCK_VALID_SCRIPTS). + constexpr int AFTER_GENESIS_START{1}; + + for (int i = AFTER_GENESIS_START; i <= snapshot_chainstate.m_chain.Height(); ++i) { index = snapshot_chainstate.m_chain[i]; // Fake nTx so that LoadBlockIndex() loads assumed-valid CBlockIndex @@ -5703,7 +5762,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( index->nTx = 1; } // Fake nChainTx so that GuessVerificationProgress reports accurately - index->nChainTx = index->pprev ? index->pprev->nChainTx + index->nTx : 1; + index->nChainTx = index->pprev->nChainTx + index->nTx; // Mark unvalidated block index entries beneath the snapshot base block as assumed-valid. if (!index->IsValid(BLOCK_VALID_SCRIPTS)) { diff --git a/src/validation.h b/src/validation.h index 4b4e6726abe58..ed43f0b77b258 100644 --- a/src/validation.h +++ b/src/validation.h @@ -471,20 +471,17 @@ class BlockManager std::unique_ptr m_block_tree_db GUARDED_BY(::cs_main); - bool LoadBlockIndexDB(std::set& setBlockIndexCandidates) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool LoadBlockIndexDB(ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * Load the blocktree off disk and into memory. Populate certain metadata * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral * collections like setDirtyBlockIndex. - * - * @param[out] block_index_candidates Fill this set with any valid blocks for - * which we've downloaded all transactions. */ bool LoadBlockIndex( const Consensus::Params& consensus_params, - std::set& block_index_candidates) - EXCLUSIVE_LOCKS_REQUIRED(cs_main); + ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Clear all data members. */ void Unload() EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -678,6 +675,10 @@ class CChainState */ const std::optional m_from_snapshot_blockhash; + //! Return true if this chainstate relies on blocks that are assumed-valid. In + //! practice this means it was created based on a UTXO snapshot. + bool reliesOnAssumedValid() { return m_from_snapshot_blockhash.has_value(); } + /** * The set of all CBlockIndex entries with either BLOCK_VALID_TRANSACTIONS (for * itself and all ancestors) *or* BLOCK_ASSUMED_VALID (if using background From 732e871a6b9467c1b1d144da9ce711b8c13b331d Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:50:14 +0000 Subject: [PATCH 06/11] merge bitcoin#23785: Move stuff to ChainstateManager --- src/index/base.cpp | 2 +- src/net_processing.cpp | 4 +- src/node/interfaces.cpp | 4 +- src/validation.cpp | 87 ++++++++++++++++++++--------------------- src/validation.h | 71 +++++++++++++++++---------------- 5 files changed, 86 insertions(+), 82 deletions(-) diff --git a/src/index/base.cpp b/src/index/base.cpp index 7a0afaff8d216..866fbc96977ef 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -64,7 +64,7 @@ bool BaseIndex::Init() if (locator.IsNull()) { m_best_block_index = nullptr; } else { - m_best_block_index = m_chainstate->m_blockman.FindForkInGlobalIndex(active_chain, locator); + m_best_block_index = m_chainstate->FindForkInGlobalIndex(locator); } // Note: this will latch to true immediately if the user starts up with an empty diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6511ed614b401..d3543e1358379 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3934,7 +3934,7 @@ void PeerManagerImpl::ProcessMessage( LOCK(cs_main); // Find the last block the caller has in the main chain - const CBlockIndex* pindex = m_chainman.m_blockman.FindForkInGlobalIndex(m_chainman.ActiveChain(), locator); + const CBlockIndex* pindex = m_chainman.ActiveChainstate().FindForkInGlobalIndex(locator); // Send the rest of the chain if (pindex) @@ -4054,7 +4054,7 @@ void PeerManagerImpl::ProcessMessage( else { // Find the last block the caller has in the main chain - pindex = m_chainman.m_blockman.FindForkInGlobalIndex(m_chainman.ActiveChain(), locator); + pindex = m_chainman.ActiveChainstate().FindForkInGlobalIndex(locator); if (pindex) pindex = m_chainman.ActiveChain().Next(pindex); } diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index be4a3de4217dc..d9ff8368609a2 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -762,8 +762,8 @@ class ChainImpl : public Chain std::optional findLocatorFork(const CBlockLocator& locator) override { LOCK(cs_main); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); - if (CBlockIndex* fork = m_node.chainman->m_blockman.FindForkInGlobalIndex(active, locator)) { + const CChainState& active = Assert(m_node.chainman)->ActiveChainstate(); + if (CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { return fork->nHeight; } return std::nullopt; diff --git a/src/validation.cpp b/src/validation.cpp index 4c0ffe3162ab5..c9516bbe919e7 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -142,11 +142,6 @@ arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); -// Internal stuff -namespace { - CBlockIndex* pindexBestInvalid = nullptr; -} // namespace - // Internal stuff from blockstorage ... extern RecursiveMutex cs_LastBlockFile; extern std::vector vinfoBlockFile; @@ -164,20 +159,21 @@ CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const return it == m_block_index.end() ? nullptr : it->second; } -CBlockIndex* BlockManager::FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) +CBlockIndex* CChainState::FindForkInGlobalIndex(const CBlockLocator& locator) const { AssertLockHeld(cs_main); // Find the latest block common to locator and chain - we expect that // locator.vHave is sorted descending by height. for (const uint256& hash : locator.vHave) { - CBlockIndex* pindex = LookupBlockIndex(hash); + CBlockIndex* pindex{m_blockman.LookupBlockIndex(hash)}; if (pindex) { - if (chain.Contains(pindex)) + if (m_chain.Contains(pindex)) { return pindex; + } } } - return chain.Genesis(); + return m_chain.Genesis(); } bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector *pvChecks = nullptr); @@ -1319,7 +1315,7 @@ void CChainState::CheckForkWarningConditions() return; } - if (pindexBestInvalid && pindexBestInvalid->nChainWork > m_chain.Tip()->nChainWork + (GetBlockProof(*m_chain.Tip()) * 6)) { + if (m_chainman.m_best_invalid && m_chainman.m_best_invalid->nChainWork > m_chain.Tip()->nChainWork + (GetBlockProof(*m_chain.Tip()) * 6)) { LogPrintf("%s: Warning: Found invalid chain which has higher work (at least ~6 blocks worth of work) than our best chain.\nChain state database corruption likely.\n", __func__); SetfLargeWorkInvalidChainFound(true); } else { @@ -1333,8 +1329,9 @@ void CChainState::InvalidChainFound(CBlockIndex* pindexNew) statsClient.inc("warnings.InvalidChainFound", 1.0f); - if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) - pindexBestInvalid = pindexNew; + if (!m_chainman.m_best_invalid || pindexNew->nChainWork > m_chainman.m_best_invalid->nChainWork) { + m_chainman.m_best_invalid = pindexNew; + } if (pindexBestHeader != nullptr && pindexBestHeader->GetAncestor(pindexNew->nHeight) == pindexNew) { pindexBestHeader = m_chain.Tip(); } @@ -1373,7 +1370,7 @@ void CChainState::InvalidBlockFound(CBlockIndex *pindex, const BlockValidationSt statsClient.inc("warnings.InvalidBlockFound", 1.0f); if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; - m_blockman.m_failed_blocks.insert(pindex); + m_chainman.m_failed_blocks.insert(pindex); setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); @@ -2814,8 +2811,9 @@ CBlockIndex* CChainState::FindMostWorkChain() { bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); if (fFailedChain || fMissingData || fConflictingChain) { // Candidate chain is not usable (either invalid or conflicting or missing data) - if (fFailedChain && (pindexBestInvalid == nullptr || pindexNew->nChainWork > pindexBestInvalid->nChainWork)) - pindexBestInvalid = pindexNew; + if (fFailedChain && (m_chainman.m_best_invalid == nullptr || pindexNew->nChainWork > m_chainman.m_best_invalid->nChainWork)) { + m_chainman.m_best_invalid = pindexNew; + } CBlockIndex *pindexFailed = pindexNew; // Remove the entire chain from the set. while (pindexTest != pindexFailed) { @@ -3181,7 +3179,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind const CBlockIndex* pindexOldTip = m_chain.Tip(); if (pindex == pindexBestHeader) { - pindexBestInvalid = pindexBestHeader; + m_chainman.m_best_invalid = pindexBestHeader; pindexBestHeader = pindexBestHeader->pprev; } @@ -3199,7 +3197,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind assert(invalid_walk_tip->pprev == m_chain.Tip()); if (pindexOldTip == pindexBestHeader) { - pindexBestInvalid = pindexBestHeader; + m_chainman.m_best_invalid = pindexBestHeader; pindexBestHeader = pindexBestHeader->pprev; } @@ -3248,7 +3246,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind to_mark_failed->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(to_mark_failed); setBlockIndexCandidates.erase(to_mark_failed); - m_blockman.m_failed_blocks.insert(to_mark_failed); + m_chainman.m_failed_blocks.insert(to_mark_failed); // If any new blocks somehow arrived while we were disconnecting // (above), then the pre-calculation of what should go into @@ -3382,10 +3380,10 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); if (!pindex) { - if (pindexBestInvalid && pindexBestInvalid->GetAncestor(m_chain.Height()) == m_chain.Tip()) { + if (m_chainman.m_best_invalid && m_chainman.m_best_invalid->GetAncestor(m_chain.Height()) == m_chain.Tip()) { LogPrintf("%s: the best known invalid block (%s) is ahead of our tip, reconsidering\n", - __func__, pindexBestInvalid->GetBlockHash().ToString()); - pindex = pindexBestInvalid; + __func__, m_chainman.m_best_invalid->GetBlockHash().ToString()); + pindex = m_chainman.m_best_invalid; } else { return; } @@ -3402,11 +3400,11 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && !(it->second->nStatus & BLOCK_CONFLICT_CHAINLOCK) && it->second->HaveTxsDownloaded() && setBlockIndexCandidates.value_comp()(m_chain.Tip(), it->second)) { setBlockIndexCandidates.insert(it->second); } - if (it->second == pindexBestInvalid) { + if (it->second == m_chainman.m_best_invalid) { // Reset invalid block marker if it was pointing to one of those. - pindexBestInvalid = nullptr; + m_chainman.m_best_invalid = nullptr; } - m_blockman.m_failed_blocks.erase(it->second); + m_chainman.m_failed_blocks.erase(it->second); } it++; } @@ -3419,17 +3417,17 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && !(pindex->nStatus & BLOCK_CONFLICT_CHAINLOCK) && pindex->HaveTxsDownloaded() && setBlockIndexCandidates.value_comp()(m_chain.Tip(), pindex)) { setBlockIndexCandidates.insert(pindex); } - if (pindex == pindexBestInvalid) { + if (pindex == m_chainman.m_best_invalid) { // Reset invalid block marker if it was pointing to one of those. - pindexBestInvalid = nullptr; + m_chainman.m_best_invalid = nullptr; } - m_blockman.m_failed_blocks.erase(pindex); + m_chainman.m_failed_blocks.erase(pindex); // Mark all nearest BLOCK_FAILED_CHILD descendants (if any) as BLOCK_FAILED_VALID auto itp = m_blockman.m_prev_block_index.equal_range(pindex->GetBlockHash()); for (auto jt = itp.first; jt != itp.second; ++jt) { if (jt->second->nStatus & BLOCK_FAILED_CHILD) { jt->second->nStatus |= BLOCK_FAILED_VALID; - m_blockman.m_failed_blocks.insert(jt->second); + m_chainman.m_failed_blocks.insert(jt->second); setDirtyBlockIndex.insert(jt->second); setBlockIndexCandidates.erase(jt->second); } @@ -3769,16 +3767,16 @@ static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& stat return true; } -bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) +bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) { AssertLockHeld(cs_main); // Check for duplicate uint256 hash = block.GetHash(); - BlockMap::iterator miSelf = m_block_index.find(hash); // TODO : ENABLE BLOCK CACHE IN SPECIFIC CASES + BlockMap::iterator miSelf{m_blockman.m_block_index.find(hash)}; if (hash != chainparams.GetConsensus().hashGenesisBlock) { - if (miSelf != m_block_index.end()) { + if (miSelf != m_blockman.m_block_index.end()) { // Block header is already known. CBlockIndex* pindex = miSelf->second; if (ppindex) @@ -3801,8 +3799,8 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS // Get prev block index CBlockIndex* pindexPrev = nullptr; - BlockMap::iterator mi = m_block_index.find(block.hashPrevBlock); - if (mi == m_block_index.end()) { + BlockMap::iterator mi{m_blockman.m_block_index.find(block.hashPrevBlock)}; + if (mi == m_blockman.m_block_index.end()) { LogPrintf("ERROR: %s: prev block not found\n", __func__); return state.Invalid(BlockValidationResult::BLOCK_MISSING_PREV, "prev-blk-not-found"); } @@ -3820,8 +3818,9 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS return state.Invalid(BlockValidationResult::BLOCK_CHAINLOCK, "bad-prevblk-chainlock"); } - if (!ContextualCheckBlockHeader(block, state, *this, chainparams, pindexPrev, GetAdjustedTime())) + if (!ContextualCheckBlockHeader(block, state, m_blockman, chainparams, pindexPrev, GetAdjustedTime())) { return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); + } /* Determine if this block descends from any block which has been found * invalid (m_failed_blocks), then mark pindexPrev and any blocks between @@ -3863,14 +3862,14 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS } if (llmq::chainLocksHandler->HasConflictingChainLock(pindexPrev->nHeight + 1, hash)) { - if (miSelf == m_block_index.end()) { - AddToBlockIndex(block, hash, BLOCK_CONFLICT_CHAINLOCK); + if (miSelf == m_blockman.m_block_index.end()) { + m_blockman.AddToBlockIndex(block, hash, BLOCK_CONFLICT_CHAINLOCK); } LogPrintf("ERROR: %s: header %s conflicts with chainlock\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_CHAINLOCK, "bad-chainlock"); } } - CBlockIndex* pindex = AddToBlockIndex(block, hash); + CBlockIndex* pindex{m_blockman.AddToBlockIndex(block, hash)}; if (ppindex) *ppindex = pindex; @@ -3889,8 +3888,7 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector& LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - bool accepted = m_blockman.AcceptBlockHeader( - header, state, chainparams, &pindex); + bool accepted{AcceptBlockHeader(header, state, chainparams, &pindex)}; ActiveChainstate().CheckBlockIndex(); if (!accepted) { @@ -3925,7 +3923,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr& pblock, Block CBlockIndex *pindexDummy = nullptr; CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; - bool accepted_header = m_blockman.AcceptBlockHeader(block, state, m_params, &pindex); + bool accepted_header{m_chainman.AcceptBlockHeader(block, state, m_params, &pindex)}; CheckBlockIndex(); if (!accepted_header) @@ -4348,8 +4346,9 @@ bool BlockManager::LoadBlockIndex( } } } - if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) - pindexBestInvalid = pindex; + if (pindex->nStatus & BLOCK_FAILED_MASK && (!chainman.m_best_invalid || pindex->nChainWork > chainman.m_best_invalid->nChainWork)) { + chainman.m_best_invalid = pindex; + } if (pindex->pprev) pindex->BuildSkip(); if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == nullptr || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) @@ -4360,7 +4359,6 @@ bool BlockManager::LoadBlockIndex( } void BlockManager::Unload() { - m_failed_blocks.clear(); m_blocks_unlinked.clear(); for (const BlockMap::value_type& entry : m_block_index) { @@ -4790,7 +4788,6 @@ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman) { LOCK(cs_main); chainman.Unload(); - pindexBestInvalid = nullptr; pindexBestHeader = nullptr; if (mempool) mempool->clear(); vinfoBlockFile.clear(); @@ -5808,7 +5805,9 @@ void ChainstateManager::Unload() chainstate->UnloadBlockIndex(); } + m_failed_blocks.clear(); m_blockman.Unload(); + m_best_invalid = nullptr; } void ChainstateManager::Reset() diff --git a/src/validation.h b/src/validation.h index ed43f0b77b258..090666c09172b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -443,26 +443,6 @@ class BlockManager BlockMap m_block_index GUARDED_BY(cs_main); PrevBlockMap m_prev_block_index GUARDED_BY(cs_main); - /** In order to efficiently track invalidity of headers, we keep the set of - * blocks which we tried to connect and found to be invalid here (ie which - * were set to BLOCK_FAILED_VALID since the last restart). We can then - * walk this set and check if a new header is a descendant of something in - * this set, preventing us from having to walk m_block_index when we try - * to connect a bad block and fail. - * - * While this is more complicated than marking everything which descends - * from an invalid block as invalid at the time we discover it to be - * invalid, doing so would require walking all of m_block_index to find all - * descendants. Since this case should be very rare, keeping track of all - * BLOCK_FAILED_VALID blocks in a set should be just fine and work just as - * well. - * - * Because we already walk m_block_index in height-order at startup, we go - * ahead and mark descendants of invalid blocks as FAILED_CHILD at that time, - * instead of putting things in this set. - */ - std::set m_failed_blocks; - /** * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. * Pruned nodes may have entries where B is missing data. @@ -492,21 +472,8 @@ class BlockManager //! Mark one block file as pruned (modify associated database entries) void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** - * If a block header hasn't already been seen, call CheckBlockHeader on it, ensure - * that it doesn't descend from an invalid block, and then add it to m_block_index. - */ - bool AcceptBlockHeader( - const CBlockHeader& block, - BlockValidationState& state, - const CChainParams& chainparams, - CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Find the last common block between the parameter chain and a locator. */ - CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - //! Returns last CBlockIndex* that is a checkpoint CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -807,6 +774,9 @@ class CChainState /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload() const; + /** Find the last common block of this chain and a locator. */ + CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** * Make various assertions about the state of the block index. * @@ -953,18 +923,53 @@ class ChainstateManager //! by the background validation chainstate. bool m_snapshot_validated{false}; + CBlockIndex* m_best_invalid; + friend bool BlockManager::LoadBlockIndex(const Consensus::Params&, ChainstateManager&); + //! Internal helper for ActivateSnapshot(). [[nodiscard]] bool PopulateAndValidateSnapshot( CChainState& snapshot_chainstate, CAutoFile& coins_file, const SnapshotMetadata& metadata); + /** + * If a block header hasn't already been seen, call CheckBlockHeader on it, ensure + * that it doesn't descend from an invalid block, and then add it to m_block_index. + */ + bool AcceptBlockHeader( + const CBlockHeader& block, + BlockValidationState& state, + const CChainParams& chainparams, + CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + friend CChainState; + public: std::thread m_load_block; //! A single BlockManager instance is shared across each constructed //! chainstate to avoid duplicating block metadata. BlockManager m_blockman GUARDED_BY(::cs_main); + /** + * In order to efficiently track invalidity of headers, we keep the set of + * blocks which we tried to connect and found to be invalid here (ie which + * were set to BLOCK_FAILED_VALID since the last restart). We can then + * walk this set and check if a new header is a descendant of something in + * this set, preventing us from having to walk m_block_index when we try + * to connect a bad block and fail. + * + * While this is more complicated than marking everything which descends + * from an invalid block as invalid at the time we discover it to be + * invalid, doing so would require walking all of m_block_index to find all + * descendants. Since this case should be very rare, keeping track of all + * BLOCK_FAILED_VALID blocks in a set should be just fine and work just as + * well. + * + * Because we already walk m_block_index in height-order at startup, we go + * ahead and mark descendants of invalid blocks as FAILED_CHILD at that time, + * instead of putting things in this set. + */ + std::set m_failed_blocks; + //! The total number of bytes available for us to use across all in-memory //! coins caches. This will be split somehow across chainstates. int64_t m_total_coinstip_cache{0}; From 301163c65e2365ea470d0f3222419ea910d41c10 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sun, 2 Jan 2022 17:05:43 +0100 Subject: [PATCH 07/11] merge bitcoin#23581: Move BlockManager to node/blockstorage --- src/node/blockstorage.cpp | 399 +++++++++++++++++++++++++++++++++++++ src/node/blockstorage.h | 93 +++++++++ src/validation.cpp | 400 -------------------------------------- src/validation.h | 91 +-------- 4 files changed, 493 insertions(+), 490 deletions(-) diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 2b2f21ca6b6fa..a907713cc01fb 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -53,6 +54,404 @@ static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); static FlatFileSeq UndoFileSeq(); +CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const +{ + AssertLockHeld(cs_main); + BlockMap::const_iterator it = m_block_index.find(hash); + return it == m_block_index.end() ? nullptr : it->second; +} + +CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, const uint256& hash, enum BlockStatus nStatus) +{ + assert(!(nStatus & BLOCK_FAILED_MASK)); // no failed blocks allowed + AssertLockHeld(cs_main); + + // Check for duplicate + BlockMap::iterator it = m_block_index.find(hash); + if (it != m_block_index.end()) { + return it->second; + } + + // Construct new block index object + CBlockIndex* pindexNew = new CBlockIndex(block); + // We assign the sequence id to blocks only when the full data is available, + // to avoid miners withholding blocks but broadcasting headers, to get a + // competitive advantage. + pindexNew->nSequenceId = 0; + BlockMap::iterator mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; + pindexNew->phashBlock = &((*mi).first); + BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock); + if (miPrev != m_block_index.end()) { + pindexNew->pprev = (*miPrev).second; + pindexNew->nHeight = pindexNew->pprev->nHeight + 1; + pindexNew->BuildSkip(); + } + pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime); + pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew); + if (nStatus & BLOCK_VALID_MASK) { + pindexNew->RaiseValidity(nStatus); + if (pindexBestHeader == nullptr || pindexBestHeader->nChainWork < pindexNew->nChainWork) { + pindexBestHeader = pindexNew; + } + } else { + pindexNew->RaiseValidity(BLOCK_VALID_TREE); // required validity level + pindexNew->nStatus |= nStatus; + } + + setDirtyBlockIndex.insert(pindexNew); + + // track prevBlockHash -> pindex (multimap) + if (pindexNew->pprev) { + m_prev_block_index.emplace(pindexNew->pprev->GetBlockHash(), pindexNew); + } + + return pindexNew; +} + +void BlockManager::PruneOneBlockFile(const int fileNumber) +{ + AssertLockHeld(cs_main); + LOCK(cs_LastBlockFile); + + for (const auto& entry : m_block_index) { + CBlockIndex* pindex = entry.second; + if (pindex->nFile == fileNumber) { + pindex->nStatus &= ~BLOCK_HAVE_DATA; + pindex->nStatus &= ~BLOCK_HAVE_UNDO; + pindex->nFile = 0; + pindex->nDataPos = 0; + pindex->nUndoPos = 0; + setDirtyBlockIndex.insert(pindex); + + // Prune from m_blocks_unlinked -- any block we prune would have + // to be downloaded again in order to consider its chain, at which + // point it would be considered as a candidate for + // m_blocks_unlinked or setBlockIndexCandidates. + auto range = m_blocks_unlinked.equal_range(pindex->pprev); + while (range.first != range.second) { + std::multimap::iterator _it = range.first; + range.first++; + if (_it->second == pindex) { + m_blocks_unlinked.erase(_it); + } + } + } + } + + vinfoBlockFile[fileNumber].SetNull(); + setDirtyFileInfo.insert(fileNumber); +} + +void BlockManager::FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight, int chain_tip_height) +{ + assert(fPruneMode && nManualPruneHeight > 0); + + LOCK2(cs_main, cs_LastBlockFile); + if (chain_tip_height < 0) { + return; + } + + // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip) + unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP); + int count = 0; + for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { + if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { + continue; + } + PruneOneBlockFile(fileNumber); + setFilesToPrune.insert(fileNumber); + count++; + } + LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count); +} + +void BlockManager::FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd) +{ + LOCK2(cs_main, cs_LastBlockFile); + if (chain_tip_height < 0 || nPruneTarget == 0) { + return; + } + if ((uint64_t)chain_tip_height <= nPruneAfterHeight) { + return; + } + + unsigned int nLastBlockWeCanPrune{(unsigned)std::min(prune_height, chain_tip_height - static_cast(MIN_BLOCKS_TO_KEEP))}; + uint64_t nCurrentUsage = CalculateCurrentUsage(); + // We don't check to prune until after we've allocated new space for files + // So we should leave a buffer under our target to account for another allocation + // before the next pruning. + uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; + uint64_t nBytesToPrune; + int count = 0; + + if (nCurrentUsage + nBuffer >= nPruneTarget) { + // On a prune event, the chainstate DB is flushed. + // To avoid excessive prune events negating the benefit of high dbcache + // values, we should not prune too rapidly. + // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon. + if (is_ibd) { + // Since this is only relevant during IBD, we use a fixed 10% + nBuffer += nPruneTarget / 10; + } + + for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { + nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; + + if (vinfoBlockFile[fileNumber].nSize == 0) { + continue; + } + + if (nCurrentUsage + nBuffer < nPruneTarget) { // are we below our target? + break; + } + + // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning + if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { + continue; + } + + PruneOneBlockFile(fileNumber); + // Queue up the files for removal + setFilesToPrune.insert(fileNumber); + nCurrentUsage -= nBytesToPrune; + count++; + } + } + + LogPrint(BCLog::PRUNE, "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", + nPruneTarget/1024/1024, nCurrentUsage/1024/1024, + ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, + nLastBlockWeCanPrune, count); +} + +CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) +{ + AssertLockHeld(cs_main); + + if (hash.IsNull()) { + return nullptr; + } + + // Return existing + BlockMap::iterator mi = m_block_index.find(hash); + if (mi != m_block_index.end()) { + return (*mi).second; + } + + // Create new + CBlockIndex* pindexNew = new CBlockIndex(); + mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; + pindexNew->phashBlock = &((*mi).first); + + return pindexNew; +} + +bool BlockManager::LoadBlockIndex( + const Consensus::Params& consensus_params, + ChainstateManager& chainman) +{ + if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { + return false; + } + + // Calculate nChainWork + std::vector> vSortedByHeight; + vSortedByHeight.reserve(m_block_index.size()); + for (const std::pair& item : m_block_index) { + CBlockIndex* pindex = item.second; + vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); + + // build m_blockman.m_prev_block_index + if (pindex->pprev) { + m_prev_block_index.emplace(pindex->pprev->GetBlockHash(), pindex); + } + } + sort(vSortedByHeight.begin(), vSortedByHeight.end()); + + // Find start of assumed-valid region. + int first_assumed_valid_height = std::numeric_limits::max(); + + for (const auto& [height, block] : vSortedByHeight) { + if (block->IsAssumedValid()) { + auto chainstates = chainman.GetAll(); + + // If we encounter an assumed-valid block index entry, ensure that we have + // one chainstate that tolerates assumed-valid entries and another that does + // not (i.e. the background validation chainstate), since assumed-valid + // entries should always be pending validation by a fully-validated chainstate. + auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; + assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); + assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); + + first_assumed_valid_height = height; + break; + } + } + + for (const std::pair& item : vSortedByHeight) { + if (ShutdownRequested()) return false; + CBlockIndex* pindex = item.second; + pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); + pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); + + // We can link the chain of blocks for which we've received transactions at some point, or + // blocks that are assumed-valid on the basis of snapshot load (see + // PopulateAndValidateSnapshot()). + // Pruned nodes may have deleted the block. + if (pindex->nTx > 0) { + if (pindex->pprev) { + if (pindex->pprev->nChainTx > 0) { + pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + } else { + pindex->nChainTx = 0; + m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex)); + } + } else { + pindex->nChainTx = pindex->nTx; + } + } + if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { + pindex->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(pindex); + } + if (pindex->IsAssumedValid() || + (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && + (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { + + // Fill each chainstate's block candidate set. Only add assumed-valid + // blocks to the tip candidate set if the chainstate is allowed to rely on + // assumed-valid blocks. + // + // If all setBlockIndexCandidates contained the assumed-valid blocks, the + // background chainstate's ActivateBestChain() call would add assumed-valid + // blocks to the chain (based on how FindMostWorkChain() works). Obviously + // we don't want this since the purpose of the background validation chain + // is to validate assued-valid blocks. + // + // Note: This is considering all blocks whose height is greater or equal to + // the first assumed-valid block to be assumed-valid blocks, and excluding + // them from the background chainstate's setBlockIndexCandidates set. This + // does mean that some blocks which are not technically assumed-valid + // (later blocks on a fork beginning before the first assumed-valid block) + // might not get added to the the background chainstate, but this is ok, + // because they will still be attached to the active chainstate if they + // actually contain more work. + // + // Instad of this height-based approach, an earlier attempt was made at + // detecting "holistically" whether the block index under consideration + // relied on an assumed-valid ancestor, but this proved to be too slow to + // be practical. + for (CChainState* chainstate : chainman.GetAll()) { + if (chainstate->reliesOnAssumedValid() || + pindex->nHeight < first_assumed_valid_height) { + chainstate->setBlockIndexCandidates.insert(pindex); + } + } + } + if (pindex->nStatus & BLOCK_FAILED_MASK && (!chainman.m_best_invalid || pindex->nChainWork > chainman.m_best_invalid->nChainWork)) { + chainman.m_best_invalid = pindex; + } + if (pindex->pprev) { + pindex->BuildSkip(); + } + if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == nullptr || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) + pindexBestHeader = pindex; + } + + return true; +} + +void BlockManager::Unload() +{ + m_blocks_unlinked.clear(); + + for (const BlockMap::value_type& entry : m_block_index) { + delete entry.second; + } + + m_block_index.clear(); + m_prev_block_index.clear(); +} + +bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) +{ + if (!LoadBlockIndex(::Params().GetConsensus(), chainman)) { + return false; + } + + // Load block file info + m_block_tree_db->ReadLastBlockFile(nLastBlockFile); + vinfoBlockFile.resize(nLastBlockFile + 1); + LogPrintf("%s: last block file = %i\n", __func__, nLastBlockFile); + for (int nFile = 0; nFile <= nLastBlockFile; nFile++) { + m_block_tree_db->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); + } + LogPrintf("%s: last block file info: %s\n", __func__, vinfoBlockFile[nLastBlockFile].ToString()); + for (int nFile = nLastBlockFile + 1; true; nFile++) { + CBlockFileInfo info; + if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) { + vinfoBlockFile.push_back(info); + } else { + break; + } + } + + // Check presence of blk files + LogPrintf("Checking all blk files are present...\n"); + std::set setBlkDataFiles; + for (const std::pair& item : m_block_index) { + CBlockIndex* pindex = item.second; + if (pindex->nStatus & BLOCK_HAVE_DATA) { + setBlkDataFiles.insert(pindex->nFile); + } + } + for (std::set::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { + FlatFilePos pos(*it, 0); + if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { + return false; + } + } + + // Check whether we have ever pruned block & undo files + m_block_tree_db->ReadFlag("prunedblockfiles", fHavePruned); + if (fHavePruned) { + LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); + } + + // Check whether we need to continue reindexing + bool fReindexing = false; + m_block_tree_db->ReadReindexing(fReindexing); + if (fReindexing) fReindex = true; + + // Check whether we have an address index + m_block_tree_db->ReadFlag("addressindex", fAddressIndex); + LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + + // Check whether we have a timestamp index + m_block_tree_db->ReadFlag("timestampindex", fTimestampIndex); + LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + + // Check whether we have a spent index + m_block_tree_db->ReadFlag("spentindex", fSpentIndex); + LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); + + return true; +} + +CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) +{ + const MapCheckpoints& checkpoints = data.mapCheckpoints; + + for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) { + const uint256& hash = i.second; + CBlockIndex* pindex = LookupBlockIndex(hash); + if (pindex) { + return pindex; + } + } + return nullptr; +} + bool IsBlockPruned(const CBlockIndex* pblockindex) { return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 378a733dbb080..f04bd6af4e747 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -7,6 +7,7 @@ #include #include // For CMessageHeader::MessageStartChars +#include #include #include @@ -20,9 +21,11 @@ class CBlockIndex; class CBlockUndo; class CChain; class CChainParams; +class CChainState; class CDeterministicMNManager; class CDSNotificationInterface; class ChainstateManager; +struct CCheckpointData; struct FlatFilePos; namespace Consensus { struct Params; @@ -57,6 +60,96 @@ extern bool fTimestampIndex; /** True if we're running in -spentindex mode. */ extern bool fSpentIndex; +typedef std::unordered_map BlockMap; +typedef std::unordered_multimap PrevBlockMap; + +struct CBlockIndexWorkComparator { + bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const; +}; + +/** + * Maintains a tree of blocks (stored in `m_block_index`) which is consulted + * to determine where the most-work tip is. + * + * This data is used mostly in `CChainState` - information about, e.g., + * candidate tips is not maintained here. + */ +class BlockManager +{ + friend CChainState; + +private: + /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ + void FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight, int chain_tip_height); + + /** + * Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a user-defined target. + * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new + * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex + * (which in this case means the blockchain must be re-downloaded.) + * + * Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set. + * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.) + * Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 1000 on regtest). + * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip. + * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files. + * A db flag records the fact that at least some block files have been pruned. + * + * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned + */ + void FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd); + +public: + BlockMap m_block_index GUARDED_BY(cs_main); + PrevBlockMap m_prev_block_index GUARDED_BY(cs_main); + + /** + * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. + * Pruned nodes may have entries where B is missing data. + */ + std::multimap m_blocks_unlinked; + + std::unique_ptr m_block_tree_db GUARDED_BY(::cs_main); + + bool LoadBlockIndexDB(ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + /** + * Load the blocktree off disk and into memory. Populate certain metadata + * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral + * collections like setDirtyBlockIndex. + */ + bool LoadBlockIndex( + const Consensus::Params& consensus_params, + ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** Clear all data members. */ + void Unload() EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + CBlockIndex* AddToBlockIndex(const CBlockHeader& block, const uint256& hash, enum BlockStatus nStatus = BLOCK_VALID_TREE) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Create a new block index entry for a given block hash */ + CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Mark one block file as pruned (modify associated database entries) + void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Returns last CBlockIndex* that is a checkpoint + CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** + * Return the spend height, which is one more than the inputs.GetBestBlock(). + * While checking, GetBestBlock() refers to the parent block. (protected by cs_main) + * This is also true for mempool checks. + */ + int GetSpendHeight(const CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + ~BlockManager() + { + Unload(); + } +}; + //! Check whether the block associated with this index entry is pruned or not. bool IsBlockPruned(const CBlockIndex* pblockindex); diff --git a/src/validation.cpp b/src/validation.cpp index c9516bbe919e7..9f5c06bececd8 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -152,13 +152,6 @@ extern std::set setDirtyFileInfo; void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); // ... TODO move fully to blockstorage -CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const -{ - AssertLockHeld(cs_main); - BlockMap::const_iterator it = m_block_index.find(hash); - return it == m_block_index.end() ? nullptr : it->second; -} - CBlockIndex* CChainState::FindForkInGlobalIndex(const CBlockLocator& locator) const { AssertLockHeld(cs_main); @@ -3437,52 +3430,6 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { } } -CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, const uint256& hash, enum BlockStatus nStatus) -{ - assert(!(nStatus & BLOCK_FAILED_MASK)); // no failed blocks allowed - AssertLockHeld(cs_main); - - // Check for duplicate - BlockMap::iterator it = m_block_index.find(hash); - if (it != m_block_index.end()) - return it->second; - - // Construct new block index object - CBlockIndex* pindexNew = new CBlockIndex(block); - // We assign the sequence id to blocks only when the full data is available, - // to avoid miners withholding blocks but broadcasting headers, to get a - // competitive advantage. - pindexNew->nSequenceId = 0; - BlockMap::iterator mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; - pindexNew->phashBlock = &((*mi).first); - BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock); - if (miPrev != m_block_index.end()) - { - pindexNew->pprev = (*miPrev).second; - pindexNew->nHeight = pindexNew->pprev->nHeight + 1; - pindexNew->BuildSkip(); - } - pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime); - pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew); - if (nStatus & BLOCK_VALID_MASK) { - pindexNew->RaiseValidity(nStatus); - if (pindexBestHeader == nullptr || pindexBestHeader->nChainWork < pindexNew->nChainWork) - pindexBestHeader = pindexNew; - } else { - pindexNew->RaiseValidity(BLOCK_VALID_TREE); // required validity level - pindexNew->nStatus |= nStatus; - } - - setDirtyBlockIndex.insert(pindexNew); - - // track prevBlockHash -> pindex (multimap) - if (pindexNew->pprev) { - m_prev_block_index.emplace(pindexNew->pprev->GetBlockHash(), pindexNew); - } - - return pindexNew; -} - /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) { @@ -3617,21 +3564,6 @@ bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensu return true; } -CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) -{ - const MapCheckpoints& checkpoints = data.mapCheckpoints; - - for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) - { - const uint256& hash = i.second; - CBlockIndex* pindex = LookupBlockIndex(hash); - if (pindex) { - return pindex; - } - } - return nullptr; -} - /** Context-dependent validity checks. * By "context", we mean only the previous block headers, but not the UTXO * set; UTXO-related validity checks are done in ConnectBlock(). @@ -4093,67 +4025,6 @@ bool TestBlockValidity(BlockValidationState& state, return true; } -/** - * BLOCK PRUNING CODE - */ - -void BlockManager::PruneOneBlockFile(const int fileNumber) -{ - AssertLockHeld(cs_main); - LOCK(cs_LastBlockFile); - - for (const auto& entry : m_block_index) { - CBlockIndex* pindex = entry.second; - if (pindex->nFile == fileNumber) { - pindex->nStatus &= ~BLOCK_HAVE_DATA; - pindex->nStatus &= ~BLOCK_HAVE_UNDO; - pindex->nFile = 0; - pindex->nDataPos = 0; - pindex->nUndoPos = 0; - setDirtyBlockIndex.insert(pindex); - - // Prune from m_blocks_unlinked -- any block we prune would have - // to be downloaded again in order to consider its chain, at which - // point it would be considered as a candidate for - // m_blocks_unlinked or setBlockIndexCandidates. - auto range = m_blocks_unlinked.equal_range(pindex->pprev); - while (range.first != range.second) { - std::multimap::iterator _it = range.first; - range.first++; - if (_it->second == pindex) { - m_blocks_unlinked.erase(_it); - } - } - } - } - - vinfoBlockFile[fileNumber].SetNull(); - setDirtyFileInfo.insert(fileNumber); -} - -void BlockManager::FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight, int chain_tip_height) -{ - assert(fPruneMode && nManualPruneHeight > 0); - - LOCK2(cs_main, cs_LastBlockFile); - if (chain_tip_height < 0) { - return; - } - - // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip) - unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP); - int count = 0; - for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { - if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { - continue; - } - PruneOneBlockFile(fileNumber); - setFilesToPrune.insert(fileNumber); - count++; - } - LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count); -} - /* This function is called from the RPC code for pruneblockchain */ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeight) { @@ -4163,277 +4034,6 @@ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeigh } } -void BlockManager::FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd) -{ - LOCK2(cs_main, cs_LastBlockFile); - if (chain_tip_height < 0 || nPruneTarget == 0) { - return; - } - if ((uint64_t)chain_tip_height <= nPruneAfterHeight) { - return; - } - - unsigned int nLastBlockWeCanPrune{(unsigned)std::min(prune_height, chain_tip_height - static_cast(MIN_BLOCKS_TO_KEEP))}; - uint64_t nCurrentUsage = CalculateCurrentUsage(); - // We don't check to prune until after we've allocated new space for files - // So we should leave a buffer under our target to account for another allocation - // before the next pruning. - uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; - uint64_t nBytesToPrune; - int count = 0; - - if (nCurrentUsage + nBuffer >= nPruneTarget) { - // On a prune event, the chainstate DB is flushed. - // To avoid excessive prune events negating the benefit of high dbcache - // values, we should not prune too rapidly. - // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon. - if (is_ibd) { - // Since this is only relevant during IBD, we use a fixed 10% - nBuffer += nPruneTarget / 10; - } - - for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { - nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; - - if (vinfoBlockFile[fileNumber].nSize == 0) { - continue; - } - - if (nCurrentUsage + nBuffer < nPruneTarget) { // are we below our target? - break; - } - - // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning - if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { - continue; - } - - PruneOneBlockFile(fileNumber); - // Queue up the files for removal - setFilesToPrune.insert(fileNumber); - nCurrentUsage -= nBytesToPrune; - count++; - } - } - - LogPrint(BCLog::PRUNE, "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", - nPruneTarget/1024/1024, nCurrentUsage/1024/1024, - ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, - nLastBlockWeCanPrune, count); -} - -CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash) -{ - AssertLockHeld(cs_main); - - if (hash.IsNull()) - return nullptr; - - // Return existing - BlockMap::iterator mi = m_block_index.find(hash); - if (mi != m_block_index.end()) - return (*mi).second; - - // Create new - CBlockIndex* pindexNew = new CBlockIndex(); - mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; - pindexNew->phashBlock = &((*mi).first); - - return pindexNew; -} - -bool BlockManager::LoadBlockIndex( - const Consensus::Params& consensus_params, - ChainstateManager& chainman) -{ - if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { - return false; - } - - // Calculate nChainWork - std::vector > vSortedByHeight; - vSortedByHeight.reserve(m_block_index.size()); - for (const std::pair& item : m_block_index) - { - CBlockIndex* pindex = item.second; - vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); - - // build m_blockman.m_prev_block_index - if (pindex->pprev) { - m_prev_block_index.emplace(pindex->pprev->GetBlockHash(), pindex); - } - } - sort(vSortedByHeight.begin(), vSortedByHeight.end()); - - // Find start of assumed-valid region. - int first_assumed_valid_height = std::numeric_limits::max(); - - for (const auto& [height, block] : vSortedByHeight) { - if (block->IsAssumedValid()) { - auto chainstates = chainman.GetAll(); - - // If we encounter an assumed-valid block index entry, ensure that we have - // one chainstate that tolerates assumed-valid entries and another that does - // not (i.e. the background validation chainstate), since assumed-valid - // entries should always be pending validation by a fully-validated chainstate. - auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; - assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); - assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); - - first_assumed_valid_height = height; - break; - } - } - - for (const std::pair& item : vSortedByHeight) - { - if (ShutdownRequested()) return false; - CBlockIndex* pindex = item.second; - pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); - pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); - - // We can link the chain of blocks for which we've received transactions at some point, or - // blocks that are assumed-valid on the basis of snapshot load (see - // PopulateAndValidateSnapshot()). - // Pruned nodes may have deleted the block. - if (pindex->nTx > 0) { - if (pindex->pprev) { - if (pindex->pprev->nChainTx > 0) { - pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; - } else { - pindex->nChainTx = 0; - m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex)); - } - } else { - pindex->nChainTx = pindex->nTx; - } - } - if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { - pindex->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(pindex); - } - if (pindex->IsAssumedValid() || - (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && - (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { - - // Fill each chainstate's block candidate set. Only add assumed-valid - // blocks to the tip candidate set if the chainstate is allowed to rely on - // assumed-valid blocks. - // - // If all setBlockIndexCandidates contained the assumed-valid blocks, the - // background chainstate's ActivateBestChain() call would add assumed-valid - // blocks to the chain (based on how FindMostWorkChain() works). Obviously - // we don't want this since the purpose of the background validation chain - // is to validate assued-valid blocks. - // - // Note: This is considering all blocks whose height is greater or equal to - // the first assumed-valid block to be assumed-valid blocks, and excluding - // them from the background chainstate's setBlockIndexCandidates set. This - // does mean that some blocks which are not technically assumed-valid - // (later blocks on a fork beginning before the first assumed-valid block) - // might not get added to the the background chainstate, but this is ok, - // because they will still be attached to the active chainstate if they - // actually contain more work. - // - // Instad of this height-based approach, an earlier attempt was made at - // detecting "holistically" whether the block index under consideration - // relied on an assumed-valid ancestor, but this proved to be too slow to - // be practical. - for (CChainState* chainstate : chainman.GetAll()) { - if (chainstate->reliesOnAssumedValid() || - pindex->nHeight < first_assumed_valid_height) { - chainstate->setBlockIndexCandidates.insert(pindex); - } - } - } - if (pindex->nStatus & BLOCK_FAILED_MASK && (!chainman.m_best_invalid || pindex->nChainWork > chainman.m_best_invalid->nChainWork)) { - chainman.m_best_invalid = pindex; - } - if (pindex->pprev) - pindex->BuildSkip(); - if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == nullptr || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) - pindexBestHeader = pindex; - } - - return true; -} - -void BlockManager::Unload() { - m_blocks_unlinked.clear(); - - for (const BlockMap::value_type& entry : m_block_index) { - delete entry.second; - } - - m_block_index.clear(); - m_prev_block_index.clear(); -} - -bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) -{ - if (!LoadBlockIndex(::Params().GetConsensus(), chainman)) { - return false; - } - - // Load block file info - m_block_tree_db->ReadLastBlockFile(nLastBlockFile); - vinfoBlockFile.resize(nLastBlockFile + 1); - LogPrintf("%s: last block file = %i\n", __func__, nLastBlockFile); - for (int nFile = 0; nFile <= nLastBlockFile; nFile++) { - m_block_tree_db->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); - } - LogPrintf("%s: last block file info: %s\n", __func__, vinfoBlockFile[nLastBlockFile].ToString()); - for (int nFile = nLastBlockFile + 1; true; nFile++) { - CBlockFileInfo info; - if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) { - vinfoBlockFile.push_back(info); - } else { - break; - } - } - - // Check presence of blk files - LogPrintf("Checking all blk files are present...\n"); - std::set setBlkDataFiles; - for (const std::pair& item : m_block_index) { - CBlockIndex* pindex = item.second; - if (pindex->nStatus & BLOCK_HAVE_DATA) { - setBlkDataFiles.insert(pindex->nFile); - } - } - for (std::set::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) - { - FlatFilePos pos(*it, 0); - if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { - return false; - } - } - - // Check whether we have ever pruned block & undo files - m_block_tree_db->ReadFlag("prunedblockfiles", fHavePruned); - if (fHavePruned) - LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); - - // Check whether we need to continue reindexing - bool fReindexing = false; - m_block_tree_db->ReadReindexing(fReindexing); - if(fReindexing) fReindex = true; - - // Check whether we have an address index - m_block_tree_db->ReadFlag("addressindex", fAddressIndex); - LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); - - // Check whether we have a timestamp index - m_block_tree_db->ReadFlag("timestampindex", fTimestampIndex); - LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); - - // Check whether we have a spent index - m_block_tree_db->ReadFlag("spentindex", fSpentIndex); - LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); - - return true; -} - void CChainState::LoadMempool(const ArgsManager& args) { if (!m_mempool) return; diff --git a/src/validation.h b/src/validation.h index 090666c09172b..df09832464995 100644 --- a/src/validation.h +++ b/src/validation.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include