diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 1a805a1dd1f50..ee349bbc6f248 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -9,6 +9,7 @@ EXTRA_LIBRARIES += \ TEST_UTIL_H = \ test/util/blockfilter.h \ + test/util/chainstate.h \ test/util/index.h \ test/util/logging.h \ test/util/mining.h \ diff --git a/src/chain.h b/src/chain.h index c1d122234ea6e..810d68261a474 100644 --- a/src/chain.h +++ b/src/chain.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,8 @@ static constexpr int64_t TIMESTAMP_WINDOW = MAX_FUTURE_BLOCK_TIME; */ static constexpr int64_t MAX_BLOCK_TIME_GAP = 25 * 60; +extern RecursiveMutex cs_main; + class CBlockFileInfo { public: @@ -124,6 +127,14 @@ enum BlockStatus: uint32_t { BLOCK_FAILED_MASK = BLOCK_FAILED_VALID | BLOCK_FAILED_CHILD, BLOCK_CONFLICT_CHAINLOCK = 128, //!< conflicts with chainlock system + + /** + * If set, this indicates that the block index entry is assumed-valid. + * Certain diagnostics will be skipped in e.g. CheckBlockIndex(). + * It almost certainly means that the block's full validation is pending + * on a background chainstate. See `doc/assumeutxo.md`. + */ + BLOCK_ASSUMED_VALID = 256, }; /** The block chain is a tree shaped structure starting with the @@ -147,13 +158,13 @@ class CBlockIndex int nHeight{0}; //! Which # file this block is stored in (blk?????.dat) - int nFile{0}; + int nFile GUARDED_BY(::cs_main){0}; //! Byte offset within blk?????.dat where this block's data is stored - unsigned int nDataPos{0}; + unsigned int nDataPos GUARDED_BY(::cs_main){0}; //! Byte offset within rev?????.dat where this block's undo data is stored - unsigned int nUndoPos{0}; + unsigned int nUndoPos GUARDED_BY(::cs_main){0}; //! (memory only) Total amount of work (expected number of hashes) in the chain up to and including this block arith_uint256 nChainWork{}; @@ -181,7 +192,7 @@ class CBlockIndex //! load to avoid the block index being spuriously rewound. //! @sa NeedsRedownload //! @sa ActivateSnapshot - uint32_t nStatus{0}; + uint32_t nStatus GUARDED_BY(::cs_main){0}; //! block header int32_t nVersion{0}; @@ -209,7 +220,9 @@ class CBlockIndex { } - FlatFilePos GetBlockPos() const { + FlatFilePos GetBlockPos() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + AssertLockHeld(::cs_main); FlatFilePos ret; if (nStatus & BLOCK_HAVE_DATA) { ret.nFile = nFile; @@ -218,7 +231,9 @@ class CBlockIndex return ret; } - FlatFilePos GetUndoPos() const { + FlatFilePos GetUndoPos() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + AssertLockHeld(::cs_main); FlatFilePos ret; if (nStatus & BLOCK_HAVE_UNDO) { ret.nFile = nFile; @@ -284,21 +299,38 @@ class CBlockIndex //! Check whether this block index entry is valid up to the passed validity level. bool IsValid(enum BlockStatus nUpTo = BLOCK_VALID_TRANSACTIONS) const + EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + AssertLockHeld(::cs_main); assert(!(nUpTo & ~BLOCK_VALID_MASK)); // Only validity flags allowed. if (nStatus & BLOCK_FAILED_MASK) return false; return ((nStatus & BLOCK_VALID_MASK) >= nUpTo); } + //! @returns true if the block is assumed-valid; this means it is queued to be + //! validated by a background chainstate. + bool IsAssumedValid() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + AssertLockHeld(::cs_main); + return nStatus & BLOCK_ASSUMED_VALID; + } + //! Raise the validity level of this block index entry. //! Returns true if the validity was changed. - bool RaiseValidity(enum BlockStatus nUpTo) + bool RaiseValidity(enum BlockStatus nUpTo) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + AssertLockHeld(::cs_main); assert(!(nUpTo & ~BLOCK_VALID_MASK)); // Only validity flags allowed. - if (nStatus & BLOCK_FAILED_MASK) - return false; + if (nStatus & BLOCK_FAILED_MASK) return false; + if ((nStatus & BLOCK_VALID_MASK) < nUpTo) { + // If this block had been marked assumed-valid and we're raising + // its validity to a certain point, there is no longer an assumption. + if (nStatus & BLOCK_ASSUMED_VALID && nUpTo >= BLOCK_VALID_SCRIPTS) { + nStatus &= ~BLOCK_ASSUMED_VALID; + } + nStatus = (nStatus & ~BLOCK_VALID_MASK) | nUpTo; return true; } @@ -339,6 +371,7 @@ class CDiskBlockIndex : public CBlockIndex SERIALIZE_METHODS(CDiskBlockIndex, obj) { + LOCK(::cs_main); int _nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT_MODE(_nVersion, VarIntMode::NONNEGATIVE_SIGNED)); diff --git a/src/index/base.cpp b/src/index/base.cpp index 7a0afaff8d216..866fbc96977ef 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -64,7 +64,7 @@ bool BaseIndex::Init() if (locator.IsNull()) { m_best_block_index = nullptr; } else { - m_best_block_index = m_chainstate->m_blockman.FindForkInGlobalIndex(active_chain, locator); + m_best_block_index = m_chainstate->FindForkInGlobalIndex(locator); } // Note: this will latch to true immediately if the user starts up with an empty diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 238b2312a091b..f37c7ca59a9c0 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -272,7 +272,7 @@ bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* n { LOCK(cs_main); - CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; + const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; const auto& consensus_params{Params().GetConsensus()}; do { diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 7b405436f844d..c5f9e9fbbc997 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -203,7 +204,7 @@ bool TxIndex::Init() // Attempt to migrate txindex from the old database to the new one. Even if // chain_tip is null, the node could be reindexing and we still want to // delete txindex records in the old database. - if (!m_db->MigrateData(*pblocktree, m_chainstate->m_chain.GetLocator())) { + if (!m_db->MigrateData(*m_chainstate->m_blockman.m_block_tree_db, m_chainstate->m_chain.GetLocator())) { return false; } @@ -215,7 +216,9 @@ bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) // Exclude genesis block transaction because outputs are not spendable. if (pindex->nHeight == 0) return true; - CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); + CDiskTxPos pos{ + WITH_LOCK(::cs_main, return pindex->GetBlockPos()), + GetSizeOfCompactSize(block.vtx.size())}; std::vector> vPos; vPos.reserve(block.vtx.size()); for (const auto& tx : block.vtx) { diff --git a/src/init.cpp b/src/init.cpp index 27b9b10dc64be..d0d2d067f4df9 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -338,7 +338,6 @@ void PrepareShutdown(NodeContext& node) chainstate->ResetCoinsViews(); } } - pblocktree.reset(); node.chain_helper.reset(); if (node.mnhf_manager) { node.mnhf_manager->DisconnectManagers(); @@ -831,47 +830,6 @@ static void BlockNotifyGenesisWait(const CBlockIndex* pBlockIndex) } } -// If we're using -prune with -reindex, then delete block files that will be ignored by the -// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile -// is missing, do the same here to delete any later block files after a gap. Also delete all -// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile -// is in sync with what's actually on disk by the time we start downloading, so that pruning -// works correctly. -static void CleanupBlockRevFiles() -{ - std::map mapBlockFiles; - - // Glob all blk?????.dat and rev?????.dat files from the blocks directory. - // Remove the rev files immediately and insert the blk file paths into an - // ordered map keyed by block file index. - LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - fs::path blocksdir = gArgs.GetBlocksDirPath(); - for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { - if (fs::is_regular_file(*it) && - it->path().filename().string().length() == 12 && - it->path().filename().string().substr(8,4) == ".dat") - { - if (it->path().filename().string().substr(0,3) == "blk") - mapBlockFiles[it->path().filename().string().substr(3,5)] = it->path(); - else if (it->path().filename().string().substr(0,3) == "rev") - remove(it->path()); - } - } - - // Remove all block files that aren't part of a contiguous set starting at - // zero by walking the ordered map (keys are block file indices) by - // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) - // start removing block files. - int nContigCounter = 0; - for (const std::pair& item : mapBlockFiles) { - if (LocaleIndependentAtoi(item.first) == nContigCounter) { - nContigCounter++; - continue; - } - remove(item.second); - } -} - #if HAVE_SYSTEM static void StartupNotify(const ArgsManager& args) { @@ -1992,6 +1950,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) UnloadBlockIndex(node.mempool.get(), chainman); + auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); diff --git a/src/llmq/instantsend.cpp b/src/llmq/instantsend.cpp index 5197384bee594..652285c222f24 100644 --- a/src/llmq/instantsend.cpp +++ b/src/llmq/instantsend.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/miner.cpp b/src/miner.cpp index 12ea59def7404..c9d47ab9382f5 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -57,7 +57,8 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam return nNewTime - nOldTime; } -BlockAssembler::Options::Options() { +BlockAssembler::Options::Options() +{ blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); nBlockMaxSize = DEFAULT_BLOCK_MAX_SIZE; } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index b12b7b3224eb9..fa3f4d0004354 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3926,7 +3926,7 @@ void PeerManagerImpl::ProcessMessage( LOCK(cs_main); // Find the last block the caller has in the main chain - const CBlockIndex* pindex = m_chainman.m_blockman.FindForkInGlobalIndex(m_chainman.ActiveChain(), locator); + const CBlockIndex* pindex = m_chainman.ActiveChainstate().FindForkInGlobalIndex(locator); // Send the rest of the chain if (pindex) @@ -4046,7 +4046,7 @@ void PeerManagerImpl::ProcessMessage( else { // Find the last block the caller has in the main chain - pindex = m_chainman.m_blockman.FindForkInGlobalIndex(m_chainman.ActiveChain(), locator); + pindex = m_chainman.ActiveChainstate().FindForkInGlobalIndex(locator); if (pindex) pindex = m_chainman.ActiveChain().Next(pindex); } diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index d271f93ccac07..648711e2907bf 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -6,21 +6,693 @@ #include #include +#include #include #include #include #include #include +#include #include #include +#include #include #include +#include #include #include #include -// From validation. TODO move here -bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false); +std::atomic_bool fImporting(false); +std::atomic_bool fReindex(false); +bool fHavePruned = false; +bool fPruneMode = false; +uint64_t nPruneTarget = 0; + +bool fAddressIndex = DEFAULT_ADDRESSINDEX; +bool fTimestampIndex = DEFAULT_TIMESTAMPINDEX; +bool fSpentIndex = DEFAULT_SPENTINDEX; + +bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const +{ + // First sort by most total work, ... + if (pa->nChainWork > pb->nChainWork) return false; + if (pa->nChainWork < pb->nChainWork) return true; + + // ... then by earliest time received, ... + if (pa->nSequenceId < pb->nSequenceId) return false; + if (pa->nSequenceId > pb->nSequenceId) return true; + + // Use pointer address as tie breaker (should only happen with blocks + // loaded from disk, as those all have id 0). + if (pa < pb) return false; + if (pa > pb) return true; + + // Identical blocks. + return false; +} + +bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const +{ + return pa->nHeight < pb->nHeight; +} + +static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); + +std::vector BlockManager::GetAllBlockIndices() +{ + AssertLockHeld(cs_main); + std::vector rv; + rv.reserve(m_block_index.size()); + for (auto& [_, block_index] : m_block_index) { + rv.push_back(&block_index); + } + return rv; +} + +CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) +{ + AssertLockHeld(cs_main); + BlockMap::iterator it = m_block_index.find(hash); + return it == m_block_index.end() ? nullptr : &it->second; +} + +const CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const +{ + AssertLockHeld(cs_main); + BlockMap::const_iterator it = m_block_index.find(hash); + return it == m_block_index.end() ? nullptr : &it->second; +} + +CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, const uint256& hash, enum BlockStatus nStatus) +{ + assert(!(nStatus & BLOCK_FAILED_MASK)); // no failed blocks allowed + AssertLockHeld(cs_main); + + auto [mi, inserted] = m_block_index.try_emplace(hash, block); + if (!inserted) { + return &mi->second; + } + CBlockIndex* pindexNew = &(*mi).second; + + // We assign the sequence id to blocks only when the full data is available, + // to avoid miners withholding blocks but broadcasting headers, to get a + // competitive advantage. + pindexNew->nSequenceId = 0; + + pindexNew->phashBlock = &((*mi).first); + BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock); + if (miPrev != m_block_index.end()) { + pindexNew->pprev = &(*miPrev).second; + pindexNew->nHeight = pindexNew->pprev->nHeight + 1; + pindexNew->BuildSkip(); + } + pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime); + pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew); + if (nStatus & BLOCK_VALID_MASK) { + pindexNew->RaiseValidity(nStatus); + if (pindexBestHeader == nullptr || pindexBestHeader->nChainWork < pindexNew->nChainWork) { + pindexBestHeader = pindexNew; + } + } else { + pindexNew->RaiseValidity(BLOCK_VALID_TREE); // required validity level + pindexNew->nStatus |= nStatus; + } + + m_dirty_blockindex.insert(pindexNew); + + // track prevBlockHash -> pindex (multimap) + if (pindexNew->pprev) { + m_prev_block_index.emplace(pindexNew->pprev->GetBlockHash(), pindexNew); + } + + return pindexNew; +} + +void BlockManager::PruneOneBlockFile(const int fileNumber) +{ + AssertLockHeld(cs_main); + LOCK(cs_LastBlockFile); + + for (auto& entry : m_block_index) { + CBlockIndex* pindex = &entry.second; + if (pindex->nFile == fileNumber) { + pindex->nStatus &= ~BLOCK_HAVE_DATA; + pindex->nStatus &= ~BLOCK_HAVE_UNDO; + pindex->nFile = 0; + pindex->nDataPos = 0; + pindex->nUndoPos = 0; + m_dirty_blockindex.insert(pindex); + + // Prune from m_blocks_unlinked -- any block we prune would have + // to be downloaded again in order to consider its chain, at which + // point it would be considered as a candidate for + // m_blocks_unlinked or setBlockIndexCandidates. + auto range = m_blocks_unlinked.equal_range(pindex->pprev); + while (range.first != range.second) { + std::multimap::iterator _it = range.first; + range.first++; + if (_it->second == pindex) { + m_blocks_unlinked.erase(_it); + } + } + } + } + + m_blockfile_info[fileNumber].SetNull(); + m_dirty_fileinfo.insert(fileNumber); +} + +void BlockManager::FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight, int chain_tip_height) +{ + assert(fPruneMode && nManualPruneHeight > 0); + + LOCK2(cs_main, cs_LastBlockFile); + if (chain_tip_height < 0) { + return; + } + + // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip) + unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP); + int count = 0; + for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) { + if (m_blockfile_info[fileNumber].nSize == 0 || m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) { + continue; + } + PruneOneBlockFile(fileNumber); + setFilesToPrune.insert(fileNumber); + count++; + } + LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count); +} + +void BlockManager::FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd) +{ + LOCK2(cs_main, cs_LastBlockFile); + if (chain_tip_height < 0 || nPruneTarget == 0) { + return; + } + if ((uint64_t)chain_tip_height <= nPruneAfterHeight) { + return; + } + + unsigned int nLastBlockWeCanPrune{(unsigned)std::min(prune_height, chain_tip_height - static_cast(MIN_BLOCKS_TO_KEEP))}; + uint64_t nCurrentUsage = CalculateCurrentUsage(); + // We don't check to prune until after we've allocated new space for files + // So we should leave a buffer under our target to account for another allocation + // before the next pruning. + uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; + uint64_t nBytesToPrune; + int count = 0; + + if (nCurrentUsage + nBuffer >= nPruneTarget) { + // On a prune event, the chainstate DB is flushed. + // To avoid excessive prune events negating the benefit of high dbcache + // values, we should not prune too rapidly. + // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon. + if (is_ibd) { + // Since this is only relevant during IBD, we use a fixed 10% + nBuffer += nPruneTarget / 10; + } + + for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) { + nBytesToPrune = m_blockfile_info[fileNumber].nSize + m_blockfile_info[fileNumber].nUndoSize; + + if (m_blockfile_info[fileNumber].nSize == 0) { + continue; + } + + if (nCurrentUsage + nBuffer < nPruneTarget) { // are we below our target? + break; + } + + // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning + if (m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) { + continue; + } + + PruneOneBlockFile(fileNumber); + // Queue up the files for removal + setFilesToPrune.insert(fileNumber); + nCurrentUsage -= nBytesToPrune; + count++; + } + } + + LogPrint(BCLog::PRUNE, "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", + nPruneTarget/1024/1024, nCurrentUsage/1024/1024, + ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, + nLastBlockWeCanPrune, count); +} + +CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) +{ + AssertLockHeld(cs_main); + + if (hash.IsNull()) { + return nullptr; + } + + const auto [mi, inserted]{m_block_index.try_emplace(hash)}; + CBlockIndex* pindex = &(*mi).second; + if (inserted) { + pindex->phashBlock = &((*mi).first); + } + return pindex; +} + +bool BlockManager::LoadBlockIndex(const Consensus::Params& consensus_params) +{ + if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { + return false; + } + + for (auto& [_, block_index] : m_block_index) { + // build m_blockman.m_prev_block_index + if (block_index.pprev) { + m_prev_block_index.emplace(block_index.pprev->GetBlockHash(), &block_index); + } + } + + // Calculate nChainWork + std::vector vSortedByHeight{GetAllBlockIndices()}; + std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), + CBlockIndexHeightOnlyComparator()); + + for (CBlockIndex* pindex : vSortedByHeight) { + if (ShutdownRequested()) return false; + pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); + pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); + + // We can link the chain of blocks for which we've received transactions at some point, or + // blocks that are assumed-valid on the basis of snapshot load (see + // PopulateAndValidateSnapshot()). + // Pruned nodes may have deleted the block. + if (pindex->nTx > 0) { + if (pindex->pprev) { + if (pindex->pprev->nChainTx > 0) { + pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + } else { + pindex->nChainTx = 0; + m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex)); + } + } else { + pindex->nChainTx = pindex->nTx; + } + } + if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { + pindex->nStatus |= BLOCK_FAILED_CHILD; + m_dirty_blockindex.insert(pindex); + } + if (pindex->pprev) { + pindex->BuildSkip(); + } + if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == nullptr || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) + pindexBestHeader = pindex; + } + + return true; +} + +void BlockManager::Unload() +{ + m_blocks_unlinked.clear(); + + m_block_index.clear(); + m_prev_block_index.clear(); + + m_blockfile_info.clear(); + m_last_blockfile = 0; + m_dirty_blockindex.clear(); + m_dirty_fileinfo.clear(); +} + +bool BlockManager::WriteBlockIndexDB() +{ + std::vector> vFiles; + vFiles.reserve(m_dirty_fileinfo.size()); + for (std::set::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end();) { + vFiles.push_back(std::make_pair(*it, &m_blockfile_info[*it])); + m_dirty_fileinfo.erase(it++); + } + std::vector vBlocks; + vBlocks.reserve(m_dirty_blockindex.size()); + for (std::set::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end();) { + vBlocks.push_back(*it); + m_dirty_blockindex.erase(it++); + } + if (!m_block_tree_db->WriteBatchSync(vFiles, m_last_blockfile, vBlocks)) { + return false; + } + return true; +} + +bool BlockManager::LoadBlockIndexDB() +{ + if (!LoadBlockIndex(::Params().GetConsensus())) { + return false; + } + + // Load block file info + m_block_tree_db->ReadLastBlockFile(m_last_blockfile); + m_blockfile_info.resize(m_last_blockfile + 1); + LogPrintf("%s: last block file = %i\n", __func__, m_last_blockfile); + for (int nFile = 0; nFile <= m_last_blockfile; nFile++) { + m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]); + } + LogPrintf("%s: last block file info: %s\n", __func__, m_blockfile_info[m_last_blockfile].ToString()); + for (int nFile = m_last_blockfile + 1; true; nFile++) { + CBlockFileInfo info; + if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) { + m_blockfile_info.push_back(info); + } else { + break; + } + } + + // Check presence of blk files + LogPrintf("Checking all blk files are present...\n"); + std::set setBlkDataFiles; + for (const auto& [_, block_index] : m_block_index) { + if (block_index.nStatus & BLOCK_HAVE_DATA) { + setBlkDataFiles.insert(block_index.nFile); + } + } + for (std::set::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { + FlatFilePos pos(*it, 0); + if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { + return false; + } + } + + // Check whether we have ever pruned block & undo files + m_block_tree_db->ReadFlag("prunedblockfiles", fHavePruned); + if (fHavePruned) { + LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); + } + + // Check whether we need to continue reindexing + bool fReindexing = false; + m_block_tree_db->ReadReindexing(fReindexing); + if (fReindexing) fReindex = true; + + // Check whether we have an address index + m_block_tree_db->ReadFlag("addressindex", fAddressIndex); + LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + + // Check whether we have a timestamp index + m_block_tree_db->ReadFlag("timestampindex", fTimestampIndex); + LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + + // Check whether we have a spent index + m_block_tree_db->ReadFlag("spentindex", fSpentIndex); + LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); + + return true; +} + +const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) +{ + const MapCheckpoints& checkpoints = data.mapCheckpoints; + + for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) { + const uint256& hash = i.second; + const CBlockIndex* pindex = LookupBlockIndex(hash); + if (pindex) { + return pindex; + } + } + return nullptr; +} + +bool IsBlockPruned(const CBlockIndex* pblockindex) +{ + AssertLockHeld(::cs_main); + return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); +} + +// If we're using -prune with -reindex, then delete block files that will be ignored by the +// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile +// is missing, do the same here to delete any later block files after a gap. Also delete all +// rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info +// is in sync with what's actually on disk by the time we start downloading, so that pruning +// works correctly. +void CleanupBlockRevFiles() +{ + std::map mapBlockFiles; + + // Glob all blk?????.dat and rev?????.dat files from the blocks directory. + // Remove the rev files immediately and insert the blk file paths into an + // ordered map keyed by block file index. + LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); + fs::path blocksdir = gArgs.GetBlocksDirPath(); + for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { + if (fs::is_regular_file(*it) && + it->path().filename().string().length() == 12 && + it->path().filename().string().substr(8,4) == ".dat") + { + if (it->path().filename().string().substr(0, 3) == "blk") { + mapBlockFiles[it->path().filename().string().substr(3, 5)] = it->path(); + } else if (it->path().filename().string().substr(0, 3) == "rev") { + remove(it->path()); + } + } + } + + // Remove all block files that aren't part of a contiguous set starting at + // zero by walking the ordered map (keys are block file indices) by + // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) + // start removing block files. + int nContigCounter = 0; + for (const std::pair& item : mapBlockFiles) { + if (LocaleIndependentAtoi(item.first) == nContigCounter) { + nContigCounter++; + continue; + } + remove(item.second); + } +} + +std::string CBlockFileInfo::ToString() const +{ + return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); +} + +CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n) +{ + LOCK(cs_LastBlockFile); + + return &m_blockfile_info.at(n); +} + +static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +{ + // Open history file to append + CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) { + return error("%s: OpenUndoFile failed", __func__); + } + + // Write index header + unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); + fileout << messageStart << nSize; + + // Write undo data + long fileOutPos = ftell(fileout.Get()); + if (fileOutPos < 0) { + return error("%s: ftell failed", __func__); + } + pos.nPos = (unsigned int)fileOutPos; + fileout << blockundo; + + // calculate & write checksum + CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + hasher << hashBlock; + hasher << blockundo; + fileout << hasher.GetHash(); + + return true; +} + +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) +{ + const FlatFilePos pos{WITH_LOCK(::cs_main, return pindex->GetUndoPos())}; + + if (pos.IsNull()) { + return error("%s: no undo data available", __func__); + } + + // Open history file to read + CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) { + return error("%s: OpenUndoFile failed", __func__); + } + + // Read block + uint256 hashChecksum; + CHashVerifier verifier(&filein); // We need a CHashVerifier as reserializing may lose data + try { + verifier << pindex->pprev->GetBlockHash(); + verifier >> blockundo; + filein >> hashChecksum; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + + // Verify checksum + if (hashChecksum != verifier.GetHash()) { + return error("%s: Checksum mismatch", __func__); + } + + return true; +} + +void BlockManager::FlushUndoFile(int block_file, bool finalize) +{ + FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize); + if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { + AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); + } +} + +void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo) +{ + LOCK(cs_LastBlockFile); + FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize); + if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { + AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); + } + // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, + // e.g. during IBD or a sync after a node going offline + if (!fFinalize || finalize_undo) FlushUndoFile(m_last_blockfile, finalize_undo); +} + +uint64_t BlockManager::CalculateCurrentUsage() +{ + LOCK(cs_LastBlockFile); + + uint64_t retval = 0; + for (const CBlockFileInfo& file : m_blockfile_info) { + retval += file.nSize + file.nUndoSize; + } + return retval; +} + +void UnlinkPrunedFiles(const std::set& setFilesToPrune) +{ + for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { + FlatFilePos pos(*it, 0); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); + LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + } +} + +static FlatFileSeq BlockFileSeq() +{ + return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); +} + +static FlatFileSeq UndoFileSeq() +{ + return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); +} + +FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) +{ + return BlockFileSeq().Open(pos, fReadOnly); +} + +/** Open an undo file (rev?????.dat) */ +static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) +{ + return UndoFileSeq().Open(pos, fReadOnly); +} + +fs::path GetBlockPosFilename(const FlatFilePos& pos) +{ + return BlockFileSeq().FileName(pos); +} + +// TODO move to blockstorage +bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown) +{ + LOCK(cs_LastBlockFile); + + unsigned int nFile = fKnown ? pos.nFile : m_last_blockfile; + if (m_blockfile_info.size() <= nFile) { + m_blockfile_info.resize(nFile + 1); + } + + bool finalize_undo = false; + if (!fKnown) { + while (m_blockfile_info[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { + // when the undo file is keeping up with the block file, we want to flush it explicitly + // when it is lagging behind (more blocks arrive than are being connected), we let the + // undo block write case handle it + finalize_undo = (m_blockfile_info[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); + nFile++; + if (m_blockfile_info.size() <= nFile) { + m_blockfile_info.resize(nFile + 1); + } + } + pos.nFile = nFile; + pos.nPos = m_blockfile_info[nFile].nSize; + } + + if ((int)nFile != m_last_blockfile) { + if (!fKnown) { + LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", m_last_blockfile, m_blockfile_info[m_last_blockfile].ToString()); + } + FlushBlockFile(!fKnown, finalize_undo); + m_last_blockfile = nFile; + } + + m_blockfile_info[nFile].AddBlock(nHeight, nTime); + if (fKnown) { + m_blockfile_info[nFile].nSize = std::max(pos.nPos + nAddSize, m_blockfile_info[nFile].nSize); + } else { + m_blockfile_info[nFile].nSize += nAddSize; + } + + if (!fKnown) { + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is too low!", _("Disk space is too low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + m_check_for_pruning = true; + } + } + + m_dirty_fileinfo.insert(nFile); + return true; +} + +bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) +{ + pos.nFile = nFile; + + LOCK(cs_LastBlockFile); + + pos.nPos = m_blockfile_info[nFile].nUndoSize; + m_blockfile_info[nFile].nUndoSize += nAddSize; + m_dirty_fileinfo.insert(nFile); + + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + m_check_for_pruning = true; + } + + return true; +} static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { @@ -45,6 +717,36 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa return true; } +bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +{ + AssertLockHeld(::cs_main); + // Write undo information to disk + if (pindex->GetUndoPos().IsNull()) { + FlatFilePos _pos; + if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { + return error("ConnectBlock(): FindUndoPos failed"); + } + if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) { + return AbortNode(state, "Failed to write undo data"); + } + // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) + // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height + // in the block file info as below; note that this does not catch the case where the undo writes are keeping up + // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in + // the FindBlockPos function + if (_pos.nFile < m_last_blockfile && static_cast(pindex->nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { + FlushUndoFile(_pos.nFile, true); + } + + // update nUndoPos in block index + pindex->nUndoPos = _pos.nPos; + pindex->nStatus |= BLOCK_HAVE_UNDO; + m_dirty_blockindex.insert(pindex); + } + + return true; +} + bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); @@ -85,7 +787,7 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) +FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; @@ -147,7 +849,7 @@ void ThreadImport(ChainstateManager& chainman, CDeterministicMNManager& dmnman, } nFile++; } - pblocktree->WriteReindexing(false); + WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false)); fReindex = false; LogPrintf("Reindexing finished\n"); // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked): diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index bcb07175e8c39..897306c3e75ff 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -5,28 +5,212 @@ #ifndef BITCOIN_NODE_BLOCKSTORAGE_H #define BITCOIN_NODE_BLOCKSTORAGE_H +#include #include #include // For CMessageHeader::MessageStartChars +#include +#include #include #include +extern RecursiveMutex cs_main; + class CActiveMasternodeManager; class ArgsManager; +class BlockValidationState; class CBlock; -class CBlockIndex; +class CBlockFileInfo; class CBlockUndo; class CChain; class CChainParams; +class CChainState; class CDeterministicMNManager; class CDSNotificationInterface; class ChainstateManager; +struct CCheckpointData; struct FlatFilePos; namespace Consensus { struct Params; } +static constexpr bool DEFAULT_ADDRESSINDEX{false}; +static constexpr bool DEFAULT_SPENTINDEX{false}; static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; +static constexpr bool DEFAULT_TIMESTAMPINDEX{false}; + +/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ +static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ +static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** The maximum size of a blk?????.dat file (since 0.8) */ +static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB + +extern std::atomic_bool fImporting; +extern std::atomic_bool fReindex; +/** Pruning-related variables and constants */ +/** True if any block files have ever been pruned. */ +extern bool fHavePruned; +/** True if we're running in -prune mode. */ +extern bool fPruneMode; +/** Number of MiB of block files that we're trying to stay below. */ +extern uint64_t nPruneTarget; + +/** True if we're running in -addressindex mode. */ +extern bool fAddressIndex; +/** True if we're running in -timestampindex mode. */ +extern bool fTimestampIndex; +/** True if we're running in -spentindex mode. */ +extern bool fSpentIndex; + +// Because validation code takes pointers to the map's CBlockIndex objects, if +// we ever switch to another associative container, we need to either use a +// container that has stable addressing (true of all std associative +// containers), or make the key a `std::unique_ptr` +using BlockMap = std::unordered_map; +using PrevBlockMap = std::unordered_multimap; + +struct CBlockIndexWorkComparator { + bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const; +}; + +struct CBlockIndexHeightOnlyComparator { + /* Only compares the height of two block indices, doesn't try to tie-break */ + bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const; +}; + +/** + * Maintains a tree of blocks (stored in `m_block_index`) which is consulted + * to determine where the most-work tip is. + * + * This data is used mostly in `CChainState` - information about, e.g., + * candidate tips is not maintained here. + */ +class BlockManager +{ + friend CChainState; + friend ChainstateManager; + +private: + void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); + void FlushUndoFile(int block_file, bool finalize = false); + bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown); + bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); + + /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ + void FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight, int chain_tip_height); + + /** + * Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a user-defined target. + * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new + * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex + * (which in this case means the blockchain must be re-downloaded.) + * + * Pruning functions are called from FlushStateToDisk when the m_check_for_pruning flag has been set. + * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.) + * Pruning cannot take place until the longest chain is at least a certain length (CChainParams::nPruneAfterHeight). + * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip. + * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files. + * A db flag records the fact that at least some block files have been pruned. + * + * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned + */ + void FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd); + + RecursiveMutex cs_LastBlockFile; + std::vector m_blockfile_info; + int m_last_blockfile = 0; + /** Global flag to indicate we should check to see if there are + * block/undo files that should be deleted. Set on startup + * or if we allocate more file space when we're in prune mode + */ + bool m_check_for_pruning = false; + + /** Dirty block index entries. */ + std::set m_dirty_blockindex; + + /** Dirty block file entries. */ + std::set m_dirty_fileinfo; + +public: + BlockMap m_block_index GUARDED_BY(cs_main); + PrevBlockMap m_prev_block_index GUARDED_BY(cs_main); + + std::vector GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + /** + * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. + * Pruned nodes may have entries where B is missing data. + */ + std::multimap m_blocks_unlinked; + + std::unique_ptr m_block_tree_db GUARDED_BY(::cs_main); + + bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + /** + * Load the blocktree off disk and into memory. Populate certain metadata + * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral + * collections like m_dirty_blockindex. + */ + bool LoadBlockIndex(const Consensus::Params& consensus_params) + EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** Clear all data members. */ + void Unload() EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + CBlockIndex* AddToBlockIndex(const CBlockHeader& block, const uint256& hash, enum BlockStatus nStatus = BLOCK_VALID_TREE) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Create a new block index entry for a given block hash */ + CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Mark one block file as pruned (modify associated database entries) + void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + const CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** Get block file info entry for one block file */ + CBlockFileInfo* GetBlockFileInfo(size_t n); + + bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); + + /** Calculate the amount of disk space the block & undo files currently use */ + uint64_t CalculateCurrentUsage(); + + //! Returns last CBlockIndex* that is a checkpoint + const CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** + * Return the spend height, which is one more than the inputs.GetBestBlock(). + * While checking, GetBestBlock() refers to the parent block. (protected by cs_main) + * This is also true for mempool checks. + */ + int GetSpendHeight(const CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + ~BlockManager() + { + Unload(); + } +}; + +//! Check whether the block associated with this index entry is pruned or not. +bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + +void CleanupBlockRevFiles(); + +/** Open a block file (blk?????.dat) */ +FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false); +/** Translation to a filesystem path */ +fs::path GetBlockPosFilename(const FlatFilePos& pos); + +/** + * Actually unlink the specified files + */ +void UnlinkPrunedFiles(const std::set& setFilesToPrune); /** Functions for disk access for blocks */ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); @@ -34,8 +218,6 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); -FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); - void ThreadImport(ChainstateManager& chainman, CDeterministicMNManager& dmnman, CDSNotificationInterface& dsnfi, std::vector vImportFiles, CActiveMasternodeManager* const mn_activeman, const ArgsManager& args); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 7c51e7255ac23..f89d8c21640bb 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -757,8 +757,8 @@ class ChainImpl : public Chain std::optional findLocatorFork(const CBlockLocator& locator) override { LOCK(cs_main); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); - if (CBlockIndex* fork = m_node.chainman->m_blockman.FindForkInGlobalIndex(active, locator)) { + const CChainState& active = Assert(m_node.chainman)->ActiveChainstate(); + if (const CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { return fork->nHeight; } return std::nullopt; @@ -856,7 +856,7 @@ class ChainImpl : public Chain // used to limit the range, and passing min_height that's too low or // max_height that's too high will not crash or change the result. LOCK(::cs_main); - if (CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { + if (const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { // Check pprev to not segfault if min_height is too low diff --git a/src/rest.cpp b/src/rest.cpp index ad2f6a104912d..65ba16e52ea66 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -271,8 +271,8 @@ static bool rest_block(const CoreContext& context, return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); CBlock block; - CBlockIndex* pblockindex = nullptr; - CBlockIndex* tip = nullptr; + const CBlockIndex* pblockindex = nullptr; + const CBlockIndex* tip = nullptr; { ChainstateManager* maybe_chainman = GetChainman(context, req); if (!maybe_chainman) return false; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7ec30ab5e3ba8..f41c1efab19a3 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -96,7 +96,8 @@ static int ComputeNextBlockAndDepth(const CBlockIndex* tip, const CBlockIndex* b return blockindex == tip ? 1 : -1; } -CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) { +static const CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) +{ LOCK(::cs_main); CChain& active_chain = chainman.ActiveChain(); @@ -113,7 +114,7 @@ CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainma return active_chain[height]; } else { const uint256 hash{ParseHashV(param, "hash_or_height")}; - CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); + const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); @@ -163,7 +164,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn UniValue txs(UniValue::VARR); if (txDetails) { CBlockUndo blockUndo; - const bool have_undo = !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex); + const bool have_undo{WITH_LOCK(::cs_main, return !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex))}; for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); // coinbase transaction (i == 0) doesn't have undo data @@ -804,7 +805,8 @@ static RPCHelpMan getblockfrompeer() UniValue result = UniValue::VOBJ; - if (index->nStatus & BLOCK_HAVE_DATA) { + const bool block_has_data = WITH_LOCK(::cs_main, return index->nStatus & BLOCK_HAVE_DATA); + if (block_has_data) { result.pushKV("warnings", "Block already downloaded"); } else if (const auto err{peerman.FetchBlock(peer_id, *index)}) { throw JSONRPCError(RPC_MISC_ERROR, err.value()); @@ -835,7 +837,8 @@ static RPCHelpMan getblockhashes() unsigned int low = request.params[1].get_int(); std::vector blockHashes; - if (LOCK(::cs_main); !GetTimestampIndex(*pblocktree, high, low, blockHashes)) { + ChainstateManager& chainman = EnsureAnyChainman(request.context); + if (LOCK(::cs_main); !GetTimestampIndex(*chainman.m_blockman.m_block_tree_db, high, low, blockHashes)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for block hashes"); } @@ -872,7 +875,7 @@ static RPCHelpMan getblockhash() if (nHeight < 0 || nHeight > active_chain.Height()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Block height out of range"); - CBlockIndex* pblockindex = active_chain[nHeight]; + const CBlockIndex* pblockindex = active_chain[nHeight]; return pblockindex->GetBlockHash().GetHex(); }, }; @@ -1057,8 +1060,9 @@ static RPCHelpMan getblockheaders() }; } -static CBlock GetBlockChecked(const CBlockIndex* pblockindex) +static CBlock GetBlockChecked(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + AssertLockHeld(::cs_main); CBlock block; if (IsBlockPruned(pblockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); @@ -1074,8 +1078,9 @@ static CBlock GetBlockChecked(const CBlockIndex* pblockindex) return block; } -static CBlockUndo GetUndoChecked(const CBlockIndex* pblockindex) +static CBlockUndo GetUndoChecked(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { + AssertLockHeld(::cs_main); CBlockUndo blockUndo; if (IsBlockPruned(pblockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available (pruned data)"); @@ -1300,7 +1305,7 @@ static RPCHelpMan pruneblockchain() // too low to be a block time (corresponds to timestamp from Sep 2001). if (heightParam > 1000000000) { // Add a 2 hour buffer to include blocks which might have had old timestamps - CBlockIndex* pindex = active_chain.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW, 0); + const CBlockIndex* pindex = active_chain.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW, 0); if (!pindex) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Could not find block with at least the specified timestamp."); } @@ -1394,7 +1399,7 @@ static RPCHelpMan gettxoutsetinfo() { UniValue ret(UniValue::VOBJ); - CBlockIndex* pindex{nullptr}; + const CBlockIndex* pindex{nullptr}; const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; CCoinsStats stats{hash_type}; stats.index_requested = request.params[2].isNull() || request.params[2].get_bool(); @@ -1765,7 +1770,7 @@ RPCHelpMan getblockchaininfo() obj.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), tip)); obj.pushKV("initialblockdownload", active_chainstate.IsInitialBlockDownload()); obj.pushKV("chainwork", tip->nChainWork.GetHex()); - obj.pushKV("size_on_disk", CalculateCurrentUsage()); + obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); obj.pushKV("pruned", fPruneMode); if (fPruneMode) { const CBlockIndex* block = tip; @@ -1876,10 +1881,10 @@ static RPCHelpMan getchaintips() std::set setOrphans; std::set setPrevs; - for (const std::pair& item : chainman.BlockIndex()) { - if (!active_chain.Contains(item.second)) { - setOrphans.insert(item.second); - setPrevs.insert(item.second->pprev); + for (const auto& [_, block_index] : chainman.BlockIndex()) { + if (!active_chain.Contains(&block_index)) { + setOrphans.insert(&block_index); + setPrevs.insert(block_index.pprev); } } @@ -2341,7 +2346,7 @@ static RPCHelpMan getblockstats() ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); - CBlockIndex* pindex{ParseHashOrHeight(request.params[0], chainman)}; + const CBlockIndex* pindex{ParseHashOrHeight(request.params[0], chainman)}; CHECK_NONFATAL(pindex != nullptr); std::set stats; @@ -2804,7 +2809,7 @@ static RPCHelpMan scantxoutset() g_should_abort_scan = false; int64_t count = 0; std::unique_ptr pcursor; - CBlockIndex* tip; + const CBlockIndex* tip; NodeContext& node = EnsureAnyNodeContext(request.context); { ChainstateManager& chainman = EnsureChainman(node); @@ -2984,7 +2989,7 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil { std::unique_ptr pcursor; CCoinsStats stats{CoinStatsHashType::NONE}; - CBlockIndex* tip; + const CBlockIndex* tip; { // We need to lock cs_main to ensure that the coinsdb isn't written to diff --git a/src/rpc/evo.cpp b/src/rpc/evo.cpp index 6b82ac692be9f..9e93370a48553 100644 --- a/src/rpc/evo.cpp +++ b/src/rpc/evo.cpp @@ -1556,7 +1556,7 @@ static RPCHelpMan protx_info() g_txindex->BlockUntilSyncedToCurrentChain(); } - CBlockIndex* pindex{nullptr}; + const CBlockIndex* pindex{nullptr}; uint256 proTxHash(ParseHashV(request.params[0], "proTxHash")); diff --git a/src/rpc/index_util.cpp b/src/rpc/index_util.cpp index 4e362af88cd9a..113071cb8af93 100644 --- a/src/rpc/index_util.cpp +++ b/src/rpc/index_util.cpp @@ -5,9 +5,10 @@ #include +#include +#include #include #include -#include bool GetAddressIndex(CBlockTreeDB& block_tree_db, const uint160& addressHash, const AddressType type, std::vector& addressIndex, diff --git a/src/rpc/masternode.cpp b/src/rpc/masternode.cpp index 9a1bfddacef2a..c2a23039943ab 100644 --- a/src/rpc/masternode.cpp +++ b/src/rpc/masternode.cpp @@ -370,7 +370,7 @@ static RPCHelpMan masternode_payments() const NodeContext& node = EnsureAnyNodeContext(request.context); const ChainstateManager& chainman = EnsureChainman(node); - CBlockIndex* pindex{nullptr}; + const CBlockIndex* pindex{nullptr}; if (g_txindex) { g_txindex->BlockUntilSyncedToCurrentChain(); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 9706c94bda711..55e3f6682358a 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -813,10 +813,11 @@ static RPCHelpMan getaddressutxos() std::vector unspentOutputs; + ChainstateManager& chainman = EnsureAnyChainman(request.context); { LOCK(::cs_main); for (const auto& address : addresses) { - if (!GetAddressUnspentIndex(*pblocktree, address.first, address.second, unspentOutputs, + if (!GetAddressUnspentIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, unspentOutputs, /* height_sort = */ true)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } @@ -900,15 +901,20 @@ static RPCHelpMan getaddressdeltas() std::vector addressIndex; + ChainstateManager& chainman = EnsureAnyChainman(request.context); { LOCK(::cs_main); for (const auto& address : addresses) { if (start > 0 && end > 0) { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex, start, end)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, + addressIndex, start, end)) + { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } else { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, + addressIndex)) + { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } @@ -978,7 +984,7 @@ static RPCHelpMan getaddressbalance() { LOCK(::cs_main); for (const auto& address : addresses) { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, addressIndex)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } @@ -1056,15 +1062,18 @@ static RPCHelpMan getaddresstxids() std::vector addressIndex; + ChainstateManager& chainman = EnsureAnyChainman(request.context); { LOCK(::cs_main); for (const auto& address : addresses) { if (start > 0 && end > 0) { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex, start, end)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, + addressIndex, start, end)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } else { - if (!GetAddressIndex(*pblocktree, address.first, address.second, addressIndex)) { + if (!GetAddressIndex(*chainman.m_blockman.m_block_tree_db, address.first, address.second, + addressIndex)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } } @@ -1137,8 +1146,9 @@ static RPCHelpMan getspentinfo() CSpentIndexKey key(txid, outputIndex); CSpentIndexValue value; + ChainstateManager& chainman = EnsureAnyChainman(request.context); CTxMemPool& mempool = EnsureAnyMemPool(request.context); - if (LOCK(::cs_main); !GetSpentIndex(*pblocktree, mempool, key, value)) { + if (LOCK(::cs_main); !GetSpentIndex(*chainman.m_blockman.m_block_tree_db, mempool, key, value)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to get spent info"); } diff --git a/src/rpc/quorums.cpp b/src/rpc/quorums.cpp index 8994e54c777f4..0ed8b05b795e2 100644 --- a/src/rpc/quorums.cpp +++ b/src/rpc/quorums.cpp @@ -966,7 +966,7 @@ static RPCHelpMan verifychainlock() const ChainstateManager& chainman = EnsureChainman(node); int nBlockHeight; - CBlockIndex* pIndex{nullptr}; + const CBlockIndex* pIndex{nullptr}; if (request.params[2].isNull()) { pIndex = WITH_LOCK(cs_main, return chainman.m_blockman.LookupBlockIndex(nBlockHash)); if (pIndex == nullptr) { diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 6c7111ccd5da9..3164e7884a1a0 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -75,7 +75,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, CTxMemPool& mempo if (!tx.IsCoinBase()) { CSpentIndexValue spentInfo; CSpentIndexKey spentKey(txin.prevout.hash, txin.prevout.n); - if (GetSpentIndex(*pblocktree, mempool, spentKey, spentInfo)) { + if (GetSpentIndex(*active_chainstate.m_blockman.m_block_tree_db, mempool, spentKey, spentInfo)) { txSpentInfo.mSpentInfo.emplace(spentKey, spentInfo); } } @@ -83,7 +83,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, CTxMemPool& mempo for (unsigned int i = 0; i < tx.vout.size(); i++) { CSpentIndexValue spentInfo; CSpentIndexKey spentKey(txid, i); - if (GetSpentIndex(*pblocktree, mempool, spentKey, spentInfo)) { + if (GetSpentIndex(*active_chainstate.m_blockman.m_block_tree_db, mempool, spentKey, spentInfo)) { txSpentInfo.mSpentInfo.emplace(spentKey, spentInfo); } } @@ -93,7 +93,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, CTxMemPool& mempo bool chainLock = false; if (!hashBlock.IsNull()) { entry.pushKV("blockhash", hashBlock.GetHex()); - CBlockIndex* pindex = active_chainstate.m_blockman.LookupBlockIndex(hashBlock); + const CBlockIndex* pindex = active_chainstate.m_blockman.LookupBlockIndex(hashBlock); if (pindex) { if (active_chainstate.m_chain.Contains(pindex)) { entry.pushKV("height", pindex->nHeight); @@ -208,7 +208,7 @@ static RPCHelpMan getrawtransaction() bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); - CBlockIndex* blockindex = nullptr; + const CBlockIndex* blockindex = nullptr; if (hash == Params().GenesisBlock().hashMerkleRoot) { // Special exception for the genesis block coinbase transaction @@ -242,7 +242,8 @@ static RPCHelpMan getrawtransaction() if (!tx) { std::string errmsg; if (blockindex) { - if (!(blockindex->nStatus & BLOCK_HAVE_DATA)) { + const bool block_has_data = WITH_LOCK(::cs_main, return blockindex->nStatus & BLOCK_HAVE_DATA); + if (!block_has_data) { throw JSONRPCError(RPC_MISC_ERROR, "Block not available"); } errmsg = "No such transaction found in the provided block"; @@ -323,7 +324,7 @@ static RPCHelpMan getrawtransactionmulti() { const uint256 blockhash{uint256S(blockhash_str)}; const UniValue txids = transactions[blockhash_str].get_array(); - CBlockIndex* blockindex{blockhash.IsNull() ? nullptr : WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(blockhash))}; + const CBlockIndex* blockindex{blockhash.IsNull() ? nullptr : WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(blockhash))}; if (blockindex == nullptr && !blockhash.IsNull()) { for (const auto idx : irange::range(txids.size())) { result.pushKV(txids[idx].get_str(), "None"); @@ -609,7 +610,7 @@ static RPCHelpMan gettxoutproof() } } - CBlockIndex* pblockindex = nullptr; + const CBlockIndex* pblockindex = nullptr; uint256 hashBlock; ChainstateManager& chainman = EnsureAnyChainman(request.context); if (!request.params[1].isNull()) { diff --git a/src/test/evo_deterministicmns_tests.cpp b/src/test/evo_deterministicmns_tests.cpp index 48b73ebf99abc..caedcca2d9295 100644 --- a/src/test/evo_deterministicmns_tests.cpp +++ b/src/test/evo_deterministicmns_tests.cpp @@ -251,7 +251,7 @@ void FuncDIP3Activation(TestChainSetup& setup) int nHeight = chainman.ActiveChain().Height(); // We start one block before DIP3 activation, so mining a block with a DIP3 transaction should fail - auto block = std::make_shared(setup.CreateBlock(txns, setup.coinbaseKey)); + auto block = std::make_shared(setup.CreateBlock(txns, setup.coinbaseKey, chainman.ActiveChainstate())); chainman.ProcessNewBlock(Params(), block, true, nullptr); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight); BOOST_ASSERT(block->GetHash() != chainman.ActiveChain().Tip()->GetBlockHash()); @@ -261,7 +261,7 @@ void FuncDIP3Activation(TestChainSetup& setup) setup.CreateAndProcessBlock({}, setup.coinbaseKey); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 1); // Mining a block with a DIP3 transaction should succeed now - block = std::make_shared(setup.CreateBlock(txns, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock(txns, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 2); @@ -288,7 +288,7 @@ void FuncV19Activation(TestChainSetup& setup) int nHeight = chainman.ActiveChain().Height(); - auto block = std::make_shared(setup.CreateBlock({tx_reg}, setup.coinbaseKey)); + auto block = std::make_shared(setup.CreateBlock({tx_reg}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); BOOST_ASSERT(!DeploymentActiveAfter(chainman.ActiveChain().Tip(), Params().GetConsensus(), Consensus::DEPLOYMENT_V19)); ++nHeight; @@ -306,7 +306,7 @@ void FuncV19Activation(TestChainSetup& setup) operator_key_new.MakeNewKey(); auto tx_upreg = CreateProUpRegTx(chainman.ActiveChain(), *(setup.m_node.mempool), utxos, tx_reg_hash, owner_key, operator_key_new.GetPublicKey(), owner_key.GetPubKey().GetID(), collateralScript, setup.coinbaseKey); - block = std::make_shared(setup.CreateBlock({tx_upreg}, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock({tx_upreg}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); BOOST_ASSERT(!DeploymentActiveAfter(chainman.ActiveChain().Tip(), Params().GetConsensus(), Consensus::DEPLOYMENT_V19)); ++nHeight; @@ -326,7 +326,7 @@ void FuncV19Activation(TestChainSetup& setup) FillableSigningProvider signing_provider; signing_provider.AddKeyPubKey(collateral_key, collateral_key.GetPubKey()); BOOST_ASSERT(SignSignature(signing_provider, CTransaction(tx_reg), tx_spend, 0, SIGHASH_ALL)); - block = std::make_shared(setup.CreateBlock({tx_spend}, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock({tx_spend}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); BOOST_ASSERT(!DeploymentActiveAfter(chainman.ActiveChain().Tip(), Params().GetConsensus(), Consensus::DEPLOYMENT_V19)); ++nHeight; @@ -614,7 +614,7 @@ void FuncTestMempoolReorg(TestChainSetup& setup) FundTransaction(chainman.ActiveChain(), tx_collateral, utxos, scriptCollateral, dmn_types::Regular.collat_amount, setup.coinbaseKey); SignTransaction(*(setup.m_node.mempool), tx_collateral, setup.coinbaseKey); - auto block = std::make_shared(setup.CreateBlock({tx_collateral}, setup.coinbaseKey)); + auto block = std::make_shared(setup.CreateBlock({tx_collateral}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); setup.m_node.dmnman->UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 1); @@ -756,7 +756,7 @@ void FuncVerifyDB(TestChainSetup& setup) FundTransaction(chainman.ActiveChain(), tx_collateral, utxos, scriptCollateral, dmn_types::Regular.collat_amount, setup.coinbaseKey); SignTransaction(*(setup.m_node.mempool), tx_collateral, setup.coinbaseKey); - auto block = std::make_shared(setup.CreateBlock({tx_collateral}, setup.coinbaseKey)); + auto block = std::make_shared(setup.CreateBlock({tx_collateral}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 1); @@ -788,7 +788,7 @@ void FuncVerifyDB(TestChainSetup& setup) auto tx_reg_hash = tx_reg.GetHash(); - block = std::make_shared(setup.CreateBlock({tx_reg}, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock({tx_reg}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 2); @@ -800,7 +800,7 @@ void FuncVerifyDB(TestChainSetup& setup) collateral_utxos.emplace(payload.collateralOutpoint, std::make_pair(1, 1000)); auto proUpRevTx = CreateProUpRevTx(chainman.ActiveChain(), *(setup.m_node.mempool), collateral_utxos, tx_reg_hash, operatorKey, collateralKey); - block = std::make_shared(setup.CreateBlock({proUpRevTx}, setup.coinbaseKey)); + block = std::make_shared(setup.CreateBlock({proUpRevTx}, setup.coinbaseKey, chainman.ActiveChainstate())); BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr)); dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip()); BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 3); diff --git a/src/test/fuzz/chain.cpp b/src/test/fuzz/chain.cpp index 79e0501e30768..1e8d71adeaba2 100644 --- a/src/test/fuzz/chain.cpp +++ b/src/test/fuzz/chain.cpp @@ -21,15 +21,18 @@ FUZZ_TARGET(chain) const uint256 zero{}; disk_block_index->phashBlock = &zero; - (void)disk_block_index->GetBlockHash(); - (void)disk_block_index->GetBlockPos(); - (void)disk_block_index->GetBlockTime(); - (void)disk_block_index->GetBlockTimeMax(); - (void)disk_block_index->GetMedianTimePast(); - (void)disk_block_index->GetUndoPos(); - (void)disk_block_index->HaveTxsDownloaded(); - (void)disk_block_index->IsValid(); - (void)disk_block_index->ToString(); + { + LOCK(::cs_main); + (void)disk_block_index->GetBlockHash(); + (void)disk_block_index->GetBlockPos(); + (void)disk_block_index->GetBlockTime(); + (void)disk_block_index->GetBlockTimeMax(); + (void)disk_block_index->GetMedianTimePast(); + (void)disk_block_index->GetUndoPos(); + (void)disk_block_index->HaveTxsDownloaded(); + (void)disk_block_index->IsValid(); + (void)disk_block_index->ToString(); + } const CBlockHeader block_header = disk_block_index->GetBlockHeader(); (void)CDiskBlockIndex{*disk_block_index}; @@ -55,7 +58,7 @@ FUZZ_TARGET(chain) if (block_status & ~BLOCK_VALID_MASK) { continue; } - (void)disk_block_index->RaiseValidity(block_status); + WITH_LOCK(::cs_main, (void)disk_block_index->RaiseValidity(block_status)); } CBlockIndex block_index{block_header}; diff --git a/src/test/interfaces_tests.cpp b/src/test/interfaces_tests.cpp index 44779f7d7cef5..5fd6041fdcf1b 100644 --- a/src/test/interfaces_tests.cpp +++ b/src/test/interfaces_tests.cpp @@ -123,6 +123,7 @@ BOOST_AUTO_TEST_CASE(findCommonAncestor) BOOST_AUTO_TEST_CASE(hasBlocks) { + LOCK(::cs_main); auto& chain = m_node.chain; const CChain& active = Assert(m_node.chainman)->ActiveChain(); diff --git a/src/test/util/blockfilter.cpp b/src/test/util/blockfilter.cpp index d1f0cad7cc8e1..2df0ffa37f14b 100644 --- a/src/test/util/blockfilter.cpp +++ b/src/test/util/blockfilter.cpp @@ -10,6 +10,8 @@ bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter) { + LOCK(::cs_main); + CBlock block; if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { return false; diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h new file mode 100644 index 0000000000000..81ea4c38f5744 --- /dev/null +++ b/src/test/util/chainstate.h @@ -0,0 +1,54 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#ifndef BITCOIN_TEST_UTIL_CHAINSTATE_H +#define BITCOIN_TEST_UTIL_CHAINSTATE_H + +#include +#include +#include +#include +#include +#include + +#include + +#include + +const auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; + +/** + * Create and activate a UTXO snapshot, optionally providing a function to + * malleate the snapshot. + */ +template +static bool +CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) +{ + // Write out a snapshot to the test's tempdir. + // + int height; + WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); + fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); + FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; + CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; + + UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); + BOOST_TEST_MESSAGE( + "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); + + // Read the written snapshot in and then activate it. + // + FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; + CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; + SnapshotMetadata metadata; + auto_infile >> metadata; + + malleation(auto_infile, metadata); + + return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); +} + + +#endif // BITCOIN_TEST_UTIL_CHAINSTATE_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index f0846d95378cb..670530feffd0f 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -220,12 +220,11 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve m_node.scheduler->m_service_thread = std::thread(util::TraceThread, "scheduler", [&] { m_node.scheduler->serviceQueue(); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); - pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - m_node.fee_estimator = std::make_unique(); m_node.mempool = std::make_unique(m_node.fee_estimator.get(), 1); m_node.chainman = std::make_unique(); + m_node.chainman->m_blockman.m_block_tree_db = std::make_unique(1 << 20, true); m_node.connman = std::make_unique(0x1337, 0x1337, *m_node.addrman); // Deterministic randomness for tests. @@ -260,7 +259,6 @@ ChainTestingSetup::~ChainTestingSetup() m_node.scheduler.reset(); m_node.chainman->Reset(); m_node.chainman.reset(); - pblocktree.reset(); } TestingSetup::TestingSetup(const std::string& chainName, const std::vector& extra_args) @@ -371,10 +369,17 @@ void TestChainSetup::mineBlocks(int num_blocks) } } -CBlock TestChainSetup::CreateAndProcessBlock(const std::vector& txns, const CScript& scriptPubKey) +CBlock TestChainSetup::CreateAndProcessBlock( + const std::vector& txns, + const CScript& scriptPubKey, + CChainState* chainstate) { + if (!chainstate) { + chainstate = &Assert(m_node.chainman)->ActiveChainstate(); + } + const CChainParams& chainparams = Params(); - auto block = CreateBlock(txns, scriptPubKey); + auto block = this->CreateBlock(txns, scriptPubKey, *chainstate); std::shared_ptr shared_pblock = std::make_shared(block); Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr); @@ -382,17 +387,23 @@ CBlock TestChainSetup::CreateAndProcessBlock(const std::vector& txns, const CKey& scriptKey) +CBlock TestChainSetup::CreateAndProcessBlock( + const std::vector& txns, + const CKey& scriptKey, + CChainState* chainstate) { CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - return CreateAndProcessBlock(txns, scriptPubKey); + return CreateAndProcessBlock(txns, scriptPubKey, chainstate); } -CBlock TestChainSetup::CreateBlock(const std::vector& txns, const CScript& scriptPubKey) +CBlock TestChainSetup::CreateBlock( + const std::vector& txns, + const CScript& scriptPubKey, + CChainState& chainstate) { const CChainParams& chainparams = Params(); CTxMemPool empty_pool; - CBlock block = BlockAssembler(m_node.chainman->ActiveChainstate(), m_node, empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; + CBlock block = BlockAssembler(chainstate, m_node, empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; std::vector llmqCommitments; for (const auto& tx : block.vtx) { @@ -441,10 +452,13 @@ CBlock TestChainSetup::CreateBlock(const std::vector& txns, return result; } -CBlock TestChainSetup::CreateBlock(const std::vector& txns, const CKey& scriptKey) +CBlock TestChainSetup::CreateBlock( + const std::vector& txns, + const CKey& scriptKey, + CChainState& chainstate) { CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - return CreateBlock(txns, scriptPubKey); + return CreateBlock(txns, scriptPubKey, chainstate); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index d3795b5b865a5..7b623355bf2ca 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -127,15 +127,25 @@ struct TestChainSetup : public RegTestingSetup /** * Create a new block with just given transactions, coinbase paying to * scriptPubKey, and try to add it to the current chain. + * If no chainstate is specified, default to the active. */ CBlock CreateAndProcessBlock(const std::vector& txns, - const CScript& scriptPubKey); + const CScript& scriptPubKey, + CChainState* chainstate = nullptr); CBlock CreateAndProcessBlock(const std::vector& txns, - const CKey& scriptKey); + const CKey& scriptKey, + CChainState* chainstate = nullptr); + + /** + * Create a new block with just given transactions, coinbase paying to + * scriptPubKey. + */ CBlock CreateBlock(const std::vector& txns, - const CScript& scriptPubKey); + const CScript& scriptPubKey, + CChainState& chainstate); CBlock CreateBlock(const std::vector& txns, - const CKey& scriptKey); + const CKey& scriptKey, + CChainState& chainstate); //! Mine a series of new blocks on the active chain. void mineBlocks(int num_blocks); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index f2fbaf72618a4..7dccfe902799c 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // +#include #include #include #include @@ -9,6 +10,8 @@ #include #include #include +#include +#include #include #include #include @@ -24,6 +27,7 @@ BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) { ChainstateManager manager; + WITH_LOCK(::cs_main, manager.m_blockman.m_block_tree_db = std::make_unique(1 << 20, true)); CTxMemPool mempool; //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given view. @@ -43,6 +47,7 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); + BOOST_REQUIRE(c1.LoadGenesisBlock()); // Need at least one block loaded to be able to flush caches // Add a coin to the in-memory cache, upsize once, then downsize. { @@ -71,9 +76,79 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) // The view cache should be empty since we had to destruct to downsize. BOOST_CHECK(!c1.CoinsTip().HaveCoinInCache(outpoint)); } +} + +//! Test UpdateTip behavior for both active and background chainstates. +//! +//! When run on the background chainstate, UpdateTip should do a subset +//! of what it does for the active chainstate. +BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) +{ + ChainstateManager& chainman = *Assert(m_node.chainman); + uint256 curr_tip = ::g_best_block; + + // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can + // be found. + mineBlocks(10); + + // After adding some blocks to the tip, best block should have changed. + BOOST_CHECK(::g_best_block != curr_tip); + + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + + // Ensure our active chain is the snapshot chainstate. + BOOST_CHECK(chainman.IsSnapshotActive()); + + curr_tip = ::g_best_block; + + // Mine a new block on top of the activated snapshot chainstate. + mineBlocks(1); // Defined in TestChain100Setup. + + // After adding some blocks to the snapshot tip, best block should have changed. + BOOST_CHECK(::g_best_block != curr_tip); + + curr_tip = ::g_best_block; + + CChainState* background_cs; + + BOOST_CHECK_EQUAL(chainman.GetAll().size(), 2); + for (CChainState* cs : chainman.GetAll()) { + if (cs != &chainman.ActiveChainstate()) { + background_cs = cs; + } + } + BOOST_CHECK(background_cs); + + // Create a block to append to the validation chain. + std::vector noTxns; + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, *background_cs); + auto pblock = std::make_shared(validation_block); + BlockValidationState state; + CBlockIndex* pindex = nullptr; + const CChainParams& chainparams = Params(); + bool newblock = false; + + // TODO: much of this is inlined from ProcessNewBlock(); just reuse PNB() + // once it is changed to support multiple chainstates. + { + LOCK(::cs_main); + bool checked = CheckBlock(*pblock, state, chainparams.GetConsensus()); + BOOST_CHECK(checked); + bool accepted = background_cs->AcceptBlock( + pblock, state, &pindex, true, nullptr, &newblock); + BOOST_CHECK(accepted); + } + // UpdateTip is called here + bool block_added = background_cs->ActivateBestChain(state, pblock); + + // Ensure tip is as expected + BOOST_CHECK_EQUAL(background_cs->m_chain.Tip()->GetBlockHash(), validation_block.GetHash()); - // Avoid triggering the address sanitizer. - WITH_LOCK(::cs_main, manager.Unload()); + // g_best_block should be unchanged after adding a block to the background + // validation chain. + BOOST_CHECK(block_added); + BOOST_CHECK_EQUAL(curr_tip, ::g_best_block); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index e4d6c5d62d97b..d3374dcdc636a 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -14,13 +14,13 @@ #include #include #include +#include #include #include #include #include #include -#include #include @@ -53,7 +53,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); - BOOST_CHECK(!manager.IsBackgroundIBD(&c1)); auto all = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); @@ -66,9 +65,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto exp_tip = c1.m_chain.Tip(); BOOST_CHECK_EQUAL(active_tip, exp_tip); - auto& validated_cs = manager.ValidatedChainstate(); - BOOST_CHECK_EQUAL(&validated_cs, &c1); - BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); DashTestSetupClose(m_node); @@ -96,8 +92,8 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK(manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); - BOOST_CHECK(manager.IsBackgroundIBD(&c1)); - BOOST_CHECK(!manager.IsBackgroundIBD(&c2)); + BOOST_CHECK_EQUAL(&c2, &manager.ActiveChainstate()); + BOOST_CHECK(&c1 != &manager.ActiveChainstate()); auto all2 = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all2.begin(), all2.end(), chainstates.begin(), chainstates.end()); @@ -114,22 +110,10 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // CCoinsViewCache instances. BOOST_CHECK(exp_tip != exp_tip2); - auto& validated_cs2 = manager.ValidatedChainstate(); - BOOST_CHECK_EQUAL(&validated_cs2, &c1); - - auto& validated_chain = manager.ValidatedChain(); - BOOST_CHECK_EQUAL(&validated_chain, &c1.m_chain); - - auto validated_tip = manager.ValidatedTip(); - exp_tip = c1.m_chain.Tip(); - BOOST_CHECK_EQUAL(validated_tip, exp_tip); - // Let scheduler events finish running to avoid accessing memory that is going to be unloaded SyncWithValidationInterfaceQueue(); DashTestSetupClose(m_node); - - WITH_LOCK(::cs_main, manager.Unload()); } //! Test rebalancing the caches associated with each chainstate. @@ -185,36 +169,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } -auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; - -template -static bool -CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) -{ - // Write out a snapshot to the test's tempdir. - // - int height; - WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); - fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); - FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; - CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; - - UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); - BOOST_TEST_MESSAGE( - "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); - - // Read the written snapshot in and then activate it. - // - FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; - CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; - SnapshotMetadata metadata; - auto_infile >> metadata; - - malleation(auto_infile, metadata); - - return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); -} - //! Test basic snapshot activation. BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) { @@ -293,6 +247,9 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) *chainman.ActiveChainstate().m_from_snapshot_blockhash, *chainman.SnapshotBlockhash()); + // Ensure that the genesis block was not marked assumed-valid. + BOOST_CHECK(WITH_LOCK(::cs_main, return !chainman.ActiveChain().Genesis()->IsAssumedValid())); + const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); const CBlockIndex* tip = chainman.ActiveTip(); @@ -338,27 +295,27 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) { LOCK(::cs_main); size_t coins_in_active{0}; - size_t coins_in_ibd{0}; - size_t coins_missing_ibd{0}; + size_t coins_in_background{0}; + size_t coins_missing_from_background{0}; for (CChainState* chainstate : chainman.GetAll()) { BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); CCoinsViewCache& coinscache = chainstate->CoinsTip(); - bool is_ibd = chainman.IsBackgroundIBD(chainstate); + bool is_background = chainstate != &chainman.ActiveChainstate(); for (CTransactionRef& txn : m_coinbase_txns) { COutPoint op{txn->GetHash(), 0}; if (coinscache.HaveCoin(op)) { - (is_ibd ? coins_in_ibd : coins_in_active)++; - } else if (is_ibd) { - coins_missing_ibd++; + (is_background ? coins_in_background : coins_in_active)++; + } else if (is_background) { + coins_missing_from_background++; } } } BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); - BOOST_CHECK_EQUAL(coins_in_ibd, initial_total_coins); - BOOST_CHECK_EQUAL(coins_missing_ibd, new_coins); + BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); + BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); } // Snapshot should refuse to load after one has already loaded. @@ -370,4 +327,82 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) loaded_snapshot_blockhash); } +//! Test LoadBlockIndex behavior when multiple chainstates are in use. +//! +//! - First, verfiy that setBlockIndexCandidates is as expected when using a single, +//! fully-validating chainstate. +//! +//! - Then mark a region of the chain BLOCK_ASSUMED_VALID and introduce a second chainstate +//! that will tolerate assumed-valid blocks. Run LoadBlockIndex() and ensure that the first +//! chainstate only contains fully validated blocks and the other chainstate contains all blocks, +//! even those assumed-valid. +//! +BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) +{ + ChainstateManager& chainman = *Assert(m_node.chainman); + CTxMemPool& mempool = *m_node.mempool; + CChainState& cs1 = chainman.ActiveChainstate(); + + int num_indexes{0}; + int num_assumed_valid{0}; + const int expected_assumed_valid{20}; + const int last_assumed_valid_idx{40}; + const int assumed_valid_start_idx = last_assumed_valid_idx - expected_assumed_valid; + + CBlockIndex* validated_tip{nullptr}; + CBlockIndex* assumed_tip{chainman.ActiveChain().Tip()}; + + auto reload_all_block_indexes = [&]() { + for (CChainState* cs : chainman.GetAll()) { + LOCK(::cs_main); + cs->UnloadBlockIndex(); + BOOST_CHECK(cs->setBlockIndexCandidates.empty()); + } + + WITH_LOCK(::cs_main, chainman.LoadBlockIndex()); + }; + + // Ensure that without any assumed-valid BlockIndex entries, all entries are considered + // tip candidates. + reload_all_block_indexes(); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), cs1.m_chain.Height() + 1); + + // Mark some region of the chain assumed-valid. + for (int i = 0; i <= cs1.m_chain.Height(); ++i) { + LOCK(::cs_main); + auto index = cs1.m_chain[i]; + + if (i < last_assumed_valid_idx && i >= assumed_valid_start_idx) { + index->nStatus = BlockStatus::BLOCK_VALID_TREE | BlockStatus::BLOCK_ASSUMED_VALID; + } + + ++num_indexes; + if (index->IsAssumedValid()) ++num_assumed_valid; + + // Note the last fully-validated block as the expected validated tip. + if (i == (assumed_valid_start_idx - 1)) { + validated_tip = index; + BOOST_CHECK(!index->IsAssumedValid()); + } + } + + BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid); + + CChainState& cs2 = WITH_LOCK(::cs_main, + return chainman.InitializeChainstate(&mempool, *m_node.evodb, m_node.chain_helper, llmq::chainLocksHandler, llmq::quorumInstantSendManager, GetRandHash())); + + reload_all_block_indexes(); + + // The fully validated chain only has candidates up to the start of the assumed-valid + // blocks. + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(validated_tip), 1); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(assumed_tip), 0); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), assumed_valid_start_idx); + + // The assumed-valid tolerant chain has all blocks as candidates. + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(validated_tip), 1); + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(assumed_tip), 1); + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index 3cd01c6f33880..5e1007d3930fe 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -12,7 +12,7 @@ #include -BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, BasicTestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, ChainTestingSetup) //! Test utilities for detecting when we need to flush the coins cache based //! on estimated memory usage. @@ -23,7 +23,8 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) { CTxMemPool mempool; BlockManager blockman{}; - CChainState chainstate(&mempool, blockman, *m_node.evodb, m_node.chain_helper, llmq::chainLocksHandler, llmq::quorumInstantSendManager); + CChainState chainstate(&mempool, blockman, *Assert(m_node.chainman), *m_node.evodb, m_node.chain_helper, + llmq::chainLocksHandler, llmq::quorumInstantSendManager); chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); diff --git a/src/txdb.cpp b/src/txdb.cpp index 038432c8fadf9..5a9cf4e91980d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -423,19 +423,23 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = insertBlockIndex(diskindex.hashPrev); pindexNew->nHeight = diskindex.nHeight; - pindexNew->nFile = diskindex.nFile; - pindexNew->nDataPos = diskindex.nDataPos; - pindexNew->nUndoPos = diskindex.nUndoPos; pindexNew->nVersion = diskindex.nVersion; pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->nTime = diskindex.nTime; pindexNew->nBits = diskindex.nBits; pindexNew->nNonce = diskindex.nNonce; - pindexNew->nStatus = diskindex.nStatus; pindexNew->nTx = diskindex.nTx; + { + LOCK(::cs_main); + pindexNew->nFile = diskindex.nFile; + pindexNew->nDataPos = diskindex.nDataPos; + pindexNew->nUndoPos = diskindex.nUndoPos; + pindexNew->nStatus = diskindex.nStatus; + } - if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) + if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) { return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString()); + } pcursor->Next(); } else { diff --git a/src/validation.cpp b/src/validation.cpp index 15be0df400ea4..75c7ba19eebfe 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -67,6 +67,7 @@ #include +#include #include #include #include @@ -77,10 +78,6 @@ /** Maximum kilobytes for transactions to store for processing during reorg */ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; -/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ -static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB -/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ -static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** Time to wait between writing blocks/block index to disk. */ static constexpr std::chrono::hours DATABASE_WRITE_INTERVAL{1}; /** Time to wait between flushing chainstate to disk. */ @@ -96,25 +93,6 @@ const std::vector CHECKLEVEL_DOC { "each level includes the checks of the previous levels", }; -bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const -{ - // First sort by most total work, ... - if (pa->nChainWork > pb->nChainWork) return false; - if (pa->nChainWork < pb->nChainWork) return true; - - // ... then by earliest time received, ... - if (pa->nSequenceId < pb->nSequenceId) return false; - if (pa->nSequenceId > pb->nSequenceId) return true; - - // Use pointer address as tie breaker (should only happen with blocks - // loaded from disk, as those all have id 0). - if (pa < pb) return false; - if (pa > pb) return true; - - // Identical blocks. - return false; -} - /** * Mutex to guard access to validation specific variables, such as reading * or changing the chainstate. @@ -132,17 +110,9 @@ Mutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; bool g_parallel_script_checks{false}; -std::atomic_bool fImporting(false); -std::atomic_bool fReindex(false); -bool fAddressIndex = DEFAULT_ADDRESSINDEX; -bool fTimestampIndex = DEFAULT_TIMESTAMPINDEX; -bool fSpentIndex = DEFAULT_SPENTINDEX; -bool fHavePruned = false; -bool fPruneMode = false; bool fRequireStandard = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; -uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; // TODO: drop this global variable. Used by net.cpp module only @@ -153,55 +123,24 @@ arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); -// Internal stuff -namespace { - CBlockIndex* pindexBestInvalid = nullptr; - - RecursiveMutex cs_LastBlockFile; - std::vector vinfoBlockFile; - int nLastBlockFile = 0; - /** Global flag to indicate we should check to see if there are - * block/undo files that should be deleted. Set on startup - * or if we allocate more file space when we're in prune mode - */ - bool fCheckForPruning = false; - - /** Dirty block index entries. */ - std::set setDirtyBlockIndex; - - /** Dirty block file entries. */ - std::set setDirtyFileInfo; -} // anon namespace - -CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const -{ - AssertLockHeld(cs_main); - BlockMap::const_iterator it = m_block_index.find(hash); - return it == m_block_index.end() ? nullptr : it->second; -} - -CBlockIndex* BlockManager::FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) +const CBlockIndex* CChainState::FindForkInGlobalIndex(const CBlockLocator& locator) const { AssertLockHeld(cs_main); // Find the latest block common to locator and chain - we expect that // locator.vHave is sorted descending by height. for (const uint256& hash : locator.vHave) { - CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex{m_blockman.LookupBlockIndex(hash)}; if (pindex) { - if (chain.Contains(pindex)) + if (m_chain.Contains(pindex)) { return pindex; + } } } - return chain.Genesis(); + return m_chain.Genesis(); } -std::unique_ptr pblocktree; - bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector *pvChecks = nullptr); -static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); -static FlatFileSeq BlockFileSeq(); -static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags) { @@ -1246,6 +1185,7 @@ void CoinsViews::InitCache() CChainState::CChainState(CTxMemPool* mempool, BlockManager& blockman, + ChainstateManager& chainman, CEvoDB& evoDb, const std::unique_ptr& chain_helper, const std::unique_ptr& clhandler, @@ -1258,6 +1198,7 @@ CChainState::CChainState(CTxMemPool* mempool, m_isman(isman), m_evoDb(evoDb), m_blockman(blockman), + m_chainman(chainman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( @@ -1338,7 +1279,7 @@ void CChainState::CheckForkWarningConditions() return; } - if (pindexBestInvalid && pindexBestInvalid->nChainWork > m_chain.Tip()->nChainWork + (GetBlockProof(*m_chain.Tip()) * 6)) { + if (m_chainman.m_best_invalid && m_chainman.m_best_invalid->nChainWork > m_chain.Tip()->nChainWork + (GetBlockProof(*m_chain.Tip()) * 6)) { LogPrintf("%s: Warning: Found invalid chain which has higher work (at least ~6 blocks worth of work) than our best chain.\nChain state database corruption likely.\n", __func__); SetfLargeWorkInvalidChainFound(true); } else { @@ -1352,8 +1293,9 @@ void CChainState::InvalidChainFound(CBlockIndex* pindexNew) statsClient.inc("warnings.InvalidChainFound", 1.0f); - if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) - pindexBestInvalid = pindexNew; + if (!m_chainman.m_best_invalid || pindexNew->nChainWork > m_chainman.m_best_invalid->nChainWork) { + m_chainman.m_best_invalid = pindexNew; + } if (pindexBestHeader != nullptr && pindexBestHeader->GetAncestor(pindexNew->nHeight) == pindexNew) { pindexBestHeader = m_chain.Tip(); } @@ -1392,8 +1334,8 @@ void CChainState::InvalidBlockFound(CBlockIndex *pindex, const BlockValidationSt statsClient.inc("warnings.InvalidBlockFound", 1.0f); if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; - m_blockman.m_failed_blocks.insert(pindex); - setDirtyBlockIndex.insert(pindex); + m_chainman.m_failed_blocks.insert(pindex); + m_blockman.m_dirty_blockindex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); } @@ -1557,65 +1499,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const C return true; } -static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) -{ - // Open history file to append - CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); - if (fileout.IsNull()) - return error("%s: OpenUndoFile failed", __func__); - - // Write index header - unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); - fileout << messageStart << nSize; - - // Write undo data - long fileOutPos = ftell(fileout.Get()); - if (fileOutPos < 0) - return error("%s: ftell failed", __func__); - pos.nPos = (unsigned int)fileOutPos; - fileout << blockundo; - - // calculate & write checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - fileout << hasher.GetHash(); - - return true; -} - -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) -{ - FlatFilePos pos = pindex->GetUndoPos(); - if (pos.IsNull()) { - return error("%s: no undo data available", __func__); - } - - // Open history file to read - CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); - if (filein.IsNull()) - return error("%s: OpenUndoFile failed", __func__); - - // Read block - uint256 hashChecksum; - CHashVerifier verifier(&filein); // We need a CHashVerifier as reserializing may lose data - try { - verifier << pindex->pprev->GetBlockHash(); - verifier >> blockundo; - filein >> hashChecksum; - } - catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - - // Verify checksum - if (hashChecksum != verifier.GetHash()) - return error("%s: Checksum mismatch", __func__); - - return true; -} - -static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str()) +bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage) { AbortNode(strMessage, userMessage); return state.Error(strMessage); @@ -1778,25 +1662,25 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI if (fSpentIndex) { - if (!pblocktree->UpdateSpentIndex(spentIndex)) { + if (!m_blockman.m_block_tree_db->UpdateSpentIndex(spentIndex)) { AbortNode("Failed to delete spent index"); return DISCONNECT_FAILED; } } if (fAddressIndex) { - if (!pblocktree->EraseAddressIndex(addressIndex)) { + if (!m_blockman.m_block_tree_db->EraseAddressIndex(addressIndex)) { AbortNode("Failed to delete address index"); return DISCONNECT_FAILED; } - if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + if (!m_blockman.m_block_tree_db->UpdateAddressUnspentIndex(addressUnspentIndex)) { AbortNode("Failed to write address unspent index"); return DISCONNECT_FAILED; } } if (fTimestampIndex) { - if (!pblocktree->EraseTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) { + if (!m_blockman.m_block_tree_db->EraseTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) { AbortNode("Failed to delete timestamp index"); return DISCONNECT_FAILED; } @@ -1819,55 +1703,6 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -static void FlushUndoFile(int block_file, bool finalize = false) -{ - FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); - if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { - AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); - } -} - -static void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) -{ - LOCK(cs_LastBlockFile); - FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); - if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { - AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); - } - // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, - // e.g. during IBD or a sync after a node going offline - if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); -} - -static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); - -static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) -{ - // Write undo information to disk - if (pindex->GetUndoPos().IsNull()) { - FlatFilePos _pos; - if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) - return error("ConnectBlock(): FindUndoPos failed"); - if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) - return AbortNode(state, "Failed to write undo data"); - // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) - // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height - // in the block file info as below; note that this does not catch the case where the undo writes are keeping up - // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in - // the FindBlockPos function - if (_pos.nFile < nLastBlockFile && static_cast(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { - FlushUndoFile(_pos.nFile, true); - } - - // update nUndoPos in block index - pindex->nUndoPos = _pos.nPos; - pindex->nStatus |= BLOCK_HAVE_UNDO; - setDirtyBlockIndex.insert(pindex); - } - - return true; -} - static CCheckQueue scriptcheckqueue(128); void StartScriptCheckWorkerThreads(int threads_num) @@ -2052,7 +1887,7 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // effectively caching the result of part of the verification. BlockMap::const_iterator it = m_blockman.m_block_index.find(hashAssumeValid); if (it != m_blockman.m_block_index.end()) { - if (it->second->GetAncestor(pindex->nHeight) == pindex && + if (it->second.GetAncestor(pindex->nHeight) == pindex && pindexBestHeader->GetAncestor(pindex->nHeight) == pindex && pindexBestHeader->nChainWork >= nMinimumChainWork) { // This block is a member of the assumed verified chain and an ancestor of the best header. @@ -2385,33 +2220,33 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, if (fJustCheck) return true; - if (!WriteUndoDataForBlock(blockundo, state, pindex, m_params)) { + if (!m_blockman.WriteUndoDataForBlock(blockundo, state, pindex, m_params)) { return false; } if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); - setDirtyBlockIndex.insert(pindex); + m_blockman.m_dirty_blockindex.insert(pindex); } int64_t nTime6 = GetTimeMicros(); if (fAddressIndex) { - if (!pblocktree->WriteAddressIndex(addressIndex)) { + if (!m_blockman.m_block_tree_db->WriteAddressIndex(addressIndex)) { return AbortNode(state, "Failed to write address index"); } - if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + if (!m_blockman.m_block_tree_db->UpdateAddressUnspentIndex(addressUnspentIndex)) { return AbortNode(state, "Failed to write address unspent index"); } } if (fSpentIndex) - if (!pblocktree->UpdateSpentIndex(spentIndex)) + if (!m_blockman.m_block_tree_db->UpdateSpentIndex(spentIndex)) return AbortNode(state, "Failed to write transaction index"); if (fTimestampIndex) - if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + if (!m_blockman.m_block_tree_db->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) return AbortNode(state, "Failed to write timestamp index"); int64_t nTime7 = GetTimeMicros(); nTimeIndexWrite += nTime7 - nTime6; @@ -2492,8 +2327,8 @@ bool CChainState::FlushStateToDisk( bool fDoFullFlush = false; CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); - LOCK(cs_LastBlockFile); - if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { + LOCK(m_blockman.cs_LastBlockFile); + if (fPruneMode && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) { // make sure we don't prune above the blockfilterindexes bestblocks // pruning is height-based int last_prune = m_chain.Height(); // last height we can prune @@ -2509,12 +2344,12 @@ bool CChainState::FlushStateToDisk( LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCHMARK); m_blockman.FindFilesToPrune(setFilesToPrune, m_params.PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); - fCheckForPruning = false; + m_blockman.m_check_for_pruning = false; } if (!setFilesToPrune.empty()) { fFlushForPrune = true; if (!fHavePruned) { - pblocktree->WriteFlag("prunedblockfiles", true); + m_blockman.m_block_tree_db->WriteFlag("prunedblockfiles", true); fHavePruned = true; } } @@ -2550,26 +2385,14 @@ bool CChainState::FlushStateToDisk( LOG_TIME_MILLIS_WITH_CATEGORY("write block and undo data to disk", BCLog::BENCHMARK); // First make sure all block and undo data is flushed to disk. - FlushBlockFile(); + m_blockman.FlushBlockFile(); } // Then update all block file information (which may refer to block and undo files). { LOG_TIME_MILLIS_WITH_CATEGORY("write block index to disk", BCLog::BENCHMARK); - std::vector > vFiles; - vFiles.reserve(setDirtyFileInfo.size()); - for (std::set::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { - vFiles.push_back(std::make_pair(*it, &vinfoBlockFile[*it])); - setDirtyFileInfo.erase(it++); - } - std::vector vBlocks; - vBlocks.reserve(setDirtyBlockIndex.size()); - for (std::set::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { - vBlocks.push_back(*it); - setDirtyBlockIndex.erase(it++); - } - if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { + if (!m_blockman.WriteBlockIndexDB()) { return AbortNode(state, "Failed to write to block index database"); } } @@ -2630,8 +2453,7 @@ void CChainState::ForceFlushStateToDisk() void CChainState::PruneAndFlush() { BlockValidationState state; - fCheckForPruning = true; - + m_blockman.m_check_for_pruning = true; if (!this->FlushStateToDisk(state, FlushStateMode::NONE)) { LogPrintf("%s: failed to flush state (%s)\n", __func__, state.ToString()); } @@ -2654,8 +2476,44 @@ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) res += warn; } +static void UpdateTipLog( + const CCoinsViewCache& coins_tip, + const CBlockIndex* tip, + const CChainParams& params, + const CEvoDB& evo_db, + const std::string& func_name, + const std::string& prefix, + const std::string& warning_messages) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +{ + + AssertLockHeld(::cs_main); + LogPrintf("%s%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo) evodb_cache=%.1fMiB%s\n", + prefix, func_name, + tip->GetBlockHash().ToString(), tip->nHeight, tip->nVersion, + log(tip->nChainWork.getdouble()) / log(2.0), (unsigned long)tip->nChainTx, + FormatISO8601DateTime(tip->GetBlockTime()), + GuessVerificationProgress(params.TxData(), tip), + coins_tip.DynamicMemoryUsage() * (1.0 / (1 << 20)), + coins_tip.GetCacheSize(), + evo_db.GetMemoryUsage() * (1.0 / (1 << 20)), + !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages) : ""); +} + void CChainState::UpdateTip(const CBlockIndex* pindexNew) { + const auto& coins_tip = this->CoinsTip(); + + // The remainder of the function isn't relevant if we are not acting on + // the active chainstate, so return if need be. + if (this != &m_chainman.ActiveChainstate()) { + // Only log every so often so that we don't bury log messages at the tip. + constexpr int BACKGROUND_LOG_INTERVAL = 2000; + if (pindexNew->nHeight % BACKGROUND_LOG_INTERVAL == 0) { + UpdateTipLog(coins_tip, pindexNew, m_params, m_evoDb, __func__, "[background validation] ", ""); + } + return; + } + // New best block if (m_mempool) { m_mempool->AddTransactionsUpdated(1); @@ -2684,13 +2542,7 @@ void CChainState::UpdateTip(const CBlockIndex* pindexNew) } } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo) evodb_cache=%.1fMiB%s\n", __func__, - pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, - log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, - FormatISO8601DateTime(pindexNew->GetBlockTime()), - GuessVerificationProgress(m_params.TxData(), pindexNew), this->CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), this->CoinsTip().GetCacheSize(), - m_evoDb.GetMemoryUsage() * (1.0 / (1<<20)), - !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages.original) : ""); + UpdateTipLog(coins_tip, pindexNew, m_params, m_evoDb, __func__, "", warning_messages.original); } /** Disconnect m_chain's tip. @@ -2910,8 +2762,9 @@ CBlockIndex* CChainState::FindMostWorkChain() { bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); if (fFailedChain || fMissingData || fConflictingChain) { // Candidate chain is not usable (either invalid or conflicting or missing data) - if (fFailedChain && (pindexBestInvalid == nullptr || pindexNew->nChainWork > pindexBestInvalid->nChainWork)) - pindexBestInvalid = pindexNew; + if (fFailedChain && (m_chainman.m_best_invalid == nullptr || pindexNew->nChainWork > m_chainman.m_best_invalid->nChainWork)) { + m_chainman.m_best_invalid = pindexNew; + } CBlockIndex *pindexFailed = pindexNew; // Remove the entire chain from the set. while (pindexTest != pindexFailed) { @@ -3244,8 +3097,8 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind { LOCK(cs_main); - for (const auto& entry : m_blockman.m_block_index) { - CBlockIndex *candidate = entry.second; + for (auto& entry : m_blockman.m_block_index) { + CBlockIndex* candidate = &entry.second; // We don't need to put anything in our active chain into the // multimap, because those candidates will be found and considered // as we disconnect. @@ -3277,7 +3130,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind const CBlockIndex* pindexOldTip = m_chain.Tip(); if (pindex == pindexBestHeader) { - pindexBestInvalid = pindexBestHeader; + m_chainman.m_best_invalid = pindexBestHeader; pindexBestHeader = pindexBestHeader->pprev; } @@ -3295,7 +3148,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind assert(invalid_walk_tip->pprev == m_chain.Tip()); if (pindexOldTip == pindexBestHeader) { - pindexBestInvalid = pindexBestHeader; + m_chainman.m_best_invalid = pindexBestHeader; pindexBestHeader = pindexBestHeader->pprev; } @@ -3305,14 +3158,14 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind // are no blocks that meet the "have data and are not invalid per // nStatus" criteria for inclusion in setBlockIndexCandidates). invalid_walk_tip->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(invalid_walk_tip); + m_blockman.m_dirty_blockindex.insert(invalid_walk_tip); setBlockIndexCandidates.erase(invalid_walk_tip); setBlockIndexCandidates.insert(invalid_walk_tip->pprev); if (invalid_walk_tip->pprev == to_mark_failed && (to_mark_failed->nStatus & BLOCK_FAILED_VALID)) { // We only want to mark the last disconnected block as BLOCK_FAILED_VALID; its children // need to be BLOCK_FAILED_CHILD instead. to_mark_failed->nStatus = (to_mark_failed->nStatus ^ BLOCK_FAILED_VALID) | BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(to_mark_failed); + m_blockman.m_dirty_blockindex.insert(to_mark_failed); } // Add any equal or more work headers to setBlockIndexCandidates @@ -3342,9 +3195,9 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind // Mark pindex (or the last disconnected block) as invalid, even when it never was in the main chain to_mark_failed->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(to_mark_failed); + m_blockman.m_dirty_blockindex.insert(to_mark_failed); setBlockIndexCandidates.erase(to_mark_failed); - m_blockman.m_failed_blocks.insert(to_mark_failed); + m_chainman.m_failed_blocks.insert(to_mark_failed); // If any new blocks somehow arrived while we were disconnecting // (above), then the pre-calculation of what should go into @@ -3353,12 +3206,10 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind // it up here, this should be an essentially unobservable error. // Loop back over all block index entries and add any missing entries // to setBlockIndexCandidates. - BlockMap::iterator it = m_blockman.m_block_index.begin(); - while (it != m_blockman.m_block_index.end()) { - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && !(it->second->nStatus & BLOCK_CONFLICT_CHAINLOCK) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, m_chain.Tip())) { - setBlockIndexCandidates.insert(it->second); + for (auto& [_, block_index] : m_blockman.m_block_index) { + if (block_index.IsValid(BLOCK_VALID_TRANSACTIONS) && !(block_index.nStatus & BLOCK_CONFLICT_CHAINLOCK) && block_index.HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(&block_index, m_chain.Tip())) { + setBlockIndexCandidates.insert(&block_index); } - it++; } InvalidChainFound(to_mark_failed); @@ -3457,8 +3308,8 @@ bool CChainState::MarkConflictingBlock(BlockValidationState& state, CBlockIndex // add it again. BlockMap::iterator it = m_blockman.m_block_index.begin(); while (it != m_blockman.m_block_index.end()) { - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && !(it->second->nStatus & BLOCK_CONFLICT_CHAINLOCK) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, m_chain.Tip())) { - setBlockIndexCandidates.insert(it->second); + if (it->second.IsValid(BLOCK_VALID_TRANSACTIONS) && !(it->second.nStatus & BLOCK_CONFLICT_CHAINLOCK) && it->second.HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(&it->second, m_chain.Tip())) { + setBlockIndexCandidates.insert(&it->second); } it++; } @@ -3478,10 +3329,10 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); if (!pindex) { - if (pindexBestInvalid && pindexBestInvalid->GetAncestor(m_chain.Height()) == m_chain.Tip()) { + if (m_chainman.m_best_invalid && m_chainman.m_best_invalid->GetAncestor(m_chain.Height()) == m_chain.Tip()) { LogPrintf("%s: the best known invalid block (%s) is ahead of our tip, reconsidering\n", - __func__, pindexBestInvalid->GetBlockHash().ToString()); - pindex = pindexBestInvalid; + __func__, m_chainman.m_best_invalid->GetBlockHash().ToString()); + pindex = m_chainman.m_best_invalid; } else { return; } @@ -3490,43 +3341,41 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { int nHeight = pindex->nHeight; // Remove the invalidity flag from this block and all its descendants. - BlockMap::iterator it = m_blockman.m_block_index.begin(); - while (it != m_blockman.m_block_index.end()) { - if (!it->second->IsValid() && it->second->GetAncestor(nHeight) == pindex) { - it->second->nStatus &= ~BLOCK_FAILED_MASK; - setDirtyBlockIndex.insert(it->second); - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && !(it->second->nStatus & BLOCK_CONFLICT_CHAINLOCK) && it->second->HaveTxsDownloaded() && setBlockIndexCandidates.value_comp()(m_chain.Tip(), it->second)) { - setBlockIndexCandidates.insert(it->second); + for (auto& [_, block_index] : m_blockman.m_block_index) { + if (!block_index.IsValid() && block_index.GetAncestor(nHeight) == pindex) { + block_index.nStatus &= ~BLOCK_FAILED_MASK; + m_blockman.m_dirty_blockindex.insert(&block_index); + if (block_index.IsValid(BLOCK_VALID_TRANSACTIONS) && !(block_index.nStatus & BLOCK_CONFLICT_CHAINLOCK) && block_index.HaveTxsDownloaded() && setBlockIndexCandidates.value_comp()(m_chain.Tip(), &block_index)) { + setBlockIndexCandidates.insert(&block_index); } - if (it->second == pindexBestInvalid) { + if (&block_index == m_chainman.m_best_invalid) { // Reset invalid block marker if it was pointing to one of those. - pindexBestInvalid = nullptr; + m_chainman.m_best_invalid = nullptr; } - m_blockman.m_failed_blocks.erase(it->second); + m_chainman.m_failed_blocks.erase(&block_index); } - it++; } // Remove the invalidity flag from all ancestors too. while (pindex != nullptr) { if (pindex->nStatus & BLOCK_FAILED_MASK) { pindex->nStatus &= ~BLOCK_FAILED_MASK; - setDirtyBlockIndex.insert(pindex); + m_blockman.m_dirty_blockindex.insert(pindex); if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && !(pindex->nStatus & BLOCK_CONFLICT_CHAINLOCK) && pindex->HaveTxsDownloaded() && setBlockIndexCandidates.value_comp()(m_chain.Tip(), pindex)) { setBlockIndexCandidates.insert(pindex); } - if (pindex == pindexBestInvalid) { + if (pindex == m_chainman.m_best_invalid) { // Reset invalid block marker if it was pointing to one of those. - pindexBestInvalid = nullptr; + m_chainman.m_best_invalid = nullptr; } - m_blockman.m_failed_blocks.erase(pindex); + m_chainman.m_failed_blocks.erase(pindex); // Mark all nearest BLOCK_FAILED_CHILD descendants (if any) as BLOCK_FAILED_VALID auto itp = m_blockman.m_prev_block_index.equal_range(pindex->GetBlockHash()); for (auto jt = itp.first; jt != itp.second; ++jt) { if (jt->second->nStatus & BLOCK_FAILED_CHILD) { jt->second->nStatus |= BLOCK_FAILED_VALID; - m_blockman.m_failed_blocks.insert(jt->second); - setDirtyBlockIndex.insert(jt->second); + m_chainman.m_failed_blocks.insert(jt->second); + m_blockman.m_dirty_blockindex.insert(jt->second); setBlockIndexCandidates.erase(jt->second); } } @@ -3535,52 +3384,6 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { } } -CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, const uint256& hash, enum BlockStatus nStatus) -{ - assert(!(nStatus & BLOCK_FAILED_MASK)); // no failed blocks allowed - AssertLockHeld(cs_main); - - // Check for duplicate - BlockMap::iterator it = m_block_index.find(hash); - if (it != m_block_index.end()) - return it->second; - - // Construct new block index object - CBlockIndex* pindexNew = new CBlockIndex(block); - // We assign the sequence id to blocks only when the full data is available, - // to avoid miners withholding blocks but broadcasting headers, to get a - // competitive advantage. - pindexNew->nSequenceId = 0; - BlockMap::iterator mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; - pindexNew->phashBlock = &((*mi).first); - BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock); - if (miPrev != m_block_index.end()) - { - pindexNew->pprev = (*miPrev).second; - pindexNew->nHeight = pindexNew->pprev->nHeight + 1; - pindexNew->BuildSkip(); - } - pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime); - pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew); - if (nStatus & BLOCK_VALID_MASK) { - pindexNew->RaiseValidity(nStatus); - if (pindexBestHeader == nullptr || pindexBestHeader->nChainWork < pindexNew->nChainWork) - pindexBestHeader = pindexNew; - } else { - pindexNew->RaiseValidity(BLOCK_VALID_TREE); // required validity level - pindexNew->nStatus |= nStatus; - } - - setDirtyBlockIndex.insert(pindexNew); - - // track prevBlockHash -> pindex (multimap) - if (pindexNew->pprev) { - m_prev_block_index.emplace(pindexNew->pprev->GetBlockHash(), pindexNew); - } - - return pindexNew; -} - /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) { @@ -3591,7 +3394,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); - setDirtyBlockIndex.insert(pindexNew); + m_blockman.m_dirty_blockindex.insert(pindexNew); if (pindexNew->pprev == nullptr || pindexNew->pprev->HaveTxsDownloaded()) { // If pindexNew is the genesis block or all parents are BLOCK_VALID_TRANSACTIONS. @@ -3624,83 +3427,6 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -// TODO move to blockstorage -bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) -{ - LOCK(cs_LastBlockFile); - - unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); - } - - bool finalize_undo = false; - if (!fKnown) { - while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { - // when the undo file is keeping up with the block file, we want to flush it explicitly - // when it is lagging behind (more blocks arrive than are being connected), we let the - // undo block write case handle it - finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); - nFile++; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); - } - } - pos.nFile = nFile; - pos.nPos = vinfoBlockFile[nFile].nSize; - } - - if ((int)nFile != nLastBlockFile) { - if (!fKnown) { - LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); - } - FlushBlockFile(!fKnown, finalize_undo); - nLastBlockFile = nFile; - } - - vinfoBlockFile[nFile].AddBlock(nHeight, nTime); - if (fKnown) - vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); - else - vinfoBlockFile[nFile].nSize += nAddSize; - - if (!fKnown) { - bool out_of_space; - size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - return AbortNode("Disk space is too low!", _("Disk space is too low!")); - } - if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; - } - } - - setDirtyFileInfo.insert(nFile); - return true; -} - -static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) -{ - pos.nFile = nFile; - - LOCK(cs_LastBlockFile); - - pos.nPos = vinfoBlockFile[nFile].nUndoSize; - vinfoBlockFile[nFile].nUndoSize += nAddSize; - setDirtyFileInfo.insert(nFile); - - bool out_of_space; - size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); - } - if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; - } - - return true; -} - static bool CheckBlockHeader(const CBlockHeader& block, const uint256& hash, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true) { // Check proof of work matches claimed amount @@ -3792,21 +3518,6 @@ bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensu return true; } -CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) -{ - const MapCheckpoints& checkpoints = data.mapCheckpoints; - - for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) - { - const uint256& hash = i.second; - CBlockIndex* pindex = LookupBlockIndex(hash); - if (pindex) { - return pindex; - } - } - return nullptr; -} - /** Context-dependent validity checks. * By "context", we mean only the previous block headers, but not the UTXO * set; UTXO-related validity checks are done in ConnectBlock(). @@ -3843,7 +3554,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // BlockIndex(). - CBlockIndex* pcheckpoint = blockman.GetLastCheckpoint(params.Checkpoints()); + const CBlockIndex* pcheckpoint = blockman.GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); return state.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "bad-fork-prior-to-checkpoint"); @@ -3942,18 +3653,18 @@ static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& stat return true; } -bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) +bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) { AssertLockHeld(cs_main); // Check for duplicate uint256 hash = block.GetHash(); - BlockMap::iterator miSelf = m_block_index.find(hash); // TODO : ENABLE BLOCK CACHE IN SPECIFIC CASES + BlockMap::iterator miSelf{m_blockman.m_block_index.find(hash)}; if (hash != chainparams.GetConsensus().hashGenesisBlock) { - if (miSelf != m_block_index.end()) { + if (miSelf != m_blockman.m_block_index.end()) { // Block header is already known. - CBlockIndex* pindex = miSelf->second; + CBlockIndex* pindex = &(miSelf->second); if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) { @@ -3974,12 +3685,12 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS // Get prev block index CBlockIndex* pindexPrev = nullptr; - BlockMap::iterator mi = m_block_index.find(block.hashPrevBlock); - if (mi == m_block_index.end()) { + BlockMap::iterator mi{m_blockman.m_block_index.find(block.hashPrevBlock)}; + if (mi == m_blockman.m_block_index.end()) { LogPrintf("ERROR: %s: prev block not found\n", __func__); return state.Invalid(BlockValidationResult::BLOCK_MISSING_PREV, "prev-blk-not-found"); } - pindexPrev = (*mi).second; + pindexPrev = &((*mi).second); assert(pindexPrev); if (pindexPrev->nStatus & BLOCK_FAILED_MASK) { @@ -3993,8 +3704,9 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS return state.Invalid(BlockValidationResult::BLOCK_CHAINLOCK, "bad-prevblk-chainlock"); } - if (!ContextualCheckBlockHeader(block, state, *this, chainparams, pindexPrev, GetAdjustedTime())) + if (!ContextualCheckBlockHeader(block, state, m_blockman, chainparams, pindexPrev, GetAdjustedTime())) { return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); + } /* Determine if this block descends from any block which has been found * invalid (m_failed_blocks), then mark pindexPrev and any blocks between @@ -4026,7 +3738,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS CBlockIndex* invalid_walk = pindexPrev; while (invalid_walk != failedit) { invalid_walk->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(invalid_walk); + m_blockman.m_dirty_blockindex.insert(invalid_walk); invalid_walk = invalid_walk->pprev; } LogPrintf("ERROR: %s: prev block invalid\n", __func__); @@ -4036,14 +3748,14 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS } if (llmq::chainLocksHandler->HasConflictingChainLock(pindexPrev->nHeight + 1, hash)) { - if (miSelf == m_block_index.end()) { - AddToBlockIndex(block, hash, BLOCK_CONFLICT_CHAINLOCK); + if (miSelf == m_blockman.m_block_index.end()) { + m_blockman.AddToBlockIndex(block, hash, BLOCK_CONFLICT_CHAINLOCK); } LogPrintf("ERROR: %s: header %s conflicts with chainlock\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_CHAINLOCK, "bad-chainlock"); } } - CBlockIndex* pindex = AddToBlockIndex(block, hash); + CBlockIndex* pindex{m_blockman.AddToBlockIndex(block, hash)}; if (ppindex) *ppindex = pindex; @@ -4062,8 +3774,7 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector& LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - bool accepted = m_blockman.AcceptBlockHeader( - header, state, chainparams, &pindex); + bool accepted{AcceptBlockHeader(header, state, chainparams, &pindex)}; ActiveChainstate().CheckBlockIndex(); if (!accepted) { @@ -4098,7 +3809,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr& pblock, Block CBlockIndex *pindexDummy = nullptr; CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; - bool accepted_header = m_blockman.AcceptBlockHeader(block, state, m_params, &pindex); + bool accepted_header{m_chainman.AcceptBlockHeader(block, state, m_params, &pindex)}; CheckBlockIndex(); if (!accepted_header) @@ -4141,7 +3852,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr& pblock, Block !ContextualCheckBlock(block, state, m_params.GetConsensus(), pindex->pprev)) { if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); + m_blockman.m_dirty_blockindex.insert(pindex); } return error("%s: %s", __func__, state.ToString()); } @@ -4154,7 +3865,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr& pblock, Block // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, m_chain, m_params, dbp); + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, m_params, dbp)}; if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -4268,90 +3979,6 @@ bool TestBlockValidity(BlockValidationState& state, return true; } -/** - * BLOCK PRUNING CODE - */ - -/* Calculate the amount of disk space the block & undo files currently use */ -uint64_t CalculateCurrentUsage() -{ - LOCK(cs_LastBlockFile); - - uint64_t retval = 0; - for (const CBlockFileInfo &file : vinfoBlockFile) { - retval += file.nSize + file.nUndoSize; - } - return retval; -} - -void BlockManager::PruneOneBlockFile(const int fileNumber) -{ - AssertLockHeld(cs_main); - LOCK(cs_LastBlockFile); - - for (const auto& entry : m_block_index) { - CBlockIndex* pindex = entry.second; - if (pindex->nFile == fileNumber) { - pindex->nStatus &= ~BLOCK_HAVE_DATA; - pindex->nStatus &= ~BLOCK_HAVE_UNDO; - pindex->nFile = 0; - pindex->nDataPos = 0; - pindex->nUndoPos = 0; - setDirtyBlockIndex.insert(pindex); - - // Prune from m_blocks_unlinked -- any block we prune would have - // to be downloaded again in order to consider its chain, at which - // point it would be considered as a candidate for - // m_blocks_unlinked or setBlockIndexCandidates. - auto range = m_blocks_unlinked.equal_range(pindex->pprev); - while (range.first != range.second) { - std::multimap::iterator _it = range.first; - range.first++; - if (_it->second == pindex) { - m_blocks_unlinked.erase(_it); - } - } - } - } - - vinfoBlockFile[fileNumber].SetNull(); - setDirtyFileInfo.insert(fileNumber); -} - - -void UnlinkPrunedFiles(const std::set& setFilesToPrune) -{ - for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - FlatFilePos pos(*it, 0); - fs::remove(BlockFileSeq().FileName(pos)); - fs::remove(UndoFileSeq().FileName(pos)); - LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); - } -} - -void BlockManager::FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight, int chain_tip_height) -{ - assert(fPruneMode && nManualPruneHeight > 0); - - LOCK2(cs_main, cs_LastBlockFile); - if (chain_tip_height < 0) { - return; - } - - // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip) - unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP); - int count = 0; - for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { - if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { - continue; - } - PruneOneBlockFile(fileNumber); - setFilesToPrune.insert(fileNumber); - count++; - } - LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count); -} - /* This function is called from the RPC code for pruneblockchain */ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeight) { @@ -4361,248 +3988,6 @@ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeigh } } -void BlockManager::FindFilesToPrune(std::set& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd) -{ - LOCK2(cs_main, cs_LastBlockFile); - if (chain_tip_height < 0 || nPruneTarget == 0) { - return; - } - if ((uint64_t)chain_tip_height <= nPruneAfterHeight) { - return; - } - - unsigned int nLastBlockWeCanPrune{(unsigned)std::min(prune_height, chain_tip_height - static_cast(MIN_BLOCKS_TO_KEEP))}; - uint64_t nCurrentUsage = CalculateCurrentUsage(); - // We don't check to prune until after we've allocated new space for files - // So we should leave a buffer under our target to account for another allocation - // before the next pruning. - uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; - uint64_t nBytesToPrune; - int count = 0; - - if (nCurrentUsage + nBuffer >= nPruneTarget) { - // On a prune event, the chainstate DB is flushed. - // To avoid excessive prune events negating the benefit of high dbcache - // values, we should not prune too rapidly. - // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon. - if (is_ibd) { - // Since this is only relevant during IBD, we use a fixed 10% - nBuffer += nPruneTarget / 10; - } - - for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { - nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; - - if (vinfoBlockFile[fileNumber].nSize == 0) { - continue; - } - - if (nCurrentUsage + nBuffer < nPruneTarget) { // are we below our target? - break; - } - - // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning - if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { - continue; - } - - PruneOneBlockFile(fileNumber); - // Queue up the files for removal - setFilesToPrune.insert(fileNumber); - nCurrentUsage -= nBytesToPrune; - count++; - } - } - - LogPrint(BCLog::PRUNE, "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", - nPruneTarget/1024/1024, nCurrentUsage/1024/1024, - ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, - nLastBlockWeCanPrune, count); -} - -static FlatFileSeq BlockFileSeq() -{ - return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); -} - -static FlatFileSeq UndoFileSeq() -{ - return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); -} - -FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { - return BlockFileSeq().Open(pos, fReadOnly); -} - -/** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { - return UndoFileSeq().Open(pos, fReadOnly); -} - -fs::path GetBlockPosFilename(const FlatFilePos &pos) -{ - return BlockFileSeq().FileName(pos); -} - -CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash) -{ - AssertLockHeld(cs_main); - - if (hash.IsNull()) - return nullptr; - - // Return existing - BlockMap::iterator mi = m_block_index.find(hash); - if (mi != m_block_index.end()) - return (*mi).second; - - // Create new - CBlockIndex* pindexNew = new CBlockIndex(); - mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; - pindexNew->phashBlock = &((*mi).first); - - return pindexNew; -} - -bool BlockManager::LoadBlockIndex( - const Consensus::Params& consensus_params, - CBlockTreeDB& blocktree, - std::set& block_index_candidates) -{ - if (!blocktree.LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) - return false; - - // Calculate nChainWork - std::vector > vSortedByHeight; - vSortedByHeight.reserve(m_block_index.size()); - for (const std::pair& item : m_block_index) - { - CBlockIndex* pindex = item.second; - vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); - - // build m_blockman.m_prev_block_index - if (pindex->pprev) { - m_prev_block_index.emplace(pindex->pprev->GetBlockHash(), pindex); - } - } - sort(vSortedByHeight.begin(), vSortedByHeight.end()); - for (const std::pair& item : vSortedByHeight) - { - if (ShutdownRequested()) return false; - CBlockIndex* pindex = item.second; - pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); - pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); - // We can link the chain of blocks for which we've received transactions at some point. - // Pruned nodes may have deleted the block. - if (pindex->nTx > 0) { - if (pindex->pprev) { - if (pindex->pprev->HaveTxsDownloaded()) { - pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; - } else { - pindex->nChainTx = 0; - m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex)); - } - } else { - pindex->nChainTx = pindex->nTx; - } - } - if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { - pindex->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(pindex); - } - if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr)) { - block_index_candidates.insert(pindex); - } - if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) - pindexBestInvalid = pindex; - if (pindex->pprev) - pindex->BuildSkip(); - if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == nullptr || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) - pindexBestHeader = pindex; - } - - return true; -} - -void BlockManager::Unload() { - m_failed_blocks.clear(); - m_blocks_unlinked.clear(); - - for (const BlockMap::value_type& entry : m_block_index) { - delete entry.second; - } - - m_block_index.clear(); - m_prev_block_index.clear(); -} - -bool CChainState::LoadBlockIndexDB() -{ - if (!m_blockman.LoadBlockIndex( - m_params.GetConsensus(), *pblocktree, - setBlockIndexCandidates)) { - return false; - } - - // Load block file info - pblocktree->ReadLastBlockFile(nLastBlockFile); - vinfoBlockFile.resize(nLastBlockFile + 1); - LogPrintf("%s: last block file = %i\n", __func__, nLastBlockFile); - for (int nFile = 0; nFile <= nLastBlockFile; nFile++) { - pblocktree->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); - } - LogPrintf("%s: last block file info: %s\n", __func__, vinfoBlockFile[nLastBlockFile].ToString()); - for (int nFile = nLastBlockFile + 1; true; nFile++) { - CBlockFileInfo info; - if (pblocktree->ReadBlockFileInfo(nFile, info)) { - vinfoBlockFile.push_back(info); - } else { - break; - } - } - - // Check presence of blk files - LogPrintf("Checking all blk files are present...\n"); - std::set setBlkDataFiles; - for (const std::pair& item : m_blockman.m_block_index) { - CBlockIndex* pindex = item.second; - if (pindex->nStatus & BLOCK_HAVE_DATA) { - setBlkDataFiles.insert(pindex->nFile); - } - } - for (std::set::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) - { - FlatFilePos pos(*it, 0); - if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { - return false; - } - } - - // Check whether we have ever pruned block & undo files - pblocktree->ReadFlag("prunedblockfiles", fHavePruned); - if (fHavePruned) - LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); - - // Check whether we need to continue reindexing - bool fReindexing = false; - pblocktree->ReadReindexing(fReindexing); - if(fReindexing) fReindex = true; - - // Check whether we have an address index - pblocktree->ReadFlag("addressindex", fAddressIndex); - LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); - - // Check whether we have a timestamp index - pblocktree->ReadFlag("timestampindex", fTimestampIndex); - LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); - - // Check whether we have a spent index - pblocktree->ReadFlag("spentindex", fSpentIndex); - LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); - - return true; -} - void CChainState::LoadMempool(const ArgsManager& args) { if (!m_mempool) return; @@ -4847,22 +4232,22 @@ bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& i } if (fAddressIndex) { - if (!pblocktree->WriteAddressIndex(addressIndex)) { + if (!m_blockman.m_block_tree_db->WriteAddressIndex(addressIndex)) { return error("RollforwardBlock(DASH): Failed to write address index"); } - if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + if (!m_blockman.m_block_tree_db->UpdateAddressUnspentIndex(addressUnspentIndex)) { return error("RollforwardBlock(DASH): Failed to write address unspent index"); } } if (fSpentIndex) { - if (!pblocktree->UpdateSpentIndex(spentIndex)) + if (!m_blockman.m_block_tree_db->UpdateSpentIndex(spentIndex)) return error("RollforwardBlock(DASH): Failed to write transaction index"); } if (fTimestampIndex) { - if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + if (!m_blockman.m_block_tree_db->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) return error("RollforwardBlock(DASH): Failed to write timestamp index"); } @@ -4890,13 +4275,13 @@ bool CChainState::ReplayBlocks() if (m_blockman.m_block_index.count(hashHeads[0]) == 0) { return error("ReplayBlocks(): reorganization to unknown block requested"); } - pindexNew = m_blockman.m_block_index[hashHeads[0]]; + pindexNew = &(m_blockman.m_block_index[hashHeads[0]]); if (!hashHeads[1].IsNull()) { // The old tip is allowed to be 0, indicating it's the first flush. if (m_blockman.m_block_index.count(hashHeads[1]) == 0) { return error("ReplayBlocks(): reorganization from unknown block requested"); } - pindexOld = m_blockman.m_block_index[hashHeads[1]]; + pindexOld = &(m_blockman.m_block_index[hashHeads[1]]); pindexFork = LastCommonAncestor(pindexOld, pindexNew); assert(pindexFork != nullptr); const bool fDIP0003Active = pindexOld->nHeight >= m_params.GetConsensus().DIP0003Height; @@ -4957,13 +4342,8 @@ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman) { LOCK(cs_main); chainman.Unload(); - pindexBestInvalid = nullptr; pindexBestHeader = nullptr; if (mempool) mempool->clear(); - vinfoBlockFile.clear(); - nLastBlockFile = 0; - setDirtyBlockIndex.clear(); - setDirtyFileInfo.clear(); g_versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { warningcache[b].clear(); @@ -4977,8 +4357,74 @@ bool ChainstateManager::LoadBlockIndex() // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = ActiveChainstate().LoadBlockIndexDB(); + bool ret = m_blockman.LoadBlockIndexDB(); if (!ret) return false; + + std::vector vSortedByHeight{m_blockman.GetAllBlockIndices()}; + std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), + CBlockIndexHeightOnlyComparator()); + + // Find start of assumed-valid region. + int first_assumed_valid_height = std::numeric_limits::max(); + + for (const CBlockIndex* block : vSortedByHeight) { + if (block->IsAssumedValid()) { + auto chainstates = GetAll(); + + // If we encounter an assumed-valid block index entry, ensure that we have + // one chainstate that tolerates assumed-valid entries and another that does + // not (i.e. the background validation chainstate), since assumed-valid + // entries should always be pending validation by a fully-validated chainstate. + auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; + assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); + assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); + + first_assumed_valid_height = block->nHeight; + break; + } + } + + for (CBlockIndex* pindex : vSortedByHeight) { + if (ShutdownRequested()) return false; + if (pindex->IsAssumedValid() || + (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && + (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { + + // Fill each chainstate's block candidate set. Only add assumed-valid + // blocks to the tip candidate set if the chainstate is allowed to rely on + // assumed-valid blocks. + // + // If all setBlockIndexCandidates contained the assumed-valid blocks, the + // background chainstate's ActivateBestChain() call would add assumed-valid + // blocks to the chain (based on how FindMostWorkChain() works). Obviously + // we don't want this since the purpose of the background validation chain + // is to validate assued-valid blocks. + // + // Note: This is considering all blocks whose height is greater or equal to + // the first assumed-valid block to be assumed-valid blocks, and excluding + // them from the background chainstate's setBlockIndexCandidates set. This + // does mean that some blocks which are not technically assumed-valid + // (later blocks on a fork beginning before the first assumed-valid block) + // might not get added to the background chainstate, but this is ok, + // because they will still be attached to the active chainstate if they + // actually contain more work. + // + // Instead of this height-based approach, an earlier attempt was made at + // detecting "holistically" whether the block index under consideration + // relied on an assumed-valid ancestor, but this proved to be too slow to + // be practical. + for (CChainState* chainstate : GetAll()) { + if (chainstate->reliesOnAssumedValid() || + pindex->nHeight < first_assumed_valid_height) { + chainstate->setBlockIndexCandidates.insert(pindex); + } + } + } + if (pindex->nStatus & BLOCK_FAILED_MASK && (!m_best_invalid || pindex->nChainWork > m_best_invalid->nChainWork)) { + m_best_invalid = pindex; + } + } + needs_init = m_blockman.m_block_index.empty(); } @@ -4993,24 +4439,25 @@ bool ChainstateManager::LoadBlockIndex() // Use the provided setting for -addressindex in the new database fAddressIndex = gArgs.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); - pblocktree->WriteFlag("addressindex", fAddressIndex); + m_blockman.m_block_tree_db->WriteFlag("addressindex", fAddressIndex); // Use the provided setting for -timestampindex in the new database fTimestampIndex = gArgs.GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); - pblocktree->WriteFlag("timestampindex", fTimestampIndex); + m_blockman.m_block_tree_db->WriteFlag("timestampindex", fTimestampIndex); // Use the provided setting for -spentindex in the new database fSpentIndex = gArgs.GetBoolArg("-spentindex", DEFAULT_SPENTINDEX); - pblocktree->WriteFlag("spentindex", fSpentIndex); + m_blockman.m_block_tree_db->WriteFlag("spentindex", fSpentIndex); } return true; } bool CChainState::AddGenesisBlock(const CBlock& block, BlockValidationState& state) { - FlatFilePos blockPos = SaveBlockToDisk(block, 0, m_chain, m_params, nullptr); - if (blockPos.IsNull()) + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, m_params, nullptr)}; + if (blockPos.IsNull()) { return error("%s: writing genesis block to disk failed (%s)", __func__, state.ToString()); + } CBlockIndex* pindex = m_blockman.AddToBlockIndex(block, block.GetHash()); ReceivedBlockTransactions(block, pindex, blockPos); return true; @@ -5109,7 +4556,7 @@ void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) } // process in case the block isn't known yet - CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); + const CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { BlockValidationState state; if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr)) { @@ -5186,8 +4633,8 @@ void CChainState::CheckBlockIndex() // Build forward-pointing map of the entire block tree. std::multimap forward; - for (const std::pair& entry : m_blockman.m_block_index) { - forward.insert(std::make_pair(entry.second->pprev, entry.second)); + for (auto& [_, block_index] : m_blockman.m_block_index) { + forward.emplace(block_index.pprev, &block_index); } assert(forward.size() == m_blockman.m_block_index.size()); @@ -5214,12 +4661,33 @@ void CChainState::CheckBlockIndex() nNodes++; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; if (pindexFirstConflicing == nullptr && pindex->nStatus & BLOCK_CONFLICT_CHAINLOCK) pindexFirstConflicing = pindex; - if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + // Assumed-valid index entries will not have data since we haven't downloaded the + // full block yet. + if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA) && !pindex->IsAssumedValid()) { + pindexFirstMissing = pindex; + } if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotTransactionsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotChainValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotScriptsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) pindexFirstNotScriptsValid = pindex; + + if (pindex->pprev != nullptr && !pindex->IsAssumedValid()) { + // Skip validity flag checks for BLOCK_ASSUMED_VALID index entries, since these + // *_VALID_MASK flags will not be present for index entries we are temporarily assuming + // valid. + if (pindexFirstNotTransactionsValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) { + pindexFirstNotTransactionsValid = pindex; + } + + if (pindexFirstNotChainValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) { + pindexFirstNotChainValid = pindex; + } + + if (pindexFirstNotScriptsValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) { + pindexFirstNotScriptsValid = pindex; + } + } // Begin: actual consistency checks. if (pindex->pprev == nullptr) { @@ -5230,7 +4698,9 @@ void CChainState::CheckBlockIndex() if (!pindex->HaveTxsDownloaded()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. - if (!fHavePruned) { + // Unless these indexes are assumed valid and pending block download on a + // background chainstate. + if (!fHavePruned && !pindex->IsAssumedValid()) { // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); assert(pindexFirstMissing == pindexFirstNeverProcessed); @@ -5239,7 +4709,16 @@ void CChainState::CheckBlockIndex() if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); } if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); - assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + if (pindex->IsAssumedValid()) { + // Assumed-valid blocks should have some nTx value. + assert(pindex->nTx > 0); + // Assumed-valid blocks should connect to the main chain. + assert((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE); + } else { + // Otherwise there should only be an nTx value if we have + // actually seen a block's transactions. + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + } // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to HaveTxsDownloaded(). assert((pindexFirstNeverProcessed == nullptr) == pindex->HaveTxsDownloaded()); assert((pindexFirstNotTransactionsValid == nullptr) == pindex->HaveTxsDownloaded()); @@ -5260,11 +4739,17 @@ void CChainState::CheckBlockIndex() } if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { if (pindexFirstInvalid == nullptr && pindexFirstConflicing == nullptr) { + const bool is_active = this == &m_chainman.ActiveChainstate(); + // If this block sorts at least as good as the current tip and // is valid and we have all data for its parents, it must be in // setBlockIndexCandidates. m_chain.Tip() must also be there // even if some data has been pruned. - if (pindexFirstMissing == nullptr || pindex == m_chain.Tip()) { + // + // Don't perform this check for the background chainstate since + // its setBlockIndexCandidates shouldn't have some entries (i.e. those past the + // snapshot block) which do exist in the block index for the active chainstate. + if (is_active && (pindexFirstMissing == nullptr || pindex == m_chain.Tip())) { assert(setBlockIndexCandidates.count(pindex)); } // If some parent is missing, then it could be that this block was in @@ -5398,17 +4883,6 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) return ret; } -std::string CBlockFileInfo::ToString() const { - return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); -} - -CBlockFileInfo* GetBlockFileInfo(size_t n) -{ - LOCK(cs_LastBlockFile); - - return &vinfoBlockFile.at(n); -} - static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) @@ -5617,7 +5091,7 @@ CChainState& ChainstateManager::InitializeChainstate(CTxMemPool* mempool, throw std::logic_error("should not be overwriting a chainstate"); } - to_modify.reset(new CChainState(mempool, m_blockman, evoDb, chain_helper, clhandler, isman, snapshot_blockhash)); + to_modify.reset(new CChainState(mempool, m_blockman, *this, evoDb, chain_helper, clhandler, isman, snapshot_blockhash)); // Snapshot chainstates and initial IBD chaintates always become active. if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { @@ -5687,7 +5161,7 @@ bool ChainstateManager::ActivateSnapshot( } auto snapshot_chainstate = WITH_LOCK(::cs_main, return std::make_unique( - /* mempool */ nullptr, m_blockman, + /* mempool */ nullptr, m_blockman, *this, this->ActiveChainstate().m_evoDb, this->ActiveChainstate().m_chain_helper, this->ActiveChainstate().m_clhandler, @@ -5886,7 +5360,14 @@ bool ChainstateManager::PopulateAndValidateSnapshot( // Fake various pieces of CBlockIndex state: CBlockIndex* index = nullptr; - for (int i = 0; i <= snapshot_chainstate.m_chain.Height(); ++i) { + + // Don't make any modifications to the genesis block. + // This is especially important because we don't want to erroneously + // apply BLOCK_ASSUMED_VALID to genesis, which would happen if we didn't skip + // it here (since it apparently isn't BLOCK_VALID_SCRIPTS). + constexpr int AFTER_GENESIS_START{1}; + + for (int i = AFTER_GENESIS_START; i <= snapshot_chainstate.m_chain.Height(); ++i) { index = snapshot_chainstate.m_chain[i]; // Fake nTx so that LoadBlockIndex() loads assumed-valid CBlockIndex @@ -5895,7 +5376,21 @@ bool ChainstateManager::PopulateAndValidateSnapshot( index->nTx = 1; } // Fake nChainTx so that GuessVerificationProgress reports accurately - index->nChainTx = index->pprev ? index->pprev->nChainTx + index->nTx : 1; + index->nChainTx = index->pprev->nChainTx + index->nTx; + + // Mark unvalidated block index entries beneath the snapshot base block as assumed-valid. + if (!index->IsValid(BLOCK_VALID_SCRIPTS)) { + // This flag will be removed once the block is fully validated by a + // background chainstate. + index->nStatus |= BLOCK_ASSUMED_VALID; + } + + m_blockman.m_dirty_blockindex.insert(index); + // Changes to the block index will be flushed to disk after this call + // returns in `ActivateSnapshot()`, when `MaybeRebalanceCaches()` is + // called, since we've added a snapshot chainstate and therefore will + // have to downsize the IBD chainstate, which will result in a call to + // `FlushStateToDisk(ALWAYS)`. } assert(index); @@ -5920,22 +5415,6 @@ bool ChainstateManager::IsSnapshotActive() const return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } -CChainState& ChainstateManager::ValidatedChainstate() const -{ - LOCK(::cs_main); - if (m_snapshot_chainstate && IsSnapshotValidated()) { - return *m_snapshot_chainstate.get(); - } - assert(m_ibd_chainstate); - return *m_ibd_chainstate.get(); -} - -bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const -{ - LOCK(::cs_main); - return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); -} - void ChainstateManager::Unload() { for (CChainState* chainstate : this->GetAll()) { @@ -5943,7 +5422,9 @@ void ChainstateManager::Unload() chainstate->UnloadBlockIndex(); } + m_failed_blocks.clear(); m_blockman.Unload(); + m_best_invalid = nullptr; } void ChainstateManager::Reset() diff --git a/src/validation.h b/src/validation.h index a5d891db06aa7..7803865b8c919 100644 --- a/src/validation.h +++ b/src/validation.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include