diff --git a/.cirrus.yml b/.cirrus.yml index f14f85da997c13..bca589a1734a60 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -169,6 +169,7 @@ task: docker_arguments: CI_IMAGE_NAME_TAG: debian:bullseye FILE_ENV: "./ci/test/00_setup_env_arm.sh" + << : *CREDITS_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV QEMU_USER_CMD: "" # Disable qemu and run the test natively @@ -185,11 +186,11 @@ task: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: '32-bit + dash [CentOS 8]' + name: '32-bit + dash [CentOS 9]' << : *GLOBAL_TASK_TEMPLATE container: docker_arguments: - CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream8 + CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream9 FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" # For faster CI feedback, immediately schedule one task that runs all tests << : *CREDITS_TEMPLATE @@ -211,8 +212,8 @@ task: name: '[TSan, depends] [lunar]' << : *GLOBAL_TASK_TEMPLATE container: - cpu: 6 # Increase CPU and Memory to avoid timeout - memory: 24G + cpu: 4 + memory: 16G # The default memory is too small, so double everything docker_arguments: CI_IMAGE_NAME_TAG: ubuntu:lunar FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" @@ -266,7 +267,7 @@ task: << : *GLOBAL_TASK_TEMPLATE container: cpu: 4 - memory: 16G # The default memory is sometimes just a bit too small, so double everything + memory: 16G # The default memory is too small, so double everything docker_arguments: CI_IMAGE_NAME_TAG: ubuntu:focal FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" diff --git a/autogen.sh b/autogen.sh index de16260b563279..d0ac7ef7ed3f13 100755 --- a/autogen.sh +++ b/autogen.sh @@ -14,3 +14,12 @@ fi command -v autoreconf >/dev/null || \ (echo "configuration failed, please install autoconf first" && exit 1) autoreconf --install --force --warnings=all + +if expr "'$(build-aux/config.guess --timestamp)" \< "'$(depends/config.guess --timestamp)" > /dev/null; then + cp depends/config.guess build-aux + cp depends/config.guess src/secp256k1/build-aux +fi +if expr "'$(build-aux/config.sub --timestamp)" \< "'$(depends/config.sub --timestamp)" > /dev/null; then + cp depends/config.sub build-aux + cp depends/config.sub src/secp256k1/build-aux +fi diff --git a/build-aux/m4/l_filesystem.m4 b/build-aux/m4/l_filesystem.m4 deleted file mode 100644 index ca3a0cd41c061b..00000000000000 --- a/build-aux/m4/l_filesystem.m4 +++ /dev/null @@ -1,47 +0,0 @@ -dnl Copyright (c) 2022 The Bitcoin Core developers -dnl Distributed under the MIT software license, see the accompanying -dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# GCC 8.1 and earlier requires -lstdc++fs -# Clang 8.0.0 (libc++) and earlier requires -lc++fs - -m4_define([_CHECK_FILESYSTEM_testbody], [[ - #include - - namespace fs = std::filesystem; - - int main() { - (void)fs::current_path().root_name(); - return 0; - } -]]) - -AC_DEFUN([CHECK_FILESYSTEM], [ - - AC_LANG_PUSH(C++) - - AC_MSG_CHECKING([whether std::filesystem can be used without link library]) - - AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no]) - SAVED_LIBS="$LIBS" - LIBS="$SAVED_LIBS -lstdc++fs" - AC_MSG_CHECKING([whether std::filesystem needs -lstdc++fs]) - AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no]) - AC_MSG_CHECKING([whether std::filesystem needs -lc++fs]) - LIBS="$SAVED_LIBS -lc++fs" - AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_FAILURE([cannot figure out how to use std::filesystem]) - ]) - ]) - ]) - - AC_LANG_POP -]) diff --git a/build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj.in b/build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj.in index 556e56936d61b0..d4e13880edf155 100644 --- a/build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj.in +++ b/build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj.in @@ -12,7 +12,7 @@ @SOURCE_FILES@ - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index 29a3b540c0c6ec..bb6cac65f1a243 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -2,7 +2,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.4.33213.308 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoinconsensus", "libbitcoinconsensus\libbitcoinconsensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_consensus", "libbitcoin_consensus\libbitcoin_consensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "navcoind", "navcoind\navcoind.vcxproj", "{D4513DDF-6013-44DC-ADCC-12EAF6D1F038}" EndProject diff --git a/build_msvc/bitcoin_config.h.in b/build_msvc/bitcoin_config.h.in index 02d8fc41c2a6a9..17166474866923 100644 --- a/build_msvc/bitcoin_config.h.in +++ b/build_msvc/bitcoin_config.h.in @@ -38,15 +38,12 @@ /* Define to 1 to enable SQLite wallet */ #define USE_SQLITE 1 -/* Define to 1 to enable ZMQ functions */ +/* Define this symbol to enable ZMQ functions */ #define ENABLE_ZMQ 1 /* define if external signer support is enabled (requires Boost::Process) */ #define ENABLE_EXTERNAL_SIGNER /**/ -/* Define this symbol if the consensus lib has been built */ -#define HAVE_CONSENSUS_LIB 1 - /* Define to 1 if you have the declaration of `be16toh', and to 0 if you don't. */ #define HAVE_DECL_BE16TOH 0 diff --git a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj similarity index 80% rename from build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj rename to build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj index e8c3234c4db69f..d8cb8f92984152 100644 --- a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj +++ b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj @@ -17,15 +17,6 @@ - - - - - - - - - diff --git a/build_msvc/libtest_util/libtest_util.vcxproj.in b/build_msvc/libtest_util/libtest_util.vcxproj.in index c4c41401a20080..69c387505008b6 100644 --- a/build_msvc/libtest_util/libtest_util.vcxproj.in +++ b/build_msvc/libtest_util/libtest_util.vcxproj.in @@ -14,6 +14,7 @@ {1DBB979A-C212-45CD-9563-446A96F87F71} + @SOURCE_FILES@ diff --git a/build_msvc/navcoin-tx/navcoin-tx.vcxproj b/build_msvc/navcoin-tx/navcoin-tx.vcxproj index 22e9cc10b7ade0..bc2a650ccd3b6c 100644 --- a/build_msvc/navcoin-tx/navcoin-tx.vcxproj +++ b/build_msvc/navcoin-tx/navcoin-tx.vcxproj @@ -12,7 +12,7 @@ - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} diff --git a/build_msvc/navcoin-util/navcoin-util.vcxproj b/build_msvc/navcoin-util/navcoin-util.vcxproj index 1bd5d9efcb8e3a..935188c2a51e13 100644 --- a/build_msvc/navcoin-util/navcoin-util.vcxproj +++ b/build_msvc/navcoin-util/navcoin-util.vcxproj @@ -12,7 +12,7 @@ - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} diff --git a/build_msvc/navcoin-wallet/navcoin-wallet.vcxproj b/build_msvc/navcoin-wallet/navcoin-wallet.vcxproj index d8cce6d7e94cf6..33f3a1228db904 100644 --- a/build_msvc/navcoin-wallet/navcoin-wallet.vcxproj +++ b/build_msvc/navcoin-wallet/navcoin-wallet.vcxproj @@ -15,7 +15,7 @@ - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} diff --git a/build_msvc/navcoind/navcoind.vcxproj b/build_msvc/navcoind/navcoind.vcxproj index ea0b48a44a2968..755dc5e19f7a7b 100644 --- a/build_msvc/navcoind/navcoind.vcxproj +++ b/build_msvc/navcoind/navcoind.vcxproj @@ -15,7 +15,7 @@ - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} diff --git a/build_msvc/test_navcoin/test_navcoin.vcxproj b/build_msvc/test_navcoin/test_navcoin.vcxproj index 7e0772bcb80204..998ec62ef82e44 100644 --- a/build_msvc/test_navcoin/test_navcoin.vcxproj +++ b/build_msvc/test_navcoin/test_navcoin.vcxproj @@ -22,7 +22,7 @@ {542007e3-be0d-4b0d-a6b0-aa8813e2558d} - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 4a54f47b03f5a5..d98f05ca6bca18 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -37,7 +37,6 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} export RUN_TIDY=${RUN_TIDY:-false} -export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # By how much to scale the test_runner timeouts (option --timeout-factor). # This is needed because some ci machines have slow CPU or disk, so sanitizers # might be slow or a reindex might be waiting on disk IO. diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index e800fc75b004d4..8bb04a5c6f0f31 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -8,12 +8,10 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream8" -# Use minimum supported python3.8 and gcc-8, see doc/dependencies.md -export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python38 python38-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" +export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream9" +export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux" export PIP_PACKAGES="pyzmq" export GOAL="install" -export NO_WERROR=1 # GCC 8 +export NO_WERROR=1 # Suppress error: #warning _FORTIFY_SOURCE > 2 is treated like 2 on this platform [-Werror=cpp] export BITCOIN_CONFIG="--enable-zmq --enable-reduce-exports" export CONFIG_SHELL="/bin/dash" -export TEST_RUNNER_ENV="LC_ALL=en_US.UTF-8" diff --git a/ci/test/00_setup_env_native.sh b/ci/test/00_setup_env_native.sh index a025ef6d540968..120926018eae2d 100755 --- a/ci/test/00_setup_env_native.sh +++ b/ci/test/00_setup_env_native.sh @@ -8,14 +8,14 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native export CI_IMAGE_NAME_TAG="ubuntu:20.04" -# Use minimum supported python3.8 and gcc-8 (or best-effort gcc-9), see doc/dependencies.md +# Use minimum supported python3.8 and gcc-9, see doc/dependencies.md export PACKAGES="gcc-9 g++-9 python3-zmq libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-9 CXX=g++-9" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export NO_WERROR=1 +export NO_WERROR=1 # -Werror=maybe-uninitialized export DOWNLOAD_PREVIOUS_RELEASES="true" export BITCOIN_CONFIG="--enable-zmq --with-libs=no --enable-reduce-exports \ --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index b86fa6f3212129..8e3c935c6fdd52 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -8,9 +8,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel export CI_IMAGE_NAME_TAG="ubuntu:20.04" -# Use minimum supported python3.8 and clang-8, see doc/dependencies.md -export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" -export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" +# Use minimum supported python3.8 and clang-10, see doc/dependencies.md +export PACKAGES="python3-zmq clang-10 llvm-10 libc++abi-10-dev libc++-10-dev" +export DEP_OPTS="NO_WALLET=1 CC=clang-10 CXX='clang++-10 -stdlib=libc++'" export GOAL="install" -export NO_WERROR=1 -export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" +export BITCOIN_CONFIG="--enable-reduce-exports --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 521bf82e0421b5..a9d4f8dfba30b2 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="ubuntu:lunar" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export CONTAINER_NAME=ci_native_tidy -export PACKAGES="clang-16 libclang-16-dev llvm-16-dev libomp-16-dev clang-tidy-16 bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libsqlite3-dev libdb++-dev" +export PACKAGES="clang-16 libclang-16-dev llvm-16-dev libomp-16-dev clang-tidy-16 jq bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 641ff964f33ab4..beb5aa92426c33 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -18,8 +18,8 @@ if [ -n "$DPKG_ADD_ARCH" ]; then fi if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - ${CI_RETRY_EXE} bash -c "dnf -y install epel-release" - ${CI_RETRY_EXE} bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" + bash -c "dnf -y install epel-release" + bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" elif [ "$CI_USE_APT_INSTALL" != "no" ]; then if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index a534b4cdadde29..e9c54139a70030 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -74,39 +74,3 @@ CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" || CI_EXEC git config --global --add safe.directory \"*\" CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" - -if [ "$CI_OS_NAME" == "macos" ]; then - top -l 1 -s 0 | awk ' /PhysMem/ {print}' - echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" -else - CI_EXEC free -m -h - CI_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) - CI_EXEC echo "$(lscpu | grep Endian)" -fi -CI_EXEC echo "Free disk space:" -CI_EXEC df -h - -if [ "$RUN_FUZZ_TESTS" = "true" ]; then - export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ - if [ ! -d "$DIR_FUZZ_IN" ]; then - CI_EXEC git clone --depth=1 https://github.com/navcoin/qa-assets "${DIR_QA_ASSETS}" - fi -elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ - if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then - CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA" - CI_EXEC curl --location --fail https://github.com/navcoin/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" - fi -fi - -CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" - -if [ "$USE_BUSY_BOX" = "true" ]; then - echo "Setup to use BusyBox utils" - # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) - # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) - # shellcheck disable=SC1010 - CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done - # Print BusyBox version - CI_EXEC patch --help -fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh deleted file mode 100755 index 199cdd64a7c6c0..00000000000000 --- a/ci/test/05_before_script.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -# Make sure default datadir does not exist and is never read by creating a dummy file -if [ "$CI_OS_NAME" == "macos" ]; then - echo > "${HOME}/Library/Application Support/Bitcoin" -else - CI_EXEC echo \> \$HOME/.bitcoin -fi - -if [ -z "$NO_DEPENDS" ]; then - if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - # CentOS has problems building the depends if the config shell is not explicitly set - # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to - # an error as the first command is executed) - SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/dash" - else - SHELL_OPTS="CONFIG_SHELL=" - fi - CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1 -fi -if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then - CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" -fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh deleted file mode 100755 index ed6b04807b4147..00000000000000 --- a/ci/test/06_script_a.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" -if [ -z "$NO_DEPENDS" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" -fi -if [ -z "$NO_WERROR" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" -fi - -CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" -PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" - -if [ -n "$ANDROID_TOOLS_URL" ]; then - CI_EXEC make distclean || true - CI_EXEC ./autogen.sh - CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - CI_EXEC "make $MAKEJOBS" - CI_EXEC "${PRINT_CCACHE_STATISTICS}" - exit 0 -fi - -BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" - -if [ -n "$CONFIG_SHELL" ]; then - CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" -else - CI_EXEC ./autogen.sh -fi - -CI_EXEC mkdir -p "${BASE_BUILD_DIR}" -export P_CI_DIR="${BASE_BUILD_DIR}" - -CI_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - -CI_EXEC make distdir VERSION="$HOST" - -export P_CI_DIR="${BASE_BUILD_DIR}/navcoin-$HOST" - -CI_EXEC ./configure --cache-file=../config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - -set -o errtrace -trap 'CI_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR - -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - # MemorySanitizer (MSAN) does not support tracking memory initialization done by - # using the Linux getrandom syscall. Avoid using getrandom by undefining - # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for - # details. - CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' -fi - -if [[ "${RUN_TIDY}" == "true" ]]; then - MAYBE_BEAR="bear --config src/.bear-tidy-config" - MAYBE_TOKEN="--" -fi - -CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) - -CI_EXEC "${PRINT_CCACHE_STATISTICS}" -CI_EXEC du -sh "${DEPENDS_DIR}"/*/ -CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 6326dfb9d6ddbc..035a844f2a4186 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -8,6 +8,115 @@ export LC_ALL=C.UTF-8 set -ex +if [ "$CI_OS_NAME" == "macos" ]; then + top -l 1 -s 0 | awk ' /PhysMem/ {print}' + echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" +else + free -m -h + echo "Number of CPUs (nproc): $(nproc)" + lscpu | grep Endian +fi +echo "Free disk space:" +df -h + +if [ "$RUN_FUZZ_TESTS" = "true" ]; then + export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + if [ ! -d "$DIR_FUZZ_IN" ]; then + git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}" + fi + ( + cd "${DIR_QA_ASSETS}" + echo "Using qa-assets repo from commit ..." + git log -1 + ) +elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ + if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then + mkdir -p "$DIR_UNIT_TEST_DATA" + curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" + fi +fi + +mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" + +if [ "$USE_BUSY_BOX" = "true" ]; then + echo "Setup to use BusyBox utils" + # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) + # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) + for util in $(busybox --list | grep -v "^ar$" | grep -v "^tar$" ); do ln -s "$(command -v busybox)" "${BINS_SCRATCH_DIR}/$util"; done + # Print BusyBox version + patch --help +fi + +# Make sure default datadir does not exist and is never read by creating a dummy file +if [ "$CI_OS_NAME" == "macos" ]; then + echo > "${HOME}/Library/Application Support/Bitcoin" +else + echo > "${HOME}/.bitcoin" +fi + +if [ -z "$NO_DEPENDS" ]; then + if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then + SHELL_OPTS="CONFIG_SHELL=/bin/dash" + else + SHELL_OPTS="CONFIG_SHELL=" + fi + bash -c "$SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS LOG=1" +fi +if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then + test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" +fi + +BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" +if [ -z "$NO_DEPENDS" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" +fi +if [ -z "$NO_WERROR" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" +fi + +ccache --zero-stats --max-size="${CCACHE_SIZE}" +PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" + +if [ -n "$ANDROID_TOOLS_URL" ]; then + make distclean || true + ./autogen.sh + bash -c "./configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) + make "${MAKEJOBS}" && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk + bash -c "${PRINT_CCACHE_STATISTICS}" + exit 0 +fi + +BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" + +if [ -n "$CONFIG_SHELL" ]; then + "$CONFIG_SHELL" -c "./autogen.sh" +else + ./autogen.sh +fi + +mkdir -p "${BASE_BUILD_DIR}" +cd "${BASE_BUILD_DIR}" + +bash -c "${BASE_ROOT_DIR}/configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) + +make distdir VERSION="$HOST" + +cd "${BASE_BUILD_DIR}/bitcoin-$HOST" + +bash -c "./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) + +if [[ "${RUN_TIDY}" == "true" ]]; then + MAYBE_BEAR="bear --config src/.bear-tidy-config" + MAYBE_TOKEN="--" +fi + +bash -c "${MAYBE_BEAR} ${MAYBE_TOKEN} make $MAKEJOBS $GOAL" || ( echo "Build failure. Verbose build follows." && make "$GOAL" V=1 ; false ) + +bash -c "${PRINT_CCACHE_STATISTICS}" +du -sh "${DEPENDS_DIR}"/*/ +du -sh "${PREVIOUS_RELEASES_DIR}" + if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 @@ -42,62 +151,25 @@ if [ "${RUN_TIDY}" = "true" ]; then set -eo pipefail cd "${BASE_BUILD_DIR}/navcoin-$HOST/src/" ( run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error" + # Filter out files by regex here, because regex may not be + # accepted in src/.bear-tidy-config + # Filter out: + # * qt qrc and moc generated files + # * walletutil (temporarily) + # * secp256k1 + jq 'map(select(.file | test("src/qt/qrc_.*\\.cpp$|/moc_.*\\.cpp$|src/wallet/walletutil|src/secp256k1/src/") | not))' ../compile_commands.json > tmp.json + mv tmp.json ../compile_commands.json cd "${BASE_BUILD_DIR}/navcoin-$HOST/" python3 "${DIR_IWYU}/include-what-you-use/iwyu_tool.py" \ - src/common/args.cpp \ - src/common/config.cpp \ - src/common/init.cpp \ - src/common/url.cpp \ - src/compat \ - src/dbwrapper.cpp \ - src/init \ - src/kernel \ - src/node/chainstate.cpp \ - src/node/chainstatemanager_args.cpp \ - src/node/mempool_args.cpp \ - src/node/minisketchwrapper.cpp \ - src/node/utxo_snapshot.cpp \ - src/node/validation_cache_args.cpp \ - src/policy/feerate.cpp \ - src/policy/packages.cpp \ - src/policy/settings.cpp \ - src/primitives/transaction.cpp \ - src/random.cpp \ - src/rpc/fees.cpp \ - src/rpc/signmessage.cpp \ - src/test/fuzz/string.cpp \ - src/test/fuzz/txorphan.cpp \ - src/test/fuzz/util \ - src/test/util/coins.cpp \ - src/uint256.cpp \ - src/util/bip32.cpp \ - src/util/bytevectorhash.cpp \ - src/util/check.cpp \ - src/util/error.cpp \ - src/util/exception.cpp \ - src/util/getuniquepath.cpp \ - src/util/hasher.cpp \ - src/util/message.cpp \ - src/util/moneystr.cpp \ - src/util/serfloat.cpp \ - src/util/spanparsing.cpp \ - src/util/strencodings.cpp \ - src/util/string.cpp \ - src/util/syserror.cpp \ - src/util/threadinterrupt.cpp \ - src/zmq \ -p . "${MAKEJOBS}" \ -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_BUILD_DIR}/navcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" \ + -Xiwyu --max_line_length=160 \ 2>&1 | tee /tmp/iwyu_ci.out cd "${BASE_ROOT_DIR}/src" python3 "${DIR_IWYU}/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out git --no-pager diff fi -if [ "$RUN_SECURITY_TESTS" = "true" ]; then - make test-security-check -fi - if [ "$RUN_FUZZ_TESTS" = "true" ]; then bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN}" fi diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh index 751a4056df7bcf..2284a2903b7c0a 100755 --- a/ci/test_run_all.sh +++ b/ci/test_run_all.sh @@ -8,8 +8,6 @@ export LC_ALL=C.UTF-8 set -o errexit; source ./ci/test/00_setup_env.sh set -o errexit; source ./ci/test/04_install.sh -set -o errexit; source ./ci/test/05_before_script.sh -set -o errexit; source ./ci/test/06_script_a.sh set -o errexit CI_EXEC "${BASE_ROOT_DIR}/ci/test/06_script_b.sh" diff --git a/configure.ac b/configure.ac index 432eccc389f329..98992c08260a1b 100644 --- a/configure.ac +++ b/configure.ac @@ -68,11 +68,12 @@ else fi AC_PROG_CXX -dnl By default, libtool for mingw refuses to link static libs into a dll for -dnl fear of mixing pic/non-pic objects, and import/export complications. Since -dnl we have those under control, re-enable that functionality. +dnl libtool overrides case $host in *mingw*) + dnl By default, libtool for mingw refuses to link static libs into a dll for + dnl fear of mixing pic/non-pic objects, and import/export complications. Since + dnl we have those under control, re-enable that functionality. lt_cv_deplibs_check_method="pass_all" dnl Remove unwanted -DDLL_EXPORT from these variables. @@ -81,6 +82,16 @@ case $host in lt_cv_prog_compiler_pic="-DPIC" lt_cv_prog_compiler_pic_CXX="-DPIC" ;; + *darwin*) + dnl Because it prints a verbose warning, lld fails the following check + dnl for "-Wl,-single_module" from libtool.m4: + dnl # If there is a non-empty error log, and "single_module" + dnl # appears in it, assume the flag caused a linker warning + dnl "-single_module" works fine on ld64 and lld, so just bypass the test. + dnl Failure to set this to "yes" causes libtool to use a very broken + dnl link-line for shared libs. + lt_cv_apple_cc_single_mod="yes" + ;; esac AC_ARG_WITH([seccomp], @@ -102,9 +113,6 @@ else AX_CXX_COMPILE_STDCXX([20], [noext], [mandatory]) fi -dnl check if additional link flags are required for std::filesystem -CHECK_FILESYSTEM - dnl Unless the user specified OBJCXX, force it to be the same as CXX. This ensures dnl that we get the same -std flags for both. m4_ifdef([AC_PROG_OBJCXX],[ @@ -1051,7 +1059,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], dnl Check for posix_fallocate AC_MSG_CHECKING([for posix_fallocate]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - // same as in src/util/system.cpp + // same as in src/util/fs_helpers.cpp #ifdef __linux__ #ifdef _POSIX_C_SOURCE #undef _POSIX_C_SOURCE @@ -1149,17 +1157,16 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], ) dnl Check for different ways of gathering OS randomness -AC_MSG_CHECKING([for Linux getrandom syscall]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include - #include - #include ]], - [[ syscall(SYS_getrandom, nullptr, 32, 0); ]])], - [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYS_GETRANDOM], [1], [Define this symbol if the Linux getrandom system call is available]) ], +AC_MSG_CHECKING([for Linux getrandom function]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include ]], + [[ getrandom(nullptr, 32, 0); ]])], + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETRANDOM], [1], [Define this symbol if the Linux getrandom function call is available]) ], [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING([for getentropy via random.h]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include +AC_MSG_CHECKING([for getentropy via sys/random.h]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include ]], [[ getentropy(nullptr, 32) ]])], [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETENTROPY_RAND], [1], [Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], @@ -1372,7 +1379,9 @@ if test "$use_usdt" != "no"; then AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [#include ], - [DTRACE_PROBE("context", "event");] + [DTRACE_PROBE(context, event); + int a, b, c, d, e, f, g; + DTRACE_PROBE7(context, event, a, b, c, d, e, f, g);] )], [AC_MSG_RESULT([yes]); AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable tracepoints for Userspace, Statically Defined Tracing])], [AC_MSG_RESULT([no]); use_usdt=no;] @@ -1587,12 +1596,9 @@ dnl ZMQ check if test "$use_zmq" = "yes"; then PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], - AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), - [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) - AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) + AC_DEFINE([ENABLE_ZMQ], [1], [Define this symbol to enable ZMQ functions]), + [AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) use_zmq=no]) -else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) fi if test "$use_zmq" = "yes"; then @@ -1604,6 +1610,8 @@ if test "$use_zmq" = "yes"; then esac fi +AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) + dnl libmultiprocess library check libmultiprocess_found=no @@ -1759,7 +1767,43 @@ else fi fi -AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) +dnl these are only used when qt is enabled +BUILD_TEST_QT="" +if test "$bitcoin_enable_qt" != "no"; then + dnl enable dbus support + AC_MSG_CHECKING([whether to build GUI with support for D-Bus]) + if test "$bitcoin_enable_qt_dbus" != "no"; then + AC_DEFINE([USE_DBUS], [1], [Define if dbus support should be compiled in]) + fi + AC_MSG_RESULT([$bitcoin_enable_qt_dbus]) + + dnl enable qr support + AC_MSG_CHECKING([whether to build GUI with support for QR codes]) + if test "$have_qrencode" = "no"; then + if test "$use_qr" = "yes"; then + AC_MSG_ERROR([QR support requested but cannot be built. Use --without-qrencode]) + fi + use_qr=no + else + if test "$use_qr" != "no"; then + AC_DEFINE([USE_QRCODE], [1], [Define if QR support should be compiled in]) + use_qr=yes + fi + fi + AC_MSG_RESULT([$use_qr]) + + if test "$XGETTEXT" = ""; then + AC_MSG_WARN([xgettext is required to update qt translations]) + fi + + AC_MSG_CHECKING([whether to build test_bitcoin-qt]) + if test "$use_gui_tests$bitcoin_enable_qt_test" = "yesyes"; then + AC_MSG_RESULT([yes]) + BUILD_TEST_QT="yes" + else + AC_MSG_RESULT([no]) + fi +fi AC_MSG_CHECKING([whether to build test_navcoin]) if test "$use_tests" = "yes"; then diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index d83ff08713d50d..54ddd473532fb4 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -559,9 +559,7 @@ inspecting signatures in Mach-O binaries.") (sha256 (base32 "0azpb9cvnbv25zg8019rqz48h8i2257ngyjg566dlnp74ivrs9vq")) - (patches (search-our-patches "glibc-ldd-x86_64.patch" - "glibc-versioned-locpath.patch" - "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" + (patches (search-our-patches "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" "glibc-2.27-fcommon.patch" "glibc-2.27-guix-prefix.patch")))))) diff --git a/contrib/guix/patches/gcc-broken-longjmp.patch b/contrib/guix/patches/gcc-broken-longjmp.patch index 1cfc0918b090a1..56568813c0d834 100644 --- a/contrib/guix/patches/gcc-broken-longjmp.patch +++ b/contrib/guix/patches/gcc-broken-longjmp.patch @@ -29,6 +29,8 @@ Date: Wed May 5 22:48:51 2021 +0200 gcc/testsuite/ * gcc.c-torture/execute/20210505-1.c: New test. + This patch can be dropped when we are building with GCC 10.4.0 or later. + diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c index 2f838840e96..06ad1b2274e 100644 --- a/gcc/config/i386/i386.c diff --git a/contrib/guix/patches/glibc-2.27-fcommon.patch b/contrib/guix/patches/glibc-2.27-fcommon.patch index f3baacab98eed9..817aa85bb95fb3 100644 --- a/contrib/guix/patches/glibc-2.27-fcommon.patch +++ b/contrib/guix/patches/glibc-2.27-fcommon.patch @@ -5,7 +5,7 @@ Date: Fri May 6 11:03:04 2022 +0100 build: use -fcommon to retain legacy behaviour with GCC 10 GCC 10 started using -fno-common by default, which causes issues with - the powerpc builds using gibc 2.24. A patch was commited to glibc to fix + the powerpc builds using gibc 2.27. A patch was commited to glibc to fix the issue, 18363b4f010da9ba459b13310b113ac0647c2fcc but is non-trvial to backport, and was broken in at least one way, see the followup in commit 7650321ce037302bfc2f026aa19e0213b8d02fe6. @@ -17,6 +17,8 @@ Date: Fri May 6 11:03:04 2022 +0100 https://sourceware.org/git/?p=glibc.git;a=commit;h=18363b4f010da9ba459b13310b113ac0647c2fcc https://sourceware.org/git/?p=glibc.git;a=commit;h=7650321ce037302bfc2f026aa19e0213b8d02fe6 + This patch can be dropped when we are building with glibc 2.31+. + diff --git a/Makeconfig b/Makeconfig index 86a71e5802..aa2166be60 100644 --- a/Makeconfig diff --git a/contrib/guix/patches/glibc-2.27-guix-prefix.patch b/contrib/guix/patches/glibc-2.27-guix-prefix.patch index 6648bc6c053dfe..dc515907ff887d 100644 --- a/contrib/guix/patches/glibc-2.27-guix-prefix.patch +++ b/contrib/guix/patches/glibc-2.27-guix-prefix.patch @@ -5,7 +5,7 @@ In order to be reproducible regardless of the architecture used to build the package, map all guix store prefixes to something fixed, e.g. /usr. We might be able to drop this in favour of using --with-nonshared-cflags -when we being using newer versions of glibc. +when we begin using newer versions of glibc. --- a/Makeconfig +++ b/Makeconfig diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch index c0f8495c41de11..ab8ae9c023231f 100644 --- a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch @@ -4,6 +4,8 @@ See also: http://lists.busybox.net/pipermail/buildroot/2020-July/590376.html. https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e +This patch can be dropped when we are building with glibc 2.28+. + From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 From: "H.J. Lu" Date: Mon, 31 Dec 2018 09:26:42 -0800 diff --git a/contrib/guix/patches/glibc-ldd-x86_64.patch b/contrib/guix/patches/glibc-ldd-x86_64.patch deleted file mode 100644 index a23b095caa7509..00000000000000 --- a/contrib/guix/patches/glibc-ldd-x86_64.patch +++ /dev/null @@ -1,10 +0,0 @@ -By default, 'RTDLLIST' in 'ldd' refers to 'lib64/ld-linux-x86-64.so', whereas -it's in 'lib/' for us. This patch fixes that. - ---- a/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed -+++ b/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed -@@ -1,3 +1,3 @@ - /LD_TRACE_LOADED_OBJECTS=1/a\ - add_env="$add_env LD_LIBRARY_VERSION=\\$verify_out" --s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \264\4-x86-64\6 \2x32\4-x32\6"_ -+s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \2\4-x86-64\6 \2x32\4-x32\6"_ diff --git a/contrib/guix/patches/glibc-versioned-locpath.patch b/contrib/guix/patches/glibc-versioned-locpath.patch deleted file mode 100644 index bc7652127fa5a2..00000000000000 --- a/contrib/guix/patches/glibc-versioned-locpath.patch +++ /dev/null @@ -1,240 +0,0 @@ -The format of locale data can be incompatible between libc versions, and -loading incompatible data can lead to 'setlocale' returning EINVAL at best -or triggering an assertion failure at worst. See -https://lists.gnu.org/archive/html/guix-devel/2015-09/msg00717.html -for background information. - -To address that, this patch changes libc to honor a new 'GUIX_LOCPATH' -variable, and to look for locale data in version-specific sub-directories of -that variable. So, if GUIX_LOCPATH=/foo:/bar, locale data is searched for in -/foo/X.Y and /bar/X.Y, where X.Y is the libc version number. - -That way, a single 'GUIX_LOCPATH' setting can work even if different libc -versions coexist on the system. - ---- a/locale/newlocale.c -+++ b/locale/newlocale.c -@@ -30,6 +30,7 @@ - /* Lock for protecting global data. */ - __libc_rwlock_define (extern , __libc_setlocale_lock attribute_hidden) - -+extern error_t compute_locale_search_path (char **, size_t *); - - /* Use this when we come along an error. */ - #define ERROR_RETURN \ -@@ -48,7 +49,6 @@ __newlocale (int category_mask, const char *locale, __locale_t base) - __locale_t result_ptr; - char *locale_path; - size_t locale_path_len; -- const char *locpath_var; - int cnt; - size_t names_len; - -@@ -102,17 +102,8 @@ __newlocale (int category_mask, const char *locale, __locale_t base) - locale_path = NULL; - locale_path_len = 0; - -- locpath_var = getenv ("LOCPATH"); -- if (locpath_var != NULL && locpath_var[0] != '\0') -- { -- if (__argz_create_sep (locpath_var, ':', -- &locale_path, &locale_path_len) != 0) -- return NULL; -- -- if (__argz_add_sep (&locale_path, &locale_path_len, -- _nl_default_locale_path, ':') != 0) -- return NULL; -- } -+ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) -+ return NULL; - - /* Get the names for the locales we are interested in. We either - allow a composite name or a single name. */ -diff --git a/locale/setlocale.c b/locale/setlocale.c -index ead030d..0c0e314 100644 ---- a/locale/setlocale.c -+++ b/locale/setlocale.c -@@ -215,12 +215,65 @@ setdata (int category, struct __locale_data *data) - } - } - -+/* Return in *LOCALE_PATH and *LOCALE_PATH_LEN the locale data search path as -+ a colon-separated list. Return ENOMEN on error, zero otherwise. */ -+error_t -+compute_locale_search_path (char **locale_path, size_t *locale_path_len) -+{ -+ char* guix_locpath_var = getenv ("GUIX_LOCPATH"); -+ char *locpath_var = getenv ("LOCPATH"); -+ -+ if (guix_locpath_var != NULL && guix_locpath_var[0] != '\0') -+ { -+ /* Entries in 'GUIX_LOCPATH' take precedence over 'LOCPATH'. These -+ entries are systematically prefixed with "/X.Y" where "X.Y" is the -+ libc version. */ -+ if (__argz_create_sep (guix_locpath_var, ':', -+ locale_path, locale_path_len) != 0 -+ || __argz_suffix_entries (locale_path, locale_path_len, -+ "/" VERSION) != 0) -+ goto bail_out; -+ } -+ -+ if (locpath_var != NULL && locpath_var[0] != '\0') -+ { -+ char *reg_locale_path = NULL; -+ size_t reg_locale_path_len = 0; -+ -+ if (__argz_create_sep (locpath_var, ':', -+ ®_locale_path, ®_locale_path_len) != 0) -+ goto bail_out; -+ -+ if (__argz_append (locale_path, locale_path_len, -+ reg_locale_path, reg_locale_path_len) != 0) -+ goto bail_out; -+ -+ free (reg_locale_path); -+ } -+ -+ if (*locale_path != NULL) -+ { -+ /* Append the system default locale directory. */ -+ if (__argz_add_sep (locale_path, locale_path_len, -+ _nl_default_locale_path, ':') != 0) -+ goto bail_out; -+ } -+ -+ return 0; -+ -+ bail_out: -+ free (*locale_path); -+ *locale_path = NULL; -+ *locale_path_len = 0; -+ -+ return ENOMEM; -+} -+ - char * - setlocale (int category, const char *locale) - { - char *locale_path; - size_t locale_path_len; -- const char *locpath_var; - char *composite; - - /* Sanity check for CATEGORY argument. */ -@@ -251,17 +304,10 @@ setlocale (int category, const char *locale) - locale_path = NULL; - locale_path_len = 0; - -- locpath_var = getenv ("LOCPATH"); -- if (locpath_var != NULL && locpath_var[0] != '\0') -+ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) - { -- if (__argz_create_sep (locpath_var, ':', -- &locale_path, &locale_path_len) != 0 -- || __argz_add_sep (&locale_path, &locale_path_len, -- _nl_default_locale_path, ':') != 0) -- { -- __libc_rwlock_unlock (__libc_setlocale_lock); -- return NULL; -- } -+ __libc_rwlock_unlock (__libc_setlocale_lock); -+ return NULL; - } - - if (category == LC_ALL) -diff --git a/string/Makefile b/string/Makefile -index 8424a61..f925503 100644 ---- a/string/Makefile -+++ b/string/Makefile -@@ -38,7 +38,7 @@ routines := strcat strchr strcmp strcoll strcpy strcspn \ - swab strfry memfrob memmem rawmemchr strchrnul \ - $(addprefix argz-,append count create ctsep next \ - delete extract insert stringify \ -- addsep replace) \ -+ addsep replace suffix) \ - envz basename \ - strcoll_l strxfrm_l string-inlines memrchr \ - xpg-strerror strerror_l -diff --git a/string/argz-suffix.c b/string/argz-suffix.c -new file mode 100644 -index 0000000..505b0f2 ---- /dev/null -+++ b/string/argz-suffix.c -@@ -0,0 +1,56 @@ -+/* Copyright (C) 2015 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ Contributed by Ludovic Courtès . -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ . */ -+ -+#include -+#include -+#include -+#include -+ -+ -+error_t -+__argz_suffix_entries (char **argz, size_t *argz_len, const char *suffix) -+ -+{ -+ size_t suffix_len = strlen (suffix); -+ size_t count = __argz_count (*argz, *argz_len); -+ size_t new_argz_len = *argz_len + count * suffix_len; -+ char *new_argz = malloc (new_argz_len); -+ -+ if (new_argz) -+ { -+ char *p = new_argz, *entry; -+ -+ for (entry = *argz; -+ entry != NULL; -+ entry = argz_next (*argz, *argz_len, entry)) -+ { -+ p = stpcpy (p, entry); -+ p = stpcpy (p, suffix); -+ p++; -+ } -+ -+ free (*argz); -+ *argz = new_argz; -+ *argz_len = new_argz_len; -+ -+ return 0; -+ } -+ else -+ return ENOMEM; -+} -+weak_alias (__argz_suffix_entries, argz_suffix_entries) -diff --git a/string/argz.h b/string/argz.h -index bb62a31..d276a35 100644 ---- a/string/argz.h -+++ b/string/argz.h -@@ -134,6 +134,16 @@ extern error_t argz_replace (char **__restrict __argz, - const char *__restrict __str, - const char *__restrict __with, - unsigned int *__restrict __replace_count); -+ -+/* Suffix each entry of ARGZ & ARGZ_LEN with SUFFIX. Return 0 on success, -+ and ENOMEN if memory cannot be allocated. */ -+extern error_t __argz_suffix_entries (char **__restrict __argz, -+ size_t *__restrict __argz_len, -+ const char *__restrict __suffix); -+extern error_t argz_suffix_entries (char **__restrict __argz, -+ size_t *__restrict __argz_len, -+ const char *__restrict __suffix); -+ - - /* Returns the next entry in ARGZ & ARGZ_LEN after ENTRY, or NULL if there - are no more. If entry is NULL, then the first entry is returned. This diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch index 072f76eafd38f0..7976b864af0f29 100644 --- a/contrib/guix/patches/vmov-alignment.patch +++ b/contrib/guix/patches/vmov-alignment.patch @@ -1,6 +1,7 @@ Description: Use unaligned VMOV instructions Author: Stephen Kitt Bug-Debian: https://bugs.debian.org/939559 +See also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412 Based on a patch originally by Claude Heiland-Allen diff --git a/depends/README.md b/depends/README.md index f7c0fc0a5fc3bb..e5188594c651ea 100644 --- a/depends/README.md +++ b/depends/README.md @@ -15,6 +15,7 @@ For example: **Bitcoin Core's `configure` script by default will ignore the depends output.** In order for it to pick up libraries, tools, and settings from the depends build, you must set the `CONFIG_SITE` environment variable to point to a `config.site` settings file. +Make sure that `CONFIG_SITE` is an absolute path. In the above example, a file named `depends/x86_64-w64-mingw32/share/config.site` will be created. To use it during compilation: diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 522a6b17efc159..08d81cbc997370 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -63,7 +63,7 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$( # Explicitly point to our binaries (e.g. cctools) so that they are # ensured to be found and preferred over other possibilities. # -# -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 +# -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 # # Forces clang to use the libc++ headers from our SDK and completely # forget about the libc++ headers from the standard directories @@ -107,7 +107,6 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ -isysroot$(OSX_SDK) \ - -stdlib=libc++ \ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \ -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk index f2712294ab20ea..b11037b83ee807 100644 --- a/depends/packages/native_clang.mk +++ b/depends/packages/native_clang.mk @@ -9,10 +9,6 @@ $(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16 $(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231 endif -define $(package)_preprocess_cmds - rm -f $($(package)_extract_dir)/lib/libc++abi.so* -endef - define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin && \ diff --git a/doc/bips.md b/doc/bips.md index 1d5c91b8bdb326..94213f80482ace 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,4 +1,4 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v24.0**): +BIPs that are implemented by Bitcoin Core: * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1** ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575)) * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). diff --git a/doc/build-unix.md b/doc/build-unix.md index a29250141e9088..82e794b88c45ed 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -4,16 +4,6 @@ Some notes on how to build Bitcoin Core in Unix. (For BSD specific instructions, see `build-*bsd.md` in this directory.) -Note ---------------------- -Always use absolute paths to configure and compile Bitcoin Core and the dependencies. -For example, when specifying the path of the dependency: - - ../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX - -Here BDB_PREFIX must be an absolute path - it is defined using $(pwd) which ensures -the usage of the absolute path. - To Build --------------------- @@ -24,12 +14,11 @@ make # use "-j N" for N parallel jobs make install # optional ``` -This will build navcoin-qt as well, if the dependencies are met. - -See [dependencies.md](dependencies.md) for a complete overview. +See below for instructions on how to [install the dependencies on popular Linux +distributions](#linux-distribution-specific-instructions), or the +[dependencies](#dependencies) section for a complete overview. -Memory Requirements --------------------- +## Memory Requirements C++ compilers are memory-hungry. It is recommended to have at least 1.5 GB of memory available when compiling Bitcoin Core. On systems with less, gcc can be @@ -57,7 +46,7 @@ Build requirements: sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils python3 -Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: +Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: sudo apt-get install libevent-dev libboost-dev @@ -65,7 +54,7 @@ SQLite is required for the descriptor wallet: sudo apt install libsqlite3-dev -Berkeley DB is required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, +Berkeley DB is only required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, but these will install Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). @@ -114,7 +103,7 @@ Build requirements: sudo dnf install gcc-c++ libtool make autoconf automake python3 -Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: +Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: sudo dnf install libevent-devel boost-devel @@ -126,7 +115,7 @@ Berkeley DB is required for the legacy wallet: sudo dnf install libdb4-devel libdb4-cxx-devel -Newer Fedora releases, since Fedora 33, have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install +Berkeley DB is only required for the legacy wallet. Newer Fedora releases have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on Berkeley DB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). @@ -166,27 +155,13 @@ libqrencode (optional) can be installed with: Once these are installed, they will be found by configure and a navcoin-qt executable will be built by default. -Notes ------ -The release is built with GCC and then "strip navcoind" to strip the debug -symbols, which reduces the executable size by about 90%. +## Dependencies -miniupnpc ---------- +See [dependencies.md](dependencies.md) for a complete overview, and +[depends](/depends/README.md) on how to compile them yourself, if you wish to +not use the packages of your Linux distribution. -[miniupnpc](https://miniupnp.tuxfamily.org) may be used for UPnP port mapping. It can be downloaded from [here]( -https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and -turned off by default. - -libnatpmp ---------- - -[libnatpmp](https://miniupnp.tuxfamily.org/libnatpmp.html) may be used for NAT-PMP port mapping. It can be downloaded -from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and -turned off by default. - -Berkeley DB ------------ +### Berkeley DB The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is recommended to use Berkeley DB 4.8. If you have to build it yourself, and don't @@ -205,53 +180,9 @@ export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-pc-linux-gnu" BDB_CFLAGS="-I${BDB_PREFIX}/include" ``` -**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). - -Security --------- -To help make your Bitcoin Core installation more secure by making certain attacks impossible to -exploit even if a vulnerability is found, binaries are hardened by default. -This can be disabled with: - -Hardening Flags: +**Note**: Make sure that `BDB_PREFIX` is an absolute path. - ./configure --enable-hardening - ./configure --disable-hardening - - -Hardening enables the following features: -* _Position Independent Executable_: Build position independent code to take advantage of Address Space Layout Randomization - offered by some kernels. Attackers who can cause execution of code at an arbitrary memory - location are thwarted if they don't know where anything useful is located. - The stack and heap are randomly located by default, but this allows the code section to be - randomly located as well. - - On an AMD64 processor where a library was not compiled with -fPIC, this will cause an error - such as: "relocation R_X86_64_32 against `......' can not be used when making a shared object;" - - To test that you have built PIE executable, install scanelf, part of paxutils, and use: - - scanelf -e ./bitcoin - - The output should contain: - - TYPE - ET_DYN - -* _Non-executable Stack_: If the stack is executable then trivial stack-based buffer overflow exploits are possible if - vulnerable buffers are found. By default, Bitcoin Core should be built with a non-executable stack, - but if one of the libraries it uses asks for an executable stack or someone makes a mistake - and uses a compiler extension which requires an executable stack, it will silently build an - executable without the non-executable stack protection. - - To verify that the stack is non-executable after compiling use: - `scanelf -e ./bitcoin` - - The output should contain: - STK/REL/PTL - RW- R-- RW- - - The STK RW- means that the stack is readable and writeable but not executable. +**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). Disable-wallet mode -------------------- diff --git a/doc/dependencies.md b/doc/dependencies.md index c8417f49e6ed58..804f796abe1fcd 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -8,8 +8,8 @@ You can find installation instructions in the `build-*.md` file for your platfor | --- | --- | | [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | | [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | -| [Clang](https://clang.llvm.org) | [8.0](https://github.com/bitcoin/bitcoin/pull/24164) | -| [GCC](https://gcc.gnu.org) | [8.1](https://github.com/bitcoin/bitcoin/pull/23060) | +| [Clang](https://clang.llvm.org) | [10.0](https://github.com/bitcoin/bitcoin/pull/27682) | +| [GCC](https://gcc.gnu.org) | [9.1](https://github.com/bitcoin/bitcoin/pull/27662) | | [Python](https://www.python.org) (scripts, tests) | [3.8](https://github.com/bitcoin/bitcoin/pull/27483) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | @@ -20,7 +20,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.81.0](https://github.com/bitcoin/bitcoin/pull/26557) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | [libevent](../depends/packages/libevent.mk) | [link](https://github.com/libevent/libevent/releases) | [2.1.12-stable](https://github.com/bitcoin/bitcoin/pull/21991) | [2.1.8](https://github.com/bitcoin/bitcoin/pull/24681) | No | | glibc | [link](https://www.gnu.org/software/libc/) | N/A | [2.27](https://github.com/bitcoin/bitcoin/pull/27029) | Yes | -| Linux Kernel | [link](https://www.kernel.org/) | N/A | 3.2.0 | Yes | +| Linux Kernel | [link](https://www.kernel.org/) | N/A | [3.17.0](https://github.com/bitcoin/bitcoin/pull/27699) | Yes | ## Optional diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 348ea52120358d..13f050db7eecd0 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -43,7 +43,7 @@ threads take up 8MiB for the thread stack on a 64-bit system, and 4MiB in a ## Linux specific -By default, since glibc `2.10`, the C library will create up to two heap arenas per core. This is known to cause excessive memory usage in some scenarios. To avoid this make a script that sets `MALLOC_ARENA_MAX` before starting navcoind: +By default, glibc will create up to two heap arenas per core. This is known to cause excessive memory usage in some scenarios. To avoid this make a script that sets `MALLOC_ARENA_MAX` before starting navcoind: ```bash #!/usr/bin/env bash diff --git a/doc/release-notes-27302.md b/doc/release-notes-27302.md new file mode 100644 index 00000000000000..e67a6c8b061e2d --- /dev/null +++ b/doc/release-notes-27302.md @@ -0,0 +1,4 @@ +Configuration +--- + +- `bitcoind` and `bitcoin-qt` will now raise an error on startup if a datadir that is being used contains a bitcoin.conf file that will be ignored, which can happen when a datadir= line is used in a bitcoin.conf file. The error message is just a diagnostic intended to prevent accidental misconfiguration, and it can be disabled to restore the previous behavior of using the datadir while ignoring the bitcoin.conf contained in it. diff --git a/doc/release-notes/release-notes-23.2.md b/doc/release-notes/release-notes-23.2.md new file mode 100644 index 00000000000000..2d6354ae5dfae5 --- /dev/null +++ b/doc/release-notes/release-notes-23.2.md @@ -0,0 +1,72 @@ +23.2 Release Notes +================== + +Bitcoin Core version 23.2 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### P2P + +- #26909 net: prevent peers.dat corruptions by only serializing once +- #27608 p2p: Avoid prematurely clearing download state for other peers +- #27610 Improve performance of p2p inv to send queues + +### Build system + +- #25436 build: suppress array-bounds errors in libxkbcommon +- #25763 bdb: disable Werror for format-security +- #26944 depends: fix systemtap download URL +- #27462 depends: fix compiling bdb with clang-16 on aarch64 + +### Miscellaneous + +- #25444 ci: macOS task imrovements +- #26388 ci: Use macos-ventura-xcode:14.1 image for "macOS native" task +- #26924 refactor: Add missing includes to fix gcc-13 compile error + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Anthony Towns +- Hennadii Stepanov +- MacroFake +- Martin Zumsande +- Michael Ford +- Suhas Daftuar + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). \ No newline at end of file diff --git a/doc/release-notes/release-notes-24.1.md b/doc/release-notes/release-notes-24.1.md new file mode 100644 index 00000000000000..f46434cc43f3d4 --- /dev/null +++ b/doc/release-notes/release-notes-24.1.md @@ -0,0 +1,99 @@ +24.1 Release Notes +================== + +Bitcoin Core version 24.1 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### P2P + +- #26878 I2P network optimizations +- #26909 net: prevent peers.dat corruptions by only serializing once +- #27608 p2p: Avoid prematurely clearing download state for other peers +- #27610 Improve performance of p2p inv to send queues + +### RPC and other APIs + +- #26515 rpc: Require NodeStateStats object in getpeerinfo +- #27279 doc: fix/improve warning helps in {create,load,unload,restore}wallet +- #27468 rest: avoid segfault for invalid URI + +### Build System + +- #26944 depends: fix systemtap download URL +- #27462 depends: fix compiling bdb with clang-16 on aarch64 + +### Wallet + +- #26595 wallet: be able to specify a wallet name and passphrase to migratewallet +- #26675 wallet: For feebump, ignore abandoned descendant spends +- #26679 wallet: Skip rescanning if wallet is more recent than tip +- #26761 wallet: fully migrate address book entries for watchonly/solvable wallets +- #27053 wallet: reuse change dest when re-creating TX with avoidpartialspends +- #27080 wallet: Zero out wallet master key upon locking so it doesn't persist in memory +- #27473 wallet: Properly handle "unknown" Address Type + +### GUI changes + +- gui#687 Load PSBTs using istreambuf_iterator rather than istream_iterator +- gui#704 Correctly limit overview transaction list + +### Miscellaneous + +- #26880 ci: replace Intel macOS CI job +- #26924 refactor: Add missing includes to fix gcc-13 compile error + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- Anthony Towns +- Hennadii Stepanov +- John Moffett +- Jon Atack +- Marco Falke +- Martin Zumsande +- Matthew Zipkin +- Michael Ford +- pablomartin4btc +- Sebastian Falbesoner +- Suhas Daftuar +- Thomas Nguyen +- Vasil Dimov + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). \ No newline at end of file diff --git a/doc/release-notes/release-notes-25.0.md b/doc/release-notes/release-notes-25.0.md new file mode 100644 index 00000000000000..919cb3b2f3444e --- /dev/null +++ b/doc/release-notes/release-notes-25.0.md @@ -0,0 +1,340 @@ +25.0 Release Notes +================== + +Bitcoin Core version 25.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +- Transactions of non-witness size 65 bytes and above are now allowed by mempool + and relay policy. This is to better reflect the actual afforded protections + against CVE-2017-12842 and open up additional use-cases of smaller transaction sizes. (#26265) + +New RPCs +-------- + +- The scanblocks RPC returns the relevant blockhashes from a set of descriptors by + scanning all blockfilters in the given range. It can be used in combination with + the getblockheader and rescanblockchain RPCs to achieve fast wallet rescans. Note + that this functionality can only be used if a compact block filter index + (-blockfilterindex=1) has been constructed by the node. (#23549) + +Updated RPCs +------------ + +- All JSON-RPC methods accept a new [named + parameter](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md#parameter-passing) called `args` that can + contain positional parameter values. This is a convenience to allow some + parameter values to be passed by name without having to name every value. The + python test framework and `bitcoin-cli` tool both take advantage of this, so + for example: + +```sh +bitcoin-cli -named createwallet wallet_name=mywallet load_on_startup=1 +``` + +Can now be shortened to: + +```sh +bitcoin-cli -named createwallet mywallet load_on_startup=1 +``` + +- The `verifychain` RPC will now return `false` if the checks didn't fail, + but couldn't be completed at the desired depth and level. This could be due + to missing data while pruning, due to an insufficient dbcache or due to + the node being shutdown before the call could finish. (#25574) + +- `sendrawtransaction` has a new, optional argument, `maxburnamount` with a default value of `0`. + Any transaction containing an unspendable output with a value greater than `maxburnamount` will + not be submitted. At present, the outputs deemed unspendable are those with scripts that begin + with an `OP_RETURN` code (known as 'datacarriers'), scripts that exceed the maximum script size, + and scripts that contain invalid opcodes. + +- The `testmempoolaccept` RPC now returns 2 additional results within the "fees" result: + "effective-feerate" is the feerate including fees and sizes of transactions validated together if + package validation was used, and also includes any modified fees from prioritisetransaction. The + "effective-includes" result lists the wtxids of transactions whose modified fees and sizes were used + in the effective-feerate (#26646). + +- `decodescript` may now infer a Miniscript descriptor under P2WSH context if it is not lacking + information. (#27037) + +- `finalizepsbt` is now able to finalize a transaction with inputs spending Miniscript-compatible + P2WSH scripts. (#24149) + +Changes to wallet related RPCs can be found in the Wallet section below. + +Build System +------------ + +- The `--enable-upnp-default` and `--enable-natpmp-default` options + have been removed. If you want to use port mapping, you can + configure it using a .conf file, or by passing the relevant + options at runtime. (#26896) + +Updated settings +---------------- + +- If the `-checkblocks` or `-checklevel` options are explicitly provided by the +user, but the verification checks cannot be completed due to an insufficient +dbcache, Bitcoin Core will now return an error at startup. (#25574) + +- Ports specified in `-port` and `-rpcport` options are now validated at startup. + Values that previously worked and were considered valid can now result in errors. (#22087) + +- Setting `-blocksonly` will now reduce the maximum mempool memory + to 5MB (users may still use `-maxmempool` to override). Previously, + the default 300MB would be used, leading to unexpected memory usage + for users running with `-blocksonly` expecting it to eliminate + mempool memory usage. + + As unused mempool memory is shared with dbcache, this also reduces + the dbcache size for users running with `-blocksonly`, potentially + impacting performance. +- Setting `-maxconnections=0` will now disable `-dnsseed` + and `-listen` (users may still set them to override). + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings +------------ + +- The `shutdownnotify` option is used to specify a command to execute synchronously +before Bitcoin Core has begun its shutdown sequence. (#23395) + + +Wallet +------ + +- The `minconf` option, which allows a user to specify the minimum number +of confirmations a UTXO being spent has, and the `maxconf` option, +which allows specifying the maximum number of confirmations, have been +added to the following RPCs in #25375: + - `fundrawtransaction` + - `send` + - `walletcreatefundedpsbt` + - `sendall` + +- Added a new `next_index` field in the response in `listdescriptors` to + have the same format as `importdescriptors` (#26194) + +- RPC `listunspent` now has a new argument `include_immature_coinbase` + to include coinbase UTXOs that don't meet the minimum spendability + depth requirement (which before were silently skipped). (#25730) + +- Rescans for descriptor wallets are now significantly faster if compact + block filters (BIP158) are available. Since those are not constructed + by default, the configuration option "-blockfilterindex=1" has to be + provided to take advantage of the optimization. This improves the + performance of the RPC calls `rescanblockchain`, `importdescriptors` + and `restorewallet`. (#25957) + +- RPC `unloadwallet` now fails if a rescan is in progress. (#26618) + +- Wallet passphrases may now contain null characters. + Prior to this change, only characters up to the first + null character were recognized and accepted. (#27068) + +- Address Purposes strings are now restricted to the currently known values of "send", + "receive", and "refund". Wallets that have unrecognized purpose strings will have + loading warnings, and the `listlabels` RPC will raise an error if an unrecognized purpose + is requested. (#27217) + +- In the `createwallet`, `loadwallet`, `unloadwallet`, and `restorewallet` RPCs, the + "warning" string field is deprecated in favor of a "warnings" field that + returns a JSON array of strings to better handle multiple warning messages and + for consistency with other wallet RPCs. The "warning" field will be fully + removed from these RPCs in v26. It can be temporarily re-enabled during the + deprecation period by launching bitcoind with the configuration option + `-deprecatedrpc=walletwarningfield`. (#27279) + +- Descriptor wallets can now spend coins sent to P2WSH Miniscript descriptors. (#24149) + +GUI changes +----------- + +- The "Mask values" is a persistent option now. (gui#701) +- The "Mask values" option affects the "Transaction" view now, in addition to the + "Overview" one. (gui#708) + +REST +---- + +- A new `/rest/deploymentinfo` endpoint has been added for fetching various + state info regarding deployments of consensus changes. (#25412) + +Binary verification +---- + +- The binary verification script has been updated. In previous releases it + would verify that the binaries had been signed with a single "release key". + In this release and moving forward it will verify that the binaries are + signed by a _threshold of trusted keys_. For more details and + examples, see: + https://github.com/bitcoin/bitcoin/blob/master/contrib/verify-binaries/README.md + (#27358) + +Low-level changes +================= + +RPC +--- + +- The JSON-RPC server now rejects requests where a parameter is specified multiple + times with the same name, instead of silently overwriting earlier parameter values + with later ones. (#26628) +- RPC `listsinceblock` now accepts an optional `label` argument + to fetch incoming transactions having the specified label. (#25934) +- Previously `setban`, `addpeeraddress`, `walletcreatefundedpsbt`, methods + allowed non-boolean and non-null values to be passed as boolean parameters. + Any string, number, array, or object value that was passed would be treated + as false. After this change, passing any value except `true`, `false`, or + `null` now triggers a JSON value is not of expected type error. (#26213) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 721217.xyz +- @RandyMcMillan +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- Aurèle Oulès +- Ben Woosley +- Bitcoin Hodler +- brunoerg +- Bushstar +- Carl Dong +- Chris Geihsler +- Cory Fields +- David Gumberg +- dergoegge +- Dhruv Mehta +- Dimitris Tsapakidis +- dougEfish +- Douglas Chimento +- ekzyis +- Elichai Turkel +- Ethan Heilman +- Fabian Jahr +- FractalEncrypt +- furszy +- Gleb Naumenko +- glozow +- Greg Sanders +- Hennadii Stepanov +- hernanmarino +- ishaanam +- ismaelsadeeq +- James O'Beirne +- jdjkelly@gmail.com +- Jeff Ruane +- Jeffrey Czyz +- Jeremy Rubin +- Jesse Barton +- João Barbosa +- JoaoAJMatos +- John Moffett +- Jon Atack +- Jonas Schnelli +- jonatack +- Joshua Kelly +- josibake +- Juan Pablo Civile +- kdmukai +- klementtan +- Kolby ML +- kouloumos +- Kristaps Kaupe +- laanwj +- Larry Ruane +- Leonardo Araujo +- Leonardo Lazzaro +- Luke Dashjr +- MacroFake +- MarcoFalke +- Martin Leitner-Ankerl +- Martin Zumsande +- Matt Whitlock +- Matthew Zipkin +- Michael Ford +- Miles Liu +- mruddy +- Murray Nesbitt +- muxator +- omahs +- pablomartin4btc +- Pasta +- Pieter Wuille +- Pttn +- Randall Naar +- Riahiamirreza +- roconnor-blockstream +- Russell O'Connor +- Ryan Ofsky +- S3RK +- Sebastian Falbesoner +- Seibart Nedor +- sinetek +- Sjors Provoost +- Skuli Dulfari +- SomberNight +- Stacie Waleyko +- stickies-v +- stratospher +- Suhas Daftuar +- Suriyaa Sundararuban +- TheCharlatan +- Vasil Dimov +- Vasil Stoyanov +- virtu +- w0xlt +- willcl-ark +- yancy +- Yusuf Sahin HAMZA + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). \ No newline at end of file diff --git a/doc/release-process.md b/doc/release-process.md index 32306ab4f15a7b..9cb0b630769e42 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -284,7 +284,7 @@ cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.navcoin-qt/pull/2 - - Push the snap, see https://github.com/bitcoin-core/packaging/blob/master/snap/build.md + - Push the snap, see https://github.com/bitcoin-core/packaging/blob/main/snap/local/build.md - This repo diff --git a/src/Makefile.am b/src/Makefile.am index 534e4a48ba3370..67b258fb4724a3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -276,6 +276,7 @@ BITCOIN_CORE_H = \ node/mempool_args.h \ node/mempool_persist_args.h \ node/miner.h \ + node/mini_miner.h \ node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ @@ -416,7 +417,7 @@ BITCOIN_CORE_H = \ obj/build.h: FORCE @$(MKDIR_P) $(builddir)/obj - @$(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ + $(AM_V_GEN) $(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h @@ -490,6 +491,7 @@ libbitcoin_node_a_SOURCES = \ node/mempool_args.cpp \ node/mempool_persist_args.cpp \ node/miner.cpp \ + node/mini_miner.cpp \ node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ @@ -1162,7 +1164,7 @@ clean-local: -rm -rf test/__pycache__ .rc.o: - @test -f $(WINDRES) + @test -f $(WINDRES) || (echo "windres $(WINDRES) not found, but is required to compile windows resource files"; exit 1) ## FIXME: How to get the appropriate modulename_CPPFLAGS in here? $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ @@ -1217,12 +1219,11 @@ endif %.raw.h: %.raw @$(MKDIR_P) $(@D) - @{ \ + $(AM_V_GEN) { \ echo "static unsigned const char $(*F)_raw[] = {" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};"; \ } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" include Makefile.minisketch.include diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 26eacf1760fa3d..dc4c35171a9915 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -127,6 +127,7 @@ BITCOIN_TESTS =\ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/miniminer_tests.cpp \ test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ @@ -308,6 +309,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/message.cpp \ test/fuzz/miniscript.cpp \ test/fuzz/minisketch.cpp \ + test/fuzz/mini_miner.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ test/fuzz/net.cpp \ @@ -443,10 +445,9 @@ endif %.json.h: %.json @$(MKDIR_P) $(@D) - @{ \ + $(AM_V_GEN) { \ echo "namespace json_tests{" && \ echo "static unsigned const char $(*F)[] = {" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" diff --git a/src/addrdb.cpp b/src/addrdb.cpp index b679ad060299e5..fcb0f0719b2d86 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -183,10 +183,10 @@ void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) DeserializeDB(ssPeers, addr, false); } -std::optional LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr& addrman) +util::Result> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args) { auto check_addrman = std::clamp(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + auto addrman{std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman)}; const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -200,19 +200,17 @@ std::optional LoadAddrman(const NetGroupManager& netgroupman, con DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { if (!RenameOver(path_addr, (fs::path)path_addr + ".bak")) { - addrman = nullptr; - return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); + return util::Error{strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again."))}; } // Addrman can be in an inconsistent state after failure, reset it addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { - addrman = nullptr; - return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), - e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr))); + return util::Error{strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), + e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr)))}; } - return std::nullopt; + return std::move(addrman); // std::move should be unneccessary but is temporarily needed to work around clang bug (https://github.com/bitcoin/bitcoin/pull/25977#issuecomment-1561270092) } void DumpAnchors(const fs::path& anchors_db_path, const std::vector& anchors) diff --git a/src/addrdb.h b/src/addrdb.h index 08d86d0f01beeb..3da630fff8eb24 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -9,6 +9,7 @@ #include // For banmap_t #include #include +#include #include #include @@ -49,7 +50,7 @@ class CBanDB }; /** Returns an error string on failure */ -std::optional LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr& addrman); +util::Result> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 11ce14728def1e..0e110a653af9f3 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -20,7 +21,7 @@ using wallet::CWallet; using wallet::CWalletTx; using wallet::CoinEligibilityFilter; using wallet::CoinSelectionParams; -using wallet::CreateDummyWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::OutputGroup; using wallet::SelectCoinsBnB; using wallet::TxStateInactive; @@ -46,7 +47,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateMockableWalletDatabase()); std::vector> wtxs; LOCK(wallet.cs_wallet); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index d5d057e96dcd95..6cf71f4b83b801 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -15,7 +16,7 @@ #include using wallet::CWallet; -using wallet::CreateMockWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::DBErrors; using wallet::GetBalance; using wallet::WALLET_FLAG_DESCRIPTORS; @@ -28,7 +29,10 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + // Set clock to genesis block, so the descriptors/keys creation time don't interfere with the blocks scanning process. + // The reason is 'generatetoaddress', which creates a chain with deterministic timestamps in the past. + SetMockTime(test_setup->m_node.chainman->GetParams().GenesisBlock().nTime); + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/bench/wallet_create_tx.cpp b/src/bench/wallet_create_tx.cpp index cb31421598c602..13b0019fd2f8bb 100644 --- a/src/bench/wallet_create_tx.cpp +++ b/src/bench/wallet_create_tx.cpp @@ -15,7 +15,7 @@ #include using wallet::CWallet; -using wallet::CreateMockWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::DBErrors; using wallet::WALLET_FLAG_DESCRIPTORS; @@ -83,7 +83,7 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type { const auto test_setup = MakeNoLogFileContext(); - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -136,7 +136,7 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type static void AvailableCoins(benchmark::Bench& bench, const std::vector& output_type) { const auto test_setup = MakeNoLogFileContext(); - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index 6b09adcc9df487..cf32fde51eae6a 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -17,18 +17,17 @@ #include using wallet::CWallet; -using wallet::DatabaseFormat; -using wallet::DatabaseOptions; +using wallet::CreateMockableWalletDatabase; using wallet::TxStateInactive; using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WalletContext; using wallet::WalletDatabase; -static std::shared_ptr BenchLoadWallet(std::unique_ptr database, WalletContext& context, DatabaseOptions& options) +static std::shared_ptr BenchLoadWallet(std::unique_ptr database, WalletContext& context, uint64_t create_flags) { bilingual_str error; std::vector warnings; - auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); + auto wallet = CWallet::Create(context, "", std::move(database), create_flags, error, warnings); NotifyWalletLoaded(context, wallet); if (context.chain) { wallet->postInitProcess(); @@ -55,7 +54,6 @@ static void AddTx(CWallet& wallet) static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) { const auto test_setup = MakeNoLogFileContext(); - test_setup->m_args.ForceSetArg("-unsafesqlitesync", "1"); WalletContext context; context.args = &test_setup->m_args; @@ -63,31 +61,28 @@ static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) // Setup the wallet // Loading the wallet will also create it - DatabaseOptions options; - if (legacy_wallet) { - options.require_format = DatabaseFormat::BERKELEY; - } else { - options.create_flags = WALLET_FLAG_DESCRIPTORS; - options.require_format = DatabaseFormat::SQLITE; + uint64_t create_flags = 0; + if (!legacy_wallet) { + create_flags = WALLET_FLAG_DESCRIPTORS; } - auto database = CreateMockWalletDatabase(options); - auto wallet = BenchLoadWallet(std::move(database), context, options); + auto database = CreateMockableWalletDatabase(); + auto wallet = BenchLoadWallet(std::move(database), context, create_flags); // Generate a bunch of transactions and addresses to put into the wallet for (int i = 0; i < 1000; ++i) { AddTx(*wallet); } - database = DuplicateMockDatabase(wallet->GetDatabase(), options); + database = DuplicateMockDatabase(wallet->GetDatabase()); // reload the wallet for the actual benchmark BenchUnloadWallet(std::move(wallet)); bench.epochs(5).run([&] { - wallet = BenchLoadWallet(std::move(database), context, options); + wallet = BenchLoadWallet(std::move(database), context, create_flags); // Cleanup - database = DuplicateMockDatabase(wallet->GetDatabase(), options); + database = DuplicateMockDatabase(wallet->GetDatabase()); BenchUnloadWallet(std::move(wallet)); }); } diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 52e697a78d90ca..c143385d134db7 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -60,7 +60,7 @@ int main(int argc, char* argv[]) // We can't use a goto here, but we can use an assert since none of the // things instantiated so far requires running the epilogue to be torn down // properly - assert(!kernel::SanityChecks(kernel_context).has_value()); + assert(kernel::SanityChecks(kernel_context)); // Necessary for CheckInputScripts (eventually called by ProcessNewBlock), // which will try the script cache first and fall back to actually @@ -89,6 +89,7 @@ int main(int argc, char* argv[]) }; const node::BlockManager::Options blockman_opts{ .chainparams = chainman_opts.chainparams, + .blocks_dir = gArgs.GetBlocksDirPath(), }; ChainstateManager chainman{chainman_opts, blockman_opts}; diff --git a/src/common/args.cpp b/src/common/args.cpp index 62dca8fc10772a..2e2804dd216f2d 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -716,7 +716,8 @@ bool CheckDataDirOption(const ArgsManager& args) fs::path ArgsManager::GetConfigFilePath() const { - return GetConfigFile(*this, GetPathArg("-conf", BITCOIN_CONF_FILENAME)); + LOCK(cs_args); + return *Assert(m_config_path); } ChainType ArgsManager::GetChainType() const diff --git a/src/common/args.h b/src/common/args.h index 537a64fcfd97c8..7569297a74c943 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -28,7 +28,6 @@ extern const char * const BITCOIN_SETTINGS_FILENAME; // Return true if -datadir option points to a valid directory or is not specified. bool CheckDataDirOption(const ArgsManager& args); -fs::path GetConfigFile(const ArgsManager& args, const fs::path& configuration_file_path); /** * Most paths passed as configuration arguments are treated as relative to @@ -138,6 +137,7 @@ class ArgsManager std::map> m_available_args GUARDED_BY(cs_args); bool m_accept_any_command GUARDED_BY(cs_args){true}; std::list m_config_sections GUARDED_BY(cs_args); + std::optional m_config_path GUARDED_BY(cs_args); mutable fs::path m_cached_blocks_path GUARDED_BY(cs_args); mutable fs::path m_cached_datadir_path GUARDED_BY(cs_args); mutable fs::path m_cached_network_datadir_path GUARDED_BY(cs_args); diff --git a/src/common/config.cpp b/src/common/config.cpp index 5efb5efb67f1f1..e25b4fe2dfebc5 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -27,11 +27,6 @@ #include #include -fs::path GetConfigFile(const ArgsManager& args, const fs::path& configuration_file_path) -{ - return AbsPathForConfigVal(args, configuration_file_path, /*net_specific=*/false); -} - static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector>& options, std::list& sections) { std::string str, prefix; @@ -126,6 +121,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) LOCK(cs_args); m_settings.ro_config.clear(); m_config_sections.clear(); + m_config_path = AbsPathForConfigVal(*this, GetPathArg("-conf", BITCOIN_CONF_FILENAME), /*net_specific=*/false); } const auto conf_path{GetConfigFilePath()}; @@ -176,7 +172,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) const size_t default_includes = add_includes({}); for (const std::string& conf_file_name : conf_file_names) { - std::ifstream conf_file_stream{GetConfigFile(*this, fs::PathFromString(conf_file_name))}; + std::ifstream conf_file_stream{AbsPathForConfigVal(*this, fs::PathFromString(conf_file_name), /*net_specific=*/false)}; if (conf_file_stream.good()) { if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; diff --git a/src/common/init.cpp b/src/common/init.cpp index 8933d59c2707ce..412d73aec7016c 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,19 @@ std::optional InitConfig(ArgsManager& args, SettingsAbortFn setting if (!CheckDataDirOption(args)) { return ConfigError{ConfigStatus::FAILED, strprintf(_("Specified data directory \"%s\" does not exist."), args.GetArg("-datadir", ""))}; } + + // Record original datadir and config paths before parsing the config + // file. It is possible for the config file to contain a datadir= line + // that changes the datadir path after it is parsed. This is useful for + // CLI tools to let them use a different data storage location without + // needing to pass it every time on the command line. (It is not + // possible for the config file to cause another configuration to be + // used, though. Specifying a conf= option in the config file causes a + // parse error, and specifying a datadir= location containing another + // bitcoin.conf file just ignores the other file.) + const fs::path orig_datadir_path{args.GetDataDirBase()}; + const fs::path orig_config_path{AbsPathForConfigVal(args, args.GetPathArg("-conf", BITCOIN_CONF_FILENAME), /*net_specific=*/false)}; + std::string error; if (!args.ReadConfigFiles(error, true)) { return ConfigError{ConfigStatus::FAILED, strprintf(_("Error reading configuration file: %s"), error)}; @@ -48,6 +62,32 @@ std::optional InitConfig(ArgsManager& args, SettingsAbortFn setting fs::create_directories(net_path / "wallets"); } + // Show an error or warning if there is a bitcoin.conf file in the + // datadir that is being ignored. + const fs::path base_config_path = base_path / BITCOIN_CONF_FILENAME; + if (fs::exists(base_config_path) && !fs::equivalent(orig_config_path, base_config_path)) { + const std::string cli_config_path = args.GetArg("-conf", ""); + const std::string config_source = cli_config_path.empty() + ? strprintf("data directory %s", fs::quoted(fs::PathToString(orig_datadir_path))) + : strprintf("command line argument %s", fs::quoted("-conf=" + cli_config_path)); + const std::string error = strprintf( + "Data directory %1$s contains a %2$s file which is ignored, because a different configuration file " + "%3$s from %4$s is being used instead. Possible ways to address this would be to:\n" + "- Delete or rename the %2$s file in data directory %1$s.\n" + "- Change datadir= or conf= options to specify one configuration file, not two, and use " + "includeconf= to include any other configuration files.\n" + "- Set allowignoredconf=1 option to treat this condition as a warning, not an error.", + fs::quoted(fs::PathToString(base_path)), + fs::quoted(BITCOIN_CONF_FILENAME), + fs::quoted(fs::PathToString(orig_config_path)), + config_source); + if (args.GetBoolArg("-allowignoredconf", false)) { + LogPrintf("Warning: %s\n", error); + } else { + return ConfigError{ConfigStatus::FAILED, Untranslated(error)}; + } + } + // Create settings.json if -nosettings was not specified. if (args.GetSettingsPath()) { std::vector details; diff --git a/src/index/base.cpp b/src/index/base.cpp index 237c8e8be05dd6..74445793957b18 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -23,7 +23,7 @@ #include #include -using node::ReadBlockFromDisk; +using node::g_indexes_ready_to_sync; constexpr uint8_t DB_BEST_BLOCK{'B'}; @@ -94,14 +94,22 @@ bool BaseIndex::Init() if (locator.IsNull()) { SetBestBlockIndex(nullptr); } else { - SetBestBlockIndex(m_chainstate->FindForkInGlobalIndex(locator)); + // Setting the best block to the locator's top block. If it is not part of the + // best chain, we will rewind to the fork point during index sync + const CBlockIndex* locator_index{m_chainstate->m_blockman.LookupBlockIndex(locator.vHave.at(0))}; + if (!locator_index) { + return InitError(strprintf(Untranslated("%s: best block of the index not found. Please rebuild the index."), GetName())); + } + SetBestBlockIndex(locator_index); } // Note: this will latch to true immediately if the user starts up with an empty // datadir and an index enabled. If this is the case, indexation will happen solely // via `BlockConnected` signals until, possibly, the next restart. m_synced = m_best_block_index.load() == active_chain.Tip(); - if (!m_synced) { + + // Skip pruning check if indexes are not ready to sync (because reindex-chainstate has wiped the chain). + if (!m_synced && g_indexes_ready_to_sync) { bool prune_violation = false; if (!m_best_block_index) { // index is not built yet @@ -157,10 +165,14 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& void BaseIndex::ThreadSync() { SetSyscallSandboxPolicy(SyscallSandboxPolicy::TX_INDEX); + // Wait for a possible reindex-chainstate to finish until continuing + // with the index sync + while (!g_indexes_ready_to_sync) { + if (!m_interrupt.sleep_for(std::chrono::milliseconds(500))) return; + } + const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { - auto& consensus_params = Params().GetConsensus(); - std::chrono::steady_clock::time_point last_log_time{0s}; std::chrono::steady_clock::time_point last_locator_write_time{0s}; while (true) { @@ -207,7 +219,7 @@ void BaseIndex::ThreadSync() CBlock block; interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex); - if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + if (!m_chainstate->m_blockman.ReadBlockFromDisk(block, *pindex)) { FatalError("%s: Failed to read block %s from disk", __func__, pindex->GetBlockHash().ToString()); return; diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index e6300ce3dd8d09..a860b3a94d6db8 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -12,8 +12,6 @@ #include #include -using node::UndoReadFromDisk; - /* The index database stores three items for each block: the disk location of the encoded filter, * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by * height, and those belonging to blocks that have been reorganized out of the active chain are @@ -223,7 +221,7 @@ bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) // pindex variable gives indexing code access to node internals. It // will be removed in upcoming commit const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); - if (!UndoReadFromDisk(block_undo, pindex)) { + if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 5bc6ad3d3167cf..d80885f8422880 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -19,9 +19,6 @@ using kernel::CCoinsStats; using kernel::GetBogoSize; using kernel::TxOutSer; -using node::ReadBlockFromDisk; -using node::UndoReadFromDisk; - static constexpr uint8_t DB_BLOCK_HASH{'s'}; static constexpr uint8_t DB_BLOCK_HEIGHT{'t'}; static constexpr uint8_t DB_MUHASH{'M'}; @@ -125,7 +122,7 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) // pindex variable gives indexing code access to node internals. It // will be removed in upcoming commit const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); - if (!UndoReadFromDisk(block_undo, pindex)) { + if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } @@ -282,12 +279,11 @@ bool CoinStatsIndex::CustomRewind(const interfaces::BlockKey& current_tip, const LOCK(cs_main); const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip.hash)}; const CBlockIndex* new_tip_index{m_chainstate->m_blockman.LookupBlockIndex(new_tip.hash)}; - const auto& consensus_params{Params().GetConsensus()}; do { CBlock block; - if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { + if (!m_chainstate->m_blockman.ReadBlockFromDisk(block, *iter_tip)) { return error("%s: Failed to read block %s from disk", __func__, iter_tip->GetBlockHash().ToString()); } @@ -409,7 +405,7 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex // Ignore genesis block if (pindex->nHeight > 0) { - if (!UndoReadFromDisk(block_undo, pindex)) { + if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 49bddf2d4dfb8e..2e07a35d0d69a1 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -10,8 +10,6 @@ #include #include -using node::OpenBlockFile; - constexpr uint8_t DB_TXINDEX{'t'}; std::unique_ptr g_txindex; @@ -80,7 +78,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe return false; } - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + CAutoFile file(m_chainstate->m_blockman.OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); if (file.IsNull()) { return error("%s: OpenBlockFile failed", __func__); } diff --git a/src/init.cpp b/src/init.cpp index 4ef1e1551c8cb0..887bf6bcedaab7 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -112,22 +112,24 @@ #include #endif +using kernel::DEFAULT_STOPAFTERBLOCKIMPORT; using kernel::DumpMempool; using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; +using node::BlockManager; using node::CacheSizes; using node::CalculateCacheSizes; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; -using node::DEFAULT_STOPAFTERBLOCKIMPORT; +using node::fReindex; +using node::g_indexes_ready_to_sync; using node::LoadChainstate; using node::MempoolPath; -using node::ShouldPersistMempool; using node::NodeContext; +using node::ShouldPersistMempool; using node::ThreadImport; using node::VerifyLoadedChainstate; -using node::fReindex; static constexpr bool DEFAULT_PROXYRANDOMIZE{true}; static constexpr bool DEFAULT_REST_ENABLE{false}; @@ -328,9 +330,8 @@ void Shutdown(NodeContext& node) #if ENABLE_ZMQ if (g_zmq_notification_interface) { - UnregisterValidationInterface(g_zmq_notification_interface); - delete g_zmq_notification_interface; - g_zmq_notification_interface = nullptr; + UnregisterValidationInterface(g_zmq_notification_interface.get()); + g_zmq_notification_interface.reset(); } #endif @@ -443,6 +444,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-dbcache=", strprintf("Maximum database cache size MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-includeconf=", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-loadblock=", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxmempool=", strprintf("Keep the transaction memory pool below megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE_MB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxorphantx=", strprintf("Keep at most unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -456,7 +458,7 @@ void SetupServerArgs(ArgsManager& argsman) "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk. This will also rebuild active optional indexes.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead. Deactivate all optional indexes before running this.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-settings=", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #if HAVE_SYSTEM argsman.AddArg("-startupnotify=", "Execute command on startup.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -984,19 +986,6 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); - if (args.GetBoolArg("-reindex-chainstate", false)) { - // indexes that must be deactivated to prevent index corruption, see #24630 - if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - return InitError(_("-reindex-chainstate option is not compatible with -coinstatsindex. Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - if (g_enabled_filter_types.count(BlockFilterType::BASIC)) { - return InitError(_("-reindex-chainstate option is not compatible with -blockfilterindex. Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - return InitError(_("-reindex-chainstate option is not compatible with -txindex. Please temporarily disable txindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - } - #if defined(USE_SYSCALL_SANDBOX) if (args.IsArgSet("-sandbox") && !args.IsArgNegated("-sandbox")) { const std::string sandbox_arg{args.GetArg("-sandbox", "")}; @@ -1037,14 +1026,17 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb .chainparams = chainparams, .datadir = args.GetDataDirNet(), }; - if (const auto error{ApplyArgsManOptions(args, chainman_opts_dummy)}) { - return InitError(*error); + auto chainman_result{ApplyArgsManOptions(args, chainman_opts_dummy)}; + if (!chainman_result) { + return InitError(util::ErrorString(chainman_result)); } - node::BlockManager::Options blockman_opts_dummy{ + BlockManager::Options blockman_opts_dummy{ .chainparams = chainman_opts_dummy.chainparams, + .blocks_dir = args.GetBlocksDirPath(), }; - if (const auto error{ApplyArgsManOptions(args, blockman_opts_dummy)}) { - return InitError(*error); + auto blockman_result{ApplyArgsManOptions(args, blockman_opts_dummy)}; + if (!blockman_result) { + return InitError(util::ErrorString(blockman_result)); } } @@ -1067,8 +1059,9 @@ static bool LockDataDirectory(bool probeOnly) bool AppInitSanityChecks(const kernel::Context& kernel) { // ********************************************************* Step 4: sanity checks - if (auto error = kernel::SanityChecks(kernel)) { - InitError(*error); + auto result{kernel::SanityChecks(kernel)}; + if (!result) { + InitError(util::ErrorString(result)); return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); } @@ -1243,9 +1236,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Initialize addrman assert(!node.addrman); uiInterface.InitMessage(_("Loading P2P addresses…").translated); - if (const auto error{LoadAddrman(*node.netgroupman, args, node.addrman)}) { - return InitError(*error); - } + auto addrman{LoadAddrman(*node.netgroupman, args)}; + if (!addrman) return InitError(util::ErrorString(addrman)); + node.addrman = std::move(*addrman); } assert(!node.banman); @@ -1427,10 +1420,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } #if ENABLE_ZMQ - g_zmq_notification_interface = CZMQNotificationInterface::Create(); + g_zmq_notification_interface = CZMQNotificationInterface::Create( + [&chainman = node.chainman](CBlock& block, const CBlockIndex& index) { + assert(chainman); + return chainman->m_blockman.ReadBlockFromDisk(block, index); + }); if (g_zmq_notification_interface) { - RegisterValidationInterface(g_zmq_notification_interface); + RegisterValidationInterface(g_zmq_notification_interface.get()); } #endif @@ -1443,12 +1440,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) .datadir = args.GetDataDirNet(), .adjusted_time_callback = GetAdjustedTime, }; - Assert(!ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction + Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction - node::BlockManager::Options blockman_opts{ + BlockManager::Options blockman_opts{ .chainparams = chainman_opts.chainparams, + .blocks_dir = args.GetBlocksDirPath(), }; - Assert(!ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction + Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction // cache size calculations CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size()); @@ -1471,8 +1469,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) .estimator = node.fee_estimator.get(), .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, }; - if (const auto err{ApplyArgsManOptions(args, chainparams, mempool_opts)}) { - return InitError(*err); + auto result{ApplyArgsManOptions(args, chainparams, mempool_opts)}; + if (!result) { + return InitError(util::ErrorString(result)); } mempool_opts.check_ratio = std::clamp(mempool_opts.check_ratio, 0, 1'000'000); @@ -1567,9 +1566,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) RegisterValidationInterface(node.peerman.get()); // ********************************************************* Step 8: start indexers + + // If reindex-chainstate was specified, delay syncing indexes until ThreadImport has reindexed the chain + if (!fReindexChainState) g_indexes_ready_to_sync = true; if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - if (const auto error{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}) { - return InitError(*error); + auto result{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}; + if (!result) { + return InitError(util::ErrorString(result)); } g_txindex = std::make_unique(interfaces::MakeChain(node), cache_sizes.tx_index, false, fReindex); @@ -1676,7 +1679,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &chainman, &args] { - ThreadImport(chainman, vImportFiles, args, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); + ThreadImport(chainman, vImportFiles, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); }); // Wait for genesis block to be processed diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index a3fa753a9807e5..40bf0b680c22a1 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -87,6 +87,9 @@ struct BlockInfo { unsigned data_pos = 0; const CBlock* data = nullptr; const CBlockUndo* undo_data = nullptr; + // The maximum time in the chain up to and including this block. + // A timestamp that can only move forward. + unsigned int chain_time_max{0}; BlockInfo(const uint256& hash LIFETIMEBOUND) : hash(hash) {} }; diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h index 5584a66e0205f9..8f26422f72c3cc 100644 --- a/src/kernel/blockmanager_opts.h +++ b/src/kernel/blockmanager_opts.h @@ -5,10 +5,16 @@ #ifndef BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H #define BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H +#include + +#include + class CChainParams; namespace kernel { +static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; + /** * An options struct for `BlockManager`, more ergonomically referred to as * `BlockManager::Options` due to the using-declaration in `BlockManager`. @@ -16,6 +22,9 @@ namespace kernel { struct BlockManagerOpts { const CChainParams& chainparams; uint64_t prune_target{0}; + bool fast_prune{false}; + bool stop_after_block_import{DEFAULT_STOPAFTERBLOCKIMPORT}; + const fs::path blocks_dir; }; } // namespace kernel diff --git a/src/kernel/chain.cpp b/src/kernel/chain.cpp index 82e77125d7f369..1c877866d0a28c 100644 --- a/src/kernel/chain.cpp +++ b/src/kernel/chain.cpp @@ -16,6 +16,7 @@ interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* index, const CBlock* data if (index) { info.prev_hash = index->pprev ? index->pprev->phashBlock : nullptr; info.height = index->nHeight; + info.chain_time_max = index->GetBlockTimeMax(); LOCK(::cs_main); info.file_number = index->nFile; info.data_pos = index->nDataPos; diff --git a/src/kernel/checks.cpp b/src/kernel/checks.cpp index 4c303c172c2d17..bf8a2ec74c0475 100644 --- a/src/kernel/checks.cpp +++ b/src/kernel/checks.cpp @@ -13,21 +13,21 @@ namespace kernel { -std::optional SanityChecks(const Context&) +util::Result SanityChecks(const Context&) { if (!ECC_InitSanityCheck()) { - return Untranslated("Elliptic curve cryptography sanity check failure. Aborting."); + return util::Error{Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")}; } if (!Random_SanityCheck()) { - return Untranslated("OS cryptographic RNG sanity check failure. Aborting."); + return util::Error{Untranslated("OS cryptographic RNG sanity check failure. Aborting.")}; } if (!ChronoSanityCheck()) { - return Untranslated("Clock epoch mismatch. Aborting."); + return util::Error{Untranslated("Clock epoch mismatch. Aborting.")}; } - return std::nullopt; + return {}; } } diff --git a/src/kernel/checks.h b/src/kernel/checks.h index 3eb14824fb6f93..8cff4e3be4617a 100644 --- a/src/kernel/checks.h +++ b/src/kernel/checks.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_KERNEL_CHECKS_H #define BITCOIN_KERNEL_CHECKS_H -#include +#include struct bilingual_str; @@ -16,7 +16,7 @@ struct Context; /** * Ensure a usable environment with all necessary library support. */ -std::optional SanityChecks(const Context&); +util::Result SanityChecks(const Context&); } diff --git a/src/key_io.cpp b/src/key_io.cpp index 4659a595447e75..454a96df5e5bcb 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -124,7 +124,11 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par data.clear(); const auto dec = bech32::Decode(str); - if ((dec.encoding == bech32::Encoding::BECH32 || dec.encoding == bech32::Encoding::BECH32M) && dec.data.size() > 0) { + if (dec.encoding == bech32::Encoding::BECH32 || dec.encoding == bech32::Encoding::BECH32M) { + if (dec.data.empty()) { + error_str = "Empty Bech32 data section"; + return CNoDestination(); + } // Bech32 decoding if (dec.hrp != params.Bech32HRP()) { error_str = strprintf("Invalid or unsupported prefix for Segwit (Bech32) address (expected %s, got %s).", params.Bech32HRP(), dec.hrp); @@ -142,6 +146,9 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par // The rest of the symbols are converted witness program bytes. data.reserve(((dec.data.size() - 1) * 5) / 8); if (ConvertBits<5, 8, false>([&](unsigned char c) { data.push_back(c); }, dec.data.begin() + 1, dec.data.end())) { + + std::string_view byte_str{data.size() == 1 ? "byte" : "bytes"}; + if (version == 0) { { WitnessV0KeyHash keyid; @@ -158,7 +165,7 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } } - error_str = "Invalid Bech32 v0 address data size"; + error_str = strprintf("Invalid Bech32 v0 address program size (%d %s), per BIP141", data.size(), byte_str); return CNoDestination(); } @@ -175,7 +182,7 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } if (data.size() < 2 || data.size() > BECH32_WITNESS_PROG_MAX_LEN) { - error_str = "Invalid Bech32 address data size"; + error_str = strprintf("Invalid Bech32 address program size (%d %s)", data.size(), byte_str); return CNoDestination(); } @@ -184,6 +191,9 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par std::copy(data.begin(), data.end(), unk.program); unk.length = data.size(); return unk; + } else { + error_str = strprintf("Invalid padding in Bech32 data section"); + return CNoDestination(); } } diff --git a/src/net.h b/src/net.h index c5f35f4c70ffa6..a6cb03f5929676 100644 --- a/src/net.h +++ b/src/net.h @@ -201,7 +201,9 @@ class CNodeStats int nVersion; std::string cleanSubVer; bool fInbound; + // We requested high bandwidth connection to peer bool m_bip152_highbandwidth_to; + // Peer requested high bandwidth connection bool m_bip152_highbandwidth_from; int m_starting_height; uint64_t nSendBytes; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 3e678142e60b56..39816c4ee9e6c6 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -51,9 +51,6 @@ #include #include -using node::ReadBlockFromDisk; -using node::ReadRawBlockFromDisk; - /** How long to cache transactions in mapRelay for normal relay */ static constexpr auto RELAY_TX_CACHE_TIME = 15min; /** How long a transaction has to be in the mempool before it can unconditionally be relayed (even when not in mapRelay). */ @@ -436,7 +433,6 @@ struct CNodeState { std::list vBlocksInFlight; //! When the first entry in vBlocksInFlight started downloading. Don't care when vBlocksInFlight is empty. std::chrono::microseconds m_downloading_since{0us}; - int nBlocksInFlight{0}; //! Whether we consider this a preferred download peer. bool fPreferredDownload{false}; /** Whether this peer wants invs or cmpctblocks (when possible) for block announcements. */ @@ -887,6 +883,9 @@ class PeerManagerImpl final : public PeerManager /** Have we requested this block from a peer */ bool IsBlockRequested(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Have we requested this block from an outbound peer */ + bool IsBlockRequestedFromOutbound(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Remove this block from our tracked requested blocks. Called if: * - the block has been received from a peer * - the request for the block has timed out @@ -909,7 +908,9 @@ class PeerManagerImpl final : public PeerManager */ void FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - std::map::iterator> > mapBlocksInFlight GUARDED_BY(cs_main); + /* Multimap used to preserve insertion order */ + typedef std::multimap::iterator>> BlockDownloadMap; + BlockDownloadMap mapBlocksInFlight GUARDED_BY(cs_main); /** When our tip was last updated. */ std::atomic m_last_tip_update{0s}; @@ -1127,40 +1128,55 @@ std::chrono::microseconds PeerManagerImpl::NextInvToInbounds(std::chrono::micros bool PeerManagerImpl::IsBlockRequested(const uint256& hash) { - return mapBlocksInFlight.find(hash) != mapBlocksInFlight.end(); + return mapBlocksInFlight.count(hash); } -void PeerManagerImpl::RemoveBlockRequest(const uint256& hash, std::optional from_peer) +bool PeerManagerImpl::IsBlockRequestedFromOutbound(const uint256& hash) { - auto it = mapBlocksInFlight.find(hash); - if (it == mapBlocksInFlight.end()) { - // Block was not requested - return; + for (auto range = mapBlocksInFlight.equal_range(hash); range.first != range.second; range.first++) { + auto [nodeid, block_it] = range.first->second; + CNodeState& nodestate = *Assert(State(nodeid)); + if (!nodestate.m_is_inbound) return true; } - auto [node_id, list_it] = it->second; + return false; +} - if (from_peer && node_id != *from_peer) { - // Block was requested by another peer +void PeerManagerImpl::RemoveBlockRequest(const uint256& hash, std::optional from_peer) +{ + auto range = mapBlocksInFlight.equal_range(hash); + if (range.first == range.second) { + // Block was not requested from any peer return; } - CNodeState *state = State(node_id); - assert(state != nullptr); + // We should not have requested too many of this block + Assume(mapBlocksInFlight.count(hash) <= MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK); - if (state->vBlocksInFlight.begin() == list_it) { - // First block on the queue was received, update the start download time for the next one - state->m_downloading_since = std::max(state->m_downloading_since, GetTime()); - } - state->vBlocksInFlight.erase(list_it); + while (range.first != range.second) { + auto [node_id, list_it] = range.first->second; + + if (from_peer && *from_peer != node_id) { + range.first++; + continue; + } + + CNodeState& state = *Assert(State(node_id)); + + if (state.vBlocksInFlight.begin() == list_it) { + // First block on the queue was received, update the start download time for the next one + state.m_downloading_since = std::max(state.m_downloading_since, GetTime()); + } + state.vBlocksInFlight.erase(list_it); + + if (state.vBlocksInFlight.empty()) { + // Last validated block on the queue for this peer was received. + m_peers_downloading_from--; + } + state.m_stalling_since = 0us; - state->nBlocksInFlight--; - if (state->nBlocksInFlight == 0) { - // Last validated block on the queue was received. - m_peers_downloading_from--; + range.first = mapBlocksInFlight.erase(range.first); } - state->m_stalling_since = 0us; - mapBlocksInFlight.erase(it); } bool PeerManagerImpl::BlockRequested(NodeId nodeid, const CBlockIndex& block, std::list::iterator** pit) @@ -1170,27 +1186,29 @@ bool PeerManagerImpl::BlockRequested(NodeId nodeid, const CBlockIndex& block, st CNodeState *state = State(nodeid); assert(state != nullptr); + Assume(mapBlocksInFlight.count(hash) <= MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK); + // Short-circuit most stuff in case it is from the same node - std::map::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); - if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { - if (pit) { - *pit = &itInFlight->second.second; + for (auto range = mapBlocksInFlight.equal_range(hash); range.first != range.second; range.first++) { + if (range.first->second.first == nodeid) { + if (pit) { + *pit = &range.first->second.second; + } + return false; } - return false; } - // Make sure it's not listed somewhere already. - RemoveBlockRequest(hash, std::nullopt); + // Make sure it's not being fetched already from same peer. + RemoveBlockRequest(hash, nodeid); std::list::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), {&block, std::unique_ptr(pit ? new PartiallyDownloadedBlock(&m_mempool) : nullptr)}); - state->nBlocksInFlight++; - if (state->nBlocksInFlight == 1) { + if (state->vBlocksInFlight.size() == 1) { // We're starting a block download (batch) from this peer. state->m_downloading_since = GetTime(); m_peers_downloading_from++; } - itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))).first; + auto itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))); if (pit) { *pit = &itInFlight->second.second; } @@ -1393,7 +1411,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co } } else if (waitingfor == -1) { // This is the first already-in-flight block. - waitingfor = mapBlocksInFlight[pindex->GetBlockHash()].first; + waitingfor = mapBlocksInFlight.lower_bound(pindex->GetBlockHash())->second.first; } } } @@ -1523,13 +1541,21 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) nSyncStarted--; for (const QueuedBlock& entry : state->vBlocksInFlight) { - mapBlocksInFlight.erase(entry.pindex->GetBlockHash()); + auto range = mapBlocksInFlight.equal_range(entry.pindex->GetBlockHash()); + while (range.first != range.second) { + auto [node_id, list_it] = range.first->second; + if (node_id != nodeid) { + range.first++; + } else { + range.first = mapBlocksInFlight.erase(range.first); + } + } } m_orphanage.EraseForPeer(nodeid); m_txrequest.DisconnectedPeer(nodeid); if (m_txreconciliation) m_txreconciliation->ForgetPeer(nodeid); m_num_preferred_download_peers -= state->fPreferredDownload; - m_peers_downloading_from -= (state->nBlocksInFlight != 0); + m_peers_downloading_from -= (!state->vBlocksInFlight.empty()); assert(m_peers_downloading_from >= 0); m_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; assert(m_outbound_peers_with_protect_from_disconnect >= 0); @@ -1770,11 +1796,10 @@ std::optional PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl LOCK(cs_main); - // Mark block as in-flight unless it already is (for this peer). - // If the peer does not send us a block, vBlocksInFlight remains non-empty, - // causing us to timeout and disconnect. - // If a block was already in-flight for a different peer, its BLOCKTXN - // response will be dropped. + // Forget about all prior requests + RemoveBlockRequest(block_index.GetBlockHash(), std::nullopt); + + // Mark block as in-flight if (!BlockRequested(peer_id, block_index)) return "Already requested from this peer"; // Construct message to request the block @@ -2196,7 +2221,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& // Fast-path: in this case it is possible to serve the block directly from disk, // as the network format matches the format on disk std::vector block_data; - if (!ReadRawBlockFromDisk(block_data, pindex->GetBlockPos(), m_chainparams.MessageStart())) { + if (!m_chainman.m_blockman.ReadRawBlockFromDisk(block_data, pindex->GetBlockPos(), m_chainparams.MessageStart())) { assert(!"cannot load block from disk"); } m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, Span{block_data})); @@ -2204,7 +2229,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } else { // Send block from disk std::shared_ptr pblockRead = std::make_shared(); - if (!ReadBlockFromDisk(*pblockRead, pindex, m_chainparams.GetConsensus())) { + if (!m_chainman.m_blockman.ReadBlockFromDisk(*pblockRead, *pindex)) { assert(!"cannot load block from disk"); } pblock = pblockRead; @@ -2710,7 +2735,7 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, c std::vector vGetData; // Download as much as possible, from earliest to latest. for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (nodestate->vBlocksInFlight.size() >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { // Can't download any more from this peer break; } @@ -3916,7 +3941,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (pindex->nHeight >= m_chainman.ActiveChain().Height() - MAX_BLOCKTXN_DEPTH) { CBlock block; - bool ret = ReadBlockFromDisk(block, pindex, m_chainparams.GetConsensus()); + const bool ret{m_chainman.m_blockman.ReadBlockFromDisk(block, *pindex)}; assert(ret); SendBlockTransactions(pfrom, *peer, block, req); @@ -4313,15 +4338,27 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, nodestate->m_last_block_announcement = GetTime(); } - std::map::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); - bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); - if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here return; + auto range_flight = mapBlocksInFlight.equal_range(pindex->GetBlockHash()); + size_t already_in_flight = std::distance(range_flight.first, range_flight.second); + bool requested_block_from_this_peer{false}; + + // Multimap ensures ordering of outstanding requests. It's either empty or first in line. + bool first_in_flight = already_in_flight == 0 || (range_flight.first->second.first == pfrom.GetId()); + + while (range_flight.first != range_flight.second) { + if (range_flight.first->second.first == pfrom.GetId()) { + requested_block_from_this_peer = true; + break; + } + range_flight.first++; + } + if (pindex->nChainWork <= m_chainman.ActiveChain().Tip()->nChainWork || // We know something better pindex->nTx != 0) { // We had this block at some point, but pruned it - if (fAlreadyInFlight) { + if (requested_block_from_this_peer) { // We requested this block for some reason, but our mempool will probably be useless // so we just grab the block via normal getdata std::vector vInv(1); @@ -4332,15 +4369,15 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } // If we're not close to tip yet, give up and let parallel block fetch work its magic - if (!fAlreadyInFlight && !CanDirectFetch()) { + if (!already_in_flight && !CanDirectFetch()) { return; } // We want to be a bit conservative just to be extra careful about DoS // possibilities in compact block processing... if (pindex->nHeight <= m_chainman.ActiveChain().Height() + 2) { - if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || - (fAlreadyInFlight && blockInFlightIt->second.first == pfrom.GetId())) { + if ((already_in_flight < MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK && nodestate->vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || + requested_block_from_this_peer) { std::list::iterator* queuedBlockIt = nullptr; if (!BlockRequested(pfrom.GetId(), *pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) @@ -4359,10 +4396,15 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, Misbehaving(*peer, 100, "invalid compact block"); return; } else if (status == READ_STATUS_FAILED) { - // Duplicate txindexes, the block is now in-flight, so just request it - std::vector vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + if (first_in_flight) { + // Duplicate txindexes, the block is now in-flight, so just request it + std::vector vInv(1); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + } else { + // Give up for this peer and wait for other peer(s) + RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); + } return; } @@ -4377,9 +4419,24 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, txn.blockhash = blockhash; blockTxnMsg << txn; fProcessBLOCKTXN = true; - } else { + } else if (first_in_flight) { + // We will try to round-trip any compact blocks we get on failure, + // as long as it's first... req.blockhash = pindex->GetBlockHash(); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + } else if (pfrom.m_bip152_highbandwidth_to && + (!pfrom.IsInboundConn() || + IsBlockRequestedFromOutbound(blockhash) || + already_in_flight < MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK - 1)) { + // ... or it's a hb relay peer and: + // - peer is outbound, or + // - we already have an outbound attempt in flight(so we'll take what we can get), or + // - it's not the final parallel download slot (which we may reserve for first outbound) + req.blockhash = pindex->GetBlockHash(); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + } else { + // Give up for this peer and wait for other peer(s) + RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); } } else { // This block is either already in flight from a different @@ -4400,7 +4457,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } } } else { - if (fAlreadyInFlight) { + if (requested_block_from_this_peer) { // We requested this block, but its far into the future, so our // mempool will probably be useless - request the block normally std::vector vInv(1); @@ -4472,24 +4529,44 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { LOCK(cs_main); - std::map::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); - if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || - it->second.first != pfrom.GetId()) { + auto range_flight = mapBlocksInFlight.equal_range(resp.blockhash); + size_t already_in_flight = std::distance(range_flight.first, range_flight.second); + bool requested_block_from_this_peer{false}; + + // Multimap ensures ordering of outstanding requests. It's either empty or first in line. + bool first_in_flight = already_in_flight == 0 || (range_flight.first->second.first == pfrom.GetId()); + + while (range_flight.first != range_flight.second) { + auto [node_id, block_it] = range_flight.first->second; + if (node_id == pfrom.GetId() && block_it->partialBlock) { + requested_block_from_this_peer = true; + break; + } + range_flight.first++; + } + + if (!requested_block_from_this_peer) { LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom.GetId()); return; } - PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + PartiallyDownloadedBlock& partialBlock = *range_flight.first->second.second->partialBlock; ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); if (status == READ_STATUS_INVALID) { RemoveBlockRequest(resp.blockhash, pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect Misbehaving(*peer, 100, "invalid compact block/non-matching block transactions"); return; } else if (status == READ_STATUS_FAILED) { - // Might have collided, fall back to getdata now :( - std::vector invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(*peer), resp.blockhash)); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + if (first_in_flight) { + // Might have collided, fall back to getdata now :( + std::vector invs; + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(*peer), resp.blockhash)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + } else { + RemoveBlockRequest(resp.blockhash, pfrom.GetId()); + LogPrint(BCLog::NET, "Peer %d sent us a compact block but it failed to reconstruct, waiting on first download to complete\n", pfrom.GetId()); + return; + } } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -5082,14 +5159,14 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) // valid headers chain with at least as much work as our tip. CNodeState *node_state = State(pnode->GetId()); if (node_state == nullptr || - (now - pnode->m_connected >= MINIMUM_CONNECT_TIME && node_state->nBlocksInFlight == 0)) { + (now - pnode->m_connected >= MINIMUM_CONNECT_TIME && node_state->vBlocksInFlight.empty())) { pnode->fDisconnect = true; LogPrint(BCLog::NET, "disconnecting extra block-relay-only peer=%d (last block received at time %d)\n", pnode->GetId(), count_seconds(pnode->m_last_block_time)); return true; } else { LogPrint(BCLog::NET, "keeping block-relay-only peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", - pnode->GetId(), count_seconds(pnode->m_connected), node_state->nBlocksInFlight); + pnode->GetId(), count_seconds(pnode->m_connected), node_state->vBlocksInFlight.size()); } return false; }); @@ -5129,13 +5206,13 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) // Also don't disconnect any peer we're trying to download a // block from. CNodeState &state = *State(pnode->GetId()); - if (now - pnode->m_connected > MINIMUM_CONNECT_TIME && state.nBlocksInFlight == 0) { + if (now - pnode->m_connected > MINIMUM_CONNECT_TIME && state.vBlocksInFlight.empty()) { LogPrint(BCLog::NET, "disconnecting extra outbound peer=%d (last block announcement received at time %d)\n", pnode->GetId(), oldest_block_announcement); pnode->fDisconnect = true; return true; } else { LogPrint(BCLog::NET, "keeping outbound peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", - pnode->GetId(), count_seconds(pnode->m_connected), state.nBlocksInFlight); + pnode->GetId(), count_seconds(pnode->m_connected), state.vBlocksInFlight.size()); return false; } }); @@ -5623,7 +5700,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) m_connman.PushMessage(pto, std::move(cached_cmpctblock_msg.value())); } else { CBlock block; - bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); + const bool ret{m_chainman.m_blockman.ReadBlockFromDisk(block, *pBestIndex)}; assert(ret); CBlockHeaderAndShortTxIDs cmpctblock{block}; m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); @@ -5774,7 +5851,9 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // especially since we have many peers and some will draw much shorter delays. unsigned int nRelayedTransactions = 0; LOCK(tx_relay->m_bloom_filter_mutex); - while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { + size_t broadcast_max{INVENTORY_BROADCAST_MAX + (tx_relay->m_tx_inventory_to_send.size()/1000)*5}; + broadcast_max = std::min(1000, broadcast_max); + while (!vInvTx.empty() && nRelayedTransactions < broadcast_max) { // Fetch the top element from the heap std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); std::set::iterator it = vInvTx.back(); @@ -5862,7 +5941,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->GetId()); + LogPrintf("Peer=%d%s is stalling block download, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; // Increase timeout for the next peer so that we don't disconnect multiple peers if our own // bandwidth is insufficient. @@ -5881,7 +5960,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = m_peers_downloading_from - 1; if (current_time > state.m_downloading_since + std::chrono::seconds{consensusParams.nPowTargetSpacing} * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->GetId()); + LogPrintf("Timeout downloading block %s from peer=%d%s, disconnecting\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; return true; } @@ -5897,11 +5976,11 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // disconnect our sync peer for stalling; we have bigger // problems if we can't get any outbound peers. if (!pto->HasPermission(NetPermissionFlags::NoBan)) { - LogPrintf("Timeout downloading headers from peer=%d, disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from peer=%d%s, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; return true; } else { - LogPrintf("Timeout downloading headers from noban peer=%d, not disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from noban peer=%d%s, not disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); // Reset the headers sync state so that we have a // chance to try downloading from a different peer. // Note: this will also result in at least one more @@ -5927,10 +6006,10 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Message: getdata (blocks) // std::vector vGetData; - if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector vToDownload; NodeId staller = -1; - FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); + FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.vBlocksInFlight.size(), vToDownload, staller); for (const CBlockIndex *pindex : vToDownload) { uint32_t nFetchFlags = GetFetchFlags(*peer); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); @@ -5938,7 +6017,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), pindex->nHeight, pto->GetId()); } - if (state.nBlocksInFlight == 0 && staller != -1) { + if (state.vBlocksInFlight.empty() && staller != -1) { if (State(staller)->m_stalling_since == 0us) { State(staller)->m_stalling_since = current_time; LogPrint(BCLog::NET, "Stall started peer=%d\n", staller); diff --git a/src/net_processing.h b/src/net_processing.h index af9a02139b14ab..deebb24c94e805 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -22,6 +22,8 @@ static const bool DEFAULT_PEERBLOOMFILTERS = false; static const bool DEFAULT_PEERBLOCKFILTERS = false; /** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ static const int DISCOURAGEMENT_THRESHOLD{100}; +/** Maximum number of outstanding CMPCTBLOCK requests for the same block. */ +static const unsigned int MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK = 3; struct CNodeStateStats { int nSyncHeight = -1; diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index 06a1934947c5e6..4b296db1b09223 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -5,26 +5,35 @@ #include #include +#include +#include +#include +#include #include +#include + namespace node { -std::optional ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts) +util::Result ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts) { // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg{args.GetIntArg("-prune", opts.prune_target)}; if (nPruneArg < 0) { - return _("Prune cannot be configured with a negative value."); + return util::Error{_("Prune cannot be configured with a negative value.")}; } uint64_t nPruneTarget{uint64_t(nPruneArg) * 1024 * 1024}; if (nPruneArg == 1) { // manual pruning: -prune=1 nPruneTarget = BlockManager::PRUNE_TARGET_MANUAL; } else if (nPruneTarget) { if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { - return strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024); + return util::Error{strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)}; } } opts.prune_target = nPruneTarget; - return std::nullopt; + if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value; + if (auto value{args.GetBoolArg("-stopafterblockimport")}) opts.stop_after_block_import = *value; + + return {}; } } // namespace node diff --git a/src/node/blockmanager_args.h b/src/node/blockmanager_args.h index e657c6bb4568f5..84540b19225f66 100644 --- a/src/node/blockmanager_args.h +++ b/src/node/blockmanager_args.h @@ -7,14 +7,12 @@ #define BITCOIN_NODE_BLOCKMANAGER_ARGS_H #include - -#include +#include class ArgsManager; -struct bilingual_str; namespace node { -std::optional ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts); +util::Result ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts); } // namespace node #endif // BITCOIN_NODE_BLOCKMANAGER_ARGS_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index bafee4e2371514..f8d4e6c1da5e5a 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -28,6 +27,7 @@ namespace node { std::atomic_bool fReindex(false); +std::atomic_bool g_indexes_ready_to_sync{false}; bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const { @@ -53,10 +53,6 @@ bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CB return pa->nHeight < pb->nHeight; } -static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); -static FlatFileSeq BlockFileSeq(); -static FlatFileSeq UndoFileSeq(); - std::vector BlockManager::GetAllBlockIndices() { AssertLockHeld(cs_main); @@ -423,7 +419,7 @@ const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& start_bl // rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info // is in sync with what's actually on disk by the time we start downloading, so that pruning // works correctly. -void CleanupBlockRevFiles() +void BlockManager::CleanupBlockRevFiles() const { std::map mapBlockFiles; @@ -431,8 +427,7 @@ void CleanupBlockRevFiles() // Remove the rev files immediately and insert the blk file paths into an // ordered map keyed by block file index. LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - const fs::path& blocksdir = gArgs.GetBlocksDirPath(); - for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { + for (fs::directory_iterator it(m_opts.blocks_dir); it != fs::directory_iterator(); it++) { const std::string path = fs::PathToString(it->path().filename()); if (fs::is_regular_file(*it) && path.length() == 12 && @@ -467,7 +462,7 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n) return &m_blockfile_info.at(n); } -static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const { // Open history file to append AutoFile fileout{OpenUndoFile(pos)}; @@ -496,9 +491,9 @@ static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const return true; } -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) +bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const { - const FlatFilePos pos{WITH_LOCK(::cs_main, return pindex->GetUndoPos())}; + const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())}; if (pos.IsNull()) { return error("%s: no undo data available", __func__); @@ -514,7 +509,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) uint256 hashChecksum; HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a try { - verifier << pindex->pprev->GetBlockHash(); + verifier << index.pprev->GetBlockHash(); verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -570,7 +565,7 @@ uint64_t BlockManager::CalculateCurrentUsage() return retval; } -void UnlinkPrunedFiles(const std::set& setFilesToPrune) +void BlockManager::UnlinkPrunedFiles(const std::set& setFilesToPrune) const { std::error_code ec; for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { @@ -583,28 +578,28 @@ void UnlinkPrunedFiles(const std::set& setFilesToPrune) } } -static FlatFileSeq BlockFileSeq() +FlatFileSeq BlockManager::BlockFileSeq() const { - return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); + return FlatFileSeq(m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); } -static FlatFileSeq UndoFileSeq() +FlatFileSeq BlockManager::UndoFileSeq() const { - return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); + return FlatFileSeq(m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE); } -FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) +FILE* BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const { return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) +FILE* BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const { return UndoFileSeq().Open(pos, fReadOnly); } -fs::path GetBlockPosFilename(const FlatFilePos& pos) +fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const { return BlockFileSeq().FileName(pos); } @@ -623,7 +618,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE}; // Use smaller blockfiles in test-only -fastprune mode - but avoid // the possibility of having a block not fit into the block file. - if (gArgs.GetBoolArg("-fastprune", false)) { + if (m_opts.fast_prune) { max_blockfile_size = 0x10000; // 64kiB if (nAddSize >= max_blockfile_size) { // dynamically adjust the blockfile size to be larger than the added size @@ -697,7 +692,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP return true; } -static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) +bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -750,7 +745,7 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid return true; } -bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) +bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const { block.SetNull(); @@ -768,33 +763,33 @@ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::P } // Check the header - if (!CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) { + if (!CheckProofOfWork(block.GetHash(), block.nBits, GetConsensus())) { return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); } // Signet only: check block solution - if (consensusParams.signet_blocks && !CheckSignetBlockSolution(block, consensusParams)) { + if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) { return error("ReadBlockFromDisk: Errors in block solution at %s", pos.ToString()); } return true; } -bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) +bool BlockManager::ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const { - const FlatFilePos block_pos{WITH_LOCK(cs_main, return pindex->GetBlockPos())}; + const FlatFilePos block_pos{WITH_LOCK(cs_main, return index.GetBlockPos())}; - if (!ReadBlockFromDisk(block, block_pos, consensusParams)) { + if (!ReadBlockFromDisk(block, block_pos)) { return false; } - if (block.GetHash() != pindex->GetBlockHash()) { + if (block.GetHash() != index.GetBlockHash()) { return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", - pindex->ToString(), block_pos.ToString()); + index.ToString(), block_pos.ToString()); } return true; } -bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) +bool BlockManager::ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) const { FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header @@ -872,7 +867,7 @@ class ImportingNow } }; -void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles, const ArgsManager& args, const fs::path& mempool_path) +void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles, const fs::path& mempool_path) { SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); @@ -888,10 +883,10 @@ void ThreadImport(ChainstateManager& chainman, std::vector vImportFile std::multimap blocks_with_unknown_parent; while (true) { FlatFilePos pos(nFile, 0); - if (!fs::exists(GetBlockPosFilename(pos))) { + if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) { break; // No block files left to reindex } - FILE* file = OpenBlockFile(pos, true); + FILE* file = chainman.m_blockman.OpenBlockFile(pos, true); if (!file) { break; // This error is logged in OpenBlockFile } @@ -939,12 +934,13 @@ void ThreadImport(ChainstateManager& chainman, std::vector vImportFile } } - if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { + if (chainman.m_blockman.StopAfterBlockImport()) { LogPrintf("Stopping after block import\n"); StartShutdown(); return; } } // End scope of ImportingNow chainman.ActiveChainstate().LoadMempool(mempool_path); + g_indexes_ready_to_sync = true; } } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 8f699f8e68a1c8..a1ebb0df0a8787 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -20,7 +20,6 @@ #include #include -class ArgsManager; class BlockValidationState; class CBlock; class CBlockFileInfo; @@ -36,7 +35,6 @@ struct Params; } namespace node { -static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB @@ -49,6 +47,7 @@ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); extern std::atomic_bool fReindex; +extern std::atomic_bool g_indexes_ready_to_sync; // Because validation code takes pointers to the map's CBlockIndex objects, if // we ever switch to another associative container, we need to either use a @@ -96,6 +95,14 @@ class BlockManager bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown); bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); + FlatFileSeq BlockFileSeq() const; + FlatFileSeq UndoFileSeq() const; + + FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; + + bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const; + bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const; + /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ void FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight, int chain_tip_height); @@ -202,6 +209,8 @@ class BlockManager [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; } + [[nodiscard]] bool StopAfterBlockImport() const { return m_opts.stop_after_block_import; } + /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); @@ -219,28 +228,29 @@ class BlockManager //! Create or update a prune lock identified by its name void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); -}; -void CleanupBlockRevFiles(); + /** Open a block file (blk?????.dat) */ + FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const; -/** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false); -/** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const FlatFilePos& pos); + /** Translation to a filesystem path */ + fs::path GetBlockPosFilename(const FlatFilePos& pos) const; -/** - * Actually unlink the specified files - */ -void UnlinkPrunedFiles(const std::set& setFilesToPrune); + /** + * Actually unlink the specified files + */ + void UnlinkPrunedFiles(const std::set& setFilesToPrune) const; -/** Functions for disk access for blocks */ -bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); -bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); -bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); + /** Functions for disk access for blocks */ + bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const; + bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const; + bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) const; -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); + bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const; + + void CleanupBlockRevFiles() const; +}; -void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles, const ArgsManager& args, const fs::path& mempool_path); +void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles, const fs::path& mempool_path); } // namespace node #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index f60ff83a0dca4d..40b609d0691a4c 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -51,7 +51,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( pblocktree->WriteReindexing(true); //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (options.prune) { - CleanupBlockRevFiles(); + chainman.m_blockman.CleanupBlockRevFiles(); } } diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index 470b6739489dd4..58fa2e155d700c 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -11,16 +11,16 @@ #include #include #include +#include #include #include #include #include -#include #include namespace node { -std::optional ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) +util::Result ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) { if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; @@ -30,7 +30,7 @@ std::optional ApplyArgsManOptions(const ArgsManager& args, Chains if (auto value{args.GetArg("-minimumchainwork")}) { if (!IsHexNumber(*value)) { - return strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value); + return util::Error{strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value)}; } opts.minimum_chain_work = UintToArith256(uint256S(*value)); } @@ -43,6 +43,6 @@ std::optional ApplyArgsManOptions(const ArgsManager& args, Chains ReadDatabaseArgs(args, opts.coins_db); ReadCoinsViewArgs(args, opts.coins_view); - return std::nullopt; + return {}; } } // namespace node diff --git a/src/node/chainstatemanager_args.h b/src/node/chainstatemanager_args.h index 6c46b998f213a6..82eb0373682197 100644 --- a/src/node/chainstatemanager_args.h +++ b/src/node/chainstatemanager_args.h @@ -5,15 +5,13 @@ #ifndef BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H #define BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H +#include #include -#include - class ArgsManager; -struct bilingual_str; namespace node { -std::optional ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); +util::Result ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); } // namespace node #endif // BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 188bbc5f452b67..599147469c8eda 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -394,7 +394,7 @@ class NodeImpl : public Node NodeContext* m_context{nullptr}; }; -bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock& lock, const CChain& active) +bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock& lock, const CChain& active, const BlockManager& blockman) { if (!index) return false; if (block.m_hash) *block.m_hash = index->GetBlockHash(); @@ -404,10 +404,10 @@ bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLockGetMedianTimePast(); if (block.m_in_active_chain) *block.m_in_active_chain = active[index->nHeight] == index; if (block.m_locator) { *block.m_locator = GetLocator(index); } - if (block.m_next_block) FillBlock(active[index->nHeight] == index ? active[index->nHeight + 1] : nullptr, *block.m_next_block, lock, active); + if (block.m_next_block) FillBlock(active[index->nHeight] == index ? active[index->nHeight + 1] : nullptr, *block.m_next_block, lock, active, blockman); if (block.m_data) { REVERSE_LOCK(lock); - if (!ReadBlockFromDisk(*block.m_data, index, Params().GetConsensus())) block.m_data->SetNull(); + if (!blockman.ReadBlockFromDisk(*block.m_data, *index)) block.m_data->SetNull(); } block.found = true; return true; @@ -557,13 +557,13 @@ class ChainImpl : public Chain bool findBlock(const uint256& hash, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); - return FillBlock(chainman().m_blockman.LookupBlockIndex(hash), block, lock, chainman().ActiveChain()); + return FillBlock(chainman().m_blockman.LookupBlockIndex(hash), block, lock, chainman().ActiveChain(), chainman().m_blockman); } bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); const CChain& active = chainman().ActiveChain(); - return FillBlock(active.FindEarliestAtLeast(min_time, min_height), block, lock, active); + return FillBlock(active.FindEarliestAtLeast(min_time, min_height), block, lock, active, chainman().m_blockman); } bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out) override { @@ -571,10 +571,10 @@ class ChainImpl : public Chain const CChain& active = chainman().ActiveChain(); if (const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { - return FillBlock(ancestor, ancestor_out, lock, active); + return FillBlock(ancestor, ancestor_out, lock, active, chainman().m_blockman); } } - return FillBlock(nullptr, ancestor_out, lock, active); + return FillBlock(nullptr, ancestor_out, lock, active, chainman().m_blockman); } bool findAncestorByHash(const uint256& block_hash, const uint256& ancestor_hash, const FoundBlock& ancestor_out) override { @@ -582,7 +582,7 @@ class ChainImpl : public Chain const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash); const CBlockIndex* ancestor = chainman().m_blockman.LookupBlockIndex(ancestor_hash); if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; - return FillBlock(ancestor, ancestor_out, lock, chainman().ActiveChain()); + return FillBlock(ancestor, ancestor_out, lock, chainman().ActiveChain(), chainman().m_blockman); } bool findCommonAncestor(const uint256& block_hash1, const uint256& block_hash2, const FoundBlock& ancestor_out, const FoundBlock& block1_out, const FoundBlock& block2_out) override { @@ -594,9 +594,9 @@ class ChainImpl : public Chain // Using & instead of && below to avoid short circuiting and leaving // output uninitialized. Cast bool to int to avoid -Wbitwise-instead-of-logical // compiler warnings. - return int{FillBlock(ancestor, ancestor_out, lock, active)} & - int{FillBlock(block1, block1_out, lock, active)} & - int{FillBlock(block2, block2_out, lock, active)}; + return int{FillBlock(ancestor, ancestor_out, lock, active, chainman().m_blockman)} & + int{FillBlock(block1, block1_out, lock, active, chainman().m_blockman)} & + int{FillBlock(block2, block2_out, lock, active, chainman().m_blockman)}; } void findCoins(std::map& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index 294111a58a60e2..5381902263b51e 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -38,7 +38,7 @@ void ApplyArgsManOptions(const ArgsManager& argsman, MemPoolLimits& mempool_limi } } -std::optional ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts) +util::Result ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts) { mempool_opts.check_ratio = argsman.GetIntArg("-checkmempool", mempool_opts.check_ratio); @@ -52,7 +52,7 @@ std::optional ApplyArgsManOptions(const ArgsManager& argsman, con if (std::optional inc_relay_fee = ParseMoney(argsman.GetArg("-incrementalrelayfee", ""))) { mempool_opts.incremental_relay_feerate = CFeeRate{inc_relay_fee.value()}; } else { - return AmountErrMsg("incrementalrelayfee", argsman.GetArg("-incrementalrelayfee", "")); + return util::Error{AmountErrMsg("incrementalrelayfee", argsman.GetArg("-incrementalrelayfee", ""))}; } } @@ -61,7 +61,7 @@ std::optional ApplyArgsManOptions(const ArgsManager& argsman, con // High fee check is done afterward in CWallet::Create() mempool_opts.min_relay_feerate = CFeeRate{min_relay_feerate.value()}; } else { - return AmountErrMsg("minrelaytxfee", argsman.GetArg("-minrelaytxfee", "")); + return util::Error{AmountErrMsg("minrelaytxfee", argsman.GetArg("-minrelaytxfee", ""))}; } } else if (mempool_opts.incremental_relay_feerate > mempool_opts.min_relay_feerate) { // Allow only setting incremental fee to control both @@ -75,7 +75,7 @@ std::optional ApplyArgsManOptions(const ArgsManager& argsman, con if (std::optional parsed = ParseMoney(argsman.GetArg("-dustrelayfee", ""))) { mempool_opts.dust_relay_feerate = CFeeRate{parsed.value()}; } else { - return AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", "")); + return util::Error{AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", ""))}; } } @@ -89,12 +89,12 @@ std::optional ApplyArgsManOptions(const ArgsManager& argsman, con mempool_opts.require_standard = !argsman.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (!chainparams.IsTestChain() && !mempool_opts.require_standard) { - return strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString()); + return util::Error{strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString())}; } mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf); ApplyArgsManOptions(argsman, mempool_opts.limits); - return std::nullopt; + return {}; } diff --git a/src/node/mempool_args.h b/src/node/mempool_args.h index 52d8b4f265542d..630fee6421bda2 100644 --- a/src/node/mempool_args.h +++ b/src/node/mempool_args.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_NODE_MEMPOOL_ARGS_H #define BITCOIN_NODE_MEMPOOL_ARGS_H -#include +#include class ArgsManager; class CChainParams; @@ -21,7 +21,7 @@ struct MemPoolOptions; * @param[in] argsman The ArgsManager in which to check set options. * @param[in,out] mempool_opts The MemPoolOptions to modify according to \p argsman. */ -[[nodiscard]] std::optional ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, kernel::MemPoolOptions& mempool_opts); +[[nodiscard]] util::Result ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, kernel::MemPoolOptions& mempool_opts); #endif // BITCOIN_NODE_MEMPOOL_ARGS_H diff --git a/src/node/mini_miner.cpp b/src/node/mini_miner.cpp new file mode 100644 index 00000000000000..71ae9d23c79de2 --- /dev/null +++ b/src/node/mini_miner.cpp @@ -0,0 +1,366 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace node { + +MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector& outpoints) +{ + LOCK(mempool.cs); + // Find which outpoints to calculate bump fees for. + // Anything that's spent by the mempool is to-be-replaced + // Anything otherwise unavailable just has a bump fee of 0 + for (const auto& outpoint : outpoints) { + if (!mempool.exists(GenTxid::Txid(outpoint.hash))) { + // This UTXO is either confirmed or not yet submitted to mempool. + // If it's confirmed, no bump fee is required. + // If it's not yet submitted, we have no information, so return 0. + m_bump_fees.emplace(outpoint, 0); + continue; + } + + // UXTO is created by transaction in mempool, add to map. + // Note: This will either create a missing entry or add the outpoint to an existing entry + m_requested_outpoints_by_txid[outpoint.hash].push_back(outpoint); + + if (const auto ptx{mempool.GetConflictTx(outpoint)}) { + // This outpoint is already being spent by another transaction in the mempool. We + // assume that the caller wants to replace this transaction and its descendants. It + // would be unusual for the transaction to have descendants as the wallet won’t normally + // attempt to replace transactions with descendants. If the outpoint is from a mempool + // transaction, we still need to calculate its ancestors bump fees (added to + // m_requested_outpoints_by_txid below), but after removing the to-be-replaced entries. + // + // Note that the descendants of a transaction include the transaction itself. Also note, + // that this is only calculating bump fees. RBF fee rules should be handled separately. + CTxMemPool::setEntries descendants; + mempool.CalculateDescendants(mempool.GetIter(ptx->GetHash()).value(), descendants); + for (const auto& desc_txiter : descendants) { + m_to_be_replaced.insert(desc_txiter->GetTx().GetHash()); + } + } + } + + // No unconfirmed UTXOs, so nothing mempool-related needs to be calculated. + if (m_requested_outpoints_by_txid.empty()) return; + + // Calculate the cluster and construct the entry map. + std::vector txids_needed; + txids_needed.reserve(m_requested_outpoints_by_txid.size()); + for (const auto& [txid, _]: m_requested_outpoints_by_txid) { + txids_needed.push_back(txid); + } + const auto cluster = mempool.GatherClusters(txids_needed); + if (cluster.empty()) { + // An empty cluster means that at least one of the transactions is missing from the mempool + // (should not be possible given processing above) or DoS limit was hit. + m_ready_to_calculate = false; + return; + } + + // Add every entry to m_entries_by_txid and m_entries, except the ones that will be replaced. + for (const auto& txiter : cluster) { + if (!m_to_be_replaced.count(txiter->GetTx().GetHash())) { + auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), MiniMinerMempoolEntry(txiter)); + m_entries.push_back(mapiter); + } else { + auto outpoints_it = m_requested_outpoints_by_txid.find(txiter->GetTx().GetHash()); + if (outpoints_it != m_requested_outpoints_by_txid.end()) { + // This UTXO is the output of a to-be-replaced transaction. Bump fee is 0; spending + // this UTXO is impossible as it will no longer exist after the replacement. + for (const auto& outpoint : outpoints_it->second) { + m_bump_fees.emplace(outpoint, 0); + } + m_requested_outpoints_by_txid.erase(outpoints_it); + } + } + } + + // Build the m_descendant_set_by_txid cache. + for (const auto& txiter : cluster) { + const auto& txid = txiter->GetTx().GetHash(); + // Cache descendants for future use. Unlike the real mempool, a descendant MiniMinerMempoolEntry + // will not exist without its ancestor MiniMinerMempoolEntry, so these sets won't be invalidated. + std::vector cached_descendants; + const bool remove{m_to_be_replaced.count(txid) > 0}; + CTxMemPool::setEntries descendants; + mempool.CalculateDescendants(txiter, descendants); + Assume(descendants.count(txiter) > 0); + for (const auto& desc_txiter : descendants) { + const auto txid_desc = desc_txiter->GetTx().GetHash(); + const bool remove_desc{m_to_be_replaced.count(txid_desc) > 0}; + auto desc_it{m_entries_by_txid.find(txid_desc)}; + Assume((desc_it == m_entries_by_txid.end()) == remove_desc); + if (remove) Assume(remove_desc); + // It's possible that remove=false but remove_desc=true. + if (!remove && !remove_desc) { + cached_descendants.push_back(desc_it); + } + } + if (remove) { + Assume(cached_descendants.empty()); + } else { + m_descendant_set_by_txid.emplace(txid, cached_descendants); + } + } + + // Release the mempool lock; we now have all the information we need for a subset of the entries + // we care about. We will solely operate on the MiniMinerMempoolEntry map from now on. + Assume(m_in_block.empty()); + Assume(m_requested_outpoints_by_txid.size() <= outpoints.size()); + SanityCheck(); +} + +// Compare by min(ancestor feerate, individual feerate), then iterator +// +// Under the ancestor-based mining approach, high-feerate children can pay for parents, but high-feerate +// parents do not incentive inclusion of their children. Therefore the mining algorithm only considers +// transactions for inclusion on basis of the minimum of their own feerate or their ancestor feerate. +struct AncestorFeerateComparator +{ + template + bool operator()(const I& a, const I& b) const { + auto min_feerate = [](const MiniMinerMempoolEntry& e) -> CFeeRate { + const CAmount ancestor_fee{e.GetModFeesWithAncestors()}; + const int64_t ancestor_size{e.GetSizeWithAncestors()}; + const CAmount tx_fee{e.GetModifiedFee()}; + const int64_t tx_size{e.GetTxSize()}; + // Comparing ancestor feerate with individual feerate: + // ancestor_fee / ancestor_size <= tx_fee / tx_size + // Avoid division and possible loss of precision by + // multiplying both sides by the sizes: + return ancestor_fee * tx_size < tx_fee * ancestor_size ? + CFeeRate(ancestor_fee, ancestor_size) : + CFeeRate(tx_fee, tx_size); + }; + CFeeRate a_feerate{min_feerate(a->second)}; + CFeeRate b_feerate{min_feerate(b->second)}; + if (a_feerate != b_feerate) { + return a_feerate > b_feerate; + } + // Use txid as tiebreaker for stable sorting + return a->first < b->first; + } +}; + +void MiniMiner::DeleteAncestorPackage(const std::set& ancestors) +{ + Assume(ancestors.size() >= 1); + // "Mine" all transactions in this ancestor set. + for (auto& anc : ancestors) { + Assume(m_in_block.count(anc->first) == 0); + m_in_block.insert(anc->first); + m_total_fees += anc->second.GetModifiedFee(); + m_total_vsize += anc->second.GetTxSize(); + auto it = m_descendant_set_by_txid.find(anc->first); + // Each entry’s descendant set includes itself + Assume(it != m_descendant_set_by_txid.end()); + for (auto& descendant : it->second) { + // If these fail, we must be double-deducting. + Assume(descendant->second.GetModFeesWithAncestors() >= anc->second.GetModifiedFee()); + Assume(descendant->second.vsize_with_ancestors >= anc->second.GetTxSize()); + descendant->second.fee_with_ancestors -= anc->second.GetModifiedFee(); + descendant->second.vsize_with_ancestors -= anc->second.GetTxSize(); + } + } + // Delete these entries. + for (const auto& anc : ancestors) { + m_descendant_set_by_txid.erase(anc->first); + // The above loop should have deducted each ancestor's size and fees from each of their + // respective descendants exactly once. + Assume(anc->second.GetModFeesWithAncestors() == 0); + Assume(anc->second.GetSizeWithAncestors() == 0); + auto vec_it = std::find(m_entries.begin(), m_entries.end(), anc); + Assume(vec_it != m_entries.end()); + m_entries.erase(vec_it); + m_entries_by_txid.erase(anc); + } +} + +void MiniMiner::SanityCheck() const +{ + // m_entries, m_entries_by_txid, and m_descendant_set_by_txid all same size + Assume(m_entries.size() == m_entries_by_txid.size()); + Assume(m_entries.size() == m_descendant_set_by_txid.size()); + // Cached ancestor values should be at least as large as the transaction's own fee and size + Assume(std::all_of(m_entries.begin(), m_entries.end(), [](const auto& entry) { + return entry->second.GetSizeWithAncestors() >= entry->second.GetTxSize() && + entry->second.GetModFeesWithAncestors() >= entry->second.GetModifiedFee();})); + // None of the entries should be to-be-replaced transactions + Assume(std::all_of(m_to_be_replaced.begin(), m_to_be_replaced.end(), + [&](const auto& txid){return m_entries_by_txid.find(txid) == m_entries_by_txid.end();})); +} + +void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate) +{ + while (!m_entries_by_txid.empty()) { + // Sort again, since transaction removal may change some m_entries' ancestor feerates. + std::sort(m_entries.begin(), m_entries.end(), AncestorFeerateComparator()); + + // Pick highest ancestor feerate entry. + auto best_iter = m_entries.begin(); + Assume(best_iter != m_entries.end()); + const auto ancestor_package_size = (*best_iter)->second.GetSizeWithAncestors(); + const auto ancestor_package_fee = (*best_iter)->second.GetModFeesWithAncestors(); + // Stop here. Everything that didn't "make it into the block" has bumpfee. + if (ancestor_package_fee < target_feerate.GetFee(ancestor_package_size)) { + break; + } + + // Calculate ancestors on the fly. This lookup should be fairly cheap, and ancestor sets + // change at every iteration, so this is more efficient than maintaining a cache. + std::set ancestors; + { + std::set to_process; + to_process.insert(*best_iter); + while (!to_process.empty()) { + auto iter = to_process.begin(); + Assume(iter != to_process.end()); + ancestors.insert(*iter); + for (const auto& input : (*iter)->second.GetTx().vin) { + if (auto parent_it{m_entries_by_txid.find(input.prevout.hash)}; parent_it != m_entries_by_txid.end()) { + if (ancestors.count(parent_it) == 0) { + to_process.insert(parent_it); + } + } + } + to_process.erase(iter); + } + } + DeleteAncestorPackage(ancestors); + SanityCheck(); + } + Assume(m_in_block.empty() || m_total_fees >= target_feerate.GetFee(m_total_vsize)); + // Do not try to continue building the block template with a different feerate. + m_ready_to_calculate = false; +} + +std::map MiniMiner::CalculateBumpFees(const CFeeRate& target_feerate) +{ + if (!m_ready_to_calculate) return {}; + // Build a block template until the target feerate is hit. + BuildMockTemplate(target_feerate); + + // Each transaction that "made it into the block" has a bumpfee of 0, i.e. they are part of an + // ancestor package with at least the target feerate and don't need to be bumped. + for (const auto& txid : m_in_block) { + // Not all of the block transactions were necessarily requested. + auto it = m_requested_outpoints_by_txid.find(txid); + if (it != m_requested_outpoints_by_txid.end()) { + for (const auto& outpoint : it->second) { + m_bump_fees.emplace(outpoint, 0); + } + m_requested_outpoints_by_txid.erase(it); + } + } + + // A transactions and its ancestors will only be picked into a block when + // both the ancestor set feerate and the individual feerate meet the target + // feerate. + // + // We had to convince ourselves that after running the mini miner and + // picking all eligible transactions into our MockBlockTemplate, there + // could still be transactions remaining that have a lower individual + // feerate than their ancestor feerate. So here is an example: + // + // ┌─────────────────┐ + // │ │ + // │ Grandparent │ + // │ 1700 vB │ + // │ 1700 sats │ Target feerate: 10 s/vB + // │ 1 s/vB │ GP Ancestor Set Feerate (ASFR): 1 s/vB + // │ │ P1_ASFR: 9.84 s/vB + // └──────▲───▲──────┘ P2_ASFR: 2.47 s/vB + // │ │ C_ASFR: 10.27 s/vB + // ┌───────────────┐ │ │ ┌──────────────┐ + // │ ├────┘ └────┤ │ ⇒ C_FR < TFR < C_ASFR + // │ Parent 1 │ │ Parent 2 │ + // │ 200 vB │ │ 200 vB │ + // │ 17000 sats │ │ 3000 sats │ + // │ 85 s/vB │ │ 15 s/vB │ + // │ │ │ │ + // └───────────▲───┘ └───▲──────────┘ + // │ │ + // │ ┌───────────┐ │ + // └────┤ ├────┘ + // │ Child │ + // │ 100 vB │ + // │ 900 sats │ + // │ 9 s/vB │ + // │ │ + // └───────────┘ + // + // We therefore calculate both the bump fee that is necessary to elevate + // the individual transaction to the target feerate: + // target_feerate × tx_size - tx_fees + // and the bump fee that is necessary to bump the entire ancestor set to + // the target feerate: + // target_feerate × ancestor_set_size - ancestor_set_fees + // By picking the maximum from the two, we ensure that a transaction meets + // both criteria. + for (const auto& [txid, outpoints] : m_requested_outpoints_by_txid) { + auto it = m_entries_by_txid.find(txid); + Assume(it != m_entries_by_txid.end()); + if (it != m_entries_by_txid.end()) { + Assume(target_feerate.GetFee(it->second.GetSizeWithAncestors()) > std::min(it->second.GetModifiedFee(), it->second.GetModFeesWithAncestors())); + CAmount bump_fee_with_ancestors = target_feerate.GetFee(it->second.GetSizeWithAncestors()) - it->second.GetModFeesWithAncestors(); + CAmount bump_fee_individual = target_feerate.GetFee(it->second.GetTxSize()) - it->second.GetModifiedFee(); + const CAmount bump_fee{std::max(bump_fee_with_ancestors, bump_fee_individual)}; + Assume(bump_fee >= 0); + for (const auto& outpoint : outpoints) { + m_bump_fees.emplace(outpoint, bump_fee); + } + } + } + return m_bump_fees; +} + +std::optional MiniMiner::CalculateTotalBumpFees(const CFeeRate& target_feerate) +{ + if (!m_ready_to_calculate) return std::nullopt; + // Build a block template until the target feerate is hit. + BuildMockTemplate(target_feerate); + + // All remaining ancestors that are not part of m_in_block must be bumped, but no other relatives + std::set ancestors; + std::set to_process; + for (const auto& [txid, outpoints] : m_requested_outpoints_by_txid) { + // Skip any ancestors that already have a miner score higher than the target feerate + // (already "made it" into the block) + if (m_in_block.count(txid)) continue; + auto iter = m_entries_by_txid.find(txid); + if (iter == m_entries_by_txid.end()) continue; + to_process.insert(iter); + ancestors.insert(iter); + } + while (!to_process.empty()) { + auto iter = to_process.begin(); + const CTransaction& tx = (*iter)->second.GetTx(); + for (const auto& input : tx.vin) { + if (auto parent_it{m_entries_by_txid.find(input.prevout.hash)}; parent_it != m_entries_by_txid.end()) { + to_process.insert(parent_it); + ancestors.insert(parent_it); + } + } + to_process.erase(iter); + } + const auto ancestor_package_size = std::accumulate(ancestors.cbegin(), ancestors.cend(), int64_t{0}, + [](int64_t sum, const auto it) {return sum + it->second.GetTxSize();}); + const auto ancestor_package_fee = std::accumulate(ancestors.cbegin(), ancestors.cend(), CAmount{0}, + [](CAmount sum, const auto it) {return sum + it->second.GetModifiedFee();}); + return target_feerate.GetFee(ancestor_package_size) - ancestor_package_fee; +} +} // namespace node diff --git a/src/node/mini_miner.h b/src/node/mini_miner.h new file mode 100644 index 00000000000000..db07e6d1bf0dfb --- /dev/null +++ b/src/node/mini_miner.h @@ -0,0 +1,121 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_MINI_MINER_H +#define BITCOIN_NODE_MINI_MINER_H + +#include + +#include +#include +#include + +namespace node { + +// Container for tracking updates to ancestor feerate as we include ancestors in the "block" +class MiniMinerMempoolEntry +{ + const CAmount fee_individual; + const CTransactionRef tx; + const int64_t vsize_individual; + +// This class must be constructed while holding mempool.cs. After construction, the object's +// methods can be called without holding that lock. +public: + CAmount fee_with_ancestors; + int64_t vsize_with_ancestors; + explicit MiniMinerMempoolEntry(CTxMemPool::txiter entry) : + fee_individual{entry->GetModifiedFee()}, + tx{entry->GetSharedTx()}, + vsize_individual(entry->GetTxSize()), + fee_with_ancestors{entry->GetModFeesWithAncestors()}, + vsize_with_ancestors(entry->GetSizeWithAncestors()) + { } + + CAmount GetModifiedFee() const { return fee_individual; } + CAmount GetModFeesWithAncestors() const { return fee_with_ancestors; } + int64_t GetTxSize() const { return vsize_individual; } + int64_t GetSizeWithAncestors() const { return vsize_with_ancestors; } + const CTransaction& GetTx() const LIFETIMEBOUND { return *tx; } +}; + +// Comparator needed for std::set +struct IteratorComparator +{ + template + bool operator()(const I& a, const I& b) const + { + return &(*a) < &(*b); + } +}; + +/** A minimal version of BlockAssembler. Allows us to run the mining algorithm on a subset of + * mempool transactions, ignoring consensus rules, to calculate mining scores. */ +class MiniMiner +{ + // When true, a caller may use CalculateBumpFees(). Becomes false if we failed to retrieve + // mempool entries (i.e. cluster size too large) or bump fees have already been calculated. + bool m_ready_to_calculate{true}; + + // Set once per lifetime, fill in during initialization. + // txids of to-be-replaced transactions + std::set m_to_be_replaced; + + // If multiple argument outpoints correspond to the same transaction, cache them together in + // a single entry indexed by txid. Then we can just work with txids since all outpoints from + // the same tx will have the same bumpfee. Excludes non-mempool transactions. + std::map> m_requested_outpoints_by_txid; + + // What we're trying to calculate. + std::map m_bump_fees; + + // The constructed block template + std::set m_in_block; + + // Information on the current status of the block + CAmount m_total_fees{0}; + int32_t m_total_vsize{0}; + + /** Main data structure holding the entries, can be indexed by txid */ + std::map m_entries_by_txid; + using MockEntryMap = decltype(m_entries_by_txid); + + /** Vector of entries, can be sorted by ancestor feerate. */ + std::vector m_entries; + + /** Map of txid to its descendants. Should be inclusive. */ + std::map> m_descendant_set_by_txid; + + /** Consider this ancestor package "mined" so remove all these entries from our data structures. */ + void DeleteAncestorPackage(const std::set& ancestors); + + /** Perform some checks. */ + void SanityCheck() const; + +public: + /** Returns true if CalculateBumpFees may be called, false if not. */ + bool IsReadyToCalculate() const { return m_ready_to_calculate; } + + /** Build a block template until the target feerate is hit. */ + void BuildMockTemplate(const CFeeRate& target_feerate); + + /** Returns set of txids in the block template if one has been constructed. */ + std::set GetMockTemplateTxids() const { return m_in_block; } + + MiniMiner(const CTxMemPool& mempool, const std::vector& outpoints); + + /** Construct a new block template and, for each outpoint corresponding to a transaction that + * did not make it into the block, calculate the cost of bumping those transactions (and their + * ancestors) to the minimum feerate. Returns a map from outpoint to bump fee, or an empty map + * if they cannot be calculated. */ + std::map CalculateBumpFees(const CFeeRate& target_feerate); + + /** Construct a new block template and, calculate the cost of bumping all transactions that did + * not make it into the block to the target feerate. Returns the total bump fee, or std::nullopt + * if it cannot be calculated. */ + std::optional CalculateTotalBumpFees(const CFeeRate& target_feerate); +}; +} // namespace node + +#endif // BITCOIN_NODE_MINI_MINER_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index e4ae3a90e0d5b2..cdbf2ab4bffa3d 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -122,7 +122,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t return TransactionError::OK; } -CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, uint256& hashBlock, const BlockManager& blockman) { if (mempool && !block_index) { CTransactionRef ptx = mempool->get(hash); @@ -143,7 +143,7 @@ CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMe } if (block_index) { CBlock block; - if (ReadBlockFromDisk(block, block_index, consensusParams)) { + if (blockman.ReadBlockFromDisk(block, *block_index)) { for (const auto& tx : block.vtx) { if (tx->GetHash() == hash) { hashBlock = block_index->GetBlockHash(); diff --git a/src/node/transaction.h b/src/node/transaction.h index 45f174f13cd2c6..168273594ce50d 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -16,6 +16,7 @@ struct Params; } namespace node { +class BlockManager; struct NodeContext; /** Maximum fee rate for sendrawtransaction and testmempoolaccept RPC calls. @@ -53,11 +54,10 @@ static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; * @param[in] block_index The block to read from disk, or nullptr * @param[in] mempool If provided, check mempool for tx * @param[in] hash The txid - * @param[in] consensusParams The params * @param[out] hashBlock The block hash, if the tx was found via -txindex or block_index * @returns The tx if found, otherwise nullptr */ -CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, uint256& hashBlock, const BlockManager& blockman); } // namespace node #endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/random.cpp b/src/random.cpp index f4c51574ccc960..54500e6cc6be20 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -28,14 +28,10 @@ #include #endif -#ifdef HAVE_SYS_GETRANDOM -#include -#include -#endif -#if defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) -#include +#if defined(HAVE_GETRANDOM) || (defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX)) #include #endif + #ifdef HAVE_SYSCTL_ARND #include #endif @@ -252,7 +248,7 @@ static void Strengthen(const unsigned char (&seed)[32], SteadyClock::duration du /** Fallback: get 32 bytes of system entropy from /dev/urandom. The most * compatible way to get cryptographic randomness on UNIX-ish platforms. */ -static void GetDevURandom(unsigned char *ent32) +[[maybe_unused]] static void GetDevURandom(unsigned char *ent32) { int f = open("/dev/urandom", O_RDONLY); if (f == -1) { @@ -285,23 +281,14 @@ void GetOSRand(unsigned char *ent32) RandFailure(); } CryptReleaseContext(hProvider, 0); -#elif defined(HAVE_SYS_GETRANDOM) +#elif defined(HAVE_GETRANDOM) /* Linux. From the getrandom(2) man page: * "If the urandom source has been initialized, reads of up to 256 bytes * will always return as many bytes as requested and will not be * interrupted by signals." */ - int rv = syscall(SYS_getrandom, ent32, NUM_OS_RANDOM_BYTES, 0); - if (rv != NUM_OS_RANDOM_BYTES) { - if (rv < 0 && errno == ENOSYS) { - /* Fallback for kernel <3.17: the return value will be -1 and errno - * ENOSYS if the syscall is not available, in that case fall back - * to /dev/urandom. - */ - GetDevURandom(ent32); - } else { - RandFailure(); - } + if (getrandom(ent32, NUM_OS_RANDOM_BYTES, 0) != NUM_OS_RANDOM_BYTES) { + RandFailure(); } #elif defined(__OpenBSD__) /* OpenBSD. From the arc4random(3) man page: @@ -311,16 +298,10 @@ void GetOSRand(unsigned char *ent32) The function call is always successful. */ arc4random_buf(ent32, NUM_OS_RANDOM_BYTES); - // Silence a compiler warning about unused function. - (void)GetDevURandom; #elif defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) - /* getentropy() is available on macOS 10.12 and later. - */ if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { RandFailure(); } - // Silence a compiler warning about unused function. - (void)GetDevURandom; #elif defined(HAVE_SYSCTL_ARND) /* FreeBSD, NetBSD and similar. It is possible for the call to return less * bytes than requested, so need to read in a loop. @@ -334,8 +315,6 @@ void GetOSRand(unsigned char *ent32) } have += len; } while (have < NUM_OS_RANDOM_BYTES); - // Silence a compiler warning about unused function. - (void)GetDevURandom; #else /* Fall back to /dev/urandom if there is no specific method implemented to * get system entropy for this OS. diff --git a/src/rest.cpp b/src/rest.cpp index e46406f1ade456..dae064f89d52df 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -36,7 +36,6 @@ using node::GetTransaction; using node::NodeContext; -using node::ReadBlockFromDisk; static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000; @@ -311,7 +310,7 @@ static bool rest_block(const std::any& context, } - if (!ReadBlockFromDisk(block, pblockindex, chainman.GetParams().GetConsensus())) { + if (!chainman.m_blockman.ReadBlockFromDisk(block, *pblockindex)) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } @@ -716,7 +715,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string const NodeContext* const node = GetNodeContext(context, req); if (!node) return false; uint256 hashBlock = uint256(); - const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, node->mempool.get(), hash, Params().GetConsensus(), hashBlock); + const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, node->mempool.get(), hash, hashBlock, node->chainman->m_blockman); if (!tx) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e7a40f812c7bd0..4c25dbc3451d37 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -58,9 +58,7 @@ using kernel::CoinStatsHashType; using node::BlockManager; using node::NodeContext; -using node::ReadBlockFromDisk; using node::SnapshotMetadata; -using node::UndoReadFromDisk; struct CUpdatedBlock { @@ -183,7 +181,7 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: CBlockUndo blockUndo; const bool is_not_pruned{WITH_LOCK(::cs_main, return !blockman.IsBlockPruned(blockindex))}; - const bool have_undo{is_not_pruned && UndoReadFromDisk(blockUndo, blockindex)}; + const bool have_undo{is_not_pruned && blockman.UndoReadFromDisk(blockUndo, *blockindex)}; for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); @@ -430,7 +428,7 @@ static RPCHelpMan getblockfrompeer() "getblockfrompeer", "Attempt to fetch block from a given peer.\n\n" "We must have the header for this block, e.g. using submitheader.\n" - "Subsequent calls for the same block and a new peer will cause the response from the previous peer to be ignored.\n" + "Subsequent calls for the same block may cause the response from the previous peer to be ignored.\n" "Peers generally ignore requests for a stale block that they never fully verified, or one that is more than a month old.\n" "When a peer does not respond with a block, we will disconnect.\n" "Note: The block could be re-pruned as soon as it is received.\n\n" @@ -587,7 +585,7 @@ static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex* pblocki } } - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { + if (!blockman.ReadBlockFromDisk(block, *pblockindex)) { // Block not found on disk. This could be because we have the block // header in our index but not yet have the block or did not accept the // block. Or if the block was pruned right after we released the lock above. @@ -611,7 +609,7 @@ static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex* pblo } } - if (!UndoReadFromDisk(blockUndo, pblockindex)) { + if (!blockman.UndoReadFromDisk(blockUndo, *pblockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Can't read undo data from disk"); } diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index f3c19003ff9952..d08e2d55d1dbb6 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -133,6 +133,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "walletprocesspsbt", 1, "sign" }, { "walletprocesspsbt", 3, "bip32derivs" }, { "walletprocesspsbt", 4, "finalize" }, + { "descriptorprocesspsbt", 1, "descriptors"}, + { "descriptorprocesspsbt", 3, "bip32derivs" }, + { "descriptorprocesspsbt", 4, "finalize" }, { "createpsbt", 0, "inputs" }, { "createpsbt", 1, "outputs" }, { "createpsbt", 2, "locktime" }, diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 5fc3510c87f0b4..f79b8e55156c4d 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -163,7 +163,7 @@ static RPCHelpMan getmemoryinfo() { {"mode", RPCArg::Type::STR, RPCArg::Default{"stats"}, "determines what kind of information is returned.\n" " - \"stats\" returns general statistics about memory usage in the daemon.\n" - " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc 2.10+)."}, + " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc)."}, }, { RPCResult{"mode \"stats\"", diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 83c9b142040869..da4addb7509353 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -51,8 +51,6 @@ using node::FindCoins; using node::GetTransaction; using node::NodeContext; using node::PSBTAnalysis; -using node::ReadBlockFromDisk; -using node::UndoReadFromDisk; static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, Chainstate& active_chainstate, const CTxUndo* txundo = nullptr, @@ -207,8 +205,9 @@ static std::vector CreateTxDoc() }; } -// Update PSBT with information from the mempool, the UTXO set, the txindex, and the provided descriptors -PartiallySignedTransaction ProcessPSBT(const std::string& psbt_string, const std::any& context, const HidingSigningProvider& provider) +// Update PSBT with information from the mempool, the UTXO set, the txindex, and the provided descriptors. +// Optionally, sign the inputs that we can using information from the descriptors. +PartiallySignedTransaction ProcessPSBT(const std::string& psbt_string, const std::any& context, const HidingSigningProvider& provider, int sighash_type, bool finalize) { // Unserialize the transactions PartiallySignedTransaction psbtx; @@ -277,9 +276,10 @@ PartiallySignedTransaction ProcessPSBT(const std::string& psbt_string, const std } // Update script/keypath information using descriptor data. - // Note that SignPSBTInput does a lot more than just constructing ECDSA signatures - // we don't actually care about those here, in fact. - SignPSBTInput(provider, psbtx, /*index=*/i, &txdata, /*sighash=*/1); + // Note that SignPSBTInput does a lot more than just constructing ECDSA signatures. + // We only actually care about those if our signing provider doesn't hide private + // information, as is the case with `descriptorprocesspsbt` + SignPSBTInput(provider, psbtx, /*index=*/i, &txdata, sighash_type, /*out_sigdata=*/nullptr, finalize); } // Update script/keypath information using descriptor data. @@ -397,7 +397,7 @@ static RPCHelpMan getrawtransaction() } uint256 hash_block; - const CTransactionRef tx = GetTransaction(blockindex, node.mempool.get(), hash, chainman.GetConsensus(), hash_block); + const CTransactionRef tx = GetTransaction(blockindex, node.mempool.get(), hash, hash_block, chainman.m_blockman); if (!tx) { std::string errmsg; if (blockindex) { @@ -441,7 +441,7 @@ static RPCHelpMan getrawtransaction() if (tx->IsCoinBase() || !blockindex || is_block_pruned || - !(UndoReadFromDisk(blockUndo, blockindex) && ReadBlockFromDisk(block, blockindex, Params().GetConsensus()))) { + !(chainman.m_blockman.UndoReadFromDisk(blockUndo, *blockindex) && chainman.m_blockman.ReadBlockFromDisk(block, *blockindex))) { TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate()); return result; } @@ -1666,7 +1666,9 @@ static RPCHelpMan utxoupdatepsbt() const PartiallySignedTransaction& psbtx = ProcessPSBT( request.params[0].get_str(), request.context, - HidingSigningProvider(&provider, /*hide_secret=*/true, /*hide_origin=*/false)); + HidingSigningProvider(&provider, /*hide_secret=*/true, /*hide_origin=*/false), + /*sighash_type=*/SIGHASH_ALL, + /*finalize=*/false); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; @@ -1867,6 +1869,82 @@ static RPCHelpMan analyzepsbt() }; } +RPCHelpMan descriptorprocesspsbt() +{ + return RPCHelpMan{"descriptorprocesspsbt", + "\nUpdate all segwit inputs in a PSBT with information from output descriptors, the UTXO set or the mempool. \n" + "Then, sign the inputs we are able to with information from the output descriptors. ", + { + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction base64 string"}, + {"descriptors", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of either strings or objects", { + {"", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with an output descriptor and extra information", { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, + {"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "Up to what index HD chains should be explored (either end or [begin,end])"}, + }}, + }}, + {"sighashtype", RPCArg::Type::STR, RPCArg::Default{"DEFAULT for Taproot, ALL otherwise"}, "The signature hash type to sign with if not specified by the PSBT. Must be one of\n" + " \"DEFAULT\"\n" + " \"ALL\"\n" + " \"NONE\"\n" + " \"SINGLE\"\n" + " \"ALL|ANYONECANPAY\"\n" + " \"NONE|ANYONECANPAY\"\n" + " \"SINGLE|ANYONECANPAY\""}, + {"bip32derivs", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include BIP 32 derivation paths for public keys if we know them"}, + {"finalize", RPCArg::Type::BOOL, RPCArg::Default{true}, "Also finalize inputs if possible"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "psbt", "The base64-encoded partially signed transaction"}, + {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, + } + }, + RPCExamples{ + HelpExampleCli("descriptorprocesspsbt", "\"psbt\" \"[\\\"descriptor1\\\", \\\"descriptor2\\\"]\"") + + HelpExampleCli("descriptorprocesspsbt", "\"psbt\" \"[{\\\"desc\\\":\\\"mydescriptor\\\", \\\"range\\\":21}]\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + // Add descriptor information to a signing provider + FlatSigningProvider provider; + + auto descs = request.params[1].get_array(); + for (size_t i = 0; i < descs.size(); ++i) { + EvalDescriptorStringOrObject(descs[i], provider, /*expand_priv=*/true); + } + + int sighash_type = ParseSighashString(request.params[2]); + bool bip32derivs = request.params[3].isNull() ? true : request.params[3].get_bool(); + bool finalize = request.params[4].isNull() ? true : request.params[4].get_bool(); + + const PartiallySignedTransaction& psbtx = ProcessPSBT( + request.params[0].get_str(), + request.context, + HidingSigningProvider(&provider, /*hide_secret=*/false, !bip32derivs), + sighash_type, + finalize); + + // Check whether or not all of the inputs are now signed + bool complete = true; + for (const auto& input : psbtx.inputs) { + complete &= PSBTInputSigned(input); + } + + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + + UniValue result(UniValue::VOBJ); + + result.pushKV("psbt", EncodeBase64(ssTx)); + result.pushKV("complete", complete); + + return result; +}, + }; +} + void RegisterRawTransactionRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ @@ -1882,6 +1960,7 @@ void RegisterRawTransactionRPCCommands(CRPCTable& t) {"rawtransactions", &createpsbt}, {"rawtransactions", &converttopsbt}, {"rawtransactions", &utxoupdatepsbt}, + {"rawtransactions", &descriptorprocesspsbt}, {"rawtransactions", &joinpsbts}, {"rawtransactions", &analyzepsbt}, }; diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp index 24b5d041158a09..d74959cecce911 100644 --- a/src/rpc/txoutproof.cpp +++ b/src/rpc/txoutproof.cpp @@ -18,7 +18,6 @@ #include using node::GetTransaction; -using node::ReadBlockFromDisk; static RPCHelpMan gettxoutproof() { @@ -85,7 +84,7 @@ static RPCHelpMan gettxoutproof() } if (pblockindex == nullptr) { - const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, /*mempool=*/nullptr, *setTxids.begin(), chainman.GetConsensus(), hashBlock); + const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, /*mempool=*/nullptr, *setTxids.begin(), hashBlock, chainman.m_blockman); if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); } @@ -98,7 +97,7 @@ static RPCHelpMan gettxoutproof() } CBlock block; - if (!ReadBlockFromDisk(block, pblockindex, chainman.GetConsensus())) { + if (!chainman.m_blockman.ReadBlockFromDisk(block, *pblockindex)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); } diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index bff9bc2b29ca14..73e529a28c7d53 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1126,7 +1126,7 @@ std::pair ParseDescriptorRange(const UniValue& value) return {low, high}; } -std::vector EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider) +std::vector EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv) { std::string desc_str; std::pair range = {0, 1000}; @@ -1159,6 +1159,9 @@ std::vector EvalDescriptorStringOrObject(const UniValue& scanobject, Fl if (!desc->Expand(i, provider, scripts, provider)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys: '%s'", desc_str)); } + if (expand_priv) { + desc->ExpandPrivate(/*pos=*/i, provider, /*out=*/provider); + } std::move(scripts.begin(), scripts.end(), std::back_inserter(ret)); } return ret; diff --git a/src/rpc/util.h b/src/rpc/util.h index bb5c30a2f44b8b..3ff02582a65f64 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -110,7 +110,7 @@ UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_s std::pair ParseDescriptorRange(const UniValue& value); /** Evaluate a descriptor given as a string, or as a {"desc":...,"range":...} object, with default range of 1000. */ -std::vector EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider); +std::vector EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv = false); /** Returns, given services flags, a list of humanly readable (known) network services */ UniValue GetServicesNames(ServiceFlags services); diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index d22dbcf671c5d5..1ff5d6cf59151b 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -19,6 +19,7 @@ #include using node::BlockAssembler; +using node::BlockManager; using node::CBlockTemplate; BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) @@ -29,10 +30,10 @@ struct BuildChainTestingSetup : public TestChain100Setup { }; static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index, - uint256& last_header) + uint256& last_header, const BlockManager& blockman) { BlockFilter expected_filter; - if (!ComputeFilter(filter_index.GetFilterType(), block_index, expected_filter)) { + if (!ComputeFilter(filter_index.GetFilterType(), *block_index, expected_filter, blockman)) { BOOST_ERROR("ComputeFilter failed on block " << block_index->nHeight); return false; } @@ -155,7 +156,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) for (block_index = m_node.chainman->ActiveChain().Genesis(); block_index != nullptr; block_index = m_node.chainman->ActiveChain().Next(block_index)) { - CheckFilterLookups(filter_index, block_index, last_header); + CheckFilterLookups(filter_index, block_index, last_header, m_node.chainman->m_blockman); } } @@ -189,7 +190,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainA_last_header); + CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman); } // Reorg to chain B. @@ -207,7 +208,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainB_last_header); + CheckFilterLookups(filter_index, block_index, chainB_last_header, m_node.chainman->m_blockman); } // Check that filters for stale blocks on A can be retrieved. @@ -221,7 +222,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainA_last_header); + CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman); } // Reorg back to chain A. @@ -241,14 +242,14 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) block_index = m_node.chainman->m_blockman.LookupBlockIndex(chainA[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainA_last_header); + CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman); { LOCK(cs_main); block_index = m_node.chainman->m_blockman.LookupBlockIndex(chainB[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainB_last_header); + CheckFilterLookups(filter_index, block_index, chainB_last_header, m_node.chainman->m_blockman); } // Test lookups for a range of filters/hashes. diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index fd33a2114704da..f094766886c469 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -14,7 +14,6 @@ using node::BlockManager; using node::BLOCK_SERIALIZATION_HEADER_SIZE; using node::MAX_BLOCKFILE_SIZE; -using node::OpenBlockFile; // use BasicTestingSetup here for the data directory configuration, setup, and cleanup BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) @@ -22,8 +21,9 @@ BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) { const auto params {CreateChainParams(ArgsManager{}, ChainType::MAIN)}; - node::BlockManager::Options blockman_opts{ + const BlockManager::Options blockman_opts{ .chainparams = *params, + .blocks_dir = m_args.GetBlocksDirPath(), }; BlockManager blockman{blockman_opts}; CChain chain {}; @@ -67,13 +67,13 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain // Check that the file is not unlinked after ScanAndUnlinkAlreadyPrunedFiles // if m_have_pruned is not yet set WITH_LOCK(chainman->GetMutex(), blockman.ScanAndUnlinkAlreadyPrunedFiles()); - BOOST_CHECK(!AutoFile(OpenBlockFile(pos, true)).IsNull()); + BOOST_CHECK(!AutoFile(blockman.OpenBlockFile(pos, true)).IsNull()); // Check that the file is unlinked after ScanAndUnlinkAlreadyPrunedFiles // once m_have_pruned is set blockman.m_have_pruned = true; WITH_LOCK(chainman->GetMutex(), blockman.ScanAndUnlinkAlreadyPrunedFiles()); - BOOST_CHECK(AutoFile(OpenBlockFile(pos, true)).IsNull()); + BOOST_CHECK(AutoFile(blockman.OpenBlockFile(pos, true)).IsNull()); // Check that calling with already pruned files doesn't cause an error WITH_LOCK(chainman->GetMutex(), blockman.ScanAndUnlinkAlreadyPrunedFiles()); @@ -83,7 +83,7 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain BOOST_CHECK_NE(old_tip, new_tip); const int new_file_number{WITH_LOCK(chainman->GetMutex(), return new_tip->GetBlockPos().nFile)}; const FlatFilePos new_pos(new_file_number, 0); - BOOST_CHECK(!AutoFile(OpenBlockFile(new_pos, true)).IsNull()); + BOOST_CHECK(!AutoFile(blockman.OpenBlockFile(new_pos, true)).IsNull()); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 1c5197b7f98d62..10bc7ee6dc9505 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -14,14 +14,19 @@ #include #include +#include +#include +#include #include #include #include +#include #include #include #include #include #include +#include #include const std::function G_TEST_LOG_FUN{}; @@ -80,13 +85,13 @@ void initialize() return WrappedGetAddrInfo(name, false); }; - bool should_abort{false}; + bool should_exit{false}; if (std::getenv("PRINT_ALL_FUZZ_TARGETS_AND_ABORT")) { for (const auto& t : FuzzTargets()) { if (std::get<2>(t.second)) continue; std::cout << t.first << std::endl; } - should_abort = true; + should_exit = true; } if (const char* out_path = std::getenv("WRITE_ALL_FUZZ_TARGETS_AND_ABORT")) { std::cout << "Writing all fuzz target names to '" << out_path << "'." << std::endl; @@ -95,13 +100,23 @@ void initialize() if (std::get<2>(t.second)) continue; out_stream << t.first << std::endl; } - should_abort = true; + should_exit= true; + } + if (should_exit){ + std::exit(EXIT_SUCCESS); + } + if (const auto* env_fuzz{std::getenv("FUZZ")}) { + // To allow for easier fuzz executable binary modification, + static std::string g_copy{env_fuzz}; // create copy to avoid compiler optimizations, and + g_fuzz_target = g_copy.c_str(); // strip string after the first null-char. + } else { + std::cerr << "Must select fuzz target with the FUZZ env var." << std::endl; + std::cerr << "Hint: Set the PRINT_ALL_FUZZ_TARGETS_AND_ABORT=1 env var to see all compiled targets." << std::endl; + std::exit(EXIT_FAILURE); } - Assert(!should_abort); - g_fuzz_target = Assert(std::getenv("FUZZ")); const auto it = FuzzTargets().find(g_fuzz_target); if (it == FuzzTargets().end()) { - std::cerr << "No fuzzer for " << g_fuzz_target << "." << std::endl; + std::cerr << "No fuzz target compiled for " << g_fuzz_target << "." << std::endl; std::exit(EXIT_FAILURE); } Assert(!g_test_one_input); diff --git a/src/test/fuzz/mini_miner.cpp b/src/test/fuzz/mini_miner.cpp new file mode 100644 index 00000000000000..f49d9403931993 --- /dev/null +++ b/src/test/fuzz/mini_miner.cpp @@ -0,0 +1,192 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace { + +const TestingSetup* g_setup; +std::deque g_available_coins; +void initialize_miner() +{ + static const auto testing_setup = MakeNoLogFileContext(); + g_setup = testing_setup.get(); + for (uint32_t i = 0; i < uint32_t{100}; ++i) { + g_available_coins.push_back(COutPoint{uint256::ZERO, i}); + } +} + +// Test that the MiniMiner can run with various outpoints and feerates. +FUZZ_TARGET_INIT(mini_miner, initialize_miner) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CTxMemPool pool{CTxMemPool::Options{}}; + std::vector outpoints; + std::deque available_coins = g_available_coins; + LOCK2(::cs_main, pool.cs); + // Cluster size cannot exceed 500 + LIMITED_WHILE(!available_coins.empty(), 500) + { + CMutableTransaction mtx = CMutableTransaction(); + const size_t num_inputs = fuzzed_data_provider.ConsumeIntegralInRange(1, available_coins.size()); + const size_t num_outputs = fuzzed_data_provider.ConsumeIntegralInRange(1, 50); + for (size_t n{0}; n < num_inputs; ++n) { + auto prevout = available_coins.front(); + mtx.vin.push_back(CTxIn(prevout, CScript())); + available_coins.pop_front(); + } + for (uint32_t n{0}; n < num_outputs; ++n) { + mtx.vout.push_back(CTxOut(100, P2WSH_OP_TRUE)); + } + CTransactionRef tx = MakeTransactionRef(mtx); + TestMemPoolEntryHelper entry; + const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; + assert(MoneyRange(fee)); + pool.addUnchecked(entry.Fee(fee).FromTx(tx)); + + // All outputs are available to spend + for (uint32_t n{0}; n < num_outputs; ++n) { + if (fuzzed_data_provider.ConsumeBool()) { + available_coins.push_back(COutPoint{tx->GetHash(), n}); + } + } + + if (fuzzed_data_provider.ConsumeBool() && !tx->vout.empty()) { + // Add outpoint from this tx (may or not be spent by a later tx) + outpoints.push_back(COutPoint{tx->GetHash(), + (uint32_t)fuzzed_data_provider.ConsumeIntegralInRange(0, tx->vout.size())}); + } else { + // Add some random outpoint (will be interpreted as confirmed or not yet submitted + // to mempool). + auto outpoint = ConsumeDeserializable(fuzzed_data_provider); + if (outpoint.has_value() && std::find(outpoints.begin(), outpoints.end(), *outpoint) == outpoints.end()) { + outpoints.push_back(*outpoint); + } + } + + } + + const CFeeRate target_feerate{CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/1000)}}; + std::optional total_bumpfee; + CAmount sum_fees = 0; + { + node::MiniMiner mini_miner{pool, outpoints}; + assert(mini_miner.IsReadyToCalculate()); + const auto bump_fees = mini_miner.CalculateBumpFees(target_feerate); + for (const auto& outpoint : outpoints) { + auto it = bump_fees.find(outpoint); + assert(it != bump_fees.end()); + assert(it->second >= 0); + sum_fees += it->second; + } + assert(!mini_miner.IsReadyToCalculate()); + } + { + node::MiniMiner mini_miner{pool, outpoints}; + assert(mini_miner.IsReadyToCalculate()); + total_bumpfee = mini_miner.CalculateTotalBumpFees(target_feerate); + assert(total_bumpfee.has_value()); + assert(!mini_miner.IsReadyToCalculate()); + } + // Overlapping ancestry across multiple outpoints can only reduce the total bump fee. + assert (sum_fees >= *total_bumpfee); +} + +// Test that MiniMiner and BlockAssembler build the same block given the same transactions and constraints. +FUZZ_TARGET_INIT(mini_miner_selection, initialize_miner) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CTxMemPool pool{CTxMemPool::Options{}}; + // Make a copy to preserve determinism. + std::deque available_coins = g_available_coins; + std::vector transactions; + + LOCK2(::cs_main, pool.cs); + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) + { + CMutableTransaction mtx = CMutableTransaction(); + const size_t num_inputs = 2; + const size_t num_outputs = fuzzed_data_provider.ConsumeIntegralInRange(2, 5); + for (size_t n{0}; n < num_inputs; ++n) { + auto prevout = available_coins.front(); + mtx.vin.push_back(CTxIn(prevout, CScript())); + available_coins.pop_front(); + } + for (uint32_t n{0}; n < num_outputs; ++n) { + mtx.vout.push_back(CTxOut(100, P2WSH_OP_TRUE)); + } + CTransactionRef tx = MakeTransactionRef(mtx); + + // First 2 outputs are available to spend. The rest are added to outpoints to calculate bumpfees. + // There is no overlap between spendable coins and outpoints passed to MiniMiner because the + // MiniMiner interprets spent coins as to-be-replaced and excludes them. + for (uint32_t n{0}; n < num_outputs - 1; ++n) { + if (fuzzed_data_provider.ConsumeBool()) { + available_coins.push_front(COutPoint{tx->GetHash(), n}); + } else { + available_coins.push_back(COutPoint{tx->GetHash(), n}); + } + } + + // Stop if pool reaches DEFAULT_BLOCK_MAX_WEIGHT because BlockAssembler will stop when the + // block template reaches that, but the MiniMiner will keep going. + if (pool.GetTotalTxSize() + GetVirtualTransactionSize(*tx) >= DEFAULT_BLOCK_MAX_WEIGHT) break; + TestMemPoolEntryHelper entry; + const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; + assert(MoneyRange(fee)); + pool.addUnchecked(entry.Fee(fee).FromTx(tx)); + transactions.push_back(tx); + } + std::vector outpoints; + for (const auto& coin : g_available_coins) { + if (!pool.GetConflictTx(coin)) outpoints.push_back(coin); + } + for (const auto& tx : transactions) { + assert(pool.exists(GenTxid::Txid(tx->GetHash()))); + for (uint32_t n{0}; n < tx->vout.size(); ++n) { + COutPoint coin{tx->GetHash(), n}; + if (!pool.GetConflictTx(coin)) outpoints.push_back(coin); + } + } + const CFeeRate target_feerate{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; + + node::BlockAssembler::Options miner_options; + miner_options.blockMinFeeRate = target_feerate; + miner_options.nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; + miner_options.test_block_validity = false; + + node::BlockAssembler miner{g_setup->m_node.chainman->ActiveChainstate(), &pool, miner_options}; + node::MiniMiner mini_miner{pool, outpoints}; + assert(mini_miner.IsReadyToCalculate()); + + CScript spk_placeholder = CScript() << OP_0; + // Use BlockAssembler as oracle. BlockAssembler and MiniMiner should select the same + // transactions, stopping once packages do not meet target_feerate. + const auto blocktemplate{miner.CreateNewBlock(spk_placeholder)}; + mini_miner.BuildMockTemplate(target_feerate); + assert(!mini_miner.IsReadyToCalculate()); + auto mock_template_txids = mini_miner.GetMockTemplateTxids(); + // MiniMiner doesn't add a coinbase tx. + assert(mock_template_txids.count(blocktemplate->block.vtx[0]->GetHash()) == 0); + mock_template_txids.emplace(blocktemplate->block.vtx[0]->GetHash()); + assert(mock_template_txids.size() <= blocktemplate->block.vtx.size()); + assert(mock_template_txids.size() >= blocktemplate->block.vtx.size()); + assert(mock_template_txids.size() == blocktemplate->block.vtx.size()); + for (const auto& tx : blocktemplate->block.vtx) { + assert(mock_template_txids.count(tx->GetHash())); + } +} +} // namespace diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 919977d8bf694d..744ff4701d22bb 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -2,15 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include -#include #include #include #include +#include #include -#include #include