Merge pull request #116 from mxaddict/btc-master

Upstream changes 05/11/2023

.cirrus.yml

@@ -41,6 +41,7 @@ container_depends_template: &CONTAINER_DEPENDS_TEMPLATE
     cpu: 2
     greedy: true
     memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
+    dockerfile: ci/test_imagefile  # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment
   depends_built_cache:
     folder: "depends/built"
     fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends)
@@ -74,17 +75,18 @@ task:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: 'tidy [bookworm]'
+  name: 'tidy [lunar]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: debian:bookworm
     cpu: 2
     memory: 5G
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: ubuntu:lunar
+      FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh"
   # For faster CI feedback, immediately schedule the linters
   << : *CREDITS_TEMPLATE
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh"
 
 task:
   name: "Win64 native [vs2022]"
@@ -161,37 +163,42 @@ task:
   name: 'ARM [unit tests, no functional tests] [bullseye]'
   << : *GLOBAL_TASK_TEMPLATE
   arm_container:
-    image: debian:bullseye
     cpu: 2
     memory: 8G
+    dockerfile: ci/test_imagefile
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: debian:bullseye
+      FILE_ENV: "./ci/test/00_setup_env_arm.sh"
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_arm.sh"
     QEMU_USER_CMD: ""  # Disable qemu and run the test natively
 
 task:
   name: 'Win64 [unit tests, no boost::process, no functional tests] [jammy]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:jammy
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: ubuntu:jammy
+      FILE_ENV: "./ci/test/00_setup_env_win64.sh"
+  << : *CREDITS_TEMPLATE
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_win64.sh"
 
 task:
   name: '32-bit + dash [CentOS 8]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: quay.io/centos/centos:stream8
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream8
+      FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
   # For faster CI feedback, immediately schedule one task that runs all tests
   << : *CREDITS_TEMPLATE
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     PACKAGE_MANAGER_INSTALL: "yum install -y"
-    FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
 
 task:
-  name: '[previous releases, dev package and some depends packages, DEBUG] [unsigned char] [buster]'
+  name: '[previous releases, dev package and depends packages, DEBUG] [focal]'
   previous_releases_cache:
     folder: "releases"
   << : *GLOBAL_TASK_TEMPLATE
@@ -201,37 +208,40 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_native.sh"
 
 task:
-  name: '[TSan, depends] [jammy]'
+  name: '[TSan, depends] [lunar]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:jammy
     cpu: 6  # Increase CPU and Memory to avoid timeout
     memory: 24G
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: ubuntu:lunar
+      FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
 
 task:
-  name: '[MSan, depends] [focal]'
+  name: '[MSan, depends] [lunar]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:focal
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: ubuntu:lunar
+      FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
     MAKEJOBS: "-j4"  # Avoid excessive memory use due to MSan
 
 task:
-  name: '[ASan + LSan + UBSan + integer, no depends, USDT] [jammy]'
+  name: '[ASan + LSan + UBSan + integer, no depends, USDT] [lunar]'
   << : *GLOBAL_TASK_TEMPLATE
   # We can't use a 'container' for the USDT interface tests as the CirrusCI
   # containers don't have privileges to hook into bitcoind. CirrusCI uses
   # Google Compute Engine instances: https://cirrus-ci.org/guide/custom-vms/
   # Images can be found here: https://cloud.google.com/compute/docs/images/os-details
   compute_engine_instance:
     image_project: ubuntu-os-cloud
-    image: family/ubuntu-2204-lts # when upgrading, check if we can drop "ADD_UNTRUSTED_BPFCC_PPA"
+    image: family/ubuntu-2304-amd64 # https://cirrus-ci.org/guide/custom-vms/#custom-compute-engine-vms
     cpu: 4
+    disk: 100
     memory: 12G
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
@@ -240,49 +250,55 @@ task:
     MAKEJOBS: "-j4"  # Avoid excessive memory use
 
 task:
-  name: '[fuzzer,address,undefined,integer, no depends] [jammy]'
+  name: '[fuzzer,address,undefined,integer, no depends] [lunar]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:jammy
-    cpu: 8  # Increase CPU and memory to avoid timeout
-    memory: 32G
+    cpu: 4  # Increase CPU and memory to avoid timeout
+    memory: 16G
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: ubuntu:lunar
+      FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
 
 task:
   name: '[multiprocess, i686, DEBUG] [focal]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:focal
     cpu: 4
     memory: 16G  # The default memory is sometimes just a bit too small, so double everything
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: ubuntu:focal
+      FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
 
 task:
-  name: '[no wallet, libbitcoinkernel] [buster]'
+  name: '[no wallet, libbitcoinkernel] [focal]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: debian:buster
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: ubuntu:focal
+      FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
+  << : *CREDITS_TEMPLATE
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
 
 task:
   name: 'macOS 10.15 [no tests] [focal]'
   << : *CONTAINER_DEPENDS_TEMPLATE
   container:
-    image: ubuntu:focal
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: ubuntu:focal
+      FILE_ENV: "./ci/test/00_setup_env_mac.sh"
+  << : *CREDITS_TEMPLATE
   macos_sdk_cache:
     folder: "depends/SDKs/$MACOS_SDK"
     fingerprint_key: "$MACOS_SDK"
   << : *MAIN_TEMPLATE
   env:
     MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers"
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_mac.sh"
 
 task:
   name: 'macOS 13 native arm64 [sqlite only] [no depends]'
@@ -305,7 +321,10 @@ task:
   name: 'ARM64 Android APK [jammy]'
   << : *CONTAINER_DEPENDS_TEMPLATE
   container:
-    image: ubuntu:jammy
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: ubuntu:jammy
+      FILE_ENV: "./ci/test/00_setup_env_android.sh"
+  << : *CREDITS_TEMPLATE
   android_sdk_cache:
     folder: "depends/SDKs/android"
     fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.2.8568313"
@@ -315,4 +334,3 @@ task:
   << : *MAIN_TEMPLATE
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_android.sh"

.python-version

@@ -1 +1 @@
-3.7.16
+3.8.16

build_msvc/libsecp256k1/libsecp256k1.vcxproj

@@ -15,6 +15,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <PreprocessorDefinitions>ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <UndefinePreprocessorDefinitions>USE_ASM_X86_64;%(UndefinePreprocessorDefinitions)</UndefinePreprocessorDefinitions>
       <AdditionalIncludeDirectories>..\..\src\secp256k1;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <DisableSpecificWarnings>4146;4244;4267;4334</DisableSpecificWarnings>
     </ClCompile>

build_msvc/msvc-autogen.py

@@ -111,7 +111,6 @@ def main():
         set_properties(vcxproj_filename, '@SOURCE_FILES@\n', content)
     parse_config_into_btc_config()
     copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h'))
-    copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h'))
 
 if __name__ == '__main__':
     main()

build_msvc/vcpkg.json

@@ -13,5 +13,12 @@
       "features": ["thread"]
     },
     "zeromq"
+  ],
+  "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996",
+  "overrides": [
+    {
+      "name": "libevent",
+      "version": "2.1.12#7"
+    }
   ]
 }

ci/README.md

@@ -14,10 +14,10 @@ testing compared to other parts of the codebase. If you want to keep the work tr
 system in a virtual machine with a Linux operating system of your choice.
 
 To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
-requires `docker` to be installed. To install all requirements on Ubuntu, run
+requires `bash`, `docker`, and `python3` to be installed. To install all requirements on Ubuntu, run
 
 ```
-sudo apt install docker.io bash
+sudo apt install bash docker.io python3
 ```
 
 To run the default test stage,

ci/lint/Dockerfile

@@ -5,7 +5,7 @@
 # entire repo as docker context during build; if it lived elsewhere, it wouldn't be
 # able to make back-references to pull in the install scripts. So here it lives.
 
-FROM python:3.7-buster
+FROM python:3.8-buster
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV LC_ALL=C.UTF-8

ci/test/00_setup_env.sh

@@ -65,6 +65,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
 # The folder for previous release binaries.
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
+export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu"
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}
 export GOAL=${GOAL:-install}

ci/test/00_setup_env_i686_centos.sh

@@ -8,7 +8,8 @@ export LC_ALL=C.UTF-8
 
 export HOST=i686-pc-linux-gnu
 export CONTAINER_NAME=ci_i686_centos
-export CI_IMAGE_NAME_TAG=quay.io/centos/centos:stream8
+export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream8"
+# Use minimum supported python3.8 and gcc-8, see doc/dependencies.md
 export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python38 python38-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison"
 export PIP_PACKAGES="pyzmq"
 export GOAL="install"

ci/test/00_setup_env_native.sh

@@ -7,15 +7,15 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native
-export CI_IMAGE_NAME_TAG=debian:buster
-# Use minimum supported python3.7 and gcc-8, see doc/dependencies.md
-export PACKAGES="gcc-8 g++-8 python3-zmq libdbus-1-dev libharfbuzz-dev"
-export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8"
+export CI_IMAGE_NAME_TAG="ubuntu:20.04"
+# Use minimum supported python3.8 and gcc-8 (or best-effort gcc-9), see doc/dependencies.md
+export PACKAGES="gcc-9 g++-9 python3-zmq libdbus-1-dev libharfbuzz-dev"
+export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-9 CXX=g++-9"
 export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
 export RUN_UNIT_TESTS_SEQUENTIAL="true"
 export RUN_UNIT_TESTS="false"
 export GOAL="install"
 export NO_WERROR=1
 export DOWNLOAD_PREVIOUS_RELEASES="true"
 export BITCOIN_CONFIG="--enable-zmq --with-libs=no --enable-reduce-exports \
---enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8"
+--enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\""

ci/test/00_setup_env_native_asan.sh

@@ -8,19 +8,14 @@ export LC_ALL=C.UTF-8
 
 # Only install BCC tracing packages in Cirrus CI.
 if [[ "${CIRRUS_CI}" == "true" ]]; then
-  # We install an up-to-date 'bpfcc-tools' package from an untrusted PPA.
-  # This can be dropped with the next Ubuntu or Debian release that includes up-to-date packages.
-  # See the if-then in ci/test/04_install.sh too.
-  export ADD_UNTRUSTED_BPFCC_PPA=true
   export BPFCC_PACKAGE="bpfcc-tools"
 else
-  export ADD_UNTRUSTED_BPFCC_PPA=false
   export BPFCC_PACKAGE=""
 fi
 
 export CONTAINER_NAME=ci_native_asan
-export PACKAGES="systemtap-sdt-dev clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}"
-export CI_IMAGE_NAME_TAG=ubuntu:22.04
+export PACKAGES="systemtap-sdt-dev clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev ${BPFCC_PACKAGE}"
+export CI_IMAGE_NAME_TAG=ubuntu:23.04 # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
 export NO_DEPENDS=1
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"
+export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang-16 CXX=clang++-16"

ci/test/00_setup_env_native_fuzz.sh

@@ -6,13 +6,13 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
 export CONTAINER_NAME=ci_native_fuzz
-export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev"
+export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang -ftrivial-auto-var-init=pattern' CXX='clang++ -ftrivial-auto-var-init=pattern'"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'"
 export CCACHE_SIZE=200M

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -6,14 +6,14 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:20.04"
+export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
 LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
 
 export CONTAINER_NAME="ci_native_fuzz_msan"
-export PACKAGES="clang-12 llvm-12 cmake"
+export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake"
 # BDB generates false-positives and will be removed in future
 export DEP_OPTS="NO_BDB=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -6,15 +6,15 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="debian:bookworm"
 export CONTAINER_NAME=ci_native_fuzz_valgrind
-export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind"
+export PACKAGES="clang llvm libclang-rt-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export FUZZ_TESTS_CONFIG="--valgrind"
 export GOAL="install"
-# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5
+# Temporarily pin dwarf 4, until using Valgrind 3.20 or later
 export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'"
 export CCACHE_SIZE=200M