Merge remote-tracking branch 'bitcoin/27.x'

.cirrus.yml

@@ -22,7 +22,7 @@ env:  # Global defaults
 # - The ./ci/ depedencies (with cirrus-cli) should be installed:
 #
 #   ```
-#   apt update && apt install screen python3 bash podman-docker curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus
+#   apt update && apt install git screen python3 bash podman-docker curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus
 #   ```
 #
 # - There are no strict requirements on the hardware, because having less CPUs
@@ -43,13 +43,12 @@ env:  # Global defaults
 # The following specific types should exist, with the following requirements:
 # - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory.
 # - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory.
-# - noble: For a machine running the Linux kernel shipped with exaclty Ubuntu Noble 24.04. The machine is recommended to have 4 CPUs and 16 GB of memory.
 # - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory.
 
 base_template: &BASE_TEMPLATE
   merge_base_script:
     # Unconditionally install git (used in fingerprint_script).
-    - bash -c "$PACKAGE_MANAGER_INSTALL git"
+    - git --version || bash -c "$PACKAGE_MANAGER_INSTALL git"
     - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
     - git fetch --depth=1 $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge"
     - git checkout FETCH_HEAD  # Use merged changes to detect silent merge conflicts
@@ -152,20 +151,6 @@ task:
   env:
     FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
 
-task:
-  name: 'ASan + LSan + UBSan + integer, no depends, USDT'
-  enable_bpfcc_script:
-    # In the image build step, no external environment variables are available,
-    # so any settings will need to be written to the settings env file:
-    - sed -i "s|\${CIRRUS_CI}|true|g" ./ci/test/00_setup_env_native_asan.sh
-  << : *GLOBAL_TASK_TEMPLATE
-  persistent_worker:
-    labels:
-      type: noble  # Must use this specific worker (needed for USDT functional tests)
-  timeout_in: 300m  # Use longer timeout for the *rare* case where a full build (llvm + msan + depends + ...) needs to be done.
-  env:
-    FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
-
 task:
   name: 'fuzzer,address,undefined,integer, no depends'
   << : *GLOBAL_TASK_TEMPLATE

.github/workflows/ci.yml

@@ -18,7 +18,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  DANGER_RUN_CI_ON_HOST: 1
   CI_FAILFAST_TEST_LEAVE_DANGLING: 1  # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error
   MAKEJOBS: '-j10'
 
@@ -83,6 +82,7 @@ jobs:
     timeout-minutes: 120
 
     env:
+      DANGER_RUN_CI_ON_HOST: 1
       FILE_ENV: './ci/test/00_setup_env_mac_native.sh'
       BASE_ROOT_DIR: ${{ github.workspace }}
 
@@ -98,14 +98,17 @@ jobs:
       - name: Install Homebrew packages
         env:
           HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
-        run: brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq
+        run: |
+          # A workaround for "The `brew link` step did not complete successfully" error.
+          brew install python@3 || brew link --overwrite python@3
+          brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq
 
       - name: Set Ccache directory
         run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"
 
       - name: Restore Ccache cache
         id: ccache-cache
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ${{ env.CCACHE_DIR }}
           key: ${{ github.job }}-ccache-${{ github.run_id }}
@@ -115,7 +118,7 @@ jobs:
         run: ./ci/test_run_all.sh
 
       - name: Save Ccache cache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
@@ -147,16 +150,17 @@ jobs:
         with:
           arch: x64
 
-      - name: Check MSBuild
+      - name: Get tool information
         run: |
           msbuild -version | Out-File -FilePath "$env:GITHUB_WORKSPACE\msbuild_version"
           Get-Content -Path "$env:GITHUB_WORKSPACE\msbuild_version"
           $env:VCToolsVersion | Out-File -FilePath "$env:GITHUB_WORKSPACE\toolset_version"
-          Get-Content -Path "$env:GITHUB_WORKSPACE\toolset_version"
+          Write-Host "VCToolsVersion $(Get-Content -Path "$env:GITHUB_WORKSPACE\toolset_version")"
+          py -3 --version
 
       - name: Ccache installation cache
         id: ccache-installation-cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             C:\ProgramData\chocolatey\lib\ccache
@@ -173,7 +177,7 @@ jobs:
 
       - name: Restore Ccache cache
         id: ccache-cache
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ~/AppData/Local/ccache
           key: ${{ github.job }}-ccache-${{ github.run_id }}
@@ -189,13 +193,13 @@ jobs:
           Get-Content -Path "$env:GITHUB_WORKSPACE\vcpkg_commit"
 
       - name: vcpkg tools cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: C:/vcpkg/downloads/tools
           key: ${{ github.job }}-vcpkg-tools
 
       - name: vcpkg binary cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/AppData/Local/vcpkg/archives
           key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('vcpkg_commit', 'msbuild_version', 'toolset_version', 'build_msvc/vcpkg.json') }}
@@ -213,7 +217,7 @@ jobs:
         run: ccache --show-stats
 
       - name: Save Ccache cache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
         with:
           path: ~/AppData/Local/ccache
@@ -235,4 +239,44 @@ jobs:
       - name: Run functional tests
         env:
           TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }}
-        run: py -3 test\functional\test_runner.py --jobs $env:NUMBER_OF_PROCESSORS --ci --quiet --tmpdirprefix=$env:RUNNER_TEMP --combinedlogslen=99999999 --timeout-factor=$env:TEST_RUNNER_TIMEOUT_FACTOR $env:TEST_RUNNER_EXTRA --exclude mempool_limit
+        shell: cmd
+        run: py -3 test\functional\test_runner.py --jobs %NUMBER_OF_PROCESSORS% --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% %TEST_RUNNER_EXTRA%
+
+  asan-lsan-ubsan-integer-no-depends-usdt:
+    name: 'ASan + LSan + UBSan + integer, no depends, USDT'
+    runs-on: ubuntu-24.04 # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools
+    # No need to run on the read-only mirror, unless it is a PR.
+    if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request'
+    timeout-minutes: 120
+    env:
+      FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set Ccache directory
+        run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"
+
+      - name: Restore Ccache cache
+        id: ccache-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ${{ github.job }}-ccache-${{ github.run_id }}
+          restore-keys: ${{ github.job }}-ccache-
+
+      - name: Enable bpfcc script
+        # In the image build step, no external environment variables are available,
+        # so any settings will need to be written to the settings env file:
+        run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh
+
+      - name: CI script
+        run: ./ci/test_run_all.sh
+
+      - name: Save Ccache cache
+        uses: actions/cache/save@v4
+        if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+          key: ${{ github.job }}-ccache-${{ github.run_id }}

.tx/config

@@ -1,7 +1,7 @@
 [main]
 host = https://www.transifex.com
 
-[o:bitcoin:p:bitcoin:r:qt-translation-026x]
+[o:bitcoin:p:bitcoin:r:qt-translation-027x]
 file_filter = src/qt/locale/bitcoin_<lang>.xlf
 source_file = src/qt/locale/bitcoin_en.xlf
 source_lang = en

CONTRIBUTING.md

@@ -58,6 +58,11 @@ Most communication about Navio Core development happens on Discord, in the
 Discussion about codebase improvements happens in GitHub issues and pull
 requests.
 
+The developer
+[mailing list](https://groups.google.com/g/bitcoindev)
+should be used to discuss complicated or controversial consensus or P2P protocol changes before working on
+a patch set.
+Archives can be found on [https://gnusha.org/pi/bitcoindev/](https://gnusha.org/pi/bitcoindev/).
 
 Contributor Workflow
 --------------------

build-aux/m4/l_atomic.m4

@@ -7,7 +7,7 @@ dnl warranty.
 # Clang, when building for 32-bit,
 # and linking against libstdc++, requires linking with
 # -latomic if using the C++ atomic library.
-# Can be tested with: clang++ test.cpp -m32
+# Can be tested with: clang++ -std=c++20 test.cpp -m32
 #
 # Sourced from http://bugs.debian.org/797228
 
@@ -27,8 +27,11 @@ m4_define([_CHECK_ATOMIC_testbody], [[
     auto t1 = t.load();
     t.compare_exchange_strong(t1, 3s);
 
-    std::atomic<int64_t> a{};
+    std::atomic<double> d{};
+    d.store(3.14);
+    auto d1 = d.load();
 
+    std::atomic<int64_t> a{};
     int64_t v = 5;
     int64_t r = a.fetch_add(v);
     return static_cast<int>(r);

build_msvc/test_navio/test_navio.vcxproj

@@ -10,7 +10,7 @@
   </PropertyGroup>
   <ItemGroup>
     <ClCompile Include="..\..\src\test\*_properties.cpp" />
-    <ClCompile Include="..\..\src\test\*_tests.cpp" />
+    <ClCompile Include="..\..\src\test\*_tests.cpp" Exclude="..\..\src\test\ipc_tests.cpp" />
     <ClCompile Include="..\..\src\test\gen\*_gen.cpp" />
     <ClCompile Include="..\..\src\test\main.cpp" />
     <ClCompile Include="..\..\src\test\util\*.cpp" />

ci/README.md

@@ -14,10 +14,10 @@ testing compared to other parts of the codebase. If you want to keep the work tr
 system in a virtual machine with a Linux operating system of your choice.
 
 To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
-requires `bash`, `docker`, and `python3` to be installed. To install all requirements on Ubuntu, run
+requires `bash`, `docker`, and `python3` to be installed. To run on different architectures than the host `qemu` is also required. To install all requirements on Ubuntu, run
 
 ```
-sudo apt install bash docker.io python3
+sudo apt install bash docker.io python3 qemu-user-static
 ```
 
 It is recommended to run the ci system in a clean env. To run the test stage

ci/lint/04_install.sh

@@ -10,10 +10,11 @@ export PATH=$PWD/ci/retry:$PATH
 
 ${CI_RETRY_EXE} apt-get update
 # Lint dependencies:
+# - automake pkg-config libtool (for lint_includes_build_config)
 # - curl/xz-utils (to install shellcheck)
 # - git (used in many lint scripts)
 # - gpg (used by verify-commits)
-${CI_RETRY_EXE} apt-get install -y curl xz-utils git gpg
+${CI_RETRY_EXE} apt-get install -y automake pkg-config libtool curl xz-utils git gpg
 
 PYTHON_PATH="/python_build"
 if [ ! -d "${PYTHON_PATH}/bin" ]; then

ci/test/00_setup_env_mac_native.sh

@@ -7,7 +7,9 @@
 export LC_ALL=C.UTF-8
 
 export HOST=x86_64-apple-darwin
-export PIP_PACKAGES="zmq"
+# Homebrew's [email protected] is marked as externally managed (PEP 668).
+# Therefore, `--break-system-packages` is needed.
+export PIP_PACKAGES="--break-system-packages zmq"
 export GOAL="install"
 export BITCOIN_CONFIG="--with-miniupnpc --with-natpmp --enable-reduce-exports"
 export CI_OS_NAME="macos"

ci/test/00_setup_env_native_asan.sh

@@ -7,8 +7,10 @@
 export LC_ALL=C.UTF-8
 
 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
-# Only install BCC tracing packages in Cirrus CI.
-if [[ "${CIRRUS_CI}" == "true" ]]; then
+
+# Only install BCC tracing packages in CI. Container has to match the host for BCC to work.
+if [[ "${INSTALL_BCC_TRACING_TOOLS}" == "true" ]]; then
+  # Required for USDT functional tests to run
   BPFCC_PACKAGE="bpfcc-tools linux-headers-$(uname --kernel-release)"
   export CI_CONTAINER_CAP="--privileged -v /sys/kernel:/sys/kernel:rw"
 else

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
@@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
 export CONTAINER_NAME="ci_native_fuzz_msan"
 export PACKAGES="ninja-build"
 # BDB generates false-positives and will be removed in future
-export DEP_OPTS="NO_BDB=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export DEP_OPTS="DEBUG=1 NO_BDB=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export USE_MEMORY_SANITIZER="true"

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -16,5 +16,5 @@ export RUN_FUZZ_TESTS=true
 export FUZZ_TESTS_CONFIG="--valgrind"
 export GOAL="install"
 # Temporarily pin dwarf 4, until using Valgrind 3.20 or later
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC='clang -gdwarf-4' CXX='clang++ -gdwarf-4'"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS=-gdwarf-4 CXXFLAGS=-gdwarf-4"
 export CCACHE_MAXSIZE=200M

ci/test/00_setup_env_native_msan.sh

@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls -fsanitize-blacklist=${BASE_ROOT_DIR}/test/sanitizer_suppressions/msan"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
@@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
 export CONTAINER_NAME="ci_native_msan"
 export PACKAGES="ninja-build"
 # BDB generates false-positives and will be removed in future
-export DEP_OPTS="NO_BDB=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export DEP_OPTS="DEBUG=1 NO_BDB=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
 export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export USE_MEMORY_SANITIZER="true"

ci/test/00_setup_env_native_valgrind.sh

@@ -14,4 +14,4 @@ export NO_DEPENDS=1
 export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
 export GOAL="install"
 # Temporarily pin dwarf 4, until using Valgrind 3.20 or later
-export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb CC='clang -gdwarf-4' CXX='clang++ -gdwarf-4'"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb CC=clang CXX=clang++ CFLAGS=-gdwarf-4 CXXFLAGS=-gdwarf-4"  # TODO enable GUI

ci/test/00_setup_env_s390x.sh

@@ -9,8 +9,8 @@ export LC_ALL=C.UTF-8
 export HOST=s390x-linux-gnu
 export PACKAGES="python3-zmq"
 export CONTAINER_NAME=ci_s390x
-export CI_IMAGE_NAME_TAG="docker.io/s390x/debian:bookworm"
-export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
+export CI_IMAGE_NAME_TAG="docker.io/s390x/ubuntu:24.04"
+export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
 export RUN_FUNCTIONAL_TESTS=true
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-reduce-exports"

ci/test/01_base_install.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018-2022 The Bitcoin Core developers
+# Copyright (c) 2018-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -36,7 +36,7 @@ if [ -n "$PIP_PACKAGES" ]; then
 fi
 
 if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
-  ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-17.0.2 /msan/llvm-project
+  ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-18.1.1" /msan/llvm-project
 
   cmake -G Ninja -B /msan/clang_build/ \
     -DLLVM_ENABLE_PROJECTS="clang" \
@@ -53,13 +53,14 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
   update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100
 
   cmake -G Ninja -B /msan/cxx_build/ \
-    -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \
+    -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
     -DCMAKE_BUILD_TYPE=Release \
     -DLLVM_USE_SANITIZER=MemoryWithOrigins \
     -DCMAKE_C_COMPILER=clang \
     -DCMAKE_CXX_COMPILER=clang++ \
     -DLLVM_TARGETS_TO_BUILD=Native \
     -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \
+    -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \
     -DLIBCXX_HARDENING_MODE=debug \
     -S /msan/llvm-project/runtimes