diff --git a/.cirrus.yml b/.cirrus.yml index b49c99291eead..1c25ee1668133 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -3,19 +3,48 @@ env: # Global defaults PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache - CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling process and setting this variable avoids killing the CI script itself on error - CCACHE_SIZE: "200M" + CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling processes and setting this variable avoids killing the CI script itself on error + CCACHE_MAXSIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine -cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - DANGER_RUN_CI_ON_HOST: "1" # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system - -persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV - RESTART_CI_DOCKER_BEFORE_RUN: "1" - -persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE - persistent_worker: {} # https://cirrus-ci.org/guide/persistent-workers/ +# https://cirrus-ci.org/guide/persistent-workers/ +# +# It is possible to select a specific persistent worker by label. Refer to the +# Cirrus CI docs for more details. +# +# Generally, a persistent worker must run Ubuntu 23.04+ or Debian 12+. +# Specifically, +# - apt-get is required due to PACKAGE_MANAGER_INSTALL +# - podman-docker-4.1+ is required due to the use of `podman` when +# RESTART_CI_DOCKER_BEFORE_RUN is set and 4.1+ due to the bugfix in 4.1 +# (https://github.com/bitcoin/bitcoin/pull/21652#issuecomment-1657098200) +# - The ./ci/ depedencies (with cirrus-cli) should be installed: +# +# ``` +# apt update && apt install screen python3 bash podman-docker curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus +# ``` +# +# - There are no strict requirements on the hardware, because having less CPUs +# runs the same CI script (maybe slower). To avoid rare and intermittent OOM +# due to short memory usage spikes, it is recommended to add (and persist) +# swap: +# +# ``` +# fallocate -l 16G /swapfile_ci && chmod 600 /swapfile_ci && mkswap /swapfile_ci && swapon /swapfile_ci && ( echo '/swapfile_ci none swap sw 0 0' | tee -a /etc/fstab ) +# ``` +# +# - To register the persistent worker, open a `screen` session and run: +# +# ``` +# RESTART_CI_DOCKER_BEFORE_RUN=1 screen cirrus worker run --labels type=todo_fill_in_type --token todo_fill_in_token +# ``` +# +# The following specific types should exist, with the following requirements: +# - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory. +# - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory. +# - noble: For a machine running the Linux kernel shipped with exaclty Ubuntu Noble 24.04. The machine is recommended to have 4 CPUs and 16 GB of memory. +# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory. base_template: &BASE_TEMPLATE merge_base_script: @@ -28,26 +57,11 @@ base_template: &BASE_TEMPLATE main_template: &MAIN_TEMPLATE timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out - ccache_cache: - folder: "/tmp/ccache_dir" ci_script: - ./ci/test_run_all.sh -container_depends_template: &CONTAINER_DEPENDS_TEMPLATE - << : *BASE_TEMPLATE - container: - # https://cirrus-ci.org/faq/#are-there-any-limits - # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel - cpu: 2 - greedy: true - memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers - dockerfile: ci/test_imagefile # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment - depends_built_cache: - folder: "depends/built" - fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends) - global_task_template: &GLOBAL_TASK_TEMPLATE - << : *CONTAINER_DEPENDS_TEMPLATE + << : *BASE_TEMPLATE << : *MAIN_TEMPLATE compute_credits_template: &CREDITS_TEMPLATE @@ -56,7 +70,7 @@ compute_credits_template: &CREDITS_TEMPLATE use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'navcoin/navcoin' && $CIRRUS_PR != "" task: - name: 'lint [bookworm]' + name: 'lint' << : *BASE_TEMPLATE container: image: debian:bookworm @@ -64,269 +78,135 @@ task: memory: 1G # For faster CI feedback, immediately schedule the linters << : *CREDITS_TEMPLATE + test_runner_cache: + folder: "/lint_test_runner" + fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:test/lint/test_runner) python_cache: - folder: "/tmp/python" + folder: "/python_build" fingerprint_script: cat .python-version /etc/os-release unshallow_script: - git fetch --unshallow --no-tags lint_script: - ./ci/lint_run_all.sh - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: 'tidy [lunar]' + name: 'tidy' << : *GLOBAL_TASK_TEMPLATE - container: - cpu: 2 - memory: 5G - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:23.04 - FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh" - # For faster CI feedback, immediately schedule the linters - << : *CREDITS_TEMPLATE + persistent_worker: + labels: + type: medium env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh" task: - name: "Win64 native [vs2022]" - windows_container: - cpu: 8 - memory: 16G - image: cirrusci/windowsservercore:visualstudio2022 - timeout_in: 120m - env: - PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%' - PYTHONUTF8: 1 - CI_VCPKG_TAG: '2023.01.09' - VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' - VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' - CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' - WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat' - x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' - IgnoreWarnIntDirInTempDetected: 'true' - merge_script: - - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; } - vcpkg_tools_cache: - folder: '%VCPKG_DOWNLOADS%\tools' - reupload_on_changes: false - fingerprint_script: - - echo %CI_VCPKG_TAG% - - msbuild -version - vcpkg_binary_cache: - folder: '%VCPKG_DEFAULT_BINARY_CACHE%' - reupload_on_changes: true - fingerprint_script: - - echo %CI_VCPKG_TAG% - - type build_msvc\vcpkg.json - - msbuild -version - populate_script: - - mkdir %VCPKG_DEFAULT_BINARY_CACHE% - ccache_cache: - folder: '%CCACHE_DIR%' - install_tools_script: - - choco install --yes --no-progress ccache --version=4.7.4 - - choco install --yes --no-progress python3 --version=3.9.6 - - pip install zmq - - ccache --version - - python -VV - install_vcpkg_script: - - cd .. - - git clone --quiet https://github.com/microsoft/vcpkg.git - - cd vcpkg - - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% - - .\bootstrap-vcpkg -disableMetrics - - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake - - .\vcpkg integrate install - - .\vcpkg version - build_script: - - '%x64_NATIVE_TOOLS%' - - cd %CIRRUS_WORKING_DIR% - - ccache --zero-stats --max-size=%CCACHE_SIZE% - - python build_msvc\msvc-autogen.py - - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL%;UseMultiToolTask=true;Configuration=Release -verbosity:minimal -noLogo - - ccache --show-stats - check_script: - - build_msvc\x64\Release\test_navcoin.exe -l test_suite - - build_msvc\x64\Release\bench_bench_navcoin.exe --sanity-check - - python test\util\test_runner.py - - python test\util\rpcauth-test.py - functional_tests_script: - # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". - # See: https://learn.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance - - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - - netsh int ipv6 set dynamicport tcp start=1025 num=64511 - # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash - -task: - name: 'ARM [unit tests, no functional tests] [bookworm]' + name: 'ARM, unit tests, no functional tests' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: debian:bookworm - FILE_ENV: "./ci/test/00_setup_env_arm.sh" + persistent_worker: + labels: + type: arm64 # Use arm64 worker to sidestep qemu and avoid a slow CI: https://github.com/bitcoin/bitcoin/pull/28087#issuecomment-1649399453 env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: - name: 'Win64 [unit tests, no boost::process, no functional tests] [jammy]' + name: 'Win64, unit tests, no functional tests' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:jammy - FILE_ENV: "./ci/test/00_setup_env_win64.sh" - << : *CREDITS_TEMPLATE + persistent_worker: + labels: + type: small env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_win64.sh" task: - name: '32-bit + dash [CentOS 8]' + name: '32-bit CentOS, dash' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream8 - FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" - # For faster CI feedback, immediately schedule one task that runs all tests - << : *CREDITS_TEMPLATE + persistent_worker: + labels: + type: small env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - PACKAGE_MANAGER_INSTALL: "yum install -y" + FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" -task: - name: '[previous releases, dev package and depends packages, DEBUG] [focal]' - previous_releases_cache: - folder: "releases" - << : *GLOBAL_TASK_TEMPLATE - << : *PERSISTENT_WORKER_TEMPLATE - env: - << : *PERSISTENT_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native.sh" +# task: +# name: 'previous releases, depends DEBUG' +# << : *GLOBAL_TASK_TEMPLATE +# persistent_worker: +# labels: +# type: small +# env: +# FILE_ENV: "./ci/test/00_setup_env_native_previous_releases.sh" task: - name: '[TSan, depends] [lunar]' + name: 'TSan, depends' << : *GLOBAL_TASK_TEMPLATE - container: - cpu: 6 # Increase CPU and Memory to avoid timeout - memory: 24G - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:23.04 - FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" + persistent_worker: + labels: + type: medium env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: - name: '[MSan, depends] [lunar]' + name: 'MSan, depends' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:lunar - FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" + persistent_worker: + labels: + type: medium + timeout_in: 300m # Use longer timeout for the *rare* case where a full build (llvm + msan + depends + ...) needs to be done. env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan + FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: - name: '[ASan + LSan + UBSan + integer, no depends, USDT] [lunar]' + name: 'ASan + LSan + UBSan + integer, no depends, USDT' + enable_bpfcc_script: + # In the image build step, no external environment variables are available, + # so any settings will need to be written to the settings env file: + - sed -i "s|\${CIRRUS_CI}|true|g" ./ci/test/00_setup_env_native_asan.sh << : *GLOBAL_TASK_TEMPLATE - # We can't use a 'container' for the USDT interface tests as the CirrusCI - # containers don't have privileges to hook into bitcoind. CirrusCI uses - # Google Compute Engine instances: https://cirrus-ci.org/guide/custom-vms/ - # Images can be found here: https://cloud.google.com/compute/docs/images/os-details - compute_engine_instance: - image_project: ubuntu-os-cloud - image: family/ubuntu-2304-amd64 # https://cirrus-ci.org/guide/custom-vms/#custom-compute-engine-vms - cpu: 4 - disk: 100 - memory: 12G + persistent_worker: + labels: + type: noble # Must use this specific worker (needed for USDT functional tests) env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - HOME: /root/ # Only needed for compute_engine_instance FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" - MAKEJOBS: "-j4" # Avoid excessive memory use task: - name: '[fuzzer,address,undefined,integer, no depends] [lunar]' + name: 'fuzzer,address,undefined,integer, no depends' << : *GLOBAL_TASK_TEMPLATE - container: - cpu: 4 # Increase CPU and memory to avoid timeout - memory: 16G - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:lunar - FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" + persistent_worker: + labels: + type: medium env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: '[multiprocess, i686, DEBUG] [focal]' + name: 'multiprocess, i686, DEBUG' << : *GLOBAL_TASK_TEMPLATE - container: - cpu: 4 - memory: 16G # The default memory is sometimes just a bit too small, so double everything - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:focal - FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" + persistent_worker: + labels: + type: medium env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" task: - name: '[no wallet, libbitcoinkernel] [focal]' + name: 'no wallet, libblsct' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:focal - FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" - << : *CREDITS_TEMPLATE - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - -task: - name: 'macOS 10.15 [no tests] [focal]' - << : *CONTAINER_DEPENDS_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:focal - FILE_ENV: "./ci/test/00_setup_env_mac.sh" - << : *CREDITS_TEMPLATE - macos_sdk_cache: - folder: "depends/SDKs/$MACOS_SDK" - fingerprint_key: "$MACOS_SDK" - << : *MAIN_TEMPLATE + persistent_worker: + labels: + type: small env: - MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers" - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_libblsct_only.sh" task: - name: 'macOS 13 native arm64 [sqlite only] [no depends]' - macos_instance: - # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: ghcr.io/cirruslabs/macos-ventura-xcode:14.1 # https://cirrus-ci.org/guide/macOS - << : *BASE_TEMPLATE - check_clang_script: - - clang --version - brew_install_script: - - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt - << : *MAIN_TEMPLATE + name: 'no wallet, libnavcoinkernel' + << : *GLOBAL_TASK_TEMPLATE + persistent_worker: + labels: + type: small env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - CI_USE_APT_INSTALL: "no" - PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do - FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh" + FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libnavcoinkernel.sh" task: - name: 'ARM64 Android APK [jammy]' - << : *CONTAINER_DEPENDS_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:jammy - FILE_ENV: "./ci/test/00_setup_env_android.sh" - << : *CREDITS_TEMPLATE - android_sdk_cache: - folder: "depends/SDKs/android" - fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.2.8568313" - depends_sources_cache: - folder: "depends/sources" - fingerprint_script: git rev-parse HEAD:depends/packages - << : *MAIN_TEMPLATE + name: 'macOS-cross, no tests' + << : *GLOBAL_TASK_TEMPLATE + persistent_worker: + labels: + type: small env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_mac_cross.sh" diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.yml b/.github/ISSUE_TEMPLATE/good_first_issue.yml index c40dad9687a05..133937c011af6 100644 --- a/.github/ISSUE_TEMPLATE/good_first_issue.yml +++ b/.github/ISSUE_TEMPLATE/good_first_issue.yml @@ -29,8 +29,10 @@ body: attributes: label: Useful Skills description: For example, “`std::thread`”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”. - validations: - required: false + value: | + * Compiling Bitcoin Core from source + * Running the C++ unit tests and the Python functional tests + * ... - type: textarea attributes: label: Guidance for new contributors diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000..33783a0c6fd72 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,236 @@ +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +name: CI +on: + # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request. + pull_request: + # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push. + push: + branches: + - '**' + tags-ignore: + - '**' + +concurrency: + group: ${{ github.event_name != 'pull_request' && github.run_id || github.ref }} + cancel-in-progress: true + +env: + DANGER_RUN_CI_ON_HOST: 1 + CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error + MAKEJOBS: '-j10' + +jobs: + test-each-commit: + name: 'test each commit' + runs-on: ubuntu-22.04 + if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 + timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. Assuming a worst case time of 1 hour per commit, this leads to a --max-count=6 below. + env: + MAX_COUNT: 6 + steps: + - name: Determine fetch depth + run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV" + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: ${{ env.FETCH_DEPTH }} + - name: Determine commit range + run: | + # Checkout HEAD~ and find the test base commit + # Checkout HEAD~ because it would be wasteful to rerun tests on the PR + # head commit that are already run by other jobs. + git checkout HEAD~ + # Figure out test base commit by listing ancestors of HEAD, excluding + # ancestors of the most recent merge commit, limiting the list to the + # newest MAX_COUNT ancestors, ordering it from oldest to newest, and + # taking the first one. + # + # If the branch contains up to MAX_COUNT ancestor commits after the + # most recent merge commit, all of those commits will be tested. If it + # contains more, only the most recent MAX_COUNT commits will be + # tested. + # + # In the command below, the ^@ suffix is used to refer to all parents + # of the merge commit as described in: + # https://git-scm.com/docs/git-rev-parse#_other_rev_parent_shorthand_notations + # and the ^ prefix is used to exclude these parents and all their + # ancestors from the rev-list output as described in: + # https://git-scm.com/docs/git-rev-list + echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD ^$(git rev-list -n1 --merges HEAD)^@ | head -1)" >> "$GITHUB_ENV" + - run: | + sudo apt-get update + sudo apt-get install clang-15 ccache build-essential libtool autotools-dev automake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev -y + - name: Compile and run tests + run: | + # Run tests on commits after the last merge commit and before the PR head commit + # Use clang++, because it is a bit faster and uses less memory than g++ + git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && ./autogen.sh && CC=clang-15 CXX=clang++-15 ./configure && make clean && make -j $(nproc) check && ./test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }} + + macos-native-x86_64: + name: 'macOS 13 native, x86_64, no depends, sqlite only' + # Use latest image, but hardcode version to avoid silent upgrades (and breaks). + # See: https://github.com/actions/runner-images#available-images. + runs-on: macos-13 + + # No need to run on the read-only mirror, unless it is a PR. + if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' + + timeout-minutes: 120 + + env: + FILE_ENV: './ci/test/00_setup_env_mac_native.sh' + BASE_ROOT_DIR: ${{ github.workspace }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Clang version + run: | + sudo xcode-select --switch /Applications/Xcode_15.0.app + clang --version + + - name: Install Homebrew packages + env: + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 + run: brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq + + - name: Set Ccache directory + run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" + + - name: Restore Ccache cache + id: ccache-cache + uses: actions/cache/restore@v3 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ github.job }}-ccache-${{ github.run_id }} + restore-keys: ${{ github.job }}-ccache- + + - name: CI script + run: ./ci/test_run_all.sh + + - name: Save Ccache cache + uses: actions/cache/save@v3 + if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true' + with: + path: ${{ env.CCACHE_DIR }} + # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache + key: ${{ github.job }}-ccache-${{ github.run_id }} + + win64-native: + name: 'Win64 native, VS 2022' + # Use latest image, but hardcode version to avoid silent upgrades (and breaks). + # See: https://github.com/actions/runner-images#available-images. + runs-on: windows-2022 + + # No need to run on the read-only mirror, unless it is a PR. + if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' + + env: + CCACHE_MAXSIZE: '200M' + CI_CCACHE_VERSION: '4.7.5' + PYTHONUTF8: 1 + TEST_RUNNER_TIMEOUT_FACTOR: 40 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure Developer Command Prompt for Microsoft Visual C++ + # Using microsoft/setup-msbuild is not enough. + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Check MSBuild + run: | + msbuild -version | Out-File -FilePath "$env:GITHUB_WORKSPACE\msbuild_version" + Get-Content -Path "$env:GITHUB_WORKSPACE\msbuild_version" + $env:VCToolsVersion | Out-File -FilePath "$env:GITHUB_WORKSPACE\toolset_version" + Get-Content -Path "$env:GITHUB_WORKSPACE\toolset_version" + + - name: Ccache installation cache + id: ccache-installation-cache + uses: actions/cache@v3 + with: + path: | + C:\ProgramData\chocolatey\lib\ccache + C:\ProgramData\chocolatey\bin\ccache.exe + C:\ccache\cl.exe + key: ${{ github.job }}-ccache-installation-${{ env.CI_CCACHE_VERSION }} + + - name: Install Ccache + if: steps.ccache-installation-cache.outputs.cache-hit != 'true' + run: | + choco install --yes --no-progress ccache --version=$env:CI_CCACHE_VERSION + New-Item -ItemType Directory -Path "C:\ccache" + Copy-Item -Path "$env:ChocolateyInstall\lib\ccache\tools\ccache-$env:CI_CCACHE_VERSION-windows-x86_64\ccache.exe" -Destination "C:\ccache\cl.exe" + + - name: Restore Ccache cache + id: ccache-cache + uses: actions/cache/restore@v3 + with: + path: ~/AppData/Local/ccache + key: ${{ github.job }}-ccache-${{ github.run_id }} + restore-keys: ${{ github.job }}-ccache- + + - name: Using vcpkg with MSBuild + run: | + Set-Location "$env:VCPKG_INSTALLATION_ROOT" + Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" + Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_PLATFORM_TOOLSET_VERSION $env:VCToolsVersion)" + .\vcpkg.exe --vcpkg-root "$env:VCPKG_INSTALLATION_ROOT" integrate install + git rev-parse HEAD | Out-File -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit" + Get-Content -Path "$env:GITHUB_WORKSPACE\vcpkg_commit" + + - name: vcpkg tools cache + uses: actions/cache@v3 + with: + path: C:/vcpkg/downloads/tools + key: ${{ github.job }}-vcpkg-tools + + - name: vcpkg binary cache + uses: actions/cache@v3 + with: + path: ~/AppData/Local/vcpkg/archives + key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('vcpkg_commit', 'msbuild_version', 'toolset_version', 'build_msvc/vcpkg.json') }} + + - name: Generate project files + run: py -3 build_msvc\msvc-autogen.py + + - name: Build + shell: cmd + run: | + ccache --zero-stats + msbuild build_msvc\bitcoin.sln -property:CLToolPath=C:\ccache;CLToolExe=cl.exe;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + + - name: Ccache stats + run: ccache --show-stats + + - name: Save Ccache cache + uses: actions/cache/save@v3 + if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true' + with: + path: ~/AppData/Local/ccache + # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache + key: ${{ github.job }}-ccache-${{ github.run_id }} + + - name: Run unit tests + run: src\test_navcoin.exe -l test_suite + + - name: Run benchmarks + run: src\bench_navcoin.exe -sanity-check + + - name: Run util tests + run: py -3 test\util\test_runner.py + + - name: Run rpcauth test + run: py -3 test\util\rpcauth-test.py + + - name: Run functional tests + env: + TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }} + run: py -3 test\functional\test_runner.py --jobs $env:NUMBER_OF_PROCESSORS --ci --quiet --tmpdirprefix=$env:RUNNER_TEMP --combinedlogslen=99999999 --timeout-factor=$env:TEST_RUNNER_TIMEOUT_FACTOR $env:TEST_RUNNER_EXTRA diff --git a/.gitignore b/.gitignore index 8461e24d554cd..334d50fd90efb 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ src/navcoin-chainstate src/navcoin-wallet src/test/fuzz/fuzz src/test/test_navcoin -src/qt/test/test_navcoin-qt # autoreconf Makefile.in @@ -41,20 +40,6 @@ src/config/bitcoin-config.h src/config/bitcoin-config.h.in src/config/stamp-h1 src/obj -share/setup.nsi -share/qt/Info.plist - -src/qt/*.moc -src/qt/moc_*.cpp -src/qt/forms/ui_*.h - -src/qt/test/moc*.cpp - -src/qt/bitcoin-qt.config -src/qt/bitcoin-qt.creator -src/qt/bitcoin-qt.creator.user -src/qt/bitcoin-qt.files -src/qt/bitcoin-qt.includes .deps .dirstamp @@ -71,10 +56,11 @@ src/qt/bitcoin-qt.includes *.pb.cc *.pb.h *.dat +*.tmp *.log *.trs -*.dmg +*.zip *.json.h *.raw.h @@ -92,15 +78,15 @@ src/qt/bitcoin-qt.includes *.qm Makefile !depends/Makefile -src/qt/navcoin-qt -Navcoin-Qt.app # Qt Creator Makefile.am.user # Unit-tests Makefile.test -navcoin-qt_test + +# Visual Studio +*.vcxproj.user # Resources cpp qrc_*.cpp @@ -135,7 +121,7 @@ test/.mypy_cache/ /doc/doxygen/ -libbitcoinconsensus.pc +libnavcoinconsensus.pc contrib/devtools/split-debug.sh # Output from running db4 installation diff --git a/.python-version b/.python-version index eee6392d5c718..43077b246094f 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.8.16 +3.9.18 diff --git a/.style.yapf b/.style.yapf index 69d8c6aee417d..350ac638554e5 100644 --- a/.style.yapf +++ b/.style.yapf @@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True i18n_comment= # The i18n function call names. The presence of this function stops -# reformattting on that line, because the string it has cannot be moved +# reformatting on that line, because the string it has cannot be moved # away from the i18n comment. i18n_function_call= diff --git a/.tx/config b/.tx/config index ab1fb933d833e..e75ce0afcb7cf 100644 --- a/.tx/config +++ b/.tx/config @@ -1,7 +1,7 @@ [main] host = https://www.transifex.com -[o:bitcoin:p:bitcoin:r:qt-translation-025x] +[o:bitcoin:p:bitcoin:r:qt-translation-026x] file_filter = src/qt/locale/bitcoin_.xlf source_file = src/qt/locale/bitcoin_en.xlf source_lang = en diff --git a/COPYING b/COPYING index 2f7add71aca6a..e6d6e9fe57070 100644 --- a/COPYING +++ b/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2023 The Bitcoin Core developers -Copyright (c) 2009-2023 Bitcoin Developers +Copyright (c) 2009-2024 The Bitcoin Core developers +Copyright (c) 2009-2024 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile.am b/Makefile.am index 6372e8731209d..b9d85f41e4f8f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,23 +14,18 @@ endif .PHONY: deploy FORCE .INTERMEDIATE: $(COVERAGE_INFO) -export PYTHONPATH - if BUILD_BITCOIN_LIBS pkgconfigdir = $(libdir)/pkgconfig -pkgconfig_DATA = libbitcoinconsensus.pc +pkgconfig_DATA = libnavcoinconsensus.pc endif -BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT) -BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) -BITCOIN_TEST_BIN=$(top_builddir)/src/test/$(BITCOIN_TEST_NAME)$(EXEEXT) -BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT) -BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT) -BITCOIN_UTIL_BIN=$(top_builddir)/src/$(BITCOIN_UTIL_NAME)$(EXEEXT) -BITCOIN_WALLET_BIN=$(top_builddir)/src/$(BITCOIN_WALLET_TOOL_NAME)$(EXEEXT) +BITCOIND_BIN=$(top_builddir)/src/$(NAVCOIN_DAEMON_NAME)$(EXEEXT) +BITCOIN_TEST_BIN=$(top_builddir)/src/test/$(NAVCOIN_TEST_NAME)$(EXEEXT) +BITCOIN_CLI_BIN=$(top_builddir)/src/$(NAVCOIN_CLI_NAME)$(EXEEXT) +NAVCOIN_TX_BIN=$(top_builddir)/src/$(NAVCOIN_TX_NAME)$(EXEEXT) +BITCOIN_UTIL_BIN=$(top_builddir)/src/$(NAVCOIN_UTIL_NAME)$(EXEEXT) +BITCOIN_WALLET_BIN=$(top_builddir)/src/$(NAVCOIN_WALLET_TOOL_NAME)$(EXEEXT) BITCOIN_NODE_BIN=$(top_builddir)/src/$(BITCOIN_MP_NODE_NAME)$(EXEEXT) -BITCOIN_GUI_BIN=$(top_builddir)/src/$(BITCOIN_MP_GUI_NAME)$(EXEEXT) -BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win64-setup$(EXEEXT) empty := space := $(empty) $(empty) @@ -66,7 +61,7 @@ $(BITCOIND_BIN): FORCE $(BITCOIN_CLI_BIN): FORCE $(MAKE) -C src $(@F) -$(BITCOIN_TX_BIN): FORCE +$(NAVCOIN_TX_BIN): FORCE $(MAKE) -C src $(@F) $(BITCOIN_UTIL_BIN): FORCE @@ -99,7 +94,7 @@ $(COV_TOOL_WRAPPER): @chmod +x $(COV_TOOL_WRAPPER) baseline.info: $(COV_TOOL_WRAPPER) - $(LCOV) -c -i -d $(abs_builddir)/src -o $@ + $(LCOV) $(LCOV_OPTS) -c -i -d $(abs_builddir)/src -o $@ baseline_filtered.info: baseline.info $(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@ @@ -133,13 +128,13 @@ functional_test_filtered.info: functional_test.info $(LCOV) -a $@ $(LCOV_OPTS) -o $@ fuzz_coverage.info: fuzz_filtered.info - $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a fuzz_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt + $(LCOV) $(LCOV_OPTS) -a baseline_filtered.info -a fuzz_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt test_bitcoin_coverage.info: baseline_filtered.info test_bitcoin_filtered.info - $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -o $@ + $(LCOV) $(LCOV_OPTS) -a baseline_filtered.info -a test_bitcoin_filtered.info -o $@ total_coverage.info: test_bitcoin_filtered.info functional_test_filtered.info - $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt + $(LCOV) $(LCOV_OPTS) -a baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt fuzz.coverage/.dirstamp: fuzz_coverage.info $(GENHTML) -s $(LCOV_OPTS) $< -o $(@D) @@ -222,10 +217,12 @@ EXTRA_DIST += \ test/util/data/txcreatesignv1.hex \ test/util/data/txcreatesignv1.json \ test/util/data/txcreatesignv2.hex \ + test/util/data/txreplace1.hex \ + test/util/data/txreplacenoinputs.hex \ + test/util/data/txreplaceomittedn.hex \ + test/util/data/txreplacesingleinput.hex \ test/util/rpcauth-test.py -CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) - DISTCHECK_CONFIGURE_FLAGS = --enable-man doc/doxygen/.stamp: doc/Doxyfile FORCE diff --git a/autogen.sh b/autogen.sh index de16260b56327..69c892ffa0e72 100755 --- a/autogen.sh +++ b/autogen.sh @@ -14,3 +14,16 @@ fi command -v autoreconf >/dev/null || \ (echo "configuration failed, please install autoconf first" && exit 1) autoreconf --install --force --warnings=all + +if expr "'$(build-aux/config.guess --timestamp)" \< "'$(depends/config.guess --timestamp)" > /dev/null; then + chmod ug+w build-aux/config.guess + chmod ug+w src/secp256k1/build-aux/config.guess + cp depends/config.guess build-aux + cp depends/config.guess src/secp256k1/build-aux +fi +if expr "'$(build-aux/config.sub --timestamp)" \< "'$(depends/config.sub --timestamp)" > /dev/null; then + chmod ug+w build-aux/config.sub + chmod ug+w src/secp256k1/build-aux/config.sub + cp depends/config.sub build-aux + cp depends/config.sub src/secp256k1/build-aux +fi diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 index 51a35054d08ce..8a2df5627ff3a 100644 --- a/build-aux/m4/ax_cxx_compile_stdcxx.m4 +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -983,7 +983,7 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_20], [[ #error "This is not a C++ compiler" -#elif __cplusplus < 202002L +#elif __cplusplus < 201709L // Temporary patch on top of upstream to allow g++-10 #error "This is not a C++20 compiler" diff --git a/build-aux/m4/bitcoin_runtime_lib.m4 b/build-aux/m4/bitcoin_runtime_lib.m4 deleted file mode 100644 index 1a6922deca485..0000000000000 --- a/build-aux/m4/bitcoin_runtime_lib.m4 +++ /dev/null @@ -1,42 +0,0 @@ -# On some platforms clang builtin implementations -# require compiler-rt as a runtime library to use. -# -# See: -# - https://bugs.llvm.org/show_bug.cgi?id=28629 - -m4_define([_CHECK_RUNTIME_testbody], [[ - bool f(long long x, long long y, long long* p) - { - return __builtin_mul_overflow(x, y, p); - } - int main() { return 0; } -]]) - -AC_DEFUN([CHECK_RUNTIME_LIB], [ - - AC_LANG_PUSH([C++]) - - AC_MSG_CHECKING([for __builtin_mul_overflow]) - AC_LINK_IFELSE( - [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])], - [ - AC_MSG_RESULT([yes]) - AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow]) - ], - [ - ax_check_save_flags="$LDFLAGS" - LDFLAGS="$LDFLAGS --rtlib=compiler-rt -lgcc_s" - AC_LINK_IFELSE( - [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])], - [ - AC_MSG_RESULT([yes, with additional linker flags]) - RUNTIME_LDFLAGS="--rtlib=compiler-rt -lgcc_s" - AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow]) - ], - [AC_MSG_RESULT([no])]) - LDFLAGS="$ax_check_save_flags" - ]) - - AC_LANG_POP - AC_SUBST([RUNTIME_LDFLAGS]) -]) diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index 602b57fe43263..aa00168fce85b 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -4,8 +4,10 @@ dnl permitted in any medium without royalty provided the copyright notice dnl and this notice are preserved. This file is offered as-is, without any dnl warranty. -# Some versions of gcc/libstdc++ require linking with -latomic if -# using the C++ atomic library. +# Clang, when building for 32-bit, +# and linking against libstdc++, requires linking with +# -latomic if using the C++ atomic library. +# Can be tested with: clang++ test.cpp -m32 # # Sourced from http://bugs.debian.org/797228 @@ -22,6 +24,8 @@ m4_define([_CHECK_ATOMIC_testbody], [[ std::atomic t{0s}; t.store(2s); + auto t1 = t.load(); + t.compare_exchange_strong(t1, 3s); std::atomic a{}; diff --git a/build-aux/m4/l_filesystem.m4 b/build-aux/m4/l_filesystem.m4 deleted file mode 100644 index ca3a0cd41c061..0000000000000 --- a/build-aux/m4/l_filesystem.m4 +++ /dev/null @@ -1,47 +0,0 @@ -dnl Copyright (c) 2022 The Bitcoin Core developers -dnl Distributed under the MIT software license, see the accompanying -dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# GCC 8.1 and earlier requires -lstdc++fs -# Clang 8.0.0 (libc++) and earlier requires -lc++fs - -m4_define([_CHECK_FILESYSTEM_testbody], [[ - #include - - namespace fs = std::filesystem; - - int main() { - (void)fs::current_path().root_name(); - return 0; - } -]]) - -AC_DEFUN([CHECK_FILESYSTEM], [ - - AC_LANG_PUSH(C++) - - AC_MSG_CHECKING([whether std::filesystem can be used without link library]) - - AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no]) - SAVED_LIBS="$LIBS" - LIBS="$SAVED_LIBS -lstdc++fs" - AC_MSG_CHECKING([whether std::filesystem needs -lstdc++fs]) - AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no]) - AC_MSG_CHECKING([whether std::filesystem needs -lc++fs]) - LIBS="$SAVED_LIBS -lc++fs" - AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_FAILURE([cannot figure out how to use std::filesystem]) - ]) - ]) - ]) - - AC_LANG_POP -]) diff --git a/build_msvc/.gitignore b/build_msvc/.gitignore index 64038e9c43e4d..133f53f044c71 100644 --- a/build_msvc/.gitignore +++ b/build_msvc/.gitignore @@ -20,13 +20,11 @@ libbitcoin_util/libbitcoin_util.vcxproj libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj libbitcoin_wallet/libbitcoin_wallet.vcxproj libbitcoin_zmq/libbitcoin_zmq.vcxproj -bench_bitcoin/bench_bitcoin.vcxproj +bench_navcoin/bench_navcoin.vcxproj libtest_util/libtest_util.vcxproj /bitcoin_config.h /common.init.vcxproj */Win32 -libbitcoin_qt/QtGeneratedFiles/* -test_bitcoin-qt/QtGeneratedFiles/* vcpkg_installed diff --git a/build_msvc/README.md b/build_msvc/README.md index d4c187a170fe1..a1f7b985174c5 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -32,7 +32,9 @@ Qt --------------------- To build Navcoin Core with the GUI, a static build of Qt is required. -1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.5.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.11.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.11/single/qt-everywhere-opensource-src-5.15.11.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. + +> 💡 **Tip:** If you use the default path with "Extract All" for the Qt source code zip file, and end up with something like `C:\dev\qt-everywhere-opensource-src-5.15.11\qt-everywhere-src-5.15.11`, you are likely to encounter a "path too long" error when building. To fix the problem move the source files to a shorter path such as the recommended `C:\dev\qt-source`. 2. Open "x64 Native Tools Command Prompt for VS 2022", and input the following commands: ```cmd diff --git a/build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj b/build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj deleted file mode 100644 index f81f98db6e6e9..0000000000000 --- a/build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj +++ /dev/null @@ -1,172 +0,0 @@ - - - - - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB} - - - Application - $(SolutionDir)$(Platform)\$(Configuration)\ - - - - $(IntDir)bench_addrman.obj - - - $(IntDir)bench_base58.obj - - - $(IntDir)bench_bech32.obj - - - $(IntDir)bench_bech32_mod.obj - - - $(IntDir)bench_bench.obj - - - $(IntDir)bench_bench_bitcoin.obj - - - $(IntDir)bench_block_assemble.obj - - - $(IntDir)bench_ccoins_caching.obj - - - $(IntDir)bench_chacha20.obj - - - $(IntDir)bench_chacha_poly_aead.obj - - - $(IntDir)bench_checkblock.obj - - - $(IntDir)bench_checkqueue.obj - - - $(IntDir)bench_crypto_hash.obj - - - $(IntDir)bench_data.obj - - - $(IntDir)bench_descriptors.obj - - - $(IntDir)bench_duplicate_inputs.obj - - - $(IntDir)bench_examples.obj - - - $(IntDir)bench_gcs_filter.obj - - - $(IntDir)bench_hashpadding.obj - - - $(IntDir)bench_load_external.obj - - - $(IntDir)bench_lockedpool.obj - - - $(IntDir)bench_logging.obj - - - $(IntDir)bench_mempool_eviction.obj - - - $(IntDir)bench_mempool_stress.obj - - - $(IntDir)bench_merkle_root.obj - - - $(IntDir)bench_nanobench.obj - - - $(IntDir)bench_peer_eviction.obj - - - $(IntDir)bench_poly1305.obj - - - $(IntDir)bench_prevector.obj - - - $(IntDir)bench_rollingbloom.obj - - - $(IntDir)bench_rpc_blockchain.obj - - - $(IntDir)bench_rpc_mempool.obj - - - $(IntDir)bench_strencodings.obj - - - $(IntDir)bench_util_time.obj - - - $(IntDir)bench_verify_script.obj - - - - - {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} - - - {7c87e378-df58-482e-aa2f-1bc129bc19ce} - - - {6190199c-6cf4-4dad-bfbd-93fa72a760c1} - - - {460fee33-1fe1-483f-b3bf-931ff8e969a5} - - - {b53a5535-ee9d-4c6f-9a26-f79ee3bc3754} - - - {93b86837-b543-48a5-a89b-7c87abb77df2} - - - {792d487f-f14c-49fc-a9de-3fc150f31c3f} - - - {5724ba7d-a09a-4ba8-800b-c4c1561b3d69} - - - {bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6} - - - {18430fef-6b61-4c53-b396-718e02850f1b} - - - {1e065f03-3566-47d0-8fa9-daa72b084e7d} - - - {1DBB979A-C212-45CD-9563-446A96F87F72} - - - {1DBB979A-C212-45CD-9563-446A96F87F71} - - - - - There was an error executing the raw bench header generation task. - - - - - - - - - - - diff --git a/build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj.in b/build_msvc/bench_navcoin/bench_navcoin.vcxproj.in similarity index 88% rename from build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj.in rename to build_msvc/bench_navcoin/bench_navcoin.vcxproj.in index 556e56936d61b..a7343941ec59b 100644 --- a/build_msvc/bench_bench_navcoin/bench_bench_navcoin.vcxproj.in +++ b/build_msvc/bench_navcoin/bench_navcoin.vcxproj.in @@ -12,7 +12,7 @@ @SOURCE_FILES@ - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} @@ -45,12 +45,6 @@ {1e065f03-3566-47d0-8fa9-daa72b084e7d} - - {1DBB979A-C212-45CD-9563-446A96F87F72} - - - {1DBB979A-C212-45CD-9563-446A96F87F71} - diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index 29a3b540c0c6e..4a6c994daf4b5 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -2,7 +2,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.4.33213.308 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoinconsensus", "libbitcoinconsensus\libbitcoinconsensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_consensus", "libbitcoin_consensus\libbitcoin_consensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "navcoind", "navcoind\navcoind.vcxproj", "{D4513DDF-6013-44DC-ADCC-12EAF6D1F038}" EndProject @@ -26,7 +26,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_cli", "libbitcoi EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "navcoin-cli", "navcoin-cli\navcoin-cli.vcxproj", "{0B2D7431-F876-4A58-87BF-F748338CD3BF}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bench_bench_navcoin", "bench_bench_navcoin\bench_bench_navcoin.vcxproj", "{1125654E-E1B2-4431-8B5C-62EA9A2FEECB}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bench_navcoin", "bench_navcoin\bench_navcoin.vcxproj", "{1125654E-E1B2-4431-8B5C-62EA9A2FEECB}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "navcoin-tx", "navcoin-tx\navcoin-tx.vcxproj", "{D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}" EndProject @@ -46,7 +46,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libminisketch", "libminiske EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mcl", "..\src\bls\mcl\src\proj\mcl.vcxproj", "{1DBB979A-C212-45CD-9563-446A96F87F71}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bls", "..\src\bls\src\proj\bls384_256.vcxproj", "{1DBB979A-C212-45CD-9563-446A96F87F72}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bls384_256", "..\src\bls\src\proj\bls384_256.vcxproj", "{1DBB979A-C212-45CD-9563-446A96F87F72}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -128,10 +128,14 @@ Global {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x64.Build.0 = Release|x64 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x86.ActiveCfg = Release|x64 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x86.Build.0 = Release|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.ActiveCfg = Debug|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.Build.0 = Debug|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.ActiveCfg = Release|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.Build.0 = Release|x64 + {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.ActiveCfg = Debug|x64 + {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.Build.0 = Debug|x64 + {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x86.ActiveCfg = Debug|x64 + {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x86.Build.0 = Debug|x64 + {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.ActiveCfg = Release|x64 + {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.Build.0 = Release|x64 + {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x86.ActiveCfg = Release|x64 + {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x86.Build.0 = Release|x64 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x64.ActiveCfg = Debug|x64 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x64.Build.0 = Debug|x64 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x86.ActiveCfg = Debug|x64 @@ -241,7 +245,7 @@ Global HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {58AAB032-7274-49BD-845E-5EF4DBB69B70} SolutionGuid = {C82D227F-890F-4566-A1D9-4DD7921AD106} - SolutionGuid = {58AAB032-7274-49BD-845E-5EF4DBB69B70} EndGlobalSection EndGlobal diff --git a/build_msvc/bitcoin_config.h.in b/build_msvc/bitcoin_config.h.in index 02d8fc41c2a6a..40b7a408908ab 100644 --- a/build_msvc/bitcoin_config.h.in +++ b/build_msvc/bitcoin_config.h.in @@ -38,15 +38,9 @@ /* Define to 1 to enable SQLite wallet */ #define USE_SQLITE 1 -/* Define to 1 to enable ZMQ functions */ +/* Define this symbol to enable ZMQ functions */ #define ENABLE_ZMQ 1 -/* define if external signer support is enabled (requires Boost::Process) */ -#define ENABLE_EXTERNAL_SIGNER /**/ - -/* Define this symbol if the consensus lib has been built */ -#define HAVE_CONSENSUS_LIB 1 - /* Define to 1 if you have the declaration of `be16toh', and to 0 if you don't. */ #define HAVE_DECL_BE16TOH 0 @@ -119,16 +113,16 @@ #define HAVE_DLLEXPORT_ATTRIBUTE 1 /* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "https://github.com/bitcoin/bitcoin/issues" +#define PACKAGE_BUGREPORT "https://github.com/navcoin/navcoin/issues" /* Define to the full name of this package. */ -#define PACKAGE_NAME "Bitcoin Core" +#define PACKAGE_NAME "Navcoin Core" /* Define to the full name and version of this package. */ #define PACKAGE_STRING $ /* Define to the home page for this package. */ -#define PACKAGE_URL "https://bitcoincore.org/" +#define PACKAGE_URL "https://navcoin.org/" /* Define to the version of this package. */ #define PACKAGE_VERSION $ diff --git a/build_msvc/common.init.vcxproj.in b/build_msvc/common.init.vcxproj.in index 7c9e0d89c0e55..b217762294316 100644 --- a/build_msvc/common.init.vcxproj.in +++ b/build_msvc/common.init.vcxproj.in @@ -58,7 +58,7 @@ - Disabled + MaxSpeed false true true @@ -91,7 +91,7 @@ /utf-8 /Zc:__cplusplus /std:c++20 %(AdditionalOptions) 4018;4244;4267;4715;4805 true - _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions) + _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;SECP256K1_STATIC;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions) ..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;..\..\src\bls\include;..\..\src\bls\mcl\include;..\..\src\bls\src;..\..\src\bls\mcl\src;%(AdditionalIncludeDirectories) @@ -99,9 +99,6 @@ Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) true - - /ignore:4221 - diff --git a/build_msvc/common.props b/build_msvc/common.props index e444c7efd3758..34ad7c420d948 100644 --- a/build_msvc/common.props +++ b/build_msvc/common.props @@ -4,7 +4,7 @@ - $(SolutionDir)\..\src\bls\mcl\../cybozulib/include;$(SolutionDir)\..\src\bls\mcl\../cybozulib_ext/include;$(SolutionDir)\..\src\bls\mcl\include;$(SolutionDir)\..\src\bls\mcl\../xbyak;$(SolutionDir)..\src\bls\include;$(SolutionDir)..\src;$(SolutionDir)..\src\univalue\include;$(SolutionDir)..\src\leveldb\include;$(SolutionDir)..\src\minisketch\include;$(SolutionDir)..\src\secp256k1\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\..\src\bls\cybozulib\include;$(SolutionDir)\..\src\bls\cybozulib_ext\include;$(SolutionDir)\..\src\bls\mcl\include;$(SolutionDir)\..\src\bls\xbyak;$(SolutionDir)..\src\bls\include;$(SolutionDir)..\src;$(SolutionDir)..\src\univalue\include;$(SolutionDir)..\src\leveldb\include;$(SolutionDir)..\src\minisketch\include;$(SolutionDir)..\src\secp256k1\include;%(AdditionalIncludeDirectories) @@ -13,10 +13,10 @@ MultiThreaded - _MBCS;%(PreprocessorDefinitions);NOMINMAX;BLS_ETH;MCL_USE_VINT;MCL_VINT_FIXED_BUFFER;MCL_DONT_USE_OPENSSL;MCL_DONT_USE_XBYAK;MCLBN_DONT_EXPORT;MT;MCL_SIZEOF_UNIT=4;MCL_MAX_BIT_SIZE=384;CYBOZU_MINIMUM_EXCEPTION;%(PreprocessorDefinitions) + _MBCS;NOMINMAX;BLS_ETH;MCL_USE_VINT;MCL_VINT_FIXED_BUFFER;MCL_DONT_USE_OPENSSL;MCL_DONT_USE_XBYAK;MCLBN_DONT_EXPORT;MT;MCL_SIZEOF_UNIT=4;MCL_MAX_BIT_SIZE=384;CYBOZU_MINIMUM_EXCEPTION;%(PreprocessorDefinitions) - $(SolutionDir)../cybozulib_ext/lib;$(SolutionDir)lib;%(AdditionalLibraryDirectories) + $(SolutionDir)..\cybozulib_ext\lib;$(SolutionDir)lib;%(AdditionalLibraryDirectories) diff --git a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in index 482e4333f7967..f0ded37f2d1b3 100644 --- a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in +++ b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in @@ -9,7 +9,13 @@ -@SOURCE_FILES@ + @SOURCE_FILES@ + + {1DBB979A-C212-45CD-9563-446A96F87F72} + + + {1DBB979A-C212-45CD-9563-446A96F87F71} + diff --git a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj similarity index 77% rename from build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj rename to build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj index 19c63fe38da08..24b97ca2b6cc9 100644 --- a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj +++ b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj @@ -8,29 +8,14 @@ StaticLibrary - - {1DBB979A-C212-45CD-9563-446A96F87F72} - - - {1DBB979A-C212-45CD-9563-446A96F87F71} - - - - - - - - - - - + @@ -41,6 +26,12 @@ + + {1DBB979A-C212-45CD-9563-446A96F87F72} + + + {1DBB979A-C212-45CD-9563-446A96F87F71} + diff --git a/build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in b/build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in index a9cd7c780c63f..c0d429e70f19a 100644 --- a/build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in +++ b/build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in @@ -23,4 +23,4 @@ - \ No newline at end of file + diff --git a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in index 613d5c7199159..5b730596af3e3 100644 --- a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in +++ b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in @@ -12,6 +12,12 @@ @SOURCE_FILES@ + + {1DBB979A-C212-45CD-9563-446A96F87F72} + + + {1DBB979A-C212-45CD-9563-446A96F87F71} + diff --git a/build_msvc/libleveldb/libleveldb.vcxproj b/build_msvc/libleveldb/libleveldb.vcxproj index 2914eb2cfbb3e..eacfbb2641977 100644 --- a/build_msvc/libleveldb/libleveldb.vcxproj +++ b/build_msvc/libleveldb/libleveldb.vcxproj @@ -51,7 +51,7 @@ HAVE_CRC32C=0;HAVE_SNAPPY=0;LEVELDB_IS_BIG_ENDIAN=0;_UNICODE;UNICODE;_CRT_NONSTDC_NO_DEPRECATE;LEVELDB_PLATFORM_WINDOWS;%(PreprocessorDefinitions) - 4244;4267 + 4244;4267;4722 ..\..\src\leveldb;..\..\src\leveldb\include;%(AdditionalIncludeDirectories) diff --git a/build_msvc/libminisketch/libminisketch.vcxproj b/build_msvc/libminisketch/libminisketch.vcxproj index b34593fe5c717..60e57caa579e3 100644 --- a/build_msvc/libminisketch/libminisketch.vcxproj +++ b/build_msvc/libminisketch/libminisketch.vcxproj @@ -28,7 +28,7 @@ - 4060;4065;4146;4244;4267;4554 + 4060;4065;4146;4244;4267 HAVE_CLMUL;DISABLE_DEFAULT_FIELDS;ENABLE_FIELD_32;%(PreprocessorDefinitions) diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index 585c28e503be8..7ea4b96534690 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -14,10 +14,10 @@ - ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions) + ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;ENABLE_MODULE_ELLSWIFT;%(PreprocessorDefinitions) USE_ASM_X86_64;%(UndefinePreprocessorDefinitions) ..\..\src\secp256k1;%(AdditionalIncludeDirectories) - 4146;4244;4267;4334 + 4146;4244;4267 diff --git a/build_msvc/libtest_util/libtest_util.vcxproj.in b/build_msvc/libtest_util/libtest_util.vcxproj.in index c4c41401a2008..f30fd0d58716f 100644 --- a/build_msvc/libtest_util/libtest_util.vcxproj.in +++ b/build_msvc/libtest_util/libtest_util.vcxproj.in @@ -8,13 +8,14 @@ StaticLibrary + +@SOURCE_FILES@ {1DBB979A-C212-45CD-9563-446A96F87F72} {1DBB979A-C212-45CD-9563-446A96F87F71} -@SOURCE_FILES@ diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index 33f82f13542be..40ae7fc2bfa32 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -20,7 +20,7 @@ 'libbitcoin_wallet_tool', 'libbitcoin_wallet', 'libbitcoin_zmq', - 'bench_bench_navcoin', + 'bench_navcoin', 'libtest_util', ] diff --git a/build_msvc/navcoin-tx/navcoin-tx.vcxproj b/build_msvc/navcoin-tx/navcoin-tx.vcxproj index 22e9cc10b7ade..0f21567d08e0c 100644 --- a/build_msvc/navcoin-tx/navcoin-tx.vcxproj +++ b/build_msvc/navcoin-tx/navcoin-tx.vcxproj @@ -12,7 +12,7 @@ - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} @@ -30,12 +30,6 @@ {bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6} - - {1DBB979A-C212-45CD-9563-446A96F87F72} - - - {1DBB979A-C212-45CD-9563-446A96F87F71} - diff --git a/build_msvc/navcoin-util/navcoin-util.vcxproj b/build_msvc/navcoin-util/navcoin-util.vcxproj index 1bd5d9efcb8e3..2e460dc70bd8e 100644 --- a/build_msvc/navcoin-util/navcoin-util.vcxproj +++ b/build_msvc/navcoin-util/navcoin-util.vcxproj @@ -12,7 +12,7 @@ - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} @@ -30,12 +30,6 @@ {bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6} - - {1DBB979A-C212-45CD-9563-446A96F87F72} - - - {1DBB979A-C212-45CD-9563-446A96F87F71} - diff --git a/build_msvc/navcoin-wallet/navcoin-wallet.vcxproj b/build_msvc/navcoin-wallet/navcoin-wallet.vcxproj index d8cce6d7e94cf..8fbf64154ac43 100644 --- a/build_msvc/navcoin-wallet/navcoin-wallet.vcxproj +++ b/build_msvc/navcoin-wallet/navcoin-wallet.vcxproj @@ -15,7 +15,7 @@ - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} @@ -39,12 +39,6 @@ {bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6} - - {1DBB979A-C212-45CD-9563-446A96F87F71} - - - {1DBB979A-C212-45CD-9563-446A96F87F71} - diff --git a/build_msvc/navcoind/navcoind.vcxproj b/build_msvc/navcoind/navcoind.vcxproj index ea0b48a44a296..cd7e7da346425 100644 --- a/build_msvc/navcoind/navcoind.vcxproj +++ b/build_msvc/navcoind/navcoind.vcxproj @@ -9,13 +9,11 @@ $(SolutionDir)$(Platform)\$(Configuration)\ - - - $(IntDir)init_bitcoind.obj + + + $(IntDir)init_navcoind.obj - - - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} @@ -45,13 +43,8 @@ {18430fef-6b61-4c53-b396-718e02850f1b} - - {1DBB979A-C212-45CD-9563-446A96F87F72} - - - {1DBB979A-C212-45CD-9563-446A96F87F71} - + @@ -79,16 +72,20 @@ Replace="@USE_SQLITE_TRUE@" By=""> + + Replace="@ENABLE_FUZZ_BINARY_TRUE@" By="#"> + Replace="@ENABLE_EXTERNAL_SIGNER_TRUE@" By="#"> + diff --git a/build_msvc/test_navcoin/test_navcoin.vcxproj b/build_msvc/test_navcoin/test_navcoin.vcxproj index 7e0772bcb8020..8d2f8ca59a9f8 100644 --- a/build_msvc/test_navcoin/test_navcoin.vcxproj +++ b/build_msvc/test_navcoin/test_navcoin.vcxproj @@ -22,7 +22,7 @@ {542007e3-be0d-4b0d-a6b0-aa8813e2558d} - + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} @@ -58,13 +58,12 @@ {18430fef-6b61-4c53-b396-718e02850f1b} - - {1DBB979A-C212-45CD-9563-446A96F87F72} - - - {1DBB979A-C212-45CD-9563-446A96F87F71} - + + + 4018;4244;4267;4703;4715;4805 + + There was an error executing the JSON test header generation task. @@ -74,7 +73,7 @@ - + diff --git a/build_msvc/vcpkg.json b/build_msvc/vcpkg.json index 64eaaa0d6d4a6..3555afb2ea45a 100644 --- a/build_msvc/vcpkg.json +++ b/build_msvc/vcpkg.json @@ -3,18 +3,15 @@ "version-string": "1", "dependencies": [ "berkeleydb", + "boost-date-time", "boost-multi-index", - "boost-process", "boost-signals2", "boost-test", + "libevent", "sqlite3", - { - "name": "libevent", - "features": ["thread"] - }, "zeromq" ], - "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996", + "builtin-baseline": "9edb1b8e590cc086563301d735cae4b6e732d2d2", "overrides": [ { "name": "libevent", diff --git a/ci/README.md b/ci/README.md index d014565f4473c..7cf3b1f563b36 100644 --- a/ci/README.md +++ b/ci/README.md @@ -20,16 +20,11 @@ requires `bash`, `docker`, and `python3` to be installed. To install all require sudo apt install bash docker.io python3 ``` -To run the default test stage, +It is recommended to run the ci system in a clean env. To run the test stage +with a specific configuration, ``` -./ci/test_run_all.sh -``` - -To run the test stage with a specific configuration, - -``` -FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh +env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh' ``` ### Configurations @@ -44,14 +39,11 @@ the system package manager to install build dependencies. This guarantees that the tester is using the same versions as the release builds, which also use `./depends`. -If no `FILE_ENV` has been specified or values are left out, `00_setup_env.sh` -is used as the default configuration with fallback values. - It is also possible to force a specific configuration without modifying the file. For example, ``` -MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh +env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh' ``` The files starting with `0n` (`n` greater than 0) are the scripts that are run @@ -60,5 +52,5 @@ in order. ### Cache In order to avoid rebuilding all dependencies for each build, the binaries are -cached and re-used when possible. Changes in the dependency-generator will +cached and reused when possible. Changes in the dependency-generator will trigger cache-invalidation and rebuilds as necessary. diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index f7147582dc1da..476417d04b9ee 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -6,6 +6,8 @@ export LC_ALL=C +export PATH=$PWD/ci/retry:$PATH + ${CI_RETRY_EXE} apt-get update # Lint dependencies: # - curl/xz-utils (to install shellcheck) @@ -13,31 +15,42 @@ ${CI_RETRY_EXE} apt-get update # - gpg (used by verify-commits) ${CI_RETRY_EXE} apt-get install -y curl xz-utils git gpg -if [ -z "${SKIP_PYTHON_INSTALL}" ]; then - PYTHON_PATH=/tmp/python - if [ ! -d "${PYTHON_PATH}/bin" ]; then - ( - git clone https://github.com/pyenv/pyenv.git - cd pyenv/plugins/python-build || exit 1 - ./install.sh - ) - # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment - ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ - clang - env CC=clang python-build "$(cat "${BASE_ROOT_DIR}/.python-version")" "${PYTHON_PATH}" - fi - export PATH="${PYTHON_PATH}/bin:${PATH}" - command -v python3 - python3 --version +PYTHON_PATH="/python_build" +if [ ! -d "${PYTHON_PATH}/bin" ]; then + ( + ${CI_RETRY_EXE} git clone https://github.com/pyenv/pyenv.git + cd pyenv/plugins/python-build || exit 1 + ./install.sh + ) + # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment + ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev curl llvm \ + libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ + clang + env CC=clang python-build "$(cat "./.python-version")" "${PYTHON_PATH}" +fi +export PATH="${PYTHON_PATH}/bin:${PATH}" +command -v python3 +python3 --version + +export LINT_RUNNER_PATH="/lint_test_runner" +if [ ! -d "${LINT_RUNNER_PATH}" ]; then + ${CI_RETRY_EXE} apt-get install -y cargo + ( + cd ./test/lint/test_runner || exit 1 + cargo build + mkdir -p "${LINT_RUNNER_PATH}" + mv target/debug/test_runner "${LINT_RUNNER_PATH}" + ) fi -${CI_RETRY_EXE} pip3 install codespell==2.2.1 -${CI_RETRY_EXE} pip3 install flake8==5.0.4 -${CI_RETRY_EXE} pip3 install mypy==0.971 -${CI_RETRY_EXE} pip3 install pyzmq==24.0.1 -${CI_RETRY_EXE} pip3 install vulture==2.6 +${CI_RETRY_EXE} pip3 install \ + codespell==2.2.5 \ + flake8==6.1.0 \ + lief==0.13.2 \ + mypy==1.4.1 \ + pyzmq==25.1.0 \ + vulture==2.6 SHELLCHECK_VERSION=v0.8.0 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | \ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 5d62451553c84..83732e4f35b41 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -6,6 +6,8 @@ export LC_ALL=C +set -ex + if [ -n "$LOCAL_BRANCH" ]; then # To faithfully recreate CI linting locally, specify all commits on the current # branch. @@ -21,15 +23,7 @@ else fi export COMMIT_RANGE -# This only checks that the trees are pure subtrees, it is not doing a full -# check with -r to not have to fetch all the remotes. -test/lint/git-subtree-check.sh src/crypto/ctaes -test/lint/git-subtree-check.sh src/secp256k1 -test/lint/git-subtree-check.sh src/minisketch -test/lint/git-subtree-check.sh src/leveldb -test/lint/git-subtree-check.sh src/crc32c -test/lint/check-doc.py -test/lint/all-lint.py +RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner" if [ "$CIRRUS_REPO_FULL_NAME" = "navcoin/navcoin" ] && [ "$CIRRUS_PR" = "" ] ; then # Sanity check only the last few commits to get notified of missing sigs, diff --git a/ci/lint/Dockerfile b/ci/lint/Dockerfile deleted file mode 100644 index a0a45164803ff..0000000000000 --- a/ci/lint/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# See test/lint/README.md for usage. -# -# This container basically has to live in this directory in order to pull in the CI -# install scripts. If it lived in the root directory, it would have to pull in the -# entire repo as docker context during build; if it lived elsewhere, it wouldn't be -# able to make back-references to pull in the install scripts. So here it lives. - -FROM python:3.8-buster - -ENV DEBIAN_FRONTEND=noninteractive -ENV LC_ALL=C.UTF-8 - -# This is used by the 04_install.sh script; we can't read the Python version from -# .python-version for the same reasons as above, and it's more efficient to pull a -# preexisting Python image than it is to build from source. -ENV SKIP_PYTHON_INSTALL=1 - -# Must be built from ./ci/lint/ for these paths to work. -COPY ./docker-entrypoint.sh /entrypoint.sh -COPY ./04_install.sh /install.sh - -RUN /install.sh && \ - echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \ - chmod 755 /entrypoint.sh && \ - rm -rf /var/lib/apt/lists/* - - -WORKDIR /bitcoin -ENTRYPOINT ["/entrypoint.sh"] diff --git a/ci/lint/docker-entrypoint.sh b/ci/lint/container-entrypoint.sh similarity index 53% rename from ci/lint/docker-entrypoint.sh rename to ci/lint/container-entrypoint.sh index 3fdbbb0761c0a..a403f923a21a5 100755 --- a/ci/lint/docker-entrypoint.sh +++ b/ci/lint/container-entrypoint.sh @@ -1,10 +1,18 @@ #!/usr/bin/env bash +# +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + export LC_ALL=C # Fixes permission issues when there is a container UID/GID mismatch with the owner # of the mounted bitcoin src dir. git config --global --add safe.directory /bitcoin +export PATH="/python_build/bin:${PATH}" +export LINT_RUNNER_PATH="/lint_test_runner" + if [ -z "$1" ]; then LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh" else diff --git a/ci/lint_imagefile b/ci/lint_imagefile new file mode 100644 index 0000000000000..d32b35b19d0a7 --- /dev/null +++ b/ci/lint_imagefile @@ -0,0 +1,24 @@ +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +# See test/lint/README.md for usage. + +FROM debian:bookworm + +ENV DEBIAN_FRONTEND=noninteractive +ENV LC_ALL=C.UTF-8 + +COPY ./.python-version /.python-version +COPY ./ci/lint/container-entrypoint.sh /entrypoint.sh +COPY ./ci/lint/04_install.sh /install.sh +COPY ./test/lint/test_runner /test/lint/test_runner + +RUN /install.sh && \ + echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \ + chmod 755 /entrypoint.sh && \ + rm -rf /var/lib/apt/lists/* + + +WORKDIR /bitcoin +ENTRYPOINT ["/entrypoint.sh"] diff --git a/ci/lint_run_all.sh b/ci/lint_run_all.sh index 7adfe71674149..b56ee0d303f9b 100755 --- a/ci/lint_run_all.sh +++ b/ci/lint_run_all.sh @@ -8,4 +8,5 @@ export LC_ALL=C.UTF-8 set -o errexit; source ./ci/test/00_setup_env.sh set -o errexit; source ./ci/lint/04_install.sh -set -o errexit; source ./ci/lint/06_script.sh +set -o errexit +./ci/lint/06_script.sh diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 4a54f47b03f5a..b9e6818afbccd 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -6,16 +6,25 @@ export LC_ALL=C.UTF-8 -# The root dir. +set -ex + +# The source root dir, usually from git, usually read-only. # The ci system copies this folder. -BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) -export BASE_ROOT_DIR +BASE_READ_ONLY_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_READ_ONLY_DIR +# The destination root dir inside the container. +# This folder will also hold any SDKs. +# This folder only exists on the ci guest and will be a copy of BASE_READ_ONLY_DIR +export BASE_ROOT_DIR="${BASE_ROOT_DIR:-/ci_container_base}" # The depends dir. # This folder exists only on the ci guest, and on the ci host as a volume. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} -# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) -# This folder only exists on the ci host. +# A folder for the ci system to put temporary files (build result, datadirs for tests, ...) +# This folder only exists on the ci guest. export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} +# A folder for the ci system to put executables. +# This folder only exists on the ci guest. +export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/" echo "Setting specific values in env" if [ -n "${FILE_ENV}" ]; then @@ -27,48 +36,37 @@ fi echo "Fallback to default values in env (if not yet set)" # The number of parallel jobs to pass down to make and test_runner.py export MAKEJOBS=${MAKEJOBS:--j4} -# What host to compile for. See also ./depends/README.md -# Tests that need cross-compilation export the appropriate HOST. -# Tests that run natively guess the host -export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} # Whether to prefer BusyBox over GNU utilities export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} export RUN_TIDY=${RUN_TIDY:-false} -export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # By how much to scale the test_runner timeouts (option --timeout-factor). # This is needed because some ci machines have slow CPU or disk, so sanitizers # might be slow or a reindex might be waiting on disk IO. export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-40} -export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} -export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} -export CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG:-ubuntu:20.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} # See man 7 debconf export DEBIAN_FRONTEND=noninteractive -export CCACHE_SIZE=${CCACHE_SIZE:-100M} +export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-100M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # The cache dir. # This folder exists only on the ci guest, and on the ci host as a volume. export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} # Folder where the build result is put (bin and lib). -export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # Folder where the build is done (dist and out-of-tree build). export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. -export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} -export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" -export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} +export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} -export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index 8ce8f9ca8febc..5c75c14d38033 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2023 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export HOST=aarch64-linux-android export PACKAGES="unzip openjdk-8-jdk gradle" export CONTAINER_NAME=ci_android -export CI_IMAGE_NAME_TAG="ubuntu:jammy" +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:22.04" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index c26518ae94c57..0de15e435788a 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -7,18 +7,10 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf -# The host arch is unknown, so we run the tests through qemu. -# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi export DPKG_ADD_ARCH="armhf" export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" -if [ -n "$QEMU_USER_CMD" ]; then - # Likely cross-compiling, so install the needed gcc and qemu-user - export PACKAGES="$PACKAGES qemu-user" -fi export CONTAINER_NAME=ci_arm_linux -# Use debian to avoid 404 apt errors when cross compiling -export CI_IMAGE_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/arm64v8/debian:bookworm" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index e800fc75b004d..e6e6e5afd9966 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -8,12 +8,10 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream8" -# Use minimum supported python3.8 and gcc-8, see doc/dependencies.md -export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python38 python38-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" +export CI_IMAGE_NAME_TAG="quay.io/centos/amd64:stream9" +export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux e2fsprogs cmake" export PIP_PACKAGES="pyzmq" export GOAL="install" -export NO_WERROR=1 # GCC 8 +export NO_WERROR=1 # Suppress error: #warning _FORTIFY_SOURCE > 2 is treated like 2 on this platform [-Werror=cpp] export BITCOIN_CONFIG="--enable-zmq --enable-reduce-exports" export CONFIG_SHELL="/bin/dash" -export TEST_RUNNER_ENV="LC_ALL=en_US.UTF-8" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index c598ee50424bd..f31de86ca1105 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2022 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,10 +8,10 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:20.04" -export PACKAGES="cmake python3 llvm clang g++-multilib" +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:22.04" +export PACKAGES="llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" -export TEST_RUNNER_ENV="BITCOIND=navcoin-node" -export TEST_RUNNER_EXTRA="--nosandbox" +export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' \ +CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE'" +export BITCOIND=navcoin-node # Used in functional tests diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh deleted file mode 100755 index e454282b695e7..0000000000000 --- a/ci/test/00_setup_env_mac.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export CONTAINER_NAME=ci_macos_cross -export CI_IMAGE_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to macos -export HOST=x86_64-apple-darwin -export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso zip" -export XCODE_VERSION=12.2 -export XCODE_BUILD_ID=12B45b -export RUN_UNIT_TESTS=false -export RUN_FUNCTIONAL_TESTS=false -export GOAL="deploy" -export BITCOIN_CONFIG="--enable-reduce-exports LDFLAGS=-Wno-error=unused-command-line-argument" diff --git a/ci/test/00_setup_env_mac_cross.sh b/ci/test/00_setup_env_mac_cross.sh new file mode 100755 index 0000000000000..69a114c31427b --- /dev/null +++ b/ci/test/00_setup_env_mac_cross.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} + +export CONTAINER_NAME=ci_macos_cross +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +export HOST=x86_64-apple-darwin +export PACKAGES="zip" +export XCODE_VERSION=15.0 +export XCODE_BUILD_ID=15A240d +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export GOAL="deploy" +export BITCOIN_CONFIG="--enable-reduce-exports" diff --git a/ci/test/00_setup_env_mac_native_arm64.sh b/ci/test/00_setup_env_mac_native.sh similarity index 72% rename from ci/test/00_setup_env_mac_native_arm64.sh rename to ci/test/00_setup_env_mac_native.sh index ceaf2ce821d5b..cd5be896d9e89 100755 --- a/ci/test/00_setup_env_mac_native_arm64.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export HOST=arm64-apple-darwin +export HOST=x86_64-apple-darwin export PIP_PACKAGES="zmq" export GOAL="install" export BITCOIN_CONFIG="--with-miniupnpc --with-natpmp --enable-reduce-exports" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" -export CCACHE_SIZE=300M +export CCACHE_MAXSIZE=400M +export RUN_FUZZ_TESTS=true diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 028b54c9e05eb..c5f6171e6790a 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -1,21 +1,26 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" # Only install BCC tracing packages in Cirrus CI. if [[ "${CIRRUS_CI}" == "true" ]]; then - export BPFCC_PACKAGE="bpfcc-tools" + BPFCC_PACKAGE="bpfcc-tools linux-headers-$(uname --kernel-release)" + export CI_CONTAINER_CAP="--privileged -v /sys/kernel:/sys/kernel:rw" else - export BPFCC_PACKAGE="" + BPFCC_PACKAGE="" + export CI_CONTAINER_CAP="--cap-add SYS_PTRACE" # If run with (ASan + LSan), the container needs access to ptrace (https://github.com/google/sanitizers/issues/764) fi export CONTAINER_NAME=ci_native_asan -export PACKAGES="systemtap-sdt-dev clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev ${BPFCC_PACKAGE}" -export CI_IMAGE_NAME_TAG=ubuntu:23.04 # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). +export PACKAGES="systemtap-sdt-dev clang-17 llvm-17 libclang-rt-17-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang-16 CXX=clang++-16" +export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb \ +CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ +--with-sanitizers=address,float-divide-by-zero,integer,undefined \ +CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index fce8794071069..abee3c154169a 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -1,18 +1,21 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2023 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" +export PACKAGES="clang-17 llvm-17 libclang-rt-17-dev libevent-dev libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'" -export CCACHE_SIZE=200M +export CI_CONTAINER_CAP="--cap-add SYS_PTRACE" # If run with (ASan + LSan), the container needs access to ptrace (https://github.com/google/sanitizers/issues/764) +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \ +CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" +export CCACHE_MAXSIZE=200M +export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-17" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 9cea5b84de4a7..06697aa440296 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -1,25 +1,25 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2023 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" -LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_fuzz_msan" -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" export RUN_FUZZ_TESTS=true -export CCACHE_SIZE=250M +export CCACHE_MAXSIZE=250M diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 3511f65000564..1f60c4680368c 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2023 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm libclang-rt-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" +export PACKAGES="clang llvm libclang-rt-dev libevent-dev libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -16,5 +16,5 @@ export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" # Temporarily pin dwarf 4, until using Valgrind 3.20 or later -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" -export CCACHE_SIZE=200M +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC='clang -gdwarf-4' CXX='clang++ -gdwarf-4'" +export CCACHE_MAXSIZE=200M diff --git a/ci/test/00_setup_env_native_libblsct_only.sh b/ci/test/00_setup_env_native_libblsct_only.sh new file mode 100755 index 0000000000000..550cb580afcaa --- /dev/null +++ b/ci/test/00_setup_env_native_libblsct_only.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024-present The Navcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_libblsct_only +export CI_IMAGE_NAME_TAG="docker.io/debian:bullseye" +export PACKAGES="clang-13 llvm-13 libc++abi-13-dev libc++-13-dev" +export DEP_OPTS="NO_WALLET=1 CC=clang-13 CXX='clang++-13 -stdlib=libc++'" +export GOAL="install" +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export BITCOIN_CONFIG="--enable-build-libblsct-only" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 03690e757d5d8..adff79b333d34 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2023 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls -fsanitize-blacklist=${BASE_ROOT_DIR}/test/sanitizer_suppressions/msan" -LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" -export CCACHE_SIZE=250M +export CCACHE_MAXSIZE=250M diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh deleted file mode 100755 index 73e9ea1fc5799..0000000000000 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel -export CI_IMAGE_NAME_TAG="docker.io/ubuntu:20.04" -# Use minimum supported python3.8 and clang-8, see doc/dependencies.md -export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" -export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" -export GOAL="install" -export NO_WERROR=1 -export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_nowallet_libnavcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libnavcoinkernel.sh new file mode 100755 index 0000000000000..beee83a6f9f4b --- /dev/null +++ b/ci/test/00_setup_env_native_nowallet_libnavcoinkernel.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_nowallet_libnavcoinkernel +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +# Use minimum supported python3.9 (or best-effort 3.10) and clang-14, see doc/dependencies.md +export PACKAGES="python3-zmq clang-14 llvm-14 libc++abi-14-dev libc++-14-dev" +export DEP_OPTS="NO_WALLET=1 CC=clang-14 CXX='clang++-14 -stdlib=libc++'" +export GOAL="install" +export BITCOIN_CONFIG="--enable-reduce-exports --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native.sh b/ci/test/00_setup_env_native_previous_releases.sh similarity index 53% rename from ci/test/00_setup_env_native.sh rename to ci/test/00_setup_env_native_previous_releases.sh index a025ef6d54096..a6d1505d920ab 100755 --- a/ci/test/00_setup_env_native.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -1,21 +1,20 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CONTAINER_NAME=ci_native -export CI_IMAGE_NAME_TAG="ubuntu:20.04" -# Use minimum supported python3.8 and gcc-8 (or best-effort gcc-9), see doc/dependencies.md -export PACKAGES="gcc-9 g++-9 python3-zmq libdbus-1-dev libharfbuzz-dev" -export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-9 CXX=g++-9" +export CONTAINER_NAME=ci_native_previous_releases +export CI_IMAGE_NAME_TAG="docker.io/debian:bullseye" +# Use minimum supported python3.9 and gcc-10, see doc/dependencies.md +export PACKAGES="gcc-10 g++-10 python3-zmq" +export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-10 CXX=g++-10" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export NO_WERROR=1 export DOWNLOAD_PREVIOUS_RELEASES="true" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --enable-reduce-exports \ ---enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --enable-reduce-exports --enable-debug \ +CFLAGS=\"-g0 -O2 -funsigned-char\" CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index ee9e6810312b9..2b3ab57e33443 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -1,19 +1,20 @@ #!/usr/bin/env bash # -# Copyright (c) 2023 The Bitcoin Core developers +# Copyright (c) 2023-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:lunar" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_tidy -export PACKAGES="clang-16 libclang-16-dev llvm-16-dev libomp-16-dev clang-tidy-16 bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libsqlite3-dev libdb++-dev" +export TIDY_LLVM_V="17" +export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq bear libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=false export RUN_TIDY=true export GOAL="install" -export BITCOIN_CONFIG="CC=clang-16 CXX=clang++-16 --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0 -I/usr/lib/llvm-16/lib/clang/16/include'" -export CCACHE_SIZE=200M +export BITCOIN_CONFIG="CC=clang-${TIDY_LLVM_V} CXX=clang++-${TIDY_LLVM_V} --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0 -I/usr/lib/llvm-${TIDY_LLVM_V}/lib/clang/${TIDY_LLVM_V}/include'" +export CCACHE_MAXSIZE=200M diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 8ebb1fa563ccd..aa23bad8091fd 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2023 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export CI_IMAGE_NAME_TAG=ubuntu:23.04 # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libc++abi-16-dev libc++-16-dev python3-zmq" -export DEP_OPTS="CC=clang-16 CXX='clang++-16 -stdlib=libc++'" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" +export PACKAGES="clang-17 llvm-17 libclang-rt-17-dev libc++abi-17-dev libc++-17-dev python3-zmq" +export DEP_OPTS="CC=clang-17 CXX='clang++-17 -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 2f2a715a5675f..7f90aabe04d9d 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" # Temporarily pin dwarf 4, until using Valgrind 3.20 or later -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb CC='clang -gdwarf-4' CXX='clang++ -gdwarf-4'" diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 523e81c94a2da..ca84ecce5153c 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -1,25 +1,15 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export HOST=s390x-linux-gnu -# The host arch is unknown, so we run the tests through qemu. -# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi export PACKAGES="python3-zmq" -if [ -n "$QEMU_USER_CMD" ]; then - # Likely cross-compiling, so install the needed gcc and qemu-user - export DPKG_ADD_ARCH="s390x" - export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x" -fi -# Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export CI_IMAGE_NAME_TAG="debian:bookworm" -export TEST_RUNNER_ENV="LC_ALL=C" +export CI_IMAGE_NAME_TAG="docker.io/s390x/debian:bookworm" export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 9bf4d1948344b..b85521b4f2a4c 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,16 +1,19 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2023 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export CI_IMAGE_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:22.04" # Check that Jammy can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" -export PACKAGES="python3 g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" +export PACKAGES="nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports --enable-external-signer" +# Prior to 11.0.0, the mingw-w64 headers were missing noreturn attributes, causing warnings when +# cross-compiling for Windows. https://sourceforge.net/p/mingw-w64/bugs/306/ +# https://github.com/mingw-w64/mingw-w64/commit/1690994f515910a31b9fb7c7bd3a52d4ba987abe +export BITCOIN_CONFIG="--enable-reduce-exports CXXFLAGS=-Wno-return-type" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 641ff964f33ab..b15df4b6cc860 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -6,6 +6,8 @@ export LC_ALL=C.UTF-8 +set -ex + CFG_DONE="ci.base-install-done" # Use a global git setting to remember whether this script ran to avoid running it twice if [ "$(git config --global ${CFG_DONE})" == "true" ]; then @@ -18,9 +20,9 @@ if [ -n "$DPKG_ADD_ARCH" ]; then fi if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - ${CI_RETRY_EXE} bash -c "dnf -y install epel-release" - ${CI_RETRY_EXE} bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" -elif [ "$CI_USE_APT_INSTALL" != "no" ]; then + bash -c "dnf -y install epel-release" + bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" +elif [ "$CI_OS_NAME" != "macos" ]; then if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list fi @@ -29,28 +31,45 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then fi if [ -n "$PIP_PACKAGES" ]; then - if [ "$CI_OS_NAME" == "macos" ]; then - sudo -H pip3 install --upgrade pip - # shellcheck disable=SC2086 - IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES - else - # shellcheck disable=SC2086 - ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES - fi + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - update-alternatives --install /usr/bin/clang++ clang++ "$(which clang++-16)" 100 - update-alternatives --install /usr/bin/clang clang "$(which clang-16)" 100 - git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.1 "${BASE_SCRATCH_DIR}"/msan/llvm-project - cmake -B "${BASE_SCRATCH_DIR}"/msan/build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLIBCXX_ENABLE_DEBUG_MODE=ON -DLIBCXX_ENABLE_ASSERTIONS=ON -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes - make -C "${BASE_SCRATCH_DIR}"/msan/build/ "$MAKEJOBS" + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-17.0.2 /msan/llvm-project + + cmake -G Ninja -B /msan/clang_build/ \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ + -S /msan/llvm-project/llvm + + ninja -C /msan/clang_build/ "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds + ninja -C /msan/clang_build/ install-runtimes + + update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100 + update-alternatives --install /usr/bin/clang clang /msan/clang_build/bin/clang 100 + update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100 + + cmake -G Ninja -B /msan/cxx_build/ \ + -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_HARDENING_MODE=debug \ + -S /msan/llvm-project/runtimes + + ninja -C /msan/cxx_build/ "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds fi if [[ "${RUN_TIDY}" == "true" ]]; then - git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_16 "${DIR_IWYU}"/include-what-you-use - cmake -B "${DIR_IWYU}"/build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-16 -S "${DIR_IWYU}"/include-what-you-use - make -C "${DIR_IWYU}"/build/ install "$MAKEJOBS" + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${TIDY_LLVM_V}" /include-what-you-use + cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${TIDY_LLVM_V}" -S /include-what-you-use + make -C /iwyu-build/ install "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds fi mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" @@ -61,7 +80,7 @@ if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ] OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" if [ ! -f "$OSX_SDK_PATH" ]; then - curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" + ${CI_RETRY_EXE} curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" fi tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi @@ -69,11 +88,11 @@ fi if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip if [ ! -f "$ANDROID_TOOLS_PATH" ]; then - curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" + ${CI_RETRY_EXE} curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" fi mkdir -p "$ANDROID_HOME" unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME" - yes | "${ANDROID_HOME}"/cmdline-tools/bin/sdkmanager --sdk_root="${ANDROID_HOME}" --install "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" "platform-tools" "platforms;android-${ANDROID_API_LEVEL}" "ndk;${ANDROID_NDK_VERSION}" + yes | "${ANDROID_HOME}"/cmdline-tools/bin/sdkmanager --sdk_root="${ANDROID_HOME}" --install "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" "platform-tools" "platforms;android-31" "platforms;android-${ANDROID_API_LEVEL}" "ndk;${ANDROID_NDK_VERSION}" fi git config --global ${CFG_DONE} "true" diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh new file mode 100755 index 0000000000000..8a7a978994673 --- /dev/null +++ b/ci/test/02_run_container.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 +export CI_IMAGE_LABEL="bitcoin-ci-test" + +set -ex + +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then + # Export all env vars to avoid missing some. + # Though, exclude those with newlines to avoid parsing problems. + python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" != key and "PATH" != key and "USER" != key]' | tee /tmp/env + # System-dependent env vars must be kept as is. So read them from the container. + docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append /tmp/env + echo "Creating $CI_IMAGE_NAME_TAG container to run in" + DOCKER_BUILDKIT=1 docker build \ + --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ + --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ + --build-arg "FILE_ENV=${FILE_ENV}" \ + --label="${CI_IMAGE_LABEL}" \ + --tag="${CONTAINER_NAME}" \ + "${BASE_READ_ONLY_DIR}" + docker volume create "${CONTAINER_NAME}_ccache" || true + docker volume create "${CONTAINER_NAME}_depends" || true + docker volume create "${CONTAINER_NAME}_depends_sources" || true + docker volume create "${CONTAINER_NAME}_depends_SDKs_android" || true + docker volume create "${CONTAINER_NAME}_previous_releases" || true + + if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then + echo "Restart docker before run to stop and clear all containers started with --rm" + podman container rm --force --all # Similar to "systemctl restart docker" + + # Still prune everything in case the filtered pruning doesn't work, or if labels were not set + # on a previous run. Belt and suspenders approach, should be fine to remove in the future. + # Prune images used by --external containers (e.g. build containers) when + # using podman. + echo "Prune all dangling images" + podman image prune --force --external + fi + echo "Prune all dangling $CI_IMAGE_LABEL images" + # When detecting podman-docker, `--external` should be added. + docker image prune --force --filter "label=$CI_IMAGE_LABEL" + + # shellcheck disable=SC2086 + CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \ + --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \ + --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \ + --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built" \ + --mount "type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources" \ + --mount "type=volume,src=${CONTAINER_NAME}_depends_SDKs_android,dst=$DEPENDS_DIR/SDKs/android" \ + --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ + --env-file /tmp/env \ + --name "$CONTAINER_NAME" \ + "$CONTAINER_NAME") + export CI_CONTAINER_ID + export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" +else + echo "Running on host system without docker wrapper" + echo "Create missing folders" + mkdir -p "${CCACHE_DIR}" + mkdir -p "${PREVIOUS_RELEASES_DIR}" +fi + +if [ "$CI_OS_NAME" == "macos" ]; then + IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" + export IN_GETOPT_BIN +fi + +CI_EXEC () { + $CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*" +} +export -f CI_EXEC + +# Normalize all folders to BASE_ROOT_DIR +CI_EXEC rsync --archive --stats --human-readable "${BASE_READ_ONLY_DIR}/" "${BASE_ROOT_DIR}" || echo "Nothing to copy from ${BASE_READ_ONLY_DIR}/" +CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh" + +# Fixes permission issues when there is a container UID/GID mismatch with the owner +# of the git source code directory. +CI_EXEC git config --global --add safe.directory \"*\" + +CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" + +CI_EXEC "${BASE_ROOT_DIR}/ci/test/03_test_script.sh" + +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then + echo "Stop and remove CI container by ID" + docker container kill "${CI_CONTAINER_ID}" +fi diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh new file mode 100755 index 0000000000000..4c4fd0ec98146 --- /dev/null +++ b/ci/test/03_test_script.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +set -ex + +export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" +export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" +export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" + +if [ "$CI_OS_NAME" == "macos" ]; then + top -l 1 -s 0 | awk ' /PhysMem/ {print}' + echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" +else + free -m -h + echo "Number of CPUs (nproc): $(nproc)" + echo "System info: $(uname --kernel-name --kernel-release)" + lscpu +fi +echo "Free disk space:" +df -h + +# What host to compile for. See also ./depends/README.md +# Tests that need cross-compilation export the appropriate HOST. +# Tests that run natively guess the host +export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} + +( + # compact->outputs[i].file_size is uninitialized memory, so reading it is UB. + # The statistic bytes_written is only used for logging, which is disabled in + # CI, so as a temporary minimal fix to work around UB and CI failures, leave + # bytes_written unmodified. + # See https://github.com/bitcoin/bitcoin/pull/28359#issuecomment-1698694748 + echo 'diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc +index 65e31724bc..f61b471953 100644 +--- a/src/leveldb/db/db_impl.cc ++++ b/src/leveldb/db/db_impl.cc +@@ -1028,9 +1028,6 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { + stats.bytes_read += compact->compaction->input(which, i)->file_size; + } + } +- for (size_t i = 0; i < compact->outputs.size(); i++) { +- stats.bytes_written += compact->outputs[i].file_size; +- } + + mutex_.Lock(); + stats_[compact->compaction->level() + 1].Add(stats);' | patch -p1 + git diff +) + +if [ "$RUN_FUZZ_TESTS" = "true" ]; then + export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + if [ ! -d "$DIR_FUZZ_IN" ]; then + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/navcoin/qa-assets "${DIR_QA_ASSETS}" + fi + ( + cd "${DIR_QA_ASSETS}" + echo "Using qa-assets repo from commit ..." + git log -1 + ) +elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ + if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then + mkdir -p "$DIR_UNIT_TEST_DATA" + ${CI_RETRY_EXE} curl --location --fail https://github.com/navcoin/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" + fi +fi + +mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" + +if [ "$USE_BUSY_BOX" = "true" ]; then + echo "Setup to use BusyBox utils" + # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) + # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) + for util in $(busybox --list | grep -v "^ar$" | grep -v "^tar$" ); do ln -s "$(command -v busybox)" "${BINS_SCRATCH_DIR}/$util"; done + # Print BusyBox version + patch --help +fi + +# Make sure default datadir does not exist and is never read by creating a dummy file +if [ "$CI_OS_NAME" == "macos" ]; then + echo > "${HOME}/Library/Application Support/Navcoin" +else + echo > "${HOME}/.navcoin" +fi + +if [ -z "$NO_DEPENDS" ]; then + if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then + SHELL_OPTS="CONFIG_SHELL=/bin/dash" + else + SHELL_OPTS="CONFIG_SHELL=" + fi + bash -c "$SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS LOG=1" +fi +if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then + test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" +fi + +BITCOIN_CONFIG_ALL="--disable-dependency-tracking" +if [ -z "$NO_DEPENDS" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" +fi +if [ -z "$NO_WERROR" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" +fi + +ccache --zero-stats +PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" + +if [ -n "$ANDROID_TOOLS_URL" ]; then + make distclean || true + ./autogen.sh + bash -c "./configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) + make "${MAKEJOBS}" && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk + bash -c "${PRINT_CCACHE_STATISTICS}" + exit 0 +fi + +BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" + +if [ -n "$CONFIG_SHELL" ]; then + "$CONFIG_SHELL" -c "./autogen.sh" +else + ./autogen.sh +fi + +mkdir -p "${BASE_BUILD_DIR}" +cd "${BASE_BUILD_DIR}" + +bash -c "${BASE_ROOT_DIR}/configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) + +make distdir VERSION="$HOST" + +cd "${BASE_BUILD_DIR}/navcoin-$HOST" + +bash -c "./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) + +if [[ "${RUN_TIDY}" == "true" ]]; then + MAYBE_BEAR="bear --config src/.bear-tidy-config" + MAYBE_TOKEN="--" +fi + +bash -c "${MAYBE_BEAR} ${MAYBE_TOKEN} make $MAKEJOBS $GOAL" || ( echo "Build failure. Verbose build follows." && make "$GOAL" V=1 ; false ) + +bash -c "${PRINT_CCACHE_STATISTICS}" +du -sh "${DEPENDS_DIR}"/*/ +du -sh "${PREVIOUS_RELEASES_DIR}" + +if [[ $HOST = *-mingw32 ]]; then + # Generate all binaries, so that they can be wrapped + make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1 + "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" +fi + +if [ -n "$USE_VALGRIND" ]; then + "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" +fi + +if [ "$RUN_UNIT_TESTS" = "true" ]; then + DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" make "${MAKEJOBS}" check VERBOSE=1 +fi + +if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}"/bin/test_navcoin --catch_system_errors=no -l test_suite +fi + +if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then + # shellcheck disable=SC2086 + LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/functional/test_runner.py --ci "${MAKEJOBS}" --tmpdirprefix "${BASE_SCRATCH_DIR}"/test_runner/ --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" ${TEST_RUNNER_EXTRA} --quiet --failfast +fi + +if [ "${RUN_TIDY}" = "true" ]; then + cmake -B /tidy-build -DLLVM_DIR=/usr/lib/llvm-"${TIDY_LLVM_V}"/cmake -DCMAKE_BUILD_TYPE=Release -S "${BASE_ROOT_DIR}"/contrib/devtools/bitcoin-tidy + cmake --build /tidy-build "$MAKEJOBS" + cmake --build /tidy-build --target bitcoin-tidy-tests "$MAKEJOBS" + + set -eo pipefail + cd "${BASE_BUILD_DIR}/navcoin-$HOST/src/" + ( run-clang-tidy-"${TIDY_LLVM_V}" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" ) | grep -C5 "error" + # Filter out files by regex here, because regex may not be + # accepted in src/.bear-tidy-config + # Filter out: + # * qt qrc and moc generated files + jq 'map(select(.file | test("src/qt/qrc_.*\\.cpp$|/moc_.*\\.cpp$") | not))' ../compile_commands.json > tmp.json + mv tmp.json ../compile_commands.json + cd "${BASE_BUILD_DIR}/navcoin-$HOST/" + python3 "/include-what-you-use/iwyu_tool.py" \ + -p . "${MAKEJOBS}" \ + -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_BUILD_DIR}/navcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" \ + -Xiwyu --max_line_length=160 \ + 2>&1 | tee /tmp/iwyu_ci.out + cd "${BASE_ROOT_DIR}/src" + python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + git --no-pager diff +fi + +if [ "$RUN_FUZZ_TESTS" = "true" ]; then + # shellcheck disable=SC2086 + LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} "${MAKEJOBS}" -l DEBUG "${DIR_FUZZ_IN}" --empty_min_time=60 +fi diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh deleted file mode 100755 index 5ab6a7f9820e6..0000000000000 --- a/ci/test/04_install.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -if [[ $QEMU_USER_CMD == qemu-s390* ]]; then - export LC_ALL=C -fi - -# Create folders that are mounted into the docker -mkdir -p "${CCACHE_DIR}" -mkdir -p "${PREVIOUS_RELEASES_DIR}" - -export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" -export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" -export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" -if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) - CI_CONTAINER_CAP="--cap-add SYS_PTRACE" -fi - -export P_CI_DIR="$PWD" -export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/" - -if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - # Export all env vars to avoid missing some. - # Though, exclude those with newlines to avoid parsing problems. - python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value]' | tee /tmp/env - echo "Creating $CI_IMAGE_NAME_TAG container to run in" - DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \ - --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ - --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ - --build-arg "FILE_ENV=${FILE_ENV}" \ - --tag="${CONTAINER_NAME}" \ - "${BASE_ROOT_DIR}" - docker volume create "${CONTAINER_NAME}_ccache" || true - docker volume create "${CONTAINER_NAME}_depends" || true - docker volume create "${CONTAINER_NAME}_previous_releases" || true - - if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then - echo "Prune stopped containers" - podman container prune -f - echo "Prune all dangling images" - docker image prune --force - fi - - # shellcheck disable=SC2086 - CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ - --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \ - --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \ - --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ - -w $BASE_ROOT_DIR \ - --env-file /tmp/env \ - --name $CONTAINER_NAME \ - $CONTAINER_NAME) - export CI_CONTAINER_ID - export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" -else - echo "Running on host system without docker wrapper" -fi - -CI_EXEC () { - $CI_EXEC_CMD_PREFIX bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*" -} -export -f CI_EXEC - -CI_EXEC rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}" || echo "/ci_base_install/ missing" -CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh" -CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" || echo "Nothing to copy from ro_base" -# Fixes permission issues when there is a container UID/GID mismatch with the owner -# of the git source code directory. -CI_EXEC git config --global --add safe.directory \"*\" - -CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" - -if [ "$CI_OS_NAME" == "macos" ]; then - top -l 1 -s 0 | awk ' /PhysMem/ {print}' - echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" -else - CI_EXEC free -m -h - CI_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) - CI_EXEC echo "$(lscpu | grep Endian)" -fi -CI_EXEC echo "Free disk space:" -CI_EXEC df -h - -if [ "$RUN_FUZZ_TESTS" = "true" ]; then - export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ - if [ ! -d "$DIR_FUZZ_IN" ]; then - CI_EXEC git clone --depth=1 https://github.com/navcoin/qa-assets "${DIR_QA_ASSETS}" - fi -elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ - if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then - CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA" - CI_EXEC curl --location --fail https://github.com/navcoin/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" - fi -fi - -CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" - -if [ "$USE_BUSY_BOX" = "true" ]; then - echo "Setup to use BusyBox utils" - # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) - # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) - # shellcheck disable=SC1010 - CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done - # Print BusyBox version - CI_EXEC patch --help -fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh deleted file mode 100755 index 199cdd64a7c6c..0000000000000 --- a/ci/test/05_before_script.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -# Make sure default datadir does not exist and is never read by creating a dummy file -if [ "$CI_OS_NAME" == "macos" ]; then - echo > "${HOME}/Library/Application Support/Bitcoin" -else - CI_EXEC echo \> \$HOME/.bitcoin -fi - -if [ -z "$NO_DEPENDS" ]; then - if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - # CentOS has problems building the depends if the config shell is not explicitly set - # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to - # an error as the first command is executed) - SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/dash" - else - SHELL_OPTS="CONFIG_SHELL=" - fi - CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1 -fi -if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then - CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" -fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh deleted file mode 100755 index ed6b04807b414..0000000000000 --- a/ci/test/06_script_a.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" -if [ -z "$NO_DEPENDS" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" -fi -if [ -z "$NO_WERROR" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" -fi - -CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" -PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" - -if [ -n "$ANDROID_TOOLS_URL" ]; then - CI_EXEC make distclean || true - CI_EXEC ./autogen.sh - CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - CI_EXEC "make $MAKEJOBS" - CI_EXEC "${PRINT_CCACHE_STATISTICS}" - exit 0 -fi - -BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" - -if [ -n "$CONFIG_SHELL" ]; then - CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" -else - CI_EXEC ./autogen.sh -fi - -CI_EXEC mkdir -p "${BASE_BUILD_DIR}" -export P_CI_DIR="${BASE_BUILD_DIR}" - -CI_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - -CI_EXEC make distdir VERSION="$HOST" - -export P_CI_DIR="${BASE_BUILD_DIR}/navcoin-$HOST" - -CI_EXEC ./configure --cache-file=../config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - -set -o errtrace -trap 'CI_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR - -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - # MemorySanitizer (MSAN) does not support tracking memory initialization done by - # using the Linux getrandom syscall. Avoid using getrandom by undefining - # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for - # details. - CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' -fi - -if [[ "${RUN_TIDY}" == "true" ]]; then - MAYBE_BEAR="bear --config src/.bear-tidy-config" - MAYBE_TOKEN="--" -fi - -CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) - -CI_EXEC "${PRINT_CCACHE_STATISTICS}" -CI_EXEC du -sh "${DEPENDS_DIR}"/*/ -CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh deleted file mode 100755 index 6326dfb9d6ddb..0000000000000 --- a/ci/test/06_script_b.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -set -ex - -if [[ $HOST = *-mingw32 ]]; then - # Generate all binaries, so that they can be wrapped - make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 - make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1 - "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" -fi - -if [ -n "$QEMU_USER_CMD" ]; then - # Generate all binaries, so that they can be wrapped - make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 - make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 - "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" -fi - -if [ -n "$USE_VALGRIND" ]; then - "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" -fi - -if [ "$RUN_UNIT_TESTS" = "true" ]; then - bash -c "${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib make $MAKEJOBS check VERBOSE=1" -fi - -if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - bash -c "${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib ${BASE_OUTDIR}/bin/test_navcoin --catch_system_errors=no -l test_suite" -fi - -if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix ${BASE_SCRATCH_DIR}/test_runner/ --ansi --combinedlogslen=99999999 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast" -fi - -if [ "${RUN_TIDY}" = "true" ]; then - set -eo pipefail - cd "${BASE_BUILD_DIR}/navcoin-$HOST/src/" - ( run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error" - cd "${BASE_BUILD_DIR}/navcoin-$HOST/" - python3 "${DIR_IWYU}/include-what-you-use/iwyu_tool.py" \ - src/common/args.cpp \ - src/common/config.cpp \ - src/common/init.cpp \ - src/common/url.cpp \ - src/compat \ - src/dbwrapper.cpp \ - src/init \ - src/kernel \ - src/node/chainstate.cpp \ - src/node/chainstatemanager_args.cpp \ - src/node/mempool_args.cpp \ - src/node/minisketchwrapper.cpp \ - src/node/utxo_snapshot.cpp \ - src/node/validation_cache_args.cpp \ - src/policy/feerate.cpp \ - src/policy/packages.cpp \ - src/policy/settings.cpp \ - src/primitives/transaction.cpp \ - src/random.cpp \ - src/rpc/fees.cpp \ - src/rpc/signmessage.cpp \ - src/test/fuzz/string.cpp \ - src/test/fuzz/txorphan.cpp \ - src/test/fuzz/util \ - src/test/util/coins.cpp \ - src/uint256.cpp \ - src/util/bip32.cpp \ - src/util/bytevectorhash.cpp \ - src/util/check.cpp \ - src/util/error.cpp \ - src/util/exception.cpp \ - src/util/getuniquepath.cpp \ - src/util/hasher.cpp \ - src/util/message.cpp \ - src/util/moneystr.cpp \ - src/util/serfloat.cpp \ - src/util/spanparsing.cpp \ - src/util/strencodings.cpp \ - src/util/string.cpp \ - src/util/syserror.cpp \ - src/util/threadinterrupt.cpp \ - src/zmq \ - -p . "${MAKEJOBS}" \ - -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_BUILD_DIR}/navcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" \ - 2>&1 | tee /tmp/iwyu_ci.out - cd "${BASE_ROOT_DIR}/src" - python3 "${DIR_IWYU}/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out - git --no-pager diff -fi - -if [ "$RUN_SECURITY_TESTS" = "true" ]; then - make test-security-check -fi - -if [ "$RUN_FUZZ_TESTS" = "true" ]; then - bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN}" -fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh deleted file mode 100755 index e028ede37888f..0000000000000 --- a/ci/test/wrap-qemu.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}; do - # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do - echo "Wrap $b ..." - mv "$b" "${b}_orig" - echo '#!/usr/bin/env bash' > "$b" - echo "$QEMU_USER_CMD \"${b}_orig\" \"\$@\"" >> "$b" - chmod +x "$b" - done -done diff --git a/ci/test/wrapped-cl.bat b/ci/test/wrapped-cl.bat deleted file mode 100644 index fc2a604c580be..0000000000000 --- a/ci/test/wrapped-cl.bat +++ /dev/null @@ -1 +0,0 @@ -ccache cl %* diff --git a/ci/test_imagefile b/ci/test_imagefile index 4854708d1a9b2..f8b5eea1c88ab 100644 --- a/ci/test_imagefile +++ b/ci/test_imagefile @@ -1,3 +1,9 @@ +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +# See ci/README.md for usage. + ARG CI_IMAGE_NAME_TAG FROM ${CI_IMAGE_NAME_TAG} @@ -5,6 +11,6 @@ ARG FILE_ENV ENV FILE_ENV=${FILE_ENV} COPY ./ci/retry/retry /usr/bin/retry -COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_base_install/ci/test/ +COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_container_base/ci/test/ -RUN ["bash", "-c", "cd /ci_base_install/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] +RUN ["bash", "-c", "cd /ci_container_base/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh index 751a4056df7bc..3afc47b23edae 100755 --- a/ci/test_run_all.sh +++ b/ci/test_run_all.sh @@ -1,19 +1,11 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 set -o errexit; source ./ci/test/00_setup_env.sh -set -o errexit; source ./ci/test/04_install.sh -set -o errexit; source ./ci/test/05_before_script.sh -set -o errexit; source ./ci/test/06_script_a.sh set -o errexit -CI_EXEC "${BASE_ROOT_DIR}/ci/test/06_script_b.sh" - -if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Stop and remove CI container by ID" - docker container kill "${CI_CONTAINER_ID}" -fi +"./ci/test/02_run_container.sh" diff --git a/configure.ac b/configure.ac index 34291ebf2b971..92773826c13be 100644 --- a/configure.ac +++ b/configure.ac @@ -1,10 +1,10 @@ AC_PREREQ([2.69]) -define(_CLIENT_VERSION_MAJOR, 25) +define(_CLIENT_VERSION_MAJOR, 26) define(_CLIENT_VERSION_MINOR, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2023) +define(_COPYRIGHT_YEAR, 2024) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin Core]]) AC_INIT([Navcoin Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_BUILD)m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/navcoin/navcoin/issues],[navcoin],[https://navcoin.org/]) @@ -29,13 +29,13 @@ if test -n "$PKG_CONFIG_LIBDIR"; then PKG_CONFIG="env PKG_CONFIG_LIBDIR=$PKG_CONFIG_LIBDIR $PKG_CONFIG" fi -BITCOIN_DAEMON_NAME=navcoind -BITCOIN_TEST_NAME=test_navcoin -BITCOIN_CLI_NAME=navcoin-cli -BITCOIN_TX_NAME=navcoin-tx -BITCOIN_UTIL_NAME=navcoin-util -BITCOIN_CHAINSTATE_NAME=navcoin-chainstate -BITCOIN_WALLET_TOOL_NAME=navcoin-wallet +NAVCOIN_DAEMON_NAME=navcoind +NAVCOIN_TEST_NAME=test_navcoin +NAVCOIN_CLI_NAME=navcoin-cli +NAVCOIN_TX_NAME=navcoin-tx +NAVCOIN_UTIL_NAME=navcoin-util +NAVCOIN_CHAINSTATE_NAME=navcoin-chainstate +NAVCOIN_WALLET_TOOL_NAME=navcoin-wallet dnl Multi Process BITCOIN_MP_NODE_NAME=navcoin-node @@ -68,11 +68,12 @@ else fi AC_PROG_CXX -dnl By default, libtool for mingw refuses to link static libs into a dll for -dnl fear of mixing pic/non-pic objects, and import/export complications. Since -dnl we have those under control, re-enable that functionality. +dnl libtool overrides case $host in *mingw*) + dnl By default, libtool for mingw refuses to link static libs into a dll for + dnl fear of mixing pic/non-pic objects, and import/export complications. Since + dnl we have those under control, re-enable that functionality. lt_cv_deplibs_check_method="pass_all" dnl Remove unwanted -DDLL_EXPORT from these variables. @@ -81,29 +82,20 @@ case $host in lt_cv_prog_compiler_pic="-DPIC" lt_cv_prog_compiler_pic_CXX="-DPIC" ;; + *darwin*) + dnl Because it prints a verbose warning, lld fails the following check + dnl for "-Wl,-single_module" from libtool.m4: + dnl # If there is a non-empty error log, and "single_module" + dnl # appears in it, assume the flag caused a linker warning + dnl "-single_module" works fine on ld64 and lld, so just bypass the test. + dnl Failure to set this to "yes" causes libtool to use a very broken + dnl link-line for shared libs. + lt_cv_apple_cc_single_mod="yes" + ;; esac -AC_ARG_WITH([seccomp], - [AS_HELP_STRING([--with-seccomp], - [enable experimental syscall sandbox feature (-sandbox), default is yes if seccomp-bpf is detected under Linux x86_64])], - [seccomp_found=$withval], - [seccomp_found=auto]) - -AC_ARG_ENABLE([c++20], - [AS_HELP_STRING([--enable-c++20], - [enable compilation in c++20 mode (disabled by default)])], - [use_cxx20=$enableval], - [use_cxx20=no]) - -dnl Require C++17 compiler (no GNU extensions) -if test "$use_cxx20" = "no"; then -AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) -else +dnl Require C++20 compiler (no GNU extensions) AX_CXX_COMPILE_STDCXX([20], [noext], [mandatory]) -fi - -dnl check if additional link flags are required for std::filesystem -CHECK_FILESYSTEM dnl Unless the user specified OBJCXX, force it to be the same as CXX. This ensures dnl that we get the same -std flags for both. @@ -124,8 +116,8 @@ AC_PATH_TOOL([AR], [ar]) AC_PATH_TOOL([GCOV], [gcov]) AC_PATH_TOOL([LLVM_COV], [llvm-cov]) AC_PATH_PROG([LCOV], [lcov]) -dnl Python 3.8 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.8 python3.9 python3.10 python3.11 python3.12 python3 python]) +dnl The minimum supported version is specified in .python-version and should be used if available, see doc/dependencies.md +AC_PATH_PROGS([PYTHON], [python3.9 python3.10 python3.11 python3.12 python3 python]) AC_PATH_PROG([GENHTML], [genhtml]) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG([CCACHE], [ccache]) @@ -135,8 +127,6 @@ AC_PATH_TOOL([OBJCOPY], [objcopy]) AC_PATH_PROG([DOXYGEN], [doxygen]) AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"]) -AC_ARG_VAR([PYTHONPATH], [Augments the default search path for python module files]) - AC_ARG_ENABLE([wallet], [AS_HELP_STRING([--disable-wallet], [disable wallet (enabled by default)])], @@ -223,10 +213,10 @@ dnl May be useful if warnings from external headers clutter the build output dnl too much, so that it becomes difficult to spot Navcoin Core warnings dnl or if they cause a build failure with --enable-werror. AC_ARG_ENABLE([suppress-external-warnings], - [AS_HELP_STRING([--enable-suppress-external-warnings], - [Suppress warnings from external headers (default is no)])], + [AS_HELP_STRING([--disable-suppress-external-warnings], + [Do not suppress warnings from external headers (default is to suppress)])], [suppress_external_warnings=$enableval], - [suppress_external_warnings=no]) + [suppress_external_warnings=yes]) AC_ARG_ENABLE([lcov], [AS_HELP_STRING([--enable-lcov], @@ -318,11 +308,6 @@ AC_ARG_ENABLE([external-signer], [use_external_signer=$enableval], [use_external_signer=auto]) -AC_ARG_ENABLE([lto], - [AS_HELP_STRING([--enable-lto],[build using LTO (default is no)])], - [enable_lto=$enableval], - [enable_lto=no]) - AC_ARG_ENABLE([build-libblsct-only], [AS_HELP_STRING([--enable-build-libblsct-only], [build libblsct.a only and do not build others (default is to build everything)])], @@ -380,11 +365,6 @@ if test "$enable_debug" = "yes"; then AX_CHECK_COMPILE_FLAG([-ftrapv], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"], [], [$CXXFLAG_WERROR]) fi -if test "$enable_lto" = "yes"; then - AX_CHECK_COMPILE_FLAG([-flto], [LTO_CXXFLAGS="$LTO_CXXFLAGS -flto"], [AC_MSG_ERROR([compile failed with -flto])], [$CXXFLAG_WERROR]) - AX_CHECK_LINK_FLAG([-flto], [LTO_LDFLAGS="$LTO_LDFLAGS -flto"], [AC_MSG_ERROR([link failed with -flto])], [$CXXFLAG_WERROR]) -fi - if test "$use_sanitizers" != ""; then dnl First check if the compiler accepts flags. If an incompatible pair like dnl -fsanitize=address,thread is used here, this check will fail. This will also @@ -418,12 +398,6 @@ if test "$enable_werror" = "yes"; then AC_MSG_ERROR([enable-werror set but -Werror is not usable]) fi ERROR_CXXFLAGS=$CXXFLAG_WERROR - - dnl -Wreturn-type is broken in GCC for MinGW-w64. - dnl https://sourceforge.net/p/mingw-w64/bugs/306/ - AX_CHECK_COMPILE_FLAG([-Werror=return-type], [], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Wno-error=return-type"], [$CXXFLAG_WERROR], - [AC_LANG_SOURCE([[#include - int f(){ assert(false); }]])]) fi if test "$CXXFLAGS_overridden" = "no"; then @@ -444,12 +418,9 @@ if test "$CXXFLAGS_overridden" = "no"; then AX_CHECK_COMPILE_FLAG([-Wduplicated-cond], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wlogical-op], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"], [], [$CXXFLAG_WERROR]) - dnl -Wsuggest-override is broken with GCC before 9.2 - dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 - AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"], [], [$CXXFLAG_WERROR], - [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) - AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wunreachable-code], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code"], [], [$CXXFLAG_WERROR]) if test "$suppress_external_warnings" != "no" ; then AX_CHECK_COMPILE_FLAG([-Wdocumentation], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"], [], [$CXXFLAG_WERROR]) @@ -468,6 +439,12 @@ fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) +dnl Currently all versions of gcc are subject to a class of bugs, see the +dnl gccbug_90348 test case (only reproduces on GCC 11 and earlier) and +dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111843. To work around that, set +dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) +AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fstack-reuse=none"]) + enable_arm_crc=no enable_arm_shani=no enable_sse42=no @@ -575,7 +552,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ CXXFLAGS="$TEMP_CXXFLAGS" # ARM -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-march=armv8-a+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crypto"], [], [$CXXFLAG_WERROR]) TEMP_CXXFLAGS="$CXXFLAGS" @@ -619,6 +596,9 @@ fi CORE_CPPFLAGS="$CORE_CPPFLAGS -DHAVE_BUILD_INFO" +# We need to have have BLS_ETH=1 for bls and mcl libs +CORE_CPPFLAGS="$CORE_CPPFLAGS -DBLS_ETH=1" + AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], [build navcoin-cli navcoin-tx navcoin-util navcoin-wallet (default=yes)])], @@ -663,15 +643,15 @@ AC_ARG_WITH([libs], AC_ARG_WITH([experimental-kernel-lib], [AS_HELP_STRING([--with-experimental-kernel-lib], - [build experimental bitcoinkernel library (default is to build if we're building libraries and the experimental build-chainstate executable)])], + [build experimental navcoinkernel library (default is to build if we're building libraries and the experimental build-chainstate executable)])], [build_experimental_kernel_lib=$withval], [build_experimental_kernel_lib=auto]) AC_ARG_WITH([daemon], [AS_HELP_STRING([--with-daemon], [build navcoind daemon (default=yes)])], - [build_bitcoind=$withval], - [build_bitcoind=yes]) + [build_navcoind=$withval], + [build_navcoind=yes]) case $host in *mingw*) @@ -705,6 +685,8 @@ case $host in AC_MSG_ERROR([windres not found]) fi + CORE_CPPFLAGS="$CORE_CPPFLAGS -DSECP256K1_STATIC" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" dnl Prevent the definition of min/max macros. dnl We always want to use the standard library. @@ -720,6 +702,10 @@ case $host in dnl We require Windows 7 (NT 6.1) or later AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) + + dnl Avoid the use of aligned vector instructions when building for Windows. + dnl See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412. + AX_CHECK_COMPILE_FLAG([-Wa,-muse-unaligned-vector-move], [CORE_CXXFLAGS="$CORE_CXXFLAGS -Wa,-muse-unaligned-vector-move"], [], [$CXXFLAG_WERROR]) ;; *darwin*) TARGET_OS=darwin @@ -740,7 +726,7 @@ case $host in dnl option to system-ify all /usr/local/include paths without adding it to the list dnl of search paths in case it's not already there. if test "$suppress_external_warnings" != "no"; then - AX_CHECK_PREPROC_FLAG([-Xclang -internal-isystem/usr/local/include], [CORE_CPPFLAGS="$CORE_CPPFLAGS -Xclang -internal-isystem/usr/local/include"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-Xclang -internal-isystem -Xclang /usr/local/include/], [CORE_CPPFLAGS="$CORE_CPPFLAGS -Xclang -internal-isystem -Xclang /usr/local/include/"], [], [$CXXFLAG_WERROR]) fi if test "$use_bdb" != "no" && $BREW list --versions berkeley-db@4 >/dev/null && test "$BDB_CFLAGS" = "" && test "$BDB_LIBS" = ""; then @@ -790,7 +776,7 @@ case $host in AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil]) AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], [install_name_tool]) AC_PATH_TOOL([OTOOL], [otool], [otool]) - AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs]) + AC_PATH_PROG([ZIP], [zip], [zip]) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, @@ -910,8 +896,6 @@ if test "$ac_cv_sys_large_files" != "" && CORE_CPPFLAGS="$CORE_CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi -AC_SEARCH_LIBS([clock_gettime],[rt]) - if test "$enable_gprof" = "yes"; then dnl -pg is incompatible with -pie. Since hardening and profiling together doesn't make sense, dnl we simply make them mutually exclusive here. Additionally, hardened toolchains may force @@ -934,12 +918,6 @@ if test "$TARGET_OS" != "windows"; then AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi -dnl Versions of gcc prior to 12.1 (commit -dnl https://github.com/gcc-mirror/gcc/commit/551aa75778a4c5165d9533cd447c8fc822f583e1) -dnl are subject to a bug, see the gccbug_90348 test case and -dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set -dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) -AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) if test "$use_hardening" != "no"; then use_hardening=yes AX_CHECK_COMPILE_FLAG([-Wstack-protector], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) @@ -949,7 +927,8 @@ if test "$use_hardening" != "no"; then case $host in *mingw*) - dnl stack-clash-protection doesn't currently work, and likely should just be skipped for Windows. + dnl stack-clash-protection doesn't compile with GCC 10 and earlier. + dnl In any case, it is a no-op for Windows. dnl See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90458 for more details. ;; *) @@ -957,6 +936,11 @@ if test "$use_hardening" != "no"; then ;; esac + case $host in + *aarch64*) + AX_CHECK_COMPILE_FLAG([-mbranch-protection=bti], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -mbranch-protection=bti"]) + ;; + esac dnl When enable_debug is yes, all optimizations are disabled. dnl However, FORTIFY_SOURCE requires that there is some level of optimization, otherwise it does nothing and just creates a compiler warning. @@ -978,12 +962,6 @@ if test "$use_hardening" != "no"; then AX_CHECK_LINK_FLAG([-Wl,-z,now], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-z,separate-code], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,separate-code"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-fPIE -pie], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"], [], [$CXXFLAG_WERROR]) - - case $host in - *mingw*) - AC_CHECK_LIB([ssp], [main], [], [AC_MSG_ERROR([libssp missing])]) - ;; - esac fi dnl These flags are specific to ld64, and may cause issues with other linkers. @@ -992,7 +970,7 @@ dnl "ad_strip" as the symbol for the entry point. if test "$TARGET_OS" = "darwin"; then AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) - AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-fixup_chains], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-fixup_chains"], [], [$LDFLAG_WERROR]) fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) @@ -1057,7 +1035,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], dnl Check for posix_fallocate AC_MSG_CHECKING([for posix_fallocate]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - // same as in src/util/system.cpp + // same as in src/util/fs_helpers.cpp #ifdef __linux__ #ifdef _POSIX_C_SOURCE #undef _POSIX_C_SOURCE @@ -1155,17 +1133,16 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], ) dnl Check for different ways of gathering OS randomness -AC_MSG_CHECKING([for Linux getrandom syscall]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include - #include - #include ]], - [[ syscall(SYS_getrandom, nullptr, 32, 0); ]])], - [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYS_GETRANDOM], [1], [Define this symbol if the Linux getrandom system call is available]) ], +AC_MSG_CHECKING([for Linux getrandom function]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include ]], + [[ getrandom(nullptr, 32, 0); ]])], + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETRANDOM], [1], [Define this symbol if the Linux getrandom function call is available]) ], [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING([for getentropy via random.h]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include +AC_MSG_CHECKING([for getentropy via sys/random.h]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include ]], [[ getentropy(nullptr, 32) ]])], [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETENTROPY_RAND], [1], [Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], @@ -1303,7 +1280,7 @@ if test "$enable_fuzz" = "yes"; then build_bitcoin_util=no build_bitcoin_chainstate=no build_bitcoin_wallet=no - build_bitcoind=no + build_navcoind=no build_bitcoin_libs=no use_bench=no use_tests=no @@ -1324,7 +1301,7 @@ if test "$enable_build_libblsct_only" = "yes"; then build_bitcoin_util=no build_bitcoin_chainstate=no build_bitcoin_wallet=no - build_bitcoind=no + build_navcoind=no build_bitcoin_libs=no use_bench=no use_tests=no @@ -1351,8 +1328,6 @@ if test "$enable_fuzz_binary" = "yes"; then ]],[[ */ int not_main() { ]])]) - - CHECK_RUNTIME_LIB fi if test "$enable_wallet" != "no"; then @@ -1398,7 +1373,9 @@ if test "$use_usdt" != "no"; then AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [#include ], - [DTRACE_PROBE("context", "event");] + [DTRACE_PROBE(context, event); + int a, b, c, d, e, f, g; + DTRACE_PROBE7(context, event, a, b, c, d, e, f, g);] )], [AC_MSG_RESULT([yes]); AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable tracepoints for Userspace, Statically Defined Tracing])], [AC_MSG_RESULT([no]); use_usdt=no;] @@ -1406,7 +1383,7 @@ if test "$use_usdt" != "no"; then fi AM_CONDITIONAL([ENABLE_USDT_TRACEPOINTS], [test "$use_usdt" = "yes"]) -if test "$build_bitcoind$use_bench$use_tests" = "nononono"; then +if test "$build_navcoind$use_bench$use_tests" = "nonono"; then use_upnp=no use_natpmp=no use_zmq=no @@ -1416,15 +1393,13 @@ dnl Check for libminiupnpc (optional) if test "$use_upnp" != "no"; then TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $MINIUPNPC_CPPFLAGS" - AC_CHECK_HEADERS( - [miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], - [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS])], - [have_miniupnpc=no] - ) + AC_CHECK_HEADERS([miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], [], [have_miniupnpc=no]) - dnl The minimum supported miniUPnPc API version is set to 17. This excludes - dnl versions with known vulnerabilities. if test "$have_miniupnpc" != "no"; then + AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS]) + + dnl The minimum supported miniUPnPc API version is set to 17. This excludes + dnl versions with known vulnerabilities. AC_MSG_CHECKING([whether miniUPnPc API version is supported]) AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ @%:@include @@ -1449,13 +1424,16 @@ dnl Check for libnatpmp (optional). if test "$use_natpmp" != "no"; then TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $NATPMP_CPPFLAGS" - AC_CHECK_HEADERS([natpmp.h], - [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS="$NATPMP_LIBS -lnatpmp"], [have_natpmp=no], [$NATPMP_LIBS])], - [have_natpmp=no]) + AC_CHECK_HEADERS([natpmp.h], [], [have_natpmp=no]) + + if test "$have_natpmp" != "no"; then + AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS="$NATPMP_LIBS -lnatpmp"], [have_natpmp=no], [$NATPMP_LIBS]) + fi + CPPFLAGS="$TEMP_CPPFLAGS" fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$use_tests$use_bench$enable_fuzz_binary" = "nonononononononono"; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_navcoind$use_tests$use_bench$enable_fuzz_binary" = "nononononononono"; then use_boost=no else use_boost=yes @@ -1464,9 +1442,9 @@ fi if test "$use_boost" = "yes"; then dnl Check for Boost headers - AX_BOOST_BASE([1.64.0],[],[AC_MSG_ERROR([Boost is not available!])]) + AX_BOOST_BASE([1.73.0],[],[AC_MSG_ERROR([Boost is not available!])]) if test "$want_boost" = "no"; then - AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost]) + AC_MSG_ERROR([only libnavcoinconsensus can be built without Boost]) fi dnl we don't use multi_index serialization @@ -1479,10 +1457,6 @@ if test "$use_boost" = "yes"; then AX_CHECK_PREPROC_FLAG([-DBOOST_NO_CXX98_FUNCTION_BASE], [BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE"], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([[#include ]])]) - if test "$enable_debug" = "yes" || test "$enable_fuzz" = "yes"; then - BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE" - fi - if test "$suppress_external_warnings" != "no"; then BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) fi @@ -1519,9 +1493,19 @@ if test "$use_external_signer" != "no"; then CXXFLAGS="$TEMP_CXXFLAGS" AC_MSG_RESULT([$have_boost_process]) if test "$have_boost_process" = "yes"; then - use_external_signer="yes" - AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) - AC_DEFINE([BOOST_PROCESS_USE_STD_FS], [1], [Defined to avoid Boost::Process trying to use Boost Filesystem]) + case $host in + dnl Boost Process for Windows uses Boost ASIO. Boost ASIO performs + dnl pre-main init of Windows networking libraries, which we do not + dnl want. + *mingw*) + use_external_signer="no" + ;; + *) + use_external_signer="yes" + AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) + AC_DEFINE([BOOST_PROCESS_USE_STD_FS], [1], [Defined to avoid Boost::Process trying to use Boost Filesystem]) + ;; + esac else if test "$use_external_signer" = "yes"; then AC_MSG_ERROR([External signing is not supported for this Boost version]) @@ -1531,36 +1515,6 @@ if test "$use_external_signer" != "no"; then fi AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) -dnl Do not compile with syscall sandbox support when compiling under the sanitizers. -dnl The sanitizers introduce use of syscalls that are not typically used in navcoind -dnl (such as execve when the sanitizers execute llvm-symbolizer). -if test "$use_sanitizers" != ""; then - AC_MSG_WARN([Specifying --with-sanitizers forces --without-seccomp since the sanitizers introduce use of syscalls not allowed by the navcoind syscall sandbox (-sandbox=).]) - seccomp_found=no -fi -if test "$seccomp_found" != "no"; then - AC_MSG_CHECKING([for seccomp-bpf (Linux x86-64)]) - AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ - @%:@include - ]], [[ - #if !defined(__x86_64__) - # error Syscall sandbox is an experimental feature currently available only under Linux x86-64. - #endif - ]])],[ - AC_MSG_RESULT([yes]) - seccomp_found="yes" - AC_DEFINE([USE_SYSCALL_SANDBOX], [1], [Define this symbol to build with syscall sandbox support.]) - ],[ - AC_MSG_RESULT([no]) - seccomp_found="no" - ]) -fi -dnl Currently only enable -sandbox= feature if seccomp is found. -dnl In the future, sandboxing could be also be supported with other -dnl sandboxing mechanisms besides seccomp. -use_syscall_sandbox=$seccomp_found -AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "$use_syscall_sandbox" != "no"]) - dnl Check for reduced exports if test "$use_reduce_exports" = "yes"; then AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fvisibility=hidden"], @@ -1578,7 +1532,7 @@ fi dnl libevent check use_libevent=no -if test "$build_bitcoin_cli$build_bitcoind$enable_fuzz_binary$use_tests$use_bench" != "nonononono"; then +if test "$build_bitcoin_cli$build_navcoind$enable_fuzz_binary$use_tests$use_bench" != "nonononono"; then PKG_CHECK_MODULES([EVENT], [libevent >= 2.1.8], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.1.8 or greater not found.])]) if test "$TARGET_OS" != "windows"; then PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.1.8], [], [AC_MSG_ERROR([libevent_pthreads version 2.1.8 or greater not found.])]) @@ -1613,12 +1567,9 @@ dnl ZMQ check if test "$use_zmq" = "yes"; then PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], - AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), - [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) - AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) + AC_DEFINE([ENABLE_ZMQ], [1], [Define this symbol to enable ZMQ functions]), + [AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) use_zmq=no]) -else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) fi if test "$use_zmq" = "yes"; then @@ -1630,6 +1581,8 @@ if test "$use_zmq" = "yes"; then esac fi +AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) + dnl libmultiprocess library check libmultiprocess_found=no @@ -1669,9 +1622,9 @@ if test "$build_multiprocess" != "no"; then AC_SUBST(MPGEN_PREFIX) fi -AC_MSG_CHECKING([whether to build bitcoind]) -AM_CONDITIONAL([BUILD_BITCOIND], [test $build_bitcoind = "yes"]) -AC_MSG_RESULT($build_bitcoind) +AC_MSG_CHECKING([whether to build navcoind]) +AM_CONDITIONAL([BUILD_BITCOIND], [test $build_navcoind = "yes"]) +AC_MSG_RESULT($build_navcoind) AC_MSG_CHECKING([whether to build bitcoin-cli]) AM_CONDITIONAL([BUILD_BITCOIN_CLI], [test $build_bitcoin_cli = "yes"]) @@ -1692,7 +1645,7 @@ AC_MSG_RESULT($build_bitcoin_util) AC_MSG_CHECKING([whether to build experimental bitcoin-chainstate]) if test "$build_bitcoin_chainstate" = "yes"; then if test "$build_experimental_kernel_lib" = "no"; then - AC_MSG_ERROR([experimental bitcoin-chainstate cannot be built without the experimental bitcoinkernel library. Use --with-experimental-kernel-lib]); + AC_MSG_ERROR([experimental bitcoin-chainstate cannot be built without the experimental navcoinkernel library. Use --with-experimental-kernel-lib]); fi fi AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"]) @@ -1703,7 +1656,7 @@ AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"]) if test "$build_bitcoin_libs" = "yes"; then AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built]) - AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) + AC_CONFIG_FILES([libnavcoinconsensus.pc:libnavcoinconsensus.pc.in]) fi AM_CONDITIONAL([BUILD_BITCOIN_KERNEL_LIB], [test "$build_experimental_kernel_lib" != "no" && ( test "$build_experimental_kernel_lib" = "yes" || test "$build_bitcoin_chainstate" = "yes" )]) @@ -1785,8 +1738,6 @@ else fi fi -AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) - AC_MSG_CHECKING([whether to build test_navcoin]) if test "$use_tests" = "yes"; then if test "$enable_fuzz" = "yes"; then @@ -1807,7 +1758,7 @@ else AC_MSG_RESULT([no]) fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoin_libs$build_bitcoind$enable_fuzz_binary$use_bench$use_tests" = "nononononononononono"; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoin_libs$build_navcoind$enable_fuzz_binary$use_bench$use_tests$enable_build_libblsct_only" = "nononononononononono"; then AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --enable-fuzz(-binary) --enable-bench or --enable-tests]) fi @@ -1858,13 +1809,13 @@ AC_SUBST(COPYRIGHT_YEAR, _COPYRIGHT_YEAR) AC_SUBST(COPYRIGHT_HOLDERS, "_COPYRIGHT_HOLDERS") AC_SUBST(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION") AC_SUBST(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL") -AC_SUBST(BITCOIN_DAEMON_NAME) -AC_SUBST(BITCOIN_TEST_NAME) -AC_SUBST(BITCOIN_CLI_NAME) -AC_SUBST(BITCOIN_TX_NAME) -AC_SUBST(BITCOIN_UTIL_NAME) -AC_SUBST(BITCOIN_CHAINSTATE_NAME) -AC_SUBST(BITCOIN_WALLET_TOOL_NAME) +AC_SUBST(NAVCOIN_DAEMON_NAME) +AC_SUBST(NAVCOIN_TEST_NAME) +AC_SUBST(NAVCOIN_CLI_NAME) +AC_SUBST(NAVCOIN_TX_NAME) +AC_SUBST(NAVCOIN_UTIL_NAME) +AC_SUBST(NAVCOIN_CHAINSTATE_NAME) +AC_SUBST(NAVCOIN_WALLET_TOOL_NAME) AC_SUBST(BITCOIN_MP_NODE_NAME) AC_SUBST(RELDFLAGS) @@ -1881,8 +1832,6 @@ AC_SUBST(GPROF_LDFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) -AC_SUBST(LTO_CXXFLAGS) -AC_SUBST(LTO_LDFLAGS) AC_SUBST(PIC_FLAGS) AC_SUBST(PIE_FLAGS) AC_SUBST(SANITIZER_CXXFLAGS) @@ -1946,11 +1895,21 @@ case ${OS} in ;; esac +dnl An old hack similar to a98356fee to remove hard-coded +dnl bind_at_load flag from libtool +case $host in + *darwin*) + AC_MSG_RESULT([Removing -Wl,bind_at_load from libtool.]) + sed < libtool > libtool-2 '/bind_at_load/d' + mv libtool-2 libtool + chmod 755 libtool + ;; +esac + echo echo "Options used to compile and link:" echo " external signer = $use_external_signer" echo " multiprocess = $build_multiprocess" -echo " with experimental syscall sandbox support = $use_syscall_sandbox" echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" if test "$enable_wallet" != "no"; then @@ -1973,7 +1932,6 @@ echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" -echo " LTO = $enable_lto" echo " libblsct only = $enable_build_libblsct_only" echo echo " target os = $host_os" @@ -1983,8 +1941,8 @@ echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CORE_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $CORE_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CORE_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $CORE_LDFLAGS $LDFLAGS" echo " AR = $AR" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/README.md b/contrib/README.md index 3c6e978061169..1de8a867ad020 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -13,7 +13,7 @@ Construct a linear, no-fork, best version of the blockchain. ### [Qos](/contrib/qos) ### -A Linux bash script that will set up traffic control (tc) to limit the outgoing bandwidth for connections to the Bitcoin network. This means one can have an always-on bitcoind instance running, and another local bitcoind/bitcoin-qt instance which connects to this node and receives blocks from it. +A Linux bash script that will set up traffic control (tc) to limit the outgoing bandwidth for connections to the Bitcoin network. This means one can have an always-on navcoind instance running, and another local navcoind/bitcoin-qt instance which connects to this node and receives blocks from it. ### [Seeds](/contrib/seeds) ### Utility to generate the pnSeed[] array that is compiled into the client. @@ -35,7 +35,7 @@ Test and Verify Tools ### [TestGen](/contrib/testgen) ### Utilities to generate test vectors for the data-driven Bitcoin tests. -### [Verify Binaries](/contrib/verifybinaries) ### +### [Verify-Binaries](/contrib/verify-binaries) ### This script attempts to download and verify the signature file SHA256SUMS.asc from bitcoin.org. Command Line Tools diff --git a/contrib/completions/bash/bitcoin-cli.bash-completion b/contrib/completions/bash/bitcoin-cli.bash similarity index 100% rename from contrib/completions/bash/bitcoin-cli.bash-completion rename to contrib/completions/bash/bitcoin-cli.bash diff --git a/contrib/completions/bash/bitcoin-tx.bash-completion b/contrib/completions/bash/bitcoin-tx.bash similarity index 100% rename from contrib/completions/bash/bitcoin-tx.bash-completion rename to contrib/completions/bash/bitcoin-tx.bash diff --git a/contrib/completions/bash/bitcoind.bash-completion b/contrib/completions/bash/navcoind.bash similarity index 81% rename from contrib/completions/bash/bitcoind.bash-completion rename to contrib/completions/bash/navcoind.bash index c11d99ef3169c..81cd5b2b5ea6e 100644 --- a/contrib/completions/bash/bitcoind.bash-completion +++ b/contrib/completions/bash/navcoind.bash @@ -1,15 +1,15 @@ -# bash programmable completion for bitcoind(1) and bitcoin-qt(1) +# bash programmable completion for navcoind(1) and bitcoin-qt(1) # Copyright (c) 2012-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -_bitcoind() { +_navcoind() { local cur prev words=() cword - local bitcoind + local navcoind - # save and use original argument to invoke bitcoind for -help + # save and use original argument to invoke navcoind for -help # it might not be in $PATH - bitcoind="$1" + navcoind="$1" COMPREPLY=() _get_comp_words_by_ref -n = cur prev words cword @@ -33,7 +33,7 @@ _bitcoind() { # only parse -help if sensible if [[ -z "$cur" || "$cur" =~ ^- ]]; then local helpopts - helpopts=$($bitcoind -help 2>&1 | awk '$1 ~ /^-/ { sub(/=.*/, "="); print $1 }' ) + helpopts=$($navcoind -help 2>&1 | awk '$1 ~ /^-/ { sub(/=.*/, "="); print $1 }' ) COMPREPLY=( $( compgen -W "$helpopts" -- "$cur" ) ) fi @@ -45,7 +45,7 @@ _bitcoind() { ;; esac } && -complete -F _bitcoind bitcoind bitcoin-qt +complete -F _navcoind navcoind bitcoin-qt # Local variables: # mode: shell-script diff --git a/contrib/completions/fish/bitcoin-qt.fish b/contrib/completions/fish/bitcoin-qt.fish deleted file mode 100644 index 15a355ae88f5d..0000000000000 --- a/contrib/completions/fish/bitcoin-qt.fish +++ /dev/null @@ -1,35 +0,0 @@ -# Disable files from being included in completions by default -complete --command bitcoin-qt --no-files - -# Extract options -function __fish_bitcoinqt_get_options - argparse 'nofiles' -- $argv - set --local cmd (commandline -opc)[1] - set --local options - - if set -q _flag_nofiles - set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match --invert -r '^.*=$') - else - set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match -r '^.*=$') - end - - for option in $options - echo $option - end -end - - -# Add options with file completion -complete \ - --command bitcoin-qt \ - --arguments "(__fish_bitcoinqt_get_options)" -# Enable file completions only if the commandline now contains a `*.=` style option -complete -c bitcoin-qt \ - --condition 'string match --regex -- ".*=" (commandline -pt)' \ - --force-files - -# Add options without file completion -complete \ - --command bitcoin-qt \ - --arguments "(__fish_bitcoinqt_get_options --nofiles)" - diff --git a/contrib/completions/fish/bitcoind.fish b/contrib/completions/fish/navcoind.fish similarity index 77% rename from contrib/completions/fish/bitcoind.fish rename to contrib/completions/fish/navcoind.fish index fa245ae17f470..8cfa52f19a516 100644 --- a/contrib/completions/fish/bitcoind.fish +++ b/contrib/completions/fish/navcoind.fish @@ -1,8 +1,8 @@ # Disable files from being included in completions by default -complete --command bitcoind --no-files +complete --command navcoind --no-files # Extract options -function __fish_bitcoind_get_options +function __fish_navcoind_get_options argparse 'nofiles' -- $argv set --local cmd (commandline -opc)[1] set --local options @@ -21,15 +21,15 @@ end # Add options with file completion complete \ - --command bitcoind \ - --arguments "(__fish_bitcoind_get_options)" + --command navcoind \ + --arguments "(__fish_navcoind_get_options)" # Enable file completions only if the commandline now contains a `*.=` style option -complete --command bitcoind \ +complete --command navcoind \ --condition 'string match --regex -- ".*=" (commandline -pt)' \ --force-files # Add options without file completion complete \ - --command bitcoind \ - --arguments "(__fish_bitcoind_get_options --nofiles)" + --command navcoind \ + --arguments "(__fish_navcoind_get_options --nofiles)" diff --git a/contrib/debian/copyright b/contrib/debian/copyright index ca430170a1d6e..dafc92f8ad72d 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,7 +5,7 @@ Upstream-Contact: Satoshi Nakamoto Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2023, Bitcoin Core Developers +Copyright: 2009-2024, Bitcoin Core Developers License: Expat Comment: The Bitcoin Core Developers encompasses all contributors to the project, listed in the release notes or the git log. diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 76aae5c3f5905..e99603f3d57f8 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -83,14 +83,24 @@ A small script to automatically create manpages in ../../doc/man by running the This requires help2man which can be found at: https://www.gnu.org/software/help2man/ With in-tree builds this tool can be run from any directory within the -repostitory. To use this tool with out-of-tree builds set `BUILDDIR`. For +repository. To use this tool with out-of-tree builds set `BUILDDIR`. For example: ```bash BUILDDIR=$PWD/build contrib/devtools/gen-manpages.py ``` -gen-bitcoin-conf.sh +headerssync-params.py +===================== + +A script to generate optimal parameters for the headerssync module (src/headerssync.cpp). It takes no command-line +options, as all its configuration is set at the top of the file. It runs many times faster inside PyPy. Invocation: + +```bash +pypy3 contrib/devtools/headerssync-params.py +``` + +gen-navcoin-conf.sh =================== Generates a navcoin.conf file in `share/examples/` by parsing the output from `navcoind --help`. This script is run during the @@ -102,7 +112,7 @@ repository. To use this tool with out-of-tree builds set `BUILDDIR`. For example: ```bash -BUILDDIR=$PWD/build contrib/devtools/gen-bitcoin-conf.sh +BUILDDIR=$PWD/build contrib/devtools/gen-navcoin-conf.sh ``` security-check.py and test-security-check.py diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt new file mode 100644 index 0000000000000..35e60d1d87e2b --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -0,0 +1,56 @@ +cmake_minimum_required(VERSION 3.9) + +project(bitcoin-tidy VERSION 1.0.0 DESCRIPTION "clang-tidy checks for Bitcoin Core") + +include(GNUInstallDirs) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_CXX_EXTENSIONS False) + +# TODO: Figure out how to avoid the terminfo check +find_package(LLVM REQUIRED CONFIG) +find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy" HINTS ${LLVM_TOOLS_BINARY_DIR}) +message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") +message(STATUS "Found clang-tidy: ${CLANG_TIDY_EXE}") + +add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp logprintf.cpp) +target_include_directories(bitcoin-tidy SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) + +# Disable RTTI and exceptions as necessary +if (MSVC) + target_compile_options(bitcoin-tidy PRIVATE /GR-) +else() + target_compile_options(bitcoin-tidy PRIVATE -fno-rtti) + target_compile_options(bitcoin-tidy PRIVATE -fno-exceptions) +endif() + +if(CMAKE_HOST_APPLE) + # ld64 expects no undefined symbols by default + target_link_options(bitcoin-tidy PRIVATE -Wl,-flat_namespace) + target_link_options(bitcoin-tidy PRIVATE -Wl,-undefined -Wl,suppress) +endif() + +# Add warnings +if (MSVC) + target_compile_options(bitcoin-tidy PRIVATE /W4) +else() + target_compile_options(bitcoin-tidy PRIVATE -Wall) + target_compile_options(bitcoin-tidy PRIVATE -Wextra) +endif() + +if(CMAKE_VERSION VERSION_LESS 3.27) + set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=${CMAKE_BINARY_DIR}/${CMAKE_SHARED_MODULE_PREFIX}bitcoin-tidy${CMAKE_SHARED_MODULE_SUFFIX}" "-checks=-*,bitcoin-*") +else() + # CLANG_TIDY_COMMAND supports generator expressions as of 3.27 + set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=$" "-checks=-*,bitcoin-*") +endif() + +# Create a dummy library that runs clang-tidy tests as a side-effect of building +add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_logprintf.cpp) +add_dependencies(bitcoin-tidy-tests bitcoin-tidy) + +set_target_properties(bitcoin-tidy-tests PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_COMMAND}") + + +install(TARGETS bitcoin-tidy LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/contrib/devtools/bitcoin-tidy/README b/contrib/devtools/bitcoin-tidy/README new file mode 100644 index 0000000000000..c15e07c4ede54 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/README @@ -0,0 +1,11 @@ +# Bitcoin Tidy + +Example Usage: + +```bash +cmake -S . -B build -DLLVM_DIR=$(llvm-config --cmakedir) -DCMAKE_BUILD_TYPE=Release + +cmake --build build -j$(nproc) + +cmake --build build --target bitcoin-tidy-tests -j$(nproc) +``` diff --git a/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp new file mode 100644 index 0000000000000..0f34d37793877 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "logprintf.h" + +#include +#include + +class BitcoinModule final : public clang::tidy::ClangTidyModule +{ +public: + void addCheckFactories(clang::tidy::ClangTidyCheckFactories& CheckFactories) override + { + CheckFactories.registerCheck("bitcoin-unterminated-logprintf"); + } +}; + +static clang::tidy::ClangTidyModuleRegistry::Add + X("bitcoin-module", "Adds bitcoin checks."); + +volatile int BitcoinModuleAnchorSource = 0; diff --git a/contrib/devtools/bitcoin-tidy/example_logprintf.cpp b/contrib/devtools/bitcoin-tidy/example_logprintf.cpp new file mode 100644 index 0000000000000..a12a666c086d6 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/example_logprintf.cpp @@ -0,0 +1,108 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +// Test for bitcoin-unterminated-logprintf + +enum LogFlags { + NONE +}; + +enum Level { + None +}; + +template +static inline void LogPrintf_(const std::string& logging_function, const std::string& source_file, const int source_line, const LogFlags flag, const Level level, const char* fmt, const Args&... args) +{ +} + +#define LogPrintLevel_(category, level, ...) LogPrintf_(__func__, __FILE__, __LINE__, category, level, __VA_ARGS__) +#define LogPrintf(...) LogPrintLevel_(LogFlags::NONE, Level::None, __VA_ARGS__) + +#define LogPrint(category, ...) \ + do { \ + LogPrintf(__VA_ARGS__); \ + } while (0) + + +class CWallet +{ + std::string GetDisplayName() const + { + return "default wallet"; + } + +public: + template + void WalletLogPrintf(const char* fmt, Params... parameters) const + { + LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); + }; +}; + +struct ScriptPubKeyMan +{ + std::string GetDisplayName() const + { + return "default wallet"; + } + + template + void WalletLogPrintf(const char* fmt, Params... parameters) const + { + LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); + }; +}; + +void good_func() +{ + LogPrintf("hello world!\n"); +} +void good_func2() +{ + CWallet wallet; + wallet.WalletLogPrintf("hi\n"); + ScriptPubKeyMan spkm; + spkm.WalletLogPrintf("hi\n"); + + const CWallet& walletref = wallet; + walletref.WalletLogPrintf("hi\n"); + + auto* walletptr = new CWallet(); + walletptr->WalletLogPrintf("hi\n"); + delete walletptr; +} +void bad_func() +{ + LogPrintf("hello world!"); +} +void bad_func2() +{ + LogPrintf(""); +} +void bad_func3() +{ + // Ending in "..." has no special meaning. + LogPrintf("hello world!..."); +} +void bad_func4_ignored() +{ + LogPrintf("hello world!"); // NOLINT(bitcoin-unterminated-logprintf) +} +void bad_func5() +{ + CWallet wallet; + wallet.WalletLogPrintf("hi"); + ScriptPubKeyMan spkm; + spkm.WalletLogPrintf("hi"); + + const CWallet& walletref = wallet; + walletref.WalletLogPrintf("hi"); + + auto* walletptr = new CWallet(); + walletptr->WalletLogPrintf("hi"); + delete walletptr; +} diff --git a/contrib/devtools/bitcoin-tidy/logprintf.cpp b/contrib/devtools/bitcoin-tidy/logprintf.cpp new file mode 100644 index 0000000000000..36beac28c86c3 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/logprintf.cpp @@ -0,0 +1,60 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "logprintf.h" + +#include +#include + + +namespace { +AST_MATCHER(clang::StringLiteral, unterminated) +{ + size_t len = Node.getLength(); + if (len > 0 && Node.getCodeUnit(len - 1) == '\n') { + return false; + } + return true; +} +} // namespace + +namespace bitcoin { + +void LogPrintfCheck::registerMatchers(clang::ast_matchers::MatchFinder* finder) +{ + using namespace clang::ast_matchers; + + /* + Logprintf(..., ..., ..., ..., ..., "foo", ...) + */ + + finder->addMatcher( + callExpr( + callee(functionDecl(hasName("LogPrintf_"))), + hasArgument(5, stringLiteral(unterminated()).bind("logstring"))), + this); + + /* + auto walletptr = &wallet; + wallet.WalletLogPrintf("foo"); + wallet->WalletLogPrintf("foo"); + */ + finder->addMatcher( + cxxMemberCallExpr( + callee(cxxMethodDecl(hasName("WalletLogPrintf"))), + hasArgument(0, stringLiteral(unterminated()).bind("logstring"))), + this); +} + +void LogPrintfCheck::check(const clang::ast_matchers::MatchFinder::MatchResult& Result) +{ + if (const clang::StringLiteral* lit = Result.Nodes.getNodeAs("logstring")) { + const clang::ASTContext& ctx = *Result.Context; + const auto user_diag = diag(lit->getEndLoc(), "Unterminated format string used with LogPrintf"); + const auto& loc = lit->getLocationOfByte(lit->getByteLength(), *Result.SourceManager, ctx.getLangOpts(), ctx.getTargetInfo()); + user_diag << clang::FixItHint::CreateInsertion(loc, "\\n"); + } +} + +} // namespace bitcoin diff --git a/contrib/devtools/bitcoin-tidy/logprintf.h b/contrib/devtools/bitcoin-tidy/logprintf.h new file mode 100644 index 0000000000000..db95dfe143e84 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/logprintf.h @@ -0,0 +1,29 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef LOGPRINTF_CHECK_H +#define LOGPRINTF_CHECK_H + +#include + +namespace bitcoin { + +// Warn about any use of LogPrintf that does not end with a newline. +class LogPrintfCheck final : public clang::tidy::ClangTidyCheck +{ +public: + LogPrintfCheck(clang::StringRef Name, clang::tidy::ClangTidyContext* Context) + : clang::tidy::ClangTidyCheck(Name, Context) {} + + bool isLanguageVersionSupported(const clang::LangOptions& LangOpts) const override + { + return LangOpts.CPlusPlus; + } + void registerMatchers(clang::ast_matchers::MatchFinder* Finder) override; + void check(const clang::ast_matchers::MatchFinder::MatchResult& Result) override; +}; + +} // namespace bitcoin + +#endif // LOGPRINTF_CHECK_H diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index b1d9f2b7db23b..b742a8cea6718 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -5,7 +5,6 @@ import sys import re -from typing import Dict, List, Set MAPPING = { 'core_read.cpp': 'core_io.cpp', @@ -33,7 +32,7 @@ def module_name(path): return None files = dict() -deps: Dict[str, Set[str]] = dict() +deps: dict[str, set[str]] = dict() RE = re.compile("^#include <(.*)>") @@ -65,7 +64,7 @@ def module_name(path): shortest_cycle = None for module in sorted(deps.keys()): # Build the transitive closure of dependencies of module - closure: Dict[str, List[str]] = dict() + closure: dict[str, list[str]] = dict() for dep in deps[module]: closure[dep] = [] while True: diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 420bf7ff33002..e2b661d65d235 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -1,166 +1,190 @@ #!/usr/bin/env python3 # -#===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===# +# ===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===# # -# The LLVM Compiler Infrastructure +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -# This file is distributed under the University of Illinois Open Source -# License. -# -# ============================================================ -# -# University of Illinois/NCSA -# Open Source License -# -# Copyright (c) 2007-2015 University of Illinois at Urbana-Champaign. -# All rights reserved. -# -# Developed by: -# -# LLVM Team -# -# University of Illinois at Urbana-Champaign -# -# http://llvm.org -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal with -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimers. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimers in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the names of the LLVM Team, University of Illinois at -# Urbana-Champaign, nor the names of its contributors may be used to -# endorse or promote products derived from this Software without specific -# prior written permission. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -# SOFTWARE. -# -# ============================================================ -# -#===------------------------------------------------------------------------===# - -r""" -ClangFormat Diff Reformatter -============================ +# ===------------------------------------------------------------------------===# +""" This script reads input from a unified diff and reformats all the changed lines. This is useful to reformat all the lines touched by a specific patch. Example usage for git/svn users: - git diff -U0 HEAD^ | clang-format-diff.py -p1 -i - svn diff --diff-cmd=diff -x-U0 | clang-format-diff.py -i + git diff -U0 --no-color --relative HEAD^ | {clang_format_diff} -p1 -i + svn diff --diff-cmd=diff -x-U0 | {clang_format_diff} -i +It should be noted that the filename contained in the diff is used unmodified +to determine the source file to update. Users calling this script directly +should be careful to ensure that the path in the diff is correct relative to the +current working directory. """ +from __future__ import absolute_import, division, print_function import argparse import difflib -import io import re import subprocess import sys - -# Change this to the full path if clang-format is not on the path. -binary = 'clang-format' +from io import StringIO def main(): - parser = argparse.ArgumentParser(description= - 'Reformat changed lines in diff. Without -i ' - 'option just output the diff that would be ' - 'introduced.') - parser.add_argument('-i', action='store_true', default=False, - help='apply edits to files instead of displaying a diff') - parser.add_argument('-p', metavar='NUM', default=0, - help='strip the smallest prefix containing P slashes') - parser.add_argument('-regex', metavar='PATTERN', default=None, - help='custom pattern selecting file paths to reformat ' - '(case sensitive, overrides -iregex)') - parser.add_argument('-iregex', metavar='PATTERN', default= - r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc|js|ts|proto' - r'|protodevel|java)', - help='custom pattern selecting file paths to reformat ' - '(case insensitive, overridden by -regex)') - parser.add_argument('-sort-includes', action='store_true', default=False, - help='let clang-format sort include blocks') - parser.add_argument('-v', '--verbose', action='store_true', - help='be more verbose, ineffective without -i') - args = parser.parse_args() - - # Extract changed lines for each file. - filename = None - lines_by_file = {} - for line in sys.stdin: - match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line) - if match: - filename = match.group(2) - if filename is None: - continue - - if args.regex is not None: - if not re.match('^%s$' % args.regex, filename): - continue - else: - if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE): - continue - - match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line) - if match: - start_line = int(match.group(1)) - line_count = 1 - if match.group(3): - line_count = int(match.group(3)) - if line_count == 0: - continue - end_line = start_line + line_count - 1 - lines_by_file.setdefault(filename, []).extend( - ['-lines', str(start_line) + ':' + str(end_line)]) - - # Reformat files containing changes in place. - for filename, lines in lines_by_file.items(): - if args.i and args.verbose: - print('Formatting {}'.format(filename)) - command = [binary, filename] - if args.i: - command.append('-i') - if args.sort_includes: - command.append('-sort-includes') - command.extend(lines) - command.extend(['-style=file', '-fallback-style=none']) - p = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=None, - stdin=subprocess.PIPE, - text=True) - stdout, stderr = p.communicate() - if p.returncode != 0: - sys.exit(p.returncode) - - if not args.i: - with open(filename, encoding="utf8") as f: - code = f.readlines() - formatted_code = io.StringIO(stdout).readlines() - diff = difflib.unified_diff(code, formatted_code, - filename, filename, - '(before formatting)', '(after formatting)') - diff_string = ''.join(diff) - if len(diff_string) > 0: - sys.stdout.write(diff_string) - -if __name__ == '__main__': - main() + parser = argparse.ArgumentParser( + description=__doc__.format(clang_format_diff="%(prog)s"), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "-i", + action="store_true", + default=False, + help="apply edits to files instead of displaying a diff", + ) + parser.add_argument( + "-p", + metavar="NUM", + default=0, + help="strip the smallest prefix containing P slashes", + ) + parser.add_argument( + "-regex", + metavar="PATTERN", + default=None, + help="custom pattern selecting file paths to reformat " + "(case sensitive, overrides -iregex)", + ) + parser.add_argument( + "-iregex", + metavar="PATTERN", + default=r".*\.(?:cpp|cc|c\+\+|cxx|cppm|ccm|cxxm|c\+\+m|c|cl|h|hh|hpp" + r"|hxx|m|mm|inc|js|ts|proto|protodevel|java|cs|json|s?vh?)", + help="custom pattern selecting file paths to reformat " + "(case insensitive, overridden by -regex)", + ) + parser.add_argument( + "-sort-includes", + action="store_true", + default=False, + help="let clang-format sort include blocks", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="be more verbose, ineffective without -i", + ) + parser.add_argument( + "-style", + help="formatting style to apply (LLVM, GNU, Google, Chromium, " + "Microsoft, Mozilla, WebKit)", + ) + parser.add_argument( + "-fallback-style", + help="The name of the predefined style used as a" + "fallback in case clang-format is invoked with" + "-style=file, but can not find the .clang-format" + "file to use.", + ) + parser.add_argument( + "-binary", + default="clang-format", + help="location of binary to use for clang-format", + ) + args = parser.parse_args() + + # Extract changed lines for each file. + filename = None + lines_by_file = {} + for line in sys.stdin: + match = re.search(r"^\+\+\+\ (.*?/){%s}(\S*)" % args.p, line) + if match: + filename = match.group(2) + if filename is None: + continue + + if args.regex is not None: + if not re.match("^%s$" % args.regex, filename): + continue + else: + if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE): + continue + + match = re.search(r"^@@.*\+(\d+)(?:,(\d+))?", line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(2): + line_count = int(match.group(2)) + # The input is something like + # + # @@ -1, +0,0 @@ + # + # which means no lines were added. + if line_count == 0: + continue + # Also format lines range if line_count is 0 in case of deleting + # surrounding statements. + end_line = start_line + if line_count != 0: + end_line += line_count - 1 + lines_by_file.setdefault(filename, []).extend( + ["-lines", str(start_line) + ":" + str(end_line)] + ) + + # Reformat files containing changes in place. + for filename, lines in lines_by_file.items(): + if args.i and args.verbose: + print("Formatting {}".format(filename)) + command = [args.binary, filename] + if args.i: + command.append("-i") + if args.sort_includes: + command.append("-sort-includes") + command.extend(lines) + if args.style: + command.extend(["-style", args.style]) + if args.fallback_style: + command.extend(["-fallback-style", args.fallback_style]) + + try: + p = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=None, + stdin=subprocess.PIPE, + universal_newlines=True, + ) + except OSError as e: + # Give the user more context when clang-format isn't + # found/isn't executable, etc. + raise RuntimeError( + 'Failed to run "%s" - %s"' % (" ".join(command), e.strerror) + ) + + stdout, stderr = p.communicate() + if p.returncode != 0: + sys.exit(p.returncode) + + if not args.i: + with open(filename, encoding="utf8") as f: + code = f.readlines() + formatted_code = StringIO(stdout).readlines() + diff = difflib.unified_diff( + code, + formatted_code, + filename, + filename, + "(before formatting)", + "(after formatting)", + ) + diff_string = "".join(diff) + if len(diff_string) > 0: + sys.stdout.write(diff_string) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py index 2860e7db99441..658cec988636f 100755 --- a/contrib/devtools/gen-manpages.py +++ b/contrib/devtools/gen-manpages.py @@ -8,7 +8,7 @@ import tempfile BINARIES = [ -'src/bitcoind', +'src/navcoind', 'src/bitcoin-cli', 'src/bitcoin-tx', 'src/bitcoin-wallet', diff --git a/contrib/devtools/gen-bitcoin-conf.sh b/contrib/devtools/gen-navcoin-conf.sh similarity index 86% rename from contrib/devtools/gen-bitcoin-conf.sh rename to contrib/devtools/gen-navcoin-conf.sh index 6923e823b12c2..1a2a447124fa4 100755 --- a/contrib/devtools/gen-bitcoin-conf.sh +++ b/contrib/devtools/gen-navcoin-conf.sh @@ -7,9 +7,9 @@ export LC_ALL=C TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} BUILDDIR=${BUILDDIR:-$TOPDIR} BINDIR=${BINDIR:-$BUILDDIR/src} -BITCOIND=${BITCOIND:-$BINDIR/bitcoind} +BITCOIND=${BITCOIND:-$BINDIR/navcoind} SHARE_EXAMPLES_DIR=${SHARE_EXAMPLES_DIR:-$TOPDIR/share/examples} -EXAMPLE_CONF_FILE=${EXAMPLE_CONF_FILE:-$SHARE_EXAMPLES_DIR/bitcoin.conf} +EXAMPLE_CONF_FILE=${EXAMPLE_CONF_FILE:-$SHARE_EXAMPLES_DIR/navcoin.conf} [ ! -x "$BITCOIND" ] && echo "$BITCOIND not found or not executable." && exit 1 @@ -22,10 +22,10 @@ fi if [ -n "$DIRTY" ] then echo -e "WARNING: $BITCOIND was built from a dirty tree.\n" - echo -e "To safely generate a bitcoin.conf file, please commit your changes to $BITCOIND, rebuild, then run this script again.\n" + echo -e "To safely generate a navcoin.conf file, please commit your changes to $BITCOIND, rebuild, then run this script again.\n" fi -echo 'Generating example bitcoin.conf file in share/examples/' +echo 'Generating example navcoin.conf file in share/examples/' # create the directory, if it doesn't exist mkdir -p "${SHARE_EXAMPLES_DIR}" @@ -33,8 +33,8 @@ mkdir -p "${SHARE_EXAMPLES_DIR}" # create the header text cat > "${EXAMPLE_CONF_FILE}" << 'EOF' ## -## bitcoin.conf configuration file. -## Generated by contrib/devtools/gen-bitcoin-conf.sh. +## navcoin.conf configuration file. +## Generated by contrib/devtools/gen-navcoin-conf.sh. ## ## Lines beginning with # are comments. ## All possible configuration options are provided. To use, copy this file @@ -46,7 +46,7 @@ cat > "${EXAMPLE_CONF_FILE}" << 'EOF' ### Options EOF -# parse the output from bitcoind --help +# parse the output from navcoind --help # adding newlines is a bit funky to ensure portability for BSD # see here for more details: https://stackoverflow.com/a/24575385 ${BITCOIND} --help \ diff --git a/contrib/devtools/headerssync-params.py b/contrib/devtools/headerssync-params.py new file mode 100644 index 0000000000000..0198f5db99f8b --- /dev/null +++ b/contrib/devtools/headerssync-params.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Script to find the optimal parameters for the headerssync module through simulation.""" + +from math import log, exp, sqrt +from datetime import datetime, timedelta +import random + +# Parameters: + +# Aim for still working fine at some point in the future. [datetime] +TIME = datetime(2026, 10, 5) + +# Expected block interval. [timedelta] +BLOCK_INTERVAL = timedelta(seconds=600) + +# The number of headers corresponding to the minchainwork parameter. [headers] +MINCHAINWORK_HEADERS = 804000 + +# Combined processing bandwidth from all attackers to one victim. [bit/s] +# 6 Gbit/s is approximately the speed at which a single thread of a Ryzen 5950X CPU thread can hash +# headers. In practice, the victim's network bandwidth and network processing overheads probably +# impose a far lower number, but it's a useful upper bound. +ATTACK_BANDWIDTH = 6000000000 + +# How much additional permanent memory usage are attackers (jointly) allowed to cause in the victim, +# expressed as fraction of the normal memory usage due to mainchain growth, for the duration the +# attack is sustained. [unitless] +# 0.2 means that attackers, while they keep up the attack, can cause permanent memory usage due to +# headers storage to grow at 1.2 header per BLOCK_INTERVAL. +ATTACK_FRACTION = 0.2 + +# When this is set, the mapping from period size to memory usage (at optimal buffer size for that +# period) is assumed to be convex. This greatly speeds up the computation, and does not appear +# to influence the outcome. Set to False for a stronger guarantee to get the optimal result. +ASSUME_CONVEX = True + +# Explanation: +# +# The headerssync module implements a DoS protection against low-difficulty header spam which does +# not rely on checkpoints. In short it works as follows: +# +# - (initial) header synchronization is split into two phases: +# - A commitment phase, in which headers are downloaded from the peer, and a very compact +# commitment to them is remembered in per-peer memory. The commitment phase ends when the +# received chain's combined work reaches a predetermined threshold. +# - A redownload phase, during which the headers are downloaded a second time from the same peer, +# and compared against the commitment constructed in the first phase. If there is a match, the +# redownloaded headers are fed to validation and accepted into permanent storage. +# +# This separation guarantees that no headers are accepted into permanent storage without +# requiring the peer to first prove the chain actually has sufficient work. +# +# - To actually implement this commitment mechanism, the following approach is used: +# - Keep a *1 bit* commitment (constructed using a salted hash function), for every block whose +# height is a multiple of {period} plus an offset value. If RANDOMIZE_OFFSET, the offset, +# like the salt, is chosen randomly when the synchronization starts and kept fixed afterwards. +# - When redownloading, headers are fed through a per-peer queue that holds {bufsize} headers, +# before passing them to validation. All the headers in this queue are verified against the +# commitment bits created in the first phase before any header is released from it. This means +# {bufsize/period} bits are checked "on top of" each header before actually processing it, +# which results in a commitment structure with roughly {bufsize/period} bits of security, as +# once a header is modified, due to the prevhash inclusion, all future headers necessarily +# change as well. +# +# The question is what these {period} and {bufsize} parameters need to be set to. This program +# exhaustively tests a range of values to find the optimal choice, taking into account: +# +# - Minimizing the (maximum of) two scenarios that trigger per-peer memory usage: +# +# - When downloading a (likely honest) chain that reaches the chainwork threshold after {n} +# blocks, and then redownloads them, we will consume per-peer memory that is sufficient to +# store {n/period} commitment bits and {bufsize} headers. We only consider attackers without +# sufficient hashpower (as otherwise they are from a PoW perspective not attackers), which +# means {n} is restricted to the honest chain's length before reaching minchainwork. +# +# - When downloading a (likely false) chain of {n} headers that never reaches the chainwork +# threshold, we will consume per-peer memory that is sufficient to store {n/period} +# commitment bits. Such a chain may be very long, by exploiting the timewarp bug to avoid +# ramping up difficulty. There is however an absolute limit on how long such a chain can be: 6 +# blocks per second since genesis, due to the increasing MTP consensus rule. +# +# - Not gratuitously preventing synchronizing any valid chain, however difficult such a chain may +# be to construct. In particular, the above scenario with an enormous timewarp-expoiting chain +# cannot simply be ignored, as it is legal that the honest main chain is like that. We however +# do not bother minimizing the memory usage in that case (because a billion-header long honest +# chain will inevitably use far larger amounts of memory than designed for). +# +# - Keep the rate at which attackers can get low-difficulty headers accepted to the block index +# negligible. Specifically, the possibility exists for an attacker to send the honest main +# chain's headers during the commitment phase, but then start deviating at an attacker-chosen +# point by sending novel low-difficulty headers instead. Depending on how high we set the +# {bufsize/period} ratio, we can make the probability that such a header makes it in +# arbitrarily small, but at the cost of higher memory during the redownload phase. It turns out, +# some rate of memory usage growth is expected anyway due to chain growth, so permitting the +# attacker to increase that rate by a small factor isn't concerning. The attacker may start +# somewhat later than genesis, as long as the difficulty doesn't get too high. This reduces +# the attacker bandwidth required at the cost of higher PoW needed for constructing the +# alternate chain. This trade-off is ignored here, as it results in at most a small constant +# factor in attack rate. + + +# System properties: + +# Headers in the redownload buffer are stored without prevhash. [bits] +COMPACT_HEADER_SIZE = 48 * 8 + +# How many bits a header uses in P2P protocol. [bits] +NET_HEADER_SIZE = 81 * 8 + +# How many headers are sent at once. [headers] +HEADER_BATCH_COUNT = 2000 + +# Whether or not the offset of which blocks heights get checksummed is randomized. +RANDOMIZE_OFFSET = True + +# Timestamp of the genesis block +GENESIS_TIME = datetime(2009, 1, 3) + +# Derived values: + +# What rate of headers worth of RAM attackers are allowed to cause in the victim. [headers/s] +LIMIT_HEADERRATE = ATTACK_FRACTION / BLOCK_INTERVAL.total_seconds() + +# How many headers can attackers (jointly) send a victim per second. [headers/s] +NET_HEADERRATE = ATTACK_BANDWIDTH / NET_HEADER_SIZE + +# What fraction of headers sent by attackers can at most be accepted by a victim [unitless] +LIMIT_FRACTION = LIMIT_HEADERRATE / NET_HEADERRATE + +# How many headers we permit attackers to cause being accepted per attack. [headers/attack] +ATTACK_HEADERS = LIMIT_FRACTION * MINCHAINWORK_HEADERS + + +def find_max_headers(when): + """Compute the maximum number of headers a valid Bitcoin chain can have at given time.""" + # When exploiting the timewarp attack, this can be up to 6 per second since genesis. + return 6 * ((when - GENESIS_TIME) // timedelta(seconds=1)) + + +def lambert_w(value): + """Solve the equation x*exp(x)=value (x > 0, value > 0).""" + # Initial approximation. + approx = max(log(value), 0.0) + for _ in range(10): + # Newton-Rhapson iteration steps. + approx += (value * exp(-approx) - approx) / (approx + 1.0) + return approx + + +def attack_rate(period, bufsize, limit=None): + """Compute maximal accepted headers per attack in (period, bufsize) configuration. + + If limit is provided, the computation is stopped early when the result is known to exceed the + value in limit. + """ + + max_rate = None + max_honest = None + # Let the current batch 0 being received be the first one in which the attacker starts lying. + # They will only ever start doing so right after a commitment block, but where that is can be + # in a number of places. Let honest be the number of honest headers in this current batch, + # preceding the forged ones. + for honest in range(HEADER_BATCH_COUNT): + # The number of headers the attack under consideration will on average get accepted. + # This is the number being computed. + rate = 0 + + # Iterate over the possible alignments of commitments w.r.t. the first batch. In case + # the alignments are randomized, try all values. If not, the attacker can know/choose + # the alignment, and will always start forging right after a commitment. + if RANDOMIZE_OFFSET: + align_choices = list(range(period)) + else: + align_choices = [(honest - 1) % period] + # Now loop over those possible alignment values, computing the average attack rate + # over them by dividing each contribution by len(align_choices). + for align in align_choices: + # These state variables capture the situation after receiving the first batch. + # - The number of headers received after the last commitment for an honest block: + after_good_commit = HEADER_BATCH_COUNT - honest + ((honest - align - 1) % period) + # - The number of forged headers in the redownload buffer: + forged_in_buf = HEADER_BATCH_COUNT - honest + + # Now iterate over the next batches of headers received, adding contributions to the + # rate variable. + while True: + # Process the first HEADER_BATCH_COUNT headers in the buffer: + accept_forged_headers = max(forged_in_buf - bufsize, 0) + forged_in_buf -= accept_forged_headers + if accept_forged_headers: + # The probability the attack has not been detected yet at this point: + prob = 0.5 ** (after_good_commit // period) + # Update attack rate, divided by align_choices to average over the alignments. + rate += accept_forged_headers * prob / len(align_choices) + # If this means we exceed limit, bail out early (performance optimization). + if limit is not None and rate >= limit: + return rate, None + # If the maximal term being added is negligible compared to rate, stop + # iterating. + if HEADER_BATCH_COUNT * prob < 1.0e-16 * rate * len(align_choices): + break + # Update state from a new incoming batch (which is all forged) + after_good_commit += HEADER_BATCH_COUNT + forged_in_buf += HEADER_BATCH_COUNT + + if max_rate is None or rate > max_rate: + max_rate = rate + max_honest = honest + + return max_rate, max_honest + + +def memory_usage(period, bufsize, when): + """How much memory (max,mainchain,timewarp) does the (period,bufsize) configuration need?""" + + # Per-peer memory usage for a timewarp chain that never meets minchainwork + mem_timewarp = find_max_headers(when) // period + # Per-peer memory usage for being fed the main chain + mem_mainchain = (MINCHAINWORK_HEADERS // period) + bufsize * COMPACT_HEADER_SIZE + # Maximum per-peer memory usage + max_mem = max(mem_timewarp, mem_mainchain) + + return max_mem, mem_mainchain, mem_timewarp + +def find_bufsize(period, attack_headers, when, max_mem=None, min_bufsize=1): + """Determine how big bufsize needs to be given a specific period length. + + Given a period, find the smallest value of bufsize such that the attack rate against the + (period, bufsize) configuration is below attack_headers. If max_mem is provided, and no + such bufsize exists that needs less than max_mem bits of memory, None is returned. + min_bufsize is the minimal result to be considered.""" + + if max_mem is None: + succ_buf = min_bufsize - 1 + fail_buf = min_bufsize + # First double iteratively until an upper bound for failure is found. + while True: + if attack_rate(period, fail_buf, attack_headers)[0] < attack_headers: + break + succ_buf, fail_buf = fail_buf, 3 * fail_buf - 2 * succ_buf + else: + # If a long low-work header chain exists that exceeds max_mem already, give up. + if find_max_headers(when) // period > max_mem: + return None + # Otherwise, verify that the maximal buffer size that permits a mainchain sync with less + # than max_mem memory is sufficient to get the attack rate below attack_headers. If not, + # also give up. + max_buf = (max_mem - (MINCHAINWORK_HEADERS // period)) // COMPACT_HEADER_SIZE + if max_buf < min_bufsize: + return None + if attack_rate(period, max_buf, attack_headers)[0] >= attack_headers: + return None + # If it is sufficient, that's an upper bound to start our search. + succ_buf = min_bufsize - 1 + fail_buf = max_buf + + # Then perform a bisection search to narrow it down. + while fail_buf > succ_buf + 1: + try_buf = (succ_buf + fail_buf) // 2 + if attack_rate(period, try_buf, attack_headers)[0] >= attack_headers: + succ_buf = try_buf + else: + fail_buf = try_buf + return fail_buf + + +def optimize(when): + """Find the best (period, bufsize) configuration.""" + + # When period*bufsize = memory_scale, the per-peer memory for a mainchain sync and a maximally + # long low-difficulty header sync are equal. + memory_scale = (find_max_headers(when) - MINCHAINWORK_HEADERS) / COMPACT_HEADER_SIZE + # Compute approximation for {bufsize/period}, using a formula for a simplified problem. + approx_ratio = lambert_w(log(4) * memory_scale / ATTACK_HEADERS**2) / log(4) + # Use those for a first attempt. + print("Searching configurations:") + period = int(sqrt(memory_scale / approx_ratio) + 0.5) + bufsize = find_bufsize(period, ATTACK_HEADERS, when) + mem = memory_usage(period, bufsize, when) + best = (period, bufsize, mem) + maps = [(period, bufsize), (MINCHAINWORK_HEADERS + 1, None)] + print(f"- Initial: period={period}, buffer={bufsize}, mem={mem[0] / 8192:.3f} KiB") + + # Consider all period values between 1 and MINCHAINWORK_HEADERS, except the one just tried. + periods = [iv for iv in range(1, MINCHAINWORK_HEADERS + 1) if iv != period] + # Iterate, picking a random element from periods, computing its corresponding bufsize, and + # then using the result to shrink the period. + while True: + # Remove all periods whose memory usage for low-work long chain sync exceed the best + # memory usage we've found so far. + periods = [p for p in periods if find_max_headers(when) // p < best[2][0]] + # Stop if there is nothing left to try. + if len(periods) == 0: + break + # Pick a random remaining option for period size, and compute corresponding bufsize. + period = periods.pop(random.randrange(len(periods))) + # The buffer size (at a given attack level) cannot shrink as the period grows. Find the + # largest period smaller than the selected one we know the buffer size for, and use that + # as a lower bound to find_bufsize. + min_bufsize = max([(p, b) for p, b in maps if p < period] + [(0,0)])[1] + bufsize = find_bufsize(period, ATTACK_HEADERS, when, best[2][0], min_bufsize) + if bufsize is not None: + # We found a (period, bufsize) configuration with better memory usage than our best + # so far. Remember it for future lower bounds. + maps.append((period, bufsize)) + mem = memory_usage(period, bufsize, when) + assert mem[0] <= best[2][0] + if ASSUME_CONVEX: + # Remove all periods that are on the other side of the former best as the new + # best. + periods = [p for p in periods if (p < best[0]) == (period < best[0])] + best = (period, bufsize, mem) + print(f"- New best: period={period}, buffer={bufsize}, mem={mem[0] / 8192:.3f} KiB") + else: + # The (period, bufsize) configuration we found is worse than what we already had. + if ASSUME_CONVEX: + # Remove all periods that are on the other side of the tried configuration as the + # best one. + periods = [p for p in periods if (p < period) == (best[0] < period)] + + # Return the result. + period, bufsize, _ = best + return period, bufsize + + +def analyze(when): + """Find the best configuration and print it out.""" + + period, bufsize = optimize(when) + # Compute accurate statistics for the best found configuration. + _, mem_mainchain, mem_timewarp = memory_usage(period, bufsize, when) + headers_per_attack, _ = attack_rate(period, bufsize) + attack_volume = NET_HEADER_SIZE * MINCHAINWORK_HEADERS + # And report them. + print() + print("Optimal configuration:") + print() + print("//! Store one header commitment per HEADER_COMMITMENT_PERIOD blocks.") + print(f"constexpr size_t HEADER_COMMITMENT_PERIOD{{{period}}};") + print() + print("//! Only feed headers to validation once this many headers on top have been") + print("//! received and validated against commitments.") + print(f"constexpr size_t REDOWNLOAD_BUFFER_SIZE{{{bufsize}}};" + f" // {bufsize}/{period} = ~{bufsize/period:.1f} commitments") + print() + print("Properties:") + print(f"- Per-peer memory for mainchain sync: {mem_mainchain / 8192:.3f} KiB") + print(f"- Per-peer memory for timewarp attack: {mem_timewarp / 8192:.3f} KiB") + print(f"- Attack rate: {1/headers_per_attack:.1f} attacks for 1 header of memory growth") + print(f" (where each attack costs {attack_volume / 8388608:.3f} MiB bandwidth)") + + +analyze(TIME) diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp index 919ffab102ddb..befc949f18ca8 100644 --- a/contrib/devtools/iwyu/bitcoin.core.imp +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -1,7 +1,4 @@ # Fixups / upstreamed changes [ - { include: [ "", private, "", public ] }, - { include: [ "", private, "", public ] }, - { include: [ "", private, "", public ] }, { include: [ "", private, "", public ] }, ] diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 6cd022ef1710e..f57e9abfeca1c 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -8,9 +8,8 @@ Otherwise the exit status will be 1 and it will log which executables failed which checks. ''' import sys -from typing import List -import lief #type:ignore +import lief def check_ELF_RELRO(binary) -> bool: ''' @@ -113,7 +112,7 @@ def check_ELF_control_flow(binary) -> bool: main = binary.get_function_address('main') content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) - if content == [243, 15, 30, 250]: # endbr64 + if content.tolist() == [243, 15, 30, 250]: # endbr64 return True return False @@ -142,7 +141,7 @@ def check_PE_control_flow(binary) -> bool: content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) - if content == [243, 15, 30, 250]: # endbr64 + if content.tolist() == [243, 15, 30, 250]: # endbr64 return True return False @@ -158,12 +157,11 @@ def check_MACHO_NOUNDEFS(binary) -> bool: ''' return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) -def check_MACHO_LAZY_BINDINGS(binary) -> bool: +def check_MACHO_FIXUP_CHAINS(binary) -> bool: ''' - Check for no lazy bindings. - We don't use or check for MH_BINDATLOAD. See #18295. + Check for use of chained fixups. ''' - return binary.dyld_info.lazy_bind == (0,0) + return binary.has_dyld_chained_fixups def check_MACHO_Canary(binary) -> bool: ''' @@ -190,7 +188,17 @@ def check_MACHO_control_flow(binary) -> bool: ''' content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) - if content == [243, 15, 30, 250]: # endbr64 + if content.tolist() == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_MACHO_branch_protection(binary) -> bool: + ''' + Check for branch protection instrumentation + ''' + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) + + if content.tolist() == [95, 36, 3, 213]: # bti return True return False @@ -214,8 +222,8 @@ def check_MACHO_control_flow(binary) -> bool: BASE_MACHO = [ ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), ('Canary', check_MACHO_Canary), + ('FIXUP_CHAINS', check_MACHO_FIXUP_CHAINS), ] CHECKS = { @@ -233,7 +241,7 @@ def check_MACHO_control_flow(binary) -> bool: lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE), ('NX', check_NX), ('CONTROL_FLOW', check_MACHO_control_flow)], - lief.ARCHITECTURES.ARM64: BASE_MACHO, + lief.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_branch_protection)], } } @@ -256,7 +264,7 @@ def check_MACHO_control_flow(binary) -> bool: retval = 1 continue - failed: List[str] = [] + failed: list[str] = [] for (name, func) in CHECKS[etype][arch]: if not func(binary): failed.append(name) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 3507f954f3665..5c941c4a05006 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -11,9 +11,8 @@ find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import sys -from typing import List, Dict -import lief #type:ignore +import lief # Debian 10 (Buster) EOL: 2024. https://wiki.debian.org/LTS # @@ -33,7 +32,7 @@ # See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. MAX_VERSIONS = { -'GCC': (4,8,0), +'GCC': (4,3,0), 'GLIBC': { lief.ELF.ARCH.x86_64: (2,27), lief.ELF.ARCH.ARM: (2,27), @@ -53,7 +52,7 @@ # Expected linker-loader names can be found here: # https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 -ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { +ELF_INTERPRETER_NAMES: dict[lief.ELF.ARCH, dict[lief.ENDIANNESS, str]] = { lief.ELF.ARCH.x86_64: { lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", }, @@ -72,7 +71,7 @@ }, } -ELF_ABIS: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, List[int]]] = { +ELF_ABIS: dict[lief.ELF.ARCH, dict[lief.ENDIANNESS, list[int]]] = { lief.ELF.ARCH.x86_64: { lief.ENDIANNESS.LITTLE: [3,2,0], }, @@ -93,12 +92,11 @@ # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { -# bitcoind and bitcoin-qt +# navcoind and bitcoin-qt 'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading 'libm.so.6', # math library -'librt.so.1', # real-time (clock) 'libatomic.so.1', 'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker @@ -129,7 +127,7 @@ } MACHO_ALLOWED_LIBRARIES = { -# bitcoind and bitcoin-qt +# navcoind and bitcoin-qt 'libc++.1.dylib', # C++ Standard Library 'libSystem.B.dylib', # libc, libm, libpthread, libinfo # bitcoin-qt only @@ -158,21 +156,21 @@ 'KERNEL32.dll', # win32 base APIs 'msvcrt.dll', # C standard library for MSVC 'SHELL32.dll', # shell API -'USER32.dll', # user interface 'WS2_32.dll', # sockets # bitcoin-qt only 'dwmapi.dll', # desktop window manager 'GDI32.dll', # graphics device interface 'IMM32.dll', # input method editor -'NETAPI32.dll', +'NETAPI32.dll', # network management 'ole32.dll', # component object model 'OLEAUT32.dll', # OLE Automation API 'SHLWAPI.dll', # light weight shell API -'USERENV.dll', -'UxTheme.dll', +'USER32.dll', # user interface +'USERENV.dll', # user management +'UxTheme.dll', # visual style 'VERSION.dll', # version checking 'WINMM.dll', # WinMM audio API -'WTSAPI32.dll', +'WTSAPI32.dll', # Remote Desktop } def check_version(max_versions, version, arch) -> bool: @@ -232,12 +230,17 @@ def check_MACHO_libraries(binary) -> bool: return ok def check_MACHO_min_os(binary) -> bool: - if binary.build_version.minos == [10,15,0]: + if binary.build_version.minos == [11,0,0]: return True return False def check_MACHO_sdk(binary) -> bool: - if binary.build_version.sdk == [11, 0, 0]: + if binary.build_version.sdk == [14, 0, 0]: + return True + return False + +def check_MACHO_ld64(binary) -> bool: + if binary.build_version.tools[0].version == [711, 0, 0]: return True return False @@ -279,6 +282,7 @@ def check_ELF_ABI(binary) -> bool: ('DYNAMIC_LIBRARIES', check_MACHO_libraries), ('MIN_OS', check_MACHO_min_os), ('SDK', check_MACHO_sdk), + ('LD64', check_MACHO_ld64), ], lief.EXE_FORMATS.PE: [ ('DYNAMIC_LIBRARIES', check_PE_libraries), @@ -297,7 +301,7 @@ def check_ELF_ABI(binary) -> bool: retval = 1 continue - failed: List[str] = [] + failed: list[str] = [] for (name, func) in CHECKS[etype]: if not func(binary): failed.append(name) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 54718fd7a1f84..48823c7e45819 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -5,10 +5,9 @@ ''' Test script for security-check.py ''' -import lief #type:ignore +import lief import os import subprocess -from typing import List import unittest from utils import determine_wellknown_cmd @@ -28,13 +27,13 @@ def clean_files(source, executable): os.remove(source) os.remove(executable) -def call_security_check(cc, source, executable, options): +def call_security_check(cc: str, source: str, executable: str, options) -> tuple: # This should behave the same as AC_TRY_LINK, so arrange well-known flags # in the same order as autoconf would. # # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for # reference. - env_flags: List[str] = [] + env_flags: list[str] = [] for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: env_flags += filter(None, os.environ.get(var, '').split(' ')) @@ -119,29 +118,31 @@ def test_MACHO(self): arch = get_arch(cc, source, executable) if arch == lief.ARCHITECTURES.X86: - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), - (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-no_fixup_chains']), + (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS Canary PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']), + (1, executable+': failed NOUNDEFS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']), (1, executable+': failed PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']), + (1, executable+': failed PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), (0, '')) else: # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS')) - self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']), - (1, executable+': failed LAZY_BINDINGS')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']), + (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS BRANCH_PROTECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains', '-mbranch-protection=bti']), + (1, executable+': failed NOUNDEFS Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti']), + (1, executable+': failed NOUNDEFS')) + self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti']), (0, '')) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index e304880140e6f..0140decb25fdd 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -7,18 +7,17 @@ ''' import os import subprocess -from typing import List import unittest from utils import determine_wellknown_cmd -def call_symbol_check(cc: List[str], source, executable, options): +def call_symbol_check(cc: list[str], source, executable, options): # This should behave the same as AC_TRY_LINK, so arrange well-known flags # in the same order as autoconf would. # # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for # reference. - env_flags: List[str] = [] + env_flags: list[str] = [] for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: env_flags += filter(None, os.environ.get(var, '').split(' ')) @@ -28,7 +27,7 @@ def call_symbol_check(cc: List[str], source, executable, options): os.remove(executable) return (p.returncode, p.stdout.rstrip()) -def get_machine(cc: List[str]): +def get_machine(cc: list[str]): p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, text=True) return p.stdout.rstrip() @@ -121,7 +120,7 @@ def test_MACHO(self): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.15', '-Wl,11.4']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,11.0', '-Wl,11.4']), (1, f'{executable}: failed SDK')) def test_PE(self): diff --git a/contrib/devtools/test_utxo_snapshots.sh b/contrib/devtools/test_utxo_snapshots.sh new file mode 100755 index 0000000000000..208f8d08b1c49 --- /dev/null +++ b/contrib/devtools/test_utxo_snapshots.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# Demonstrate the creation and usage of UTXO snapshots. +# +# A server node starts up, IBDs up to a certain height, then generates a UTXO +# snapshot at that point. +# +# The server then downloads more blocks (to create a diff from the snapshot). +# +# We bring a client up, load the UTXO snapshot, and we show the client sync to +# the "network tip" and then start a background validation of the snapshot it +# loaded. We see the background validation chainstate removed after validation +# completes. +# +# The shellcheck rule SC2086 (quoted variables) disablements are necessary +# since this rule needs to be violated in order to get navcoind to pick up on +# $EARLY_IBD_FLAGS for the script to work. + +export LC_ALL=C +set -e + +BASE_HEIGHT=${1:-30000} +INCREMENTAL_HEIGHT=20000 +FINAL_HEIGHT=$((BASE_HEIGHT + INCREMENTAL_HEIGHT)) + +SERVER_DATADIR="$(pwd)/utxodemo-data-server-$BASE_HEIGHT" +CLIENT_DATADIR="$(pwd)/utxodemo-data-client-$BASE_HEIGHT" +UTXO_DAT_FILE="$(pwd)/utxo.$BASE_HEIGHT.dat" + +# Chosen to try to not interfere with any running navcoind processes. +SERVER_PORT=8633 +SERVER_RPC_PORT=8632 + +CLIENT_PORT=8733 +CLIENT_RPC_PORT=8732 + +SERVER_PORTS="-port=${SERVER_PORT} -rpcport=${SERVER_RPC_PORT}" +CLIENT_PORTS="-port=${CLIENT_PORT} -rpcport=${CLIENT_RPC_PORT}" + +# Ensure the client exercises all indexes to test that snapshot use works +# properly with indexes. +ALL_INDEXES="-txindex -coinstatsindex -blockfilterindex=1" + +if ! command -v jq >/dev/null ; then + echo "This script requires jq to parse JSON RPC output. Please install it." + echo "(e.g. sudo apt install jq)" + exit 1 +fi + +DUMP_OUTPUT="dumptxoutset-output-$BASE_HEIGHT.json" + +finish() { + echo + echo "Killing server and client PIDs ($SERVER_PID, $CLIENT_PID) and cleaning up datadirs" + echo + rm -f "$UTXO_DAT_FILE" "$DUMP_OUTPUT" + rm -rf "$SERVER_DATADIR" "$CLIENT_DATADIR" + kill -9 "$SERVER_PID" "$CLIENT_PID" +} + +trap finish EXIT + +# Need to specify these to trick client into accepting server as a peer +# it can IBD from, otherwise the default values prevent IBD from the server node. +EARLY_IBD_FLAGS="-maxtipage=9223372036854775207 -minimumchainwork=0x00" + +server_rpc() { + ./src/bitcoin-cli -rpcport=$SERVER_RPC_PORT -datadir="$SERVER_DATADIR" "$@" +} +client_rpc() { + ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir="$CLIENT_DATADIR" "$@" +} +server_sleep_til_boot() { + while ! server_rpc ping >/dev/null 2>&1; do sleep 0.1; done +} +client_sleep_til_boot() { + while ! client_rpc ping >/dev/null 2>&1; do sleep 0.1; done +} +server_sleep_til_shutdown() { + while server_rpc ping >/dev/null 2>&1; do sleep 0.1; done +} + +mkdir -p "$SERVER_DATADIR" "$CLIENT_DATADIR" + +echo "Hi, welcome to the assumeutxo demo/test" +echo +echo "We're going to" +echo +echo " - start up a 'server' node, sync it via mainnet IBD to height ${BASE_HEIGHT}" +echo " - create a UTXO snapshot at that height" +echo " - IBD ${INCREMENTAL_HEIGHT} more blocks on top of that" +echo +echo "then we'll demonstrate assumeutxo by " +echo +echo " - starting another node (the 'client') and loading the snapshot in" +echo " * first you'll have to modify the code slightly (chainparams) and recompile" +echo " * don't worry, we'll make it easy" +echo " - observing the client sync ${INCREMENTAL_HEIGHT} blocks on top of the snapshot from the server" +echo " - observing the client validate the snapshot chain via background IBD" +echo +read -p "Press [enter] to continue" _ + +echo +echo "-- Starting the demo. You might want to run the two following commands in" +echo " separate terminal windows:" +echo +echo " watch -n0.1 tail -n 30 $SERVER_DATADIR/debug.log" +echo " watch -n0.1 tail -n 30 $CLIENT_DATADIR/debug.log" +echo +read -p "Press [enter] to continue" _ + +echo +echo "-- IBDing the blocks (height=$BASE_HEIGHT) required to the server node..." +# shellcheck disable=SC2086 +./src/navcoind -logthreadnames=1 $SERVER_PORTS \ + -datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -stopatheight="$BASE_HEIGHT" >/dev/null + +echo +echo "-- Creating snapshot at ~ height $BASE_HEIGHT ($UTXO_DAT_FILE)..." +server_sleep_til_shutdown # wait for stopatheight to be hit +# shellcheck disable=SC2086 +./src/navcoind -logthreadnames=1 $SERVER_PORTS \ + -datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -connect=0 -listen=0 >/dev/null & +SERVER_PID="$!" + +server_sleep_til_boot +server_rpc dumptxoutset "$UTXO_DAT_FILE" > "$DUMP_OUTPUT" +cat "$DUMP_OUTPUT" +kill -9 "$SERVER_PID" + +RPC_BASE_HEIGHT=$(jq -r .base_height < "$DUMP_OUTPUT") +RPC_AU=$(jq -r .txoutset_hash < "$DUMP_OUTPUT") +RPC_NCHAINTX=$(jq -r .nchaintx < "$DUMP_OUTPUT") +RPC_BLOCKHASH=$(jq -r .base_hash < "$DUMP_OUTPUT") + +server_sleep_til_shutdown + +echo +echo "-- Now: add the following to CMainParams::m_assumeutxo_data" +echo " in src/kernel/chainparams.cpp, and recompile:" +echo +echo " {${RPC_BASE_HEIGHT}, AssumeutxoHash{uint256S(\"0x${RPC_AU}\")}, ${RPC_NCHAINTX}, uint256S(\"0x${RPC_BLOCKHASH}\")}," +echo +echo +echo "-- IBDing more blocks to the server node (height=$FINAL_HEIGHT) so there is a diff between snapshot and tip..." +# shellcheck disable=SC2086 +./src/navcoind $SERVER_PORTS -logthreadnames=1 -datadir="$SERVER_DATADIR" \ + $EARLY_IBD_FLAGS -stopatheight="$FINAL_HEIGHT" >/dev/null + +echo +echo "-- Starting the server node to provide blocks to the client node..." +# shellcheck disable=SC2086 +./src/navcoind $SERVER_PORTS -logthreadnames=1 -debug=net -datadir="$SERVER_DATADIR" \ + $EARLY_IBD_FLAGS -connect=0 -listen=1 >/dev/null & +SERVER_PID="$!" +server_sleep_til_boot + +echo +echo "-- Okay, what you're about to see is the client starting up and activating the snapshot." +echo " I'm going to display the top 14 log lines from the client on top of an RPC called" +echo " getchainstates, which is like getblockchaininfo but for both the snapshot and " +echo " background validation chainstates." +echo +echo " You're going to first see the snapshot chainstate sync to the server's tip, then" +echo " the background IBD chain kicks in to validate up to the base of the snapshot." +echo +echo " Once validation of the snapshot is done, you should see log lines indicating" +echo " that we've deleted the background validation chainstate." +echo +echo " Once everything completes, exit the watch command with CTRL+C." +echo +read -p "When you're ready for all this, hit [enter]" _ + +echo +echo "-- Starting the client node to get headers from the server, then load the snapshot..." +# shellcheck disable=SC2086 +./src/navcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" \ + -connect=0 -addnode=127.0.0.1:$SERVER_PORT -debug=net $EARLY_IBD_FLAGS >/dev/null & +CLIENT_PID="$!" +client_sleep_til_boot + +echo +echo "-- Initial state of the client:" +client_rpc getchainstates + +echo +echo "-- Loading UTXO snapshot into client..." +client_rpc loadtxoutset "$UTXO_DAT_FILE" + +watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat" + +echo +echo "-- Okay, now I'm going to restart the client to make sure that the snapshot chain reloads " +echo " as the main chain properly..." +echo +echo " Press CTRL+C after you're satisfied to exit the demo" +echo +read -p "Press [enter] to continue" + +client_sleep_til_boot +# shellcheck disable=SC2086 +./src/navcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" -connect=0 \ + -addnode=127.0.0.1:$SERVER_PORT "$EARLY_IBD_FLAGS" >/dev/null & +CLIENT_PID="$!" +client_sleep_til_boot + +watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat" + +echo +echo "-- Done!" diff --git a/contrib/devtools/utils.py b/contrib/devtools/utils.py index 68ad1c3aba191..8b4c67c6c0ce2 100755 --- a/contrib/devtools/utils.py +++ b/contrib/devtools/utils.py @@ -8,10 +8,9 @@ import shutil import sys import os -from typing import List -def determine_wellknown_cmd(envvar, progname) -> List[str]: +def determine_wellknown_cmd(envvar, progname) -> list[str]: maybe_env = os.getenv(envvar) maybe_which = shutil.which(progname) if maybe_env: diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh index dee25ff67b61e..fbb8591965fd0 100755 --- a/contrib/devtools/utxo_snapshot.sh +++ b/contrib/devtools/utxo_snapshot.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2023 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -8,6 +8,8 @@ export LC_ALL=C set -ueo pipefail +NETWORK_DISABLED=false + if (( $# < 3 )); then echo 'Usage: utxo_snapshot.sh ' echo @@ -26,19 +28,67 @@ OUTPUT_PATH="${1}"; shift; # Most of the calls we make take a while to run, so pad with a lengthy timeout. BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999" +# Check if the node is pruned and get the pruned block height +PRUNED=$( ${BITCOIN_CLI_CALL} getblockchaininfo | awk '/pruneheight/ {print $2}' | tr -d ',' ) + +if (( GENERATE_AT_HEIGHT < PRUNED )); then + echo "Error: The requested snapshot height (${GENERATE_AT_HEIGHT}) should be greater than the pruned block height (${PRUNED})." + exit 1 +fi + +# Early exit if file at OUTPUT_PATH already exists +if [[ -e "$OUTPUT_PATH" ]]; then + (>&2 echo "Error: $OUTPUT_PATH already exists or is not a valid path.") + exit 1 +fi + +# Validate that the path is correct +if [[ "${OUTPUT_PATH}" != "-" && ! -d "$(dirname "${OUTPUT_PATH}")" ]]; then + (>&2 echo "Error: The directory $(dirname "${OUTPUT_PATH}") does not exist.") + exit 1 +fi + +function cleanup { + (>&2 echo "Restoring chain to original height; this may take a while") + ${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" + + if $NETWORK_DISABLED; then + (>&2 echo "Restoring network activity") + ${BITCOIN_CLI_CALL} setnetworkactive true + fi +} + +function early_exit { + (>&2 echo "Exiting due to Ctrl-C") + cleanup + exit 1 +} + +# Prompt the user to disable network activity +read -p "Do you want to disable network activity (setnetworkactive false) before running invalidateblock? (Y/n): " -r +if [[ "$REPLY" =~ ^[Yy]*$ || -z "$REPLY" ]]; then + # User input is "Y", "y", or Enter key, proceed with the action + NETWORK_DISABLED=true + (>&2 echo "Disabling network activity") + ${BITCOIN_CLI_CALL} setnetworkactive false +else + (>&2 echo "Network activity remains enabled") +fi + # Block we'll invalidate/reconsider to rewind/fast-forward the chain. PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) ) +# Trap for normal exit and Ctrl-C +trap cleanup EXIT +trap early_exit INT + (>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while") ${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}" if [[ "${OUTPUT_PATH}" = "-" ]]; then (>&2 echo "Generating txoutset info...") - ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g' + ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_3 | sed 's/^.*: "\(.\+\)\+",/\1/g' else (>&2 echo "Generating UTXO snapshot...") ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}" fi - -(>&2 echo "Restoring chain to original height; this may take a while") -${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md index bbd88e58f3dc9..35ea83e585ba3 100644 --- a/contrib/guix/INSTALL.md +++ b/contrib/guix/INSTALL.md @@ -62,9 +62,6 @@ so you should log out and log back in. Please refer to fanquake's instructions [here](https://github.com/fanquake/core-review/tree/master/guix). -Note that the `Dockerfile` is largely equivalent to running through the binary -tarball installation steps. - ## Option 4: Using a distribution-maintained package Note that this section is based on the distro packaging situation at the time of @@ -74,25 +71,15 @@ https://repology.org/project/guix/versions ### Debian / Ubuntu -Guix v1.2.0 is available as a distribution package starting in [Debian -11](https://packages.debian.org/bullseye/guix) and [Ubuntu -21.04](https://packages.ubuntu.com/search?keywords=guix). - -Note that if you intend on using Guix without using any substitutes (more -details [here][security-model]), v1.2.0 has a known problem when building GnuTLS -from source. Solutions and workarounds are documented -[here](#gnutls-test-suite-fail-status-request-revoked). - +Guix is available as a distribution package in [Debian +](https://packages.debian.org/search?keywords=guix) and [Ubuntu +](https://packages.ubuntu.com/search?keywords=guix). To install: ```sh sudo apt install guix ``` -For up-to-date information on Debian and Ubuntu's release history: -- [Debian release history](https://www.debian.org/releases/) -- [Ubuntu release history](https://ubuntu.com/about/release-cycle) - ### Arch Linux Guix is available in the AUR as @@ -167,80 +154,41 @@ For reference, the graphic below outlines Guix v1.3.0's dependency graph: ![bootstrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) -#### Consider /tmp on tmpfs - -If you use an NVME (SSD) drive, you may encounter [cryptic build errors](#coreutils-fail-teststail-2inotify-dir-recreate). Mounting a [tmpfs at /tmp](https://ubuntu.com/blog/data-driven-analysis-tmp-on-tmpfs) should prevent this and may improve performance as a bonus. - -#### Guile - -##### Choosing a Guile version and sticking to it - -One of the first things you need to decide is which Guile version you want to -use: Guile v2.2 or Guile v3.0. Unlike the python2 to python3 transition, Guile -v2.2 and Guile v3.0 are largely compatible, as evidenced by the fact that most -Guile packages and even [Guix -itself](https://guix.gnu.org/en/blog/2020/guile-3-and-guix/) support running on -both. - -What is important here is that you **choose one**, and you **remain consistent** -with your choice throughout **all Guile-related packages**, no matter if they -are installed via the distribution's package manager or installed from source. -This is because the files for Guile packages are installed to directories which -are separated based on the Guile version. - -###### Example: Checking that Ubuntu's `guile-git` is compatible with your chosen Guile version - -On Ubuntu Focal: +If you do not care about building each dependency from source, and Guix is +already packaged for your distribution, you can easily install only the build +dependencies of Guix. For example, to enable deb-src and install the Guix build +dependencies on Ubuntu/Debian: ```sh -$ apt show guile-git -Package: guile-git -... -Depends: guile-2.2, guile-bytestructures, libgit2-dev -... +sed -i 's|# deb-src|deb-src|g' /etc/apt/sources.list +apt update +apt-get build-dep -y guix ``` -As you can see, the package `guile-git` depends on `guile-2.2`, meaning that it -was likely built for Guile v2.2. This means that if you decided to use Guile -v3.0 on Ubuntu Focal, you would need to build guile-git from source instead of -using the distribution package. +If this succeeded, you can likely skip to section +["Building and Installing Guix itself"](#building-and-installing-guix-itself). -On Ubuntu Hirsute: - -```sh -$ apt show guile-git -Package: guile-git -... -Depends: guile-3.0 | guile-2.2, guile-bytestructures (>= 1.0.7-3~), libgit2-dev (>= 1.0) -... -``` - -In this case, `guile-git` depends on either `guile-3.0` or `guile-2.2`, meaning -that it would work no matter what Guile version you decided to use. +#### Guile ###### Corner case: Multiple versions of Guile on one system -It is recommended to only install one version of Guile, so that build systems do +It is recommended to only install the required version of Guile, so that build systems do not get confused about which Guile to use. -However, if you insist on having both Guile v2.2 and Guile v3.0 installed on -your system, then you need to **consistently** specify one of -`GUILE_EFFECTIVE_VERSION=3.0` or `GUILE_EFFECTIVE_VERSION=2.2` to all +However, if you insist on having more versions of Guile installed on +your system, then you need to **consistently** specify +`GUILE_EFFECTIVE_VERSION=3.0` to all `./configure` invocations for Guix and its dependencies. ##### Installing Guile -Guile is most likely already packaged for your distribution, so after you have -[chosen a Guile version](#choosing-a-guile-version-and-sticking-to-it), install -it via your distribution's package manager. - If your distribution splits packages into `-dev`-suffixed and non-`-dev`-suffixed sub-packages (as is the case for Debian-derived distributions), please make sure to install both. For example, to install Guile -v2.2 on Debian/Ubuntu: +v3.0 on Debian/Ubuntu: ```sh -apt install guile-2.2 guile-2.2-dev +apt install guile-3.0 guile-3.0-dev ``` #### Mixing distribution packages and source-built packages @@ -258,16 +206,16 @@ source-built packages, you will need to augment the `GUILE_LOAD_PATH` and `GUILE_LOAD_COMPILED_PATH` environment variables so that Guile will look under the right prefix and find your source-built packages. -For example, if you are using Guile v2.2, and have Guile packages in the +For example, if you are using Guile v3.0, and have Guile packages in the `/usr/local` prefix, either add the following lines to your `.profile` or `.bash_profile` so that the environment variable is properly set for all future shell logins, or paste the lines into a POSIX-style shell to temporarily modify the environment variables of your current shell session. ```sh -# Help Guile v2.2.x find packages in /usr/local -export GUILE_LOAD_PATH="/usr/local/share/guile/site/2.2${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" -export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/2.2/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" +# Help Guile v3.0.x find packages in /usr/local +export GUILE_LOAD_PATH="/usr/local/share/guile/site/3.0${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" +export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/3.0/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" ``` Note that these environment variables are used to check for packages during @@ -352,7 +300,7 @@ Relevant for: - Those installing `guile-git` from their distribution where `guile-git` is built against `libgit2 < 1.1` -As of v0.4.0, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, +As of v0.5.2, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, it actually requires `libgit2 >= 1.1`, otherwise, it will be confused by a reference of `origin/keyring`: instead of interpreting the reference as "the 'keyring' branch of the 'origin' remote", the reference is interpreted as "the @@ -366,20 +314,6 @@ Should you be in this situation, you need to build both `libgit2 v1.1.x` and Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 -##### `{scheme,guile}-bytestructures` v1.0.8 and v1.0.9 are broken for Guile v2.2 - -Relevant for: -- Those building `{scheme,guile}-bytestructures` from source against Guile v2.2 - -Commit -[707eea3](https://github.com/TaylanUB/scheme-bytestructures/commit/707eea3a85e1e375e86702229ebf73d496377669) -introduced a regression for Guile v2.2 and was first included in v1.0.8, this -was later corrected in commit -[ec9a721](https://github.com/TaylanUB/scheme-bytestructures/commit/ec9a721957c17bcda13148f8faa5f06934431ff7) -and included in v1.1.0. - -TL;DR If you decided to use Guile v2.2, do not use `{scheme,guile}-bytestructures` v1.0.8 or v1.0.9. - ### Building and Installing Guix itself Start by cloning Guix: @@ -389,10 +323,8 @@ git clone https://git.savannah.gnu.org/git/guix.git cd guix ``` -You will likely want to build the latest release, however, if the latest release -when you're reading this is still 1.3.0 then you may want to use 998eda30 instead -to avoid the issues described in [#25099]( -https://github.com/bitcoin/bitcoin/pull/25099). +You will likely want to build the latest release. +At the time of writing (November 2023), the latest release was `v1.4.0`. ``` git branch -a -l 'origin/version-*' # check for the latest release @@ -578,7 +510,7 @@ sudo --login guix pull --commit= ``` `guix pull` is quite a long process (especially if you're using -`--no-substitute`). If you encounter build problems, please refer to the +`--no-substitutes`). If you encounter build problems, please refer to the [troubleshooting section](#troubleshooting). Note that running a bare `guix pull` with no commit or branch specified will @@ -616,7 +548,7 @@ systemctl enable guix-daemon systemctl start guix-daemon ``` -Remember to set `--no-substitute` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. +Remember to set `--no-substitutes` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. ##### If you installed Guix via the Debian/Ubuntu distribution packages @@ -726,26 +658,18 @@ $ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build failure output for the most accurate, up-to-date information. -### openssl-1.1.1l and openssl-1.1.1n - -OpenSSL includes tests that will fail once some certificate has expired. A workaround -is to change your system clock: - -```sh -sudo timedatectl set-ntp no -sudo date --set "28 may 2022 15:00:00" -sudo --login guix build --cores=1 /gnu/store/g9alz81w4q03ncm542487xd001s6akd4-openssl-1.1.1l.drv -sudo --login guix build --cores=1 /gnu/store/mw6ax0gk33gh082anrdrxp2flrbskxv6-openssl-1.1.1n.drv -sudo timedatectl set-ntp yes -``` - ### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem which rejects characters not present in the UTF-8 character code set. An example is ZFS with the utf8only=on option set. -More information: https://bugs.python.org/issue37584 +More information: https://github.com/python/cpython/issues/81765 + +### openssl-1.1.1l and openssl-1.1.1n + +OpenSSL includes tests that will fail once some certificate has expired. +The workarounds from the GnuTLS section immediately below can be used. ### GnuTLS: test-suite FAIL: status-request-revoked @@ -781,13 +705,41 @@ authorized. This workaround was described [here](https://issues.guix.gnu.org/44559#5). Basically: -1. Turn off networking 2. Turn off NTP 3. Set system time to 2020-10-01 4. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv 5. Set system time back to accurate current time 6. Turn NTP back on -7. Turn networking back on + +For example, + +```sh +sudo timedatectl set-ntp no +sudo date --set "01 oct 2020 15:00:00" +guix build /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +sudo timedatectl set-ntp yes +``` + +#### Workaround 3: Disable the tests in the Guix source code for this single derivation + +If all of the above workarounds fail, you can also disable the `tests` phase of +the derivation via the `arguments` option, as described in the official +[`package` +reference](https://guix.gnu.org/manual/en/html_node/package-Reference.html). + +For example, to disable the openssl-1.1 check phase: + +```diff +diff --git a/gnu/packages/tls.scm b/gnu/packages/tls.scm +index f1e844b..1077c4b 100644 +--- a/gnu/packages/tls.scm ++++ b/gnu/packages/tls.scm +@@ -494,4 +494,5 @@ (define-public openssl-1.1 + (arguments + `(#:parallel-tests? #f ++ #:tests? #f + #:test-target "test" +``` ### coreutils: FAIL: tests/tail-2/inotify-dir-recreate @@ -796,7 +748,7 @@ The inotify-dir-create test fails on "remote" filesystems such as overlayfs as non-remote. A relatively easy workaround to this is to make sure that a somewhat traditional -filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds), see [/tmp on tmpfs](#consider-tmp-on-tmpfs). For +filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For Docker users, this might mean [using a volume][docker/volumes], [binding mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) [mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. diff --git a/contrib/guix/README.md b/contrib/guix/README.md index c0feb486ff2a3..6fb647f8a9d13 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -11,7 +11,7 @@ We achieve bootstrappability by using Guix as a functional package manager. # Requirements -Conservatively, you will need an x86_64 machine with: +Conservatively, you will need: - 16GB of free disk space on the partition that /gnu/store will reside in - 8GB of free disk space **per platform triple** you're planning on building @@ -259,7 +259,7 @@ details. Override the number of jobs to run simultaneously, you might want to do so on a memory-limited machine. This may be passed to: - - `guix` build commands as in `guix environment --cores="$JOBS"` + - `guix` build commands as in `guix shell --cores="$JOBS"` - `make` as in `make --jobs="$JOBS"` - `xargs` as in `xargs -P"$JOBS"` @@ -301,7 +301,7 @@ details. * _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ - Additional flags to be passed to the invocation of `guix environment` inside + Additional flags to be passed to the invocation of `guix shell` inside `guix time-machine`. # Choosing your security model diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 74b24b9612072..298e7bfbd689f 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -365,7 +365,7 @@ EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build # container specified by 'contrib/guix/manifest.scm'. # - # Explanation of `guix environment` flags: + # Explanation of `guix shell` flags: # # --container run command within an isolated container # @@ -428,7 +428,7 @@ EOF # more information. # # shellcheck disable=SC2086,SC2031 - time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ --container \ --pure \ --no-cwd \ diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 3279d431aaf34..4694209e00d5b 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -286,7 +286,7 @@ EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build # container specified by 'contrib/guix/manifest.scm'. # - # Explanation of `guix environment` flags: + # Explanation of `guix shell` flags: # # --container run command within an isolated container # @@ -343,7 +343,7 @@ EOF # more information. # # shellcheck disable=SC2086,SC2031 - time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ --container \ --pure \ --no-cwd \ diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index e0bd15493f884..f922306d34c20 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -8,7 +8,7 @@ export TZ=UTC # Although Guix _does_ set umask when building its own packages (in our case, # this is all packages in manifest.scm), it does not set it for `guix -# environment`. It does make sense for at least `guix environment --container` +# shell`. It does make sense for at least `guix shell --container` # to set umask, so if that change gets merged upstream and we bump the # time-machine to a commit which includes the aforementioned change, we can # remove this line. @@ -52,7 +52,8 @@ BASEPREFIX="${PWD}/depends" store_path() { grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ | head --lines=1 \ - | sed --expression='s|^[[:space:]]*"||' \ + | sed --expression='s|\x29*$||' \ + --expression='s|^[[:space:]]*"||' \ --expression='s|"[[:space:]]*$||' } @@ -314,7 +315,7 @@ mkdir -p "$DISTSRC" | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) ) - make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.dmg" + make deploy ${V:+V=1} OSX_ZIP="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip" ;; esac ( @@ -353,9 +354,9 @@ mkdir -p "$DISTSRC" ;; esac - # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh + # copy over the example navcoin.conf file. if contrib/devtools/gen-navcoin-conf.sh # has not been run before buildling, this file will be a stub - cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" + cp "${DISTSRC}/share/examples/navcoin.conf" "${DISTNAME}/" cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/" diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index 80277046f45f2..a7f4020270069 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -8,7 +8,7 @@ export TZ=UTC # Although Guix _does_ set umask when building its own packages (in our case, # this is all packages in manifest.scm), it does not set it for `guix -# environment`. It does make sense for at least `guix environment --container` +# shell`. It does make sense for at least `guix shell --container` # to set umask, so if that change gets merged upstream and we bump the # time-machine to a commit which includes the aforementioned change, we can # remove this line. @@ -85,11 +85,12 @@ mkdir -p "$DISTSRC" # Apply detached codesignatures to dist/ (in-place) signapple apply dist/Navcoin-Qt.app codesignatures/osx/dist - # Make a DMG from dist/ - xorrisofs -D -l -V "$(< osx_volname)" -no-pad -r -dir-mode 0755 \ - -o "${OUTDIR}/${DISTNAME}-${HOST}.dmg" \ - dist \ - -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH" + # Make a .zip from dist/ + cd dist/ + find . -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find . | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST}.zip" ;; *) exit 1 diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 3eb8fc02dae65..6c912ca748d7e 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=998eda3067c7d21e0d9bb3310d2f5a14b8f1c681 \ + --commit=d5ca4d4fd713a9f7e17e074a1e37dda99bbb09fc \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index d83ff08713d50..7335596107e62 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -1,44 +1,35 @@ -(use-modules (gnu) - (gnu packages) +(use-modules (gnu packages) (gnu packages autotools) - (gnu packages base) - (gnu packages bash) + ((gnu packages bash) #:select (bash-minimal)) (gnu packages bison) - (gnu packages certs) - (gnu packages cdrom) - (gnu packages check) - (gnu packages cmake) + ((gnu packages certs) #:select (nss-certs)) + ((gnu packages cmake) #:select (cmake-minimal)) (gnu packages commencement) (gnu packages compression) (gnu packages cross-base) - (gnu packages curl) (gnu packages file) (gnu packages gawk) (gnu packages gcc) - (gnu packages gnome) - (gnu packages installers) - (gnu packages linux) + ((gnu packages installers) #:select (nsis-x86_64)) + ((gnu packages linux) #:select (linux-libre-headers-6.1 util-linux)) (gnu packages llvm) (gnu packages mingw) (gnu packages moreutils) (gnu packages pkg-config) - (gnu packages python) - (gnu packages python-crypto) - (gnu packages python-web) - (gnu packages shells) - (gnu packages tls) - (gnu packages version-control) + ((gnu packages python) #:select (python-minimal)) + ((gnu packages python-build) #:select (python-tomli)) + ((gnu packages python-crypto) #:select (python-asn1crypto)) + ((gnu packages tls) #:select (openssl)) + ((gnu packages version-control) #:select (git-minimal)) (guix build-system cmake) (guix build-system gnu) (guix build-system python) (guix build-system trivial) - (guix download) (guix gexp) (guix git-download) ((guix licenses) #:prefix license:) (guix packages) - (guix profiles) - (guix utils)) + ((guix utils) #:select (substitute-keyword-arguments))) (define-syntax-rule (search-our-patches file-name ...) "Return the list of absolute file names corresponding to each @@ -47,41 +38,7 @@ FILE-NAME found in ./patches relative to the current file." ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) (list (search-patch file-name) ...))) -(define (make-ssp-fixed-gcc xgcc) - "Given a XGCC package, return a modified package that uses the SSP function -from glibc instead of from libssp.so. Our `symbol-check' script will complain if -we link against libssp.so, and thus will ensure that this works properly. - -Taken from: -http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" - (package - (inherit xgcc) - (arguments - (substitute-keyword-arguments (package-arguments xgcc) - ((#:make-flags flags) - `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) - -(define (make-gcc-rpath-link xgcc) - "Given a XGCC package, return a modified package that replace each instance of --rpath in the default system spec that's inserted by Guix with -rpath-link" - (package - (inherit xgcc) - (arguments - (substitute-keyword-arguments (package-arguments xgcc) - ((#:phases phases) - `(modify-phases ,phases - (add-after 'pre-configure 'replace-rpath-with-rpath-link - (lambda _ - (substitute* (cons "gcc/config/rs6000/sysv4.h" - (find-files "gcc/config" - "^gnu-user.*\\.h$")) - (("-rpath=") "-rpath-link=")) - #t)))))))) - -(define building-on (string-append (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) - -(define (explicit-cross-configure package) - (package-with-extra-configure-variable package "--build" building-on)) +(define building-on (string-append "--build=" (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) (define (make-cross-toolchain target base-gcc-for-libc @@ -92,28 +49,28 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (let* ((xbinutils (cross-binutils target)) ;; 1. Build a cross-compiling gcc without targeting any libc, derived ;; from BASE-GCC-FOR-LIBC - (xgcc-sans-libc (explicit-cross-configure (cross-gcc target - #:xgcc base-gcc-for-libc - #:xbinutils xbinutils))) + (xgcc-sans-libc (cross-gcc target + #:xgcc base-gcc-for-libc + #:xbinutils xbinutils)) ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived ;; from BASE-KERNEL-HEADERS (xkernel (cross-kernel-headers target - base-kernel-headers - xgcc-sans-libc - xbinutils)) + #:linux-headers base-kernel-headers + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils)) ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, ;; derived from BASE-LIBC - (xlibc (explicit-cross-configure (cross-libc target - base-libc - xgcc-sans-libc - xbinutils - xkernel))) + (xlibc (cross-libc target + #:libc base-libc + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils + #:xheaders xkernel)) ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from ;; BASE-GCC - (xgcc (explicit-cross-configure (cross-gcc target - #:xgcc base-gcc - #:xbinutils xbinutils - #:libc xlibc)))) + (xgcc (cross-gcc target + #:xgcc base-gcc + #:xbinutils xbinutils + #:libc xlibc))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and ;; XGCC (package @@ -123,11 +80,11 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (build-system trivial-build-system) (arguments '(#:builder (begin (mkdir %output) #t))) (propagated-inputs - `(("binutils" ,xbinutils) - ("libc" ,xlibc) - ("libc:static" ,xlibc "static") - ("gcc" ,xgcc) - ("gcc-lib" ,xgcc "lib"))) + (list xbinutils + xlibc + xgcc + `(,xlibc "static") + `(,xgcc "lib"))) (synopsis (string-append "Complete GCC tool chain for " target)) (description (string-append "This package provides a complete GCC tool chain for " target " development.")) @@ -135,23 +92,14 @@ chain for " target " development.")) (license (package-license xgcc))))) (define base-gcc gcc-10) -(define base-linux-kernel-headers linux-libre-headers-5.15) - -;; https://gcc.gnu.org/install/configure.html -(define (hardened-gcc gcc) - (package-with-extra-configure-variable ( - package-with-extra-configure-variable ( - package-with-extra-configure-variable gcc - "--enable-initfini-array" "yes") - "--enable-default-ssp" "yes") - "--enable-default-pie" "yes")) +(define base-linux-kernel-headers linux-libre-headers-6.1) (define* (make-bitcoin-cross-toolchain target #:key - (base-gcc-for-libc base-gcc) + (base-gcc-for-libc linux-base-gcc) (base-kernel-headers base-linux-kernel-headers) - (base-libc (hardened-glibc glibc-2.27)) - (base-gcc (make-gcc-rpath-link (hardened-gcc base-gcc)))) + (base-libc glibc-2.27) + (base-gcc linux-base-gcc)) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target @@ -160,26 +108,19 @@ desirable for building Bitcoin Core release binaries." base-libc base-gcc)) -(define (make-gcc-with-pthreads gcc) - (package-with-extra-configure-variable - (package-with-extra-patches gcc - (search-our-patches "gcc-10-remap-guix-store.patch")) - "--enable-threads" "posix")) - -(define (make-mingw-w64-cross-gcc cross-gcc) - (package-with-extra-patches cross-gcc - (search-our-patches "vmov-alignment.patch" - "gcc-broken-longjmp.patch"))) +(define (gcc-mingw-patches gcc) + (package-with-extra-patches gcc + (search-our-patches "gcc-remap-guix-store.patch" + "vmov-alignment.patch"))) (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" (let* ((xbinutils (cross-binutils target)) (pthreads-xlibc mingw-w64-x86_64-winpthreads) - (pthreads-xgcc (make-gcc-with-pthreads - (cross-gcc target - #:xgcc (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc)) + (pthreads-xgcc (cross-gcc target + #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) #:xbinutils xbinutils - #:libc pthreads-xlibc)))) + #:libc pthreads-xlibc))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and ;; XGCC (package @@ -189,53 +130,54 @@ desirable for building Bitcoin Core release binaries." (build-system trivial-build-system) (arguments '(#:builder (begin (mkdir %output) #t))) (propagated-inputs - `(("binutils" ,xbinutils) - ("libc" ,pthreads-xlibc) - ("gcc" ,pthreads-xgcc) - ("gcc-lib" ,pthreads-xgcc "lib"))) + (list xbinutils + pthreads-xlibc + pthreads-xgcc + `(,pthreads-xgcc "lib"))) (synopsis (string-append "Complete GCC tool chain for " target)) (description (string-append "This package provides a complete GCC tool chain for " target " development.")) (home-page (package-home-page pthreads-xgcc)) (license (package-license pthreads-xgcc))))) -(define (make-nsis-for-gcc-10 base-nsis) - (package-with-extra-patches base-nsis - (search-our-patches "nsis-gcc-10-memmove.patch" - "nsis-disable-installer-reloc.patch"))) - -(define (fix-ppc64-nx-default lief) - (package-with-extra-patches lief - (search-our-patches "lief-fix-ppc64-nx-default.patch"))) - -;; Our python-lief package can be removed once we are using -;; guix 83bfdb409787cb2737e68b093a319b247b7858e6 or later. -;; Note we currently use cmake-minimal. +;; While LIEF is packaged in Guix, we maintain our own package, +;; to simplify building, and more easily apply updates. +;; Moreover, the Guix's package uses cmake, which caused build +;; failure; see https://github.com/bitcoin/bitcoin/pull/27296. (define-public python-lief (package (name "python-lief") - (version "0.12.3") + (version "0.13.2") (source (origin (method git-fetch) (uri (git-reference (url "https://github.com/lief-project/LIEF") (commit version))) (file-name (git-file-name name version)) + (modules '((guix build utils))) + (snippet + '(begin + ;; Configure build for Python bindings. + (substitute* "api/python/config-default.toml" + (("(ninja = )true" all m) + (string-append m "false")) + (("(parallel-jobs = )0" all m) + (string-append m (number->string (parallel-job-count))))))) (sha256 (base32 - "11i6hqmcjh56y554kqhl61698n9v66j2qk1c1g63mv2w07h2z661")))) + "0y48x358ppig5xp97ahcphfipx7cg9chldj2q5zrmn610fmi4zll")))) (build-system python-build-system) - (native-inputs (list cmake-minimal)) + (native-inputs (list cmake-minimal python-tomli)) (arguments (list #:tests? #f ;needs network #:phases #~(modify-phases %standard-phases + (add-before 'build 'change-directory + (lambda _ + (chdir "api/python"))) (replace 'build (lambda _ - (invoke - "python" "setup.py" "--sdk" "build" - (string-append - "-j" (number->string (parallel-job-count))))))))) + (invoke "python" "setup.py" "build")))))) (home-page "https://github.com/lief-project/LIEF") (synopsis "Library to instrument executable formats") (description @@ -248,18 +190,15 @@ and abstract ELF, PE and MachO formats.") (name "osslsigncode") (version "2.5") (source (origin - (method url-fetch) - (uri (string-append "https://github.com/mtrojnar/" - name "/archive/" version ".tar.gz")) + (method git-fetch) + (uri (git-reference + (url "https://github.com/mtrojnar/osslsigncode") + (commit version))) (sha256 (base32 - "03by9706gg0an6dn48pljx38vcb76ziv11bgm8ilwsf293x2k4hv")))) + "1j47vwq4caxfv0xw68kw5yh00qcpbd56d7rq6c483ma3y7s96yyz")))) (build-system cmake-build-system) - (inputs - `(("openssl", openssl))) - (arguments - '(#:configure-flags - (list "-DCMAKE_DISABLE_FIND_PACKAGE_CURL=TRUE"))) + (inputs (list openssl)) (home-page "https://github.com/mtrojnar/osslsigncode") (synopsis "Authenticode signing and timestamping tool") (description "osslsigncode is a small tool that implements part of the @@ -295,7 +234,7 @@ thus should be able to compile on most platforms where these exist.") (define-public python-oscrypto (package (name "python-oscrypto") - (version "1.2.1") + (version "1.3.0") (source (origin (method git-fetch) @@ -305,7 +244,7 @@ thus should be able to compile on most platforms where these exist.") (file-name (git-file-name name version)) (sha256 (base32 - "1d4d8s4z340qhvb3g5m5v3436y3a71yc26wk4749q64m09kxqc3l")) + "1v5wkmzcyiqy39db8j2dvkdrv2nlsc48556h73x4dzjwd6kg4q0a")) (patches (search-our-patches "oscrypto-hard-code-openssl.patch")))) (build-system python-build-system) (native-search-paths @@ -316,8 +255,7 @@ thus should be able to compile on most platforms where these exist.") (files '("etc/ssl/certs/ca-certificates.crt"))))) (propagated-inputs - `(("python-asn1crypto" ,python-asn1crypto) - ("openssl" ,openssl))) + (list python-asn1crypto openssl)) (arguments `(#:phases (modify-phases %standard-phases @@ -355,7 +293,7 @@ thus should be able to compile on most platforms where these exist.") (package (inherit python-oscrypto) (name "python-oscryptotests") (propagated-inputs - `(("python-oscrypto" ,python-oscrypto))) + (list python-oscrypto)) (arguments `(#:tests? #f #:phases @@ -382,9 +320,9 @@ thus should be able to compile on most platforms where these exist.") "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) (build-system python-build-system) (propagated-inputs - `(("python-asn1crypto" ,python-asn1crypto) - ("python-oscrypto" ,python-oscrypto) - ("python-oscryptotests", python-oscryptotests))) ;; certvalidator tests import oscryptotests + (list python-asn1crypto + python-oscrypto + python-oscryptotests)) ;; certvalidator tests import oscryptotests (arguments `(#:phases (modify-phases %standard-phases @@ -432,79 +370,8 @@ certificates or paths. Supports various options, including: validation at a specific moment in time, whitelisting and revocation checks.") (license license:expat)))) -(define-public python-altgraph - (package - (name "python-altgraph") - (version "0.17") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/ronaldoussoren/altgraph") - (commit (string-append "v" version)))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "09sm4srvvkw458pn48ga9q7ykr4xlz7q8gh1h9w7nxpf001qgpwb")))) - (build-system python-build-system) - (home-page "https://github.com/ronaldoussoren/altgraph") - (synopsis "Python graph (network) package") - (description "altgraph is a fork of graphlib: a graph (network) package for -constructing graphs, BFS and DFS traversals, topological sort, shortest paths, -etc. with graphviz output.") - (license license:expat))) - - -(define-public python-macholib - (package - (name "python-macholib") - (version "1.14") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/ronaldoussoren/macholib") - (commit (string-append "v" version)))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "0aislnnfsza9wl4f0vp45ivzlc0pzhp9d4r08700slrypn5flg42")))) - (build-system python-build-system) - (propagated-inputs - `(("python-altgraph" ,python-altgraph))) - (arguments - '(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'disable-broken-tests - (lambda _ - ;; This test is broken as there is no keyboard interrupt. - (substitute* "macholib_tests/test_command_line.py" - (("^(.*)class TestCmdLine" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "macholib_tests/test_dyld.py" - (("^(.*)def test_\\S+_find" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line)) - (("^(.*)def testBasic" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line)) - ) - #t))))) - (home-page "https://github.com/ronaldoussoren/macholib") - (synopsis "Python library for analyzing and editing Mach-O headers") - (description "macholib is a Macho-O header analyzer and editor. It's -typically used as a dependency analysis tool, and also to rewrite dylib -references in Mach-O headers to be @executable_path relative. Though this tool -targets a platform specific file format, it is pure python code that is platform -and endian independent.") - (license license:expat))) - (define-public python-signapple - (let ((commit "8a945a2e7583be2665cf3a6a89d665b70ecd1ab6")) + (let ((commit "62155712e7417aba07565c9780a80e452823ae6a")) (package (name "python-signapple") (version (git-version "0.1" "1" commit)) @@ -517,15 +384,13 @@ and endian independent.") (file-name (git-file-name name commit)) (sha256 (base32 - "0fr1hangvfyiwflca6jg5g8zvg3jc9qr7vd2c12ff89pznf38dlg")))) + "1nm6rm4h4m7kbq729si4cm8rzild62mk4ni8xr5zja7l33fhv3gb")))) (build-system python-build-system) (propagated-inputs - `(("python-asn1crypto" ,python-asn1crypto) - ("python-oscrypto" ,python-oscrypto) - ("python-certvalidator" ,python-certvalidator) - ("python-elfesteem" ,python-elfesteem) - ("python-requests" ,python-requests) - ("python-macholib" ,python-macholib))) + (list python-asn1crypto + python-oscrypto + python-certvalidator + python-elfesteem)) ;; There are no tests, but attempting to run python setup.py test leads to ;; problems, just disable the test (arguments '(#:tests? #f)) @@ -535,16 +400,41 @@ and endian independent.") inspecting signatures in Mach-O binaries.") (license license:expat)))) -;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html -;; We don't use --disable-werror directly, as that would be passed through to bash, -;; and cause it's build to fail. -(define (hardened-glibc glibc) - (package-with-extra-configure-variable ( - package-with-extra-configure-variable ( - package-with-extra-configure-variable glibc - "enable_werror" "no") - "--enable-stack-protector" "all") - "--enable-bind-now" "yes")) +(define-public mingw-w64-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-threads=posix", + "--enable-default-ssp=yes", + building-on))))))) + +(define-public linux-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-initfini-array=yes", + "--enable-default-ssp=yes", + "--enable-default-pie=yes", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + ;; Given a XGCC package, return a modified package that replace each instance of + ;; -rpath in the default system spec that's inserted by Guix with -rpath-link + (add-after 'pre-configure 'replace-rpath-with-rpath-link + (lambda _ + (substitute* (cons "gcc/config/rs6000/sysv4.h" + (find-files "gcc/config" + "^gnu-user.*\\.h$")) + (("-rpath=") "-rpath-link=")) + #t)))))))) (define-public glibc-2.27 (package @@ -559,11 +449,32 @@ inspecting signatures in Mach-O binaries.") (sha256 (base32 "0azpb9cvnbv25zg8019rqz48h8i2257ngyjg566dlnp74ivrs9vq")) - (patches (search-our-patches "glibc-ldd-x86_64.patch" - "glibc-versioned-locpath.patch" - "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" + (patches (search-our-patches "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" "glibc-2.27-fcommon.patch" - "glibc-2.27-guix-prefix.patch")))))) + "glibc-2.27-guix-prefix.patch" + "glibc-2.27-no-librt.patch" + "glibc-2.27-powerpc-ldbrx.patch")))) + (arguments + (substitute-keyword-arguments (package-arguments glibc) + ((#:configure-flags flags) + `(append ,flags + ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html + (list "--enable-stack-protector=all", + "--enable-bind-now", + "--disable-werror", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + (add-before 'configure 'set-etc-rpc-installation-directory + (lambda* (#:key outputs #:allow-other-keys) + ;; Install the rpc data base file under `$out/etc/rpc'. + ;; Otherwise build will fail with "Permission denied." + (let ((out (assoc-ref outputs "out"))) + (substitute* "sunrpc/Makefile" + (("^\\$\\(inst_sysconfdir\\)/rpc(.*)$" _ suffix) + (string-append out "/etc/rpc" suffix "\n")) + (("^install-others =.*$") + (string-append "install-others = " out "/etc/rpc\n")))))))))))) (packages->manifest (append @@ -589,7 +500,7 @@ inspecting signatures in Mach-O binaries.") xz ;; Build tools gnu-make - libtool-2.4.7 + libtool autoconf-2.71 automake pkg-config @@ -598,21 +509,21 @@ inspecting signatures in Mach-O binaries.") gcc-toolchain-10 (list gcc-toolchain-10 "static") ;; Scripting - python-minimal ;; (3.9) + python-minimal ;; (3.10) ;; Git git-minimal ;; Tests - (fix-ppc64-nx-default python-lief)) + python-lief) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") - (make-nsis-for-gcc-10 nsis-x86_64) + nsis-x86_64 nss-certs osslsigncode)) ((string-contains target "-linux-") (list (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-toolchain-10 binutils cmake-minimal xorriso python-signapple)) + (list clang-toolchain-17 binutils cmake-minimal python-signapple zip)) (else '()))))) diff --git a/contrib/guix/patches/gcc-broken-longjmp.patch b/contrib/guix/patches/gcc-broken-longjmp.patch deleted file mode 100644 index 1cfc0918b090a..0000000000000 --- a/contrib/guix/patches/gcc-broken-longjmp.patch +++ /dev/null @@ -1,68 +0,0 @@ -commit eb5698897c52702498938592d7f76e67d126451f -Author: Eric Botcazou -Date: Wed May 5 22:48:51 2021 +0200 - - Fix PR target/100402 - - This is a regression for 64-bit Windows present from mainline down to the 9 - branch and introduced by the fix for PR target/99234. Again SEH, but with - a twist related to the way MinGW implements setjmp/longjmp, which turns out - to be piggybacked on SEH with recent versions of MinGW, i.e. the longjmp - performs a bona-fide unwinding of the stack, because it calls RtlUnwindEx - with the second argument initially passed to setjmp, which is the result of - __builtin_frame_address (0) in the MinGW header file: - - define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) - - This means that we directly expose the frame pointer to the SEH machinery - here (unlike with regular exception handling where we use an intermediate - CFA) and thus that we cannot do whatever we want with it. The old code - would leave it unaligned, i.e. not multiple of 16, whereas the new code - aligns it, but this breaks for some reason; at least it appears that a - .seh_setframe directive with 0 as second argument always works, so the - fix aligns it this way. - - gcc/ - PR target/100402 - * config/i386/i386.c (ix86_compute_frame_layout): For a SEH target, - always return the establisher frame for __builtin_frame_address (0). - gcc/testsuite/ - * gcc.c-torture/execute/20210505-1.c: New test. - -diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c -index 2f838840e96..06ad1b2274e 100644 ---- a/gcc/config/i386/i386.c -+++ b/gcc/config/i386/i386.c -@@ -6356,12 +6356,29 @@ ix86_compute_frame_layout (void) - area, see the SEH code in config/i386/winnt.c for the rationale. */ - frame->hard_frame_pointer_offset = frame->sse_reg_save_offset; - -- /* If we can leave the frame pointer where it is, do so. Also, return -+ /* If we can leave the frame pointer where it is, do so; however return - the establisher frame for __builtin_frame_address (0) or else if the -- frame overflows the SEH maximum frame size. */ -+ frame overflows the SEH maximum frame size. -+ -+ Note that the value returned by __builtin_frame_address (0) is quite -+ constrained, because setjmp is piggybacked on the SEH machinery with -+ recent versions of MinGW: -+ -+ # elif defined(__SEH__) -+ # if defined(__aarch64__) || defined(_ARM64_) -+ # define setjmp(BUF) _setjmp((BUF), __builtin_sponentry()) -+ # elif (__MINGW_GCC_VERSION < 40702) -+ # define setjmp(BUF) _setjmp((BUF), mingw_getsp()) -+ # else -+ # define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) -+ # endif -+ -+ and the second argument passed to _setjmp, if not null, is forwarded -+ to the TargetFrame parameter of RtlUnwindEx by longjmp (after it has -+ built an ExceptionRecord on the fly describing the setjmp buffer). */ - const HOST_WIDE_INT diff - = frame->stack_pointer_offset - frame->hard_frame_pointer_offset; -- if (diff <= 255) -+ if (diff <= 255 && !crtl->accesses_prior_frames) - { - /* The resulting diff will be a multiple of 16 lower than 255, - i.e. at most 240 as required by the unwind data structure. */ diff --git a/contrib/guix/patches/gcc-10-remap-guix-store.patch b/contrib/guix/patches/gcc-remap-guix-store.patch similarity index 100% rename from contrib/guix/patches/gcc-10-remap-guix-store.patch rename to contrib/guix/patches/gcc-remap-guix-store.patch diff --git a/contrib/guix/patches/glibc-2.27-fcommon.patch b/contrib/guix/patches/glibc-2.27-fcommon.patch index f3baacab98eed..817aa85bb95fb 100644 --- a/contrib/guix/patches/glibc-2.27-fcommon.patch +++ b/contrib/guix/patches/glibc-2.27-fcommon.patch @@ -5,7 +5,7 @@ Date: Fri May 6 11:03:04 2022 +0100 build: use -fcommon to retain legacy behaviour with GCC 10 GCC 10 started using -fno-common by default, which causes issues with - the powerpc builds using gibc 2.24. A patch was commited to glibc to fix + the powerpc builds using gibc 2.27. A patch was commited to glibc to fix the issue, 18363b4f010da9ba459b13310b113ac0647c2fcc but is non-trvial to backport, and was broken in at least one way, see the followup in commit 7650321ce037302bfc2f026aa19e0213b8d02fe6. @@ -17,6 +17,8 @@ Date: Fri May 6 11:03:04 2022 +0100 https://sourceware.org/git/?p=glibc.git;a=commit;h=18363b4f010da9ba459b13310b113ac0647c2fcc https://sourceware.org/git/?p=glibc.git;a=commit;h=7650321ce037302bfc2f026aa19e0213b8d02fe6 + This patch can be dropped when we are building with glibc 2.31+. + diff --git a/Makeconfig b/Makeconfig index 86a71e5802..aa2166be60 100644 --- a/Makeconfig diff --git a/contrib/guix/patches/glibc-2.27-guix-prefix.patch b/contrib/guix/patches/glibc-2.27-guix-prefix.patch index 6648bc6c053df..dc515907ff887 100644 --- a/contrib/guix/patches/glibc-2.27-guix-prefix.patch +++ b/contrib/guix/patches/glibc-2.27-guix-prefix.patch @@ -5,7 +5,7 @@ In order to be reproducible regardless of the architecture used to build the package, map all guix store prefixes to something fixed, e.g. /usr. We might be able to drop this in favour of using --with-nonshared-cflags -when we being using newer versions of glibc. +when we begin using newer versions of glibc. --- a/Makeconfig +++ b/Makeconfig diff --git a/contrib/guix/patches/glibc-2.27-no-librt.patch b/contrib/guix/patches/glibc-2.27-no-librt.patch new file mode 100644 index 0000000000000..4f2092ba7eca5 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-no-librt.patch @@ -0,0 +1,53 @@ +This patch can be dropped when we are building with glibc 2.30+. + +commit 6e41ef56c9baab719a02f1377b1e7ce7bff61e73 +Author: Florian Weimer +Date: Fri Feb 8 10:21:56 2019 +0100 + + rt: Turn forwards from librt to libc into compat symbols [BZ #24194] + + As the result of commit 6e6249d0b461b952d0f544792372663feb6d792a + ("BZ#14743: Move clock_* symbols from librt to libc."), in glibc 2.17, + clock_gettime, clock_getres, clock_settime, clock_getcpuclockid, + clock_nanosleep were added to libc, and the file rt/clock-compat.c + was added with forwarders to the actual implementations in libc. + These forwarders were wrapped in + + #if SHLIB_COMPAT (librt, GLIBC_2_2, GLIBC_2_17) + + so that they are not present for newer architectures (such as + powerpc64le) with a 2.17 or later ABI baseline. But the forwarders + were not marked as compatibility symbols. As a result, on older + architectures, historic configure checks such as + + AC_CHECK_LIB(rt, clock_gettime) + + still cause linking against librt, even though this is completely + unnecessary. It also creates a needless porting hazard because + architectures behave differently when it comes to symbol availability. + + Reviewed-by: Carlos O'Donell + +diff --git a/rt/clock-compat.c b/rt/clock-compat.c +index f816973c05..11e71aa890 100644 +--- a/rt/clock-compat.c ++++ b/rt/clock-compat.c +@@ -30,14 +30,16 @@ + #if HAVE_IFUNC + # undef INIT_ARCH + # define INIT_ARCH() +-# define COMPAT_REDIRECT(name, proto, arglist) libc_ifunc (name, &__##name) ++# define COMPAT_REDIRECT(name, proto, arglist) libc_ifunc (name, &__##name) \ ++ compat_symbol (librt, name, name, GLIBC_2_2); + #else + # define COMPAT_REDIRECT(name, proto, arglist) \ + int \ + name proto \ + { \ + return __##name arglist; \ +- } ++ } \ ++ compat_symbol (librt, name, name, GLIBC_2_2); + #endif + + COMPAT_REDIRECT (clock_getres, diff --git a/contrib/guix/patches/glibc-2.27-powerpc-ldbrx.patch b/contrib/guix/patches/glibc-2.27-powerpc-ldbrx.patch new file mode 100644 index 0000000000000..26716054c8fcf --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-powerpc-ldbrx.patch @@ -0,0 +1,245 @@ +From 50b0b3c9ff71ffd7ebbd74ae46844c3566478123 Mon Sep 17 00:00:00 2001 +From: "Gabriel F. T. Gomes" +Date: Mon, 27 May 2019 15:21:22 -0300 +Subject: [PATCH] powerpc: Fix build failures with current GCC + +Since GCC commit 271500 (svn), also known as the following commit on the +git mirror: + +commit e154242724b084380e3221df7c08fcdbd8460674 +Author: amodra +Date: Wed May 22 04:34:26 2019 +0000 + + [RS6000] Don't pass -many to the assembler + +glibc builds are failing when an assembly implementation does not +declare the correct '.machine' directive, or when no such directive is +declared at all. For example, when a POWER6 instruction is used, but +'.machine power6' is not declared, the assembler will fail with an error +similar to the following: + + ../sysdeps/powerpc/powerpc64/power8/strcmp.S: Assembler messages: + 24 ../sysdeps/powerpc/powerpc64/power8/strcmp.S:55: Error: unrecognized opcode: `cmpb' + +This patch adds '.machine powerN' directives where none existed, as well +as it updates '.machine power7' directives on POWER8 files, because the +minimum binutils version required to build glibc (binutils 2.25) now +provides this machine version. It also adds '-many' to the assembler +command used to build tst-set_ppr.c. + +Tested for powerpc, powerpc64, and powerpc64le, as well as with +build-many-glibcs.py for powerpc targets. + +Reviewed-by: Tulio Magno Quites Machado Filho +--- + sysdeps/powerpc/Makefile | 5 +++ + sysdeps/powerpc/powerpc64/power4/memcmp.S | 7 ++++ + sysdeps/powerpc/powerpc64/power7/strncmp.S | 1 + + .../powerpc/powerpc64/power8/fpu/s_llround.S | 1 + + sysdeps/powerpc/powerpc64/power8/strcasecmp.S | 36 ++++++------------- + sysdeps/powerpc/powerpc64/power8/strcasestr.S | 14 ++------ + sysdeps/powerpc/powerpc64/power8/strcmp.S | 1 + + 7 files changed, 28 insertions(+), 37 deletions(-) + +diff --git a/sysdeps/powerpc/Makefile b/sysdeps/powerpc/Makefile +index 6aa683b03f..23126147df 100644 +--- a/sysdeps/powerpc/Makefile ++++ b/sysdeps/powerpc/Makefile +@@ -45,6 +45,11 @@ ifeq ($(subdir),misc) + sysdep_headers += sys/platform/ppc.h + tests += test-gettimebase + tests += tst-set_ppr ++ ++# This test is expected to run and exit with EXIT_UNSUPPORTED on ++# processors that do not implement the Power ISA 2.06 or greater. ++# But the test makes use of instructions from Power ISA 2.06 and 2.07. ++CFLAGS-tst-set_ppr.c += -Wa,-many + endif + + ifneq (,$(filter %le,$(config-machine))) +diff --git a/sysdeps/powerpc/powerpc64/power4/memcmp.S b/sysdeps/powerpc/powerpc64/power4/memcmp.S +index e5319f101f..38dcf4c9a1 100644 +--- a/sysdeps/powerpc/powerpc64/power4/memcmp.S ++++ b/sysdeps/powerpc/powerpc64/power4/memcmp.S +@@ -26,7 +26,14 @@ + # define MEMCMP memcmp + #endif + ++#ifndef __LITTLE_ENDIAN__ + .machine power4 ++#else ++/* Little endian is only available since POWER8, so it's safe to ++ specify .machine as power8 (or older), even though this is a POWER4 ++ file. Since the little-endian code uses 'ldbrx', power7 is enough. */ ++ .machine power7 ++#endif + ENTRY_TOCLESS (MEMCMP, 4) + CALL_MCOUNT 3 + +diff --git a/sysdeps/powerpc/powerpc64/power7/strncmp.S b/sysdeps/powerpc/powerpc64/power7/strncmp.S +index 0c7429d19f..10f898c5a3 100644 +--- a/sysdeps/powerpc/powerpc64/power7/strncmp.S ++++ b/sysdeps/powerpc/powerpc64/power7/strncmp.S +@@ -28,6 +28,7 @@ + const char *s2 [r4], + size_t size [r5]) */ + ++ .machine power7 + ENTRY_TOCLESS (STRNCMP, 5) + CALL_MCOUNT 3 + +diff --git a/sysdeps/powerpc/powerpc64/power8/fpu/s_llround.S b/sysdeps/powerpc/powerpc64/power8/fpu/s_llround.S +index a22fc63bb3..84c76ba0f9 100644 +--- a/sysdeps/powerpc/powerpc64/power8/fpu/s_llround.S ++++ b/sysdeps/powerpc/powerpc64/power8/fpu/s_llround.S +@@ -26,6 +26,7 @@ + + /* long long [r3] llround (float x [fp1]) */ + ++ .machine power8 + ENTRY_TOCLESS (__llround) + CALL_MCOUNT 0 + frin fp1,fp1 /* Round to nearest +-0.5. */ +diff --git a/sysdeps/powerpc/powerpc64/power8/strcasecmp.S b/sysdeps/powerpc/powerpc64/power8/strcasecmp.S +index 3a2efe2a64..eeacd40c7f 100644 +--- a/sysdeps/powerpc/powerpc64/power8/strcasecmp.S ++++ b/sysdeps/powerpc/powerpc64/power8/strcasecmp.S +@@ -91,21 +91,7 @@ + 3: \ + TOLOWER() + +-#ifdef _ARCH_PWR8 +-# define VCLZD_V8_v7 vclzd v8, v7; +-# define MFVRD_R3_V1 mfvrd r3, v1; +-# define VSUBUDM_V9_V8 vsubudm v9, v9, v8; +-# define VPOPCNTD_V8_V8 vpopcntd v8, v8; +-# define VADDUQM_V7_V8 vadduqm v9, v7, v8; +-#else +-# define VCLZD_V8_v7 .long 0x11003fc2 +-# define MFVRD_R3_V1 .long 0x7c230067 +-# define VSUBUDM_V9_V8 .long 0x112944c0 +-# define VPOPCNTD_V8_V8 .long 0x110047c3 +-# define VADDUQM_V7_V8 .long 0x11274100 +-#endif +- +- .machine power7 ++ .machine power8 + + ENTRY (__STRCASECMP) + #ifdef USE_AS_STRNCASECMP +@@ -265,15 +251,15 @@ L(different): + #ifdef __LITTLE_ENDIAN__ + /* Count trailing zero. */ + vspltisb v8, -1 +- VADDUQM_V7_V8 ++ vadduqm v9, v7, v8 + vandc v8, v9, v7 +- VPOPCNTD_V8_V8 ++ vpopcntd v8, v8 + vspltb v6, v8, 15 + vcmpequb. v6, v6, v1 + blt cr6, L(shift8) + #else + /* Count leading zero. */ +- VCLZD_V8_v7 ++ vclzd v8, v7 + vspltb v6, v8, 7 + vcmpequb. v6, v6, v1 + blt cr6, L(shift8) +@@ -291,7 +277,7 @@ L(skipsum): + /* Merge and move to GPR. */ + vmrglb v6, v6, v7 + vslo v1, v6, v1 +- MFVRD_R3_V1 ++ mfvrd r3, v1 + /* Place the characters that are different in first position. */ + sldi rSTR2, rRTN, 56 + srdi rSTR2, rSTR2, 56 +@@ -301,7 +287,7 @@ L(skipsum): + vslo v6, v5, v8 + vslo v7, v4, v8 + vmrghb v1, v6, v7 +- MFVRD_R3_V1 ++ mfvrd r3, v1 + srdi rSTR2, rRTN, 48 + sldi rSTR2, rSTR2, 56 + srdi rSTR2, rSTR2, 56 +@@ -320,15 +306,15 @@ L(null_found): + #ifdef __LITTLE_ENDIAN__ + /* Count trailing zero. */ + vspltisb v8, -1 +- VADDUQM_V7_V8 ++ vadduqm v9, v7, v8 + vandc v8, v9, v7 +- VPOPCNTD_V8_V8 ++ vpopcntd v8, v8 + vspltb v6, v8, 15 + vcmpequb. v6, v6, v10 + blt cr6, L(shift_8) + #else + /* Count leading zero. */ +- VCLZD_V8_v7 ++ vclzd v8, v7 + vspltb v6, v8, 7 + vcmpequb. v6, v6, v10 + blt cr6, L(shift_8) +@@ -343,10 +329,10 @@ L(skipsum1): + vspltisb v10, 7 + vslb v10, v10, v10 + vsldoi v9, v0, v10, 1 +- VSUBUDM_V9_V8 ++ vsubudm v9, v9, v8 + vspltisb v8, 8 + vsldoi v8, v0, v8, 1 +- VSUBUDM_V9_V8 ++ vsubudm v9, v9, v8 + /* Shift and remove junk after null character. */ + #ifdef __LITTLE_ENDIAN__ + vslo v5, v5, v9 +diff --git a/sysdeps/powerpc/powerpc64/power8/strcasestr.S b/sysdeps/powerpc/powerpc64/power8/strcasestr.S +index 9fc24c29f9..e10f06fd86 100644 +--- a/sysdeps/powerpc/powerpc64/power8/strcasestr.S ++++ b/sysdeps/powerpc/powerpc64/power8/strcasestr.S +@@ -73,18 +73,8 @@ + vor reg, v8, reg; \ + vcmpequb. v6, reg, v4; + +-/* TODO: change these to the actual instructions when the minimum required +- binutils allows it. */ +-#ifdef _ARCH_PWR8 +-#define VCLZD_V8_v7 vclzd v8, v7; +-#else +-#define VCLZD_V8_v7 .long 0x11003fc2 +-#endif +- + #define FRAMESIZE (FRAME_MIN_SIZE+48) +-/* TODO: change this to .machine power8 when the minimum required binutils +- allows it. */ +- .machine power7 ++ .machine power8 + ENTRY (STRCASESTR, 4) + CALL_MCOUNT 2 + mflr r0 /* Load link register LR to r0. */ +@@ -291,7 +281,7 @@ L(nullchk1): + vcmpequb. v6, v0, v7 + /* Shift r3 by 16 bytes and proceed. */ + blt cr6, L(shift16) +- VCLZD_V8_v7 ++ vclzd v8, v7 + #ifdef __LITTLE_ENDIAN__ + vspltb v6, v8, 15 + #else +diff --git a/sysdeps/powerpc/powerpc64/power8/strcmp.S b/sysdeps/powerpc/powerpc64/power8/strcmp.S +index 15e7351d1b..d592266d1d 100644 +--- a/sysdeps/powerpc/powerpc64/power8/strcmp.S ++++ b/sysdeps/powerpc/powerpc64/power8/strcmp.S +@@ -31,6 +31,7 @@ + 64K as default, the page cross handling assumes minimum page size of + 4k. */ + ++ .machine power8 + ENTRY_TOCLESS (STRCMP, 4) + li r0,0 + +-- +2.41.0 diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch index c0f8495c41de1..ab8ae9c023231 100644 --- a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch @@ -4,6 +4,8 @@ See also: http://lists.busybox.net/pipermail/buildroot/2020-July/590376.html. https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e +This patch can be dropped when we are building with glibc 2.28+. + From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 From: "H.J. Lu" Date: Mon, 31 Dec 2018 09:26:42 -0800 diff --git a/contrib/guix/patches/glibc-ldd-x86_64.patch b/contrib/guix/patches/glibc-ldd-x86_64.patch deleted file mode 100644 index a23b095caa750..0000000000000 --- a/contrib/guix/patches/glibc-ldd-x86_64.patch +++ /dev/null @@ -1,10 +0,0 @@ -By default, 'RTDLLIST' in 'ldd' refers to 'lib64/ld-linux-x86-64.so', whereas -it's in 'lib/' for us. This patch fixes that. - ---- a/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed -+++ b/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed -@@ -1,3 +1,3 @@ - /LD_TRACE_LOADED_OBJECTS=1/a\ - add_env="$add_env LD_LIBRARY_VERSION=\\$verify_out" --s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \264\4-x86-64\6 \2x32\4-x32\6"_ -+s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \2\4-x86-64\6 \2x32\4-x32\6"_ diff --git a/contrib/guix/patches/glibc-versioned-locpath.patch b/contrib/guix/patches/glibc-versioned-locpath.patch deleted file mode 100644 index bc7652127fa5a..0000000000000 --- a/contrib/guix/patches/glibc-versioned-locpath.patch +++ /dev/null @@ -1,240 +0,0 @@ -The format of locale data can be incompatible between libc versions, and -loading incompatible data can lead to 'setlocale' returning EINVAL at best -or triggering an assertion failure at worst. See -https://lists.gnu.org/archive/html/guix-devel/2015-09/msg00717.html -for background information. - -To address that, this patch changes libc to honor a new 'GUIX_LOCPATH' -variable, and to look for locale data in version-specific sub-directories of -that variable. So, if GUIX_LOCPATH=/foo:/bar, locale data is searched for in -/foo/X.Y and /bar/X.Y, where X.Y is the libc version number. - -That way, a single 'GUIX_LOCPATH' setting can work even if different libc -versions coexist on the system. - ---- a/locale/newlocale.c -+++ b/locale/newlocale.c -@@ -30,6 +30,7 @@ - /* Lock for protecting global data. */ - __libc_rwlock_define (extern , __libc_setlocale_lock attribute_hidden) - -+extern error_t compute_locale_search_path (char **, size_t *); - - /* Use this when we come along an error. */ - #define ERROR_RETURN \ -@@ -48,7 +49,6 @@ __newlocale (int category_mask, const char *locale, __locale_t base) - __locale_t result_ptr; - char *locale_path; - size_t locale_path_len; -- const char *locpath_var; - int cnt; - size_t names_len; - -@@ -102,17 +102,8 @@ __newlocale (int category_mask, const char *locale, __locale_t base) - locale_path = NULL; - locale_path_len = 0; - -- locpath_var = getenv ("LOCPATH"); -- if (locpath_var != NULL && locpath_var[0] != '\0') -- { -- if (__argz_create_sep (locpath_var, ':', -- &locale_path, &locale_path_len) != 0) -- return NULL; -- -- if (__argz_add_sep (&locale_path, &locale_path_len, -- _nl_default_locale_path, ':') != 0) -- return NULL; -- } -+ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) -+ return NULL; - - /* Get the names for the locales we are interested in. We either - allow a composite name or a single name. */ -diff --git a/locale/setlocale.c b/locale/setlocale.c -index ead030d..0c0e314 100644 ---- a/locale/setlocale.c -+++ b/locale/setlocale.c -@@ -215,12 +215,65 @@ setdata (int category, struct __locale_data *data) - } - } - -+/* Return in *LOCALE_PATH and *LOCALE_PATH_LEN the locale data search path as -+ a colon-separated list. Return ENOMEN on error, zero otherwise. */ -+error_t -+compute_locale_search_path (char **locale_path, size_t *locale_path_len) -+{ -+ char* guix_locpath_var = getenv ("GUIX_LOCPATH"); -+ char *locpath_var = getenv ("LOCPATH"); -+ -+ if (guix_locpath_var != NULL && guix_locpath_var[0] != '\0') -+ { -+ /* Entries in 'GUIX_LOCPATH' take precedence over 'LOCPATH'. These -+ entries are systematically prefixed with "/X.Y" where "X.Y" is the -+ libc version. */ -+ if (__argz_create_sep (guix_locpath_var, ':', -+ locale_path, locale_path_len) != 0 -+ || __argz_suffix_entries (locale_path, locale_path_len, -+ "/" VERSION) != 0) -+ goto bail_out; -+ } -+ -+ if (locpath_var != NULL && locpath_var[0] != '\0') -+ { -+ char *reg_locale_path = NULL; -+ size_t reg_locale_path_len = 0; -+ -+ if (__argz_create_sep (locpath_var, ':', -+ ®_locale_path, ®_locale_path_len) != 0) -+ goto bail_out; -+ -+ if (__argz_append (locale_path, locale_path_len, -+ reg_locale_path, reg_locale_path_len) != 0) -+ goto bail_out; -+ -+ free (reg_locale_path); -+ } -+ -+ if (*locale_path != NULL) -+ { -+ /* Append the system default locale directory. */ -+ if (__argz_add_sep (locale_path, locale_path_len, -+ _nl_default_locale_path, ':') != 0) -+ goto bail_out; -+ } -+ -+ return 0; -+ -+ bail_out: -+ free (*locale_path); -+ *locale_path = NULL; -+ *locale_path_len = 0; -+ -+ return ENOMEM; -+} -+ - char * - setlocale (int category, const char *locale) - { - char *locale_path; - size_t locale_path_len; -- const char *locpath_var; - char *composite; - - /* Sanity check for CATEGORY argument. */ -@@ -251,17 +304,10 @@ setlocale (int category, const char *locale) - locale_path = NULL; - locale_path_len = 0; - -- locpath_var = getenv ("LOCPATH"); -- if (locpath_var != NULL && locpath_var[0] != '\0') -+ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) - { -- if (__argz_create_sep (locpath_var, ':', -- &locale_path, &locale_path_len) != 0 -- || __argz_add_sep (&locale_path, &locale_path_len, -- _nl_default_locale_path, ':') != 0) -- { -- __libc_rwlock_unlock (__libc_setlocale_lock); -- return NULL; -- } -+ __libc_rwlock_unlock (__libc_setlocale_lock); -+ return NULL; - } - - if (category == LC_ALL) -diff --git a/string/Makefile b/string/Makefile -index 8424a61..f925503 100644 ---- a/string/Makefile -+++ b/string/Makefile -@@ -38,7 +38,7 @@ routines := strcat strchr strcmp strcoll strcpy strcspn \ - swab strfry memfrob memmem rawmemchr strchrnul \ - $(addprefix argz-,append count create ctsep next \ - delete extract insert stringify \ -- addsep replace) \ -+ addsep replace suffix) \ - envz basename \ - strcoll_l strxfrm_l string-inlines memrchr \ - xpg-strerror strerror_l -diff --git a/string/argz-suffix.c b/string/argz-suffix.c -new file mode 100644 -index 0000000..505b0f2 ---- /dev/null -+++ b/string/argz-suffix.c -@@ -0,0 +1,56 @@ -+/* Copyright (C) 2015 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ Contributed by Ludovic Courtès . -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ . */ -+ -+#include -+#include -+#include -+#include -+ -+ -+error_t -+__argz_suffix_entries (char **argz, size_t *argz_len, const char *suffix) -+ -+{ -+ size_t suffix_len = strlen (suffix); -+ size_t count = __argz_count (*argz, *argz_len); -+ size_t new_argz_len = *argz_len + count * suffix_len; -+ char *new_argz = malloc (new_argz_len); -+ -+ if (new_argz) -+ { -+ char *p = new_argz, *entry; -+ -+ for (entry = *argz; -+ entry != NULL; -+ entry = argz_next (*argz, *argz_len, entry)) -+ { -+ p = stpcpy (p, entry); -+ p = stpcpy (p, suffix); -+ p++; -+ } -+ -+ free (*argz); -+ *argz = new_argz; -+ *argz_len = new_argz_len; -+ -+ return 0; -+ } -+ else -+ return ENOMEM; -+} -+weak_alias (__argz_suffix_entries, argz_suffix_entries) -diff --git a/string/argz.h b/string/argz.h -index bb62a31..d276a35 100644 ---- a/string/argz.h -+++ b/string/argz.h -@@ -134,6 +134,16 @@ extern error_t argz_replace (char **__restrict __argz, - const char *__restrict __str, - const char *__restrict __with, - unsigned int *__restrict __replace_count); -+ -+/* Suffix each entry of ARGZ & ARGZ_LEN with SUFFIX. Return 0 on success, -+ and ENOMEN if memory cannot be allocated. */ -+extern error_t __argz_suffix_entries (char **__restrict __argz, -+ size_t *__restrict __argz_len, -+ const char *__restrict __suffix); -+extern error_t argz_suffix_entries (char **__restrict __argz, -+ size_t *__restrict __argz_len, -+ const char *__restrict __suffix); -+ - - /* Returns the next entry in ARGZ & ARGZ_LEN after ENTRY, or NULL if there - are no more. If entry is NULL, then the first entry is returned. This diff --git a/contrib/guix/patches/lief-fix-ppc64-nx-default.patch b/contrib/guix/patches/lief-fix-ppc64-nx-default.patch deleted file mode 100644 index 101bc1ddc0cfe..0000000000000 --- a/contrib/guix/patches/lief-fix-ppc64-nx-default.patch +++ /dev/null @@ -1,29 +0,0 @@ -Correct default for Binary::has_nx on ppc64 - -From the Linux kernel source: - - * This is the default if a program doesn't have a PT_GNU_STACK - * program header entry. The PPC64 ELF ABI has a non executable stack - * stack by default, so in the absence of a PT_GNU_STACK program header - * we turn execute permission off. - -This patch can be dropped the next time we update LIEF. - -diff --git a/src/ELF/Binary.cpp b/src/ELF/Binary.cpp -index a90be1ab..fd2d9764 100644 ---- a/src/ELF/Binary.cpp -+++ b/src/ELF/Binary.cpp -@@ -1084,7 +1084,12 @@ bool Binary::has_nx() const { - return segment->type() == SEGMENT_TYPES::PT_GNU_STACK; - }); - if (it_stack == std::end(segments_)) { -- return false; -+ if (header().machine_type() == ARCH::EM_PPC64) { -+ // The PPC64 ELF ABI has a non-executable stack by default. -+ return true; -+ } else { -+ return false; -+ } - } - - return !(*it_stack)->has(ELF_SEGMENT_FLAGS::PF_X); diff --git a/contrib/guix/patches/nsis-disable-installer-reloc.patch b/contrib/guix/patches/nsis-disable-installer-reloc.patch deleted file mode 100644 index 4914527e56beb..0000000000000 --- a/contrib/guix/patches/nsis-disable-installer-reloc.patch +++ /dev/null @@ -1,30 +0,0 @@ -Patch NSIS so that it's installer stubs, produced at NSIS build time, -do not contain .reloc sections, which will exist by default when using -binutils/ld 2.36+. - -This ultimately fixes an issue when running the installer with the -"Force randomization for images (Mandatory ASLR)" setting active. - -This patch has not yet been sent upstream, because it's not clear if this -is the best fix, for the underlying issue, which seems to be that makensis -doesn't account for .reloc sections when it builds installers. - -The existence of a reloc section shouldn't be a problem, and, if anything, -is actually a requirement for working ASLR. All other Windows binaries we -produce contain them, and function correctly when under the same -"Force randomization for images (Mandatory ASLR)" setting. - -See: -https://github.com/bitcoin/bitcoin/issues/25726 -https://sourceforge.net/p/nsis/bugs/1131/ - ---- a/SCons/Config/gnu -+++ b/SCons/Config/gnu -@@ -102,6 +102,7 @@ stub_env.Append(LINKFLAGS = ['-mwindows']) # build windows executables - stub_env.Append(LINKFLAGS = ['$NODEFLIBS_FLAG']) # no standard libraries - stub_env.Append(LINKFLAGS = ['$ALIGN_FLAG']) # 512 bytes align - stub_env.Append(LINKFLAGS = ['$MAP_FLAG']) # generate map file -+stub_env.Append(LINKFLAGS = ['-Wl,--disable-reloc-section']) - - conf = FlagsConfigure(stub_env) - conf.CheckCompileFlag('-fno-tree-loop-distribute-patterns') # GCC 10: Don't generate msvcrt!memmove calls (bug #1248) diff --git a/contrib/guix/patches/nsis-gcc-10-memmove.patch b/contrib/guix/patches/nsis-gcc-10-memmove.patch deleted file mode 100644 index a1aadfd4f3618..0000000000000 --- a/contrib/guix/patches/nsis-gcc-10-memmove.patch +++ /dev/null @@ -1,23 +0,0 @@ -commit f6df41524e703dc471e283e566a48e05a735b7f2 -Author: Anders -Date: Sat Jun 27 23:18:45 2020 +0000 - - Don't let GCC 10 generate memmove calls (bug #1248) - - git-svn-id: https://svn.code.sf.net/p/nsis/code/NSIS/trunk@7189 212acab6-be3b-0410-9dea-997c60f758d6 - -diff --git a/SCons/Config/gnu b/SCons/Config/gnu -index bfcb362d..21fa446b 100644 ---- a/SCons/Config/gnu -+++ b/SCons/Config/gnu -@@ -103,6 +103,10 @@ stub_env.Append(LINKFLAGS = ['$NODEFLIBS_FLAG']) # no standard libraries - stub_env.Append(LINKFLAGS = ['$ALIGN_FLAG']) # 512 bytes align - stub_env.Append(LINKFLAGS = ['$MAP_FLAG']) # generate map file - -+conf = FlagsConfigure(stub_env) -+conf.CheckCompileFlag('-fno-tree-loop-distribute-patterns') # GCC 10: Don't generate msvcrt!memmove calls (bug #1248) -+conf.Finish() -+ - stub_uenv = stub_env.Clone() - stub_uenv.Append(CPPDEFINES = ['_UNICODE', 'UNICODE']) - diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch index 072f76eafd38f..7976b864af0f2 100644 --- a/contrib/guix/patches/vmov-alignment.patch +++ b/contrib/guix/patches/vmov-alignment.patch @@ -1,6 +1,7 @@ Description: Use unaligned VMOV instructions Author: Stephen Kitt Bug-Debian: https://bugs.debian.org/939559 +See also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412 Based on a patch originally by Claude Heiland-Allen diff --git a/contrib/init/README.md b/contrib/init/README.md index affc7c2e75063..2ee0ec9ebda1b 100644 --- a/contrib/init/README.md +++ b/contrib/init/README.md @@ -1,11 +1,11 @@ Sample configuration files for: ``` -systemd: bitcoind.service -Upstart: bitcoind.conf -OpenRC: bitcoind.openrc - bitcoind.openrcconf -CentOS: bitcoind.init -macOS: org.bitcoin.bitcoind.plist +systemd: navcoind.service +Upstart: navcoind.conf +OpenRC: navcoind.openrc + navcoind.openrcconf +CentOS: navcoind.init +macOS: org.bitcoin.navcoind.plist ``` have been made available to assist packagers in creating node packages here. diff --git a/contrib/init/bitcoind.conf b/contrib/init/navcoind.conf similarity index 83% rename from contrib/init/bitcoind.conf rename to contrib/init/navcoind.conf index dde1bd0c4d16b..1a7fd04f2d869 100644 --- a/contrib/init/bitcoind.conf +++ b/contrib/init/navcoind.conf @@ -3,14 +3,14 @@ description "Bitcoin Core Daemon" start on runlevel [2345] stop on starting rc RUNLEVEL=[016] -env BITCOIND_BIN="/usr/bin/bitcoind" +env BITCOIND_BIN="/usr/bin/navcoind" env BITCOIND_USER="bitcoin" env BITCOIND_GROUP="bitcoin" -env BITCOIND_PIDDIR="/var/run/bitcoind" +env BITCOIND_PIDDIR="/var/run/navcoind" # upstart can't handle variables constructed with other variables -env BITCOIND_PIDFILE="/var/run/bitcoind/bitcoind.pid" -env BITCOIND_CONFIGFILE="/etc/bitcoin/bitcoin.conf" -env BITCOIND_DATADIR="/var/lib/bitcoind" +env BITCOIND_PIDFILE="/var/run/navcoind/navcoind.pid" +env BITCOIND_CONFIGFILE="/etc/navcoin/navcoin.conf" +env BITCOIND_DATADIR="/var/lib/navcoind" expect fork @@ -20,12 +20,12 @@ kill timeout 600 pre-start script # this will catch non-existent config files - # bitcoind will check and exit with this very warning, but it can do so + # navcoind will check and exit with this very warning, but it can do so # long after forking, leaving upstart to think everything started fine. # since this is a commonly encountered case on install, just check and # warn here. if ! grep -qs '^rpcpassword=' "$BITCOIND_CONFIGFILE" ; then - echo "ERROR: You must set a secure rpcpassword to run bitcoind." + echo "ERROR: You must set a secure rpcpassword to run navcoind." echo "The setting must appear in $BITCOIND_CONFIGFILE" echo echo "This password is security critical to securing wallets " diff --git a/contrib/init/bitcoind.init b/contrib/init/navcoind.init similarity index 56% rename from contrib/init/bitcoind.init rename to contrib/init/navcoind.init index 19e1f76d094ce..117b39b2b2de3 100644 --- a/contrib/init/bitcoind.init +++ b/contrib/init/navcoind.init @@ -1,36 +1,36 @@ #!/usr/bin/env bash # -# bitcoind The bitcoin core server. +# navcoind The bitcoin core server. # # # chkconfig: 345 80 20 -# description: bitcoind -# processname: bitcoind +# description: navcoind +# processname: navcoind # # Source function library. . /etc/init.d/functions -# you can override defaults in /etc/sysconfig/bitcoind, see below -if [ -f /etc/sysconfig/bitcoind ]; then - . /etc/sysconfig/bitcoind +# you can override defaults in /etc/sysconfig/navcoind, see below +if [ -f /etc/sysconfig/navcoind ]; then + . /etc/sysconfig/navcoind fi RETVAL=0 -prog=bitcoind -# you can override the lockfile via BITCOIND_LOCKFILE in /etc/sysconfig/bitcoind -lockfile=${BITCOIND_LOCKFILE-/var/lock/subsys/bitcoind} +prog=navcoind +# you can override the lockfile via BITCOIND_LOCKFILE in /etc/sysconfig/navcoind +lockfile=${BITCOIND_LOCKFILE-/var/lock/subsys/navcoind} -# bitcoind defaults to /usr/bin/bitcoind, override with BITCOIND_BIN -bitcoind=${BITCOIND_BIN-/usr/bin/bitcoind} +# navcoind defaults to /usr/bin/navcoind, override with BITCOIND_BIN +navcoind=${BITCOIND_BIN-/usr/bin/navcoind} -# bitcoind opts default to -disablewallet, override with BITCOIND_OPTS -bitcoind_opts=${BITCOIND_OPTS--disablewallet} +# navcoind opts default to -disablewallet, override with BITCOIND_OPTS +navcoind_opts=${BITCOIND_OPTS--disablewallet} start() { echo -n $"Starting $prog: " - daemon $DAEMONOPTS $bitcoind $bitcoind_opts + daemon $DAEMONOPTS $navcoind $navcoind_opts RETVAL=$? echo [ $RETVAL -eq 0 ] && touch $lockfile diff --git a/contrib/init/bitcoind.openrc b/contrib/init/navcoind.openrc similarity index 86% rename from contrib/init/bitcoind.openrc rename to contrib/init/navcoind.openrc index 013a1a607027c..78bd2b63f3b21 100644 --- a/contrib/init/bitcoind.openrc +++ b/contrib/init/navcoind.openrc @@ -5,23 +5,23 @@ if [ -d "/var/lib/bitcoin/.bitcoin" ]; then BITCOIND_DEFAULT_DATADIR="/var/lib/bitcoin/.bitcoin" else - BITCOIND_DEFAULT_DATADIR="/var/lib/bitcoind" + BITCOIND_DEFAULT_DATADIR="/var/lib/navcoind" fi -BITCOIND_CONFIGFILE=${BITCOIND_CONFIGFILE:-/etc/bitcoin/bitcoin.conf} -BITCOIND_PIDDIR=${BITCOIND_PIDDIR:-/var/run/bitcoind} -BITCOIND_PIDFILE=${BITCOIND_PIDFILE:-${BITCOIND_PIDDIR}/bitcoind.pid} +BITCOIND_CONFIGFILE=${BITCOIND_CONFIGFILE:-/etc/navcoin/navcoin.conf} +BITCOIND_PIDDIR=${BITCOIND_PIDDIR:-/var/run/navcoind} +BITCOIND_PIDFILE=${BITCOIND_PIDFILE:-${BITCOIND_PIDDIR}/navcoind.pid} BITCOIND_DATADIR=${BITCOIND_DATADIR:-${BITCOIND_DEFAULT_DATADIR}} BITCOIND_USER=${BITCOIND_USER:-${BITCOIN_USER:-bitcoin}} BITCOIND_GROUP=${BITCOIND_GROUP:-bitcoin} -BITCOIND_BIN=${BITCOIND_BIN:-/usr/bin/bitcoind} +BITCOIND_BIN=${BITCOIND_BIN:-/usr/bin/navcoind} BITCOIND_NICE=${BITCOIND_NICE:-${NICELEVEL:-0}} BITCOIND_OPTS="${BITCOIND_OPTS:-${BITCOIN_OPTS}}" name="Bitcoin Core Daemon" description="Bitcoin cryptocurrency P2P network daemon" -command="/usr/bin/bitcoind" +command="/usr/bin/navcoind" command_args="-pid=\"${BITCOIND_PIDFILE}\" \ -conf=\"${BITCOIND_CONFIGFILE}\" \ -datadir=\"${BITCOIND_DATADIR}\" \ @@ -72,7 +72,7 @@ checkconfig() if grep -qs '^rpcuser=' "${BITCOIND_CONFIGFILE}" && \ ! grep -qs '^rpcpassword=' "${BITCOIND_CONFIGFILE}" ; then eerror "" - eerror "ERROR: You must set a secure rpcpassword to run bitcoind." + eerror "ERROR: You must set a secure rpcpassword to run navcoind." eerror "The setting must appear in ${BITCOIND_CONFIGFILE}" eerror "" eerror "This password is security critical to securing wallets " diff --git a/contrib/init/bitcoind.openrcconf b/contrib/init/navcoind.openrcconf similarity index 52% rename from contrib/init/bitcoind.openrcconf rename to contrib/init/navcoind.openrcconf index c8a22a08d9241..a23a8f734e23b 100644 --- a/contrib/init/bitcoind.openrcconf +++ b/contrib/init/navcoind.openrcconf @@ -1,31 +1,31 @@ -# /etc/conf.d/bitcoind: config file for /etc/init.d/bitcoind +# /etc/conf.d/navcoind: config file for /etc/init.d/navcoind # Config file location -#BITCOIND_CONFIGFILE="/etc/bitcoin/bitcoin.conf" +#BITCOIND_CONFIGFILE="/etc/navcoin/navcoin.conf" # What directory to write pidfile to? (created and owned by $BITCOIND_USER) -#BITCOIND_PIDDIR="/var/run/bitcoind" +#BITCOIND_PIDDIR="/var/run/navcoind" # What filename to give the pidfile -#BITCOIND_PIDFILE="${BITCOIND_PIDDIR}/bitcoind.pid" +#BITCOIND_PIDFILE="${BITCOIND_PIDDIR}/navcoind.pid" -# Where to write bitcoind data (be mindful that the blockchain is large) -#BITCOIND_DATADIR="/var/lib/bitcoind" +# Where to write navcoind data (be mindful that the blockchain is large) +#BITCOIND_DATADIR="/var/lib/navcoind" -# User and group to own bitcoind process +# User and group to own navcoind process #BITCOIND_USER="bitcoin" #BITCOIND_GROUP="bitcoin" -# Path to bitcoind executable -#BITCOIND_BIN="/usr/bin/bitcoind" +# Path to navcoind executable +#BITCOIND_BIN="/usr/bin/navcoind" -# Nice value to run bitcoind under +# Nice value to run navcoind under #BITCOIND_NICE=0 # Additional options (avoid -conf and -datadir, use flags above) #BITCOIND_OPTS="" -# The timeout in seconds OpenRC will wait for bitcoind to terminate +# The timeout in seconds OpenRC will wait for navcoind to terminate # after a SIGTERM has been raised. # Note that this will be mapped as argument to start-stop-daemon's # '--retry' option, which means you can specify a retry schedule diff --git a/contrib/init/bitcoind.service b/contrib/init/navcoind.service similarity index 75% rename from contrib/init/bitcoind.service rename to contrib/init/navcoind.service index 93de353bb4eb0..61d0507cc2e69 100644 --- a/contrib/init/bitcoind.service +++ b/contrib/init/navcoind.service @@ -1,11 +1,11 @@ # It is not recommended to modify this file in-place, because it will # be overwritten during package upgrades. If you want to add further # options or overwrite existing ones then use -# $ systemctl edit bitcoind.service +# $ systemctl edit navcoind.service # See "man systemd.service" for details. # Note that almost all daemon options could be specified in -# /etc/bitcoin/bitcoin.conf, but keep in mind those explicitly +# /etc/navcoin/navcoin.conf, but keep in mind those explicitly # specified as arguments in ExecStart= will override those in the # config file. @@ -18,10 +18,11 @@ After=network-online.target Wants=network-online.target [Service] -ExecStart=/usr/bin/bitcoind -daemonwait \ - -pid=/run/bitcoind/bitcoind.pid \ - -conf=/etc/bitcoin/bitcoin.conf \ - -datadir=/var/lib/bitcoind +ExecStart=/usr/bin/navcoind -pid=/run/navcoind/navcoind.pid \ + -conf=/etc/navcoin/navcoin.conf \ + -datadir=/var/lib/navcoind \ + -startupnotify='systemd-notify --ready' \ + -shutdownnotify='systemd-notify --stopping' # Make sure the config directory is readable by the service user PermissionsStartOnly=true @@ -30,8 +31,10 @@ ExecStartPre=/bin/chgrp bitcoin /etc/bitcoin # Process management #################### -Type=forking -PIDFile=/run/bitcoind/bitcoind.pid +Type=notify +NotifyAccess=all +PIDFile=/run/navcoind/navcoind.pid + Restart=on-failure TimeoutStartSec=infinity TimeoutStopSec=600 @@ -43,16 +46,16 @@ TimeoutStopSec=600 User=bitcoin Group=bitcoin -# /run/bitcoind -RuntimeDirectory=bitcoind +# /run/navcoind +RuntimeDirectory=navcoind RuntimeDirectoryMode=0710 # /etc/bitcoin ConfigurationDirectory=bitcoin ConfigurationDirectoryMode=0710 -# /var/lib/bitcoind -StateDirectory=bitcoind +# /var/lib/navcoind +StateDirectory=navcoind StateDirectoryMode=0710 # Hardening measures diff --git a/contrib/init/org.bitcoin.bitcoind.plist b/contrib/init/org.navcoin.navcoind.plist similarity index 77% rename from contrib/init/org.bitcoin.bitcoind.plist rename to contrib/init/org.navcoin.navcoind.plist index 95b5342f1ef9d..5587888042c19 100644 --- a/contrib/init/org.bitcoin.bitcoind.plist +++ b/contrib/init/org.navcoin.navcoind.plist @@ -3,10 +3,10 @@ Label - org.bitcoin.bitcoind + org.navcoin.navcoind ProgramArguments - /usr/local/bin/bitcoind + /usr/local/bin/navcoind RunAtLoad diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 25a1c7351a297..97c385d85ac5e 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -20,7 +20,7 @@ standalone hash lists but safe to use with linearize-data.py, which will output the same data no matter which byte format is chosen. The `linearize-hashes` script requires a connection, local or remote, to a -JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient. +JSON-RPC server. Running `navcoind` or `bitcoin-qt -server` will be sufficient. ## Step 2: Copy local block data @@ -38,7 +38,7 @@ will be printed. respectively, to the current time and to the timestamp of the most recent block written to the script's blockchain. * `genesis`: The hash of the genesis block in the blockchain. -* `input`: bitcoind blocks/ directory containing blkNNNNN.dat +* `input`: navcoind blocks/ directory containing blkNNNNN.dat * `hashlist`: text file containing list of block hashes created by linearize-hashes.py. * `max_out_sz`: Maximum size for files created by the `output_file` option. diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg index 5f566261ca674..65569605f1383 100644 --- a/contrib/linearize/example-linearize.cfg +++ b/contrib/linearize/example-linearize.cfg @@ -1,4 +1,4 @@ -# bitcoind RPC settings (linearize-hashes) +# navcoind RPC settings (linearize-hashes) rpcuser=someuser rpcpassword=somepassword #datadir=~/.bitcoin diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 599a0bfa6ce49..ea599df3d844b 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -6,7 +6,7 @@ The `macdeployqtplus` script should not be run manually. Instead, after building make deploy ``` -When complete, it will have produced `Bitcoin-Core.dmg`. +When complete, it will have produced `Bitcoin-Core.zip`. ## SDK Extraction @@ -14,56 +14,50 @@ When complete, it will have produced `Bitcoin-Core.dmg`. A free Apple Developer Account is required to proceed. -Our current macOS SDK -(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) -can be extracted from -[Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip). +Our macOS SDK can be extracted from +[Xcode_15.xip](https://download.developer.apple.com/Developer_Tools/Xcode_15/Xcode_15.xip). Alternatively, after logging in to your account go to 'Downloads', then 'More' -and search for [`Xcode 12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2). +and search for [`Xcode 15`](https://developer.apple.com/download/all/?q=Xcode%2015). An Apple ID and cookies enabled for the hostname are needed to download this. -The `sha256sum` of the downloaded XIP archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`. +The `sha256sum` of the downloaded XIP archive should be `4daaed2ef2253c9661779fa40bfff50655dc7ec45801aba5a39653e7bcdde48e`. -After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` -archive. This makes the SDK less-trivial to extract on non-macOS machines. One -approach (tested on Debian Buster) is outlined below: +To extract the `.xip` on Linux: ```bash # Install/clone tools needed for extracting Xcode.app apt install cpio git clone https://github.com/bitcoin-core/apple-sdk-tools.git -# Unpack Xcode_12.2.xip and place the resulting Xcode.app in your current +# Unpack the .xip and place the resulting Xcode.app in your current # working directory -python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.2.xip | cpio -d -i +python3 apple-sdk-tools/extract_xcode.py -f Xcode_15.xip | cpio -d -i ``` -On macOS the process is more straightforward: +On macOS: ```bash -xip -x Xcode_12.2.xip +xip -x Xcode_15.xip ``` -### Step 2: Generating `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` +### Step 2: Generating the SDK tarball from `Xcode.app` -To generate `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`, run -the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the -previous stage) as the first argument. +To generate the SDK, run the script [`gen-sdk`](./gen-sdk) with the +path to `Xcode.app` (extracted in the previous stage) as the first argument. ```bash -# Generate a Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz from -# the supplied Xcode.app ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` -The `sha256sum` of the generated TAR.GZ archive should be `df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619`. +The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar.gz`. +The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c282aa6892f55d`. -## Deterministic macOS DMG Notes +## Deterministic macOS App Notes -Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple -`binutils` (`ld`, `ar`, etc) and DMG authoring tools. +macOS Applications are created in Linux by combining a recent `clang` and the Apple +`binutils` (`ld`, `ar`, etc). Apple uses `clang` extensively for development and has upstreamed the necessary functionality so that a vanilla clang can take advantage. It supports the use of `-F`, @@ -93,20 +87,15 @@ created using these tools. The build process has been designed to avoid includin SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely redistributed. -[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. - -A background image is added to DMG files by inserting a `.DS_Store` during creation. - As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in order to satisfy the new Gatekeeper requirements. Because this private key cannot be shared, we'll have to be a bit creative in order for the build process to remain somewhat deterministic. Here's how it works: -- Builders use Guix to create an unsigned release. This outputs an unsigned DMG which +- Builders use Guix to create an unsigned release. This outputs an unsigned ZIP which users may choose to bless and run. It also outputs an unsigned app structure in the form - of a tarball, which also contains all of the tools that have been previously (deterministically) - built in order to create a final DMG. + of a tarball. - The Apple keyholder uses this unsigned app to create a detached signature, using the script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). - Builders feed the unsigned app + detached signature back into Guix. It uses the - pre-built tools to recombine the pieces into a deterministic DMG. + pre-built tools to recombine the pieces into a deterministic ZIP. diff --git a/contrib/macdeploy/background.tiff b/contrib/macdeploy/background.tiff deleted file mode 100644 index 1fb088c8374ac..0000000000000 Binary files a/contrib/macdeploy/background.tiff and /dev/null differ diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 5c29e9f499f4d..214088eaa4008 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -24,7 +24,7 @@ fi rm -rf ${TEMPDIR} mkdir -p ${TEMPDIR} -${SIGNAPPLE} sign -f --detach "${TEMPDIR}/${OUTROOT}" "$@" "${BUNDLE}" +${SIGNAPPLE} sign -f --detach "${TEMPDIR}/${OUTROOT}" "$@" "${BUNDLE}" --hardened-runtime tar -C "${TEMPDIR}" -czf "${OUT}" . rm -rf "${TEMPDIR}" diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index 6efaaccb8e16b..b73f5cba14c48 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -62,9 +62,6 @@ def run(): out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id) - xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1") - assert xcode_libcxx_dir.is_dir() - if args.out_sdktgz: out_sdktgz_path = pathlib.Path(args.out_sdktgz_path) else: @@ -72,7 +69,7 @@ def run(): out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name)) def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir): - """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files' + """Add all files in dir_to_add to tarfp, but prepent alt_base_dir to the files' names e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking: @@ -107,8 +104,6 @@ def run(): with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp: print("Adding MacOSX SDK {} files...".format(sdk_version)) tarfp_add_with_base_change(tarfp, sdk_dir, out_name) - print("Adding libc++ headers...") - tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name)) print("Done! Find the resulting gzipped tarball at:") print(out_sdktgz_path.resolve()) diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus deleted file mode 100755 index 7ec254482019c..0000000000000 --- a/contrib/macdeploy/macdeployqtplus +++ /dev/null @@ -1,598 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2011 Patrick "p2k" Schneider -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -import sys, re, os, platform, shutil, stat, subprocess, os.path -from argparse import ArgumentParser -from ds_store import DSStore -from mac_alias import Alias -from pathlib import Path -from subprocess import PIPE, run -from typing import List, Optional - -# This is ported from the original macdeployqt with modifications - -class FrameworkInfo(object): - def __init__(self): - self.frameworkDirectory = "" - self.frameworkName = "" - self.frameworkPath = "" - self.binaryDirectory = "" - self.binaryName = "" - self.binaryPath = "" - self.version = "" - self.installName = "" - self.deployedInstallName = "" - self.sourceFilePath = "" - self.destinationDirectory = "" - self.sourceResourcesDirectory = "" - self.sourceVersionContentsDirectory = "" - self.sourceContentsDirectory = "" - self.destinationResourcesDirectory = "" - self.destinationVersionContentsDirectory = "" - - def __eq__(self, other): - if self.__class__ == other.__class__: - return self.__dict__ == other.__dict__ - else: - return False - - def __str__(self): - return f""" Framework name: {self.frameworkName} - Framework directory: {self.frameworkDirectory} - Framework path: {self.frameworkPath} - Binary name: {self.binaryName} - Binary directory: {self.binaryDirectory} - Binary path: {self.binaryPath} - Version: {self.version} - Install name: {self.installName} - Deployed install name: {self.deployedInstallName} - Source file Path: {self.sourceFilePath} - Deployed Directory (relative to bundle): {self.destinationDirectory} -""" - - def isDylib(self): - return self.frameworkName.endswith(".dylib") - - def isQtFramework(self): - if self.isDylib(): - return self.frameworkName.startswith("libQt") - else: - return self.frameworkName.startswith("Qt") - - reOLine = re.compile(r'^(.+) \(compatibility version [0-9.]+, current version [0-9.]+\)$') - bundleFrameworkDirectory = "Contents/Frameworks" - bundleBinaryDirectory = "Contents/MacOS" - - @classmethod - def fromOtoolLibraryLine(cls, line: str) -> Optional['FrameworkInfo']: - # Note: line must be trimmed - if line == "": - return None - - # Don't deploy system libraries - if line.startswith("/System/Library/") or line.startswith("@executable_path") or line.startswith("/usr/lib/"): - return None - - m = cls.reOLine.match(line) - if m is None: - raise RuntimeError(f"otool line could not be parsed: {line}") - - path = m.group(1) - - info = cls() - info.sourceFilePath = path - info.installName = path - - if path.endswith(".dylib"): - dirname, filename = os.path.split(path) - info.frameworkName = filename - info.frameworkDirectory = dirname - info.frameworkPath = path - - info.binaryDirectory = dirname - info.binaryName = filename - info.binaryPath = path - info.version = "-" - - info.installName = path - info.deployedInstallName = f"@executable_path/../Frameworks/{info.binaryName}" - info.sourceFilePath = path - info.destinationDirectory = cls.bundleFrameworkDirectory - else: - parts = path.split("/") - i = 0 - # Search for the .framework directory - for part in parts: - if part.endswith(".framework"): - break - i += 1 - if i == len(parts): - raise RuntimeError(f"Could not find .framework or .dylib in otool line: {line}") - - info.frameworkName = parts[i] - info.frameworkDirectory = "/".join(parts[:i]) - info.frameworkPath = os.path.join(info.frameworkDirectory, info.frameworkName) - - info.binaryName = parts[i+3] - info.binaryDirectory = "/".join(parts[i+1:i+3]) - info.binaryPath = os.path.join(info.binaryDirectory, info.binaryName) - info.version = parts[i+2] - - info.deployedInstallName = f"@executable_path/../Frameworks/{os.path.join(info.frameworkName, info.binaryPath)}" - info.destinationDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, info.binaryDirectory) - - info.sourceResourcesDirectory = os.path.join(info.frameworkPath, "Resources") - info.sourceContentsDirectory = os.path.join(info.frameworkPath, "Contents") - info.sourceVersionContentsDirectory = os.path.join(info.frameworkPath, "Versions", info.version, "Contents") - info.destinationResourcesDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, "Resources") - info.destinationVersionContentsDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, "Versions", info.version, "Contents") - - return info - -class ApplicationBundleInfo(object): - def __init__(self, path: str): - self.path = path - # for backwards compatibility reasons, this must remain as Navcoin-Qt - self.binaryPath = os.path.join(path, "Contents", "MacOS", "Navcoin-Qt") - if not os.path.exists(self.binaryPath): - raise RuntimeError(f"Could not find bundle binary for {path}") - self.resourcesPath = os.path.join(path, "Contents", "Resources") - self.pluginPath = os.path.join(path, "Contents", "PlugIns") - -class DeploymentInfo(object): - def __init__(self): - self.qtPath = None - self.pluginPath = None - self.deployedFrameworks = [] - - def detectQtPath(self, frameworkDirectory: str): - parentDir = os.path.dirname(frameworkDirectory) - if os.path.exists(os.path.join(parentDir, "translations")): - # Classic layout, e.g. "/usr/local/Trolltech/Qt-4.x.x" - self.qtPath = parentDir - else: - self.qtPath = os.getenv("QTDIR", None) - - if self.qtPath is not None: - pluginPath = os.path.join(self.qtPath, "plugins") - if os.path.exists(pluginPath): - self.pluginPath = pluginPath - - def usesFramework(self, name: str) -> bool: - for framework in self.deployedFrameworks: - if framework.endswith(".framework"): - if framework.startswith(f"{name}."): - return True - elif framework.endswith(".dylib"): - if framework.startswith(f"lib{name}."): - return True - return False - -def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: - if verbose: - print(f"Inspecting with otool: {binaryPath}") - otoolbin=os.getenv("OTOOL", "otool") - otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, text=True) - if otool.returncode != 0: - sys.stderr.write(otool.stderr) - sys.stderr.flush() - raise RuntimeError(f"otool failed with return code {otool.returncode}") - - otoolLines = otool.stdout.split("\n") - otoolLines.pop(0) # First line is the inspected binary - if ".framework" in binaryPath or binaryPath.endswith(".dylib"): - otoolLines.pop(0) # Frameworks and dylibs list themselves as a dependency. - - libraries = [] - for line in otoolLines: - line = line.replace("@loader_path", os.path.dirname(binaryPath)) - info = FrameworkInfo.fromOtoolLibraryLine(line.strip()) - if info is not None: - if verbose: - print("Found framework:") - print(info) - libraries.append(info) - - return libraries - -def runInstallNameTool(action: str, *args): - installnametoolbin=os.getenv("INSTALL_NAME_TOOL", "install_name_tool") - run([installnametoolbin, "-"+action] + list(args), check=True) - -def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int): - if verbose: - print("Using install_name_tool:") - print(" in", binaryPath) - print(" change reference", oldName) - print(" to", newName) - runInstallNameTool("change", oldName, newName, binaryPath) - -def changeIdentification(id: str, binaryPath: str, verbose: int): - if verbose: - print("Using install_name_tool:") - print(" change identification in", binaryPath) - print(" to", id) - runInstallNameTool("id", id, binaryPath) - -def runStrip(binaryPath: str, verbose: int): - stripbin=os.getenv("STRIP", "strip") - if verbose: - print("Using strip:") - print(" stripped", binaryPath) - run([stripbin, "-x", binaryPath], check=True) - -def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional[str]: - if framework.sourceFilePath.startswith("Qt"): - #standard place for Nokia Qt installer's frameworks - fromPath = f"/Library/Frameworks/{framework.sourceFilePath}" - else: - fromPath = framework.sourceFilePath - toDir = os.path.join(path, framework.destinationDirectory) - toPath = os.path.join(toDir, framework.binaryName) - - if framework.isDylib(): - if not os.path.exists(fromPath): - raise RuntimeError(f"No file at {fromPath}") - - if os.path.exists(toPath): - return None # Already there - - if not os.path.exists(toDir): - os.makedirs(toDir) - - shutil.copy2(fromPath, toPath) - if verbose: - print("Copied:", fromPath) - print(" to:", toPath) - else: - to_dir = os.path.join(path, "Contents", "Frameworks", framework.frameworkName) - if os.path.exists(to_dir): - return None # Already there - - from_dir = framework.frameworkPath - if not os.path.exists(from_dir): - raise RuntimeError(f"No directory at {from_dir}") - - shutil.copytree(from_dir, to_dir, symlinks=True) - if verbose: - print("Copied:", from_dir) - print(" to:", to_dir) - - headers_link = os.path.join(to_dir, "Headers") - if os.path.exists(headers_link): - os.unlink(headers_link) - - headers_dir = os.path.join(to_dir, framework.binaryDirectory, "Headers") - if os.path.exists(headers_dir): - shutil.rmtree(headers_dir) - - permissions = os.stat(toPath) - if not permissions.st_mode & stat.S_IWRITE: - os.chmod(toPath, permissions.st_mode | stat.S_IWRITE) - - return toPath - -def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPath: str, strip: bool, verbose: int, deploymentInfo: Optional[DeploymentInfo] = None) -> DeploymentInfo: - if deploymentInfo is None: - deploymentInfo = DeploymentInfo() - - while len(frameworks) > 0: - framework = frameworks.pop(0) - deploymentInfo.deployedFrameworks.append(framework.frameworkName) - - print("Processing", framework.frameworkName, "...") - - # Get the Qt path from one of the Qt frameworks - if deploymentInfo.qtPath is None and framework.isQtFramework(): - deploymentInfo.detectQtPath(framework.frameworkDirectory) - - if framework.installName.startswith("@executable_path") or framework.installName.startswith(bundlePath): - print(framework.frameworkName, "already deployed, skipping.") - continue - - # install_name_tool the new id into the binary - changeInstallName(framework.installName, framework.deployedInstallName, binaryPath, verbose) - - # Copy framework to app bundle. - deployedBinaryPath = copyFramework(framework, bundlePath, verbose) - # Skip the rest if already was deployed. - if deployedBinaryPath is None: - continue - - if strip: - runStrip(deployedBinaryPath, verbose) - - # install_name_tool it a new id. - changeIdentification(framework.deployedInstallName, deployedBinaryPath, verbose) - # Check for framework dependencies - dependencies = getFrameworks(deployedBinaryPath, verbose) - - for dependency in dependencies: - changeInstallName(dependency.installName, dependency.deployedInstallName, deployedBinaryPath, verbose) - - # Deploy framework if necessary. - if dependency.frameworkName not in deploymentInfo.deployedFrameworks and dependency not in frameworks: - frameworks.append(dependency) - - return deploymentInfo - -def deployFrameworksForAppBundle(applicationBundle: ApplicationBundleInfo, strip: bool, verbose: int) -> DeploymentInfo: - frameworks = getFrameworks(applicationBundle.binaryPath, verbose) - if len(frameworks) == 0: - print(f"Warning: Could not find any external frameworks to deploy in {applicationBundle.path}.") - return DeploymentInfo() - else: - return deployFrameworks(frameworks, applicationBundle.path, applicationBundle.binaryPath, strip, verbose) - -def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: DeploymentInfo, strip: bool, verbose: int): - plugins = [] - if deploymentInfo.pluginPath is None: - return - for dirpath, dirnames, filenames in os.walk(deploymentInfo.pluginPath): - pluginDirectory = os.path.relpath(dirpath, deploymentInfo.pluginPath) - - if pluginDirectory not in ['styles', 'platforms']: - continue - - for pluginName in filenames: - pluginPath = os.path.join(pluginDirectory, pluginName) - - if pluginName.split('.')[0] not in ['libqminimal', 'libqcocoa', 'libqmacstyle']: - continue - - plugins.append((pluginDirectory, pluginName)) - - for pluginDirectory, pluginName in plugins: - print("Processing plugin", os.path.join(pluginDirectory, pluginName), "...") - - sourcePath = os.path.join(deploymentInfo.pluginPath, pluginDirectory, pluginName) - destinationDirectory = os.path.join(appBundleInfo.pluginPath, pluginDirectory) - if not os.path.exists(destinationDirectory): - os.makedirs(destinationDirectory) - - destinationPath = os.path.join(destinationDirectory, pluginName) - shutil.copy2(sourcePath, destinationPath) - if verbose: - print("Copied:", sourcePath) - print(" to:", destinationPath) - - if strip: - runStrip(destinationPath, verbose) - - dependencies = getFrameworks(destinationPath, verbose) - - for dependency in dependencies: - changeInstallName(dependency.installName, dependency.deployedInstallName, destinationPath, verbose) - - # Deploy framework if necessary. - if dependency.frameworkName not in deploymentInfo.deployedFrameworks: - deployFrameworks([dependency], appBundleInfo.path, destinationPath, strip, verbose, deploymentInfo) - -ap = ArgumentParser(description="""Improved version of macdeployqt. - -Outputs a ready-to-deploy app in a folder "dist" and optionally wraps it in a .dmg file. -Note, that the "dist" folder will be deleted before deploying on each run. - -Optionally, Qt translation files (.qm) can be added to the bundle.""") - -ap.add_argument("app_bundle", nargs=1, metavar="app-bundle", help="application bundle to be deployed") -ap.add_argument("appname", nargs=1, metavar="appname", help="name of the app being deployed") -ap.add_argument("-verbose", nargs="?", const=True, help="Output additional debugging information") -ap.add_argument("-no-plugins", dest="plugins", action="store_false", default=True, help="skip plugin deployment") -ap.add_argument("-no-strip", dest="strip", action="store_false", default=True, help="don't run 'strip' on the binaries") -ap.add_argument("-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image") -ap.add_argument("-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translations. Base translations will automatically be added to the bundle's resources.") - -config = ap.parse_args() - -verbose = config.verbose - -# ------------------------------------------------ - -app_bundle = config.app_bundle[0] -appname = config.appname[0] - -if not os.path.exists(app_bundle): - sys.stderr.write(f"Error: Could not find app bundle \"{app_bundle}\"\n") - sys.exit(1) - -# ------------------------------------------------ - -if os.path.exists("dist"): - print("+ Removing existing dist folder +") - shutil.rmtree("dist") - -if os.path.exists(appname + ".dmg"): - print("+ Removing existing DMG +") - os.unlink(appname + ".dmg") - -if os.path.exists(appname + ".temp.dmg"): - os.unlink(appname + ".temp.dmg") - -# ------------------------------------------------ - -target = os.path.join("dist", "Navcoin-Qt.app") - -print("+ Copying source bundle +") -if verbose: - print(app_bundle, "->", target) - -os.mkdir("dist") -shutil.copytree(app_bundle, target, symlinks=True) - -applicationBundle = ApplicationBundleInfo(target) - -# ------------------------------------------------ - -print("+ Deploying frameworks +") - -try: - deploymentInfo = deployFrameworksForAppBundle(applicationBundle, config.strip, verbose) - if deploymentInfo.qtPath is None: - deploymentInfo.qtPath = os.getenv("QTDIR", None) - if deploymentInfo.qtPath is None: - sys.stderr.write("Warning: Could not detect Qt's path, skipping plugin deployment!\n") - config.plugins = False -except RuntimeError as e: - sys.stderr.write(f"Error: {str(e)}\n") - sys.exit(1) - -# ------------------------------------------------ - -if config.plugins: - print("+ Deploying plugins +") - - try: - deployPlugins(applicationBundle, deploymentInfo, config.strip, verbose) - except RuntimeError as e: - sys.stderr.write(f"Error: {str(e)}\n") - sys.exit(1) - -# ------------------------------------------------ - -if config.translations_dir: - if not Path(config.translations_dir[0]).exists(): - sys.stderr.write(f"Error: Could not find translation dir \"{config.translations_dir[0]}\"\n") - sys.exit(1) - -print("+ Adding Qt translations +") - -translations = Path(config.translations_dir[0]) - -regex = re.compile('qt_[a-z]*(.qm|_[A-Z]*.qm)') - -lang_files = [x for x in translations.iterdir() if regex.match(x.name)] - -for file in lang_files: - if verbose: - print(file.as_posix(), "->", os.path.join(applicationBundle.resourcesPath, file.name)) - shutil.copy2(file.as_posix(), os.path.join(applicationBundle.resourcesPath, file.name)) - -# ------------------------------------------------ - -print("+ Installing qt.conf +") - -qt_conf="""[Paths] -Translations=Resources -Plugins=PlugIns -""" - -with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f: - f.write(qt_conf.encode()) - -# ------------------------------------------------ - -print("+ Generating .DS_Store +") - -output_file = os.path.join("dist", ".DS_Store") - -ds = DSStore.open(output_file, 'w+') - -ds['.']['bwsp'] = { - 'WindowBounds': '{{300, 280}, {500, 343}}', - 'PreviewPaneVisibility': False, -} - -icvp = { - 'gridOffsetX': 0.0, - 'textSize': 12.0, - 'viewOptionsVersion': 1, - 'backgroundImageAlias': b'\x00\x00\x00\x00\x02\x1e\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\x94\\\xb0H+\x00\x05\x00\x00\x00\x98\x0fbackground.tiff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xd19\xb0\xf8\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\r\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b.background\x00\x00\x10\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x11\x00\x08\x00\x00\xd19\xb0\xf8\x00\x00\x00\x01\x00\x04\x00\x00\x00\x98\x00\x0e\x00 \x00\x0f\x00b\x00a\x00c\x00k\x00g\x00r\x00o\x00u\x00n\x00d\x00.\x00t\x00i\x00f\x00f\x00\x0f\x00\x02\x00\x00\x00\x12\x00\x1c/.background/background.tiff\x00\x14\x01\x06\x00\x00\x00\x00\x01\x06\x00\x02\x00\x00\x0cMacintosh HD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xce\x97\xab\xc3H+\x00\x00\x01\x88[\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02u\xab\x8d\xd1\x94\\\xb0devrddsk\xff\xff\xff\xff\x00\x00\t \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07bitcoin\x00\x00\x10\x00\x08\x00\x00\xce\x97\xab\xc3\x00\x00\x00\x11\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x01\x00\x14\x01\x88[\x88\x00\x16\xa9\t\x00\x08\xfaR\x00\x08\xfaQ\x00\x02d\x8e\x00\x0e\x00\x02\x00\x00\x00\x0f\x00\x1a\x00\x0c\x00M\x00a\x00c\x00i\x00n\x00t\x00o\x00s\x00h\x00 \x00H\x00D\x00\x13\x00\x01/\x00\x00\x15\x00\x02\x00\x14\xff\xff\x00\x00\xff\xff\x00\x00', - 'backgroundColorBlue': 1.0, - 'iconSize': 96.0, - 'backgroundColorGreen': 1.0, - 'arrangeBy': 'none', - 'showIconPreview': True, - 'gridSpacing': 100.0, - 'gridOffsetY': 0.0, - 'showItemInfo': False, - 'labelOnBottom': True, - 'backgroundType': 2, - 'backgroundColorRed': 1.0 -} -alias = Alias().from_bytes(icvp['backgroundImageAlias']) -alias.volume.name = appname -alias.volume.posix_path = '/Volumes/' + appname -icvp['backgroundImageAlias'] = alias.to_bytes() -ds['.']['icvp'] = icvp - -ds['.']['vSrn'] = ('long', 1) - -ds['Applications']['Iloc'] = (370, 156) -ds['Navcoin-Qt.app']['Iloc'] = (128, 156) - -ds.flush() -ds.close() - -# ------------------------------------------------ - -if platform.system() == "Darwin": - subprocess.check_call(f"codesign --deep --force --sign - {target}", shell=True) - -print("+ Installing background.tiff +") - -bg_path = os.path.join('dist', '.background', 'background.tiff') -os.mkdir(os.path.dirname(bg_path)) - -tiff_path = os.path.join('contrib', 'macdeploy', 'background.tiff') -shutil.copy2(tiff_path, bg_path) - -# ------------------------------------------------ - -print("+ Generating symlink for /Applications +") - -os.symlink("/Applications", os.path.join('dist', "Applications")) - -# ------------------------------------------------ - -if config.dmg is not None: - - print("+ Preparing .dmg disk image +") - - if verbose: - print("Determining size of \"dist\"...") - size = 0 - for path, dirs, files in os.walk("dist"): - for file in files: - size += os.path.getsize(os.path.join(path, file)) - size += int(size * 0.15) - - if verbose: - print("Creating temp image for modification...") - - tempname: str = appname + ".temp.dmg" - - run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, text=True) - - if verbose: - print("Attaching temp image...") - output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, text=True, stdout=PIPE).stdout - - print("+ Finalizing .dmg disk image +") - - run(["hdiutil", "detach", f"/Volumes/{appname}"], text=True) - - run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, text=True) - - os.unlink(tempname) - -# ------------------------------------------------ - -print("+ Done +") - -sys.exit(0) diff --git a/contrib/message-capture/message-capture-docs.md b/contrib/message-capture/message-capture-docs.md index 730196846134f..571c541b2ec75 100644 --- a/contrib/message-capture/message-capture-docs.md +++ b/contrib/message-capture/message-capture-docs.md @@ -6,7 +6,7 @@ This feature allows for message capture on a per-peer basis. It answers the sim ## Usage and Functionality -* Run `bitcoind` with the `-capturemessages` option. +* Run `navcoind` with the `-capturemessages` option. * Look in the `message_capture` folder in your datadir. * Typically this will be `~/.bitcoin/message_capture`. * See that there are many folders inside, one for each peer names with its IP address and port. diff --git a/contrib/message-capture/message-capture-parser.py b/contrib/message-capture/message-capture-parser.py index d6ddc1c149a27..0f409717d4c8a 100755 --- a/contrib/message-capture/message-capture-parser.py +++ b/contrib/message-capture/message-capture-parser.py @@ -11,7 +11,7 @@ from io import BytesIO import json from pathlib import Path -from typing import Any, List, Optional +from typing import Any, Optional sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) @@ -92,7 +92,7 @@ def to_jsonable(obj: Any) -> Any: return obj -def process_file(path: str, messages: List[Any], recv: bool, progress_bar: Optional[ProgressBar]) -> None: +def process_file(path: str, messages: list[Any], recv: bool, progress_bar: Optional[ProgressBar]) -> None: with open(path, 'rb') as f_in: if progress_bar: bytes_read = 0 @@ -188,7 +188,7 @@ def main(): output = Path.cwd() / Path(args.output) if args.output else False use_progress_bar = (not args.no_progress_bar) and sys.stdout.isatty() - messages = [] # type: List[Any] + messages = [] # type: list[Any] if use_progress_bar: total_size = sum(capture.stat().st_size for capture in capturepaths) progress_bar = ProgressBar(total_size) diff --git a/contrib/qos/README.md b/contrib/qos/README.md index 0ded87c58f4ee..041f93f6d240f 100644 --- a/contrib/qos/README.md +++ b/contrib/qos/README.md @@ -2,4 +2,4 @@ This is a Linux bash script that will set up tc to limit the outgoing bandwidth for connections to the Bitcoin network. It limits outbound TCP traffic with a source or destination port of 8333, but not if the destination IP is within a LAN. -This means one can have an always-on bitcoind instance running, and another local bitcoind/bitcoin-qt instance which connects to this node and receives blocks from it. +This means one can have an always-on navcoind instance running, and another local navcoind/bitcoin-qt instance which connects to this node and receives blocks from it. diff --git a/contrib/seeds/asmap.py b/contrib/seeds/asmap.py index e28e5cf532e90..214805b5a5790 100644 --- a/contrib/seeds/asmap.py +++ b/contrib/seeds/asmap.py @@ -10,11 +10,12 @@ import ipaddress import random import unittest +from collections.abc import Callable, Iterable from enum import Enum from functools import total_ordering -from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union, overload +from typing import Optional, Union, overload -def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> List[bool]: +def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> list[bool]: """ Convert an IPv4 or IPv6 network to a prefix represented as a list of bits. @@ -32,7 +33,7 @@ def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> Li assert (netrange & ((1 << (128 - num_bits)) - 1)) == 0 return [((netrange >> (127 - i)) & 1) != 0 for i in range(num_bits)] -def prefix_to_net(prefix: List[bool]) -> Union[ipaddress.IPv4Network,ipaddress.IPv6Network]: +def prefix_to_net(prefix: list[bool]) -> Union[ipaddress.IPv4Network,ipaddress.IPv6Network]: """The reverse operation of net_to_prefix.""" # Convert to number netrange = sum(b << (127 - i) for i, b in enumerate(prefix)) @@ -47,10 +48,10 @@ def prefix_to_net(prefix: List[bool]) -> Union[ipaddress.IPv4Network,ipaddress.I return ipaddress.IPv6Network((netrange, num_bits), True) # Shortcut for (prefix, ASN) entries. -ASNEntry = Tuple[List[bool], int] +ASNEntry = tuple[list[bool], int] # Shortcut for (prefix, old ASN, new ASN) entries. -ASNDiff = Tuple[List[bool], int, int] +ASNDiff = tuple[list[bool], int, int] class _VarLenCoder: """ @@ -75,7 +76,7 @@ class _VarLenCoder: other classes start one past the last element of the class before it. """ - def __init__(self, minval: int, clsbits: List[int]): + def __init__(self, minval: int, clsbits: list[int]): """Construct a new _VarLenCoder.""" self._minval = minval self._clsbits = clsbits @@ -85,7 +86,7 @@ def can_encode(self, val: int) -> bool: """Check whether value val is in the range this coder supports.""" return self._minval <= val <= self._maxval - def encode(self, val: int, ret: List[int]) -> None: + def encode(self, val: int, ret: list[int]) -> None: """Append encoding of val onto integer list ret.""" assert self._minval <= val <= self._maxval @@ -120,7 +121,7 @@ def encode_size(self, val: int) -> int: break return ret + bits - def decode(self, stream, bitpos) -> Tuple[int,int]: + def decode(self, stream, bitpos) -> tuple[int,int]: """Decode a number starting at bitpos in stream, returning value and new bitpos.""" val = self._minval bits = 0 @@ -281,11 +282,11 @@ class ASMap: - mappings, represented by new trie nodes. """ - def update(self, prefix: List[bool], asn: int) -> None: + def update(self, prefix: list[bool], asn: int) -> None: """Update this ASMap object to map prefix to the specified asn.""" assert asn == 0 or _CODER_ASN.can_encode(asn) - def recurse(node: List, offset: int) -> None: + def recurse(node: list, offset: int) -> None: if offset == len(prefix): # Reached the end of prefix; overwrite this node. node.clear() @@ -306,7 +307,7 @@ def recurse(node: List, offset: int) -> None: node.append(oldasn) recurse(self._trie, 0) - def update_multi(self, entries: List[Tuple[List[bool], int]]) -> None: + def update_multi(self, entries: list[tuple[list[bool], int]]) -> None: """Apply multiple update operations, where longer prefixes take precedence.""" entries.sort(key=lambda entry: len(entry[0])) for prefix, asn in entries: @@ -314,7 +315,7 @@ def update_multi(self, entries: List[Tuple[List[bool], int]]) -> None: def _set_trie(self, trie) -> None: """Set trie directly. Internal use only.""" - def recurse(node: List) -> None: + def recurse(node: list) -> None: if len(node) < 2: return recurse(node[0]) @@ -342,7 +343,7 @@ def entry_key(entry): for prefix, asn in sorted(entries, key=entry_key): self.update(prefix, asn) - def lookup(self, prefix: List[bool]) -> Optional[int]: + def lookup(self, prefix: list[bool]) -> Optional[int]: """Look up a prefix. Returns ASN, or 0 if unassigned, or None if indeterminate.""" node = self._trie for bit in prefix: @@ -353,11 +354,11 @@ def lookup(self, prefix: List[bool]) -> Optional[int]: return node[0] return None - def _to_entries_flat(self, fill: bool = False) -> List[ASNEntry]: + def _to_entries_flat(self, fill: bool = False) -> list[ASNEntry]: """Convert an ASMap object to a list of non-overlapping (prefix, asn) objects.""" - prefix : List[bool] = [] + prefix : list[bool] = [] - def recurse(node: List) -> List[ASNEntry]: + def recurse(node: list) -> list[ASNEntry]: ret = [] if len(node) == 1: if node[0] > 0: @@ -375,24 +376,24 @@ def recurse(node: List) -> List[ASNEntry]: return ret return recurse(self._trie) - def _to_entries_minimal(self, fill: bool = False) -> List[ASNEntry]: + def _to_entries_minimal(self, fill: bool = False) -> list[ASNEntry]: """Convert a trie to a minimal list of ASNEntry objects, exploiting overlap.""" - prefix : List[bool] = [] + prefix : list[bool] = [] - def recurse(node: List) -> (Tuple[Dict[Optional[int], List[ASNEntry]], bool]): + def recurse(node: list) -> (tuple[dict[Optional[int], list[ASNEntry]], bool]): if len(node) == 1 and node[0] == 0: return {None if fill else 0: []}, True if len(node) == 1: return {node[0]: [], None: [(list(prefix), node[0])]}, False - ret: Dict[Optional[int], List[ASNEntry]] = {} + ret: dict[Optional[int], list[ASNEntry]] = {} prefix.append(False) left, lhole = recurse(node[0]) prefix[-1] = True right, rhole = recurse(node[1]) prefix.pop() hole = not fill and (lhole or rhole) - def candidate(ctx: Optional[int], res0: Optional[List[ASNEntry]], - res1: Optional[List[ASNEntry]]): + def candidate(ctx: Optional[int], res0: Optional[list[ASNEntry]], + res1: Optional[list[ASNEntry]]): if res0 is not None and res1 is not None: if ctx not in ret or len(res0) + len(res1) < len(ret[ctx]): ret[ctx] = res0 + res1 @@ -417,7 +418,7 @@ def __str__(self) -> str: """Convert this ASMap object to a string containing Python code constructing it.""" return f"ASMap({self._trie})" - def to_entries(self, overlapping: bool = True, fill: bool = False) -> List[ASNEntry]: + def to_entries(self, overlapping: bool = True, fill: bool = False) -> list[ASNEntry]: """ Convert the mappings in this ASMap object to a list of ASNEntry objects. @@ -448,7 +449,7 @@ def from_random(num_leaves: int = 10, max_asn: int = 6, assert max_asn >= 1 or unassigned_prob == 1 assert _CODER_ASN.can_encode(max_asn) assert 0.0 <= unassigned_prob <= 1.0 - trie: List = [] + trie: list = [] leaves = [trie] ret = ASMap() for i in range(1, num_leaves): @@ -472,12 +473,12 @@ def from_random(num_leaves: int = 10, max_asn: int = 6, def _to_binnode(self, fill: bool = False) -> _BinNode: """Convert a trie to a _BinNode object.""" - def recurse(node: List) -> Tuple[Dict[Optional[int], _BinNode], bool]: + def recurse(node: list) -> tuple[dict[Optional[int], _BinNode], bool]: if len(node) == 1 and node[0] == 0: return {(None if fill else 0): _BinNode.make_end()}, True if len(node) == 1: return {None: _BinNode.make_leaf(node[0]), node[0]: _BinNode.make_end()}, False - ret: Dict[Optional[int], _BinNode] = {} + ret: dict[Optional[int], _BinNode] = {} left, lhole = recurse(node[0]) right, rhole = recurse(node[1]) hole = (lhole or rhole) and not fill @@ -507,7 +508,7 @@ def candidate(ctx: Optional[int], arg1, arg2, func: Callable): @staticmethod def _from_binnode(binnode: _BinNode) -> "ASMap": """Construct an ASMap object from a _BinNode. Internal use only.""" - def recurse(node: _BinNode, default: int) -> List: + def recurse(node: _BinNode, default: int) -> list: if node.ins == _Instruction.RETURN: return [node.arg1] if node.ins == _Instruction.JUMP: @@ -542,7 +543,7 @@ def to_binary(self, fill: bool = False) -> bytes: Returns: A bytes object with the encoding of this ASMap object. """ - bits: List[int] = [] + bits: list[int] = [] def recurse(node: _BinNode) -> None: _CODER_INS.encode(node.ins.value, bits) @@ -582,11 +583,11 @@ def recurse(node: _BinNode) -> None: def from_binary(bindata: bytes) -> Optional["ASMap"]: """Decode an ASMap object from the provided binary encoding.""" - bits: List[int] = [] + bits: list[int] = [] for byte in bindata: bits.extend((byte >> i) & 1 for i in range(8)) - def recurse(bitpos: int) -> Tuple[_BinNode, int]: + def recurse(bitpos: int) -> tuple[_BinNode, int]: insval, bitpos = _CODER_INS.decode(bits, bitpos) ins = _Instruction(insval) if ins == _Instruction.RETURN: @@ -632,7 +633,7 @@ def __eq__(self, other: object) -> bool: def extends(self, req: "ASMap") -> bool: """Determine whether this matches req for all subranges where req is assigned.""" - def recurse(actual: List, require: List) -> bool: + def recurse(actual: list, require: list) -> bool: if len(require) == 1 and require[0] == 0: return True if len(require) == 1: @@ -646,20 +647,20 @@ def recurse(actual: List, require: List) -> bool: #pylint: disable=protected-access return recurse(self._trie, req._trie) - def diff(self, other: "ASMap") -> List[ASNDiff]: + def diff(self, other: "ASMap") -> list[ASNDiff]: """Compute the diff from self to other.""" - prefix: List[bool] = [] - ret: List[ASNDiff] = [] + prefix: list[bool] = [] + ret: list[ASNDiff] = [] - def recurse(old_node: List, new_node: List): + def recurse(old_node: list, new_node: list): if len(old_node) == 1 and len(new_node) == 1: if old_node[0] != new_node[0]: ret.append((list(prefix), old_node[0], new_node[0])) else: - old_left: List = old_node if len(old_node) == 1 else old_node[0] - old_right: List = old_node if len(old_node) == 1 else old_node[1] - new_left: List = new_node if len(new_node) == 1 else new_node[0] - new_right: List = new_node if len(new_node) == 1 else new_node[1] + old_left: list = old_node if len(old_node) == 1 else old_node[0] + old_right: list = old_node if len(old_node) == 1 else old_node[1] + new_left: list = new_node if len(new_node) == 1 else new_node[0] + new_right: list = new_node if len(new_node) == 1 else new_node[1] prefix.append(False) recurse(old_left, new_left) prefix[-1] = True @@ -760,7 +761,7 @@ def test_patching(self) -> None: # It starts off being equal to asmap. patched = copy.copy(asmap) # Keep a list of patches performed. - patches: List[ASNEntry] = [] + patches: list[ASNEntry] = [] # Initially there cannot be any difference. self.assertEqual(asmap.diff(patched), []) # Make 5 patches, each building on top of the previous ones. diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index af408c2df53ab..f03c2ab5e808f 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -11,7 +11,7 @@ import ipaddress import re import sys -from typing import List, Dict, Union +from typing import Union from asmap import ASMap, net_to_prefix @@ -117,14 +117,14 @@ def parseline(line: str) -> Union[dict, None]: 'sortkey': sortkey, } -def dedup(ips: List[Dict]) -> List[Dict]: +def dedup(ips: list[dict]) -> list[dict]: """ Remove duplicates from `ips` where multiple ips share address and port. """ d = {} for ip in ips: d[ip['ip'],ip['port']] = ip return list(d.values()) -def filtermultiport(ips: List[Dict]) -> List[Dict]: +def filtermultiport(ips: list[dict]) -> list[dict]: """ Filter out hosts with more nodes per IP""" hist = collections.defaultdict(list) for ip in ips: @@ -132,7 +132,7 @@ def filtermultiport(ips: List[Dict]) -> List[Dict]: return [value[0] for (key,value) in list(hist.items()) if len(value)==1] # Based on Greg Maxwell's seed_filter.py -def filterbyasn(asmap: ASMap, ips: List[Dict], max_per_asn: Dict, max_per_net: int) -> List[Dict]: +def filterbyasn(asmap: ASMap, ips: list[dict], max_per_asn: dict, max_per_net: int) -> list[dict]: """ Prunes `ips` by (a) trimming ips to have at most `max_per_net` ips from each net (e.g. ipv4, ipv6); and (b) trimming ips to have at most `max_per_asn` ips from each asn in each net. @@ -143,8 +143,8 @@ def filterbyasn(asmap: ASMap, ips: List[Dict], max_per_asn: Dict, max_per_net: i # Filter IPv46 by ASN, and limit to max_per_net per network result = [] - net_count: Dict[str, int] = collections.defaultdict(int) - asn_count: Dict[int, int] = collections.defaultdict(int) + net_count: dict[str, int] = collections.defaultdict(int) + asn_count: dict[int, int] = collections.defaultdict(int) for i, ip in enumerate(ips_ipv46): if net_count[ip['net']] == max_per_net: @@ -165,9 +165,9 @@ def filterbyasn(asmap: ASMap, ips: List[Dict], max_per_asn: Dict, max_per_net: i result.extend(ips_onion[0:max_per_net]) return result -def ip_stats(ips: List[Dict]) -> str: +def ip_stats(ips: list[dict]) -> str: """ Format and return pretty string from `ips`. """ - hist: Dict[str, int] = collections.defaultdict(int) + hist: dict[str, int] = collections.defaultdict(int) for ip in ips: if ip is not None: hist[ip['net']] += 1 diff --git a/contrib/signet/miner b/contrib/signet/miner index 61d9f62be7184..e5daf9f993eaa 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -30,7 +30,7 @@ logging.basicConfig( SIGNET_HEADER = b"\xec\xc7\xda\xa2" PSBT_SIGNET_BLOCK = b"\xfc\x06signetb" # proprietary PSBT global field holding the block being signed -RE_MULTIMINER = re.compile("^(\d+)(-(\d+))?/(\d+)$") +RE_MULTIMINER = re.compile(r"^(\d+)(-(\d+))?/(\d+)$") def create_coinbase(height, value, spk): cb = CTransaction() diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md index 206bec1647880..da9791764110c 100644 --- a/contrib/tracing/README.md +++ b/contrib/tracing/README.md @@ -30,12 +30,12 @@ information. For development there exist a [bpftrace Reference Guide], a ## Examples -The bpftrace examples contain a relative path to the `bitcoind` binary. By +The bpftrace examples contain a relative path to the `navcoind` binary. By default, the scripts should be run from the repository-root and assume a -self-compiled `bitcoind` binary. The paths in the examples can be changed, for +self-compiled `navcoind` binary. The paths in the examples can be changed, for example, to point to release builds if needed. See the [Bitcoin Core USDT documentation] on how to list available tracepoints in your -`bitcoind` binary. +`navcoind` binary. [Bitcoin Core USDT documentation]: ../../doc/tracing.md#listing-available-tracepoints @@ -82,7 +82,7 @@ about the connection. Peers can be selected individually to view recent P2P messages. ``` -$ python3 contrib/tracing/p2p_monitor.py ./src/bitcoind +$ python3 contrib/tracing/p2p_monitor.py ./src/navcoind ``` Lists selectable peers and traffic and connection information. @@ -150,7 +150,7 @@ lost. BCC prints: `Possibly lost 2 samples` on lost messages. ``` -$ python3 contrib/tracing/log_raw_p2p_msgs.py ./src/bitcoind +$ python3 contrib/tracing/log_raw_p2p_msgs.py ./src/navcoind ``` ``` @@ -188,7 +188,7 @@ In a different terminal, starting Bitcoin Core in SigNet mode and with re-indexing enabled. ``` -$ ./src/bitcoind -signet -reindex +$ ./src/navcoind -signet -reindex ``` This produces the following output. @@ -241,7 +241,7 @@ A BCC Python script to log the UTXO cache flushes. Based on the `utxocache:flush` tracepoint. ```bash -$ python3 contrib/tracing/log_utxocache_flush.py ./src/bitcoind +$ python3 contrib/tracing/log_utxocache_flush.py ./src/navcoind ``` ``` @@ -300,7 +300,7 @@ comprising a timestamp along with all event data available via the event's tracepoint. ```console -$ python3 contrib/tracing/mempool_monitor.py ./src/bitcoind +$ python3 contrib/tracing/mempool_monitor.py ./src/navcoind ``` ``` diff --git a/contrib/tracing/connectblock_benchmark.bt b/contrib/tracing/connectblock_benchmark.bt index 6e7a98ef07664..ae5d04ab4a91c 100755 --- a/contrib/tracing/connectblock_benchmark.bt +++ b/contrib/tracing/connectblock_benchmark.bt @@ -14,15 +14,15 @@ over . - Threshold - This script requires a 'bitcoind' binary compiled with eBPF support and the - 'validation:block_connected' USDT. By default, it's assumed that 'bitcoind' is - located in './src/bitcoind'. This can be modified in the script below. + This script requires a 'navcoind' binary compiled with eBPF support and the + 'validation:block_connected' USDT. By default, it's assumed that 'navcoind' is + located in './src/navcoind'. This can be modified in the script below. EXAMPLES: bpftrace contrib/tracing/connectblock_benchmark.bt 300000 680000 1000 - When run together 'bitcoind -reindex', this benchmarks the time it takes to + When run together 'navcoind -reindex', this benchmarks the time it takes to connect the blocks between height 300.000 and 680.000 (inclusive) and prints details about all blocks that take longer than 1000ms to connect. Prints a histogram with block connection times when the benchmark is finished. @@ -30,7 +30,7 @@ bpftrace contrib/tracing/connectblock_benchmark.bt 0 0 500 - When running together 'bitcoind', all newly connected blocks that + When running together 'navcoind', all newly connected blocks that take longer than 500ms to connect are logged. A histogram with block connection times is shown when the script is terminated. @@ -67,7 +67,7 @@ BEGIN connected block is between the start and end height (or the end height is unset). */ -usdt:./src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 || $2 == 0 )/ +usdt:./src/navcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 || $2 == 0 )/ { $height = arg1; $transactions = arg2; @@ -102,7 +102,7 @@ usdt:./src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 || $2 blocks where the time it took to connect the block is above the . */ -usdt:./src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / +usdt:./src/navcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / { $hash = arg0; $height = (int32) arg1; diff --git a/contrib/tracing/log_p2p_traffic.bt b/contrib/tracing/log_p2p_traffic.bt index f62956aa5e1c3..85dc8051fe9d9 100755 --- a/contrib/tracing/log_p2p_traffic.bt +++ b/contrib/tracing/log_p2p_traffic.bt @@ -5,7 +5,7 @@ BEGIN printf("Logging P2P traffic\n") } -usdt:./src/bitcoind:net:inbound_message +usdt:./src/navcoind:net:inbound_message { $peer_id = (int64) arg0; $peer_addr = str(arg1); @@ -15,7 +15,7 @@ usdt:./src/bitcoind:net:inbound_message printf("inbound '%s' msg from peer %d (%s, %s) with %d bytes\n", $msg_type, $peer_id, $peer_type, $peer_addr, $msg_len); } -usdt:./src/bitcoind:net:outbound_message +usdt:./src/navcoind:net:outbound_message { $peer_id = (int64) arg0; $peer_addr = str(arg1); diff --git a/contrib/tracing/log_raw_p2p_msgs.py b/contrib/tracing/log_raw_p2p_msgs.py index c0ab70410622b..dbf3374820c3b 100755 --- a/contrib/tracing/log_raw_p2p_msgs.py +++ b/contrib/tracing/log_raw_p2p_msgs.py @@ -132,15 +132,15 @@ def print_message(event, inbound): ) -def main(bitcoind_path): - bitcoind_with_usdts = USDT(path=str(bitcoind_path)) +def main(navcoind_path): + navcoind_with_usdts = USDT(path=str(navcoind_path)) # attaching the trace functions defined in the BPF program to the tracepoints - bitcoind_with_usdts.enable_probe( + navcoind_with_usdts.enable_probe( probe="inbound_message", fn_name="trace_inbound_message") - bitcoind_with_usdts.enable_probe( + navcoind_with_usdts.enable_probe( probe="outbound_message", fn_name="trace_outbound_message") - bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + bpf = BPF(text=program, usdt_contexts=[navcoind_with_usdts]) # BCC: perf buffer handle function for inbound_messages def handle_inbound(_, data, size): @@ -177,7 +177,7 @@ def handle_outbound(_, data, size): if __name__ == "__main__": if len(sys.argv) < 2: - print("USAGE:", sys.argv[0], "path/to/bitcoind") + print("USAGE:", sys.argv[0], "path/to/navcoind") exit() path = sys.argv[1] main(path) diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py index 6c568998e9923..20e3e2d0026a6 100755 --- a/contrib/tracing/log_utxocache_flush.py +++ b/contrib/tracing/log_utxocache_flush.py @@ -10,7 +10,7 @@ """Example logging Bitcoin Core utxo set cache flushes utilizing the utxocache:flush tracepoint.""" -# USAGE: ./contrib/tracing/log_utxocache_flush.py path/to/bitcoind +# USAGE: ./contrib/tracing/log_utxocache_flush.py path/to/navcoind # BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into # a sandboxed Linux kernel VM. @@ -70,14 +70,14 @@ def print_event(event): )) -def main(bitcoind_path): - bitcoind_with_usdts = USDT(path=str(bitcoind_path)) +def main(navcoind_path): + navcoind_with_usdts = USDT(path=str(navcoind_path)) # attaching the trace functions defined in the BPF program # to the tracepoints - bitcoind_with_usdts.enable_probe( + navcoind_with_usdts.enable_probe( probe="flush", fn_name="trace_flush") - b = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + b = BPF(text=program, usdt_contexts=[navcoind_with_usdts]) def handle_flush(_, data, size): """ Coins Flush handler. @@ -100,7 +100,7 @@ def handle_flush(_, data, size): if __name__ == "__main__": if len(sys.argv) < 2: - print("USAGE: ", sys.argv[0], "path/to/bitcoind") + print("USAGE: ", sys.argv[0], "path/to/navcoind") exit(1) path = sys.argv[1] diff --git a/contrib/tracing/log_utxos.bt b/contrib/tracing/log_utxos.bt index 54d5010f825e4..edda6f4fd07c8 100755 --- a/contrib/tracing/log_utxos.bt +++ b/contrib/tracing/log_utxos.bt @@ -6,9 +6,9 @@ bpftrace contrib/tracing/log_utxos.bt - This script requires a 'bitcoind' binary compiled with eBPF support and the - 'utxocache' tracepoints. By default, it's assumed that 'bitcoind' is - located in './src/bitcoind'. This can be modified in the script below. + This script requires a 'navcoind' binary compiled with eBPF support and the + 'utxocache' tracepoints. By default, it's assumed that 'navcoind' is + located in './src/navcoind'. This can be modified in the script below. NOTE: requires bpftrace v0.12.0 or above. */ @@ -22,7 +22,7 @@ BEGIN /* Attaches to the 'utxocache:add' tracepoint and prints additions to the UTXO set cache. */ -usdt:./src/bitcoind:utxocache:add +usdt:./src/navcoind:utxocache:add { $txid = arg0; $index = (uint32)arg1; @@ -44,7 +44,7 @@ usdt:./src/bitcoind:utxocache:add /* Attaches to the 'utxocache:spent' tracepoint and prints spents from the UTXO set cache. */ -usdt:./src/bitcoind:utxocache:spent +usdt:./src/navcoind:utxocache:spent { $txid = arg0; $index = (uint32)arg1; @@ -66,7 +66,7 @@ usdt:./src/bitcoind:utxocache:spent /* Attaches to the 'utxocache:uncache' tracepoint and uncache UTXOs from the UTXO set cache. */ -usdt:./src/bitcoind:utxocache:uncache +usdt:./src/navcoind:utxocache:uncache { $txid = arg0; $index = (uint32)arg1; diff --git a/contrib/tracing/mempool_monitor.py b/contrib/tracing/mempool_monitor.py index 9d427d4632a46..11c04127fcffe 100755 --- a/contrib/tracing/mempool_monitor.py +++ b/contrib/tracing/mempool_monitor.py @@ -27,7 +27,7 @@ struct added_event { u8 hash[HASH_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; }; @@ -35,7 +35,7 @@ { u8 hash[HASH_LENGTH]; char reason[MAX_REMOVAL_REASON_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; u64 entry_time; }; @@ -49,11 +49,11 @@ struct replaced_event { u8 replaced_hash[HASH_LENGTH]; - u64 replaced_vsize; + s32 replaced_vsize; s64 replaced_fee; u64 replaced_entry_time; u8 replacement_hash[HASH_LENGTH]; - u64 replacement_vsize; + s32 replacement_vsize; s64 replacement_fee; }; @@ -114,16 +114,16 @@ """ -def main(bitcoind_path): - bitcoind_with_usdts = USDT(path=str(bitcoind_path)) +def main(navcoind_path): + navcoind_with_usdts = USDT(path=str(navcoind_path)) # attaching the trace functions defined in the BPF program # to the tracepoints - bitcoind_with_usdts.enable_probe(probe="mempool:added", fn_name="trace_added") - bitcoind_with_usdts.enable_probe(probe="mempool:removed", fn_name="trace_removed") - bitcoind_with_usdts.enable_probe(probe="mempool:replaced", fn_name="trace_replaced") - bitcoind_with_usdts.enable_probe(probe="mempool:rejected", fn_name="trace_rejected") - bpf = BPF(text=PROGRAM, usdt_contexts=[bitcoind_with_usdts]) + navcoind_with_usdts.enable_probe(probe="mempool:added", fn_name="trace_added") + navcoind_with_usdts.enable_probe(probe="mempool:removed", fn_name="trace_removed") + navcoind_with_usdts.enable_probe(probe="mempool:replaced", fn_name="trace_replaced") + navcoind_with_usdts.enable_probe(probe="mempool:rejected", fn_name="trace_rejected") + bpf = BPF(text=PROGRAM, usdt_contexts=[navcoind_with_usdts]) events = [] @@ -365,7 +365,7 @@ def timestamp_age(timestamp): if __name__ == "__main__": if len(sys.argv) < 2: - print("USAGE: ", sys.argv[0], "path/to/bitcoind") + print("USAGE: ", sys.argv[0], "path/to/navcoind") exit(1) path = sys.argv[1] diff --git a/contrib/tracing/p2p_monitor.py b/contrib/tracing/p2p_monitor.py index 4ff701cac3c8c..58bcfeea8105c 100755 --- a/contrib/tracing/p2p_monitor.py +++ b/contrib/tracing/p2p_monitor.py @@ -3,7 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -""" Interactive bitcoind P2P network traffic monitor utilizing USDT and the +""" Interactive navcoind P2P network traffic monitor utilizing USDT and the net:inbound_message and net:outbound_message tracepoints. """ # This script demonstrates what USDT for Bitcoin Core can enable. It uses BCC @@ -115,17 +115,17 @@ def add_message(self, message): self.total_outbound_msgs += 1 -def main(bitcoind_path): +def main(navcoind_path): peers = dict() - bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + navcoind_with_usdts = USDT(path=str(navcoind_path)) # attaching the trace functions defined in the BPF program to the tracepoints - bitcoind_with_usdts.enable_probe( + navcoind_with_usdts.enable_probe( probe="inbound_message", fn_name="trace_inbound_message") - bitcoind_with_usdts.enable_probe( + navcoind_with_usdts.enable_probe( probe="outbound_message", fn_name="trace_outbound_message") - bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + bpf = BPF(text=program, usdt_contexts=[navcoind_with_usdts]) # BCC: perf buffer handle function for inbound_messages def handle_inbound(_, data, size): @@ -247,7 +247,7 @@ def render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_pa if __name__ == "__main__": if len(sys.argv) < 2: - print("USAGE:", sys.argv[0], "path/to/bitcoind") + print("USAGE:", sys.argv[0], "path/to/navcoind") exit() path = sys.argv[1] main(path) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index faba1f6ae6cbd..ee91acd5ef4b7 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -89,3 +89,9 @@ ... fun:_ZN5BCLog6Logger12StartLoggingEv } +{ + Suppress https://bugs.kde.org/show_bug.cgi?id=472219 - fixed in Valgrind 3.22. + Memcheck:Param + ppoll(ufds.events) + obj:/lib/ld-musl-aarch64.so.1 +} diff --git a/contrib/verify-binaries/README.md b/contrib/verify-binaries/README.md index c62d760e1a7ee..04d683e69b559 100644 --- a/contrib/verify-binaries/README.md +++ b/contrib/verify-binaries/README.md @@ -17,7 +17,7 @@ must obtain that key for your local GPG installation. You can obtain these keys by - through a browser using a key server (e.g. keyserver.ubuntu.com), - manually using the `gpg --keyserver --recv-keys ` command, or - - you can run the packaged `verify.py ... --import-keys` script to + - you can run the packaged `verify.py --import-keys ...` script to have it automatically retrieve unrecognized keys. #### Usage diff --git a/contrib/verify-binaries/verify.py b/contrib/verify-binaries/verify.py index d0749f503f865..12e6e10d8a4de 100755 --- a/contrib/verify-binaries/verify.py +++ b/contrib/verify-binaries/verify.py @@ -122,7 +122,7 @@ def download_with_wget(remote_file, local_file): return result.returncode == 0, result.stdout.decode().rstrip() -def download_lines_with_urllib(url) -> t.Tuple[bool, t.List[str]]: +def download_lines_with_urllib(url) -> tuple[bool, list[str]]: """Get (success, text lines of a file) over HTTP.""" try: return (True, [ @@ -138,7 +138,7 @@ def verify_with_gpg( filename, signature_filename, output_filename: t.Optional[str] = None -) -> t.Tuple[int, str]: +) -> tuple[int, str]: with tempfile.NamedTemporaryFile() as status_file: args = [ 'gpg', '--yes', '--verify', '--verify-options', 'show-primary-uid-only', "--status-file", status_file.name, @@ -177,12 +177,12 @@ def __repr__(self): def parse_gpg_result( - output: t.List[str] -) -> t.Tuple[t.List[SigData], t.List[SigData], t.List[SigData]]: + output: list[str] +) -> tuple[list[SigData], list[SigData], list[SigData]]: """Returns good, unknown, and bad signatures from GPG stdout.""" - good_sigs: t.List[SigData] = [] - unknown_sigs: t.List[SigData] = [] - bad_sigs: t.List[SigData] = [] + good_sigs: list[SigData] = [] + unknown_sigs: list[SigData] = [] + bad_sigs: list[SigData] = [] total_resolved_sigs = 0 # Ensure that all lines we match on include a prefix that prevents malicious input @@ -265,7 +265,7 @@ def files_are_equal(filename1, filename2): def get_files_from_hosts_and_compare( - hosts: t.List[str], path: str, filename: str, require_all: bool = False + hosts: list[str], path: str, filename: str, require_all: bool = False ) -> ReturnCode: """ Retrieve the same file from a number of hosts and ensure they have the same contents. @@ -326,7 +326,7 @@ def join_url(host: str) -> str: return ReturnCode.SUCCESS -def check_multisig(sums_file: str, sigfilename: str, args: argparse.Namespace) -> t.Tuple[int, str, t.List[SigData], t.List[SigData], t.List[SigData]]: +def check_multisig(sums_file: str, sigfilename: str, args: argparse.Namespace) -> tuple[int, str, list[SigData], list[SigData], list[SigData]]: # check signature # # We don't write output to a file because this command will almost certainly @@ -365,8 +365,8 @@ def prompt_yn(prompt) -> bool: def verify_shasums_signature( signature_file_path: str, sums_file_path: str, args: argparse.Namespace -) -> t.Tuple[ - ReturnCode, t.List[SigData], t.List[SigData], t.List[SigData], t.List[SigData] +) -> tuple[ + ReturnCode, list[SigData], list[SigData], list[SigData], list[SigData] ]: min_good_sigs = args.min_good_sigs gpg_allowed_codes = [0, 2] # 2 is returned when untrusted signatures are present. @@ -429,14 +429,14 @@ def verify_shasums_signature( return (ReturnCode.SUCCESS, good_trusted, good_untrusted, unknown, bad) -def parse_sums_file(sums_file_path: str, filename_filter: t.List[str]) -> t.List[t.List[str]]: +def parse_sums_file(sums_file_path: str, filename_filter: list[str]) -> list[list[str]]: # extract hashes/filenames of binaries to verify from hash file; # each line has the following format: " " with open(sums_file_path, 'r', encoding='utf8') as hash_file: return [line.split()[:2] for line in hash_file if len(filename_filter) == 0 or any(f in line for f in filename_filter)] -def verify_binary_hashes(hashes_to_verify: t.List[t.List[str]]) -> t.Tuple[ReturnCode, t.Dict[str, str]]: +def verify_binary_hashes(hashes_to_verify: list[list[str]]) -> tuple[ReturnCode, dict[str, str]]: offending_files = [] files_to_hashes = {} diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index d8087a4db387c..9cfa851d4b758 100755 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py @@ -7,7 +7,7 @@ ZMQ example using python3's asyncio Bitcoin should be started with the command line arguments: - bitcoind -testnet -daemon \ + navcoind -testnet -daemon \ -zmqpubrawtx=tcp://127.0.0.1:28332 \ -zmqpubrawblock=tcp://127.0.0.1:28332 \ -zmqpubhashtx=tcp://127.0.0.1:28332 \ diff --git a/depends/Makefile b/depends/Makefile index 68be151df33dc..3f335ac7ad8af 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -47,7 +47,7 @@ NO_HARDEN ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources C_STANDARD ?= c11 -CXX_STANDARD ?= c++17 +CXX_STANDARD ?= c++20 BUILD = $(shell ./config.guess) HOST ?= $(BUILD) @@ -145,8 +145,8 @@ include packages/packages.mk # 2. Before including packages/*.mk (excluding packages/packages.mk), since # they rely on the build_id variables # -build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') -$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR) 'NM='$(build_NM)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' NM='$(host_NM)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') boost_packages_$(NO_BOOST) = $(boost_packages) @@ -177,7 +177,7 @@ endif all_packages = $(packages) $(native_packages) -meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk +meta_depends = Makefile config.guess config.sub funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk $(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils) $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) @@ -236,7 +236,6 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS) $(host_$(release_type)_CXXFLAGS))|' \ -e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \ -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \ - -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \ -e 's|@no_zmq@|$(NO_ZMQ)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ -e 's|@no_bdb@|$(NO_BDB)|' \ diff --git a/depends/README.md b/depends/README.md index f7c0fc0a5fc3b..718ee620ebd44 100644 --- a/depends/README.md +++ b/depends/README.md @@ -15,6 +15,7 @@ For example: **Bitcoin Core's `configure` script by default will ignore the depends output.** In order for it to pick up libraries, tools, and settings from the depends build, you must set the `CONFIG_SITE` environment variable to point to a `config.site` settings file. +Make sure that `CONFIG_SITE` is an absolute path. In the above example, a file named `depends/x86_64-w64-mingw32/share/config.site` will be created. To use it during compilation: @@ -47,7 +48,7 @@ The paths are automatically configured and no other options are needed unless ta #### For macOS cross compilation - sudo apt-get install curl bsdmainutils cmake libz-dev python3-setuptools libtinfo5 xorriso + sudo apt-get install curl bsdmainutils cmake zip Note: You must obtain the macOS SDK before proceeding with a cross-compile. Under the depends directory, create a subdirectory named `SDKs`. @@ -97,7 +98,7 @@ The following can be set when running make: `make FOO=bar` - `SDK_PATH`: Path where SDKs can be found (used by macOS) - `FALLBACK_DOWNLOAD_PATH`: If a source file can't be fetched, try here before giving up - `C_STANDARD`: Set the C standard version used. Defaults to `c11`. -- `CXX_STANDARD`: Set the C++ standard version used. Defaults to `c++17`. +- `CXX_STANDARD`: Set the C++ standard version used. Defaults to `c++20`. - `NO_BOOST`: Don't download/build/cache Boost - `NO_LIBEVENT`: Don't download/build/cache Libevent - `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ @@ -107,9 +108,6 @@ The following can be set when running make: `make FOO=bar` - `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP - `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP - `NO_USDT`: Don't download/build/cache packages needed for enabling USDT tracepoints -- `ALLOW_HOST_PACKAGES`: Packages that are missed in dependencies (due to `NO_*` option or - build script logic) are searched for among the host system packages using - `pkg-config`. It allows building with packages of other (newer) versions - `MULTIPROCESS`: Build libmultiprocess (experimental, requires CMake) - `DEBUG`: Disable some optimizations and enable more runtime checking - `HOST_ID_SALT`: Optional salt to use when generating host package ids @@ -120,7 +118,7 @@ The following can be set when running make: `make FOO=bar` - `LOG`: Use file-based logging for individual packages. During a package build its log file resides in the `depends` directory, and the log file is printed out automatically in case of build error. After successful build log files are moved along with package archives -- `LTO`: Use LTO when building packages. +- `LTO`: Enable options needed for LTO. Does not add `-flto` related options to *FLAGS. - `NO_HARDEN=1`: Don't use hardening options when building packages If some packages are not built, for example `make NO_WALLET=1`, the appropriate diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 8ed82b276df9c..eb64c97f64780 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -11,8 +11,8 @@ build_darwin_SHA256SUM=shasum -a 256 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o #darwin host on darwin builder. overrides darwin host preferences. -darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot$(shell xcrun --show-sdk-path) -darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ -isysroot$(shell xcrun --show-sdk-path) +darwin_CC=$(shell xcrun -f clang) -isysroot$(shell xcrun --show-sdk-path) +darwin_CXX:=$(shell xcrun -f clang++) -stdlib=libc++ -isysroot$(shell xcrun --show-sdk-path) darwin_AR:=$(shell xcrun -f ar) darwin_RANLIB:=$(shell xcrun -f ranlib) darwin_STRIP:=$(shell xcrun -f strip) diff --git a/depends/config.guess b/depends/config.guess index 69188da73d743..cdfc4392047ce 100755 --- a/depends/config.guess +++ b/depends/config.guess @@ -4,7 +4,7 @@ # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2023-01-01' +timestamp='2023-08-22' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -47,7 +47,7 @@ me=`echo "$0" | sed -e 's,.*/,,'` usage="\ Usage: $0 [OPTION] -Output the configuration name of the system \`$me' is run on. +Output the configuration name of the system '$me' is run on. Options: -h, --help print this help, then exit @@ -66,7 +66,7 @@ This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -102,8 +102,8 @@ GUESS= # temporary files to be created and, as you can see below, it is a # headache to deal with in a portable fashion. -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. +# Historically, 'CC_FOR_BUILD' used to be named 'HOST_CC'. We still +# use 'HOST_CC' if defined, but it is deprecated. # Portable tmp directory creation inspired by the Autoconf team. @@ -155,6 +155,9 @@ Linux|GNU|GNU/*) set_cc_for_build cat <<-EOF > "$dummy.c" + #if defined(__ANDROID__) + LIBC=android + #else #include #if defined(__UCLIBC__) LIBC=uclibc @@ -169,6 +172,7 @@ Linux|GNU|GNU/*) LIBC=musl #endif #endif + #endif EOF cc_set_libc=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` eval "$cc_set_libc" @@ -459,7 +463,7 @@ case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in UNAME_RELEASE=`uname -v` ;; esac - # Japanese Language versions have a version number like `4.1.3-JL'. + # Japanese Language versions have a version number like '4.1.3-JL'. SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/-/_/'` GUESS=sparc-sun-sunos$SUN_REL ;; @@ -904,7 +908,7 @@ EOF fi ;; *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` + UNAME_PROCESSOR=`uname -p` case $UNAME_PROCESSOR in amd64) UNAME_PROCESSOR=x86_64 ;; @@ -976,7 +980,27 @@ EOF GUESS=$UNAME_MACHINE-unknown-minix ;; aarch64:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + set_cc_for_build + CPU=$UNAME_MACHINE + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __ARM_EABI__ + #ifdef __ARM_PCS_VFP + ABI=eabihf + #else + ABI=eabi + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + eabi | eabihf) CPU=armv8l; LIBCABI=$LIBC$ABI ;; + esac + fi + GUESS=$CPU-unknown-linux-$LIBCABI ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be @@ -1042,6 +1066,15 @@ EOF k1om:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; + kvx:Linux:*:*) + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; + kvx:cos:*:*) + GUESS=$UNAME_MACHINE-unknown-cos + ;; + kvx:mbr:*:*) + GUESS=$UNAME_MACHINE-unknown-mbr + ;; loongarch32:Linux:*:* | loongarch64:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; @@ -1197,7 +1230,7 @@ EOF GUESS=$UNAME_MACHINE-pc-sysv4.2uw$UNAME_VERSION ;; i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility + # If we were able to find 'uname', then EMX Unix compatibility # is probably installed. GUESS=$UNAME_MACHINE-pc-os2-emx ;; @@ -1338,7 +1371,7 @@ EOF GUESS=ns32k-sni-sysv fi ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + PENTIUM:*:4.0*:*) # Unisys 'ClearPath HMP IX 4000' SVR4/MP effort # says GUESS=i586-unisys-sysv4 ;; diff --git a/depends/config.site.in b/depends/config.site.in index 05c2ccbac104e..29b2a67ed0b0e 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -78,10 +78,6 @@ if test "@host_os@" = darwin; then BREW=no fi -if test -z "$enable_lto" && test -n "@lto@"; then - enable_lto=yes -fi - if test -z "$enable_hardening" && test -n "@no_harden@"; then enable_hardening=no fi @@ -89,9 +85,7 @@ fi PKG_CONFIG="$(which pkg-config) --static" PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig" -if test -z "@allow_host_packages@"; then - PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig" -fi +PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig" CPPFLAGS="-I${depends_prefix}/include/ ${CPPFLAGS}" LDFLAGS="-L${depends_prefix}/lib ${LDFLAGS}" @@ -102,7 +96,6 @@ fi if test -n "@CXX@" -a -z "${CXX}"; then CXX="@CXX@" fi -PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PATH_SEPARATOR}}${PYTHONPATH}" if test -n "@AR@"; then AR="@AR@" diff --git a/depends/config.sub b/depends/config.sub index de4259e404797..defe52c0c874b 100755 --- a/depends/config.sub +++ b/depends/config.sub @@ -4,7 +4,7 @@ # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2023-01-21' +timestamp='2023-09-19' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -82,7 +82,7 @@ This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -130,7 +130,7 @@ IFS=$saved_IFS # Separate into logical components for further validation case $1 in *-*-*-*-*) - echo Invalid configuration \`"$1"\': more than four components >&2 + echo "Invalid configuration '$1': more than four components" >&2 exit 1 ;; *-*-*-*) @@ -145,7 +145,8 @@ case $1 in nto-qnx* | linux-* | uclinux-uclibc* \ | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova* | managarm-*) + | storm-chaos* | os2-emx* | rtmk-nova* | managarm-* \ + | windows-* ) basic_machine=$field1 basic_os=$maybe_os ;; @@ -943,7 +944,7 @@ $basic_machine EOF IFS=$saved_IFS ;; - # We use `pc' rather than `unknown' + # We use 'pc' rather than 'unknown' # because (1) that's what they normally are, and # (2) the word "unknown" tends to confuse beginning users. i*86 | x86_64) @@ -1180,7 +1181,7 @@ case $cpu-$vendor in case $cpu in 1750a | 580 \ | a29k \ - | aarch64 | aarch64_be \ + | aarch64 | aarch64_be | aarch64c | arm64ec \ | abacus \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ @@ -1199,12 +1200,14 @@ case $cpu-$vendor in | d10v | d30v | dlx | dsp16xx \ | e2k | elxsi | epiphany \ | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | javascript \ | h8300 | h8500 \ | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ | i370 | i*86 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ + | kvx \ | le32 | le64 \ | lm32 \ | loongarch32 | loongarch64 \ @@ -1213,31 +1216,7 @@ case $cpu-$vendor in | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ | m88110 | m88k | maxq | mb | mcore | mep | metag \ | microblaze | microblazeel \ - | mips | mipsbe | mipseb | mipsel | mipsle \ - | mips16 \ - | mips64 | mips64eb | mips64el \ - | mips64octeon | mips64octeonel \ - | mips64orion | mips64orionel \ - | mips64r5900 | mips64r5900el \ - | mips64vr | mips64vrel \ - | mips64vr4100 | mips64vr4100el \ - | mips64vr4300 | mips64vr4300el \ - | mips64vr5000 | mips64vr5000el \ - | mips64vr5900 | mips64vr5900el \ - | mipsisa32 | mipsisa32el \ - | mipsisa32r2 | mipsisa32r2el \ - | mipsisa32r3 | mipsisa32r3el \ - | mipsisa32r5 | mipsisa32r5el \ - | mipsisa32r6 | mipsisa32r6el \ - | mipsisa64 | mipsisa64el \ - | mipsisa64r2 | mipsisa64r2el \ - | mipsisa64r3 | mipsisa64r3el \ - | mipsisa64r5 | mipsisa64r5el \ - | mipsisa64r6 | mipsisa64r6el \ - | mipsisa64sb1 | mipsisa64sb1el \ - | mipsisa64sr71k | mipsisa64sr71kel \ - | mipsr5900 | mipsr5900el \ - | mipstx39 | mipstx39el \ + | mips* \ | mmix \ | mn10200 | mn10300 \ | moxie \ @@ -1285,7 +1264,7 @@ case $cpu-$vendor in ;; *) - echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + echo "Invalid configuration '$1': machine '$cpu-$vendor' not recognized" 1>&2 exit 1 ;; esac @@ -1306,11 +1285,12 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if test x$basic_os != x +if test x"$basic_os" != x then # First recognize some ad-hoc cases, or perhaps split kernel-os, or else just # set os. +obj= case $basic_os in gnu/linux*) kernel=linux @@ -1510,10 +1490,16 @@ case $os in os=eabi ;; *) - os=elf + os= + obj=elf ;; esac ;; + aout* | coff* | elf* | pe*) + # These are machine code file formats, not OSes + obj=$os + os= + ;; *) # No normalization, but not necessarily accepted, that comes below. ;; @@ -1532,12 +1518,15 @@ else # system, and we'll never get to this point. kernel= +obj= case $cpu-$vendor in score-*) - os=elf + os= + obj=elf ;; spu-*) - os=elf + os= + obj=elf ;; *-acorn) os=riscix1.2 @@ -1547,28 +1536,35 @@ case $cpu-$vendor in os=gnu ;; arm*-semi) - os=aout + os= + obj=aout ;; c4x-* | tic4x-*) - os=coff + os= + obj=coff ;; c8051-*) - os=elf + os= + obj=elf ;; clipper-intergraph) os=clix ;; hexagon-*) - os=elf + os= + obj=elf ;; tic54x-*) - os=coff + os= + obj=coff ;; tic55x-*) - os=coff + os= + obj=coff ;; tic6x-*) - os=coff + os= + obj=coff ;; # This must come before the *-dec entry. pdp10-*) @@ -1590,19 +1586,24 @@ case $cpu-$vendor in os=sunos3 ;; m68*-cisco) - os=aout + os= + obj=aout ;; mep-*) - os=elf + os= + obj=elf ;; mips*-cisco) - os=elf + os= + obj=elf ;; mips*-*) - os=elf + os= + obj=elf ;; or32-*) - os=coff + os= + obj=coff ;; *-tti) # must be before sparc entry or we get the wrong os. os=sysv3 @@ -1611,7 +1612,8 @@ case $cpu-$vendor in os=sunos4.1.1 ;; pru-*) - os=elf + os= + obj=elf ;; *-be) os=beos @@ -1692,10 +1694,12 @@ case $cpu-$vendor in os=uxpv ;; *-rom68k) - os=coff + os= + obj=coff ;; *-*bug) - os=coff + os= + obj=coff ;; *-apple) os=macos @@ -1713,7 +1717,8 @@ esac fi -# Now, validate our (potentially fixed-up) OS. +# Now, validate our (potentially fixed-up) individual pieces (OS, OBJ). + case $os in # Sometimes we do "kernel-libc", so those need to count as OSes. musl* | newlib* | relibc* | uclibc*) @@ -1724,6 +1729,9 @@ case $os in # VxWorks passes extra cpu info in the 4th filed. simlinux | simwindows | spe) ;; + # See `case $cpu-$os` validation below + ghcjs) + ;; # Now accept the basic system types. # The portable systems comes first. # Each alternative MUST end in a * to match a version number. @@ -1732,7 +1740,7 @@ case $os in | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ | sym* | plan9* | psp* | sim* | xray* | os68k* | v88r* \ | hiux* | abug | nacl* | netware* | windows* \ - | os9* | macos* | osx* | ios* \ + | os9* | macos* | osx* | ios* | tvos* | watchos* \ | mpw* | magic* | mmixware* | mon960* | lnews* \ | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ | aos* | aros* | cloudabi* | sortix* | twizzler* \ @@ -1741,11 +1749,11 @@ case $os in | mirbsd* | netbsd* | dicos* | openedition* | ose* \ | bitrig* | openbsd* | secbsd* | solidbsd* | libertybsd* | os108* \ | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \ - | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ - | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | bosx* | nextstep* | cxux* | oabi* \ + | ptx* | ecoff* | winnt* | domain* | vsta* \ | udi* | lites* | ieee* | go32* | aux* | hcos* \ | chorusrdb* | cegcc* | glidix* | serenity* \ - | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | cygwin* | msys* | moss* | proelf* | rtems* \ | midipix* | mingw32* | mingw64* | mint* \ | uxpv* | beos* | mpeix* | udk* | moxiebox* \ | interix* | uwin* | mks* | rhapsody* | darwin* \ @@ -1758,7 +1766,7 @@ case $os in | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx* | zephyr* \ - | fiwix* | mlibc* ) + | fiwix* | mlibc* | cos* | mbr* ) ;; # This one is extra strict with allowed versions sco3.2v2 | sco3.2v[4-9]* | sco5v6*) @@ -1766,54 +1774,99 @@ case $os in ;; none) ;; - kernel* ) + kernel* | msvc* ) # Restricted further below ;; + '') + if test x"$obj" = x + then + echo "Invalid configuration '$1': Blank OS only allowed with explicit machine code file format" 1>&2 + fi + ;; + *) + echo "Invalid configuration '$1': OS '$os' not recognized" 1>&2 + exit 1 + ;; +esac + +case $obj in + aout* | coff* | elf* | pe*) + ;; + '') + # empty is fine + ;; *) - echo Invalid configuration \`"$1"\': OS \`"$os"\' not recognized 1>&2 + echo "Invalid configuration '$1': Machine code format '$obj' not recognized" 1>&2 + exit 1 + ;; +esac + +# Here we handle the constraint that a (synthetic) cpu and os are +# valid only in combination with each other and nowhere else. +case $cpu-$os in + # The "javascript-unknown-ghcjs" triple is used by GHC; we + # accept it here in order to tolerate that, but reject any + # variations. + javascript-ghcjs) + ;; + javascript-* | *-ghcjs) + echo "Invalid configuration '$1': cpu '$cpu' is not valid with os '$os$obj'" 1>&2 exit 1 ;; esac # As a final step for OS-related things, validate the OS-kernel combination # (given a valid OS), if there is a kernel. -case $kernel-$os in - linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* \ - | linux-musl* | linux-relibc* | linux-uclibc* | linux-mlibc* ) +case $kernel-$os-$obj in + linux-gnu*- | linux-dietlibc*- | linux-android*- | linux-newlib*- \ + | linux-musl*- | linux-relibc*- | linux-uclibc*- | linux-mlibc*- ) ;; - uclinux-uclibc* ) + uclinux-uclibc*- ) ;; - managarm-mlibc* | managarm-kernel* ) + managarm-mlibc*- | managarm-kernel*- ) ;; - -dietlibc* | -newlib* | -musl* | -relibc* | -uclibc* | -mlibc* ) + windows*-msvc*-) + ;; + -dietlibc*- | -newlib*- | -musl*- | -relibc*- | -uclibc*- | -mlibc*- ) # These are just libc implementations, not actual OSes, and thus # require a kernel. - echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 + echo "Invalid configuration '$1': libc '$os' needs explicit kernel." 1>&2 exit 1 ;; - -kernel* ) - echo "Invalid configuration \`$1': \`$os' needs explicit kernel." 1>&2 + -kernel*- ) + echo "Invalid configuration '$1': '$os' needs explicit kernel." 1>&2 exit 1 ;; - *-kernel* ) - echo "Invalid configuration \`$1': \`$kernel' does not support \`$os'." 1>&2 + *-kernel*- ) + echo "Invalid configuration '$1': '$kernel' does not support '$os'." 1>&2 exit 1 ;; - kfreebsd*-gnu* | kopensolaris*-gnu*) + *-msvc*- ) + echo "Invalid configuration '$1': '$os' needs 'windows'." 1>&2 + exit 1 ;; - vxworks-simlinux | vxworks-simwindows | vxworks-spe) + kfreebsd*-gnu*- | kopensolaris*-gnu*-) ;; - nto-qnx*) + vxworks-simlinux- | vxworks-simwindows- | vxworks-spe-) ;; - os2-emx) + nto-qnx*-) + ;; + os2-emx-) + ;; + *-eabi*- | *-gnueabi*-) ;; - *-eabi* | *-gnueabi*) + none--*) + # None (no kernel, i.e. freestanding / bare metal), + # can be paired with an machine code file format ;; - -*) + -*-) # Blank kernel with real OS is always fine. ;; - *-*) - echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 + --*) + # Blank kernel and OS with real machine code file format is always fine. + ;; + *-*-*) + echo "Invalid configuration '$1': Kernel '$kernel' not known to work with OS '$os'." 1>&2 exit 1 ;; esac @@ -1896,7 +1949,7 @@ case $vendor in ;; esac -echo "$cpu-$vendor-${kernel:+$kernel-}$os" +echo "$cpu-$vendor${kernel:+-$kernel}${os:+-$os}${obj:+-$obj}" exit # Local variables: diff --git a/depends/description.md b/depends/description.md index 0a6f2e6442190..69ee5bd36ccd8 100644 --- a/depends/description.md +++ b/depends/description.md @@ -6,8 +6,7 @@ There are several features that make it different from most similar systems: In theory, binaries for any target OS/architecture can be created, from a builder running any OS/architecture. In practice, build-side tools must be specified when the defaults don't fit, and packages must be amended to work -on new hosts. For now, a build architecture of x86_64 is assumed, either on -Linux or macOS. +on new hosts. ### No reliance on timestamps @@ -28,7 +27,7 @@ etc), and as well as a hash of the same data for each recursive dependency. If any portion of a package's build recipe changes, it will be rebuilt as well as any other package that depends on it. If any of the main makefiles (Makefile, funcs.mk, etc) are changed, all packages will be rebuilt. After building, the -results are cached into a tarball that can be re-used and distributed. +results are cached into a tarball that can be reused and distributed. ### Package build results are (relatively) deterministic. diff --git a/depends/gen_id b/depends/gen_id index 3341310e460a8..8518b4e67442e 100755 --- a/depends/gen_id +++ b/depends/gen_id @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Usage: env [ CC=... ] [ C_STANDARD=...] [ CXX=... ] [CXX_STANDARD=...] \ -# [ AR=... ] [ RANLIB=... ] [ STRIP=... ] [ DEBUG=... ] \ +# [ AR=... ] [ NM=... ] [ RANLIB=... ] [ STRIP=... ] [ DEBUG=... ] \ # [ LTO=... ] [ NO_HARDEN=... ] ./build-id [ID_SALT]... # # Prints to stdout a SHA256 hash representing the current toolset, used by @@ -56,6 +56,11 @@ echo "ZERO_AR_DATE=${ZERO_AR_DATE}" echo "END AR" + echo "BEGIN NM" + bash -c "${NM} --version" + env | grep '^NM_' + echo "END NM" + echo "BEGIN RANLIB" bash -c "${RANLIB} --version" env | grep '^RANLIB_' diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk index db32d02fe84a9..32e3045bc5864 100644 --- a/depends/hosts/android.mk +++ b/depends/hosts/android.mk @@ -11,11 +11,6 @@ endif android_CFLAGS=-std=$(C_STANDARD) android_CXXFLAGS=-std=$(CXX_STANDARD) -ifneq ($(LTO),) -android_CFLAGS += -flto -android_LDFLAGS += -flto -endif - android_AR=$(ANDROID_TOOLCHAIN_BIN)/llvm-ar android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/llvm-ranlib android_NM=$(ANDROID_TOOLCHAIN_BIN)/llvm-nm diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 522a6b17efc15..b94ac7d56fea2 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,8 +1,8 @@ -OSX_MIN_VERSION=10.15 -OSX_SDK_VERSION=11.0 -XCODE_VERSION=12.2 -XCODE_BUILD_ID=12B45b -LD64_VERSION=609 +OSX_MIN_VERSION=11.0 +OSX_SDK_VERSION=14.0 +XCODE_VERSION=15.0 +XCODE_BUILD_ID=15A240d +LD64_VERSION=711 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers @@ -19,7 +19,6 @@ clang_prog=$(build_prefix)/bin/clang clangxx_prog=$(clang_prog)++ llvm_config_prog=$(build_prefix)/bin/llvm-config -clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version) else # FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's # system @@ -37,7 +36,6 @@ clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config") -clang_resource_dir=$(shell clang -print-resource-dir) llvm_lib_dir=$(shell $(llvm_config_prog) --libdir) endif @@ -63,62 +61,45 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$( # Explicitly point to our binaries (e.g. cctools) so that they are # ensured to be found and preferred over other possibilities. # -# -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 +# -isysroot$(OSX_SDK) -nostdlibinc # -# Forces clang to use the libc++ headers from our SDK and completely -# forget about the libc++ headers from the standard directories +# Disable default include paths built into the compiler as well as +# those normally included for libc and libc++. The only path that +# remains implicitly is the clang resource dir. # -# -Xclang -*system \ -# -Xclang -*system \ -# -Xclang -*system ... +# -iwithsysroot / -iframeworkwithsysroot # -# Adds path_a, path_b, and path_c to the bottom of clang's list of -# include search paths. This is used to explicitly specify the list of -# system include search paths and its ordering, rather than rely on -# clang's autodetection routine. This routine has been shown to: -# 1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1 -# when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034) -# 2. Fail to pickup C headers in $SYSROOT/usr/include when -# C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9) +# Adds the desired paths from the SDK # -# Talking directly to cc1 with -Xclang here grants us access to specify -# more granular categories for these system include search paths, and we -# can use the correct categories that these search paths would have been -# placed in if the autodetection routine had worked correctly. (see: -# https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment) -# -# Furthermore, it places these search paths after any "non-Xclang" -# specified search paths. This prevents any additional clang options or -# environment variables from coming after or in between these system -# include search paths, as that would be wrong in general but would also -# break #include_next's. +# -platform_version # +# Indicate to the linker the platform, the oldest supported version, +# and the SDK used. + darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ - $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ - -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ - -isysroot$(OSX_SDK) \ - -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ - -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include + $(clang_prog) --target=$(host) \ + -B$(build_prefix)/bin \ + -isysroot$(OSX_SDK) -nostdlibinc \ + -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks + darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ - $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ - -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ - -isysroot$(OSX_SDK) \ - -stdlib=libc++ \ - -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \ - -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ - -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include - -darwin_CFLAGS=-pipe -std=$(C_STANDARD) -darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) - -ifneq ($(LTO),) -darwin_CFLAGS += -flto -darwin_CXXFLAGS += -flto -darwin_LDFLAGS += -flto + $(clangxx_prog) --target=$(host) \ + -B$(build_prefix)/bin \ + -isysroot$(OSX_SDK) -nostdlibinc \ + -iwithsysroot/usr/include/c++/v1 \ + -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks + +darwin_CFLAGS=-pipe -std=$(C_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSION) +darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSION) +darwin_LDFLAGS=-Wl,-platform_version,macos,$(OSX_MIN_VERSION),$(OSX_SDK_VERSION) + +ifneq ($(build_os),darwin) +darwin_CFLAGS += -mlinker-version=$(LD64_VERSION) +darwin_CXXFLAGS += -mlinker-version=$(LD64_VERSION) endif darwin_release_CFLAGS=-O2 diff --git a/depends/hosts/freebsd.mk b/depends/hosts/freebsd.mk index 5351d0b90095b..055097b03ddc8 100644 --- a/depends/hosts/freebsd.mk +++ b/depends/hosts/freebsd.mk @@ -1,12 +1,6 @@ freebsd_CFLAGS=-pipe -std=$(C_STANDARD) freebsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -ifneq ($(LTO),) -freebsd_CFLAGS += -flto -freebsd_CXXFLAGS += -flto -freebsd_LDFLAGS += -flto -endif - freebsd_release_CFLAGS=-O2 freebsd_release_CXXFLAGS=$(freebsd_release_CFLAGS) diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index 0e2496174e302..8be23be57db03 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -2,10 +2,6 @@ linux_CFLAGS=-pipe -std=$(C_STANDARD) linux_CXXFLAGS=-pipe -std=$(CXX_STANDARD) ifneq ($(LTO),) -linux_CFLAGS += -flto -linux_CXXFLAGS += -flto -linux_LDFLAGS += -flto - linux_AR = $(host_toolchain)gcc-ar linux_NM = $(host_toolchain)gcc-nm linux_RANLIB = $(host_toolchain)gcc-ranlib @@ -17,7 +13,7 @@ linux_release_CXXFLAGS=$(linux_release_CFLAGS) linux_debug_CFLAGS=-O1 linux_debug_CXXFLAGS=$(linux_debug_CFLAGS) -linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -D_LIBCPP_ENABLE_ASSERTIONS=1 +linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -D_LIBCPP_ENABLE_DEBUG_MODE=1 ifeq (86,$(findstring 86,$(build_arch))) i686_linux_CC=gcc -m32 diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index fc1cc1afbe5cc..15aa7cd25ad05 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -6,10 +6,6 @@ mingw32_CFLAGS=-pipe -std=$(C_STANDARD) mingw32_CXXFLAGS=-pipe -std=$(CXX_STANDARD) ifneq ($(LTO),) -mingw32_CFLAGS += -flto -mingw32_CXXFLAGS += -flto -mingw32_LDFLAGS += -flto - mingw32_AR = $(host_toolchain)gcc-ar mingw32_NM = $(host_toolchain)gcc-nm mingw32_RANLIB = $(host_toolchain)gcc-ranlib diff --git a/depends/hosts/netbsd.mk b/depends/hosts/netbsd.mk index 14121dca20f32..f33b2d2889571 100644 --- a/depends/hosts/netbsd.mk +++ b/depends/hosts/netbsd.mk @@ -2,10 +2,6 @@ netbsd_CFLAGS=-pipe -std=$(C_STANDARD) netbsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) ifneq ($(LTO),) -netbsd_CFLAGS += -flto -netbsd_CXXFLAGS += -flto -netbsd_LDFLAGS += -flto - netbsd_AR = $(host_toolchain)gcc-ar netbsd_NM = $(host_toolchain)gcc-nm netbsd_RANLIB = $(host_toolchain)gcc-ranlib diff --git a/depends/hosts/openbsd.mk b/depends/hosts/openbsd.mk index d330e94d2ed2d..bdd36dc9b35de 100644 --- a/depends/hosts/openbsd.mk +++ b/depends/hosts/openbsd.mk @@ -1,12 +1,6 @@ openbsd_CFLAGS=-pipe -std=$(C_STANDARD) openbsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -ifneq ($(LTO),) -openbsd_CFLAGS += -flto -openbsd_CXXFLAGS += -flto -openbsd_LDFLAGS += -flto -endif - openbsd_release_CFLAGS=-O2 openbsd_release_CXXFLAGS=$(openbsd_release_CFLAGS) diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index ebc097d686f58..ab43764b38f27 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -3,6 +3,11 @@ $(package)_version=1.81.0 $(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ $(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.bz2 $(package)_sha256_hash=71feeed900fbccca04a3b4f2f84a7c217186f28a940ed8b7ed4725986baf99fa +$(package)_patches=process_macos_sdk.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/process_macos_sdk.patch +endef define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/include && \ diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index f4778c1ecdc32..2465c8091bbeb 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -4,17 +4,20 @@ $(package)_download_path=$(native_$(package)_download_path) $(package)_download_file=$(native_$(package)_download_file) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) -$(package)_dependencies=native_$(package) +# Hardcode library install path to "lib" to match the PKG_CONFIG_PATH +# setting in depends/config.site.in, which also hardcodes "lib". +# Without this setting, cmake by default would use the OS library +# directory, which might be "lib64" or something else, not "lib", on multiarch systems. define $(package)_set_vars := -$(package)_config_opts := --with-external-capnp -$(package)_config_opts += CAPNP="$$(native_capnp_prefixbin)/capnp" -$(package)_config_opts += CAPNP_CXX="$$(native_capnp_prefixbin)/capnp-c++" -$(package)_config_opts_android := --disable-shared + $(package)_config_opts := -DBUILD_TESTING=OFF + $(package)_config_opts += -DWITH_OPENSSL=OFF + $(package)_config_opts += -DWITH_ZLIB=OFF + $(package)_config_opts += -DCMAKE_INSTALL_LIBDIR=lib/ endef define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_cmake) . endef define $(package)_build_cmds @@ -24,3 +27,7 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm -rf lib/pkgconfig +endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index 765d6493776bb..d237f52dbb237 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -8,7 +8,13 @@ ifneq ($(host),$(build)) $(package)_dependencies += native_capnp endif +# Hardcode library install path to "lib" to match the PKG_CONFIG_PATH +# setting in depends/config.site.in, which also hardcodes "lib". +# Without this setting, cmake by default would use the OS library +# directory, which might be "lib64" or something else, not "lib", on multiarch systems. define $(package)_set_vars := +$(package)_config_opts += -DCMAKE_INSTALL_LIBDIR=lib/ +$(package)_config_opts += -DCMAKE_POSITION_INDEPENDENT_CODE=ON ifneq ($(host),$(build)) $(package)_config_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp" $(package)_config_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++" diff --git a/depends/packages/native_capnp.mk b/depends/packages/native_capnp.mk index ed5a6deee2fdb..484e78d5d906b 100644 --- a/depends/packages/native_capnp.mk +++ b/depends/packages/native_capnp.mk @@ -1,12 +1,18 @@ package=native_capnp -$(package)_version=0.7.0 +$(package)_version=1.0.1 $(package)_download_path=https://capnproto.org/ $(package)_download_file=capnproto-c++-$($(package)_version).tar.gz $(package)_file_name=capnproto-cxx-$($(package)_version).tar.gz -$(package)_sha256_hash=c9a4c0bd88123064d483ab46ecee777f14d933359e23bff6fb4f4dbd28b4cd41 +$(package)_sha256_hash=0f7f4b8a76a2cdb284fddef20de8306450df6dd031a47a15ac95bc43c3358e09 + +define $(package)_set_vars + $(package)_config_opts := -DBUILD_TESTING=OFF + $(package)_config_opts += -DWITH_OPENSSL=OFF + $(package)_config_opts += -DWITH_ZLIB=OFF +endef define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_cmake) . endef define $(package)_build_cmds @@ -16,3 +22,7 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm -rf lib/pkgconfig +endef diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 03e9002ecd759..3148e51048c5c 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,8 +1,8 @@ package=native_cctools -$(package)_version=2ef2e931cf641547eb8a68cfebde61003587c9fd +$(package)_version=c74fafe86076713cb8e6f937af43b6df6da1f42d $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=6b73269efdf5c58a070e7357b66ee760501388549d6a12b423723f45888b074b +$(package)_sha256_hash=e2c1588d505a69c32e079f4e616e0f117d5478429040e394f624f43f2796e6bc $(package)_build_subdir=cctools $(package)_dependencies=native_libtapi @@ -17,13 +17,9 @@ endef ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_preprocess_cmds mkdir -p $($(package)_staging_prefix_dir)/lib && \ - cp $(llvm_lib_dir)/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools + cp $(llvm_lib_dir)/libLTO.so $($(package)_staging_prefix_dir)/lib/ endef else -define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools -endef endif define $(package)_config_cmds diff --git a/depends/packages/native_ds_store.mk b/depends/packages/native_ds_store.mk deleted file mode 100644 index 51a95f48ef7b1..0000000000000 --- a/depends/packages/native_ds_store.mk +++ /dev/null @@ -1,15 +0,0 @@ -package=native_ds_store -$(package)_version=1.3.0 -$(package)_download_path=https://github.com/dmgbuild/ds_store/archive/ -$(package)_file_name=v$($(package)_version).tar.gz -$(package)_sha256_hash=76b3280cd4e19e5179defa23fb594a9dd32643b0c80d774bd3108361d94fb46d -$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages - -define $(package)_build_cmds - python3 setup.py build -endef - -define $(package)_stage_cmds - mkdir -p $($(package)_install_libdir) && \ - python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) -endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index e647afba5f35c..946e885354aac 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,8 @@ package=native_libmultiprocess -$(package)_version=1af83d15239ccfa7e47b8764029320953dd7fdf1 +$(package)_version=414542f81e0997354b45b8ade13ca144a3e35ff1 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=e5587d3feedc7f8473f178a89b94163a11076629825d664964799bbbd5844da5 +$(package)_sha256_hash=8542dbaf8c4fce8fd7af6929f5dc9b34dffa51c43e9ee360e93ee0f34b180bc2 $(package)_dependencies=native_capnp define $(package)_config_cmds diff --git a/depends/packages/native_libtapi.mk b/depends/packages/native_libtapi.mk index 052bb2368933a..fb5ab0b4dccc6 100644 --- a/depends/packages/native_libtapi.mk +++ b/depends/packages/native_libtapi.mk @@ -1,13 +1,18 @@ package=native_libtapi -$(package)_version=664b8414f89612f2dfd35a9b679c345aa5389026 +$(package)_version=eb33a59f2e30ff9724dc1ea8bee8b5229b0557c9 $(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=62e419c12d1c9fad67cc1cd523132bc00db050998337c734c15bc8d73cc02b61 +$(package)_sha256_hash=d4d46c64622f13d6938cecf989046d9561011bb59e8ee835f8f39825d67f578f +$(package)_patches=disable_zlib.patch ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -$(package)_dependencies=native_clang +$(package)_dependencies=native_llvm endif +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/disable_zlib.patch +endef + define $(package)_build_cmds CC=$(clang_prog) CXX=$(clangxx_prog) INSTALLPREFIX=$($(package)_staging_prefix_dir) ./build.sh endef diff --git a/depends/packages/native_clang.mk b/depends/packages/native_llvm.mk similarity index 67% rename from depends/packages/native_clang.mk rename to depends/packages/native_llvm.mk index f2712294ab20e..09994eb01296a 100644 --- a/depends/packages/native_clang.mk +++ b/depends/packages/native_llvm.mk @@ -1,20 +1,17 @@ -package=native_clang -$(package)_version=10.0.1 +package=native_llvm +$(package)_version=17.0.6 +$(package)_major_version=$(firstword $(subst ., ,$($(package)_version))) $(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) ifneq (,$(findstring aarch64,$(BUILD))) $(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz -$(package)_sha256_hash=90dc69a4758ca15cd0ffa45d07fbf5bf4309d47d2c7745a9f0735ecffde9c31f +$(package)_sha256_hash=6dd62762285326f223f40b8e4f2864b5c372de3f7de0731cb7cd55ca5287b75a else -$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz -$(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231 +$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-22.04.tar.xz +$(package)_sha256_hash=884ee67d647d77e58740c1e645649e29ae9e8a6fe87c1376be0f3a30f3cc9ab3 endif -define $(package)_preprocess_cmds - rm -f $($(package)_extract_dir)/lib/libc++abi.so* -endef - define $(package)_stage_cmds - mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \ + mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin && \ mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \ @@ -24,5 +21,5 @@ define $(package)_stage_cmds cp include/llvm-c/ExternC.h $($(package)_staging_prefix_dir)/include/llvm-c && \ cp include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c && \ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -r lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ + cp -r lib/clang/$($(package)_major_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include/ endef diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk deleted file mode 100644 index ddd631186edf7..0000000000000 --- a/depends/packages/native_mac_alias.mk +++ /dev/null @@ -1,15 +0,0 @@ -package=native_mac_alias -$(package)_version=2.2.0 -$(package)_download_path=https://github.com/dmgbuild/mac_alias/archive/ -$(package)_file_name=v$($(package)_version).tar.gz -$(package)_sha256_hash=421e6d7586d1f155c7db3e7da01ca0dacc9649a509a253ad7077b70174426499 -$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages - -define $(package)_build_cmds - python3 setup.py build -endef - -define $(package)_stage_cmds - mkdir -p $($(package)_install_libdir) && \ - python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index bb9476f9b6b02..541f56a28498d 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -17,13 +17,13 @@ multiprocess_native_packages = native_libmultiprocess native_capnp usdt_linux_packages=systemtap -darwin_native_packages = native_ds_store native_mac_alias +darwin_native_packages = ifneq ($(build_os),darwin) darwin_native_packages += native_cctools native_libtapi ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -darwin_native_packages+= native_clang +darwin_native_packages+= native_llvm endif endif diff --git a/depends/packages/systemtap.mk b/depends/packages/systemtap.mk index 541ebeee01408..c912e18c31e77 100644 --- a/depends/packages/systemtap.mk +++ b/depends/packages/systemtap.mk @@ -3,11 +3,10 @@ $(package)_version=4.8 $(package)_download_path=https://sourceware.org/ftp/systemtap/releases/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=cbd50a4eba5b261394dc454c12448ddec73e55e6742fda7f508f9fbc1331c223 -$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch fix_variadic_warning.patch +$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch && \ - patch -p1 < $($(package)_patch_dir)/fix_variadic_warning.patch && \ mkdir -p $($(package)_staging_prefix_dir)/include/sys && \ cp includes/sys/sdt.h $($(package)_staging_prefix_dir)/include/sys/sdt.h endef diff --git a/depends/packages/xcb_proto.mk b/depends/packages/xcb_proto.mk index 9be822506dbc9..6e1c5a10a87ba 100644 --- a/depends/packages/xcb_proto.mk +++ b/depends/packages/xcb_proto.mk @@ -1,8 +1,8 @@ package=xcb_proto -$(package)_version=1.14.1 +$(package)_version=1.15.2 $(package)_download_path=https://xorg.freedesktop.org/archive/individual/proto $(package)_file_name=xcb-proto-$($(package)_version).tar.xz -$(package)_sha256_hash=f04add9a972ac334ea11d9d7eb4fc7f8883835da3e4859c9afa971efdf57fcc3 +$(package)_sha256_hash=7072beb1f680a2fe3f9e535b797c146d22528990c72f63ddb49d2f350a3653ed define $(package)_config_cmds $($(package)_autoconf) diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index d7152327934fb..cc78999dbbeb9 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -1,9 +1,9 @@ package=zeromq -$(package)_version=4.3.4 +$(package)_version=4.3.5 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=c593001a89f5a85dd2ddf564805deb860e02471171b3f204944857336295c3e5 -$(package)_patches=remove_libstd_link.patch netbsd_kevent_void.patch +$(package)_sha256_hash=6653ef5910f17954861fe72332e68b03ca6e4d9c7160eb3a8de5a5a913bfab43 +$(package)_patches=remove_libstd_link.patch define $(package)_set_vars $(package)_config_opts = --without-docs --disable-shared --disable-valgrind @@ -19,8 +19,7 @@ define $(package)_set_vars endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ - patch -p1 < $($(package)_patch_dir)/netbsd_kevent_void.patch + patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch endef define $(package)_config_cmds diff --git a/depends/patches/boost/process_macos_sdk.patch b/depends/patches/boost/process_macos_sdk.patch new file mode 100644 index 0000000000000..ebc556d97270e --- /dev/null +++ b/depends/patches/boost/process_macos_sdk.patch @@ -0,0 +1,27 @@ +Fix Boost Process compilation with macOS 14 SDK. +Can be dropped with Boost 1.84.0. +https://github.com/boostorg/process/pull/343. +https://github.com/boostorg/process/issues/342. + +diff --git a/boost/process/detail/posix/handles.hpp b/boost/process/detail/posix/handles.hpp +index cd9e1ce5a..304e77b1c 100644 +--- a/boost/process/detail/posix/handles.hpp ++++ b/boost/process/detail/posix/handles.hpp +@@ -33,7 +33,7 @@ inline std::vector get_handles(std::error_code & ec) + else + ec.clear(); + +- auto my_fd = ::dirfd(dir.get()); ++ auto my_fd = dirfd(dir.get()); + + struct ::dirent * ent_p; + +@@ -117,7 +117,7 @@ struct limit_handles_ : handler_base_ext + return; + } + +- auto my_fd = ::dirfd(dir); ++ auto my_fd = dirfd(dir); + struct ::dirent * ent_p; + + while ((ent_p = readdir(dir)) != nullptr) diff --git a/depends/patches/native_libtapi/disable_zlib.patch b/depends/patches/native_libtapi/disable_zlib.patch new file mode 100644 index 0000000000000..6c7691214aa84 --- /dev/null +++ b/depends/patches/native_libtapi/disable_zlib.patch @@ -0,0 +1,17 @@ +build: disable zlib + +This isn't needed, and causes issues when clang-tblgen +is built, but trys to reach for a system libz.so. + +diff --git a/build.sh b/build.sh +index e25d2f732..ec8422621 100755 +--- a/build.sh ++++ b/build.sh +@@ -66,6 +66,7 @@ cmake ../src/llvm \ + -DCMAKE_INSTALL_PREFIX=$INSTALLPREFIX \ + -DTAPI_REPOSITORY_STRING=$TAPI_VERSION \ + -DTAPI_FULL_VERSION=$TAPI_VERSION \ ++ -DLLVM_ENABLE_ZLIB=OFF \ + $CMAKE_EXTRA_ARGS + + echo "" diff --git a/depends/patches/qt/dont_hardcode_pwd.patch b/depends/patches/qt/dont_hardcode_pwd.patch deleted file mode 100644 index a74e9cb09872b..0000000000000 --- a/depends/patches/qt/dont_hardcode_pwd.patch +++ /dev/null @@ -1,27 +0,0 @@ -commit 0e953866fc4672486e29e1ba6d83b4207e7b2f0b -Author: fanquake -Date: Tue Aug 18 15:09:06 2020 +0800 - - Don't hardcode pwd path - - Let a man use his builtins if he wants to! Also, removes the unnecessary - assumption that pwd lives under /bin/pwd. - - See #15581. - -diff --git a/qtbase/configure b/qtbase/configure -index 08b49a8d..faea5b55 100755 ---- a/qtbase/configure -+++ b/qtbase/configure -@@ -36,9 +36,9 @@ - relconf=`basename $0` - # the directory of this script is the "source tree" - relpath=`dirname $0` --relpath=`(cd "$relpath"; /bin/pwd)` -+relpath=`(cd "$relpath"; pwd)` - # the current directory is the "build tree" or "object tree" --outpath=`/bin/pwd` -+outpath=`pwd` - - WHICH="which" - diff --git a/depends/patches/qt/dont_hardcode_x86_64.patch b/depends/patches/qt/dont_hardcode_x86_64.patch deleted file mode 100644 index a66426877ad78..0000000000000 --- a/depends/patches/qt/dont_hardcode_x86_64.patch +++ /dev/null @@ -1,119 +0,0 @@ -macOS: Don't hard-code x86_64 as the architecture when using qmake - -Upstream commit: - - Qt 6.1: 9082cc8e8d5a6441dabe5e7a95bc0cd9085b95fe - -For other Qt branches see -https://codereview.qt-project.org/q/I70db7e4c27f0d3da5d0af33cb491d72c312d3fa8 - - ---- old/qtbase/configure.json -+++ new/qtbase/configure.json -@@ -244,11 +244,18 @@ - - "testTypeDependencies": { - "linkerSupportsFlag": [ "use_bfd_linker", "use_gold_linker", "use_lld_linker" ], -- "verifySpec": [ "shared", "use_bfd_linker", "use_gold_linker", "use_lld_linker", "compiler-flags", "qmakeargs", "commit" ], -+ "verifySpec": [ -+ "shared", -+ "use_bfd_linker", "use_gold_linker", "use_lld_linker", -+ "compiler-flags", "qmakeargs", -+ "simulator_and_device", -+ "thread", -+ "commit" ], - "compile": [ "verifyspec" ], - "detectPkgConfig": [ "cross_compile", "machineTuple" ], - "library": [ "pkg-config", "compiler-flags" ], -- "getPkgConfigVariable": [ "pkg-config" ] -+ "getPkgConfigVariable": [ "pkg-config" ], -+ "architecture" : [ "verifyspec" ] - }, - - "testTypeAliases": { -@@ -762,7 +769,7 @@ - }, - "architecture": { - "label": "Architecture", -- "output": [ "architecture" ] -+ "output": [ "architecture", "commitConfig" ] - }, - "pkg-config": { - "label": "Using pkg-config", -diff --git a/configure.pri b/configure.pri -index 49755f7abfd..8be9b10d7d4 100644 ---- old/qtbase/configure.pri -+++ new/qtbase/configure.pri -@@ -662,6 +662,13 @@ defineTest(qtConfOutput_commitOptions) { - write_file($$QT_BUILD_TREE/mkspecs/qdevice.pri, $${currentConfig}.output.devicePro)|error() - } - -+# Output is written after configuring each Qt module, -+# but some tests within a module might depend on the -+# configuration output of previous tests. -+defineTest(qtConfOutput_commitConfig) { -+ qtConfProcessOutput() -+} -+ - # type (empty or 'host'), option name, default value - defineTest(processQtPath) { - out_var = config.rel_input.$${2} -diff --git a/mkspecs/common/macx.conf b/mkspecs/common/macx.conf -index d16b77acb8e..4ba0a8eaa36 100644 ---- old/qtbase/mkspecs/common/macx.conf -+++ new/qtbase/mkspecs/common/macx.conf -@@ -6,7 +6,6 @@ QMAKE_PLATFORM += macos osx macx - QMAKE_MAC_SDK = macosx - - QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.13 --QMAKE_APPLE_DEVICE_ARCHS = x86_64 - - # Should be 10.15, but as long as the CI builds with - # older SDKs we have to keep this. -diff --git a/mkspecs/features/mac/default_post.prf b/mkspecs/features/mac/default_post.prf -index 92a9112bca6..d888731ec8d 100644 ---- old/qtbase/mkspecs/features/mac/default_post.prf -+++ new/qtbase/mkspecs/features/mac/default_post.prf -@@ -95,6 +95,11 @@ app_extension_api_only { - QMAKE_LFLAGS += $$QMAKE_CFLAGS_APPLICATION_EXTENSION - } - -+# Non-universal builds do not set QMAKE_APPLE_DEVICE_ARCHS, -+# so we pick it up from what the arch test resolved instead. -+isEmpty(QMAKE_APPLE_DEVICE_ARCHS): \ -+ QMAKE_APPLE_DEVICE_ARCHS = $$QT_ARCH -+ - macx-xcode { - qmake_pkginfo_typeinfo.name = QMAKE_PKGINFO_TYPEINFO - !isEmpty(QMAKE_PKGINFO_TYPEINFO): \ -@@ -150,9 +155,6 @@ macx-xcode { - simulator: VALID_SIMULATOR_ARCHS = $$QMAKE_APPLE_SIMULATOR_ARCHS - VALID_ARCHS = $$VALID_DEVICE_ARCHS $$VALID_SIMULATOR_ARCHS - -- isEmpty(VALID_ARCHS): \ -- error("QMAKE_APPLE_DEVICE_ARCHS or QMAKE_APPLE_SIMULATOR_ARCHS must contain at least one architecture") -- - single_arch: VALID_ARCHS = $$first(VALID_ARCHS) - - ACTIVE_ARCHS = $(filter $(EXPORT_VALID_ARCHS), $(ARCHS)) -diff --git a/mkspecs/features/toolchain.prf b/mkspecs/features/toolchain.prf -index efbe7c1e55b..8add6dc8043 100644 ---- old/qtbase/mkspecs/features/toolchain.prf -+++ new/qtbase/mkspecs/features/toolchain.prf -@@ -182,9 +182,14 @@ isEmpty($${target_prefix}.INCDIRS) { - # UIKit simulator platforms will see the device SDK's sysroot in - # QMAKE_DEFAULT_*DIRS, because they're handled in a single build pass. - darwin { -- # Clang doesn't pick up the architecture from the sysroot, and will -- # default to the host architecture, so we need to manually set it. -- cxx_flags += -arch $$QMAKE_APPLE_DEVICE_ARCHS -+ uikit { -+ # Clang doesn't automatically pick up the architecture, just because -+ # we're passing the iOS sysroot below, and we will end up building the -+ # test for the host architecture, resulting in linker errors when -+ # linking against the iOS libraries. We work around this by passing -+ # the architecture explicitly. -+ cxx_flags += -arch $$first(QMAKE_APPLE_DEVICE_ARCHS) -+ } - - uikit:macx-xcode: \ - cxx_flags += -isysroot $$sdk_path_device.value diff --git a/depends/patches/qt/duplicate_lcqpafonts.patch b/depends/patches/qt/duplicate_lcqpafonts.patch deleted file mode 100644 index c460b51dcff67..0000000000000 --- a/depends/patches/qt/duplicate_lcqpafonts.patch +++ /dev/null @@ -1,104 +0,0 @@ -QtGui: Fix duplication of logging category lcQpaFonts - -Move it to qplatformfontdatabase.h. - -Upstream commit: - - Qt 6.0: ab01885e48873fb2ad71841a3f1627fe4d9cd835 - ---- a/qtbase/src/gui/text/qplatformfontdatabase.cpp -+++ b/qtbase/src/gui/text/qplatformfontdatabase.cpp -@@ -52,6 +52,8 @@ - - QT_BEGIN_NAMESPACE - -+Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") -+ - void qt_registerFont(const QString &familyname, const QString &stylename, - const QString &foundryname, int weight, - QFont::Style style, int stretch, bool antialiased, - ---- a/qtbase/src/gui/text/qplatformfontdatabase.h -+++ b/qtbase/src/gui/text/qplatformfontdatabase.h -@@ -50,6 +50,7 @@ - // - - #include -+#include - #include - #include - #include -@@ -62,6 +63,7 @@ - - QT_BEGIN_NAMESPACE - -+Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) - - class QWritingSystemsPrivate; - - ---- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm -+++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm -@@ -86,8 +86,6 @@ - - QT_BEGIN_NAMESPACE - --Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") -- - static float SYNTHETIC_ITALIC_SKEW = std::tan(14.f * std::acos(0.f) / 90.f); - - bool QCoreTextFontEngine::ct_getSfntTable(void *user_data, uint tag, uchar *buffer, uint *length) - ---- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h -+++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h -@@ -64,8 +64,6 @@ - - QT_BEGIN_NAMESPACE - --Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) -- - class QCoreTextFontEngine : public QFontEngine - { - Q_GADGET - ---- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp -+++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp -@@ -68,8 +68,6 @@ - - QT_BEGIN_NAMESPACE - --Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") -- - #ifndef QT_NO_DIRECTWRITE - // ### fixme: Consider direct linking of dwrite.dll once Windows Vista pre SP2 is dropped (QTBUG-49711) - - ---- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h -+++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h -@@ -63,8 +63,6 @@ - - QT_BEGIN_NAMESPACE - --Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) -- - class QWindowsFontEngineData - { - Q_DISABLE_COPY_MOVE(QWindowsFontEngineData) - ---- a/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp -+++ b/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp -@@ -40,6 +40,7 @@ - #include "qgenericunixthemes_p.h" - - #include "qpa/qplatformtheme_p.h" -+#include "qpa/qplatformfontdatabase.h" - - #include - #include -@@ -76,7 +77,6 @@ - QT_BEGIN_NAMESPACE - - Q_DECLARE_LOGGING_CATEGORY(qLcTray) --Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") - - ResourceHelper::ResourceHelper() - { diff --git a/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch b/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch deleted file mode 100644 index d4d6539f56dc4..0000000000000 --- a/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch +++ /dev/null @@ -1,20 +0,0 @@ -Modify the optimisation flags for FastFixedDtoa. -This fixes a non-determinism issue in the asm produced for -this function when cross-compiling on x86_64 and aarch64 for -the arm-linux-gnueabihf HOST. - ---- a/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h -+++ b/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h -@@ -48,9 +48,12 @@ namespace double_conversion { - // - // This method only works for some parameters. If it can't handle the input it - // returns false. The output is null-terminated when the function succeeds. -+#pragma GCC push_options -+#pragma GCC optimize ("-O1") - bool FastFixedDtoa(double v, int fractional_count, - Vector buffer, int* length, int* decimal_point); - -+#pragma GCC pop_options - } // namespace double_conversion - - #endif // DOUBLE_CONVERSION_FIXED_DTOA_H_ diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch deleted file mode 100644 index 936b82e1522e3..0000000000000 --- a/depends/patches/qt/fix_android_jni_static.patch +++ /dev/null @@ -1,18 +0,0 @@ ---- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp -+++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -943,6 +943,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) - __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); - return -1; - } -+ -+ const jint ret = QT_PREPEND_NAMESPACE(QtAndroidPrivate::initJNI(vm, env)); -+ if (ret != 0) -+ { -+ __android_log_print(ANDROID_LOG_FATAL, "Qt", "initJNI failed"); -+ return ret; -+ } -+ - QWindowSystemInterfacePrivate::TabletEvent::setPlatformSynthesizesMouse(false); - - m_javaVM = vm; - diff --git a/depends/patches/qt/fix_montery_include.patch b/depends/patches/qt/fix_montery_include.patch deleted file mode 100644 index 38b700addfe7a..0000000000000 --- a/depends/patches/qt/fix_montery_include.patch +++ /dev/null @@ -1,21 +0,0 @@ -From dece6f5840463ae2ddf927d65eb1b3680e34a547 -From: Øystein Heskestad -Date: Wed, 27 Oct 2021 13:07:46 +0200 -Subject: [PATCH] Add missing macOS header file that was indirectly included before - -See: https://bugreports.qt.io/browse/QTBUG-97855 - -Upstream Commits: - - Qt 6.2: c884bf138a21dd7320e35cef34d24e22e74d7ce0 - -diff --git a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h -index e070ba97..07c75b04 100644 ---- a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h -+++ b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h -@@ -40,6 +40,7 @@ - #ifndef QIOSURFACEGRAPHICSBUFFER_H - #define QIOSURFACEGRAPHICSBUFFER_H - -+#include - #include - #include diff --git a/depends/patches/qt/fix_qt_pkgconfig.patch b/depends/patches/qt/fix_qt_pkgconfig.patch deleted file mode 100644 index 73f4d89f7354e..0000000000000 --- a/depends/patches/qt/fix_qt_pkgconfig.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- old/qtbase/mkspecs/features/qt_module.prf -+++ new/qtbase/mkspecs/features/qt_module.prf -@@ -269,7 +269,7 @@ load(qt_installs) - load(qt_targets) - - # this builds on top of qt_common --!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { -+if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { - CONFIG += create_pc - QMAKE_PKGCONFIG_DESTDIR = pkgconfig - host_build: \ diff --git a/depends/patches/qt/guix_cross_lib_path.patch b/depends/patches/qt/guix_cross_lib_path.patch deleted file mode 100644 index 7911dc21d7dba..0000000000000 --- a/depends/patches/qt/guix_cross_lib_path.patch +++ /dev/null @@ -1,17 +0,0 @@ -Facilitate guix building with CROSS_LIBRARY_PATH - -See discussion in https://github.com/bitcoin/bitcoin/pull/15277. - ---- a/qtbase/mkspecs/features/toolchain.prf -+++ b/qtbase/mkspecs/features/toolchain.prf -@@ -236,8 +236,8 @@ isEmpty($${target_prefix}.INCDIRS) { - add_libraries = false - for (line, output) { - line ~= s/^[ \\t]*// # remove leading spaces -- contains(line, "LIBRARY_PATH=.*") { -- line ~= s/^LIBRARY_PATH=// # remove leading LIBRARY_PATH= -+ contains(line, "(CROSS_)?LIBRARY_PATH=.*") { -+ line ~= s/^(CROSS_)?LIBRARY_PATH=// # remove leading (CROSS_)?LIBRARY_PATH= - equals(QMAKE_HOST.os, Windows): \ - paths = $$split(line, ;) - else: \ diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf deleted file mode 100644 index cb94bf07b42d8..0000000000000 --- a/depends/patches/qt/mac-qmake.conf +++ /dev/null @@ -1,22 +0,0 @@ -MAKEFILE_GENERATOR = UNIX -CONFIG += app_bundle incremental lib_version_first absolute_library_soname -QMAKE_INCREMENTAL_STYLE = sublib -include(../common/macx.conf) -include(../common/gcc-base-mac.conf) -include(../common/clang.conf) -include(../common/clang-mac.conf) -QMAKE_MAC_SDK_PATH=$${MAC_SDK_PATH} -QMAKE_XCODE_VERSION = $${XCODE_VERSION} -QMAKE_XCODE_DEVELOPER_PATH=/Developer -QMAKE_MAC_SDK=macosx -QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} -QMAKE_MAC_SDK.macosx.platform_name = macosx -QMAKE_MAC_SDK.macosx.SDKVersion = $${MAC_SDK_VERSION} -QMAKE_MAC_SDK.macosx.PlatformPath = /phony -!host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} -!host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS -!host_build: QMAKE_CXXFLAGS += -target $${MAC_TARGET} -!host_build: QMAKE_LFLAGS += -target $${MAC_TARGET} -QMAKE_AR = $${CROSS_COMPILE}ar cq -QMAKE_RANLIB=$${CROSS_COMPILE}ranlib -load(qt_config) diff --git a/depends/patches/qt/no-xlib.patch b/depends/patches/qt/no-xlib.patch deleted file mode 100644 index d6846aaca2c29..0000000000000 --- a/depends/patches/qt/no-xlib.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 9563cef873ae82e06f60708d706d054717e801ce Mon Sep 17 00:00:00 2001 -From: Carl Dong -Date: Thu, 18 Jul 2019 17:22:05 -0400 -Subject: [PATCH] Wrap xlib related code blocks in #if's - -They are not necessary to compile QT. ---- - qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp -index 7c62c2e2b3..c05c6c0a07 100644 ---- a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp -+++ b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp -@@ -49,7 +49,9 @@ - #include - #include - #include -+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) - #include -+#endif - #include - #include - -@@ -391,6 +391,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window) - xcb_flush(xcb_connection()); - } - -+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) - static int cursorIdForShape(int cshape) - { - int cursorId = 0; -@@ -444,6 +445,7 @@ static int cursorIdForShape(int cshape) - } - return cursorId; - } -+#endif - - xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape) - { -@@ -556,7 +558,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) - xcb_cursor_t QXcbCursor::createFontCursor(int cshape) - { - xcb_connection_t *conn = xcb_connection(); -+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) - int cursorId = cursorIdForShape(cshape); -+#endif - xcb_cursor_t cursor = XCB_NONE; - - // Try Xcursor first -@@ -586,6 +590,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) - // Non-standard X11 cursors are created from bitmaps - cursor = createNonStandardCursor(cshape); - -+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) - // Create a glpyh cursor if everything else failed - if (!cursor && cursorId) { - cursor = xcb_generate_id(conn); -@@ -593,6 +598,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) - cursorId, cursorId + 1, - 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0); - } -+#endif - - if (cursor && cshape >= 0 && cshape < Qt::LastCursor && connection()->hasXFixes()) { - const char *name = cursorNames[cshape].front(); --- -2.22.0 - diff --git a/depends/patches/qt/qt.pro b/depends/patches/qt/qt.pro deleted file mode 100644 index 8f2e900a840fb..0000000000000 --- a/depends/patches/qt/qt.pro +++ /dev/null @@ -1,16 +0,0 @@ -# Create the super cache so modules will add themselves to it. -cache(, super) - -!QTDIR_build: cache(CONFIG, add, $$list(QTDIR_build)) - -prl = no_install_prl -CONFIG += $$prl -cache(CONFIG, add stash, prl) - -TEMPLATE = subdirs -SUBDIRS = qtbase qttools qttranslations - -qttools.depends = qtbase -qttranslations.depends = qttools - -load(qt_configure) diff --git a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch deleted file mode 100644 index f0c14a9400e12..0000000000000 --- a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch +++ /dev/null @@ -1,17 +0,0 @@ -The moc executable loops through headers on CPLUS_INCLUDE_PATH and stumbles -on the GCC internal _GLIBCXX_VISIBILITY macro. Tell it to ignore it as it is -not supposed to be looking there to begin with. - -Upstream report: https://bugreports.qt.io/browse/QTBUG-83160 - -diff --git a/qtbase/src/tools/moc/main.cpp b/qtbase/src/tools/moc/main.cpp ---- a/qtbase/src/tools/moc/main.cpp -+++ b/qtbase/src/tools/moc/main.cpp -@@ -238,6 +238,7 @@ int runMoc(int argc, char **argv) - dummyVariadicFunctionMacro.arguments += Symbol(0, PP_IDENTIFIER, "__VA_ARGS__"); - pp.macros["__attribute__"] = dummyVariadicFunctionMacro; - pp.macros["__declspec"] = dummyVariadicFunctionMacro; -+ pp.macros["_GLIBCXX_VISIBILITY"] = dummyVariadicFunctionMacro; - - QString filename; - QString output; diff --git a/depends/patches/qt/qttools_src.pro b/depends/patches/qt/qttools_src.pro deleted file mode 100644 index 6ef71a0942735..0000000000000 --- a/depends/patches/qt/qttools_src.pro +++ /dev/null @@ -1,6 +0,0 @@ -TEMPLATE = subdirs -SUBDIRS = linguist - -fb = force_bootstrap -CONFIG += $$fb -cache(CONFIG, add, fb) diff --git a/depends/patches/qt/rcc_hardcode_timestamp.patch b/depends/patches/qt/rcc_hardcode_timestamp.patch deleted file mode 100644 index 03f3897975646..0000000000000 --- a/depends/patches/qt/rcc_hardcode_timestamp.patch +++ /dev/null @@ -1,24 +0,0 @@ -Hardcode last modified timestamp in Qt RCC - -This change allows the already built qt package to be reused even with -the SOURCE_DATE_EPOCH variable set, e.g., for Guix builds. - - ---- old/qtbase/src/tools/rcc/rcc.cpp -+++ new/qtbase/src/tools/rcc/rcc.cpp -@@ -227,14 +227,7 @@ void RCCFileInfo::writeDataInfo(RCCResourceLibrary &lib) - - if (lib.formatVersion() >= 2) { - // last modified time stamp -- const QDateTime lastModified = m_fileInfo.lastModified(); -- quint64 lastmod = quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0); -- static const quint64 sourceDate = 1000 * qgetenv("QT_RCC_SOURCE_DATE_OVERRIDE").toULongLong(); -- if (sourceDate != 0) -- lastmod = sourceDate; -- static const quint64 sourceDate2 = 1000 * qgetenv("SOURCE_DATE_EPOCH").toULongLong(); -- if (sourceDate2 != 0) -- lastmod = sourceDate2; -+ quint64 lastmod = quint64(1); - lib.writeNumber8(lastmod); - if (text || pass1) - lib.writeChar('\n'); diff --git a/depends/patches/qt/use_android_ndk23.patch b/depends/patches/qt/use_android_ndk23.patch deleted file mode 100644 index f22367d527bec..0000000000000 --- a/depends/patches/qt/use_android_ndk23.patch +++ /dev/null @@ -1,13 +0,0 @@ -Use Android NDK r23 LTS - ---- old/qtbase/mkspecs/features/android/default_pre.prf -+++ new/qtbase/mkspecs/features/android/default_pre.prf -@@ -76,7 +76,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- - else: equals(QT_ARCH, arm64-v8a): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/aarch64-linux-android- - else: CROSS_COMPILE = $$NDK_LLVM_PATH/bin/arm-linux-androideabi- - --QMAKE_RANLIB = $${CROSS_COMPILE}ranlib -+QMAKE_RANLIB = $$NDK_LLVM_PATH/bin/llvm-ranlib - QMAKE_LINK_SHLIB = $$QMAKE_LINK - QMAKE_LFLAGS = - diff --git a/depends/patches/systemtap/fix_variadic_warning.patch b/depends/patches/systemtap/fix_variadic_warning.patch deleted file mode 100644 index 93cc2d6081d77..0000000000000 --- a/depends/patches/systemtap/fix_variadic_warning.patch +++ /dev/null @@ -1,16 +0,0 @@ -Could be dropped after a migration to C++20. -See: https://github.com/bitcoin/bitcoin/issues/26916. - -diff --git a/includes/sys/sdt.h b/includes/sys/sdt.h -index 4075a5f..7c6138c 100644 ---- a/includes/sys/sdt.h -+++ b/includes/sys/sdt.h -@@ -276,7 +276,7 @@ __extension__ extern unsigned long long __sdt_unsp; - _SDT_ASM_1(.purgem _SDT_TYPE_) \ - _SDT_ASM_1(.purgem _SDT_TYPE) - --#define _SDT_ASM_BODY(provider, name, pack_args, args, ...) \ -+#define _SDT_ASM_BODY(provider, name, pack_args, args) \ - _SDT_DEF_MACROS \ - _SDT_ASM_1(990: _SDT_NOP) \ - _SDT_ASM_3( .pushsection .note.stapsdt,_SDT_ASM_AUTOGROUP,"note") \ diff --git a/depends/patches/zeromq/netbsd_kevent_void.patch b/depends/patches/zeromq/netbsd_kevent_void.patch deleted file mode 100644 index 845c6bdda6719..0000000000000 --- a/depends/patches/zeromq/netbsd_kevent_void.patch +++ /dev/null @@ -1,57 +0,0 @@ -commit 129137d5182967dbfcfec66bad843df2a992a78f -Author: fanquake -Date: Mon Jan 3 20:13:33 2022 +0800 - - problem: kevent udata is now void* on NetBSD Current (10) - - solution: check for the intptr_t variant in configure. - -diff --git a/configure.ac b/configure.ac -index 1a571291..402f8b86 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -307,6 +307,27 @@ case "${host_os}" in - if test "x$libzmq_netbsd_has_atomic" = "xno"; then - AC_DEFINE(ZMQ_FORCE_MUTEXES, 1, [Force to use mutexes]) - fi -+ # NetBSD Current (to become 10) has changed the type of udata in it's -+ # kevent struct from intptr_t to void * to align with darwin and other -+ # BSDs, see upstream commit: -+ # https://github.com/NetBSD/src/commit/e5ead823eb916b56589d2c6c560dbcfe4a2d0afc -+ AC_MSG_CHECKING([whether kevent udata type is intptr_t]) -+ AC_LANG_PUSH([C++]) -+ AC_LINK_IFELSE([AC_LANG_PROGRAM( -+ [[#include -+ #include -+ #include ]], -+ [[struct kevent ev; -+ intptr_t udata; -+ EV_SET(&ev, 0, 0, EV_ADD, 0, 0, udata); -+ return 0;]])], -+ [libzmq_netbsd_kevent_udata_intptr_t=yes], -+ [libzmq_netbsd_kevent_udata_intptr_t=no]) -+ AC_LANG_POP([C++]) -+ AC_MSG_RESULT([$libzmq_netbsd_kevent_udata_intptr_t]) -+ if test "x$libzmq_netbsd_kevent_udata_intptr_t" = "xyes"; then -+ AC_DEFINE(ZMQ_NETBSD_KEVENT_UDATA_INTPTR_T, 1, [kevent udata type is intptr_t]) -+ fi - ;; - *openbsd*|*bitrig*) - # Define on OpenBSD to enable all library features -diff --git a/src/kqueue.cpp b/src/kqueue.cpp -index 53d82ac4..a6a7a7f2 100644 ---- a/src/kqueue.cpp -+++ b/src/kqueue.cpp -@@ -46,9 +46,9 @@ - #include "i_poll_events.hpp" - #include "likely.hpp" - --// NetBSD defines (struct kevent).udata as intptr_t, everyone else --// as void *. --#if defined ZMQ_HAVE_NETBSD -+// NetBSD up to version 9 defines (struct kevent).udata as intptr_t, -+// everyone else as void *. -+#if defined ZMQ_HAVE_NETBSD && defined(ZMQ_NETBSD_KEVENT_UDATA_INTPTR_T) - #define kevent_udata_t intptr_t - #else - #define kevent_udata_t void * diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index ffb13b6eb6bac..5ff6e33d6e7df 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -5,6 +5,41 @@ The headless daemon `navcoind` has the JSON-RPC API enabled by default, the GUI option. In the GUI it is possible to execute RPC methods in the Debug Console Dialog. +## Endpoints + +There are two JSON-RPC endpoints on the server: + +1. `/` +2. `/wallet//` + +### `/` endpoint + +This endpoint is always active. +It can always service non-wallet requests and can service wallet requests when +exactly one wallet is loaded. + +### `/wallet//` endpoint + +This endpoint is only activated when the wallet component has been compiled in. +It can service both wallet and non-wallet requests. +It MUST be used for wallet requests when two or more wallets are loaded. + +This is the endpoint used by bitcoin-cli when a `-rpcwallet=` parameter is passed in. + +Best practice would dictate using the `/wallet//` endpoint for ALL +requests when multiple wallets are in use. + +### Examples + +```sh +# Get block count from the / endpoint when rpcuser=alice and rpcport=38332 +$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getblockcount", "params": []}' -H 'content-type: text/plain;' localhost:38332/ + +# Get balance from the /wallet/walletname endpoint when rpcuser=alice, rpcport=38332 and rpcwallet=desc-wallet +$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getbalance", "params": []}' -H 'content-type: text/plain;' localhost:38332/wallet/desc-wallet + +``` + ## Parameter passing The JSON-RPC server supports both _by-position_ and _by-name_ [parameter @@ -93,7 +128,7 @@ RPC interface will be abused. Instead, expose it only on the host system's localhost, for example: `-p 127.0.0.1:8332:8332` -- **Secure authentication:** By default, Bitcoin Core generates unique +- **Secure authentication:** By default, when no `rpcpassword` is specified, Bitcoin Core generates unique login credentials each time it restarts and puts them into a file readable only by the user that started Bitcoin Core, allowing any of that user's RPC clients with read access to the file to login diff --git a/doc/README.md b/doc/README.md index 980ff00689f56..910702a3d714c 100644 --- a/doc/README.md +++ b/doc/README.md @@ -71,7 +71,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th ### Miscellaneous - [Assets Attribution](assets-attribution.md) -- [bitcoin.conf Configuration File](bitcoin-conf.md) +- [navcoin.conf Configuration File](bitcoin-conf.md) - [CJDNS Support](cjdns.md) - [Files](files.md) - [Fuzz-testing](fuzzing.md) @@ -79,6 +79,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Init Scripts (systemd/upstart/openrc)](init.md) - [Managing Wallets](managing-wallets.md) - [Multisig Tutorial](multisig-tutorial.md) +- [Offline Signing Tutorial](offline-signing-tutorial.md) - [P2P bad ports definition and list](p2p-bad-ports.md) - [PSBT support](psbt.md) - [Reduce Memory](reduce-memory.md) diff --git a/doc/bips.md b/doc/bips.md index 1d5c91b8bdb32..952d289daa6fe 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,4 +1,4 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v24.0**): +BIPs that are implemented by Bitcoin Core: * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1** ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575)) * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). @@ -49,6 +49,7 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v24.0**): * [`BIP 173`](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki): Bech32 addresses for native Segregated Witness outputs are supported as of **v0.16.0** ([PR 11167](https://github.com/bitcoin/bitcoin/pull/11167)). Bech32 addresses are generated by default as of **v0.20.0** ([PR 16884](https://github.com/bitcoin/bitcoin/pull/16884)). * [`BIP 174`](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki): RPCs to operate on Partially Signed Bitcoin Transactions (PSBT) are present as of **v0.17.0** ([PR 13557](https://github.com/bitcoin/bitcoin/pull/13557)). * [`BIP 176`](https://github.com/bitcoin/bips/blob/master/bip-0176.mediawiki): Bits Denomination [QT only] is supported as of **v0.16.0** ([PR 12035](https://github.com/bitcoin/bitcoin/pull/12035)). +* [`BIP 324`](https://github.com/bitcoin/bips/blob/master/bip-0324.mediawiki): The v2 transport protocol specified by BIP324 and the associated `NODE_P2P_V2` service bit are supported as of **v26.0**, but off by default ([PR 28331](https://github.com/bitcoin/bitcoin/pull/28331)). * [`BIP 325`](https://github.com/bitcoin/bips/blob/master/bip-0325.mediawiki): Signet test network is supported as of **v0.21.0** ([PR 18267](https://github.com/bitcoin/bitcoin/pull/18267)). * [`BIP 339`](https://github.com/bitcoin/bips/blob/master/bip-0339.mediawiki): Relay of transactions by wtxid is supported as of **v0.21.0** ([PR 18044](https://github.com/bitcoin/bitcoin/pull/18044)). * [`BIP 340`](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki) diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index 0f05cdcba7891..a7e3282cf1782 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -2,7 +2,7 @@ Updated for NetBSD [9.2](https://netbsd.org/releases/formal-9/NetBSD-9.2.html). -This guide describes how to build bitcoind, command-line utilities, and GUI on NetBSD. +This guide describes how to build navcoind, command-line utilities, and GUI on NetBSD. ## Preparation @@ -46,7 +46,7 @@ git clone https://github.com/bitcoin/bitcoin.git #### Wallet Dependencies -It is not necessary to build wallet functionality to run bitcoind or the GUI. +It is not necessary to build wallet functionality to run navcoind or the GUI. ###### Descriptor Wallet Support diff --git a/doc/build-osx.md b/doc/build-osx.md index 0c3b74bf56dd9..87f850cd720a6 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -163,14 +163,8 @@ brew install python #### Deploy Dependencies -You can deploy a `.dmg` containing the Bitcoin Core application using `make deploy`. -This command depends on a couple of python packages, so it is required that you have `python` installed. - -Ensuring that `python` is installed, you can install the deploy dependencies by running the following commands in your terminal: - -``` bash -pip3 install ds_store mac_alias -``` +You can deploy a `.zip` containing the Bitcoin Core application using `make deploy`. +It is required that you have `python` installed. ## Building Bitcoin Core @@ -181,7 +175,6 @@ There are many ways to configure Bitcoin Core, here are a few common examples: ##### Wallet (BDB + SQlite) Support, No GUI: If `berkeley-db@4` is installed, then legacy wallet support will be built. -If `berkeley-db@4` is not installed, then this will throw an error. If `sqlite` is installed, then descriptor wallet support will also be built. Additionally, this explicitly disables the GUI. @@ -230,7 +223,7 @@ make check # Run tests if Python 3 is available ### 3. Deploy (optional) -You can also create a `.dmg` containing the `.app` bundle by running the following command: +You can also create a `.zip` containing the `.app` bundle by running the following command: ``` bash make deploy @@ -255,9 +248,9 @@ Before running, you may create an empty configuration file: ```shell mkdir -p "/Users/${USER}/Library/Application Support/Bitcoin" -touch "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" +touch "/Users/${USER}/Library/Application Support/Bitcoin/navcoin.conf" -chmod 600 "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" +chmod 600 "/Users/${USER}/Library/Application Support/Bitcoin/navcoin.conf" ``` You can monitor the download process by looking at the debug.log file: diff --git a/doc/build-unix.md b/doc/build-unix.md index a29250141e908..ed38410774a30 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -4,16 +4,6 @@ Some notes on how to build Bitcoin Core in Unix. (For BSD specific instructions, see `build-*bsd.md` in this directory.) -Note ---------------------- -Always use absolute paths to configure and compile Bitcoin Core and the dependencies. -For example, when specifying the path of the dependency: - - ../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX - -Here BDB_PREFIX must be an absolute path - it is defined using $(pwd) which ensures -the usage of the absolute path. - To Build --------------------- @@ -24,12 +14,11 @@ make # use "-j N" for N parallel jobs make install # optional ``` -This will build navcoin-qt as well, if the dependencies are met. +See below for instructions on how to [install the dependencies on popular Linux +distributions](#linux-distribution-specific-instructions), or the +[dependencies](#dependencies) section for a complete overview. -See [dependencies.md](dependencies.md) for a complete overview. - -Memory Requirements --------------------- +## Memory Requirements C++ compilers are memory-hungry. It is recommended to have at least 1.5 GB of memory available when compiling Bitcoin Core. On systems with less, gcc can be @@ -57,7 +46,7 @@ Build requirements: sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils python3 -Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: +Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: sudo apt-get install libevent-dev libboost-dev @@ -65,8 +54,8 @@ SQLite is required for the descriptor wallet: sudo apt install libsqlite3-dev -Berkeley DB is required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, -but these will install Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed +Berkeley DB is only required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, +but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). @@ -114,7 +103,7 @@ Build requirements: sudo dnf install gcc-c++ libtool make autoconf automake python3 -Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: +Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: sudo dnf install libevent-devel boost-devel @@ -122,11 +111,7 @@ SQLite is required for the descriptor wallet: sudo dnf install sqlite-devel -Berkeley DB is required for the legacy wallet: - - sudo dnf install libdb4-devel libdb4-cxx-devel - -Newer Fedora releases, since Fedora 33, have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install +Berkeley DB is only required for the legacy wallet. Fedora releases have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on Berkeley DB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). @@ -143,7 +128,7 @@ ZMQ dependencies (provides ZMQ API): User-Space, Statically Defined Tracing (USDT) dependencies: - sudo dnf install systemtap + sudo dnf install systemtap-sdt-devel GUI dependencies: @@ -166,27 +151,13 @@ libqrencode (optional) can be installed with: Once these are installed, they will be found by configure and a navcoin-qt executable will be built by default. -Notes ------ -The release is built with GCC and then "strip navcoind" to strip the debug -symbols, which reduces the executable size by about 90%. - -miniupnpc ---------- - -[miniupnpc](https://miniupnp.tuxfamily.org) may be used for UPnP port mapping. It can be downloaded from [here]( -https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and -turned off by default. +## Dependencies -libnatpmp ---------- +See [dependencies.md](dependencies.md) for a complete overview, and +[depends](/depends/README.md) on how to compile them yourself, if you wish to +not use the packages of your Linux distribution. -[libnatpmp](https://miniupnp.tuxfamily.org/libnatpmp.html) may be used for NAT-PMP port mapping. It can be downloaded -from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and -turned off by default. - -Berkeley DB ------------ +### Berkeley DB The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is recommended to use Berkeley DB 4.8. If you have to build it yourself, and don't @@ -205,53 +176,9 @@ export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-pc-linux-gnu" BDB_CFLAGS="-I${BDB_PREFIX}/include" ``` -**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). - -Security --------- -To help make your Bitcoin Core installation more secure by making certain attacks impossible to -exploit even if a vulnerability is found, binaries are hardened by default. -This can be disabled with: +**Note**: Make sure that `BDB_PREFIX` is an absolute path. -Hardening Flags: - - ./configure --enable-hardening - ./configure --disable-hardening - - -Hardening enables the following features: -* _Position Independent Executable_: Build position independent code to take advantage of Address Space Layout Randomization - offered by some kernels. Attackers who can cause execution of code at an arbitrary memory - location are thwarted if they don't know where anything useful is located. - The stack and heap are randomly located by default, but this allows the code section to be - randomly located as well. - - On an AMD64 processor where a library was not compiled with -fPIC, this will cause an error - such as: "relocation R_X86_64_32 against `......' can not be used when making a shared object;" - - To test that you have built PIE executable, install scanelf, part of paxutils, and use: - - scanelf -e ./bitcoin - - The output should contain: - - TYPE - ET_DYN - -* _Non-executable Stack_: If the stack is executable then trivial stack-based buffer overflow exploits are possible if - vulnerable buffers are found. By default, Bitcoin Core should be built with a non-executable stack, - but if one of the libraries it uses asks for an executable stack or someone makes a mistake - and uses a compiler extension which requires an executable stack, it will silently build an - executable without the non-executable stack protection. - - To verify that the stack is non-executable after compiling use: - `scanelf -e ./bitcoin` - - The output should contain: - STK/REL/PTL - RW- R-- RW- - - The STK RW- means that the stack is readable and writeable but not executable. +**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). Disable-wallet mode -------------------- @@ -281,6 +208,6 @@ This example lists the steps necessary to setup and build a command line only di ./autogen.sh ./configure make check - ./src/bitcoind + ./src/navcoind If you intend to work with legacy Berkeley DB wallets, see [Berkeley DB](#berkeley-db) section. diff --git a/doc/build-windows.md b/doc/build-windows.md index 027e8f80f50fd..841693c77b55f 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -48,16 +48,9 @@ Acquire the source in the usual way: ## Building for 64-bit Windows The first step is to install the mingw-w64 cross-compilation tool chain: - - on modern systems (Ubuntu 21.04 Hirsute Hippo or newer, Debian 11 Bullseye or newer): ```sh sudo apt install g++-mingw-w64-x86-64-posix -``` - - - on older systems: - -```sh -sudo apt install g++-mingw-w64-x86-64 ``` Once the toolchain is installed the build steps are common: diff --git a/doc/cjdns.md b/doc/cjdns.md index bdb502d803ff8..0fb0b84e181d7 100644 --- a/doc/cjdns.md +++ b/doc/cjdns.md @@ -112,5 +112,4 @@ There are several ways to see your CJDNS address in Bitcoin Core: To see which CJDNS peers your node is connected to, use `navcoin-cli -netinfo 4` or the `getpeerinfo` RPC (i.e. `navcoin-cli getpeerinfo`). -To see which CJDNS addresses your node knows, use the `getnodeaddresses 0 cjdns` -RPC. +You can use the `getnodeaddresses` RPC to fetch a number of CJDNS peers known to your node; run `bitcoin-cli help getnodeaddresses` for details. diff --git a/doc/dependencies.md b/doc/dependencies.md index c8417f49e6ed5..e992b50b060a2 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -8,19 +8,19 @@ You can find installation instructions in the `build-*.md` file for your platfor | --- | --- | | [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | | [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | -| [Clang](https://clang.llvm.org) | [8.0](https://github.com/bitcoin/bitcoin/pull/24164) | -| [GCC](https://gcc.gnu.org) | [8.1](https://github.com/bitcoin/bitcoin/pull/23060) | -| [Python](https://www.python.org) (scripts, tests) | [3.8](https://github.com/bitcoin/bitcoin/pull/27483) | +| [Clang](https://clang.llvm.org) | [14.0](https://github.com/bitcoin/bitcoin/pull/29208) | +| [GCC](https://gcc.gnu.org) | [10.1](https://github.com/bitcoin/bitcoin/pull/28348) | +| [Python](https://www.python.org) (scripts, tests) | [3.9](https://github.com/bitcoin/bitcoin/pull/28211) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | ## Required | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | -| [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.81.0](https://github.com/bitcoin/bitcoin/pull/26557) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | +| [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.81.0](https://github.com/bitcoin/bitcoin/pull/26557) | [1.73.0](https://github.com/bitcoin/bitcoin/pull/29066) | No | | [libevent](../depends/packages/libevent.mk) | [link](https://github.com/libevent/libevent/releases) | [2.1.12-stable](https://github.com/bitcoin/bitcoin/pull/21991) | [2.1.8](https://github.com/bitcoin/bitcoin/pull/24681) | No | | glibc | [link](https://www.gnu.org/software/libc/) | N/A | [2.27](https://github.com/bitcoin/bitcoin/pull/27029) | Yes | -| Linux Kernel | [link](https://www.kernel.org/) | N/A | 3.2.0 | Yes | +| Linux Kernel | [link](https://www.kernel.org/) | N/A | [3.17.0](https://github.com/bitcoin/bitcoin/pull/27699) | Yes | ## Optional @@ -30,7 +30,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | | [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | | [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | -| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.5](https://github.com/bitcoin/bitcoin/pull/25719) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.11](https://github.com/bitcoin/bitcoin/pull/28769) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/doc/design/assumeutxo.md b/doc/design/assumeutxo.md index e0adb35fba489..c8b786767db67 100644 --- a/doc/design/assumeutxo.md +++ b/doc/design/assumeutxo.md @@ -1,10 +1,46 @@ # assumeutxo Assumeutxo is a feature that allows fast bootstrapping of a validating navcoind -instance with a very similar security model to assumevalid. +instance. -The RPC commands `dumptxoutset` and `loadtxoutset` (yet to be merged) are used to -respectively generate and load UTXO snapshots. The utility script +## Loading a snapshot + +There is currently no canonical source for snapshots, but any downloaded snapshot +will be checked against a hash that's been hardcoded in source code. + +Once you've obtained the snapshot, you can use the RPC command `loadtxoutset` to +load it. + +### Pruning + +A pruned node can load a snapshot. To save space, it's possible to delete the +snapshot file as soon as `loadtxoutset` finishes. + +The minimum `-dbcache` setting is 550 MiB, but this functionality ignores that +minimum and uses at least 1100 MiB. + +As the background sync continues there will be temporarily two chainstate +directories, each multiple gigabytes in size (likely growing larger than the +the downloaded snapshot). + +### Indexes + +Indexes work but don't take advantage of this feature. They always start building +from the genesis block. Once the background validation reaches the snapshot block, +indexes will continue to build all the way to the tip. + +For indexes that support pruning, note that no pruning will take place between +the snapshot and the tip, until the background sync has completed - after which +everything is pruned. Depending on how old the snapshot is, this may temporarily +use a significant amount of disk space. + +## Generating a snapshot + +The RPC command `dumptxoutset` can be used to generate a snapshot. This can be used +to create a snapshot on one node that you wish to load on another node. +It can also be used to verify the hardcoded snapshot hash in the source code. + +The utility script `./contrib/devtools/utxo_snapshot.sh` may be of use. ## General background @@ -17,10 +53,9 @@ respectively generate and load UTXO snapshots. The utility script - A new block index `nStatus` flag is introduced, `BLOCK_ASSUMED_VALID`, to mark block index entries that are required to be assumed-valid by a chainstate created - from a UTXO snapshot. This flag is mostly used as a way to modify certain + from a UTXO snapshot. This flag is used as a way to modify certain CheckBlockIndex() logic to account for index entries that are pending validation by a - chainstate running asynchronously in the background. We also use this flag to control - which index entries are added to setBlockIndexCandidates during LoadBlockIndex(). + chainstate running asynchronously in the background. - The concept of UTXO snapshots is treated as an implementation detail that lives behind the ChainstateManager interface. The external presentation of the changes diff --git a/doc/design/libraries.md b/doc/design/libraries.md index 7cda64e713a05..e137ce8a9f9ea 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -5,15 +5,15 @@ | *libbitcoin_cli* | RPC client functionality used by *bitcoin-cli* executable | | *libbitcoin_common* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_util*, but higher-level (see [Dependencies](#dependencies)). | | *libbitcoin_consensus* | Stable, backwards-compatible consensus functionality used by *libbitcoin_node* and *libbitcoin_wallet* and also exposed as a [shared library](../shared-libraries.md). | -| *libbitcoinconsensus* | Shared library build of static *libbitcoin_consensus* library | +| *libnavcoinconsensus* | Shared library build of static *libbitcoin_consensus* library | | *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node* and also exposed as a [shared library](../shared-libraries.md). | | *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables | | *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`--enable-multiprocess`](multiprocess.md) is used. | -| *libbitcoin_node* | P2P and RPC server functionality used by *bitcoind* and *bitcoin-qt* executables. | +| *libbitcoin_node* | P2P and RPC server functionality used by *navcoind* and *bitcoin-qt* executables. | | *libbitcoin_util* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_common*, but lower-level (see [Dependencies](#dependencies)). | -| *libbitcoin_wallet* | Wallet functionality used by *bitcoind* and *bitcoin-wallet* executables. | +| *libbitcoin_wallet* | Wallet functionality used by *navcoind* and *bitcoin-wallet* executables. | | *libbitcoin_wallet_tool* | Lower-level wallet functionality used by *bitcoin-wallet* executable. | -| *libbitcoin_zmq* | [ZeroMQ](../zmq.md) functionality used by *bitcoind* and *bitcoin-qt* executables. | +| *libbitcoin_zmq* | [ZeroMQ](../zmq.md) functionality used by *navcoind* and *bitcoin-qt* executables. | ## Conventions @@ -41,8 +41,8 @@ graph TD; bitcoin-cli[bitcoin-cli]-->libbitcoin_cli; -bitcoind[bitcoind]-->libbitcoin_node; -bitcoind[bitcoind]-->libbitcoin_wallet; +navcoind[navcoind]-->libbitcoin_node; +navcoind[navcoind]-->libbitcoin_wallet; bitcoin-qt[bitcoin-qt]-->libbitcoin_node; bitcoin-qt[bitcoin-qt]-->libbitcoinqt; @@ -75,7 +75,7 @@ libbitcoin_wallet_tool-->libbitcoin_wallet; libbitcoin_wallet_tool-->libbitcoin_util; classDef bold stroke-width:2px, font-weight:bold, font-size: smaller; -class bitcoin-qt,bitcoind,bitcoin-cli,bitcoin-wallet bold +class bitcoin-qt,navcoind,bitcoin-cli,bitcoin-wallet bold ``` @@ -100,5 +100,5 @@ class bitcoin-qt,bitcoind,bitcoin-cli,bitcoin-wallet bold ## Work in progress -- Validation code is moving from *libbitcoin_node* to *libbitcoin_kernel* as part of [The libbitcoinkernel Project #24303](https://github.com/bitcoin/bitcoin/issues/24303) +- Validation code is moving from *libbitcoin_node* to *libbitcoin_kernel* as part of [The libnavcoinkernel Project #24303](https://github.com/bitcoin/bitcoin/issues/24303) - Source code organization is discussed in general in [Library source code organization #15732](https://github.com/bitcoin/bitcoin/issues/15732) diff --git a/doc/design/multiprocess.md b/doc/design/multiprocess.md index bea50850d1ff3..9cd14e85431ee 100644 --- a/doc/design/multiprocess.md +++ b/doc/design/multiprocess.md @@ -1,72 +1,264 @@ -# Multiprocess Bitcoin +# Multiprocess Bitcoin Design Document -On unix systems, the `--enable-multiprocess` build option can be passed to `./configure` to build new `bitcoin-node`, `navcoin-wallet`, and `navcoin-gui` executables alongside existing `navcoind` and `navcoin-qt` executables. +Guide to the design and architecture of the Bitcoin Core multiprocess feature -`bitcoin-node` is a drop-in replacement for `navcoind`, and `navcoin-gui` is a drop-in replacement for `navcoin-qt`, and there are no differences in use or external behavior between the new and old executables. But internally (after [#10102](https://github.com/bitcoin/bitcoin/pull/10102)), `navcoin-gui` will spawn a `bitcoin-node` process to run P2P and RPC code, communicating with it across a socket pair, and `bitcoin-node` will spawn `navcoin-wallet` to run wallet code, also communicating over a socket pair. This will let node, wallet, and GUI code run in separate address spaces for better isolation, and allow future improvements like being able to start and stop components independently on different machines and environments. +_This document describes the design of the multiprocess feature. For usage information, see the top-level [multiprocess.md](../multiprocess.md) file._ -## Next steps +## Table of contents -Specific next steps after [#10102](https://github.com/bitcoin/bitcoin/pull/10102) will be: +- [Introduction](#introduction) +- [Current Architecture](#current-architecture) +- [Proposed Architecture](#proposed-architecture) +- [Component Overview: Navigating the IPC Framework](#component-overview-navigating-the-ipc-framework) +- [Design Considerations](#design-considerations) + - [Selection of Cap’n Proto](#selection-of-capn-proto) + - [Hiding IPC](#hiding-ipc) + - [Interface Definition Maintenance](#interface-definition-maintenance) + - [Interface Stability](#interface-stability) +- [Security Considerations](#security-considerations) +- [Example Use Cases and Flows](#example-use-cases-and-flows) + - [Retrieving a Block Hash](#retrieving-a-block-hash) +- [Future Enhancements](#future-enhancements) +- [Conclusion](#conclusion) +- [Appendices](#appendices) + - [Glossary of Terms](#glossary-of-terms) + - [References](#references) +- [Acknowledgements](#acknowledgements) -- [ ] Adding `-ipcbind` and `-ipcconnect` options to `bitcoin-node`, `navcoin-wallet`, and `navcoin-gui` executables so they can listen and connect to TCP ports and unix socket paths. This will allow separate processes to be started and stopped any time and connect to each other. -- [ ] Adding `-server` and `-rpcbind` options to the `navcoin-wallet` executable so wallet processes can handle RPC requests directly without going through the node. -- [ ] Supporting windows, not just unix systems. The existing socket code is already cross-platform, so the only windows-specific code that needs to be written is code spawning a process and passing a socket descriptor. This can be implemented with `CreateProcess` and `WSADuplicateSocket`. Example: https://memset.wordpress.com/2010/10/13/win32-api-passing-socket-with-ipc-method/. -- [ ] Adding sandbox features, restricting subprocess access to resources and data. See [https://eklitzke.org/multiprocess-bitcoin](https://eklitzke.org/multiprocess-bitcoin). +## Introduction -## Debugging +The Bitcoin Core software has historically employed a monolithic architecture. The existing design has integrated functionality like P2P network operations, wallet management, and a GUI into a single executable. While effective, it has limitations in flexibility, security, and scalability. This project introduces changes that transition Bitcoin Core to a more modular architecture. It aims to enhance security, improve usability, and facilitate maintenance and development of the software in the long run. -The `-debug=ipc` command line option can be used to see requests and responses between processes. +## Current Architecture -## Installation +The current system features two primary executables: `navcoind` and `bitcoin-qt`. `navcoind` combines a Bitcoin P2P node with an integrated JSON-RPC server, wallet, and indexes. `bitcoin-qt` extends this by incorporating a Qt-based GUI. This monolithic structure, although robust, presents challenges such as limited operational flexibility and increased security risks due to the tight integration of components. -The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) as dependencies. A simple way to get starting using it without installing these dependencies manually is to use the [depends system](../depends) with the `MULTIPROCESS=1` [dependency option](../depends#dependency-options) passed to make: +## Proposed Architecture +The new architecture divides the existing code into three specialized executables: + +- `bitcoin-node`: Manages the P2P node, indexes, and JSON-RPC server. +- `bitcoin-wallet`: Handles all wallet functionality. +- `bitcoin-gui`: Provides a standalone Qt-based GUI. + +This modular approach is designed to enhance security through component isolation and improve usability by allowing independent operation of each module. This allows for new use-cases, such as running the node on a dedicated machine and operating wallets and GUIs on separate machines with the flexibility to start and stop them as needed. + +This subdivision could be extended in the future. For example, indexes could be removed from the `bitcoin-node` executable and run in separate executables. And JSON-RPC servers could be added to wallet and index executables, so they can listen and respond to RPC requests on their own ports, without needing to forward RPC requests through `bitcoin-node`. + +
+ +```mermaid +flowchart LR + node[bitcoin-node] -- listens on --> socket["<datadir>/node.sock"] + wallet[bitcoin-wallet] -- connects to --> socket + gui[bitcoin-gui] -- connects to --> socket +``` + +
+Processes and socket connection. +
+ +## Component Overview: Navigating the IPC Framework + +This section describes the major components of the Inter-Process Communication (IPC) framework covering the relevant source files, generated files, tools, and libraries. + +### Abstract C++ Classes in [`src/interfaces/`](../../src/interfaces/) +- The foundation of the IPC implementation lies in the abstract C++ classes within the [`src/interfaces/`](../../src/interfaces/) directory. These classes define pure virtual methods that code in [`src/node/`](../../src/node/), [`src/wallet/`](../../src/wallet/), and [`src/qt/`](../../src/qt/) directories call to interact with each other. +- Each abstract class in this directory represents a distinct interface that the different modules (node, wallet, GUI) implement and use for cross-process communication. +- The classes are written following conventions described in [Internal Interface + Guidelines](../developer-notes.md#internal-interface-guidelines) to ensure + compatibility with Cap'n Proto. + +### Cap’n Proto Files in [`src/ipc/capnp/`](../../src/ipc/capnp/) +- Corresponding to each abstract class, there are `.capnp` files within the [`src/ipc/capnp/`](../../src/ipc/capnp/) directory. These files are used as input to the `mpgen` tool (described below) to generate C++ code. +- These Cap’n Proto files ([learn more about Cap'n Proto RPC](https://capnproto.org/rpc.html)) define the structure and format of messages that are exchanged over IPC. They serve as blueprints for generating C++ code that bridges the gap between high-level C++ interfaces and low-level socket communication. + +### The `mpgen` Code Generation Tool +- A central component of the IPC framework is the `mpgen` tool which is part the [`libmultiprocess` project](https://github.com/chaincodelabs/libmultiprocess). This tool takes the `.capnp` files as input and generates C++ code. +- The generated code handles IPC communication, translating interface calls into socket reads and writes. + +### C++ Client Subclasses in Generated Code +- In the generated code, we have C++ client subclasses that inherit from the abstract classes in [`src/interfaces/`](../../src/interfaces/). These subclasses are the workhorses of the IPC mechanism. +- They implement all the methods of the interface, marshalling arguments into a structured format, sending them as requests to the IPC server via a UNIX socket, and handling the responses. +- These subclasses effectively mask the complexity of IPC, presenting a familiar C++ interface to developers. +- Internally, the client subclasses generated by the `mpgen` tool wrap [client classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#clients), and use them to send IPC requests. + +### C++ Server Classes in Generated Code +- On the server side, corresponding generated C++ classes receive IPC requests. These server classes are responsible for unmarshalling method arguments, invoking the corresponding methods in the local [`src/interfaces/`](../../src/interfaces/) objects, and creating the IPC response. +- The server classes ensure that return values (including output argument values and thrown exceptions) are marshalled and sent back to the client, completing the communication cycle. +- Internally, the server subclasses generated by the `mpgen` tool inherit from [server classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#servers), and use them to process IPC requests. + +### The `libmultiprocess` Runtime Library +- **Core Functionality**: The `libmultiprocess` runtime library's primary function is to instantiate the generated client and server classes as needed. +- **Bootstrapping IPC Connections**: It provides functions for starting new IPC connections, specifically binding generated client and server classes for an initial `interfaces::Init` interface (defined in [`src/interfaces/init.h`](../../src/interfaces/init.h)) to a UNIX socket. This initial interface has methods returning other interfaces that different Bitcoin Core modules use to communicate after the bootstrapping phase. +- **Asynchronous I/O and Thread Management**: The library is also responsible for managing I/O and threading. Particularly, it ensures that IPC requests never block each other and that new threads on either side of a connection can always make client calls. It also manages worker threads on the server side of calls, ensuring that calls from the same client thread always execute on the same server thread (to avoid locking issues and support nested callbacks). + +### Type Hooks in [`src/ipc/capnp/*-types.h`](../../src/ipc/capnp/) +- **Custom Type Conversions**: In [`src/ipc/capnp/*-types.h`](../../src/ipc/capnp/), function overloads of two `libmultiprocess` C++ functions, `mp::CustomReadField` and `mp::CustomBuildFields`, are defined. These overloads are used for customizing the conversion of specific C++ types to and from Cap’n Proto types. +- **Handling Special Cases**: The `mpgen` tool and `libmultiprocess` library can convert most C++ types to and from Cap’n Proto types automatically, including interface types, primitive C++ types, standard C++ types like `std::vector`, `std::set`, `std::map`, `std::tuple`, and `std::function`, as well as simple C++ structs that consist of aforementioned types and whose fields correspond 1:1 with Cap’n Proto struct fields. For other types, `*-types.h` files provide custom code to convert between C++ and Cap’n Proto data representations. + +### Protocol-Agnostic IPC Code in [`src/ipc/`](../../src/ipc/) +- **Broad Applicability**: Unlike the Cap’n Proto-specific code in [`src/ipc/capnp/`](../../src/ipc/capnp/), the code in the [`src/ipc/`](../../src/ipc/) directory is protocol-agnostic. This enables potential support for other protocols, such as gRPC or a custom protocol in the future. +- **Process Management and Socket Operations**: The main purpose of this component is to provide functions for spawning new processes and creating and connecting to UNIX sockets. +- **ipc::Exception Class**: This code also defines an `ipc::Exception` class which is thrown from the generated C++ client class methods when there is an unexpected IPC error, such as a disconnection. + +
+ +```mermaid +flowchart TD + capnpFile[ipc/capnp/chain.capnp] -->|Input to| mpgenTool([mpgen Tool]) + mpgenTool -->|Generates| proxyTypesH[ipc/capnp/chain.capnp.proxy-types.h] + mpgenTool --> proxyClientCpp[ipc/capnp/chain.capnp.proxy-client.c++] + mpgenTool --> proxyServerCpp[ipc/capnp/chain.capnp.proxy-server.c++] + proxyTypesH -.->|Includes| interfaces/chain.h + proxyClientCpp -.-> interfaces/chain.h + proxyServerCpp -.-> interfaces/chain.h ``` -cd -make -C depends MULTIPROCESS=1 -CONFIG_SITE=$PWD/depends/x86_64-pc-linux-gnu/share/config.site ./configure -make -src/bitcoin-node -regtest -printtoconsole -debug=ipc -BITCOIND=bitcoin-node test/functional/test_runner.py + +
+Diagram showing generated source files and includes. +
+ +## Design Considerations + +### Selection of Cap’n Proto +The choice to use [Cap’n Proto](https://capnproto.org/) for IPC was primarily influenced by its support for passing object references and managing object lifetimes, which would have to be implemented manually with a framework that only supported plain requests and responses like [gRPC](https://grpc.io/). The support is especially helpful for passing callback objects like `std::function` and enabling bidirectional calls between processes. + +The choice to use an RPC framework at all instead of a custom protocol was necessitated by the size of Bitcoin Core internal interfaces which consist of around 150 methods that pass complex data structures and are called in complicated ways (in parallel, and from callbacks that can be nested and stored). Writing a custom protocol to wrap these complicated interfaces would be a lot more work, akin to writing a new RPC framework. + +### Hiding IPC + +The IPC mechanism is deliberately isolated from the rest of the codebase so less code has to be concerned with IPC. + +Building Bitcoin Core with IPC support is optional, and node, wallet, and GUI code can be compiled to either run in the same process or separate processes. The build system also ensures Cap’n Proto library headers can only be used within the [`src/ipc/capnp/`](../../src/ipc/capnp/) directory, not in other parts of the codebase. + +The libmultiprocess runtime is designed to place as few constraints as possible on IPC interfaces and to make IPC calls act like normal function calls. Method arguments, return values, and exceptions are automatically serialized and sent between processes. Object references and `std::function` arguments are tracked to allow invoked code to call back into invoking code at any time. And there is a 1:1 threading model where every client thread has a corresponding server thread responsible for executing incoming calls from that thread (there can be multiple calls from the same thread due to callbacks) without blocking, and holding the same thread-local variables and locks so behavior is the same whether IPC is used or not. + +### Interface Definition Maintenance + +The choice to maintain interface definitions and C++ type mappings as `.capnp` files in the [`src/ipc/capnp/`](../../src/ipc/capnp/) was mostly done for convenience, and probably something that could be improved in the future. + +In the current design, class names, method names, and parameter names are duplicated between C++ interfaces in [`src/interfaces/`](../../src/interfaces/) and Cap’n Proto files in [`src/ipc/capnp/`](../../src/ipc/capnp/). While this keeps C++ interface headers simple and free of references to IPC, it is a maintenance burden because it means inconsistencies between C++ declarations and Cap’n Proto declarations will result in compile errors. (Static type checking ensures these are not runtime errors.) + +An alternate approach could use custom [C++ Attributes](https://en.cppreference.com/w/cpp/language/attributes) embedded in interface declarations to automatically generate `.capnp` files from C++ headers. This has not been pursued because parsing C++ headers is more complicated than parsing Cap’n Proto interface definitions, especially portably on multiple platforms. + +In the meantime, the developer guide [Internal interface guidelines](developer-notes.md#internal-interface-guidelines) can provide guidance on keeping interfaces consistent and functional and avoiding compile errors. + +### Interface Stability + +The currently defined IPC interfaces are unstable, and can change freely with no backwards compatibility. The decision to allow this stems from the recognition that our current interfaces are still evolving and not yet ideal for external use. As these interfaces mature and become more refined, there may be an opportunity to declare them stable and use Cap’n Proto's support for protocol evolution ([Cap'n Proto - Evolving Your Protocol](https://capnproto.org/language.html#evolving-your-protocol)) to allow them to be extended while remaining backwards compatible. This could allow different versions of node, GUI, and wallet binaries to interoperate, and potentially open doors for external tools to utilize these interfaces, such as creating custom indexes through a stable indexing interface. However, for now, the priority is to improve the interfaces internally. Given their current state and the advantages of using JSON-RPC for most common tasks, it's more practical to focus on internal development rather than external applicability. + +## Security Considerations + +The integration of [Cap’n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) into the Bitcoin Core architecture increases its potential attack surface. Cap’n Proto, being a complex and substantial new dependency, introduces potential sources of vulnerability, particularly through the creation of new UNIX sockets. The inclusion of libmultiprocess, while a smaller external dependency, also contributes to this risk. However, plans are underway to incorporate libmultiprocess as a git subtree, aligning it more closely with the project's well-reviewed internal libraries. While adopting these multiprocess features does introduce some risk, it's worth noting that they can be disabled, allowing builds without these new dependencies. This flexibility ensures that users can balance functionality with security considerations as needed. + +## Example Use Cases and Flows + +### Retrieving a Block Hash + +Let’s walk through an example where the `bitcoin-wallet` process requests the hash of a block at a specific height from the `bitcoin-node` process. This example demonstrates the practical application of the IPC mechanism, specifically the interplay between C++ method calls and Cap’n Proto-generated RPC calls. + +
+ +```mermaid +sequenceDiagram + box "bitcoin-wallet process" + participant WalletCode as Wallet code + participant ChainClient as Generated Chain client class
ProxyClient + end + box "bitcoin-node process" + participant ChainServer as Generated Chain server class
ProxyServer + participant LocalChain as Chain object
node::ChainImpl + end + + WalletCode->>ChainClient: getBlockHash(height) + ChainClient->>ChainServer: Send RPC getBlockHash request + ChainServer->>LocalChain: getBlockHash(height) + LocalChain->>ChainServer: Return block hash + ChainServer->>ChainClient: Send response with block hash + ChainClient->>WalletCode: Return block hash ``` -The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). - -Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. - -## IPC implementation details - -Cross process Node, Wallet, and Chain interfaces are defined in -[`src/interfaces/`](../src/interfaces/). These are C++ classes which follow -[conventions](developer-notes.md#internal-interface-guidelines), like passing -serializable arguments so they can be called from different processes, and -making methods pure virtual so they can have proxy implementations that forward -calls between processes. - -When Wallet, Node, and Chain code is running in the same process, calling any -interface method invokes the implementation directly. When code is running in -different processes, calling an interface method invokes a proxy interface -implementation that communicates with a remote process and invokes the real -implementation in the remote process. The -[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) code -generation tool internally generates proxy client classes and proxy server -classes for this purpose that are thin wrappers around Cap'n Proto -[client](https://capnproto.org/cxxrpc.html#clients) and -[server](https://capnproto.org/cxxrpc.html#servers) classes, which handle the -actual serialization and socket communication. - -As much as possible, calls between processes are meant to work the same as -calls within a single process without adding limitations or requiring extra -implementation effort. Processes communicate with each other by calling regular -[C++ interface methods](../src/interfaces/README.md). Method arguments and -return values are automatically serialized and sent between processes. Object -references and `std::function` arguments are automatically tracked and mapped -to allow invoked code to call back into invoking code at any time, and there is -a 1:1 threading model where any thread invoking a method in another process has -a corresponding thread in the invoked process responsible for executing all -method calls from the source thread, without blocking I/O or holding up another -call, and using the same thread local variables, locks, and callbacks between -calls. The forwarding, tracking, and threading is implemented inside the -[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) library -which has the design goal of making calls between processes look like calls in -the same process to the extent possible. +
+Chain::getBlockHash call diagram +
+ +1. **Initiation in bitcoin-wallet** + - The wallet process calls the `getBlockHash` method on a `Chain` object. This method is defined as a virtual method in [`src/interfaces/chain.h`](../../src/interfaces/chain.h). + +2. **Translation to Cap’n Proto RPC** + - The `Chain::getBlockHash` virtual method is overridden by the `Chain` [client subclass](#c-client-subclasses-in-generated-code) to translate the method call into a Cap’n Proto RPC call. + - The client subclass is automatically generated by the `mpgen` tool from the [`chain.capnp`](https://github.com/ryanofsky/bitcoin/blob/pr/ipc/src/ipc/capnp/chain.capnp) file in [`src/ipc/capnp/`](../../src/ipc/capnp/). + +3. **Request Preparation and Dispatch** + - The `getBlockHash` method of the generated `Chain` client subclass in `bitcoin-wallet` populates a Cap’n Proto request with the `height` parameter, sends it to `bitcoin-node` process, and waits for a response. + +4. **Handling in bitcoin-node** + - Upon receiving the request, the Cap'n Proto dispatching code in the `bitcoin-node` process calls the `getBlockHash` method of the `Chain` [server class](#c-server-classes-in-generated-code). + - The server class is automatically generated by the `mpgen` tool from the [`chain.capnp`](https://github.com/ryanofsky/bitcoin/blob/pr/ipc/src/ipc/capnp/chain.capnp) file in [`src/ipc/capnp/`](../../src/ipc/capnp/). + - The `getBlockHash` method of the generated `Chain` server subclass in `bitcoin-wallet` receives a Cap’n Proto request object with the `height` parameter, and calls the `getBlockHash` method on its local `Chain` object with the provided `height`. + - When the call returns, it encapsulates the return value in a Cap’n Proto response, which it sends back to the `bitcoin-wallet` process, + +5. **Response and Return** + - The `getBlockHash` method of the generated `Chain` client subclass in `bitcoin-wallet` which sent the request now receives the response. + - It extracts the block hash value from the response, and returns it to the original caller. + +## Future Enhancements + +Further improvements are possible such as: + +- Separating indexes from `bitcoin-node`, and running indexing code in separate processes (see [indexes: Stop using node internal types #24230](https://github.com/bitcoin/bitcoin/pull/24230)). +- Enabling wallet processes to listen for JSON-RPC requests on their own ports instead of needing the node process to listen and forward requests to them. +- Automatically generating `.capnp` files from C++ interface definitions (see [Interface Definition Maintenance](#interface-definition-maintenance)). +- Simplifying and stabilizing interfaces (see [Interface Stability](#interface-stability)). +- Adding sandbox features, restricting subprocess access to resources and data (see [https://eklitzke.org/multiprocess-bitcoin](https://eklitzke.org/multiprocess-bitcoin)). +- Using Cap'n Proto's support for [other languages](https://capnproto.org/otherlang.html), such as [Rust](https://github.com/capnproto/capnproto-rust), to allow code written in other languages to call Bitcoin Core C++ code, and vice versa (see [How to rustify libmultiprocess? #56](https://github.com/chaincodelabs/libmultiprocess/issues/56)). + +## Conclusion + +This modularization represents an advancement in Bitcoin Core's architecture, offering enhanced security, flexibility, and maintainability. The project invites collaboration and feedback from the community. + +## Appendices + +### Glossary of Terms + +- **abstract class**: A class in C++ that consists of virtual functions. In the Bitcoin Core project, they define interfaces for inter-component communication. + +- **asynchronous I/O**: A form of input/output processing that allows a program to continue other operations while a transmission is in progress. + +- **Cap’n Proto**: A high-performance data serialization and RPC library, chosen for its support for object references and bidirectional communication. + +- **Cap’n Proto interface**: A set of methods defined in Cap’n Proto to facilitate structured communication between different software components. + +- **Cap’n Proto struct**: A structured data format used in Cap’n Proto, similar to structs in C++, for organizing and transporting data across different processes. + +- **client class (in generated code)**: A C++ class generated from a Cap’n Proto interface which inherits from a Bitcoin core abstract class, and implements each virtual method to send IPC requests to another process. (see also [components section](#c-client-subclasses-in-generated-code)) + +- **IPC (inter-process communication)**: Mechanisms that enable processes to exchange requests and data. + +- **ipc::Exception class**: A class within Bitcoin Core's protocol-agnostic IPC code that is thrown by client class methods when there is an IPC error. + +- **libmultiprocess**: A custom library and code generation tool used for creating IPC interfaces and managing IPC connections. + +- **marshalling**: Transforming an object’s memory representation for transmission. + +- **mpgen tool**: A tool within the `libmultiprocess` suite that generates C++ code from Cap’n Proto files, facilitating IPC. + +- **protocol-agnostic code**: Generic IPC code in [`src/ipc/`](../../src/ipc/) that does not rely on Cap’n Proto and could be used with other protocols. Distinct from code in [`src/ipc/capnp/`](../../src/ipc/capnp/) which relies on Cap’n Proto. + +- **RPC (remote procedure call)**: A protocol that enables a program to request a service from another program in a different address space or network. Bitcoin Core uses [JSON-RPC](https://en.wikipedia.org/wiki/JSON-RPC) for RPC. + +- **server class (in generated code)**: A C++ class generated from a Cap’n Proto interface which handles requests sent by a _client class_ in another process. The request handled by calling a local Bitcoin Core interface method, and the return values (if any) are sent back in a response. (see also: [components section](#c-server-classes-in-generated-code)) + +- **unix socket**: Communication endpoint which is a filesystem path, used for exchanging data between processes running on the same host. + +- **virtual method**: A function or method whose behavior can be overridden within an inheriting class by a function with the same signature. + +## References + +- **Cap’n Proto RPC protocol description**: https://capnproto.org/rpc.html +- **libmultiprocess project page**: https://github.com/chaincodelabs/libmultiprocess + +## Acknowledgements + +This design doc was written by @ryanofsky, who is grateful to all the reviewers who gave feedback and tested [multiprocess PRs](https://github.com/bitcoin/bitcoin/pull/28722), and everyone else who's helped with this project. Particular thanks to @ariard who deeply reviewed IPC code and improved the design of the IPC library and initialization process. @jnewbery who championed the early refactoring PRs and helped guide them through development and review. @sjors who has reviewed and repeatedly tested multiprocess code, reporting many issues and helping debug them. @hebasto, @fanquake, and @maflcko who made significant improvements to the build system and fixed countless build issues. @vasild and @jamesob who were brave contributors to the libmultiprocess library. And Chaincode Labs for making this work possible. Also thanks to ChatGPT, who actually wrote most of this document (not @ryanofsky). diff --git a/doc/developer-notes.md b/doc/developer-notes.md index f9399219acf2b..00ab22c1a6e4b 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -217,13 +217,11 @@ apt install clang-tidy bear clang Then, pass clang as compiler to configure, and use bear to produce the `compile_commands.json`: ```sh -./autogen.sh && ./configure CC=clang CXX=clang++ --enable-suppress-external-warnings +./autogen.sh && ./configure CC=clang CXX=clang++ make clean && bear --config src/.bear-tidy-config -- make -j $(nproc) ``` -The output is denoised of errors from external dependencies and includes with -`--enable-suppress-external-warnings` and `--config src/.bear-tidy-config`. Both -options may be omitted to view the full list of errors. +The output is denoised of errors from external dependencies. To run clang-tidy on all source files: @@ -394,8 +392,8 @@ If the code is behaving strangely, take a look in the `debug.log` file in the da error and debugging messages are written there. Debug logging can be enabled on startup with the `-debug` and `-loglevel` -configuration options and toggled while bitcoind is running with the `logging` -RPC. For instance, launching bitcoind with `-debug` or `-debug=1` will turn on +configuration options and toggled while navcoind is running with the `logging` +RPC. For instance, launching navcoind with `-debug` or `-debug=1` will turn on all log categories and `-loglevel=trace` will turn on all log severity levels. The Qt code routes `qDebug()` output to `debug.log` under category "qt": run with `-debug=qt` @@ -490,7 +488,9 @@ To enable LCOV report generation during test runs: make make cov -# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`. +# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`, +# which covers unit tests, and `./total.coverage/index.html`, which covers +# unit and functional tests. ``` ### Performance profiling with perf @@ -623,8 +623,9 @@ Threads : Started from `main()` in `navcoind.cpp`. Responsible for starting up and shutting down the application. -- [ThreadImport (`b-loadblk`)](https://doxygen.bitcoincore.org/namespacenode.html#ab4305679079866f0f420f7dbf278381d) - : Loads blocks from `blk*.dat` files or `-loadblock=` on startup. +- [Init load (`b-initload`)](https://doxygen.bitcoincore.org/namespacenode.html#ab4305679079866f0f420f7dbf278381d) + : Performs various loading tasks that are part of init but shouldn't block the node from being started: external block import, + reindex, reindex-chainstate, main chain activation, spawn indexes background sync threads and mempool load. - [CCheckQueue::Loop (`b-scriptch.x`)](https://doxygen.bitcoincore.org/class_c_check_queue.html#a6e7fa51d3a25e7cb65446d4b50e6a987) : Parallel script validation threads for transactions in blocks. @@ -724,6 +725,47 @@ General Navcoin Core - *Explanation*: If the test suite is to be updated for a change, this has to be done first. +Logging +------- + +The macros `LogInfo`, `LogDebug`, `LogTrace`, `LogWarning` and `LogError` are available for +logging messages. They should be used as follows: + +- `LogDebug(BCLog::CATEGORY, fmt, params...)` is what you want + most of the time, and it should be used for log messages that are + useful for debugging and can reasonably be enabled on a production + system (that has sufficient free storage space). They will be logged + if the program is started with `-debug=category` or `-debug=1`. + Note that `LogPrint(BCLog::CATEGORY, fmt, params...)` is a deprecated + alias for `LogDebug`. + +- `LogInfo(fmt, params...)` should only be used rarely, e.g. for startup + messages or for infrequent and important events such as a new block tip + being found or a new outbound connection being made. These log messages + are unconditional, so care must be taken that they can't be used by an + attacker to fill up storage. Note that `LogPrintf(fmt, params...)` is + a deprecated alias for `LogInfo`. + +- `LogError(fmt, params...)` should be used in place of `LogInfo` for + severe problems that require the node (or a subsystem) to shut down + entirely (e.g., insufficient storage space). + +- `LogWarning(fmt, params...)` should be used in place of `LogInfo` for + severe problems that the node admin should address, but are not + severe enough to warrant shutting down the node (e.g., system time + appears to be wrong, unknown soft fork appears to have activated). + +- `LogTrace(BCLog::CATEGORY, fmt, params...)` should be used in place of + `LogDebug` for log messages that would be unusable on a production + system, e.g. due to being too noisy in normal use, or too resource + intensive to process. These will be logged if the startup + options `-debug=category -loglevel=category:trace` or `-debug=1 + -loglevel=trace` are selected. + +Note that the format strings and parameters of `LogDebug` and `LogTrace` +are only evaluated if the logging category is enabled, so you must be +careful to avoid side-effects in those expressions. + Wallet ------- @@ -740,12 +782,6 @@ Common misconceptions are clarified in those sections: - Passing (non-)fundamental types in the [C++ Core Guideline](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-conventional). -- Assertions should not have side-effects. - - - *Rationale*: Even though the source code is set to refuse to compile - with assertions disabled, having side-effects in assertions is unexpected and - makes the code harder to understand. - - If you use the `.h`, you must link the `.cpp`. - *Rationale*: Include files define the interface for the code in implementation files. Including one but @@ -892,7 +928,7 @@ Strings and formatting `wcstoll`, `wcstombs`, `wcstoul`, `wcstoull`, `wcstoumax`, `wcswidth`, `wcsxfrm`, `wctob`, `wctomb`, `wctrans`, `wctype`, `wcwidth`, `wprintf` -- For `strprintf`, `LogPrint`, `LogPrintf` formatting characters don't need size specifiers. +- For `strprintf`, `LogInfo`, `LogDebug`, etc formatting characters don't need size specifiers. - *Rationale*: Navcoin Core uses tinyformat, which is type safe. Leave them out to avoid confusion. @@ -904,7 +940,7 @@ Strings and formatting - *Rationale*: Although this is guaranteed to be safe starting with C++11, `.data()` communicates the intent better. - - Do not use it when passing strings to `tfm::format`, `strprintf`, `LogPrint[f]`. + - Do not use it when passing strings to `tfm::format`, `strprintf`, `LogInfo`, `LogDebug`, etc. - *Rationale*: This is redundant. Tinyformat handles strings. @@ -955,7 +991,9 @@ Threads and synchronization internal to a class (private or protected) rather than public. - Combine annotations in function declarations with run-time asserts in - function definitions: + function definitions (`AssertLockNotHeld()` can be omitted if `LOCK()` is + called unconditionally after it because `LOCK()` does the same check as + `AssertLockNotHeld()` internally, for non-recursive mutexes): ```C++ // txmempool.h @@ -1409,7 +1447,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: User-facing consistency. -- Use `fs::path::u8string()` and `fs::u8path()` functions when converting path +- Use `fs::path::u8string()`/`fs::path::utf8string()` and `fs::u8path()` functions when converting path to JSON strings, not `fs::PathToString` and `fs::PathFromString` - *Rationale*: JSON strings are Unicode strings, not byte strings, and diff --git a/doc/files.md b/doc/files.md index aa15eacd2929c..5c8b6ca74c9d2 100644 --- a/doc/files.md +++ b/doc/files.md @@ -32,7 +32,7 @@ Windows | `%APPDATA%\Bitcoin\` [\[1\]](#note1) 2. A custom data directory path can be specified with the `-datadir` option. -3. All content of the data directory, except for `bitcoin.conf` file, is chain-specific. This means the actual data directory paths for non-mainnet cases differ: +3. All content of the data directory, except for `navcoin.conf` file, is chain-specific. This means the actual data directory paths for non-mainnet cases differ: Chain option | Data directory path -------------------------------|------------------------------ @@ -57,7 +57,7 @@ Subdirectory | File(s) | Description `wallets/` | | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, wallets reside in the [data directory](#data-directory-location) `./` | `anchors.dat` | Anchor IP address database, created on shutdown and deleted at startup. Anchors are last known outgoing block-relay-only peers that are tried to re-connect to on startup `./` | `banlist.json` | Stores the addresses/subnets of banned nodes. -`./` | `bitcoin.conf` | User-defined [configuration settings](bitcoin-conf.md) for `navcoind` or `navcoin-qt`. File is not written to by the software and must be created manually. Path can be specified by `-conf` option +`./` | `navcoin.conf` | User-defined [configuration settings](bitcoin-conf.md) for `navcoind` or `navcoin-qt`. File is not written to by the software and must be created manually. Path can be specified by `-conf` option `./` | `navcoind.pid` | Stores the process ID (PID) of `navcoind` or `navcoin-qt` while running; created at start and deleted on shutdown; can be specified by `-pid` option `./` | `debug.log` | Contains debug information and general logging generated by `navcoind` or `navcoin-qt`; can be specified by `-debuglogfile` option `./` | `fee_estimates.dat` | Stores statistics used to estimate minimum transaction fees required for confirmation @@ -67,7 +67,7 @@ Subdirectory | File(s) | Description `./` | `onion_v3_private_key` | Cached Tor onion service private key for `-listenonion` option `./` | `i2p_private_key` | Private key that corresponds to our I2P address. When `-i2psam=` is specified the contents of this file is used to identify ourselves for making outgoing connections to I2P peers and possibly accepting incoming ones. Automatically generated if it does not exist. `./` | `peers.dat` | Peer IP address database (custom format) -`./` | `settings.json` | Read-write settings set through GUI or RPC interfaces, augmenting manual settings from [bitcoin.conf](bitcoin-conf.md). File is created automatically if read-write settings storage is not disabled with `-nosettings` option. Path can be specified with `-settings` option +`./` | `settings.json` | Read-write settings set through GUI or RPC interfaces, augmenting manual settings from [navcoin.conf](bitcoin-conf.md). File is created automatically if read-write settings storage is not disabled with `-nosettings` option. Path can be specified with `-settings` option `./` | `.cookie` | Session RPC authentication cookie; if used, created at start and deleted on shutdown; can be specified by `-rpccookiefile` option `./` | `.lock` | Data directory lock file diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 5f3d5cb5cbcc7..59e6b7855e0c4 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -219,44 +219,45 @@ $ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang \ ./configure --disable-wallet --with-gui=no \ --with-sanitizers=address,undefined $ git apply << "EOF" -diff --git a/src/navcoind.cpp b/src/navcoind.cpp -index 455a82e39..2faa3f80f 100644 ---- a/src/navcoind.cpp -+++ b/src/navcoind.cpp -@@ -158,7 +158,11 @@ static bool AppInit(int argc, char* argv[]) - return fRet; - } - +diff --git a/src/compat/compat.h b/src/compat/compat.h +index 8195bceaec..cce2b31ff0 100644 +--- a/src/compat/compat.h ++++ b/src/compat/compat.h +@@ -90,8 +90,12 @@ typedef char* sockopt_arg_type; + // building with a binutils < 2.36 is subject to this ld bug. + #define MAIN_FUNCTION __declspec(dllexport) int main(int argc, char* argv[]) + #else +#ifdef HFND_FUZZING_ENTRY_FUNCTION_CXX -+HFND_FUZZING_ENTRY_FUNCTION_CXX(int argc, char* argv[]) ++#define MAIN_FUNCTION HFND_FUZZING_ENTRY_FUNCTION_CXX(int argc, char* argv[]) +#else - int main(int argc, char* argv[]) + #define MAIN_FUNCTION int main(int argc, char* argv[]) + #endif +#endif - { - #ifdef WIN32 - util::WinCmdLineArgs winArgs; + + // Note these both should work with the current usage of poll, but best to be safe + // WIN32 poll is broken https://daniel.haxx.se/blog/2012/10/10/wsapoll-is-broken/ diff --git a/src/net.cpp b/src/net.cpp -index cf987b699..636a4176a 100644 +index 7601a6ea84..702d0f56ce 100644 --- a/src/net.cpp +++ b/src/net.cpp -@@ -709,7 +709,7 @@ int V1TransportDeserializer::readHeader(const char *pch, unsigned int nBytes) +@@ -727,7 +727,7 @@ int V1TransportDeserializer::readHeader(Span msg_bytes) } // Check start string, network magic - if (memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { + if (false && memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { // skip network magic checking - LogPrint(BCLog::NET, "HEADER ERROR - MESSAGESTART (%s, %u bytes), received %s, peer=%d\n", hdr.GetCommand(), hdr.nMessageSize, HexStr(hdr.pchMessageStart), m_node_id); + LogPrint(BCLog::NET, "Header error: Wrong MessageStart %s received, peer=%d\n", HexStr(hdr.pchMessageStart), m_node_id); return -1; } -@@ -768,7 +768,7 @@ Optional V1TransportDeserializer::GetMessage(const std::chrono::mic +@@ -788,7 +788,7 @@ CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds RandAddEvent(ReadLE32(hash.begin())); - // Check checksum and header command string + // Check checksum and header message type string - if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { + if (false && memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { // skip checksum checking - LogPrint(BCLog::NET, "CHECKSUM ERROR (%s, %u bytes), expected %s was %s, peer=%d\n", - SanitizeString(msg->m_command), msg->m_message_size, - HexStr(Span(hash.begin(), hash.begin() + CMessageHeader::CHECKSUM_SIZE)), + LogPrint(BCLog::NET, "Header error: Wrong checksum (%s, %u bytes), expected %s was %s, peer=%d\n", + SanitizeString(msg.m_type), msg.m_message_size, + HexStr(Span{hash}.first(CMessageHeader::CHECKSUM_SIZE)), EOF $ make -C src/ navcoind $ mkdir -p inputs/ diff --git a/doc/i2p.md b/doc/i2p.md index 61cf52966aff2..2af37413c7be9 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -9,16 +9,16 @@ started with I2P terminology. ## Run Bitcoin Core with an I2P router (proxy) -A running I2P router (proxy) with [SAM](https://geti2p.net/en/docs/api/samv3) -enabled is required. Options include: +A running I2P router (proxy) is required with the [SAM](https://geti2p.net/en/docs/api/samv3) +application bridge enabled. The following routers are recommended for use with Bitcoin Core: - [i2prouter (I2P Router)](https://geti2p.net), the official implementation in - Java + Java. The SAM bridge is not enabled by default; it must be started manually, + or configured to start automatically, in the Clients page in the + router console (`http://127.0.0.1:7657/configclients`) or in the `clients.config` file. - [i2pd (I2P Daemon)](https://github.com/PurpleI2P/i2pd) ([documentation](https://i2pd.readthedocs.io/en/latest)), a lighter - alternative in C++ -- [i2p-zero](https://github.com/i2p-zero/i2p-zero) -- [other alternatives](https://en.wikipedia.org/wiki/I2P#Routers) + alternative in C++. It enables the SAM bridge by default. Note the IP address and port the SAM proxy is listening to; usually, it is `127.0.0.1:7656`. @@ -109,8 +109,7 @@ incoming I2P connections (`-i2pacceptincoming`): To see which I2P peers your node is connected to, use `navcoin-cli -netinfo 4` or the `getpeerinfo` RPC (e.g. `navcoin-cli getpeerinfo`). -To see which I2P addresses your node knows, use the `getnodeaddresses 0 i2p` -RPC. +You can use the `getnodeaddresses` RPC to fetch a number of I2P peers known to your node; run `bitcoin-cli help getnodeaddresses` for details. ## Compatibility @@ -119,8 +118,7 @@ to connect to the I2P network. Any I2P router that supports it can be used. ## Ports in I2P and Bitcoin Core -Bitcoin Core uses the [SAM v3.1](https://geti2p.net/en/docs/api/samv3) -protocol. One particularity of SAM v3.1 is that it does not support ports, +One particularity of SAM v3.1 is that it does not support ports, unlike newer versions of SAM (v3.2 and up) that do support them and default the port numbers to 0. From the point of view of peers that use newer versions of SAM or other protocols that support ports, a SAM v3.1 peer is connecting to them diff --git a/doc/init.md b/doc/init.md index fbe2ba830ab65..0a83bf4bf744a 100644 --- a/doc/init.md +++ b/doc/init.md @@ -21,7 +21,7 @@ Configuration --------------------------------- Running navcoind as a daemon does not require any manual configuration. You may -set the `rpcauth` setting in the `bitcoin.conf` configuration file to override +set the `rpcauth` setting in the `navcoin.conf` configuration file to override the default behaviour of using a special cookie for authentication. This password does not have to be remembered or typed as it is mostly used @@ -44,7 +44,7 @@ This allows for running navcoind without having to do any manual configuration. relative to the data directory. `wallet` *only* supports relative paths. For an example configuration file that describes the configuration settings, -see `share/examples/bitcoin.conf`. +see `share/examples/navcoin.conf`. Paths --------------------------------- @@ -54,7 +54,7 @@ Paths All three configurations assume several paths that might need to be adjusted. Binary: /usr/bin/navcoind - Configuration file: /etc/bitcoin/bitcoin.conf + Configuration file: /etc/navcoin/navcoin.conf Data directory: /var/lib/navcoind PID file: /var/run/navcoind/navcoind.pid (OpenRC and Upstart) or /run/navcoind/navcoind.pid (systemd) @@ -74,10 +74,10 @@ bitcoin group to do so. This does not allow for the listing of files under the directory. NOTE: It is not currently possible to override `datadir` in -`/etc/bitcoin/bitcoin.conf` with the current systemd, OpenRC, and Upstart init +`/etc/navcoin/navcoin.conf` with the current systemd, OpenRC, and Upstart init files out-of-the-box. This is because the command line options specified in the init files take precedence over the configurations in -`/etc/bitcoin/bitcoin.conf`. However, some init systems have their own +`/etc/navcoin/navcoin.conf`. However, some init systems have their own configuration mechanisms that would allow for overriding the command line options specified in the init files (e.g. setting `BITCOIND_DATADIR` for OpenRC). @@ -85,7 +85,7 @@ OpenRC). ### macOS Binary: /usr/local/bin/navcoind - Configuration file: ~/Library/Application Support/Bitcoin/bitcoin.conf + Configuration file: ~/Library/Application Support/Bitcoin/navcoin.conf Data directory: ~/Library/Application Support/Bitcoin Lock file: ~/Library/Application Support/Bitcoin/.lock diff --git a/doc/man/navcoin-qt.1 b/doc/man/navcoin-qt.1 deleted file mode 100644 index 024cbdded849a..0000000000000 --- a/doc/man/navcoin-qt.1 +++ /dev/null @@ -1,5 +0,0 @@ -.TH BITCOIN-QT "1" -.SH NAME -navcoin-qt \- manual page for navcoin-qt - -This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/multiprocess.md b/doc/multiprocess.md new file mode 100644 index 0000000000000..3c6f61e13cde1 --- /dev/null +++ b/doc/multiprocess.md @@ -0,0 +1,34 @@ +# Multiprocess Bitcoin + +_This document describes usage of the multiprocess feature. For design information, see the [design/multiprocess.md](design/multiprocess.md) file._ + +## Build Option + +On unix systems, the `--enable-multiprocess` build option can be passed to `./configure` to build new `bitcoin-node`, `bitcoin-wallet`, and `bitcoin-gui` executables alongside existing `navcoind` and `bitcoin-qt` executables. + +## Debugging + +The `-debug=ipc` command line option can be used to see requests and responses between processes. + +## Installation + +The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) as dependencies. A simple way to get started using it without installing these dependencies manually is to use the [depends system](../depends) with the `MULTIPROCESS=1` [dependency option](../depends#dependency-options) passed to make: + +``` +cd +make -C depends NO_QT=1 MULTIPROCESS=1 +CONFIG_SITE=$PWD/depends/x86_64-pc-linux-gnu/share/config.site ./configure +make +src/bitcoin-node -regtest -printtoconsole -debug=ipc +BITCOIND=bitcoin-node test/functional/test_runner.py +``` + +The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). + +Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess/blob/master/doc/install.md) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. + +## Usage + +`bitcoin-node` is a drop-in replacement for `navcoind`, and `bitcoin-gui` is a drop-in replacement for `bitcoin-qt`, and there are no differences in use or external behavior between the new and old executables. But internally after [#10102](https://github.com/bitcoin/bitcoin/pull/10102), `bitcoin-gui` will spawn a `bitcoin-node` process to run P2P and RPC code, communicating with it across a socket pair, and `bitcoin-node` will spawn `bitcoin-wallet` to run wallet code, also communicating over a socket pair. This will let node, wallet, and GUI code run in separate address spaces for better isolation, and allow future improvements like being able to start and stop components independently on different machines and environments. +[#19460](https://github.com/bitcoin/bitcoin/pull/19460) also adds a new `bitcoin-node` `-ipcbind` option and an `navcoind-wallet` `-ipcconnect` option to allow new wallet processes to connect to an existing node process. +And [#19461](https://github.com/bitcoin/bitcoin/pull/19461) adds a new `bitcoin-gui` `-ipcconnect` option to allow new GUI processes to connect to an existing node process. diff --git a/doc/bitcoin-conf.md b/doc/navcoin-conf.md similarity index 86% rename from doc/bitcoin-conf.md rename to doc/navcoin-conf.md index e08528f478c31..ef32c0c008f6d 100644 --- a/doc/bitcoin-conf.md +++ b/doc/navcoin-conf.md @@ -1,4 +1,4 @@ -# `bitcoin.conf` Configuration File +# `navcoin.conf` Configuration File The configuration file is used by `navcoind`, `navcoin-qt` and `navcoin-cli`. @@ -51,24 +51,24 @@ rpcport=4000 ## Configuration File Path -The configuration file is not automatically created; you can create it using your favorite text editor. By default, the configuration file name is `bitcoin.conf` and it is located in the Bitcoin data directory, but both the Bitcoin data directory and the configuration file path may be changed using the `-datadir` and `-conf` command-line options. +The configuration file is not automatically created; you can create it using your favorite text editor. By default, the configuration file name is `navcoin.conf` and it is located in the Bitcoin data directory, but both the Bitcoin data directory and the configuration file path may be changed using the `-datadir` and `-conf` command-line options. -The `includeconf=` option in the `bitcoin.conf` file can be used to include additional configuration files. +The `includeconf=` option in the `navcoin.conf` file can be used to include additional configuration files. ### Default configuration file locations Operating System | Data Directory | Example Path -- | -- | -- -Windows | `%APPDATA%\Bitcoin\` | `C:\Users\username\AppData\Roaming\Bitcoin\bitcoin.conf` -Linux | `$HOME/.bitcoin/` | `/home/username/.bitcoin/bitcoin.conf` -macOS | `$HOME/Library/Application Support/Bitcoin/` | `/Users/username/Library/Application Support/Bitcoin/bitcoin.conf` +Windows | `%APPDATA%\Bitcoin\` | `C:\Users\username\AppData\Roaming\Bitcoin\navcoin.conf` +Linux | `$HOME/.bitcoin/` | `/home/username/.navcoin/navcoin.conf` +macOS | `$HOME/Library/Application Support/Bitcoin/` | `/Users/username/Library/Application Support/Bitcoin/navcoin.conf` -An example configuration file can be generated by [contrib/devtools/gen-bitcoin-conf.sh](../contrib/devtools/gen-bitcoin-conf.sh). +An example configuration file can be generated by [contrib/devtools/gen-navcoin-conf.sh](../contrib/devtools/gen-navcoin-conf.sh). Run this script after compiling to generate an up-to-date configuration file. -The output is placed under `share/examples/bitcoin.conf`. +The output is placed under `share/examples/navcoin.conf`. To use the generated configuration file, copy the example file into your data directory and edit it there, like so: ``` # example copy command for linux user -cp share/examples/bitcoin.conf ~/.bitcoin +cp share/examples/navcoin.conf ~/.bitcoin ``` diff --git a/doc/offline-signing-tutorial.md b/doc/offline-signing-tutorial.md new file mode 100644 index 0000000000000..4761cf256bdb3 --- /dev/null +++ b/doc/offline-signing-tutorial.md @@ -0,0 +1,255 @@ +# Offline Signing Tutorial + +This tutorial will describe how to use two instances of Bitcoin Core, one online and one offline, to greatly increase security by not having private keys reside on a networked device. + +Maintaining an air-gap between private keys and any network connections drastically reduces the opportunity for those keys to be exfiltrated from the user. + +This workflow uses [Partially Signed Bitcoin Transactions](https://github.com/bitcoin/bitcoin/blob/master/doc/psbt.md) (PSBTs) to transfer the transaction to and from the offline wallet for signing using the private keys. + +> [!NOTE] +> While this tutorial demonstrates the process using `signet` network, you should omit the `-signet` flag in the provided commands when working with `mainnet`. + +## Overview +In this tutorial we have two hosts, both running Bitcoin v25.0 + +* `offline` host which is disconnected from all networks (internet, Tor, wifi, bluetooth etc.) and does not have, or need, a copy of the blockchain. +* `online` host which is a regular online node with a synced blockchain. + +We are going to first create an `offline_wallet` on the offline host. We will then create a `watch_only_wallet` on the online host using public key descriptors exported from the `offline_wallet`. Next we will receive some coins into the wallet. In order to spend these coins we'll create an unsigned PSBT using the `watch_only_wallet`, sign the PSBT using the private keys in the `offline_wallet`, and finally broadcast the signed PSBT using the online host. + +### Requirements +- [jq](https://jqlang.github.io/jq/) installation - This tutorial uses jq to process certain fields from JSON RPC responses, but this convenience is optional. + +### Create and Prepare the `offline_wallet` + +1. On the offline machine create a wallet named `offline_wallet` secured by a wallet `passphrase`. This wallet will contain private keys and must remain unconnected to any networks at all times. + +```sh +[offline]$ ./src/bitcoin-cli -signet -named createwallet \ + wallet_name="offline_wallet" \ + passphrase="** enter passphrase **" + +{ + "name": "offline_wallet" +} +``` + +> [!NOTE] +> The use of a passphrase is crucial to encrypt the wallet.dat file. This encryption ensures that even if an unauthorized individual gains access to the offline host, they won't be able to access the wallet's contents. Further details about securing your wallet can be found in [Managing the Wallet](https://github.com/bitcoin/bitcoin/blob/master/doc/managing-wallets.md#12-encrypting-the-wallet) + +2. Export the public key-only descriptors from the offline host to a JSON file named `descriptors.json`. We use `jq` here to extract the `.descriptors` field from the full RPC response. + +```sh +[offline]$ ./src/bitcoin-cli -signet -rpcwallet="offline_wallet" listdescriptors \ + | jq -r '.descriptors' \ + >> /path/to/descriptors.json +``` + +> [!NOTE] +> The `descriptors.json` file will be transferred to the online machine (e.g. using a USB flash drive) where it can be imported to create a related watch-only wallet. + +### Create the online `watch_only_wallet` + +1. On the online machine create a blank watch-only wallet which has private keys disabled and is named `watch_only_wallet`. This is achieved by using the `createwallet` options: `disable_private_keys=true, blank=true`. + +The `watch_only_wallet` wallet will be used to track and validate incoming transactions, create unsigned PSBTs when spending coins, and broadcast signed and finalized PSBTs. + +> [!NOTE] +> `disable_private_keys` indicates that the wallet should refuse to import private keys, i.e. will be a dedicated watch-only wallet. + +```sh +[online]$ ./src/bitcoin-cli -signet -named createwallet \ + wallet_name="watch_only_wallet" \ + disable_private_keys=true + +{ + "name": "watch_only_wallet" +} +``` + +2. Import the `offline_wallet`s public key descriptors to the online `watch_only_wallet` using the `descriptors.json` file created on the offline wallet. + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" importdescriptors "$(cat /path/to/descriptors.json)" + +[ + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + } +] +``` +> [!NOTE] +> Multiple success values indicate that multiple descriptors, for different address types, have been successfully imported. This allows generating different address types on the `watch_only_wallet`. + +### Fund the `offline_wallet` + +At this point, it's important to understand that both the `offline_wallet` and online `watch_only_wallet` share the same public keys. As a result, they generate the same addresses. Transactions can be created using either wallet, but valid signatures can only be added by the `offline_wallet` as only it has the private keys. + +1. Generate an address to receive coins. You can use _either_ the `offline_wallet` or the online `watch_only_wallet` to generate this address, as they will produce the same addresses. For the sake of this guide, we'll use the online `watch_only_wallet` to generate the address. + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" getnewaddress + +tb1qtu5qgc6ddhmqm5yqjvhg83qgk2t4ewajg0h6yh +``` + +2. Visit a faucet like https://signet.bc-2.jp and enter your address from the previous command to receive a small amount of signet coins to this address. + +3. Confirm that coins were received using the online `watch_only_wallet`. Note that the transaction may take a few moments before being received on your local node, depending on its connectivity. Just re-run the command periodically until the transaction is received. + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" listunspent + +[ + { + "txid": "0f3953dfc3eb8e753cd1633151837c5b9953992914ff32b7de08c47f1f29c762", + "vout": 1, + "address": "tb1qtu5qgc6ddhmqm5yqjvhg83qgk2t4ewajg0h6yh", + "label": "", + "scriptPubKey": "00145f2804634d6df60dd080932e83c408b2975cbbb2", + "amount": 0.01000000, + "confirmations": 4, + "spendable": true, + "solvable": true, + "desc": "wpkh([306c734f/84h/1h/0h/0/0]025932ccee7590158f7e08bb36290d135d30a0b045163da896e1cd7645ec4223a9)#xytvyr4a", + "parent_descs": [ + "wpkh([306c734f/84h/1h/0h]tpubDCJnY92ib4Zu3qd6wrBXEjG436tQdA2tDiJU2iSJYjkNS1darssPWKaBfojhjUF5vMLBcxbN2r93pmFMz2zyTEZuNx9JDo9rWqoHhATW3Uz/0/*)#7mh08dkg" + ], + "safe": true + } +] +``` + +### Create and Export an Unsigned PSBT + +1. Get a destination address for the transaction. In this tutorial we'll be sending funds to the address `tb1q9k5w0nhnhyeh78snpxh0t5t7c3lxdeg3erez32`, but if you don't need the coins for further testing you could send the coins back to the faucet. + +2. Create a funded but unsigned PSBT to the destination address with the online `watch_only_wallet` by using `send [{"address":amount},...]` and export the unsigned PSBT to a file `funded_psbt.txt` for easy portability to the `offline_wallet` for signing: + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" send \ + '{"tb1q9k5w0nhnhyeh78snpxh0t5t7c3lxdeg3erez32": 0.009}' \ + | jq -r '.psbt' \ + >> /path/to/funded_psbt.txt + +[online]$ cat /path/to/funded_psbt.txt + +cHNidP8BAHECAAAAAWLHKR9/xAjetzL/FCmZU5lbfINRMWPRPHWO68PfUzkPAQAAAAD9////AoA4AQAAAAAAFgAULajnzvO5M38eEwmu9dF+xH5m5RGs0g0AAAAAABYAFMaT0f/Wp2DCZzL6dkJ3GhWj4Y9vAAAAAAABAHECAAAAAY+dRPEBrGopkw4ugSzS9npzJDEIrE/bq1XXI0KbYnYrAQAAAAD+////ArKaXgAAAAAAFgAUwEc4LdoxSjbWo/2Ue+HS+QjwfiBAQg8AAAAAABYAFF8oBGNNbfYN0ICTLoPECLKXXLuyYW8CAAEBH0BCDwAAAAAAFgAUXygEY01t9g3QgJMug8QIspdcu7IiBgJZMszudZAVj34IuzYpDRNdMKCwRRY9qJbhzXZF7EIjqRgwbHNPVAAAgAEAAIAAAACAAAAAAAAAAAAAACICA7BlBnyAR4F2UkKuSX9MFhYCsn6j//z9i7lHDm1O0CU0GDBsc09UAACAAQAAgAAAAIABAAAAAAAAAAA= +``` +> [!NOTE] +> Leaving the `input` array empty in the above `walletcreatefundedpsbt` command is permitted and will cause the wallet to automatically select appropriate inputs for the transaction. + +### Decode and Analyze the Unsigned PSBT + +Decode and analyze the unsigned PSBT on the `offline_wallet` using the `funded_psbt.txt` file: + +```sh +[offline]$ ./src/bitcoin-cli -signet decodepsbt $(cat /path/to/funded_psbt.txt) + +{ + ... +} + +[offline]$ ./src/bitcoin-cli -signet analyzepsbt $(cat /path/to/funded_psbt.txt) + +{ + "inputs": [ + { + "has_utxo": true, + "is_final": false, + "next": "signer", + "missing": { + "signatures": [ + "5f2804634d6df60dd080932e83c408b2975cbbb2" + ] + } + } + ], + "estimated_vsize": 141, + "estimated_feerate": 0.00100000, + "fee": 0.00014100, + "next": "signer" +} +``` + +Notice that the analysis of the PSBT shows that "signatures" are missing and should be provided by the private key corresponding to the public key hash (hash160) "5f2804634d6df60dd080932e83c408b2975cbbb2" + +### Process and Sign the PSBT + +1. Unlock the `offline_wallet` with the Passphrase: + +Use the walletpassphrase command to unlock the `offline_wallet` with the passphrase. You should specify the passphrase and a timeout (in seconds) for how long you want the wallet to remain unlocked. + +```sh +[offline]$ ./src/bitcoin-cli -signet -rpcwallet="offline_wallet" walletpassphrase "** enter passphrase **" 60 +``` + +2. Process, sign and finalize the PSBT on the `offline_wallet` using the `walletprocesspsbt` command, saving the output to a file `final_psbt.txt`. + + ```sh +[offline]$ ./src/bitcoin-cli -signet -rpcwallet="offline_wallet" walletprocesspsbt \ + $(cat /path/to/funded_psbt.txt) \ + | jq -r .hex \ + >> /path/to/final_psbt.txt + ``` + +### Broadcast the Signed and Finalized PSBT +Broadcast the funded, signed and finalized PSBT `final_psbt.txt` using `sendrawtransaction` with an online node: + +```sh +[online]$ ./src/bitcoin-cli -signet sendrawtransaction $(cat /path/to/final_psbt.txt) + +c2430a0e46df472b04b0ca887bbcd5c4abf7b2ce2eb71de981444a80e2b96d52 +``` + +### Confirm Wallet Balance + +Confirm the updated balance of the offline wallet using the `watch_only_wallet`. + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" getbalances + +{ + "mine": { + "trusted": 0.00085900, + "untrusted_pending": 0.00000000, + "immature": 0.00000000 + }, + "lastprocessedblock": { + "hash": "0000003065c0669fff27edb4a71928cb48e5a6cfcdf06f491a83fd86822d18a6", + "height": 159592 + } +} +``` + + +You can also show transactions related to the wallet using `listtransactions` + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" listtransactions + +{ + ... +} +``` \ No newline at end of file diff --git a/doc/policy/README.md b/doc/policy/README.md index 27536407e7df6..4392ffb610348 100644 --- a/doc/policy/README.md +++ b/doc/policy/README.md @@ -2,8 +2,8 @@ **Policy** (Mempool or Transaction Relay Policy) is the node's set of validation rules, in addition to consensus, enforced for unconfirmed transactions before submitting them to the mempool. These -rules are local to the node and configurable (e.g. `-minrelaytxfee`, `-limitancestorsize`, -`-incrementalrelayfee`). Policy may include restrictions on the transaction itself, the transaction +rules are local to the node and configurable, see "Node relay options" when running `-help`. +Policy may include restrictions on the transaction itself, the transaction in relation to the current chain tip, and the transaction in relation to the node's mempool contents. Policy is *not* applied to transactions in blocks. diff --git a/doc/policy/packages.md b/doc/policy/packages.md index 2a5758318a9b6..dba270e494067 100644 --- a/doc/policy/packages.md +++ b/doc/policy/packages.md @@ -18,16 +18,19 @@ tip or some preceding transaction in the package. The following rules are enforced for all packages: -* Packages cannot exceed `MAX_PACKAGE_COUNT=25` count and `MAX_PACKAGE_SIZE=101KvB` total size +* Packages cannot exceed `MAX_PACKAGE_COUNT=25` count and `MAX_PACKAGE_WEIGHT=404000` total weight (#20833) - - *Rationale*: This is already enforced as mempool ancestor/descendant limits. If - transactions in a package are all related, exceeding this limit would mean that the package - can either be split up or it wouldn't pass individual mempool policy. + - *Rationale*: We want package size to be as small as possible to mitigate DoS via package + validation. However, we want to make sure that the limit does not restrict ancestor + packages that would be allowed if submitted individually. - Note that, if these mempool limits change, package limits should be reconsidered. Users may also configure their mempool limits differently. + - Note that this is transaction weight, not "virtual" size as with other limits to allow + simpler context-less checks. + * Packages must be topologically sorted. (#20833) * Packages cannot have conflicting transactions, i.e. no two transactions in a package can spend diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 348ea52120358..5bc8853e81b04 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -43,7 +43,7 @@ threads take up 8MiB for the thread stack on a 64-bit system, and 4MiB in a ## Linux specific -By default, since glibc `2.10`, the C library will create up to two heap arenas per core. This is known to cause excessive memory usage in some scenarios. To avoid this make a script that sets `MALLOC_ARENA_MAX` before starting navcoind: +By default, glibc's implementation of `malloc` may use more than one arena. This is known to cause excessive memory usage in some scenarios. To avoid this, make a script that sets `MALLOC_ARENA_MAX` before starting navcoind: ```bash #!/usr/bin/env bash diff --git a/doc/reduce-traffic.md b/doc/reduce-traffic.md index 86943b1f727cf..8926a8361aa3e 100644 --- a/doc/reduce-traffic.md +++ b/doc/reduce-traffic.md @@ -3,10 +3,10 @@ Reduce Traffic Some node operators need to deal with bandwidth caps imposed by their ISPs. -By default, Bitcoin Core allows up to 125 connections to different peers, 10 of -which are outbound. You can therefore, have at most 115 inbound connections. -Of the 10 outbound peers, there can be 8 full-relay connections and 2 -block-relay-only ones. +By default, Bitcoin Core allows up to 125 connections to different peers, 11 of +which are outbound. You can therefore, have at most 114 inbound connections. +Of the 11 outbound peers, there can be 8 full-relay connections, 2 +block-relay-only ones and occasionally 1 short-lived feeler or an extra block-relay-only connection. The default settings can result in relatively significant traffic consumption. @@ -28,7 +28,7 @@ calculating the target. ## 2. Disable "listening" (`-listen=0`) -Disabling listening will result in fewer nodes connected (remember the maximum of 10 +Disabling listening will result in fewer nodes connected (remember the maximum of 11 outbound peers). Fewer nodes will result in less traffic usage as you are relaying blocks and transactions to fewer nodes. diff --git a/doc/release-notes-25158.md b/doc/release-notes-25158.md deleted file mode 100644 index ce8ab53ddd835..0000000000000 --- a/doc/release-notes-25158.md +++ /dev/null @@ -1,6 +0,0 @@ -RPC Wallet ----------- - -- The `gettransaction`, `listtransactions`, `listsinceblock` RPCs now return - the `abandoned` field for all transactions. Previously, the "abandoned" field - was only returned for sent transactions. (#25158) \ No newline at end of file diff --git a/doc/release-notes-26076.md b/doc/release-notes-26076.md deleted file mode 100644 index f95e4be0e3913..0000000000000 --- a/doc/release-notes-26076.md +++ /dev/null @@ -1,13 +0,0 @@ -RPC ---- - -- The `listdescriptors`, `decodepsbt` and similar RPC methods now show `h` rather than apostrophe (`'`) to indicate - hardened derivation. This does not apply when using the `private` parameter, which - matches the marker used when descriptor was generated or imported. Newly created - wallets use `h`. This change makes it easier to handle descriptor strings manually. - E.g. the `importdescriptors` RPC call is easiest to use `h` as the marker: `'["desc": ".../0h/..."]'`. - With this change `listdescriptors` will use `h`, so you can copy-paste the result, - without having to add escape characters or switch `'` to 'h' manually. - Note that this changes the descriptor checksum. - For legacy wallets the `hdkeypath` field in `getaddressinfo` is unchanged, - nor is the serialization format of wallet dumps. (#26076) diff --git a/doc/release-notes-26094.md b/doc/release-notes-26094.md deleted file mode 100644 index ba73f2707e72f..0000000000000 --- a/doc/release-notes-26094.md +++ /dev/null @@ -1,6 +0,0 @@ -- The `getbalances` RPC now returns a `lastprocessedblock` JSON object which contains the wallet's last processed block - hash and height at the time the balances were calculated. This result shouldn't be cached because importing new keys could invalidate it. -- The `gettransaction` RPC now returns a `lastprocessedblock` JSON object which contains the wallet's last processed block - hash and height at the time the transaction information was generated. -- The `getwalletinfo` RPC now returns a `lastprocessedblock` JSON object which contains the wallet's last processed block - hash and height at the time the wallet information was generated. \ No newline at end of file diff --git a/doc/release-notes-28207.md b/doc/release-notes-28207.md new file mode 100644 index 0000000000000..56b88da16ab27 --- /dev/null +++ b/doc/release-notes-28207.md @@ -0,0 +1,7 @@ +mempool.dat compatibility +======================== + +The `mempool.dat` file created by -persistmempool or the savemempool RPC will +be written in a new format, which can not be read by previous software +releases. To allow for a downgrade, a temporary setting `-persistmempoolv1` has +been added to fall back to the legacy format. diff --git a/doc/release-notes-empty-template.md b/doc/release-notes-empty-template.md index 0f908064ad514..dcd93bdedd2bd 100644 --- a/doc/release-notes-empty-template.md +++ b/doc/release-notes-empty-template.md @@ -36,9 +36,9 @@ Compatibility ============== Bitcoin Core is supported and extensively tested on operating systems -using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +using the Linux Kernel 3.17+, macOS 11.0+, and Windows 7 and newer. Bitcoin Core should also work on most other Unix-like systems but is not as -frequently tested on them. It is not recommended to use Bitcoin Core on +frequently tested on them. It is not recommended to use Bitcoin Core on unsupported systems. Notable changes diff --git a/doc/release-notes/release-notes-23.2.md b/doc/release-notes/release-notes-23.2.md new file mode 100644 index 0000000000000..2d6354ae5dfae --- /dev/null +++ b/doc/release-notes/release-notes-23.2.md @@ -0,0 +1,72 @@ +23.2 Release Notes +================== + +Bitcoin Core version 23.2 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### P2P + +- #26909 net: prevent peers.dat corruptions by only serializing once +- #27608 p2p: Avoid prematurely clearing download state for other peers +- #27610 Improve performance of p2p inv to send queues + +### Build system + +- #25436 build: suppress array-bounds errors in libxkbcommon +- #25763 bdb: disable Werror for format-security +- #26944 depends: fix systemtap download URL +- #27462 depends: fix compiling bdb with clang-16 on aarch64 + +### Miscellaneous + +- #25444 ci: macOS task imrovements +- #26388 ci: Use macos-ventura-xcode:14.1 image for "macOS native" task +- #26924 refactor: Add missing includes to fix gcc-13 compile error + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Anthony Towns +- Hennadii Stepanov +- MacroFake +- Martin Zumsande +- Michael Ford +- Suhas Daftuar + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). \ No newline at end of file diff --git a/doc/release-notes/release-notes-24.1.md b/doc/release-notes/release-notes-24.1.md new file mode 100644 index 0000000000000..f46434cc43f3d --- /dev/null +++ b/doc/release-notes/release-notes-24.1.md @@ -0,0 +1,99 @@ +24.1 Release Notes +================== + +Bitcoin Core version 24.1 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### P2P + +- #26878 I2P network optimizations +- #26909 net: prevent peers.dat corruptions by only serializing once +- #27608 p2p: Avoid prematurely clearing download state for other peers +- #27610 Improve performance of p2p inv to send queues + +### RPC and other APIs + +- #26515 rpc: Require NodeStateStats object in getpeerinfo +- #27279 doc: fix/improve warning helps in {create,load,unload,restore}wallet +- #27468 rest: avoid segfault for invalid URI + +### Build System + +- #26944 depends: fix systemtap download URL +- #27462 depends: fix compiling bdb with clang-16 on aarch64 + +### Wallet + +- #26595 wallet: be able to specify a wallet name and passphrase to migratewallet +- #26675 wallet: For feebump, ignore abandoned descendant spends +- #26679 wallet: Skip rescanning if wallet is more recent than tip +- #26761 wallet: fully migrate address book entries for watchonly/solvable wallets +- #27053 wallet: reuse change dest when re-creating TX with avoidpartialspends +- #27080 wallet: Zero out wallet master key upon locking so it doesn't persist in memory +- #27473 wallet: Properly handle "unknown" Address Type + +### GUI changes + +- gui#687 Load PSBTs using istreambuf_iterator rather than istream_iterator +- gui#704 Correctly limit overview transaction list + +### Miscellaneous + +- #26880 ci: replace Intel macOS CI job +- #26924 refactor: Add missing includes to fix gcc-13 compile error + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- Anthony Towns +- Hennadii Stepanov +- John Moffett +- Jon Atack +- Marco Falke +- Martin Zumsande +- Matthew Zipkin +- Michael Ford +- pablomartin4btc +- Sebastian Falbesoner +- Suhas Daftuar +- Thomas Nguyen +- Vasil Dimov + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). \ No newline at end of file diff --git a/doc/release-notes/release-notes-24.2.md b/doc/release-notes/release-notes-24.2.md new file mode 100644 index 0000000000000..059ab5187286d --- /dev/null +++ b/doc/release-notes/release-notes-24.2.md @@ -0,0 +1,76 @@ +24.2 Release Notes +================== + +Bitcoin Core version 24.2 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### Fees + +- #27622 Fee estimation: avoid serving stale fee estimate + +### RPC and other APIs + +- #27727 rpc: Fix invalid bech32 address handling + +### Build System + +- #28097 depends: xcb-proto 1.15.2 +- #28543 build, macos: Fix qt package build with new Xcode 15 linker +- #28571 depends: fix unusable memory_resource in macos qt build + +### CI + +- #27777 ci: Prune dangling images on RESTART_CI_DOCKER_BEFORE_RUN +- #27834 ci: Nuke Android APK task, Use credits for tsan +- #27844 ci: Use podman stop over podman kill +- #27886 ci: Switch to amd64 container in "ARM" task + +### Miscellaneous +- #28452 Do not use std::vector = {} to release memory + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Abubakar Sadiq Ismail +- Hennadii Stepanov +- Marco Falke +- Michael Ford +- Pieter Wuille + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-25.0.md b/doc/release-notes/release-notes-25.0.md new file mode 100644 index 0000000000000..919cb3b2f3444 --- /dev/null +++ b/doc/release-notes/release-notes-25.0.md @@ -0,0 +1,340 @@ +25.0 Release Notes +================== + +Bitcoin Core version 25.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +- Transactions of non-witness size 65 bytes and above are now allowed by mempool + and relay policy. This is to better reflect the actual afforded protections + against CVE-2017-12842 and open up additional use-cases of smaller transaction sizes. (#26265) + +New RPCs +-------- + +- The scanblocks RPC returns the relevant blockhashes from a set of descriptors by + scanning all blockfilters in the given range. It can be used in combination with + the getblockheader and rescanblockchain RPCs to achieve fast wallet rescans. Note + that this functionality can only be used if a compact block filter index + (-blockfilterindex=1) has been constructed by the node. (#23549) + +Updated RPCs +------------ + +- All JSON-RPC methods accept a new [named + parameter](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md#parameter-passing) called `args` that can + contain positional parameter values. This is a convenience to allow some + parameter values to be passed by name without having to name every value. The + python test framework and `bitcoin-cli` tool both take advantage of this, so + for example: + +```sh +bitcoin-cli -named createwallet wallet_name=mywallet load_on_startup=1 +``` + +Can now be shortened to: + +```sh +bitcoin-cli -named createwallet mywallet load_on_startup=1 +``` + +- The `verifychain` RPC will now return `false` if the checks didn't fail, + but couldn't be completed at the desired depth and level. This could be due + to missing data while pruning, due to an insufficient dbcache or due to + the node being shutdown before the call could finish. (#25574) + +- `sendrawtransaction` has a new, optional argument, `maxburnamount` with a default value of `0`. + Any transaction containing an unspendable output with a value greater than `maxburnamount` will + not be submitted. At present, the outputs deemed unspendable are those with scripts that begin + with an `OP_RETURN` code (known as 'datacarriers'), scripts that exceed the maximum script size, + and scripts that contain invalid opcodes. + +- The `testmempoolaccept` RPC now returns 2 additional results within the "fees" result: + "effective-feerate" is the feerate including fees and sizes of transactions validated together if + package validation was used, and also includes any modified fees from prioritisetransaction. The + "effective-includes" result lists the wtxids of transactions whose modified fees and sizes were used + in the effective-feerate (#26646). + +- `decodescript` may now infer a Miniscript descriptor under P2WSH context if it is not lacking + information. (#27037) + +- `finalizepsbt` is now able to finalize a transaction with inputs spending Miniscript-compatible + P2WSH scripts. (#24149) + +Changes to wallet related RPCs can be found in the Wallet section below. + +Build System +------------ + +- The `--enable-upnp-default` and `--enable-natpmp-default` options + have been removed. If you want to use port mapping, you can + configure it using a .conf file, or by passing the relevant + options at runtime. (#26896) + +Updated settings +---------------- + +- If the `-checkblocks` or `-checklevel` options are explicitly provided by the +user, but the verification checks cannot be completed due to an insufficient +dbcache, Bitcoin Core will now return an error at startup. (#25574) + +- Ports specified in `-port` and `-rpcport` options are now validated at startup. + Values that previously worked and were considered valid can now result in errors. (#22087) + +- Setting `-blocksonly` will now reduce the maximum mempool memory + to 5MB (users may still use `-maxmempool` to override). Previously, + the default 300MB would be used, leading to unexpected memory usage + for users running with `-blocksonly` expecting it to eliminate + mempool memory usage. + + As unused mempool memory is shared with dbcache, this also reduces + the dbcache size for users running with `-blocksonly`, potentially + impacting performance. +- Setting `-maxconnections=0` will now disable `-dnsseed` + and `-listen` (users may still set them to override). + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings +------------ + +- The `shutdownnotify` option is used to specify a command to execute synchronously +before Bitcoin Core has begun its shutdown sequence. (#23395) + + +Wallet +------ + +- The `minconf` option, which allows a user to specify the minimum number +of confirmations a UTXO being spent has, and the `maxconf` option, +which allows specifying the maximum number of confirmations, have been +added to the following RPCs in #25375: + - `fundrawtransaction` + - `send` + - `walletcreatefundedpsbt` + - `sendall` + +- Added a new `next_index` field in the response in `listdescriptors` to + have the same format as `importdescriptors` (#26194) + +- RPC `listunspent` now has a new argument `include_immature_coinbase` + to include coinbase UTXOs that don't meet the minimum spendability + depth requirement (which before were silently skipped). (#25730) + +- Rescans for descriptor wallets are now significantly faster if compact + block filters (BIP158) are available. Since those are not constructed + by default, the configuration option "-blockfilterindex=1" has to be + provided to take advantage of the optimization. This improves the + performance of the RPC calls `rescanblockchain`, `importdescriptors` + and `restorewallet`. (#25957) + +- RPC `unloadwallet` now fails if a rescan is in progress. (#26618) + +- Wallet passphrases may now contain null characters. + Prior to this change, only characters up to the first + null character were recognized and accepted. (#27068) + +- Address Purposes strings are now restricted to the currently known values of "send", + "receive", and "refund". Wallets that have unrecognized purpose strings will have + loading warnings, and the `listlabels` RPC will raise an error if an unrecognized purpose + is requested. (#27217) + +- In the `createwallet`, `loadwallet`, `unloadwallet`, and `restorewallet` RPCs, the + "warning" string field is deprecated in favor of a "warnings" field that + returns a JSON array of strings to better handle multiple warning messages and + for consistency with other wallet RPCs. The "warning" field will be fully + removed from these RPCs in v26. It can be temporarily re-enabled during the + deprecation period by launching bitcoind with the configuration option + `-deprecatedrpc=walletwarningfield`. (#27279) + +- Descriptor wallets can now spend coins sent to P2WSH Miniscript descriptors. (#24149) + +GUI changes +----------- + +- The "Mask values" is a persistent option now. (gui#701) +- The "Mask values" option affects the "Transaction" view now, in addition to the + "Overview" one. (gui#708) + +REST +---- + +- A new `/rest/deploymentinfo` endpoint has been added for fetching various + state info regarding deployments of consensus changes. (#25412) + +Binary verification +---- + +- The binary verification script has been updated. In previous releases it + would verify that the binaries had been signed with a single "release key". + In this release and moving forward it will verify that the binaries are + signed by a _threshold of trusted keys_. For more details and + examples, see: + https://github.com/bitcoin/bitcoin/blob/master/contrib/verify-binaries/README.md + (#27358) + +Low-level changes +================= + +RPC +--- + +- The JSON-RPC server now rejects requests where a parameter is specified multiple + times with the same name, instead of silently overwriting earlier parameter values + with later ones. (#26628) +- RPC `listsinceblock` now accepts an optional `label` argument + to fetch incoming transactions having the specified label. (#25934) +- Previously `setban`, `addpeeraddress`, `walletcreatefundedpsbt`, methods + allowed non-boolean and non-null values to be passed as boolean parameters. + Any string, number, array, or object value that was passed would be treated + as false. After this change, passing any value except `true`, `false`, or + `null` now triggers a JSON value is not of expected type error. (#26213) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 721217.xyz +- @RandyMcMillan +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- Aurèle Oulès +- Ben Woosley +- Bitcoin Hodler +- brunoerg +- Bushstar +- Carl Dong +- Chris Geihsler +- Cory Fields +- David Gumberg +- dergoegge +- Dhruv Mehta +- Dimitris Tsapakidis +- dougEfish +- Douglas Chimento +- ekzyis +- Elichai Turkel +- Ethan Heilman +- Fabian Jahr +- FractalEncrypt +- furszy +- Gleb Naumenko +- glozow +- Greg Sanders +- Hennadii Stepanov +- hernanmarino +- ishaanam +- ismaelsadeeq +- James O'Beirne +- jdjkelly@gmail.com +- Jeff Ruane +- Jeffrey Czyz +- Jeremy Rubin +- Jesse Barton +- João Barbosa +- JoaoAJMatos +- John Moffett +- Jon Atack +- Jonas Schnelli +- jonatack +- Joshua Kelly +- josibake +- Juan Pablo Civile +- kdmukai +- klementtan +- Kolby ML +- kouloumos +- Kristaps Kaupe +- laanwj +- Larry Ruane +- Leonardo Araujo +- Leonardo Lazzaro +- Luke Dashjr +- MacroFake +- MarcoFalke +- Martin Leitner-Ankerl +- Martin Zumsande +- Matt Whitlock +- Matthew Zipkin +- Michael Ford +- Miles Liu +- mruddy +- Murray Nesbitt +- muxator +- omahs +- pablomartin4btc +- Pasta +- Pieter Wuille +- Pttn +- Randall Naar +- Riahiamirreza +- roconnor-blockstream +- Russell O'Connor +- Ryan Ofsky +- S3RK +- Sebastian Falbesoner +- Seibart Nedor +- sinetek +- Sjors Provoost +- Skuli Dulfari +- SomberNight +- Stacie Waleyko +- stickies-v +- stratospher +- Suhas Daftuar +- Suriyaa Sundararuban +- TheCharlatan +- Vasil Dimov +- Vasil Stoyanov +- virtu +- w0xlt +- willcl-ark +- yancy +- Yusuf Sahin HAMZA + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). \ No newline at end of file diff --git a/doc/release-notes/release-notes-25.1.md b/doc/release-notes/release-notes-25.1.md new file mode 100644 index 0000000000000..bfdbee4e76f1e --- /dev/null +++ b/doc/release-notes/release-notes-25.1.md @@ -0,0 +1,108 @@ +25.1 Release Notes +================== + +Bitcoin Core version 25.1 is now available from: + + + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +### P2P + +- #27626 Parallel compact block downloads, take 3 +- #27743 p2p: Unconditionally return when compact block status == READ_STATUS_FAILED + +### Fees + +- #27622 Fee estimation: avoid serving stale fee estimate + +### RPC + +- #27727 rpc: Fix invalid bech32 address handling + +### Rest + +- #27853 rest: fix crash error when calling /deploymentinfo +- #28551 http: bugfix: allow server shutdown in case of remote client disconnection + +### Wallet + +- #28038 wallet: address book migration bug fixes +- #28067 descriptors: do not return top-level only funcs as sub descriptors +- #28125 wallet: bugfix, disallow migration of invalid scripts +- #28542 wallet: Check for uninitialized last processed and conflicting heights in MarkConflicted + +### Build + +- #27724 build: disable boost multi index safe mode in debug mode +- #28097 depends: xcb-proto 1.15.2 +- #28543 build, macos: Fix qt package build with new Xcode 15 linker +- #28571 depends: fix unusable memory_resource in macos qt build + +### Gui + +- gui#751 macOS, do not process actions during shutdown + +### Miscellaneous + +- #28452 Do not use std::vector = {} to release memory + +### CI + +- #27777 ci: Prune dangling images on RESTART_CI_DOCKER_BEFORE_RUN +- #27834 ci: Nuke Android APK task, Use credits for tsan +- #27844 ci: Use podman stop over podman kill +- #27886 ci: Switch to amd64 container in "ARM" task + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Abubakar Sadiq Ismail +- Andrew Chow +- Bruno Garcia +- Gregory Sanders +- Hennadii Stepanov +- MacroFake +- Matias Furszyfer +- Michael Ford +- Pieter Wuille +- stickies-v +- Will Clark + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). \ No newline at end of file diff --git a/doc/release-notes/release-notes-26.0.md b/doc/release-notes/release-notes-26.0.md new file mode 100644 index 0000000000000..b7c7c35f65957 --- /dev/null +++ b/doc/release-notes/release-notes-26.0.md @@ -0,0 +1,357 @@ +26.0 Release Notes +================== + +Bitcoin Core version 26.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 11.0+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +- Experimental support for the v2 transport protocol defined in + [BIP324](https://github.com/bitcoin/bips/blob/master/bip-0324.mediawiki) was added. + It is off by default, but when enabled using `-v2transport` it will be negotiated + on a per-connection basis with other peers that support it too. The existing + v1 transport protocol remains fully supported. + +- Nodes with multiple reachable networks will actively try to have at least one + outbound connection to each network. This improves individual resistance to + eclipse attacks and network level resistance to partition attacks. Users no + longer need to perform active measures to ensure being connected to multiple + enabled networks. (#27213) + +Pruning +------- + +- When using assumeutxo with `-prune`, the prune budget may be exceeded if it is set + lower than 1100MB (i.e. `MIN_DISK_SPACE_FOR_BLOCK_FILES * 2`). Prune budget is normally + split evenly across each chainstate, unless the resulting prune budget per chainstate + is beneath `MIN_DISK_SPACE_FOR_BLOCK_FILES` in which case that value will be used. (#27596) + +Updated RPCs +------------ + +- Setting `-rpcserialversion=0` is deprecated and will be removed in + a future release. It can currently still be used by also adding + the `-deprecatedrpc=serialversion` option. (#28448) + +- The `hash_serialized_2` value has been removed from `gettxoutsetinfo` since the value it + calculated contained a bug and did not take all data into account. It is superseded by + `hash_serialized_3` which provides the same functionality but serves the correctly calculated hash. (#28685) + +- New fields `transport_protocol_type` and `session_id` were added to the `getpeerinfo` RPC to indicate + whether the v2 transport protocol is in use, and if so, what the session id is. + +- A new argument `v2transport` was added to the `addnode` RPC to indicate whether a v2 transaction connection + is to be attempted with the peer. + +- [Miniscript](https://bitcoin.sipa.be/miniscript/) expressions can now be used in Taproot descriptors for all RPCs working with descriptors. (#27255) + +- `finalizepsbt` is now able to finalize a PSBT with inputs spending [Miniscript](https://bitcoin.sipa.be/miniscript/)-compatible Taproot leaves. (#27255) + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +- `loadtxoutset` has been added, which allows loading a UTXO snapshot of the format + generated by `dumptxoutset`. Once this snapshot is loaded, its contents will be + deserialized into a second chainstate data structure, which is then used to sync to + the network's tip. + + Meanwhile, the original chainstate will complete the initial block download process in + the background, eventually validating up to the block that the snapshot is based upon. + + The result is a usable bitcoind instance that is current with the network tip in a + matter of minutes rather than hours. UTXO snapshot are typically obtained via + third-party sources (HTTP, torrent, etc.) which is reasonable since their contents + are always checked by hash. + + You can find more information on this process in the `assumeutxo` design + document (). + + `getchainstates` has been added to aid in monitoring the assumeutxo sync process. + +- A new `getprioritisedtransactions` RPC has been added. It returns a map of all fee deltas created by the + user with prioritisetransaction, indexed by txid. The map also indicates whether each transaction is + present in the mempool. (#27501) + +- A new RPC, `submitpackage`, has been added. It can be used to submit a list of raw hex +transactions to the mempool to be evaluated as a package using consensus and mempool policy rules. +These policies include package CPFP, allowing a child with high fees to bump a parent below the +mempool minimum feerate (but not minimum relay feerate). (#27609) + + - Warning: successful submission does not mean the transactions will propagate throughout the + network, as package relay is not supported. + + - Not all features are available. The package is limited to a child with all of its + unconfirmed parents, and no parent may spend the output of another parent. Also, package + RBF is not supported. Refer to doc/policy/packages.md for more details on package policies + and limitations. + + - This RPC is experimental. Its interface may change. + +- A new RPC `getaddrmaninfo` has been added to view the distribution of addresses in the new and tried table of the + node's address manager across different networks(ipv4, ipv6, onion, i2p, cjdns). The RPC returns count of addresses + in new and tried table as well as their sum for all networks. (#27511) + +- A new `importmempool` RPC has been added. It loads a valid `mempool.dat` file and attempts to + add its contents to the mempool. This can be useful to import mempool data from another node + without having to modify the datadir contents and without having to restart the node. (#27460) + - Warning: Importing untrusted files is dangerous, especially if metadata from the file is taken over. + - If you want to apply fee deltas, it is recommended to use the `getprioritisedtransactions` and + `prioritisetransaction` RPCs instead of the `apply_fee_delta_priority` option to avoid + double-prioritising any already-prioritised transactions in the mempool. + +Updated settings +---------------- + +- `bitcoind` and `bitcoin-qt` will now raise an error on startup + if a datadir that is being used contains a bitcoin.conf file that + will be ignored, which can happen when a datadir= line is used in + a bitcoin.conf file. The error message is just a diagnostic intended + to prevent accidental misconfiguration, and it can be disabled to + restore the previous behavior of using the datadir while ignoring + the bitcoin.conf contained in it. (#27302) + +- Passing an invalid `-debug`, `-debugexclude`, or `-loglevel` logging configuration + option now raises an error, rather than logging an easily missed warning. (#27632) + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings +------------ + +Tools and Utilities +------------------- + +- A new `bitcoinconsensus_verify_script_with_spent_outputs` function is available in libconsensus which optionally accepts the spent outputs of the transaction being verified. +- A new `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT` flag is available in libconsensus that will verify scripts with the Taproot spending rules. + +Wallet +------ + +- Wallet loading has changed in this release. Wallets with some corrupted records that could be + previously loaded (with warnings) may no longer load. For example, wallets with corrupted + address book entries may no longer load. If this happens, it is recommended + load the wallet in a previous version of Bitcoin Core and import the data into a new wallet. + Please also report an issue to help improve the software and make wallet loading more robust + in these cases. (#24914) + +- The `createwallet` RPC will no longer create legacy (BDB) wallets when + setting `descriptors=false` without also providing the + `-deprecatedrpc=create_bdb` option. This is because the legacy wallet is + being deprecated in a future release. (#28597) + +- The `gettransaction`, `listtransactions`, `listsinceblock` RPCs now return + the `abandoned` field for all transactions. Previously, the "abandoned" field + was only returned for sent transactions. (#25158) + +- The `listdescriptors`, `decodepsbt` and similar RPC methods now show `h` rather than apostrophe (`'`) to indicate + hardened derivation. This does not apply when using the `private` parameter, which + matches the marker used when descriptor was generated or imported. Newly created + wallets use `h`. This change makes it easier to handle descriptor strings manually. + E.g. the `importdescriptors` RPC call is easiest to use `h` as the marker: `'["desc": ".../0h/..."]'`. + With this change `listdescriptors` will use `h`, so you can copy-paste the result, + without having to add escape characters or switch `'` to 'h' manually. + Note that this changes the descriptor checksum. + For legacy wallets the `hdkeypath` field in `getaddressinfo` is unchanged, + nor is the serialization format of wallet dumps. (#26076) + +- The `getbalances` RPC now returns a `lastprocessedblock` JSON object which contains the wallet's last processed block + hash and height at the time the balances were calculated. This result shouldn't be cached because importing new keys could invalidate it. (#26094) + +- The `gettransaction` RPC now returns a `lastprocessedblock` JSON object which contains the wallet's last processed block + hash and height at the time the transaction information was generated. (#26094) + +- The `getwalletinfo` RPC now returns a `lastprocessedblock` JSON object which contains the wallet's last processed block + hash and height at the time the wallet information was generated. (#26094) + +- Coin selection and transaction building now accounts for unconfirmed low-feerate ancestor transactions. When it is necessary to spend unconfirmed outputs, the wallet will add fees to ensure that the new transaction with its ancestors will achieve a mining score equal to the feerate requested by the user. (#26152) + +- For RPC methods which accept `options` parameters ((`importmulti`, `listunspent`, + `fundrawtransaction`, `bumpfee`, `send`, `sendall`, `walletcreatefundedpsbt`, + `simulaterawtransaction`), it is now possible to pass the options as named + parameters without the need for a nested object. (#26485) + +This means it is possible make calls like: + +```sh +src/bitcoin-cli -named bumpfee txid fee_rate=100 +``` + +instead of + +```sh +src/bitcoin-cli -named bumpfee txid options='{"fee_rate": 100}' +``` + +- The `deprecatedrpc=walletwarningfield` configuration option has been removed. + The `createwallet`, `loadwallet`, `restorewallet` and `unloadwallet` RPCs no + longer return the "warning" string field. The same information is provided + through the "warnings" field added in v25.0, which returns a JSON array of + strings. The "warning" string field was deprecated also in v25.0. (#27757) + +- The `signrawtransactionwithkey`, `signrawtransactionwithwallet`, + `walletprocesspsbt` and `descriptorprocesspsbt` calls now return the more + specific RPC_INVALID_PARAMETER error instead of RPC_MISC_ERROR if their + sighashtype argument is malformed. (#28113) + +- RPC `walletprocesspsbt`, and `descriptorprocesspsbt` return + object now includes field `hex` (if the transaction + is complete) containing the serialized transaction + suitable for RPC `sendrawtransaction`. (#28414) + +- It's now possible to use [Miniscript](https://bitcoin.sipa.be/miniscript/) inside Taproot leaves for descriptor wallets. (#27255) + +Descriptors +----------- + +- The usage of hybrid public keys in output descriptors has been removed. Hybrid + public keys are an exotic public key encoding not supported by output descriptors + (as specified in BIP380 and documented in doc/descriptors.md). Bitcoin Core would + previously incorrectly accept descriptors containing such hybrid keys. (#28587) + +GUI changes +----------- + +- The transaction list in the GUI no longer provides a special category for "payment to yourself". Now transactions that have both inputs and outputs that affect the wallet are displayed on separate lines for spending and receiving. (gui#119) + +- A new menu option allows migrating a legacy wallet based on keys and implied output script types stored in BerkeleyDB (BDB) to a modern wallet that uses descriptors stored in SQLite. (gui#738) + +- The PSBT operations dialog marks outputs paying your own wallet with "own address". (gui#740) + +- The ability to create legacy wallets is being removed. (gui#764) + +Contrib +------- + +- Bash completion files have been renamed from `bitcoin*.bash-completion` to + `bitcoin*.bash`. This means completions can be automatically loaded on demand + based on invoked commands' names when they are put into the completion + directory (found with `pkg-config --variable=completionsdir + bash-completion`) without requiring renaming. (#28507) + +Low-level changes +================= + +Tests +----- + +- Non-standard transactions are now disabled by default on testnet + for relay and mempool acceptance. The previous behaviour can be + re-enabled by setting `-acceptnonstdtxn=1`. (#28354) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- Amiti Uttarwar +- Andrew Chow +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- Antoine Riard +- Ari +- Aurèle Oulès +- Ayush Singh +- Ben Woosley +- Brandon Odiwuor +- Brotcrunsher +- brunoerg +- Bufo +- Carl Dong +- Casey Carter +- Cory Fields +- David Álvarez Rosa +- dergoegge +- dhruv +- dimitaracev +- Erik Arvstedt +- Erik McKelvey +- Fabian Jahr +- furszy +- glozow +- Greg Sanders +- Harris +- Hennadii Stepanov +- Hernan Marino +- ishaanam +- ismaelsadeeq +- Jake Rawsthorne +- James O'Beirne +- John Moffett +- Jon Atack +- josibake +- kevkevin +- Kiminuo +- Larry Ruane +- Luke Dashjr +- MarcoFalke +- Marnix +- Martin Leitner-Ankerl +- Martin Zumsande +- Matthew Zipkin +- Michael Ford +- Michael Tidwell +- mruddy +- Murch +- ns-xvrn +- pablomartin4btc +- Pieter Wuille +- Reese Russell +- Rhythm Garg +- Ryan Ofsky +- Sebastian Falbesoner +- Sjors Provoost +- stickies-v +- stratospher +- Suhas Daftuar +- TheCharlatan +- Tim Neubauer +- Tim Ruffing +- Vasil Dimov +- virtu +- vuittont60 +- willcl-ark +- Yusuf Sahin HAMZA + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 32306ab4f15a7..a4df57cf14cf5 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -8,11 +8,11 @@ Release Process * Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). * Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). * Update manpages (after rebuilding the binaries), see [gen-manpages.py](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagespy). -* Update bitcoin.conf and commit, see [gen-bitcoin-conf.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-bitcoin-confsh). +* Update navcoin.conf and commit, see [gen-navcoin-conf.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-navcoin-confsh). ### Before every major and minor release -* Update [bips.md](bips.md) to account for changes since the last release (don't forget to bump the version number on the first line). +* Update [bips.md](bips.md) to account for changes since the last release. * Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_RC` to `0`). * Update manpages (see previous section) * Write release notes (see "Write the release notes" below). @@ -28,7 +28,7 @@ Release Process #### Before branch-off -* Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/7415) for an example. +* Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/27488) for an example. * Update the following variables in [`src/kernel/chainparams.cpp`](/src/kernel/chainparams.cpp) for mainnet, testnet, and signet: - `m_assumed_blockchain_size` and `m_assumed_chain_state_size` with the current size plus some overhead (see [this](#how-to-calculate-assumed-blockchain-and-chain-state-size) for information on how to calculate them). @@ -36,13 +36,21 @@ Release Process that causes rejection of blocks in the past history. - `chainTxData` with statistics about the transaction count and rate. Use the output of the `getchaintxstats` RPC with an `nBlocks` of 4096 (28 days) and a `bestblockhash` of RPC `getbestblockhash`; see - [this pull request](https://github.com/bitcoin/bitcoin/pull/20263) for an example. Reviewers can verify the results by running + [this pull request](https://github.com/bitcoin/bitcoin/pull/28591) for an example. Reviewers can verify the results by running `getchaintxstats ` with the `window_block_count` and `window_final_block_hash` from your output. - `defaultAssumeValid` with the output of RPC `getblockhash` using the `height` of `window_final_block_height` above (and update the block height comment with that height), taking into account the following: - On mainnet, the selected value must not be orphaned, so it may be useful to set the height two blocks back from the tip. - Testnet should be set with a height some tens of thousands back from the tip, due to reorgs there. - `nMinimumChainWork` with the "chainwork" value of RPC `getblockheader` using the same height as that selected for the previous step. +* Consider updating the headers synchronization tuning parameters to account for the chainparams updates. + The optimal values change very slowly, so this isn't strictly necessary every release, but doing so doesn't hurt. + - Update configuration variables in [`contrib/devtools/headerssync-params.py`](/contrib/devtools/headerssync-params.py): + - Set `TIME` to the software's expected supported lifetime -- after this time, its ability to defend against a high bandwidth timewarp attacker will begin to degrade. + - Set `MINCHAINWORK_HEADERS` to the height used for the `nMinimumChainWork` calculation above. + - Check that the other variables still look reasonable. + - Run the script. It works fine in CPython, but PyPy is much faster (seconds instead of minutes): `pypy3 contrib/devtools/headerssync-params.py`. + - Paste the output defining `HEADER_COMMITMENT_PERIOD` and `REDOWNLOAD_BUFFER_SIZE` into the top of [`src/headerssync.cpp`](/src/headerssync.cpp). - Clear the release notes and move them to the wiki (see "Write the release notes" below). - Translations on Transifex: - Pull translations from Transifex into the master branch. @@ -57,7 +65,7 @@ Release Process - Update the versions. - Create the draft, named "*version* Release Notes Draft", as a [collaborative wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/_new). - Clear the release notes: `cp doc/release-notes-empty-template.md doc/release-notes.md` -- Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. +- Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/27621) for an example) and provide a link to it in the release announcements where useful. - Translations on Transifex - Change the auto-update URL for the new major version's resource away from `master` and to the branch, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin//src/qt/locale/bitcoin_en.xlf`. Do not forget this or it will keep tracking the translations on master instead, drifting away from the specific major release. - Prune inputs from the qa-assets repo (See [pruning @@ -66,13 +74,15 @@ Release Process #### Before final release - Merge the release notes from [the wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/) into the branch. -- Ensure the "Needs release note" label is removed from all relevant pull requests and issues. +- Ensure the "Needs release note" label is removed from all relevant pull + requests and issues: + https://github.com/bitcoin/bitcoin/issues?q=label%3A%22Needs+release+note%22 #### Tagging a release (candidate) To tag the version (or release candidate) in git, use the `make-tag.py` script from [bitcoin-maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools). From the root of the repository run: - ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 23.0) + ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 25.0) This will perform a few last-minute consistency checks in the build system files, and if they pass, create a signed tag. @@ -96,11 +106,9 @@ Open a draft of the release notes for collaborative editing at https://github.co For the period during which the notes are being edited on the wiki, the version on the branch should be wiped and replaced with a link to the wiki which should be used for all announcements until `-final`. -Generate the change log. As this is a huge amount of work to do manually, there is the `list-pulls` script to do a pre-sorting step based on github PR metadata. See the [documentation in the README.md](https://github.com/bitcoin-core/bitcoin-maintainer-tools/blob/master/README.md#list-pulls). - Generate list of authors: - git log --format='- %aN' v(current version, e.g. 24.0)..v(new version, e.g. 24.1) | sort -fiu + git log --format='- %aN' v(current version, e.g. 25.0)..v(new version, e.g. 25.1) | grep -v 'merge-script' | sort -fiu ### Setup and perform Guix builds @@ -109,7 +117,7 @@ Checkout the Bitcoin Core version you'd like to build: ```sh pushd ./bitcoin SIGNER='(your builder key, ie bluematt, sipa, etc)' -VERSION='(new version without v-prefix, e.g. 24.0)' +VERSION='(new version without v-prefix, e.g. 25.0)' git fetch origin "v${VERSION}" git checkout "v${VERSION}" popd @@ -125,7 +133,7 @@ git -C ./guix.sigs pull ### Create the macOS SDK tarball (first time, or when SDK version changes) Create the macOS SDK tarball, see the [macdeploy -instructions](/contrib/macdeploy/README.md#deterministic-macos-dmg-notes) for +instructions](/contrib/macdeploy/README.md#deterministic-macos-app-notes) for details. ### Build and attest to build outputs @@ -144,10 +152,11 @@ Follow the relevant Guix README.md sections: pushd ./guix.sigs git add "${VERSION}/${SIGNER}"/noncodesigned.SHA256SUMS{,.asc} git commit -m "Add attestations by ${SIGNER} for ${VERSION} non-codesigned" -git push # Assuming you can push to the guix.sigs tree popd ``` +Then open a Pull Request to the [guix.sigs repository](https://github.com/bitcoin-core/guix.sigs). + ## Codesigning ### macOS codesigner only: Create detached macOS signatures (assuming [signapple](https://github.com/achow101/signapple/) is installed and up to date with master branch) @@ -202,10 +211,11 @@ popd pushd ./guix.sigs git add "${VERSION}/${SIGNER}"/all.SHA256SUMS{,.asc} git commit -m "Add attestations by ${SIGNER} for ${VERSION} codesigned" -git push # Assuming you can push to the guix.sigs tree popd ``` +Then open a Pull Request to the [guix.sigs repository](https://github.com/bitcoin-core/guix.sigs). + ## After 3 or more people have guix-built and their results match Combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`: @@ -268,23 +278,13 @@ cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc - bitcoincore.org RPC documentation update - - Install [golang](https://golang.org/doc/install) - - - Install the new Bitcoin Core release - - - Run navcoind on regtest - - - Clone the [bitcoincore.org repository](https://github.com/bitcoin-core/bitcoincore.org) - - - Run: `go run generate.go` while being in `contrib/doc-gen` folder, and with navcoin-cli in PATH - - - Add the generated files to git + - See https://github.com/bitcoin-core/bitcoincore.org/blob/master/contrib/doc-gen/ - Update packaging repo - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.navcoin-qt/pull/2 - - Push the snap, see https://github.com/bitcoin-core/packaging/blob/master/snap/build.md + - Push the snap, see https://github.com/bitcoin-core/packaging/blob/main/snap/local/build.md - This repo diff --git a/doc/shared-libraries.md b/doc/shared-libraries.md index 147e2237115ec..a9b47eb5335fb 100644 --- a/doc/shared-libraries.md +++ b/doc/shared-libraries.md @@ -1,50 +1,77 @@ Shared Libraries ================ -## bitcoinconsensus +## navcoinconsensus The purpose of this library is to make the verification functionality that is critical to Bitcoin's consensus available to other applications, e.g. to language bindings. ### API -The interface is defined in the C header `bitcoinconsensus.h` located in `src/script/bitcoinconsensus.h`. +The interface is defined in the C header `navcoinconsensus.h` located in `src/script/navcoinconsensus.h`. #### Version -`bitcoinconsensus_version` returns an `unsigned int` with the API version *(currently `1`)*. +`navcoinconsensus_version` returns an `unsigned int` with the API version *(currently `2`)*. #### Script Validation -`bitcoinconsensus_verify_script` returns an `int` with the status of the verification. It will be `1` if the input script correctly spends the previous output `scriptPubKey`. +`navcoinconsensus_verify_script`, `navcoinconsensus_verify_script_with_amount` and `navcoinconsensus_verify_script_with_spent_outputs` return an `int` with the status of the verification. It will be `1` if the input script correctly spends the previous output `scriptPubKey`. ##### Parameters +###### navcoinconsensus_verify_script - `const unsigned char *scriptPubKey` - The previous output script that encumbers spending. - `unsigned int scriptPubKeyLen` - The number of bytes for the `scriptPubKey`. - `const unsigned char *txTo` - The transaction with the input that is spending the previous output. - `unsigned int txToLen` - The number of bytes for the `txTo`. - `unsigned int nIn` - The index of the input in `txTo` that spends the `scriptPubKey`. - `unsigned int flags` - The script validation flags *(see below)*. -- `bitcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. +- `navcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. + +###### navcoinconsensus_verify_script_with_amount +- `const unsigned char *scriptPubKey` - The previous output script that encumbers spending. +- `unsigned int scriptPubKeyLen` - The number of bytes for the `scriptPubKey`. +- `int64_t amount` - The amount spent in the input +- `const unsigned char *txTo` - The transaction with the input that is spending the previous output. +- `unsigned int txToLen` - The number of bytes for the `txTo`. +- `unsigned int nIn` - The index of the input in `txTo` that spends the `scriptPubKey`. +- `unsigned int flags` - The script validation flags *(see below)*. +- `navcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. + +###### navcoinconsensus_verify_script_with_spent_outputs +- `const unsigned char *scriptPubKey` - The previous output script that encumbers spending. +- `unsigned int scriptPubKeyLen` - The number of bytes for the `scriptPubKey`. +- `int64_t amount` - The amount spent in the input +- `const unsigned char *txTo` - The transaction with the input that is spending the previous output. +- `unsigned int txToLen` - The number of bytes for the `txTo`. +- `UTXO *spentOutputs` - Previous outputs spent in the transaction. `UTXO` is a struct composed by `const unsigned char *scriptPubKey`, `unsigned int scriptPubKeySize` (the number of bytes for the `scriptPubKey`) and `unsigned int value`. +- `unsigned int spentOutputsLen` - The number of bytes for the `spentOutputs`. +- `unsigned int nIn` - The index of the input in `txTo` that spends the `scriptPubKey`. +- `unsigned int flags` - The script validation flags *(see below)*. +- `navcoinconsensus_error* err` - Will have the error/success code for the operation *(see below)*. ##### Script Flags -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NONE` -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH` - Evaluate P2SH ([BIP16](https://github.com/bitcoin/bips/blob/master/bip-0016.mediawiki)) subscripts -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG` - Enforce strict DER ([BIP66](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki)) compliance -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY` - Enforce NULLDUMMY ([BIP147](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY` - Enable CHECKLOCKTIMEVERIFY ([BIP65](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY` - Enable CHECKSEQUENCEVERIFY ([BIP112](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki)) -- `bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS` - Enable WITNESS ([BIP141](https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki)) +- `navcoinconsensus_SCRIPT_FLAGS_VERIFY_NONE` +- `navcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH` - Evaluate P2SH ([BIP16](https://github.com/bitcoin/bips/blob/master/bip-0016.mediawiki)) subscripts +- `navcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG` - Enforce strict DER ([BIP66](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki)) compliance +- `navcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY` - Enforce NULLDUMMY ([BIP147](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki)) +- `navcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY` - Enable CHECKLOCKTIMEVERIFY ([BIP65](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki)) +- `navcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY` - Enable CHECKSEQUENCEVERIFY ([BIP112](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki)) +- `navcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS` - Enable WITNESS ([BIP141](https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki)) +- `navcoinconsensus_SCRIPT_FLAGS_VERIFY_TAPROOT` - Enable TAPROOT ([BIP340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki), [BIP341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki), [BIP342](https://github.com/bitcoin/bips/blob/master/bip-0342.mediawiki)) ##### Errors -- `bitcoinconsensus_ERR_OK` - No errors with input parameters *(see the return value of `bitcoinconsensus_verify_script` for the verification status)* -- `bitcoinconsensus_ERR_TX_INDEX` - An invalid index for `txTo` -- `bitcoinconsensus_ERR_TX_SIZE_MISMATCH` - `txToLen` did not match with the size of `txTo` -- `bitcoinconsensus_ERR_DESERIALIZE` - An error deserializing `txTo` -- `bitcoinconsensus_ERR_AMOUNT_REQUIRED` - Input amount is required if WITNESS is used -- `bitcoinconsensus_ERR_INVALID_FLAGS` - Script verification `flags` are invalid (i.e. not part of the libconsensus interface) +- `navcoinconsensus_ERR_OK` - No errors with input parameters *(see the return value of `navcoinconsensus_verify_script` for the verification status)* +- `navcoinconsensus_ERR_TX_INDEX` - An invalid index for `txTo` +- `navcoinconsensus_ERR_TX_SIZE_MISMATCH` - `txToLen` did not match with the size of `txTo` +- `navcoinconsensus_ERR_DESERIALIZE` - An error deserializing `txTo` +- `navcoinconsensus_ERR_AMOUNT_REQUIRED` - Input amount is required if WITNESS is used +- `navcoinconsensus_ERR_INVALID_FLAGS` - Script verification `flags` are invalid (i.e. not part of the libconsensus interface) +- `navcoinconsensus_ERR_SPENT_OUTPUTS_REQUIRED` - Spent outputs are required if TAPROOT is used +- `navcoinconsensus_ERR_SPENT_OUTPUTS_MISMATCH` - Spent outputs size doesn't match tx inputs size ### Example Implementations - [NBitcoin](https://github.com/MetacoSA/NBitcoin/blob/5e1055cd7c4186dee4227c344af8892aea54faec/NBitcoin/Script.cs#L979-#L1031) (.NET Bindings) -- [node-libbitcoinconsensus](https://github.com/bitpay/node-libbitcoinconsensus) (Node.js Bindings) -- [java-libbitcoinconsensus](https://github.com/dexX7/java-libbitcoinconsensus) (Java Bindings) -- [bitcoinconsensus-php](https://github.com/Bit-Wasp/bitcoinconsensus-php) (PHP Bindings) +- [node-libnavcoinconsensus](https://github.com/bitpay/node-libnavcoinconsensus) (Node.js Bindings) +- [java-libnavcoinconsensus](https://github.com/dexX7/java-libnavcoinconsensus) (Java Bindings) +- [navcoinconsensus-php](https://github.com/Bit-Wasp/navcoinconsensus-php) (PHP Bindings) +- [rust-navcoinconsensus](https://github.com/rust-bitcoin/rust-navcoinconsensus) (Rust Bindings) \ No newline at end of file diff --git a/doc/tor.md b/doc/tor.md index e92d9c52b7798..b75fcac33970c 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -2,9 +2,7 @@ It is possible to run Bitcoin Core as a Tor onion service, and connect to such services. -The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. See [Tor Project FAQ:TBBSocksPort](https://www.torproject.org/docs/faq.html.en#TBBSocksPort) for how to properly -configure Tor. - +The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. ## Compatibility - Starting with version 22.0, Bitcoin Core only supports Tor version 3 hidden @@ -27,8 +25,7 @@ CLI `-addrinfo` returns the number of addresses known to your node per network. This can be useful to see how many onion peers your node knows, e.g. for `-onlynet=onion`. -To fetch a number of onion addresses that your node knows, for example seven -addresses, use the `getnodeaddresses 7 onion` RPC. +You can use the `getnodeaddresses` RPC to fetch a number of onion peers known to your node; run `bitcoin-cli help getnodeaddresses` for details. ## 1. Run Bitcoin Core behind a Tor proxy diff --git a/doc/tracing.md b/doc/tracing.md index 258a36398e6e5..91b353b755697 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -220,7 +220,7 @@ about the transaction. Arguments passed: 1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) -2. Transaction virtual size as `uint64` +2. Transaction virtual size as `int32` 3. Transaction fee as `int64` #### Tracepoint `mempool:removed` @@ -231,7 +231,7 @@ about the transaction. Arguments passed: 1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) 2. Removal reason as `pointer to C-style String` (max. length 9 characters) -3. Transaction virtual size as `uint64` +3. Transaction virtual size as `int32` 4. Transaction fee as `int64` 5. Transaction mempool entry time (epoch) as `uint64` @@ -242,11 +242,11 @@ Passes information about the replaced and replacement transactions. Arguments passed: 1. Replaced transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) -2. Replaced transaction virtual size as `uint64` +2. Replaced transaction virtual size as `int32` 3. Replaced transaction fee as `int64` 4. Replaced transaction mempool entry time (epoch) as `uint64` 5. Replacement transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) -6. Replacement transaction virtual size as `uint64` +6. Replacement transaction virtual size as `int32` 7. Replacement transaction fee as `int64` Note: In cases where a single replacement transaction replaces multiple diff --git a/doc/zmq.md b/doc/zmq.md index fe1c317be108e..df1546c854c4c 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -91,7 +91,7 @@ For instance: Each PUB notification has a topic and body, where the header corresponds to the notification type. For instance, for the notification `-zmqpubhashtx` the topic is `hashtx` (no null -terminator). These options can also be provided in bitcoin.conf. +terminator). These options can also be provided in navcoin.conf. The topics are: @@ -113,11 +113,11 @@ Where the 8-byte uints correspond to the mempool sequence number. | hashtx | <32-byte transaction hash in Little Endian> | -`rawblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`rawblock`), the second part is the serialized block, and the last part is a sequence number (representing the message count to detect lost messages). +`rawblock`: Notifies when the chain tip is updated. When assumeutxo is in use, this notification will not be issued for historical blocks connected to the background validation chainstate. Messages are ZMQ multipart messages with three parts. The first part is the topic (`rawblock`), the second part is the serialized block, and the last part is a sequence number (representing the message count to detect lost messages). | rawblock | | -`hashblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`hashblock`), the second part is the 32-byte block hash, and the last part is a sequence number (representing the message count to detect lost messages). +`hashblock`: Notifies when the chain tip is updated. When assumeutxo is in use, this notification will not be issued for historical blocks connected to the background validation chainstate. Messages are ZMQ multipart messages with three parts. The first part is the topic (`hashblock`), the second part is the 32-byte block hash, and the last part is a sequence number (representing the message count to detect lost messages). | hashblock | <32-byte block hash in Little Endian> | diff --git a/libbitcoinconsensus.pc.in b/libnavcoinconsensus.pc.in similarity index 100% rename from libbitcoinconsensus.pc.in rename to libnavcoinconsensus.pc.in diff --git a/share/examples/bitcoin.conf b/share/examples/navcoin.conf similarity index 50% rename from share/examples/bitcoin.conf rename to share/examples/navcoin.conf index 5bee4bf92e73c..70c2e093f239c 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/navcoin.conf @@ -1 +1 @@ -# This is a placeholder file. Please follow the instructions in `contrib/devtools/README.md` to generate a bitcoin.conf file. +# This is a placeholder file. Please follow the instructions in `contrib/devtools/README.md` to generate a navcoin.conf file. diff --git a/src/.bear-tidy-config b/src/.bear-tidy-config index b17a247a6c418..1e0870a93c422 100644 --- a/src/.bear-tidy-config +++ b/src/.bear-tidy-config @@ -4,8 +4,14 @@ "include_only_existing_source": true, "paths_to_include": [], "paths_to_exclude": [ + "src/crc32c", + "src/crypto/ctaes", "src/leveldb", - "src/bls" + "src/bls", + "src/minisketch", + "src/bench/nanobench.cpp", + "src/bench/nanobench.h", + "src/secp256k1" ] }, "format": { diff --git a/src/.clang-format b/src/.clang-format index 791b3b8f9f068..f20e5ee2d4ca1 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -43,5 +43,7 @@ SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false -Standard: c++17 +BreakBeforeConceptDeclarations: Always +RequiresExpressionIndentation: OuterScope +Standard: c++20 UseTab: Never diff --git a/src/.clang-tidy b/src/.clang-tidy index 39566c0cb7fcc..bfaa5ab8e7b09 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -1,11 +1,17 @@ Checks: ' -*, +bitcoin-*, bugprone-argument-comment, +bugprone-string-constructor, bugprone-use-after-move, +bugprone-lambda-function-name, misc-unused-using-decls, modernize-use-default-member-init, +modernize-use-emplace, +modernize-use-noexcept, modernize-use-nullptr, performance-*, +-performance-avoid-endl, -performance-inefficient-string-concatenation, -performance-no-int-to-ptr, -performance-noexcept-move-constructor, diff --git a/src/Makefile.am b/src/Makefile.am index 5c2a77d453890..45b3b32193625 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,8 +8,8 @@ print-%: FORCE DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) $(CORE_LDFLAGS) -AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) $(CORE_CXXFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(CORE_LDFLAGS) +AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(CORE_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) $(CORE_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) @@ -26,7 +26,7 @@ BENCHMARKS = BLS_INCLUDES=-I$(srcdir)/bls/include -I$(srcdir)/bls/mcl/include -I$(srcdir)/bls/mcl/src -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include $(BLS_INCLUDES) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(LEVELDB_CPPFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include $(BLS_INCLUDES) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a @@ -42,10 +42,10 @@ if ENABLE_ZMQ LIBBITCOIN_ZMQ=libbitcoin_zmq.a endif if BUILD_BITCOIN_LIBS -LIBBITCOINCONSENSUS=libbitcoinconsensus.la +LIBBITCOINCONSENSUS=libnavcoinconsensus.la endif if BUILD_BITCOIN_KERNEL_LIB -LIBBITCOINKERNEL=libbitcoinkernel.la +LIBBITCOINKERNEL=libnavcoinkernel.la endif if ENABLE_WALLET LIBBITCOIN_WALLET=libbitcoin_wallet.a @@ -74,21 +74,19 @@ noinst_LTLIBRARIES += $(LIBBITCOIN_CRYPTO) $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) -LIBBLS_A = bls/lib/libbls384_256.a $(LIBBLS): $(AM_V_at) \ ARCH='$(firstword $(subst -, ,$(HOST)))' \ CC='$(CC)' \ CXX='$(CXX)' \ - CFLAGS='$(CXXFLAGS) -DBLS_ETH=1' \ + CFLAGS='$(CXXFLAGS)' \ LDFLAGS='$(LDFLAGS)' \ ARFLAGS='$(ARFLAGS)' \ AR='$(AR)' \ RANLIB='$(RANLIB)' \ $(MAKE) $(AM_MAKEFLAGS) BLS_ETH=1 -C bls lib/libbls384_256.a - $(RANLIB) $(LIBBLS_A) + $(RANLIB) $(LIBBLS) -LIBMCL_A = bls/mcl/lib/libmcl.a $(LIBMCL): $(AM_V_at) \ MCL_USE_LLVM=0 \ @@ -101,7 +99,7 @@ $(LIBMCL): AR='$(AR)' \ RANLIB='$(RANLIB)' \ $(MAKE) $(AM_MAKEFLAGS) -C bls/mcl lib/libmcl.a - $(RANLIB) $(LIBMCL_A) + $(RANLIB) $(LIBMCL) # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: @@ -147,17 +145,17 @@ if BUILD_BITCOIN_CHAINSTATE endif BLSCT_H = \ + blsct/arith/elements.h \ + blsct/arith/endianness.h \ blsct/arith/mcl/mcl.h \ blsct/arith/mcl/mcl_g1point.h \ blsct/arith/mcl/mcl_init.h \ blsct/arith/mcl/mcl_scalar.h \ blsct/arith/mcl/mcl_util.h \ - blsct/arith/elements.h \ - blsct/arith/endianness.h \ blsct/bech32_mod.h \ blsct/building_block/fiat_shamir.h \ - blsct/building_block/generator_deriver.h \ blsct/building_block/g_h_gi_hi_zero_verifier.h \ + blsct/building_block/generator_deriver.h \ blsct/building_block/imp_inner_prod_arg.h \ blsct/building_block/lazy_point.h \ blsct/building_block/lazy_points.h \ @@ -166,19 +164,20 @@ BLSCT_H = \ blsct/common.h \ blsct/double_public_key.h \ blsct/eip_2333/bls12_381_keygen.h \ + blsct/external_api/blsct.h \ blsct/private_key.h \ blsct/public_key.h \ blsct/public_keys.h \ blsct/range_proof/bulletproofs/amount_recovery_request.h \ blsct/range_proof/bulletproofs/amount_recovery_result.h \ + blsct/range_proof/bulletproofs/range_proof.h \ blsct/range_proof/bulletproofs/range_proof_logic.h \ blsct/range_proof/bulletproofs/range_proof_with_transcript.h \ - blsct/range_proof/bulletproofs/range_proof.h \ blsct/range_proof/bulletproofs_plus/amount_recovery_request.h \ blsct/range_proof/bulletproofs_plus/amount_recovery_result.h \ + blsct/range_proof/bulletproofs_plus/range_proof.h \ blsct/range_proof/bulletproofs_plus/range_proof_logic.h \ blsct/range_proof/bulletproofs_plus/range_proof_with_transcript.h \ - blsct/range_proof/bulletproofs_plus/range_proof.h \ blsct/range_proof/bulletproofs_plus/util.h \ blsct/range_proof/common.h \ blsct/range_proof/generators.h \ @@ -186,9 +185,9 @@ BLSCT_H = \ blsct/range_proof/proof_base.h \ blsct/range_proof/recovered_data.h \ blsct/range_proof/setup.h \ - blsct/set_mem_proof/set_mem_proof_setup.h \ blsct/set_mem_proof/set_mem_proof.h \ blsct/set_mem_proof/set_mem_proof_prover.h \ + blsct/set_mem_proof/set_mem_proof_setup.h \ blsct/signature.h \ blsct/wallet/address.h \ blsct/wallet/hdchain.h \ @@ -199,11 +198,11 @@ BLSCT_H = \ blsct/wallet/verification.h BLSCT_CPP = \ + blsct/arith/elements.cpp \ blsct/arith/mcl/mcl_g1point.cpp \ blsct/arith/mcl/mcl_scalar.cpp \ - blsct/arith/elements.cpp \ - blsct/building_block/generator_deriver.cpp \ blsct/building_block/g_h_gi_hi_zero_verifier.cpp \ + blsct/building_block/generator_deriver.cpp \ blsct/building_block/imp_inner_prod_arg.cpp \ blsct/building_block/lazy_point.cpp \ blsct/building_block/lazy_points.cpp \ @@ -211,18 +210,19 @@ BLSCT_CPP = \ blsct/common.cpp \ blsct/double_public_key.cpp \ blsct/eip_2333/bls12_381_keygen.cpp \ + blsct/external_api/blsct.cpp \ blsct/private_key.cpp \ blsct/public_key.cpp \ blsct/public_keys.cpp \ blsct/range_proof/bulletproofs/amount_recovery_request.cpp \ blsct/range_proof/bulletproofs/amount_recovery_result.cpp \ - blsct/range_proof/bulletproofs/range_proof_logic.cpp \ blsct/range_proof/bulletproofs/range_proof.cpp \ + blsct/range_proof/bulletproofs/range_proof_logic.cpp \ blsct/range_proof/bulletproofs/range_proof_with_transcript.cpp \ blsct/range_proof/bulletproofs_plus/amount_recovery_request.cpp \ blsct/range_proof/bulletproofs_plus/amount_recovery_result.cpp \ - blsct/range_proof/bulletproofs_plus/range_proof_logic.cpp \ blsct/range_proof/bulletproofs_plus/range_proof.cpp \ + blsct/range_proof/bulletproofs_plus/range_proof_logic.cpp \ blsct/range_proof/bulletproofs_plus/range_proof_with_transcript.cpp \ blsct/range_proof/bulletproofs_plus/util.cpp \ blsct/range_proof/common.cpp \ @@ -230,21 +230,24 @@ BLSCT_CPP = \ blsct/range_proof/msg_amt_cipher.cpp \ blsct/range_proof/proof_base.cpp \ blsct/set_mem_proof/set_mem_proof.cpp \ - blsct/set_mem_proof/set_mem_proof_setup.cpp \ blsct/set_mem_proof/set_mem_proof_prover.cpp \ - blsct/wallet/verification.cpp \ - blsct/signature.cpp + blsct/set_mem_proof/set_mem_proof_setup.cpp \ + blsct/signature.cpp \ + blsct/wallet/verification.cpp + .PHONY: FORCE check-symbols check-security # navcoin core # BITCOIN_CORE_H = \ addrdb.h \ + addresstype.h \ addrman.h \ addrman_impl.h \ attributes.h \ banman.h \ base58.h \ bech32.h \ + bip324.h \ blockencodings.h \ blockfilter.h \ dandelion.h \ @@ -265,6 +268,8 @@ BITCOIN_CORE_H = \ compat/compat.h \ compat/cpuid.h \ compat/endian.h \ + common/settings.h \ + common/system.h \ compressor.h \ consensus/consensus.h \ consensus/tx_check.h \ @@ -305,10 +310,14 @@ BITCOIN_CORE_H = \ kernel/coinstats.h \ kernel/context.h \ kernel/cs_main.h \ + kernel/disconnected_transactions.h \ kernel/mempool_entry.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ kernel/mempool_persist.h \ + kernel/mempool_removal_reason.h \ + kernel/messagestartchars.h \ + kernel/notifications_interface.h \ kernel/validation_cache_sizes.h \ key.h \ key_io.h \ @@ -325,6 +334,7 @@ BITCOIN_CORE_H = \ netbase.h \ netgroup.h \ netmessagemaker.h \ + node/abort.h \ node/blockmanager_args.h \ node/blockstorage.h \ node/caches.h \ @@ -337,10 +347,14 @@ BITCOIN_CORE_H = \ node/database_args.h \ node/eviction.h \ node/interface_ui.h \ + node/kernel_notifications.h \ node/mempool_args.h \ node/mempool_persist_args.h \ node/miner.h \ + node/mini_miner.h \ node/minisketchwrapper.h \ + node/peerman_args.h \ + node/protocol_version.h \ node/psbt.h \ node/transaction.h \ node/txreconciliation.h \ @@ -380,8 +394,7 @@ BITCOIN_CORE_H = \ script/sigcache.h \ script/sign.h \ script/signingprovider.h \ - script/standard.h \ - shutdown.h \ + script/solver.h \ signet.h \ streams.h \ support/allocators/pool.h \ @@ -399,7 +412,9 @@ BITCOIN_CORE_H = \ txorphanage.h \ txrequest.h \ undo.h \ + util/any.h \ util/asmap.h \ + util/batchpriority.h \ util/bip32.h \ util/bitdeque.h \ util/bytevectorhash.h \ @@ -412,10 +427,10 @@ BITCOIN_CORE_H = \ util/fees.h \ util/fs.h \ util/fs_helpers.h \ - util/getuniquepath.h \ util/golombrice.h \ util/hash_type.h \ util/hasher.h \ + util/insert.h \ util/macros.h \ util/message.h \ util/moneystr.h \ @@ -425,19 +440,18 @@ BITCOIN_CORE_H = \ util/readwritefile.h \ util/result.h \ util/serfloat.h \ - util/settings.h \ + util/signalinterrupt.h \ util/sock.h \ util/spanparsing.h \ util/string.h \ - util/syscall_sandbox.h \ util/syserror.h \ - util/system.h \ util/thread.h \ util/threadinterrupt.h \ util/threadnames.h \ util/time.h \ util/tokenpipe.h \ util/trace.h \ + util/transaction_identifier.h \ util/translation.h \ util/types.h \ util/ui_change_type.h \ @@ -481,17 +495,18 @@ BITCOIN_CORE_H = \ obj/build.h: FORCE @$(MKDIR_P) $(builddir)/obj - @$(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ + $(AM_V_GEN) $(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # node # -libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ addrdb.cpp \ addrman.cpp \ banman.cpp \ + bip324.cpp \ blockencodings.cpp \ blockfilter.cpp \ chain.cpp \ @@ -514,11 +529,14 @@ libbitcoin_node_a_SOURCES = \ kernel/coinstats.cpp \ kernel/context.cpp \ kernel/cs_main.cpp \ + kernel/disconnected_transactions.cpp \ kernel/mempool_persist.cpp \ + kernel/mempool_removal_reason.cpp \ mapport.cpp \ net.cpp \ net_processing.cpp \ netgroup.cpp \ + node/abort.cpp \ node/blockmanager_args.cpp \ node/blockstorage.cpp \ node/caches.cpp \ @@ -532,10 +550,13 @@ libbitcoin_node_a_SOURCES = \ node/eviction.cpp \ node/interface_ui.cpp \ node/interfaces.cpp \ + node/kernel_notifications.cpp \ node/mempool_args.cpp \ node/mempool_persist_args.cpp \ node/miner.cpp \ + node/mini_miner.cpp \ node/minisketchwrapper.cpp \ + node/peerman_args.cpp \ node/psbt.cpp \ node/transaction.cpp \ node/txreconciliation.cpp \ @@ -562,7 +583,6 @@ libbitcoin_node_a_SOURCES = \ rpc/signmessage.cpp \ rpc/txoutproof.cpp \ script/sigcache.cpp \ - shutdown.cpp \ signet.cpp \ timedata.cpp \ torcontrol.cpp \ @@ -602,11 +622,11 @@ endif libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(BDB_CPPFLAGS) $(SQLITE_CFLAGS) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ + blsct/arith/elements.cpp \ blsct/arith/mcl/mcl_g1point.cpp \ blsct/arith/mcl/mcl_scalar.cpp \ - blsct/arith/elements.cpp \ - blsct/building_block/generator_deriver.cpp \ blsct/building_block/g_h_gi_hi_zero_verifier.cpp \ + blsct/building_block/generator_deriver.cpp \ blsct/building_block/imp_inner_prod_arg.cpp \ blsct/building_block/lazy_point.cpp \ blsct/building_block/lazy_points.cpp \ @@ -619,13 +639,13 @@ libbitcoin_wallet_a_SOURCES = \ blsct/public_keys.cpp \ blsct/range_proof/bulletproofs/amount_recovery_request.cpp \ blsct/range_proof/bulletproofs/amount_recovery_result.cpp \ - blsct/range_proof/bulletproofs/range_proof_logic.cpp \ blsct/range_proof/bulletproofs/range_proof.cpp \ + blsct/range_proof/bulletproofs/range_proof_logic.cpp \ blsct/range_proof/bulletproofs/range_proof_with_transcript.cpp \ blsct/range_proof/bulletproofs_plus/amount_recovery_request.cpp \ blsct/range_proof/bulletproofs_plus/amount_recovery_result.cpp \ - blsct/range_proof/bulletproofs_plus/range_proof_logic.cpp \ blsct/range_proof/bulletproofs_plus/range_proof.cpp \ + blsct/range_proof/bulletproofs_plus/range_proof_logic.cpp \ blsct/range_proof/bulletproofs_plus/range_proof_with_transcript.cpp \ blsct/range_proof/bulletproofs_plus/util.cpp \ blsct/range_proof/common.cpp \ @@ -633,8 +653,8 @@ libbitcoin_wallet_a_SOURCES = \ blsct/range_proof/msg_amt_cipher.cpp \ blsct/range_proof/proof_base.cpp \ blsct/set_mem_proof/set_mem_proof.cpp \ - blsct/set_mem_proof/set_mem_proof_setup.cpp \ blsct/set_mem_proof/set_mem_proof_prover.cpp \ + blsct/set_mem_proof/set_mem_proof_setup.cpp \ blsct/signature.cpp \ blsct/wallet/address.cpp \ blsct/wallet/keyman.cpp \ @@ -702,8 +722,8 @@ crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/aes.h \ crypto/chacha20.cpp \ crypto/chacha20.h \ - crypto/chacha_poly_aead.cpp \ - crypto/chacha_poly_aead.h \ + crypto/chacha20poly1305.cpp \ + crypto/chacha20poly1305.h \ crypto/common.h \ crypto/hkdf_sha256_32.cpp \ crypto/hkdf_sha256_32.h \ @@ -782,11 +802,11 @@ libbitcoin_consensus_a_SOURCES = \ blsct/arith/mcl/mcl_scalar.cpp \ blsct/building_block/generator_deriver.cpp \ blsct/common.cpp \ + blsct/range_proof/bulletproofs/range_proof.cpp \ + blsct/range_proof/bulletproofs_plus/range_proof.cpp \ blsct/range_proof/common.cpp \ blsct/range_proof/generators.cpp \ blsct/range_proof/proof_base.cpp \ - blsct/range_proof/bulletproofs/range_proof.cpp \ - blsct/range_proof/bulletproofs_plus/range_proof.cpp \ blsct/signature.cpp \ consensus/amount.h \ consensus/merkle.cpp \ @@ -803,7 +823,7 @@ libbitcoin_consensus_a_SOURCES = \ primitives/transaction.h \ pubkey.cpp \ pubkey.h \ - script/bitcoinconsensus.cpp \ + script/navcoinconsensus.cpp \ script/interpreter.cpp \ script/interpreter.h \ script/script.cpp \ @@ -818,14 +838,14 @@ libbitcoin_consensus_a_SOURCES = \ uint256.h \ util/moneystr.cpp \ util/strencodings.cpp \ - util/strencodings.h \ - version.h + util/strencodings.h # # common # libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ + addresstype.cpp \ base58.cpp \ bech32.cpp \ blsct/arith/mcl/mcl_g1point.cpp \ @@ -843,18 +863,15 @@ libbitcoin_common_a_SOURCES = \ blsct/private_key.cpp \ blsct/public_key.cpp \ blsct/public_keys.cpp \ - blsct/range_proof/bulletproofs/range_proof.cpp \ - blsct/range_proof/bulletproofs/range_proof_logic.cpp \ - blsct/range_proof/bulletproofs_plus/range_proof.cpp \ blsct/range_proof/bulletproofs/amount_recovery_request.cpp \ blsct/range_proof/bulletproofs/amount_recovery_result.cpp \ - blsct/range_proof/bulletproofs/range_proof_logic.cpp \ blsct/range_proof/bulletproofs/range_proof.cpp \ + blsct/range_proof/bulletproofs/range_proof_logic.cpp \ blsct/range_proof/bulletproofs/range_proof_with_transcript.cpp \ blsct/range_proof/bulletproofs_plus/amount_recovery_request.cpp \ blsct/range_proof/bulletproofs_plus/amount_recovery_result.cpp \ - blsct/range_proof/bulletproofs_plus/range_proof_logic.cpp \ blsct/range_proof/bulletproofs_plus/range_proof.cpp \ + blsct/range_proof/bulletproofs_plus/range_proof_logic.cpp \ blsct/range_proof/bulletproofs_plus/range_proof_with_transcript.cpp \ blsct/range_proof/bulletproofs_plus/util.cpp \ blsct/range_proof/common.cpp \ @@ -862,8 +879,8 @@ libbitcoin_common_a_SOURCES = \ blsct/range_proof/msg_amt_cipher.cpp \ blsct/range_proof/proof_base.cpp \ blsct/set_mem_proof/set_mem_proof.cpp \ - blsct/set_mem_proof/set_mem_proof_setup.cpp \ blsct/set_mem_proof/set_mem_proof_prover.cpp \ + blsct/set_mem_proof/set_mem_proof_setup.cpp \ blsct/signature.cpp \ blsct/wallet/address.cpp \ blsct/wallet/txfactory_global.cpp \ @@ -875,6 +892,8 @@ libbitcoin_common_a_SOURCES = \ common/init.cpp \ common/interfaces.cpp \ common/run_command.cpp \ + common/settings.cpp \ + common/system.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -903,7 +922,7 @@ libbitcoin_common_a_SOURCES = \ script/miniscript.cpp \ script/sign.cpp \ script/signingprovider.cpp \ - script/standard.cpp \ + script/solver.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) @@ -925,9 +944,11 @@ libbitcoin_util_a_SOURCES = \ logging.cpp \ random.cpp \ randomenv.cpp \ + streams.cpp \ support/cleanse.cpp \ sync.cpp \ util/asmap.cpp \ + util/batchpriority.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ util/chaintype.cpp \ @@ -937,16 +958,14 @@ libbitcoin_util_a_SOURCES = \ util/fees.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ - util/getuniquepath.cpp \ util/hasher.cpp \ util/sock.cpp \ util/syserror.cpp \ - util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ util/rbf.cpp \ util/readwritefile.cpp \ - util/settings.cpp \ + util/signalinterrupt.cpp \ util/thread.cpp \ util/threadinterrupt.cpp \ util/threadnames.cpp \ @@ -954,7 +973,6 @@ libbitcoin_util_a_SOURCES = \ util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ - util/syscall_sandbox.cpp \ util/time.cpp \ util/tokenpipe.cpp \ $(BITCOIN_CORE_H) @@ -973,13 +991,13 @@ nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h # # navcoind & navcoin-node binaries # -navcoin_daemon_sources = bitcoind.cpp +navcoin_daemon_sources = navcoind.cpp navcoin_bin_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) navcoin_bin_cxxflags = $(AM_CXXFLAGS) $(PIE_FLAGS) navcoin_bin_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) if TARGET_WINDOWS -navcoin_daemon_sources += bitcoind-res.rc +navcoin_daemon_sources += navcoind-res.rc endif navcoin_bin_ldadd = \ @@ -998,7 +1016,7 @@ navcoin_bin_ldadd = \ navcoin_bin_ldadd += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) -navcoind_SOURCES = $(navcoin_daemon_sources) init/bitcoind.cpp +navcoind_SOURCES = $(navcoin_daemon_sources) init/navcoind.cpp navcoind_CPPFLAGS = $(navcoin_bin_cppflags) navcoind_CXXFLAGS = $(navcoin_bin_cxxflags) navcoind_LDFLAGS = $(navcoin_bin_ldflags) @@ -1106,31 +1124,31 @@ navcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(PTHREAD_FLAGS) $(LIBTO navcoin_chainstate_LDADD = $(LIBBITCOINKERNEL) # libtool is unable to calculate this indirect dependency, presumably because it's a subproject. -# libsecp256k1 only needs to be linked in when libbitcoinkernel is static. +# libsecp256k1 only needs to be linked in when libnavcoinkernel is static. navcoin_chainstate_LDADD += $(LIBSECP256K1) $(LIBBLS) $(LIBMCL) # -# bitcoinkernel library # +# navcoinkernel library # if BUILD_BITCOIN_KERNEL_LIB lib_LTLIBRARIES += $(LIBBITCOINKERNEL) -libbitcoinkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS) -libbitcoinkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBBLS) $(LIBMCL) -libbitcoinkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include $(BLS_INCLUDES) -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +libnavcoinkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS) +libnavcoinkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBBLS) $(LIBMCL) +libnavcoinkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include $(BLS_INCLUDES) -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -# libbitcoinkernel requires default symbol visibility, explicitly specify that +# libnavcoinkernel requires default symbol visibility, explicitly specify that # here so that things still work even when user configures with # --enable-reduce-exports # # Note this is a quick hack that will be removed as we incrementally define what # to export from the library. -libbitcoinkernel_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -fvisibility=default +libnavcoinkernel_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -fvisibility=default -# TODO: libbitcoinkernel is a work in progress consensus engine library, as more +# TODO: libnavcoinkernel is a work in progress consensus engine library, as more # and more modules are decoupled from the consensus engine, this list will # shrink to only those which are absolutely necessary. -libbitcoinkernel_la_SOURCES = \ - kernel/bitcoinkernel.cpp \ +libnavcoinkernel_la_SOURCES = \ + kernel/navcoinkernel.cpp \ arith_uint256.cpp \ blockfilter.cpp \ blsct/arith/mcl/mcl_g1point.cpp \ @@ -1147,18 +1165,15 @@ libbitcoinkernel_la_SOURCES = \ blsct/private_key.cpp \ blsct/public_key.cpp \ blsct/public_keys.cpp \ - blsct/range_proof/bulletproofs/range_proof.cpp \ - blsct/range_proof/bulletproofs/range_proof_logic.cpp \ - blsct/range_proof/bulletproofs_plus/range_proof.cpp \ blsct/range_proof/bulletproofs/amount_recovery_request.cpp \ blsct/range_proof/bulletproofs/amount_recovery_result.cpp \ - blsct/range_proof/bulletproofs/range_proof_logic.cpp \ blsct/range_proof/bulletproofs/range_proof.cpp \ + blsct/range_proof/bulletproofs/range_proof_logic.cpp \ blsct/range_proof/bulletproofs/range_proof_with_transcript.cpp \ blsct/range_proof/bulletproofs_plus/amount_recovery_request.cpp \ blsct/range_proof/bulletproofs_plus/amount_recovery_result.cpp \ - blsct/range_proof/bulletproofs_plus/range_proof_logic.cpp \ blsct/range_proof/bulletproofs_plus/range_proof.cpp \ + blsct/range_proof/bulletproofs_plus/range_proof_logic.cpp \ blsct/range_proof/bulletproofs_plus/range_proof_with_transcript.cpp \ blsct/range_proof/bulletproofs_plus/util.cpp \ blsct/range_proof/common.cpp \ @@ -1166,18 +1181,14 @@ libbitcoinkernel_la_SOURCES = \ blsct/range_proof/msg_amt_cipher.cpp \ blsct/range_proof/proof_base.cpp \ blsct/set_mem_proof/set_mem_proof.cpp \ - blsct/set_mem_proof/set_mem_proof_setup.cpp \ blsct/set_mem_proof/set_mem_proof_prover.cpp \ + blsct/set_mem_proof/set_mem_proof_setup.cpp \ blsct/signature.cpp \ blsct/wallet/txfactory_global.cpp \ blsct/wallet/verification.cpp \ chain.cpp \ - chainparamsbase.cpp \ - chainparams.cpp \ clientversion.cpp \ coins.cpp \ - common/args.cpp \ - common/config.cpp \ compressor.cpp \ consensus/merkle.cpp \ consensus/tx_check.cpp \ @@ -1194,15 +1205,15 @@ libbitcoinkernel_la_SOURCES = \ kernel/coinstats.cpp \ kernel/context.cpp \ kernel/cs_main.cpp \ + kernel/disconnected_transactions.cpp \ kernel/mempool_persist.cpp \ + kernel/mempool_removal_reason.cpp \ key.cpp \ logging.cpp \ node/blockstorage.cpp \ node/chainstate.cpp \ - node/interface_ui.cpp \ node/utxo_snapshot.cpp \ policy/feerate.cpp \ - policy/fees.cpp \ policy/packages.cpp \ policy/policy.cpp \ policy/rbf.cpp \ @@ -1218,32 +1229,30 @@ libbitcoinkernel_la_SOURCES = \ script/script.cpp \ script/script_error.cpp \ script/sigcache.cpp \ - script/standard.cpp \ - shutdown.cpp \ + script/solver.cpp \ signet.cpp \ + streams.cpp \ support/cleanse.cpp \ support/lockedpool.cpp \ sync.cpp \ txdb.cpp \ txmempool.cpp \ uint256.cpp \ + util/batchpriority.cpp \ util/bytevectorhash.cpp \ util/chaintype.cpp \ util/check.cpp \ util/exception.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ - util/getuniquepath.cpp \ util/hasher.cpp \ util/moneystr.cpp \ util/rbf.cpp \ util/serfloat.cpp \ - util/settings.cpp \ + util/signalinterrupt.cpp \ util/strencodings.cpp \ util/string.cpp \ - util/syscall_sandbox.cpp \ util/syserror.cpp \ - util/system.cpp \ util/thread.cpp \ util/threadnames.cpp \ util/time.cpp \ @@ -1255,24 +1264,24 @@ libbitcoinkernel_la_SOURCES = \ # Required for obj/build.h to be generated first. # More details: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html -libbitcoinkernel_la-clientversion.l$(OBJEXT): obj/build.h +libnavcoinkernel_la-clientversion.l$(OBJEXT): obj/build.h endif # BUILD_BITCOIN_KERNEL_LIB # -# bitcoinconsensus library # +# navcoinconsensus library # if BUILD_BITCOIN_LIBS lib_LTLIBRARIES += $(LIBBITCOINCONSENSUS) -include_HEADERS = script/bitcoinconsensus.h -libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_la_SOURCES) $(libbitcoin_consensus_a_SOURCES) +include_HEADERS = script/navcoinconsensus.h +libnavcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_la_SOURCES) $(libbitcoin_consensus_a_SOURCES) -libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) -libbitcoinconsensus_la_LIBADD = \ +libnavcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) +libnavcoinconsensus_la_LIBADD = \ $(LIBBLS) \ $(LIBMCL) \ $(LIBSECP256K1) -libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include $(BLS_INCLUDES) -DBUILD_BITCOIN_INTERNAL -libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +libnavcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include $(BLS_INCLUDES) -DBUILD_BITCOIN_INTERNAL +libnavcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) endif # @@ -1304,15 +1313,15 @@ CLEANFILES += wallet/*.gcda wallet/*.gcno CLEANFILES += wallet/test/*.gcda wallet/test/*.gcno CLEANFILES += zmq/*.gcda zmq/*.gcno CLEANFILES += obj/build.h -CLEANFILES += $(LIBBLS_A) -CLEANFILES += $(LIBMCL_A) +CLEANFILES += $(LIBBLS) +CLEANFILES += $(LIBMCL) EXTRA_DIST = $(CTAES_DIST) # blsct library # if BUILD_LIBBLSCT_ONLY -LIBBLSCT_A = libblsct.a -noinst_LIBRARIES = $(LIBBLSCT_A) +LIBBLSCT = libblsct.a +noinst_LIBRARIES = $(LIBBLSCT) libblsct_a_SOURCES = \ crypto/sha256_arm_shani.cpp \ crypto/sha256_avx2.cpp \ @@ -1345,9 +1354,9 @@ libblsct_a_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) libblsct_a_CPPFLAGS += -DENABLE_ARM_SHANI endif -$(LIBBLSCT_A): $(LIBBLS) $(LIBMCL) +$(LIBBLSCT): $(LIBBLS) $(LIBMCL) -CLEANFILES += $(LIBBLSCT_A) blsct/include/libblsct_a-blsct.o +CLEANFILES += $(LIBBLSCT) blsct/include/libblsct_a-blsct.o endif # @@ -1360,13 +1369,14 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) clean-local: -$(MAKE) -C secp256k1 clean + -$(MAKE) -C bls/mcl clean -$(MAKE) -C bls clean -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -f config.h -rm -rf test/__pycache__ .rc.o: - @test -f $(WINDRES) + @test -f $(WINDRES) || (echo "windres $(WINDRES) not found, but is required to compile windows resource files"; exit 1) ## FIXME: How to get the appropriate modulename_CPPFLAGS in here? $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ @@ -1421,12 +1431,11 @@ endif %.raw.h: %.raw @$(MKDIR_P) $(@D) - @{ \ + $(AM_V_GEN) { \ echo "static unsigned const char $(*F)_raw[] = {" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};"; \ } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" include Makefile.minisketch.include diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index de3e77b43c529..3c4cedc0db601 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -19,17 +19,19 @@ bench_bench_navcoin_SOURCES = \ bench/bench.cpp \ bench/bench.h \ bench/bench_bitcoin.cpp \ + bench/bip324_ecdh.cpp \ bench/block_assemble.cpp \ bench/ccoins_caching.cpp \ bench/chacha20.cpp \ - bench/chacha_poly_aead.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ bench/crypto_hash.cpp \ bench/data.cpp \ bench/data.h \ bench/descriptors.cpp \ + bench/disconnected_transactions.cpp \ bench/duplicate_inputs.cpp \ + bench/ellswift.cpp \ bench/examples.cpp \ bench/gcs_filter.cpp \ bench/hashpadding.cpp \ @@ -45,13 +47,15 @@ bench_bench_navcoin_SOURCES = \ bench/poly1305.cpp \ bench/pool.cpp \ bench/prevector.cpp \ + bench/readblock.cpp \ bench/rollingbloom.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ bench/streams_findbyte.cpp \ bench/strencodings.cpp \ bench/util_time.cpp \ - bench/verify_script.cpp + bench/verify_script.cpp \ + bench/xor.cpp nodist_bench_bench_navcoin_SOURCES = $(GENERATED_BENCH_FILES) @@ -84,8 +88,9 @@ endif if ENABLE_WALLET bench_bench_navcoin_SOURCES += bench/coin_selection.cpp bench_bench_navcoin_SOURCES += bench/wallet_balance.cpp -bench_bench_navcoin_SOURCES += bench/wallet_loading.cpp +bench_bench_navcoin_SOURCES += bench/wallet_create.cpp bench_bench_navcoin_SOURCES += bench/wallet_create_tx.cpp +bench_bench_navcoin_SOURCES += bench/wallet_loading.cpp bench_bench_navcoin_LDADD += $(BDB_LIBS) $(SQLITE_LIBS) endif diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 8070635cec192..6a0c06b3c2bec 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -77,6 +77,7 @@ BITCOIN_TESTS =\ test/base58_tests.cpp \ test/base64_tests.cpp \ test/bech32_tests.cpp \ + test/bip324_tests.cpp \ test/bip32_tests.cpp \ test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ @@ -118,6 +119,7 @@ BITCOIN_TESTS =\ test/dbwrapper_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ + test/disconnected_transactions.cpp \ test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ @@ -133,9 +135,11 @@ BITCOIN_TESTS =\ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/miniminer_tests.cpp \ test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ + test/net_peer_connection_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ @@ -169,6 +173,7 @@ BITCOIN_TESTS =\ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ test/sock_tests.cpp \ + test/span_tests.cpp \ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ @@ -226,12 +231,15 @@ BITCOIN_TESTS += wallet/test/db_tests.cpp endif FUZZ_WALLET_SRC = \ + wallet/test/fuzz/coincontrol.cpp \ wallet/test/fuzz/coinselection.cpp \ + wallet/test/fuzz/fees.cpp \ wallet/test/fuzz/parse_iso8601.cpp if USE_SQLITE FUZZ_WALLET_SRC += \ - wallet/test/fuzz/notifications.cpp + wallet/test/fuzz/notifications.cpp \ + wallet/test/fuzz/scriptpubkeyman.cpp endif # USE_SQLITE BITCOIN_TEST_SUITE += \ @@ -242,7 +250,7 @@ BITCOIN_TEST_SUITE += \ endif # ENABLE_WALLET test_test_navcoin_SOURCES = $(BITCOIN_TEST_SUITE) $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) -test_test_navcoin_CPPFLAGS = $(AM_CPPFLAGS) -DBLS_ETH=1 $(BITCOIN_INCLUDES) $(TESTDEFS) $(BOOST_CPPFLAGS) $(EVENT_CFLAGS) +test_test_navcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(BOOST_CPPFLAGS) $(EVENT_CFLAGS) test_test_navcoin_LDADD = $(LIBTEST_UTIL) if ENABLE_WALLET test_test_navcoin_LDADD += $(LIBBITCOIN_WALLET) @@ -265,7 +273,7 @@ if ENABLE_FUZZ_BINARY test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fuzz_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) $(RUNTIME_LDFLAGS) +test_fuzz_fuzz_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) test_fuzz_fuzz_SOURCES = \ $(FUZZ_WALLET_SRC) \ test/fuzz/addition_overflow.cpp \ @@ -277,6 +285,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/base_encode_decode.cpp \ test/fuzz/bech32.cpp \ test/fuzz/bech32_mod.cpp \ + test/fuzz/bip324.cpp \ test/fuzz/bitdeque.cpp \ test/fuzz/block.cpp \ test/fuzz/block_header.cpp \ @@ -292,7 +301,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/crypto_aes256.cpp \ test/fuzz/crypto_aes256cbc.cpp \ test/fuzz/crypto_chacha20.cpp \ - test/fuzz/crypto_chacha20_poly1305_aead.cpp \ test/fuzz/crypto_common.cpp \ test/fuzz/crypto_diff_fuzz_chacha20.cpp \ test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ @@ -310,7 +318,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/headerssync.cpp \ test/fuzz/hex.cpp \ test/fuzz/http_request.cpp \ - test/fuzz/i2p.cpp \ test/fuzz/integer.cpp \ test/fuzz/key.cpp \ test/fuzz/key_io.cpp \ @@ -321,6 +328,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/message.cpp \ test/fuzz/miniscript.cpp \ test/fuzz/minisketch.cpp \ + test/fuzz/mini_miner.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ test/fuzz/net.cpp \ @@ -329,6 +337,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/netbase_dns_lookup.cpp \ test/fuzz/node_eviction.cpp \ test/fuzz/p2p_transport_serialization.cpp \ + test/fuzz/package_eval.cpp \ test/fuzz/parse_hd_keypath.cpp \ test/fuzz/parse_numbers.cpp \ test/fuzz/parse_script.cpp \ @@ -455,10 +464,10 @@ endif %.json.h: %.json @$(MKDIR_P) $(@D) - @{ \ + $(AM_V_GEN) { \ + echo "#include " && \ echo "namespace json_tests{" && \ - echo "static unsigned const char $(*F)[] = {" && \ + echo "static const std::string $(*F){" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index aa9c0527509b1..b4337991e42dd 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -11,6 +11,7 @@ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ test/fuzz/util.h \ + test/fuzz/util/descriptor.h \ test/fuzz/util/mempool.h \ test/fuzz/util/net.h @@ -19,6 +20,7 @@ libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ test/fuzz/util.cpp \ + test/fuzz/util/descriptor.cpp \ test/fuzz/util/mempool.cpp \ test/fuzz/util/net.cpp \ $(TEST_FUZZ_H) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index d4a9f620759bb..0610e6487c4d0 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -22,6 +22,7 @@ TEST_UTIL_H = \ test/util/blockfilter.h \ test/util/chainstate.h \ test/util/coins.h \ + test/util/index.h \ test/util/json.h \ test/util/logging.h \ test/util/mining.h \ @@ -55,10 +56,12 @@ libtest_util_a_SOURCES = \ blsct/wallet/txfactory_global.cpp \ test/util/blockfilter.cpp \ test/util/coins.cpp \ + test/util/index.cpp \ test/util/json.cpp \ test/util/logging.cpp \ test/util/mining.cpp \ test/util/net.cpp \ + test/util/random.cpp \ test/util/script.cpp \ test/util/setup_common.cpp \ test/util/str.cpp \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index b679ad060299e..8cf932bcb6396 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,6 @@ #include #include #include -#include #include namespace { @@ -47,16 +47,16 @@ bool SerializeDB(Stream& stream, const Data& data) } template -bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data, int version) +bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data) { // Generate random temporary filename const uint16_t randv{GetRand()}; std::string tmpfn = strprintf("%s.%04x", prefix, randv); - // open temp output file, and associate with CAutoFile + // open temp output file fs::path pathTmp = gArgs.GetDataDirNet() / fs::u8path(tmpfn); FILE *file = fsbridge::fopen(pathTmp, "wb"); - CAutoFile fileout(file, SER_DISK, version); + AutoFile fileout{file}; if (fileout.IsNull()) { fileout.fclose(); remove(pathTmp); @@ -86,14 +86,14 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data } template -void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) +void DeserializeDB(Stream& stream, Data&& data, bool fCheckSum = true) { - CHashVerifier verifier(&stream); + HashVerifier verifier{stream}; // de-serialize file header (network specific magic number) and .. - unsigned char pchMsgTmp[4]; + MessageStartChars pchMsgTmp; verifier >> pchMsgTmp; // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) { + if (pchMsgTmp != Params().MessageStart()) { throw std::runtime_error{"Invalid network magic number"}; } @@ -111,11 +111,10 @@ void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) } template -void DeserializeFileDB(const fs::path& path, Data& data, int version) +void DeserializeFileDB(const fs::path& path, Data&& data) { - // open input file, and associate with CAutoFile FILE* file = fsbridge::fopen(path, "rb"); - CAutoFile filein(file, SER_DISK, version); + AutoFile filein{file}; if (filein.IsNull()) { throw DbNotFoundError{}; } @@ -132,7 +131,7 @@ CBanDB::CBanDB(fs::path ban_list_path) bool CBanDB::Write(const banmap_t& banSet) { std::vector errors; - if (util::WriteSettings(m_banlist_json, {{JSON_KEY, BanMapToJson(banSet)}}, errors)) { + if (common::WriteSettings(m_banlist_json, {{JSON_KEY, BanMapToJson(banSet)}}, errors)) { return true; } @@ -152,10 +151,10 @@ bool CBanDB::Read(banmap_t& banSet) return false; } - std::map settings; + std::map settings; std::vector errors; - if (!util::ReadSettings(m_banlist_json, settings, errors)) { + if (!common::ReadSettings(m_banlist_json, settings, errors)) { for (const auto& err : errors) { LogPrintf("Cannot load banlist %s: %s\n", fs::PathToString(m_banlist_json), err); } @@ -175,23 +174,23 @@ bool CBanDB::Read(banmap_t& banSet) bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr) { const auto pathAddr = args.GetDataDirNet() / "peers.dat"; - return SerializeFileDB("peers", pathAddr, addr, CLIENT_VERSION); + return SerializeFileDB("peers", pathAddr, addr); } -void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) +void ReadFromStream(AddrMan& addr, DataStream& ssPeers) { DeserializeDB(ssPeers, addr, false); } -std::optional LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr& addrman) +util::Result> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args) { auto check_addrman = std::clamp(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + auto addrman{std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman)}; const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; try { - DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION); + DeserializeFileDB(path_addr, *addrman); LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->Size(), Ticks(SteadyClock::now() - start)); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it @@ -200,32 +199,30 @@ std::optional LoadAddrman(const NetGroupManager& netgroupman, con DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { if (!RenameOver(path_addr, (fs::path)path_addr + ".bak")) { - addrman = nullptr; - return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); + return util::Error{strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again."))}; } // Addrman can be in an inconsistent state after failure, reset it addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { - addrman = nullptr; - return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), - e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr))); + return util::Error{strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), + e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr)))}; } - return std::nullopt; + return addrman; } void DumpAnchors(const fs::path& anchors_db_path, const std::vector& anchors) { LOG_TIME_SECONDS(strprintf("Flush %d outbound block-relay-only peer addresses to anchors.dat", anchors.size())); - SerializeFileDB("anchors", anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT); + SerializeFileDB("anchors", anchors_db_path, CAddress::V2_DISK(anchors)); } std::vector ReadAnchors(const fs::path& anchors_db_path) { std::vector anchors; try { - DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT); + DeserializeFileDB(anchors_db_path, CAddress::V2_DISK(anchors)); LogPrintf("Loaded %i addresses from %s\n", anchors.size(), fs::quoted(fs::PathToString(anchors_db_path.filename()))); } catch (const std::exception&) { anchors.clear(); diff --git a/src/addrdb.h b/src/addrdb.h index 08d86d0f01bee..cc3014dce29b3 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -6,23 +6,23 @@ #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H -#include // For banmap_t -#include +#include #include +#include -#include +#include #include class ArgsManager; class AddrMan; class CAddress; -class CDataStream; +class DataStream; class NetGroupManager; -struct bilingual_str; -bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); /** Only used by tests. */ -void ReadFromStream(AddrMan& addr, CDataStream& ssPeers); +void ReadFromStream(AddrMan& addr, DataStream& ssPeers); + +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); /** Access to the banlist database (banlist.json) */ class CBanDB @@ -49,7 +49,7 @@ class CBanDB }; /** Returns an error string on failure */ -std::optional LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr& addrman); +util::Result> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/addresstype.cpp b/src/addresstype.cpp new file mode 100644 index 0000000000000..f01cd97e216c8 --- /dev/null +++ b/src/addresstype.cpp @@ -0,0 +1,175 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include