Skip to content

Commit

Permalink
Copied ci related files from btc-master branch
Browse files Browse the repository at this point in the history
  • Loading branch information
mxaddict committed Feb 13, 2024
1 parent eddf587 commit 3e655ad
Show file tree
Hide file tree
Showing 166 changed files with 3,368 additions and 2,982 deletions.
352 changes: 116 additions & 236 deletions .cirrus.yml

Large diffs are not rendered by default.

6 changes: 4 additions & 2 deletions .github/ISSUE_TEMPLATE/good_first_issue.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@ body:
attributes:
label: Useful Skills
description: For example, “`std::thread`”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”.
validations:
required: false
value: |
* Compiling Bitcoin Core from source
* Running the C++ unit tests and the Python functional tests
* ...
- type: textarea
attributes:
label: Guidance for new contributors
Expand Down
236 changes: 236 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
# Copyright (c) 2023 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

name: CI
on:
# See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request.
pull_request:
# See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push.
push:
branches:
- '**'
tags-ignore:
- '**'

concurrency:
group: ${{ github.event_name != 'pull_request' && github.run_id || github.ref }}
cancel-in-progress: true

env:
DANGER_RUN_CI_ON_HOST: 1
CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error
MAKEJOBS: '-j10'

jobs:
test-each-commit:
name: 'test each commit'
runs-on: ubuntu-22.04
if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1
timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. Assuming a worst case time of 1 hour per commit, this leads to a --max-count=6 below.
env:
MAX_COUNT: 6
steps:
- name: Determine fetch depth
run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV"
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: ${{ env.FETCH_DEPTH }}
- name: Determine commit range
run: |
# Checkout HEAD~ and find the test base commit
# Checkout HEAD~ because it would be wasteful to rerun tests on the PR
# head commit that are already run by other jobs.
git checkout HEAD~
# Figure out test base commit by listing ancestors of HEAD, excluding
# ancestors of the most recent merge commit, limiting the list to the
# newest MAX_COUNT ancestors, ordering it from oldest to newest, and
# taking the first one.
#
# If the branch contains up to MAX_COUNT ancestor commits after the
# most recent merge commit, all of those commits will be tested. If it
# contains more, only the most recent MAX_COUNT commits will be
# tested.
#
# In the command below, the ^@ suffix is used to refer to all parents
# of the merge commit as described in:
# https://git-scm.com/docs/git-rev-parse#_other_rev_parent_shorthand_notations
# and the ^ prefix is used to exclude these parents and all their
# ancestors from the rev-list output as described in:
# https://git-scm.com/docs/git-rev-list
echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD ^$(git rev-list -n1 --merges HEAD)^@ | head -1)" >> "$GITHUB_ENV"
- run: |
sudo apt-get update
sudo apt-get install clang-15 ccache build-essential libtool autotools-dev automake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev -y
- name: Compile and run tests
run: |
# Run tests on commits after the last merge commit and before the PR head commit
# Use clang++, because it is a bit faster and uses less memory than g++
git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && ./autogen.sh && CC=clang-15 CXX=clang++-15 ./configure && make clean && make -j $(nproc) check && ./test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }}
macos-native-x86_64:
name: 'macOS 13 native, x86_64, no depends, sqlite only, gui'
# Use latest image, but hardcode version to avoid silent upgrades (and breaks).
# See: https://github.com/actions/runner-images#available-images.
runs-on: macos-13

# No need to run on the read-only mirror, unless it is a PR.
if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request'

timeout-minutes: 120

env:
FILE_ENV: './ci/test/00_setup_env_mac_native.sh'
BASE_ROOT_DIR: ${{ github.workspace }}

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Clang version
run: |
sudo xcode-select --switch /Applications/Xcode_15.0.app
clang --version
- name: Install Homebrew packages
env:
HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
run: brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq

- name: Set Ccache directory
run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"

- name: Restore Ccache cache
id: ccache-cache
uses: actions/cache/restore@v3
with:
path: ${{ env.CCACHE_DIR }}
key: ${{ github.job }}-ccache-${{ github.run_id }}
restore-keys: ${{ github.job }}-ccache-

- name: CI script
run: ./ci/test_run_all.sh

- name: Save Ccache cache
uses: actions/cache/save@v3
if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
with:
path: ${{ env.CCACHE_DIR }}
# https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
key: ${{ github.job }}-ccache-${{ github.run_id }}

win64-native:
name: 'Win64 native, VS 2022'
# Use latest image, but hardcode version to avoid silent upgrades (and breaks).
# See: https://github.com/actions/runner-images#available-images.
runs-on: windows-2022

# No need to run on the read-only mirror, unless it is a PR.
if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request'

env:
CCACHE_MAXSIZE: '200M'
CI_CCACHE_VERSION: '4.7.5'
PYTHONUTF8: 1
TEST_RUNNER_TIMEOUT_FACTOR: 40

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Configure Developer Command Prompt for Microsoft Visual C++
# Using microsoft/setup-msbuild is not enough.
uses: ilammy/msvc-dev-cmd@v1
with:
arch: x64

- name: Check MSBuild
run: |
msbuild -version | Out-File -FilePath "$env:GITHUB_WORKSPACE\msbuild_version"
Get-Content -Path "$env:GITHUB_WORKSPACE\msbuild_version"
$env:VCToolsVersion | Out-File -FilePath "$env:GITHUB_WORKSPACE\toolset_version"
Get-Content -Path "$env:GITHUB_WORKSPACE\toolset_version"
- name: Ccache installation cache
id: ccache-installation-cache
uses: actions/cache@v3
with:
path: |
C:\ProgramData\chocolatey\lib\ccache
C:\ProgramData\chocolatey\bin\ccache.exe
C:\ccache\cl.exe
key: ${{ github.job }}-ccache-installation-${{ env.CI_CCACHE_VERSION }}

- name: Install Ccache
if: steps.ccache-installation-cache.outputs.cache-hit != 'true'
run: |
choco install --yes --no-progress ccache --version=$env:CI_CCACHE_VERSION
New-Item -ItemType Directory -Path "C:\ccache"
Copy-Item -Path "$env:ChocolateyInstall\lib\ccache\tools\ccache-$env:CI_CCACHE_VERSION-windows-x86_64\ccache.exe" -Destination "C:\ccache\cl.exe"
- name: Restore Ccache cache
id: ccache-cache
uses: actions/cache/restore@v3
with:
path: ~/AppData/Local/ccache
key: ${{ github.job }}-ccache-${{ github.run_id }}
restore-keys: ${{ github.job }}-ccache-

- name: Using vcpkg with MSBuild
run: |
Set-Location "$env:VCPKG_INSTALLATION_ROOT"
Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)"
Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_PLATFORM_TOOLSET_VERSION $env:VCToolsVersion)"
.\vcpkg.exe --vcpkg-root "$env:VCPKG_INSTALLATION_ROOT" integrate install
git rev-parse HEAD | Out-File -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit"
Get-Content -Path "$env:GITHUB_WORKSPACE\vcpkg_commit"
- name: vcpkg tools cache
uses: actions/cache@v3
with:
path: C:/vcpkg/downloads/tools
key: ${{ github.job }}-vcpkg-tools

- name: vcpkg binary cache
uses: actions/cache@v3
with:
path: ~/AppData/Local/vcpkg/archives
key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('vcpkg_commit', 'msbuild_version', 'toolset_version', 'build_msvc/vcpkg.json') }}

- name: Generate project files
run: py -3 build_msvc\msvc-autogen.py

- name: Build
shell: cmd
run: |
ccache --zero-stats
msbuild build_msvc\bitcoin.sln -property:CLToolPath=C:\ccache;CLToolExe=cl.exe;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
- name: Ccache stats
run: ccache --show-stats

- name: Save Ccache cache
uses: actions/cache/save@v3
if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
with:
path: ~/AppData/Local/ccache
# https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
key: ${{ github.job }}-ccache-${{ github.run_id }}

- name: Run unit tests
run: src\test_navcoin.exe -l test_suite

- name: Run benchmarks
run: src\bench_navcoin.exe -sanity-check

- name: Run util tests
run: py -3 test\util\test_runner.py

- name: Run rpcauth test
run: py -3 test\util\rpcauth-test.py

- name: Run functional tests
env:
TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }}
run: py -3 test\functional\test_runner.py --jobs $env:NUMBER_OF_PROCESSORS --ci --quiet --tmpdirprefix=$env:RUNNER_TEMP --combinedlogslen=99999999 --timeout-factor=$env:TEST_RUNNER_TIMEOUT_FACTOR $env:TEST_RUNNER_EXTRA
18 changes: 5 additions & 13 deletions ci/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,11 @@ requires `bash`, `docker`, and `python3` to be installed. To install all require
sudo apt install bash docker.io python3
```

To run the default test stage,
It is recommended to run the ci system in a clean env. To run the test stage
with a specific configuration,

```
./ci/test_run_all.sh
```

To run the test stage with a specific configuration,

```
FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh'
```

### Configurations
Expand All @@ -44,14 +39,11 @@ the system package manager to install build dependencies. This guarantees that
the tester is using the same versions as the release builds, which also use
`./depends`.

If no `FILE_ENV` has been specified or values are left out, `00_setup_env.sh`
is used as the default configuration with fallback values.

It is also possible to force a specific configuration without modifying the
file. For example,

```
MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh'
```

The files starting with `0n` (`n` greater than 0) are the scripts that are run
Expand All @@ -60,5 +52,5 @@ in order.
### Cache

In order to avoid rebuilding all dependencies for each build, the binaries are
cached and re-used when possible. Changes in the dependency-generator will
cached and reused when possible. Changes in the dependency-generator will
trigger cache-invalidation and rebuilds as necessary.
59 changes: 36 additions & 23 deletions ci/lint/04_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,38 +6,51 @@

export LC_ALL=C

export PATH=$PWD/ci/retry:$PATH

${CI_RETRY_EXE} apt-get update
# Lint dependencies:
# - curl/xz-utils (to install shellcheck)
# - git (used in many lint scripts)
# - gpg (used by verify-commits)
${CI_RETRY_EXE} apt-get install -y curl xz-utils git gpg

if [ -z "${SKIP_PYTHON_INSTALL}" ]; then
PYTHON_PATH=/tmp/python
if [ ! -d "${PYTHON_PATH}/bin" ]; then
(
git clone https://github.com/pyenv/pyenv.git
cd pyenv/plugins/python-build || exit 1
./install.sh
)
# For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment
${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \
libbz2-dev libreadline-dev libsqlite3-dev curl llvm \
libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
clang
env CC=clang python-build "$(cat "${BASE_ROOT_DIR}/.python-version")" "${PYTHON_PATH}"
fi
export PATH="${PYTHON_PATH}/bin:${PATH}"
command -v python3
python3 --version
PYTHON_PATH="/python_build"
if [ ! -d "${PYTHON_PATH}/bin" ]; then
(
${CI_RETRY_EXE} git clone https://github.com/pyenv/pyenv.git
cd pyenv/plugins/python-build || exit 1
./install.sh
)
# For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment
${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \
libbz2-dev libreadline-dev libsqlite3-dev curl llvm \
libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
clang
env CC=clang python-build "$(cat "./.python-version")" "${PYTHON_PATH}"
fi
export PATH="${PYTHON_PATH}/bin:${PATH}"
command -v python3
python3 --version

export LINT_RUNNER_PATH="/lint_test_runner"
if [ ! -d "${LINT_RUNNER_PATH}" ]; then
${CI_RETRY_EXE} apt-get install -y cargo
(
cd ./test/lint/test_runner || exit 1
cargo build
mkdir -p "${LINT_RUNNER_PATH}"
mv target/debug/test_runner "${LINT_RUNNER_PATH}"
)
fi

${CI_RETRY_EXE} pip3 install codespell==2.2.1
${CI_RETRY_EXE} pip3 install flake8==5.0.4
${CI_RETRY_EXE} pip3 install mypy==0.971
${CI_RETRY_EXE} pip3 install pyzmq==24.0.1
${CI_RETRY_EXE} pip3 install vulture==2.6
${CI_RETRY_EXE} pip3 install \
codespell==2.2.5 \
flake8==6.1.0 \
lief==0.13.2 \
mypy==1.4.1 \
pyzmq==25.1.0 \
vulture==2.6

SHELLCHECK_VERSION=v0.8.0
curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | \
Expand Down
12 changes: 3 additions & 9 deletions ci/lint/06_script.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

export LC_ALL=C

set -ex

if [ -n "$LOCAL_BRANCH" ]; then
# To faithfully recreate CI linting locally, specify all commits on the current
# branch.
Expand All @@ -21,15 +23,7 @@ else
fi
export COMMIT_RANGE

# This only checks that the trees are pure subtrees, it is not doing a full
# check with -r to not have to fetch all the remotes.
test/lint/git-subtree-check.sh src/crypto/ctaes
test/lint/git-subtree-check.sh src/secp256k1
test/lint/git-subtree-check.sh src/minisketch
test/lint/git-subtree-check.sh src/leveldb
test/lint/git-subtree-check.sh src/crc32c
test/lint/check-doc.py
test/lint/all-lint.py
RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner"

if [ "$CIRRUS_REPO_FULL_NAME" = "navcoin/navcoin" ] && [ "$CIRRUS_PR" = "" ] ; then
# Sanity check only the last few commits to get notified of missing sigs,
Expand Down
Loading

0 comments on commit 3e655ad

Please sign in to comment.