ci: build GCC 14, build nowallet depends and source with it, stage built packages in /opt, allowlist LLVM libc++

.github/workflows/build.yml

@@ -19,6 +19,8 @@ jobs:
     outputs:
       image-tag: ${{ steps.prepare.outputs.image-tag }}
       repo-name: ${{ steps.prepare.outputs.repo-name }}
+      dep-matrix: ${{ steps.prepare.outputs.dep-matrix }}
+      src-matrix: ${{ steps.prepare.outputs.src-matrix }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -32,6 +34,8 @@ jobs:
           REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
           echo "image-tag=${BRANCH_NAME}" >> $GITHUB_OUTPUT
           echo "repo-name=${REPO_NAME}" >> $GITHUB_OUTPUT
+          echo "dep-matrix=$(jq -r '.dep' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT
+          echo "src-matrix=$(jq -r '.src' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -56,18 +60,12 @@ jobs:
           cache-to: type=inline
 
   build-depends:
-    name: Build Dependencies
+    name: ${{ matrix.depends_name }}
     needs: build-image
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
-      matrix:
-        include:
-        - build_target: arm-linux
-          host: arm-linux-gnueabihf
-        - build_target: linux64
-          host: x86_64-pc-linux-gnu
-
+      matrix: ${{ fromJson(needs.build-image.outputs.dep-matrix) }}
     container:
       image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }}
       options: --user root
@@ -86,52 +84,36 @@ jobs:
           restore-keys: |
             depends-sources-
 
+      - name: Determine params
+        id: det-params
+        run: |
+          dep_name="${{ matrix.depends_name }}"
+          dep_opts="${{ matrix.dep_opts }}"
+          dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)"
+          echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\""
+          echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT
+
       - name: Cache depends
         uses: actions/cache@v4
         with:
           path: |
             depends/built
             depends/${{ matrix.host }}
-          key: ${{ runner.os }}-depends-${{ matrix.build_target }}-${{ hashFiles('depends/packages/*') }}
+          key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-${{ steps.det-params.outputs.dep_hash }}
           restore-keys: |
-            ${{ runner.os }}-depends-${{ matrix.build_target }}-${{ hashFiles('depends/packages/*') }}
-            ${{ runner.os }}-depends-${{ matrix.build_target }}
+            ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-
+            ${{ runner.os }}-depends-${{ matrix.depends_name }}-
 
       - name: Build depends
-        run: make -j$(nproc) -C depends HOST=${{ matrix.host }}
+        run: env HOST=${{ matrix.host }} ${{ matrix.dep_opts }} make -j$(nproc) -C depends
 
   build:
-    name: Build
+    name: ${{ matrix.build_target }}
     needs: [build-image, build-depends]
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
-      matrix:
-        include:
-          - build_target: arm-linux
-            host: arm-linux-gnueabihf
-            depends_on: arm-linux
-          - build_target: linux64
-            host: x86_64-pc-linux-gnu
-            depends_on: linux64
-          - build_target: linux64_tsan
-            host: x86_64-pc-linux-gnu
-            depends_on: linux64
-          - build_target: linux64_ubsan
-            host: x86_64-pc-linux-gnu
-            depends_on: linux64
-          - build_target: linux64_fuzz
-            host: x86_64-pc-linux-gnu
-            depends_on: linux64
-          - build_target: linux64_cxx20
-            host: x86_64-pc-linux-gnu
-            depends_on: linux64
-          - build_target: linux64_sqlite
-            host: x86_64-pc-linux-gnu
-            depends_on: linux64
-          - build_target: linux64_nowallet
-            host: x86_64-pc-linux-gnu
-            depends_on: linux64
+      matrix: ${{ fromJson(needs.build-image.outputs.src-matrix) }}
     container:
       image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }}
       options: --user root
@@ -141,13 +123,28 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - name: Determine params
+        id: det-params
+        run: |
+          dep_name="${{ matrix.depends_name }}"
+          dep_opts="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .dep_opts" -c .github/workflows/matrix.json)"
+          dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)"
+          echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\""
+          echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT
+          dep_host="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .host" -c .github/workflows/matrix.json)"
+          echo "\"${dep_name}\" has HOST \"${dep_host}\""
+          echo "dep_host=${dep_host}" >> $GITHUB_OUTPUT
+
       - name: Restore depends cache
         uses: actions/cache/restore@v4
         with:
           path: |
             depends/built
-            depends/${{ matrix.host }}
-          key: ${{ runner.os }}-depends-${{ matrix.depends_on }}-${{ hashFiles('depends/packages/*') }}
+            depends/${{ steps.det-params.outputs.dep_host }}
+          key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-${{ steps.det-params.outputs.dep_hash }}
+          restore-keys: |
+            ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-
+            ${{ runner.os }}-depends-${{ matrix.depends_name }}-
 
       - name: Determine PR Base SHA
         id: vars

.github/workflows/matrix.json

@@ -0,0 +1,66 @@
+{
+  "dep": {
+    "include": [
+      {
+        "depends_name": "arm-linux-gnueabihf",
+        "host": "arm-linux-gnueabihf",
+        "dep_opts": ""
+      },
+      {
+        "depends_name": "x86_64-pc-linux-gnu_debug",
+        "host": "x86_64-pc-linux-gnu",
+        "dep_opts": "DEBUG=1"
+      },
+      {
+        "depends_name": "x86_64-pc-linux-gnu_multiprocess",
+        "host": "x86_64-pc-linux-gnu",
+        "dep_opts": "DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18"
+      },
+      {
+        "depends_name": "x86_64-pc-linux-gnu_nowallet",
+        "host": "x86_64-pc-linux-gnu",
+        "dep_opts": "NO_WALLET=1"
+      }
+    ]
+  },
+  "src": {
+    "include": [
+      {
+        "build_target": "arm-linux",
+        "depends_name": "arm-linux-gnueabihf"
+      },
+      {
+        "build_target": "linux64",
+        "depends_name": "x86_64-pc-linux-gnu_debug"
+      },
+      {
+        "build_target": "linux64_cxx20",
+        "depends_name": "x86_64-pc-linux-gnu_debug"
+      },
+      {
+        "build_target": "linux64_fuzz",
+        "depends_name": "x86_64-pc-linux-gnu_debug"
+      },
+      {
+        "build_target": "linux64_multiprocess",
+        "depends_name": "x86_64-pc-linux-gnu_multiprocess"
+      },
+      {
+        "build_target": "linux64_nowallet",
+        "depends_name": "x86_64-pc-linux-gnu_nowallet"
+      },
+      {
+        "build_target": "linux64_sqlite",
+        "depends_name": "x86_64-pc-linux-gnu_debug"
+      },
+      {
+        "build_target": "linux64_tsan",
+        "depends_name": "x86_64-pc-linux-gnu_multiprocess"
+      },
+      {
+        "build_target": "linux64_ubsan",
+        "depends_name": "x86_64-pc-linux-gnu_debug"
+      }
+    ]
+  }
+}

.gitlab-ci.yml

@@ -189,27 +189,27 @@ x86_64-w64-mingw32:
   variables:
     HOST: x86_64-w64-mingw32
 
-x86_64-pc-linux-gnu-debug:
+x86_64-pc-linux-gnu_debug:
   extends: .build-depends-template
   variables:
     HOST: x86_64-pc-linux-gnu
     DEP_OPTS: "DEBUG=1"
 
-x86_64-pc-linux-gnu-nowallet:
+x86_64-pc-linux-gnu_nowallet:
   extends:
     - .build-depends-template
     - .skip-in-fast-mode-template
   variables:
     HOST: x86_64-pc-linux-gnu
     DEP_OPTS: "NO_WALLET=1"
 
-x86_64-pc-linux-gnu-multiprocess:
+x86_64-pc-linux-gnu_multiprocess:
   extends:
     - .build-depends-template
     - .skip-in-fast-mode-template
   variables:
     HOST: x86_64-pc-linux-gnu
-    DEP_OPTS: "MULTIPROCESS=1"
+    DEP_OPTS: "DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18"
 
 x86_64-apple-darwin:
   extends:
@@ -239,7 +239,7 @@ win64-build:
 linux64-build:
   extends: .build-template
   needs:
-    - x86_64-pc-linux-gnu-debug
+    - x86_64-pc-linux-gnu_debug
   variables:
     BUILD_TARGET: linux64
 
@@ -248,7 +248,7 @@ linux64_cxx20-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-pc-linux-gnu-debug
+    - x86_64-pc-linux-gnu_debug
   variables:
     BUILD_TARGET: linux64_cxx20
 
@@ -257,7 +257,7 @@ linux64_sqlite-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-pc-linux-gnu-debug
+    - x86_64-pc-linux-gnu_debug
   variables:
     BUILD_TARGET: linux64_sqlite
 
@@ -266,7 +266,7 @@ linux64_fuzz-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-pc-linux-gnu-debug
+    - x86_64-pc-linux-gnu_debug
   variables:
     BUILD_TARGET: linux64_fuzz
 
@@ -275,7 +275,7 @@ linux64_fuzz-build:
 #    - .build-template
 #    - .skip-in-fast-mode-template
 #  needs:
-#    - x86_64-pc-linux-gnu-debug
+#    - x86_64-pc-linux-gnu_debug
 #  variables:
 #    BUILD_TARGET: linux64_asan
 
@@ -284,7 +284,7 @@ linux64_tsan-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-pc-linux-gnu-debug
+    - x86_64-pc-linux-gnu_multiprocess
   variables:
     BUILD_TARGET: linux64_tsan
 
@@ -293,7 +293,7 @@ linux64_ubsan-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-pc-linux-gnu-debug
+    - x86_64-pc-linux-gnu_debug
   variables:
     BUILD_TARGET: linux64_ubsan
 
@@ -302,7 +302,7 @@ linux64_nowallet-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-pc-linux-gnu-nowallet
+    - x86_64-pc-linux-gnu_nowallet
   variables:
     BUILD_TARGET: linux64_nowallet
 
@@ -311,7 +311,7 @@ linux64_multiprocess-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-pc-linux-gnu-multiprocess
+    - x86_64-pc-linux-gnu_multiprocess
   variables:
     BUILD_TARGET: linux64_multiprocess
 
@@ -320,7 +320,7 @@ linux64_multiprocess-build:
 #    - .build-template
 #    - .skip-in-fast-mode-template
 #  needs:
-#    - x86_64-pc-linux-gnu-debug
+#    - x86_64-pc-linux-gnu_debug
 #  variables:
 #    BUILD_TARGET: linux64_valgrind
 

ci/dash/matrix.sh

@@ -18,28 +18,30 @@ export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/
 
 if [ "$BUILD_TARGET" = "arm-linux" ]; then
   source ./ci/test/00_setup_env_arm.sh
-elif [ "$BUILD_TARGET" = "win64" ]; then
-  source ./ci/test/00_setup_env_win64.sh
 elif [ "$BUILD_TARGET" = "linux64" ]; then
   source ./ci/test/00_setup_env_native_qt5.sh
 elif [ "$BUILD_TARGET" = "linux64_asan" ]; then
   source ./ci/test/00_setup_env_native_asan.sh
-elif [ "$BUILD_TARGET" = "linux64_tsan" ]; then
-  source ./ci/test/00_setup_env_native_tsan.sh
-elif [ "$BUILD_TARGET" = "linux64_ubsan" ]; then
-  source ./ci/test/00_setup_env_native_ubsan.sh
-elif [ "$BUILD_TARGET" = "linux64_fuzz" ]; then
-  source ./ci/test/00_setup_env_native_fuzz.sh
 elif [ "$BUILD_TARGET" = "linux64_cxx20" ]; then
   source ./ci/test/00_setup_env_native_cxx20.sh
-elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then
-  source ./ci/test/00_setup_env_native_sqlite.sh
+elif [ "$BUILD_TARGET" = "linux64_fuzz" ]; then
+  source ./ci/test/00_setup_env_native_fuzz.sh
+elif [ "$BUILD_TARGET" = "linux64_multiprocess" ]; then
+  source ./ci/test/00_setup_env_native_multiprocess.sh
 elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then
   source ./ci/test/00_setup_env_native_nowallet.sh
+elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then
+  source ./ci/test/00_setup_env_native_sqlite.sh
+elif [ "$BUILD_TARGET" = "linux64_tsan" ]; then
+  source ./ci/test/00_setup_env_native_tsan.sh
+elif [ "$BUILD_TARGET" = "linux64_ubsan" ]; then
+  source ./ci/test/00_setup_env_native_ubsan.sh
 elif [ "$BUILD_TARGET" = "linux64_valgrind" ]; then
   source ./ci/test/00_setup_env_native_valgrind.sh
 elif [ "$BUILD_TARGET" = "mac" ]; then
   source ./ci/test/00_setup_env_mac.sh
 elif [ "$BUILD_TARGET" = "s390x" ]; then
   source ./ci/test/00_setup_env_s390x.sh
+elif [ "$BUILD_TARGET" = "win64" ]; then
+  source ./ci/test/00_setup_env_win64.sh
 fi

ci/dash/test_integrationtests.sh

@@ -42,7 +42,7 @@ echo "Using socketevents mode: $SOCKETEVENTS"
 EXTRA_ARGS="--dashd-arg=-socketevents=$SOCKETEVENTS"
 
 set +e
-LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS
+LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS
 RESULT=$?
 set -e
 

ci/dash/test_unittests.sh

@@ -29,8 +29,8 @@ if [ "$DIRECT_WINE_EXEC_TESTS" = "true" ]; then
   wine ./src/test/test_dash.exe
 else
   if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
-    ${TEST_RUNNER_ENV} ./src/test/test_dash --catch_system_errors=no -l test_suite
+    ./src/test/test_dash --catch_system_errors=no -l test_suite
   else
-      ${TEST_RUNNER_ENV} make $MAKEJOBS check VERBOSE=1
+      make $MAKEJOBS check VERBOSE=1
   fi
 fi

ci/test/00_setup_env.sh

@@ -44,7 +44,6 @@ export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false}
 # This is needed because some ci machines have slow CPU or disk, so sanitizers
 # might be slow or a reindex might be waiting on disk IO.
 export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-4}
-export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-}
 export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
 export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000}