ci: merge bitcoin#27314, #28954, introduce dependency options in GitHub Actions builds, fix multiprocess builds, bump to Clang 18 #6400
+184
−106
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,8 @@ jobs: | |
| outputs: | ||
| image-tag: ${{ steps.prepare.outputs.image-tag }} | ||
| repo-name: ${{ steps.prepare.outputs.repo-name }} | ||
| dep-matrix: ${{ steps.prepare.outputs.dep-matrix }} | ||
| src-matrix: ${{ steps.prepare.outputs.src-matrix }} | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
|
|
@@ -32,6 +34,8 @@ jobs: | |
| REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') | ||
| echo "image-tag=${BRANCH_NAME}" >> $GITHUB_OUTPUT | ||
| echo "repo-name=${REPO_NAME}" >> $GITHUB_OUTPUT | ||
| echo "dep-matrix=$(jq -r '.dep' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT | ||
| echo "src-matrix=$(jq -r '.src' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT | ||
|
|
||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
|
|
@@ -56,18 +60,12 @@ jobs: | |
| cache-to: type=inline | ||
|
|
||
| build-depends: | ||
| name: ${{ matrix.depends_name }} | ||
| needs: build-image | ||
| runs-on: ubuntu-22.04 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: ${{ fromJson(needs.build-image.outputs.dep-matrix) }} | ||
| container: | ||
| image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} | ||
| options: --user root | ||
|
|
@@ -86,52 +84,36 @@ jobs: | |
| restore-keys: | | ||
| depends-sources- | ||
|
|
||
| - name: Determine params | ||
| id: det-params | ||
| run: | | ||
| dep_name="${{ matrix.depends_name }}" | ||
| dep_opts="${{ matrix.dep_opts }}" | ||
| dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)" | ||
| echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\"" | ||
| echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT | ||
|
|
||
| - name: Cache depends | ||
| uses: actions/cache@v4 | ||
| with: | ||
| path: | | ||
| depends/built | ||
| depends/${{ matrix.host }} | ||
| key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-${{ steps.det-params.outputs.dep_hash }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}- | ||
| ${{ runner.os }}-depends-${{ matrix.depends_name }}- | ||
|
|
||
| - name: Build depends | ||
| run: env HOST=${{ matrix.host }} ${{ matrix.dep_opts }} make -j$(nproc) -C depends | ||
|
|
||
| build: | ||
| name: ${{ matrix.build_target }} | ||
| needs: [build-image, build-depends] | ||
| runs-on: ubuntu-22.04 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: ${{ fromJson(needs.build-image.outputs.src-matrix) }} | ||
|
There was a problem hiding this comment. I fear this overcomplicates build.yml and will make it harder to improve / debug / logic on There was a problem hiding this comment. Outputs have to be generated in the previous job in order to be used in the current job and creating a new job just to read the matrix seems a bit wasteful, so the build image step is being used to also read, generate and export as output, the matrices needed for the next two jobs. We can probably try reusable workflows (source) in the future, it'll be more GitLab-like but would invite duplication and a different kind of convoluted syntax. Depends on how reliable this setup proves to be. |
||
| container: | ||
| image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} | ||
| options: --user root | ||
|
|
@@ -141,13 +123,28 @@ jobs: | |
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
|
|
||
| - name: Determine params | ||
| id: det-params | ||
| run: | | ||
| dep_name="${{ matrix.depends_name }}" | ||
| dep_opts="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .dep_opts" -c .github/workflows/matrix.json)" | ||
| dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)" | ||
kwvg marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\"" | ||
| echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT | ||
| dep_host="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .host" -c .github/workflows/matrix.json)" | ||
| echo "\"${dep_name}\" has HOST \"${dep_host}\"" | ||
| echo "dep_host=${dep_host}" >> $GITHUB_OUTPUT | ||
|
|
||
| - name: Restore depends cache | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| path: | | ||
| depends/built | ||
| depends/${{ steps.det-params.outputs.dep_host }} | ||
| key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-${{ steps.det-params.outputs.dep_hash }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}- | ||
| ${{ runner.os }}-depends-${{ matrix.depends_name }}- | ||
|
|
||
| - name: Determine PR Base SHA | ||
| id: vars | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| { | ||
| "dep": { | ||
| "include": [ | ||
| { | ||
| "depends_name": "arm-linux-gnueabihf", | ||
| "host": "arm-linux-gnueabihf", | ||
| "dep_opts": "" | ||
| }, | ||
| { | ||
| "depends_name": "x86_64-pc-linux-gnu_debug", | ||
| "host": "x86_64-pc-linux-gnu", | ||
| "dep_opts": "DEBUG=1" | ||
| }, | ||
| { | ||
| "depends_name": "x86_64-pc-linux-gnu_multiprocess", | ||
| "host": "x86_64-pc-linux-gnu", | ||
| "dep_opts": "DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18" | ||
| }, | ||
| { | ||
| "depends_name": "x86_64-pc-linux-gnu_nowallet", | ||
| "host": "x86_64-pc-linux-gnu", | ||
| "dep_opts": "NO_WALLET=1" | ||
| } | ||
| ] | ||
| }, | ||
| "src": { | ||
| "include": [ | ||
| { | ||
| "build_target": "arm-linux", | ||
| "depends_name": "arm-linux-gnueabihf" | ||
| }, | ||
| { | ||
| "build_target": "linux64", | ||
| "depends_name": "x86_64-pc-linux-gnu_debug" | ||
| }, | ||
| { | ||
| "build_target": "linux64_cxx20", | ||
| "depends_name": "x86_64-pc-linux-gnu_debug" | ||
| }, | ||
| { | ||
| "build_target": "linux64_fuzz", | ||
| "depends_name": "x86_64-pc-linux-gnu_debug" | ||
| }, | ||
| { | ||
| "build_target": "linux64_multiprocess", | ||
| "depends_name": "x86_64-pc-linux-gnu_multiprocess" | ||
| }, | ||
| { | ||
| "build_target": "linux64_nowallet", | ||
| "depends_name": "x86_64-pc-linux-gnu_nowallet" | ||
| }, | ||
| { | ||
| "build_target": "linux64_sqlite", | ||
| "depends_name": "x86_64-pc-linux-gnu_debug" | ||
| }, | ||
| { | ||
| "build_target": "linux64_tsan", | ||
| "depends_name": "x86_64-pc-linux-gnu_multiprocess" | ||
| }, | ||
| { | ||
| "build_target": "linux64_ubsan", | ||
| "depends_name": "x86_64-pc-linux-gnu_debug" | ||
| } | ||
| ] | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I kinda hate using jq in the build.yml like this; but I guess it's ok
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The alternative would be having to maintain two JSON files (they would have to be JSON if we want to share params between them without resorting to duplication), which wouldn't be the worst thing in the world but imo since
srcwill be reading fromdepfor values and they are configuring the same actions run, just different jobs, consolidation would be better.Plus we'd be using
jqlater on regardless to read through the matrix definition later on to finddep_optsandhostfromdepends_name.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
these 2 JSON seems doens't have any dependency on each other, maybe better to have 2 files?
Is there any case, when you need both of them at once?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Because they the second section (
src) refers to the first section (dep) later on and they ultimately represent two halves of one workflow.dash/.github/workflows/build.yml
Lines 126 to 136 in 3cc8c70
Examples of these are explained in the PR description.