From 05466d3e2becd1a3fe0164bd45febf5b799e83d3 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 21 Dec 2023 21:06:41 +0800 Subject: [PATCH] chore(CI): Improve github actions (#1791) This patch mainly improve the github actions for CI, including: - Separate Java client CI 'test' step into 'build_server' and 'test_java_client'. So it's not needed to build server again when retrying to run Java client tests. - Encapsulate some actions when build and test servers, including: - rebuild_thirdparty_if_needed - build_pegasus - upload_artifact - download_artifact - run_server_tests --- .github/actions/build_pegasus/action.yaml | 52 +++ .github/actions/download_artifact/action.yaml | 36 ++ .../rebuild_thirdparty_if_needed/action.yaml | 56 +++ .github/actions/run_server_tests/action.yaml | 27 ++ .github/actions/upload_artifact/action.yaml | 38 ++ .github/workflows/lint_and_test_admin-cli.yml | 1 + .../lint_and_test_admin-cli_always_pass.yml | 1 + .github/workflows/lint_and_test_collector.yml | 4 +- .github/workflows/lint_and_test_cpp.yaml | 440 +++--------------- .github/workflows/lint_and_test_go-client.yml | 1 + ...ient.yml => lint_and_test_java-client.yml} | 71 +-- .github/workflows/lint_and_test_pegic.yml | 1 + .../lint_and_test_pegic_always_pass.yml | 1 + .github/workflows/test_nodejs-client.yml | 1 + .github/workflows/test_python-client.yml | 1 + .github/workflows/test_scala-client.yml | 4 +- run.sh | 21 +- 17 files changed, 320 insertions(+), 436 deletions(-) create mode 100644 .github/actions/build_pegasus/action.yaml create mode 100644 .github/actions/download_artifact/action.yaml create mode 100644 .github/actions/rebuild_thirdparty_if_needed/action.yaml create mode 100644 .github/actions/run_server_tests/action.yaml create mode 100644 .github/actions/upload_artifact/action.yaml rename .github/workflows/{test_java-client.yml => lint_and_test_java-client.yml} (54%) diff --git a/.github/actions/build_pegasus/action.yaml b/.github/actions/build_pegasus/action.yaml new file mode 100644 index 0000000000..ad640f2fa5 --- /dev/null +++ b/.github/actions/build_pegasus/action.yaml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Pegasus +runs: + using: composite + steps: + - name: Setup cache + uses: actions/cache@v3 + with: + path: | + /github/home/.ccache + key: ${{ env.ARTIFACT_NAME }}_ccache + - name: Free Disk Space (Ubuntu) + run: | + .github/workflows/free_disk_space.sh + shell: bash + - name: Compilation + run: | + ccache -p + ccache -z + ./run.sh build -j $(nproc) --skip_thirdparty ${BUILD_OPTIONS} + ccache -s + shell: bash + - name: Pack Server + run: | + ./run.sh pack_server -j + rm -rf pegasus-server-* + shell: bash + - name: Pack Tools + run: | + ./run.sh pack_tools -j + rm -rf pegasus-tools-* + shell: bash + - name: Clear Build Files + run: | + find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + shell: bash diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/download_artifact/action.yaml new file mode 100644 index 0000000000..35ddb8ae01 --- /dev/null +++ b/.github/actions/download_artifact/action.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Download artifact +runs: + using: composite + steps: + - name: Unpack prebuilt third-parties + run: | + unzip /root/thirdparties-bin.zip -d ./thirdparty + rm -f /root/thirdparties-bin.zip + shell: bash + - name: Download tarball + uses: actions/download-artifact@v3 + with: + name: ${{ env.ARTIFACT_NAME }}_artifact_${{ github.sha }} + path: . + - name: Unpack tarball + run: | + tar -zxvf ${ARTIFACT_NAME}_builder.tar + rm -f ${ARTIFACT_NAME}_builder.tar + shell: bash diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml b/.github/actions/rebuild_thirdparty_if_needed/action.yaml new file mode 100644 index 0000000000..ff48fff299 --- /dev/null +++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Rebuild thirdparty if needed +runs: + using: composite + steps: + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + thirdparty: + - '.github/workflows/thirdparty-regular-push.yml' + - 'docker/thirdparties-src/**' + - 'docker/thirdparties-bin/**' + - 'thirdparty/**' + - name: Unpack prebuilt third-parties + if: steps.changes.outputs.thirdparty == 'false' + run: | + rm -f /root/thirdparties-src.zip + unzip /root/thirdparties-bin.zip -d ./thirdparty + rm -f /root/thirdparties-bin.zip + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs + shell: bash + - name: Rebuild third-parties + if: steps.changes.outputs.thirdparty == 'true' + working-directory: thirdparty + # Build third-parties and leave some necessary libraries and source. + run: | + rm -f /root/thirdparties-src.zip + mkdir build + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=${USE_JEMALLOC} -B build/ + cmake --build build/ -j $(nproc) + rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* + find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + ../scripts/download_hadoop.sh hadoop-bin + ../scripts/download_zk.sh zookeeper-bin + rm -rf hadoop-bin/share/doc + rm -rf zookeeper-bin/docs + shell: bash diff --git a/.github/actions/run_server_tests/action.yaml b/.github/actions/run_server_tests/action.yaml new file mode 100644 index 0000000000..816315d2bf --- /dev/null +++ b/.github/actions/run_server_tests/action.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Run server tests +runs: + using: composite + steps: + - name: Unit Testing + run: | + export LD_LIBRARY_PATH=$(pwd)/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server + ulimit -s unlimited + ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} + shell: bash diff --git a/.github/actions/upload_artifact/action.yaml b/.github/actions/upload_artifact/action.yaml new file mode 100644 index 0000000000..0fb6975d7b --- /dev/null +++ b/.github/actions/upload_artifact/action.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Upload artifact +runs: + using: composite + steps: + - name: Tar files + run: | + mv thirdparty/hadoop-bin ./ + mv thirdparty/zookeeper-bin ./ + rm -rf thirdparty + # The following operations are tricky, these directories and files don't exist if not build with '--test'. + # When build binaries for client tests, it's not needed to add '--test'. + mkdir -p build/latest/bin + mkdir -p build/latest/src/server/test + touch build/latest/src/server/test/config.ini + tar -zcvhf ${ARTIFACT_NAME}_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin + shell: bash + - name: Upload tarball + uses: actions/upload-artifact@v3 + with: + name: ${{ env.ARTIFACT_NAME }}_artifact_${{ github.sha }} + path: ${{ env.ARTIFACT_NAME }}_builder.tar diff --git a/.github/workflows/lint_and_test_admin-cli.yml b/.github/workflows/lint_and_test_admin-cli.yml index ee57ac9d89..788ba8ba3e 100644 --- a/.github/workflows/lint_and_test_admin-cli.yml +++ b/.github/workflows/lint_and_test_admin-cli.yml @@ -29,6 +29,7 @@ on: - ci-test # testing branch for github action - '*dev' paths: + - .github/workflows/lint_and_test_admin-cli.yml - admin-cli/** # for manually triggering workflow diff --git a/.github/workflows/lint_and_test_admin-cli_always_pass.yml b/.github/workflows/lint_and_test_admin-cli_always_pass.yml index 5189678d0b..9c3b18f371 100644 --- a/.github/workflows/lint_and_test_admin-cli_always_pass.yml +++ b/.github/workflows/lint_and_test_admin-cli_always_pass.yml @@ -29,6 +29,7 @@ on: - ci-test # testing branch for github action - '*dev' paths-ignore: + - .github/workflows/lint_and_test_admin-cli_always_pass.yml - admin-cli/** # for manually triggering workflow diff --git a/.github/workflows/lint_and_test_collector.yml b/.github/workflows/lint_and_test_collector.yml index 3b0a378ae0..e257dbc93d 100644 --- a/.github/workflows/lint_and_test_collector.yml +++ b/.github/workflows/lint_and_test_collector.yml @@ -29,8 +29,8 @@ on: - ci-test # testing branch for github action - '*dev' paths: - - '.github/workflows/lint_and_test_collector.yml' - - 'collector/**' + - .github/workflows/lint_and_test_collector.yml + - collector/** # for manually triggering workflow workflow_dispatch: diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 572d11b1a9..f90a18074d 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -14,12 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ---- -# Developer Notes: -# -# This config is for github actions. Before merging your changes of this file, -# it's recommended to create a PR against the ci-test branch to test if it works -# as expected. name: Cpp CI @@ -33,17 +27,18 @@ on: - ci-test # testing branch for github action - '*dev' paths: - - '.github/workflows/lint_and_test_cpp.yaml' - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'CMakeLists.txt' - - 'run.sh' - - 'scripts/pack_server.sh' - - 'scripts/pack_tools.sh' - - 'cmake_modules/**' - - 'src/**' - - 'thirdparty/**' + - .github/actions + - .github/workflows/lint_and_test_cpp.yaml + - .github/workflows/thirdparty-regular-push.yml + - docker/thirdparties-bin/** + - docker/thirdparties-src/** + - CMakeLists.txt + - cmake_modules/** + - run.sh + - scripts/pack_server.sh + - scripts/pack_tools.sh + - src/** + - thirdparty/** # for manually triggering workflow workflow_dispatch: @@ -68,6 +63,8 @@ jobs: name: IWYU needs: cpp_clang_format_linter runs-on: ubuntu-latest + env: + USE_JEMALLOC: OFF container: image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }} steps: @@ -75,29 +72,7 @@ jobs: - name: Free Disk Space (Ubuntu) run: | .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: unzip /root/thirdparties-bin.zip -d ./thirdparty - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin + - uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Build IWYU binary run: | mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git @@ -115,80 +90,17 @@ jobs: name: Build Release needs: cpp_clang_format_linter runs-on: ubuntu-latest + env: + USE_JEMALLOC: OFF + ARTIFACT_NAME: release + BUILD_OPTIONS: -t release --test container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 - - name: Setup cache - uses: actions/cache@v3 - with: - path: | - /github/home/.ccache - key: release_ccache - - name: Free Disk Space (Ubuntu) - run: | - .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - rm -f /root/thirdparties-src.zip - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin - rm -rf hadoop-bin/share/doc - rm -rf zookeeper-bin/docs - - name: Compilation - run: | - ccache -p - ccache -z - ./run.sh build --test --skip_thirdparty -j $(nproc) -t release - ccache -s - - name: Clear Build Files - run: | - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - - name: Pack Server - run: | - ./run.sh pack_server - rm -rf pegasus-server-* - - name: Pack Tools - run: | - ./run.sh pack_tools - rm -rf pegasus-tools-* - - name: Tar files - run: | - mv thirdparty/hadoop-bin ./ - mv thirdparty/zookeeper-bin ./ - rm -rf thirdparty - tar -zcvhf release__builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: release_artifact_${{ github.sha }} - path: release__builder.tar + - uses: "./.github/actions/rebuild_thirdparty_if_needed" + - uses: "./.github/actions/build_pegasus" + - uses: "./.github/actions/upload_artifact" test_Release: name: Test Release @@ -238,100 +150,31 @@ jobs: - throttle_test needs: build_Release runs-on: ubuntu-latest + env: + ARTIFACT_NAME: release container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} options: --cap-add=SYS_PTRACE steps: - uses: actions/checkout@v3 - - name: Unpack prebuilt third-parties - run: | - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - - name: Download Artifact - uses: actions/download-artifact@v3 - with: - name: release_artifact_${{ github.sha }} - path: . - - name: Tar files - run: | - tar -zxvf release__builder.tar - rm -f release__builder.tar - - name: Unit Testing - run: | - export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server - ulimit -s unlimited - ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} + - uses: "./.github/actions/download_artifact" + - uses: "./.github/actions/run_server_tests" build_ASAN: name: Build ASAN needs: cpp_clang_format_linter runs-on: ubuntu-latest + env: + USE_JEMALLOC: OFF + ARTIFACT_NAME: release_address + BUILD_OPTIONS: --sanitizer address --disable_gperf --test container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 - - name: Setup cache - uses: actions/cache@v3 - with: - path: | - /github/home/.ccache - key: asan_ccache - - name: Free Disk Space (Ubuntu) - run: | - .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - rm -f /root/thirdparties-src.zip - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin - rm -rf hadoop-bin/share/doc - rm -rf zookeeper-bin/docs - - name: Compilation - run: | - ccache -p - ccache -z - ./run.sh build --test --sanitizer address --skip_thirdparty --disable_gperf -j $(nproc) - ccache -s - - name: Clear Build Files - run: | - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - - name: Tar files - run: | - mv thirdparty/hadoop-bin ./ - mv thirdparty/zookeeper-bin ./ - rm -rf thirdparty - tar -zcvhf release_address_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: release_address_artifact_${{ github.sha }} - path: release_address_builder.tar + - uses: "./.github/actions/rebuild_thirdparty_if_needed" + - uses: "./.github/actions/build_pegasus" + - uses: "./.github/actions/upload_artifact" test_ASAN: name: Test ASAN @@ -383,29 +226,15 @@ jobs: # - throttle_test needs: build_ASAN runs-on: ubuntu-latest + env: + ARTIFACT_NAME: release_address container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} options: --cap-add=SYS_PTRACE steps: - uses: actions/checkout@v3 - - name: Unpack prebuilt third-parties - run: | - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - - name: Download Artifact - uses: actions/download-artifact@v3 - with: - name: release_address_artifact_${{ github.sha }} - path: . - - name: Tar files - run: | - tar -zxvf release_address_builder.tar - rm -f release_address_builder.tar - - name: Unit Testing - run: | - export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server - ulimit -s unlimited - ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} + - uses: "./.github/actions/download_artifact" + - uses: "./.github/actions/run_server_tests" # TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests # when we are going to release a stable version. So we disable them in regular CI @@ -414,72 +243,17 @@ jobs: # name: Build UBSAN # needs: cpp_clang_format_linter # runs-on: ubuntu-latest +# env: +# USE_JEMALLOC: OFF +# ARTIFACT_NAME: release_undefined +# BUILD_OPTIONS: --sanitizer undefined --disable_gperf --test # container: # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # steps: # - uses: actions/checkout@v3 -# - name: Setup cache -# uses: actions/cache@v3 -# with: -# path: | -# /github/home/.ccache -# key: ubsan_ccache -# - name: Free Disk Space (Ubuntu) -# run: | -# .github/workflows/free_disk_space.sh -# - uses: dorny/paths-filter@v2 -# id: changes -# with: -# filters: | -# thirdparty: -# - '.github/workflows/thirdparty-regular-push.yml' -# - 'docker/thirdparties-src/**' -# - 'docker/thirdparties-bin/**' -# - 'thirdparty/**' -# - name: Unpack prebuilt third-parties -# if: steps.changes.outputs.thirdparty == 'false' -# run: | -# rm -f /root/thirdparties-src.zip -# unzip /root/thirdparties-bin.zip -d ./thirdparty -# rm -f /root/thirdparties-bin.zip -# find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + -# rm -rf ./thirdparty/hadoop-bin/share/doc -# rm -rf ./thirdparty/zookeeper-bin/docs -# - name: Rebuild third-parties -# if: steps.changes.outputs.thirdparty == 'true' -# working-directory: thirdparty -# # Build thirdparties and leave some necessary libraries and source -# run: | -# rm -f /root/thirdparties-src.zip -# mkdir build -# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ -# cmake --build build/ -j $(nproc) -# rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* -# find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + -# ../scripts/download_hadoop.sh hadoop-bin -# ../scripts/download_zk.sh zookeeper-bin -# rm -rf hadoop-bin/share/doc -# rm -rf zookeeper-bin/docs -# - name: Compilation -# run: | -# ccache -p -# ccache -z -# ./run.sh build --test --sanitizer undefined --skip_thirdparty --disable_gperf -j $(nproc) -# ccache -s -# - name: Clear Build Files -# run: | -# find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + -# - name: Tar files -# run: | -# mv thirdparty/hadoop-bin ./ -# mv thirdparty/zookeeper-bin ./ -# rm -rf thirdparty -# tar -zcvhf release_undefined_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin -# - name: Upload Artifact -# uses: actions/upload-artifact@v3 -# with: -# name: release_undefined_artifact_${{ github.sha }} -# path: release_undefined_builder.tar +# - uses: "./.github/actions/rebuild_thirdparty_if_needed" +# - uses: "./.github/actions/build_pegasus" +# - uses: "./.github/actions/upload_artifact" # # test_UBSAN: # name: Test UBSAN @@ -527,112 +301,32 @@ jobs: # - throttle_test # needs: build_UBSAN # runs-on: ubuntu-latest +# env: +# ARTIFACT_NAME: release_undefined # container: # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # options: --cap-add=SYS_PTRACE # steps: # - uses: actions/checkout@v3 -# - name: Free Disk Space (Ubuntu) -# run: | -# .github/workflows/free_disk_space.sh -# - name: Unpack prebuilt third-parties -# run: | -# unzip /root/thirdparties-bin.zip -d ./thirdparty -# rm -f /root/thirdparties-bin.zip -# - name: Download Artifact -# uses: actions/download-artifact@v3 -# with: -# name: release_undefined_artifact_${{ github.sha }} -# path: . -# - name: Tar files -# run: | -# tar -zxvf release_undefined_builder.tar -# rm -f release_undefined_builder.tar -# - name: Unit Testing -# run: | -# export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server -# ulimit -s unlimited -# ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} +# - uses: "./.github/actions/download_artifact" +# - uses: "./.github/actions/run_server_tests" build_with_jemalloc: name: Build with jemalloc needs: cpp_clang_format_linter runs-on: ubuntu-latest + env: + USE_JEMALLOC: ON + ARTIFACT_NAME: release_jemalloc + BUILD_OPTIONS: -t release --use_jemalloc --test container: image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 - - name: Setup cache - uses: actions/cache@v3 - with: - path: | - /github/home/.ccache - key: jemalloc_ccache - - name: Free Disk Space (Ubuntu) - run: | - .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - rm -f /root/thirdparties-src.zip - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin - rm -rf hadoop-bin/share/doc - rm -rf zookeeper-bin/docs - - name: Compilation - # TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked. - run: | - ccache -p - ccache -z - ./run.sh build --test --skip_thirdparty -j $(nproc) -t release --use_jemalloc - ccache -s - - name: Clear Build Files - run: | - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - - name: Pack Server - run: | - ./run.sh pack_server -j - rm -rf pegasus-server-* - - name: Pack Tools - run: | - ./run.sh pack_tools -j - rm -rf pegasus-tools-* - - name: Tar files - run: | - mv thirdparty/hadoop-bin ./ - mv thirdparty/zookeeper-bin ./ - rm -rf thirdparty - tar -zcvhf release_jemalloc_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: release_jemalloc_artifact_${{ github.sha }} - path: release_jemalloc_builder.tar + - uses: "./.github/actions/rebuild_thirdparty_if_needed" + # TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked. + - uses: "./.github/actions/build_pegasus" + - uses: "./.github/actions/upload_artifact" test_with_jemalloc: name: Test with jemallc @@ -643,29 +337,15 @@ jobs: - dsn_utils_tests needs: build_with_jemalloc runs-on: ubuntu-latest + env: + ARTIFACT_NAME: release_jemalloc container: image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} options: --cap-add=SYS_PTRACE steps: - uses: actions/checkout@v3 - - name: Unpack prebuilt third-parties - run: | - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - - name: Download Artifact - uses: actions/download-artifact@v3 - with: - name: release_jemalloc_artifact_${{ github.sha }} - path: . - - name: Tar files - run: | - tar -zxvf release_jemalloc_builder.tar - rm -f release_jemalloc_builder.tar - - name: Unit Testing - run: | - export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server - ulimit -s unlimited - ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} + - uses: "./.github/actions/download_artifact" + - uses: "./.github/actions/run_server_tests" build_pegasus_on_macos: name: macOS diff --git a/.github/workflows/lint_and_test_go-client.yml b/.github/workflows/lint_and_test_go-client.yml index 2b660a0220..011ae97c3b 100644 --- a/.github/workflows/lint_and_test_go-client.yml +++ b/.github/workflows/lint_and_test_go-client.yml @@ -29,6 +29,7 @@ on: - ci-test # testing branch for github action - '*dev' paths: + - .github/workflows/lint_and_test_go-client.yml - go-client/** # for manually triggering workflow diff --git a/.github/workflows/test_java-client.yml b/.github/workflows/lint_and_test_java-client.yml similarity index 54% rename from .github/workflows/test_java-client.yml rename to .github/workflows/lint_and_test_java-client.yml index a3930b9635..dfe747bd80 100644 --- a/.github/workflows/test_java-client.yml +++ b/.github/workflows/lint_and_test_java-client.yml @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# name: Test - java client on: @@ -26,6 +25,7 @@ on: - ci-test # testing branch for github action - '*dev' # developing branch paths: + - .github/workflows/lint_and_test_java-client.yml - java-client/** workflow_dispatch: @@ -36,7 +36,7 @@ defaults: jobs: spotless: - name: spotless + name: Spotless runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -46,10 +46,29 @@ jobs: - name: spotless working-directory: ./java-client run: mvn spotless:check - test: - name: test + + build_server: + name: Build server needs: spotless runs-on: ubuntu-latest + env: + USE_JEMALLOC: OFF + ARTIFACT_NAME: release_for_java_client + BUILD_OPTIONS: -t release + container: + image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} + steps: + - uses: actions/checkout@v3 + - uses: "./.github/actions/rebuild_thirdparty_if_needed" + - uses: "./.github/actions/build_pegasus" + - uses: "./.github/actions/upload_artifact" + + test_java_client: + name: Test Java client + needs: build_server + runs-on: ubuntu-latest + env: + ARTIFACT_NAME: release_for_java_client container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} strategy: @@ -64,52 +83,10 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - name: Free Disk Space (Ubuntu) - run: | - .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - # TODO(yingchun): Build a docker image with prebuilt server binaries, we can use them directly without building server binaries if no server changes. - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - rm -f /root/thirdparties-src.zip - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin - rm -rf hadoop-bin/share/doc - rm -rf zookeeper-bin/docs - - name: Build Pegasus server - run: | - ./run.sh build --skip_thirdparty -j $(nproc) -t release - - name: Clear build temporary files - run: | - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} + - uses: "./.github/actions/download_artifact" - name: Start Pegasus cluster run: | apt-get update diff --git a/.github/workflows/lint_and_test_pegic.yml b/.github/workflows/lint_and_test_pegic.yml index 09b85fa92c..3788b44121 100644 --- a/.github/workflows/lint_and_test_pegic.yml +++ b/.github/workflows/lint_and_test_pegic.yml @@ -29,6 +29,7 @@ on: - ci-test # testing branch for github action - '*dev' paths: + - .github/workflows/lint_and_test_pegic.yml - pegic/** # for manually triggering workflow diff --git a/.github/workflows/lint_and_test_pegic_always_pass.yml b/.github/workflows/lint_and_test_pegic_always_pass.yml index f4ab49298b..2a9fb58d95 100644 --- a/.github/workflows/lint_and_test_pegic_always_pass.yml +++ b/.github/workflows/lint_and_test_pegic_always_pass.yml @@ -29,6 +29,7 @@ on: - ci-test # testing branch for github action - '*dev' paths-ignore: + - .github/workflows/lint_and_test_pegic_always_pass.yml - pegic/** # for manually triggering workflow diff --git a/.github/workflows/test_nodejs-client.yml b/.github/workflows/test_nodejs-client.yml index a3d7005c1c..b24065ec93 100644 --- a/.github/workflows/test_nodejs-client.yml +++ b/.github/workflows/test_nodejs-client.yml @@ -26,6 +26,7 @@ on: - ci-test # testing branch for github action - '*dev' # developing branch paths: + - .github/workflows/test_nodejs-client.yml - nodejs-client/** workflow_dispatch: diff --git a/.github/workflows/test_python-client.yml b/.github/workflows/test_python-client.yml index 623c0cfc88..bdef9d221f 100644 --- a/.github/workflows/test_python-client.yml +++ b/.github/workflows/test_python-client.yml @@ -26,6 +26,7 @@ on: - ci-test # testing branch for github action - '*dev' # developing branch paths: + - .github/workflows/test_python-client.yml - python-client/** workflow_dispatch: diff --git a/.github/workflows/test_scala-client.yml b/.github/workflows/test_scala-client.yml index 1f9eb044ae..047e550422 100644 --- a/.github/workflows/test_scala-client.yml +++ b/.github/workflows/test_scala-client.yml @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# name: Test - scala client on: @@ -24,7 +23,8 @@ on: - master - 'v[0-9]+.*' # release branch paths: - - scala-client/** + - .github/workflows/test_scala-client.yml + - scala-client/** workflow_dispatch: diff --git a/run.sh b/run.sh index c73d3e5296..dfae368f42 100755 --- a/run.sh +++ b/run.sh @@ -455,6 +455,7 @@ function run_test() mkdir -p $REPORT_DIR fi + # Run all tests if none specified. if [ "$test_modules" == "" ]; then test_modules=$(IFS=,; echo "${all_tests[*]}") fi @@ -462,7 +463,7 @@ function run_test() for module in `echo $test_modules | sed 's/,/ /g'`; do echo "====================== run $module ==========================" - # restart onebox when test pegasus + # The tests which need start onebox. local need_onebox_tests=( backup_restore_test base_api_test @@ -475,11 +476,14 @@ function run_test() restore_test throttle_test ) + # Restart onebox if needed. if [[ "${need_onebox_tests[@]}" =~ "${module}" ]]; then + # Clean up onebox at first. run_clear_onebox - m_count=3 + master_count=3 + # Update options if needed, this should be done before starting onebox to make new options take effect. if [ "${module}" == "recovery_test" ]; then - m_count=1 + master_count=1 opts="meta_state_service_type=meta_state_service_simple,distributed_lock_service_type=distributed_lock_service_simple" fi if [ "${module}" == "backup_restore_test" ]; then @@ -488,24 +492,30 @@ function run_test() if [ "${module}" == "restore_test" ]; then opts="cold_backup_disabled=false,cold_backup_checkpoint_reserve_minutes=0,cold_backup_root=onebox" fi + # Append onebox_opts if needed. [ -z ${onebox_opts} ] || opts="${opts},${onebox_opts}" - if ! run_start_onebox -m ${m_count} -w -c --opts ${opts}; then + # Start onebox. + if ! run_start_onebox -m ${master_count} -w -c --opts ${opts}; then echo "ERROR: unable to continue on testing because starting onebox failed" exit 1 fi # TODO(yingchun): remove it? sed -i "s/@LOCAL_HOSTNAME@/${LOCAL_HOSTNAME}/g" ${BUILD_LATEST_DIR}/src/server/test/config.ini else + # Restart ZK in what ever case. run_stop_zk run_start_zk fi + + # Run server test. pushd ${BUILD_LATEST_DIR}/bin/${module} REPORT_DIR=${REPORT_DIR} TEST_BIN=${module} TEST_OPTS=${test_opts} ./run.sh if [ $? != 0 ]; then echo "run test \"$module\" in `pwd` failed" exit 1 fi - # clear onebox if needed + + # Clear onebox if needed. if [[ "${need_onebox_tests[@]}" =~ "${test_modules}" ]]; then if [ "$clear_flags" == "1" ]; then run_clear_onebox @@ -519,6 +529,7 @@ function run_test() used_time=$((finish_time-start_time)) echo "Test elapsed time: $((used_time/60))m $((used_time%60))s" + # TODO(yingchun): make sure if gcov can be ran normally. if [ "$enable_gcov" == "yes" ]; then echo "Generating gcov report..." cd $ROOT