From 80ce847c1cffa1d142c14e1393c704cc6ad73927 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 19 Dec 2023 15:41:12 +0800 Subject: [PATCH] Reduce duplicate code --- .github/workflows/lint_and_test_admin-cli.yml | 6 +- .github/workflows/lint_and_test_cpp.yaml | 230 ++---------------- .github/workflows/lint_and_test_go-client.yml | 4 +- .github/workflows/regular-build.yml | 8 +- .github/workflows/test_java-client.yml | 35 +-- go-client/README.md | 2 +- run.sh | 21 +- 7 files changed, 54 insertions(+), 252 deletions(-) diff --git a/.github/workflows/lint_and_test_admin-cli.yml b/.github/workflows/lint_and_test_admin-cli.yml index a02a4afe06..12aa82e1d9 100644 --- a/.github/workflows/lint_and_test_admin-cli.yml +++ b/.github/workflows/lint_and_test_admin-cli.yml @@ -44,11 +44,11 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.14 + go-version: 1.17 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: - version: v1.29 + version: v1.55.2 working-directory: ./admin-cli build: @@ -60,7 +60,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.14 + go-version: 1.17 - name: Compile working-directory: ./admin-cli run: make diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 85b8eeb09d..f771cf27c9 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -66,6 +66,8 @@ jobs: name: IWYU needs: cpp_clang_format_linter runs-on: ubuntu-latest + env: + USE_JEMALLOC: OFF container: image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }} steps: @@ -73,29 +75,7 @@ jobs: - name: Free Disk Space (Ubuntu) run: | .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: unzip /root/thirdparties-bin.zip -d ./thirdparty - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin + - uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Build IWYU binary run: | mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git @@ -113,6 +93,9 @@ jobs: name: Build Release needs: cpp_clang_format_linter runs-on: ubuntu-latest + env: + USE_JEMALLOC: OFF + ARTIFACT_NAME: release container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: @@ -126,39 +109,7 @@ jobs: - name: Free Disk Space (Ubuntu) run: | .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - rm -f /root/thirdparties-src.zip - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin - rm -rf hadoop-bin/share/doc - rm -rf zookeeper-bin/docs + - uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Compilation run: | ccache -p @@ -176,17 +127,7 @@ jobs: run: | ./run.sh pack_tools rm -rf pegasus-tools-* - - name: Tar files - run: | - mv thirdparty/hadoop-bin ./ - mv thirdparty/zookeeper-bin ./ - rm -rf thirdparty - tar -zcvhf release__builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: release_artifact_${{ github.sha }} - path: release__builder.tar + - uses: "./.github/actions/upload_artifact/action.yaml" test_Release: name: Test Release @@ -256,7 +197,7 @@ jobs: rm -f release__builder.tar - name: Unit Testing run: | - export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server + export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server ulimit -s unlimited ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} @@ -264,6 +205,9 @@ jobs: name: Build ASAN needs: cpp_clang_format_linter runs-on: ubuntu-latest + env: + USE_JEMALLOC: OFF + ARTIFACT_NAME: release_address container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: @@ -277,39 +221,7 @@ jobs: - name: Free Disk Space (Ubuntu) run: | .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - rm -f /root/thirdparties-src.zip - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin - rm -rf hadoop-bin/share/doc - rm -rf zookeeper-bin/docs + - uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Compilation run: | ccache -p @@ -319,17 +231,7 @@ jobs: - name: Clear Build Files run: | find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - - name: Tar files - run: | - mv thirdparty/hadoop-bin ./ - mv thirdparty/zookeeper-bin ./ - rm -rf thirdparty - tar -zcvhf release_address_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: release_address_artifact_${{ github.sha }} - path: release_address_builder.tar + - uses: "./.github/actions/upload_artifact/action.yaml" test_ASAN: name: Test ASAN @@ -401,7 +303,7 @@ jobs: rm -f release_address_builder.tar - name: Unit Testing run: | - export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server + export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server ulimit -s unlimited ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} @@ -412,6 +314,9 @@ jobs: # name: Build UBSAN # needs: cpp_clang_format_linter # runs-on: ubuntu-latest +# env: +# USE_JEMALLOC: OFF +# ARTIFACT_NAME: release_undefined # container: # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # steps: @@ -425,39 +330,7 @@ jobs: # - name: Free Disk Space (Ubuntu) # run: | # .github/workflows/free_disk_space.sh -# - uses: dorny/paths-filter@v2 -# id: changes -# with: -# filters: | -# thirdparty: -# - '.github/workflows/thirdparty-regular-push.yml' -# - 'docker/thirdparties-src/**' -# - 'docker/thirdparties-bin/**' -# - 'thirdparty/**' -# - name: Unpack prebuilt third-parties -# if: steps.changes.outputs.thirdparty == 'false' -# run: | -# rm -f /root/thirdparties-src.zip -# unzip /root/thirdparties-bin.zip -d ./thirdparty -# rm -f /root/thirdparties-bin.zip -# find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + -# rm -rf ./thirdparty/hadoop-bin/share/doc -# rm -rf ./thirdparty/zookeeper-bin/docs -# - name: Rebuild third-parties -# if: steps.changes.outputs.thirdparty == 'true' -# working-directory: thirdparty -# # Build thirdparties and leave some necessary libraries and source -# run: | -# rm -f /root/thirdparties-src.zip -# mkdir build -# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ -# cmake --build build/ -j $(nproc) -# rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* -# find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + -# ../scripts/download_hadoop.sh hadoop-bin -# ../scripts/download_zk.sh zookeeper-bin -# rm -rf hadoop-bin/share/doc -# rm -rf zookeeper-bin/docs +# - uses: "./.github/actions/rebuild_thirdparty_if_needed" # - name: Compilation # run: | # ccache -p @@ -467,17 +340,7 @@ jobs: # - name: Clear Build Files # run: | # find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + -# - name: Tar files -# run: | -# mv thirdparty/hadoop-bin ./ -# mv thirdparty/zookeeper-bin ./ -# rm -rf thirdparty -# tar -zcvhf release_undefined_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin -# - name: Upload Artifact -# uses: actions/upload-artifact@v3 -# with: -# name: release_undefined_artifact_${{ github.sha }} -# path: release_undefined_builder.tar +# - uses: "./.github/actions/upload_artifact/action.yaml" # # test_UBSAN: # name: Test UBSAN @@ -548,7 +411,7 @@ jobs: # rm -f release_undefined_builder.tar # - name: Unit Testing # run: | -# export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server +# export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server # ulimit -s unlimited # ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} @@ -556,6 +419,9 @@ jobs: name: Build with jemalloc needs: cpp_clang_format_linter runs-on: ubuntu-latest + env: + USE_JEMALLOC: ON + ARTIFACT_NAME: release_jemalloc container: image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} steps: @@ -569,39 +435,7 @@ jobs: - name: Free Disk Space (Ubuntu) run: | .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - rm -f /root/thirdparties-src.zip - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin - rm -rf hadoop-bin/share/doc - rm -rf zookeeper-bin/docs + - uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Compilation run: | ccache -p @@ -619,17 +453,7 @@ jobs: run: | ./run.sh pack_tools -j rm -rf pegasus-tools-* - - name: Tar files - run: | - mv thirdparty/hadoop-bin ./ - mv thirdparty/zookeeper-bin ./ - rm -rf thirdparty - tar -zcvhf release_jemalloc_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: release_jemalloc_artifact_${{ github.sha }} - path: release_jemalloc_builder.tar + - uses: "./.github/actions/upload_artifact/action.yaml" test_with_jemalloc: name: Test with jemallc @@ -660,7 +484,7 @@ jobs: rm -f release_jemalloc_builder.tar - name: Unit Testing run: | - export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server + export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server ulimit -s unlimited ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }} diff --git a/.github/workflows/lint_and_test_go-client.yml b/.github/workflows/lint_and_test_go-client.yml index d72afcef3b..91cfceed54 100644 --- a/.github/workflows/lint_and_test_go-client.yml +++ b/.github/workflows/lint_and_test_go-client.yml @@ -49,7 +49,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.14 + go-version: 1.17 - name: Run Test run: ./test.sh working-directory: ./go-client @@ -60,5 +60,5 @@ jobs: - name: Lint uses: golangci/golangci-lint-action@v3 with: - version: v1.29 + version: v1.55.2 working-directory: ./go-client diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml index 9e9a0d06fb..aafb2c7a40 100644 --- a/.github/workflows/regular-build.yml +++ b/.github/workflows/regular-build.yml @@ -110,14 +110,14 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.14 + go-version: 1.17 - name: Build go-client run: make working-directory: ./go-client - name: Lint go-client uses: golangci/golangci-lint-action@v3 with: - version: v1.29 + version: v1.55.2 working-directory: ./go-client - name: Build admin-cli run: make @@ -125,7 +125,7 @@ jobs: - name: Lint admin-cli uses: golangci/golangci-lint-action@v3 with: - version: v1.29 + version: v1.55.2 working-directory: ./admin-cli - name: Build pegic run: make @@ -133,7 +133,7 @@ jobs: - name: Lint pegic uses: golangci/golangci-lint-action@v3 with: - version: v1.29 + version: v1.55.2 working-directory: ./pegic build_java: diff --git a/.github/workflows/test_java-client.yml b/.github/workflows/test_java-client.yml index a3930b9635..a99a13dac5 100644 --- a/.github/workflows/test_java-client.yml +++ b/.github/workflows/test_java-client.yml @@ -67,40 +67,7 @@ jobs: - name: Free Disk Space (Ubuntu) run: | .github/workflows/free_disk_space.sh - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - thirdparty: - - '.github/workflows/thirdparty-regular-push.yml' - - 'docker/thirdparties-src/**' - - 'docker/thirdparties-bin/**' - - 'thirdparty/**' - - name: Unpack prebuilt third-parties - if: steps.changes.outputs.thirdparty == 'false' - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - # TODO(yingchun): Build a docker image with prebuilt server binaries, we can use them directly without building server binaries if no server changes. - - name: Rebuild third-parties - if: steps.changes.outputs.thirdparty == 'true' - working-directory: thirdparty - # Build thirdparties and leave some necessary libraries and source - run: | - rm -f /root/thirdparties-src.zip - mkdir build - cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/ - cmake --build build/ -j $(nproc) - rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* - find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin - rm -rf hadoop-bin/share/doc - rm -rf zookeeper-bin/docs + - uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Build Pegasus server run: | ./run.sh build --skip_thirdparty -j $(nproc) -t release diff --git a/go-client/README.md b/go-client/README.md index 82e021955f..91da971a2f 100644 --- a/go-client/README.md +++ b/go-client/README.md @@ -22,7 +22,7 @@ under the License. ## Requirement -Go1.14+. +Go1.17+. ## Logging diff --git a/run.sh b/run.sh index d89ac67155..e3a3b4af42 100755 --- a/run.sh +++ b/run.sh @@ -439,6 +439,7 @@ function run_test() mkdir -p $REPORT_DIR fi + # Run all tests if none specified. if [ "$test_modules" == "" ]; then test_modules=$(IFS=,; echo "${all_tests[*]}") fi @@ -446,7 +447,7 @@ function run_test() for module in `echo $test_modules | sed 's/,/ /g'`; do echo "====================== run $module ==========================" - # restart onebox when test pegasus + # The tests which need start onebox. local need_onebox_tests=( backup_restore_test base_api_test @@ -459,11 +460,14 @@ function run_test() restore_test throttle_test ) + # Restart onebox if needed. if [[ "${need_onebox_tests[@]}" =~ "${module}" ]]; then + # Clean up onebox at first. run_clear_onebox - m_count=3 + master_count=3 + # Update options if needed, this should be done before starting onebox to make new options take effect. if [ "${module}" == "recovery_test" ]; then - m_count=1 + master_count=1 opts="meta_state_service_type=meta_state_service_simple,distributed_lock_service_type=distributed_lock_service_simple" fi if [ "${module}" == "backup_restore_test" ]; then @@ -472,24 +476,30 @@ function run_test() if [ "${module}" == "restore_test" ]; then opts="cold_backup_disabled=false,cold_backup_checkpoint_reserve_minutes=0,cold_backup_root=onebox" fi + # Append onebox_opts if needed. [ -z ${onebox_opts} ] || opts="${opts},${onebox_opts}" - if ! run_start_onebox -m ${m_count} -w -c --opts ${opts}; then + # Start onebox. + if ! run_start_onebox -m ${master_count} -w -c --opts ${opts}; then echo "ERROR: unable to continue on testing because starting onebox failed" exit 1 fi # TODO(yingchun): remove it? sed -i "s/@LOCAL_HOSTNAME@/${LOCAL_HOSTNAME}/g" ${BUILD_LATEST_DIR}/src/server/test/config.ini else + # Restart ZK in what ever case. run_stop_zk run_start_zk fi + + # Run server test. pushd ${BUILD_LATEST_DIR}/bin/${module} REPORT_DIR=${REPORT_DIR} TEST_BIN=${module} TEST_OPTS=${test_opts} ./run.sh if [ $? != 0 ]; then echo "run test \"$module\" in `pwd` failed" exit 1 fi - # clear onebox if needed + + # Clear onebox if needed. if [[ "${need_onebox_tests[@]}" =~ "${test_modules}" ]]; then if [ "$clear_flags" == "1" ]; then run_clear_onebox @@ -503,6 +513,7 @@ function run_test() used_time=$((finish_time-start_time)) echo "Test elapsed time: $((used_time/60))m $((used_time%60))s" + # TODO(yingchun): make sure if gcov can be ran normally. if [ "$enable_gcov" == "yes" ]; then echo "Generating gcov report..." cd $ROOT