Reduce duplicate code

apache · Dec 19, 2023 · 80ce847 · 80ce847
1 parent 2a6cc71
commit 80ce847
Show file tree

Hide file tree

Showing 7 changed files with 54 additions and 252 deletions.
diff --git a/.github/workflows/lint_and_test_admin-cli.yml b/.github/workflows/lint_and_test_admin-cli.yml
@@ -44,11 +44,11 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.17
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.29
+          version: v1.55.2
           working-directory: ./admin-cli
 
   build:
@@ -60,7 +60,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.17
       - name: Compile
         working-directory: ./admin-cli
         run: make

diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
@@ -66,36 +66,16 @@ jobs:
     name: IWYU
     needs: cpp_clang_format_linter
     runs-on: ubuntu-latest
+    env:
+      USE_JEMALLOC: OFF
     container:
       image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }}
     steps:
       - uses: actions/checkout@v3
       - name: Free Disk Space (Ubuntu)
         run: |
           .github/workflows/free_disk_space.sh
-      - uses: dorny/paths-filter@v2
-        id: changes
-        with:
-          filters: |
-            thirdparty:
-              - '.github/workflows/thirdparty-regular-push.yml'
-              - 'docker/thirdparties-src/**'
-              - 'docker/thirdparties-bin/**'
-              - 'thirdparty/**'
-      - name: Unpack prebuilt third-parties
-        if: steps.changes.outputs.thirdparty == 'false'
-        run: unzip /root/thirdparties-bin.zip -d ./thirdparty
-      - name: Rebuild third-parties
-        if: steps.changes.outputs.thirdparty == 'true'
-        working-directory: thirdparty
-        # Build thirdparties and leave some necessary libraries and source
-        run: |
-          mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
-          cmake --build build/ -j $(nproc)
-          rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
-          ../scripts/download_hadoop.sh hadoop-bin
-          ../scripts/download_zk.sh zookeeper-bin
+      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
       - name: Build IWYU binary
         run: |
           mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git
@@ -113,6 +93,9 @@ jobs:
     name: Build Release
     needs: cpp_clang_format_linter
     runs-on: ubuntu-latest
+    env:
+      USE_JEMALLOC: OFF
+      ARTIFACT_NAME: release
     container:
       image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
     steps:
@@ -126,39 +109,7 @@ jobs:
       - name: Free Disk Space (Ubuntu)
         run: |
           .github/workflows/free_disk_space.sh
-      - uses: dorny/paths-filter@v2
-        id: changes
-        with:
-          filters: |
-            thirdparty:
-              - '.github/workflows/thirdparty-regular-push.yml'
-              - 'docker/thirdparties-src/**'
-              - 'docker/thirdparties-bin/**'
-              - 'thirdparty/**'
-      - name: Unpack prebuilt third-parties
-        if: steps.changes.outputs.thirdparty == 'false'
-        run: |
-          rm -f /root/thirdparties-src.zip
-          unzip /root/thirdparties-bin.zip -d ./thirdparty
-          rm -f /root/thirdparties-bin.zip
-          find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-          rm -rf ./thirdparty/hadoop-bin/share/doc
-          rm -rf ./thirdparty/zookeeper-bin/docs
-      - name: Rebuild third-parties
-        if: steps.changes.outputs.thirdparty == 'true'
-        working-directory: thirdparty
-        # Build thirdparties and leave some necessary libraries and source
-        run: |
-          rm -f /root/thirdparties-src.zip
-          mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
-          cmake --build build/ -j $(nproc)
-          rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
-          find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-          ../scripts/download_hadoop.sh hadoop-bin
-          ../scripts/download_zk.sh zookeeper-bin
-          rm -rf hadoop-bin/share/doc
-          rm -rf zookeeper-bin/docs
+      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
       - name: Compilation
         run: |
           ccache -p
@@ -176,17 +127,7 @@ jobs:
         run: |
           ./run.sh pack_tools
           rm -rf pegasus-tools-*
-      - name: Tar files
-        run: |
-          mv thirdparty/hadoop-bin ./
-          mv thirdparty/zookeeper-bin ./
-          rm -rf thirdparty
-          tar -zcvhf release__builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: release_artifact_${{ github.sha }}
-          path: release__builder.tar
+      - uses: "./.github/actions/upload_artifact/action.yaml"
 
   test_Release:
     name: Test Release
@@ -256,14 +197,17 @@ jobs:
           rm -f release__builder.tar
       - name: Unit Testing
         run: |
-          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
+          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server
           ulimit -s unlimited
           ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
   build_ASAN:
     name: Build ASAN
     needs: cpp_clang_format_linter
     runs-on: ubuntu-latest
+    env:
+      USE_JEMALLOC: OFF
+      ARTIFACT_NAME: release_address
     container:
       image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
     steps:
@@ -277,39 +221,7 @@ jobs:
       - name: Free Disk Space (Ubuntu)
         run: |
           .github/workflows/free_disk_space.sh
-      - uses: dorny/paths-filter@v2
-        id: changes
-        with:
-          filters: |
-            thirdparty:
-              - '.github/workflows/thirdparty-regular-push.yml'
-              - 'docker/thirdparties-src/**'
-              - 'docker/thirdparties-bin/**'
-              - 'thirdparty/**'
-      - name: Unpack prebuilt third-parties
-        if: steps.changes.outputs.thirdparty == 'false'
-        run: |
-          rm -f /root/thirdparties-src.zip
-          unzip /root/thirdparties-bin.zip -d ./thirdparty
-          rm -f /root/thirdparties-bin.zip
-          find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-          rm -rf ./thirdparty/hadoop-bin/share/doc
-          rm -rf ./thirdparty/zookeeper-bin/docs
-      - name: Rebuild third-parties
-        if: steps.changes.outputs.thirdparty == 'true'
-        working-directory: thirdparty
-        # Build thirdparties and leave some necessary libraries and source
-        run: |
-          rm -f /root/thirdparties-src.zip
-          mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
-          cmake --build build/ -j $(nproc)
-          rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
-          find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-          ../scripts/download_hadoop.sh hadoop-bin
-          ../scripts/download_zk.sh zookeeper-bin
-          rm -rf hadoop-bin/share/doc
-          rm -rf zookeeper-bin/docs
+      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
       - name: Compilation
         run: |
           ccache -p
@@ -319,17 +231,7 @@ jobs:
       - name: Clear Build Files
         run: |
           find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-      - name: Tar files
-        run: |
-          mv thirdparty/hadoop-bin ./
-          mv thirdparty/zookeeper-bin ./
-          rm -rf thirdparty
-          tar -zcvhf release_address_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: release_address_artifact_${{ github.sha }}
-          path: release_address_builder.tar
+      - uses: "./.github/actions/upload_artifact/action.yaml"
 
   test_ASAN:
     name: Test ASAN
@@ -401,7 +303,7 @@ jobs:
           rm -f release_address_builder.tar
       - name: Unit Testing
         run: |
-          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
+          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server
           ulimit -s unlimited
           ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
@@ -412,6 +314,9 @@ jobs:
 #    name: Build UBSAN
 #    needs: cpp_clang_format_linter
 #    runs-on: ubuntu-latest
+#    env:
+#      USE_JEMALLOC: OFF
+#      ARTIFACT_NAME: release_undefined
 #    container:
 #      image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
 #    steps:
@@ -425,39 +330,7 @@ jobs:
 #      - name: Free Disk Space (Ubuntu)
 #        run: |
 #          .github/workflows/free_disk_space.sh
-#      - uses: dorny/paths-filter@v2
-#        id: changes
-#        with:
-#          filters: |
-#            thirdparty:
-#              - '.github/workflows/thirdparty-regular-push.yml'
-#              - 'docker/thirdparties-src/**'
-#              - 'docker/thirdparties-bin/**'
-#              - 'thirdparty/**'
-#      - name: Unpack prebuilt third-parties
-#        if: steps.changes.outputs.thirdparty == 'false'
-#        run: |
-#          rm -f /root/thirdparties-src.zip
-#          unzip /root/thirdparties-bin.zip -d ./thirdparty
-#          rm -f /root/thirdparties-bin.zip
-#          find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-#          rm -rf ./thirdparty/hadoop-bin/share/doc
-#          rm -rf ./thirdparty/zookeeper-bin/docs
-#      - name: Rebuild third-parties
-#        if: steps.changes.outputs.thirdparty == 'true'
-#        working-directory: thirdparty
-#        # Build thirdparties and leave some necessary libraries and source
-#        run: |
-#          rm -f /root/thirdparties-src.zip
-#          mkdir build
-#          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
-#          cmake --build build/ -j $(nproc)
-#          rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
-#          find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-#          ../scripts/download_hadoop.sh hadoop-bin
-#          ../scripts/download_zk.sh zookeeper-bin
-#          rm -rf hadoop-bin/share/doc
-#          rm -rf zookeeper-bin/docs
+#      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
 #      - name: Compilation
 #        run: |
 #          ccache -p
@@ -467,17 +340,7 @@ jobs:
 #      - name: Clear Build Files
 #        run: |
 #          find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-#      - name: Tar files
-#        run: |
-#          mv thirdparty/hadoop-bin ./
-#          mv thirdparty/zookeeper-bin ./
-#          rm -rf thirdparty
-#          tar -zcvhf release_undefined_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin
-#      - name: Upload Artifact
-#        uses: actions/upload-artifact@v3
-#        with:
-#          name: release_undefined_artifact_${{ github.sha }}
-#          path: release_undefined_builder.tar
+#      - uses: "./.github/actions/upload_artifact/action.yaml"
 #
 #  test_UBSAN:
 #    name: Test UBSAN
@@ -548,14 +411,17 @@ jobs:
 #          rm -f release_undefined_builder.tar
 #      - name: Unit Testing
 #        run: |
-#          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
+#          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server
 #          ulimit -s unlimited
 #          ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 
   build_with_jemalloc:
     name: Build with jemalloc
     needs: cpp_clang_format_linter
     runs-on: ubuntu-latest
+    env:
+      USE_JEMALLOC: ON
+      ARTIFACT_NAME: release_jemalloc
     container:
       image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }}
     steps:
@@ -569,39 +435,7 @@ jobs:
       - name: Free Disk Space (Ubuntu)
         run: |
           .github/workflows/free_disk_space.sh
-      - uses: dorny/paths-filter@v2
-        id: changes
-        with:
-          filters: |
-            thirdparty:
-              - '.github/workflows/thirdparty-regular-push.yml'
-              - 'docker/thirdparties-src/**'
-              - 'docker/thirdparties-bin/**'
-              - 'thirdparty/**'
-      - name: Unpack prebuilt third-parties
-        if: steps.changes.outputs.thirdparty == 'false'
-        run: |
-          rm -f /root/thirdparties-src.zip
-          unzip /root/thirdparties-bin.zip -d ./thirdparty
-          rm -f /root/thirdparties-bin.zip
-          find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-          rm -rf ./thirdparty/hadoop-bin/share/doc
-          rm -rf ./thirdparty/zookeeper-bin/docs
-      - name: Rebuild third-parties
-        if: steps.changes.outputs.thirdparty == 'true'
-        working-directory: thirdparty
-        # Build thirdparties and leave some necessary libraries and source
-        run: |
-          rm -f /root/thirdparties-src.zip
-          mkdir build
-          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=ON -B build/
-          cmake --build build/ -j $(nproc)
-          rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
-          find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-          ../scripts/download_hadoop.sh hadoop-bin
-          ../scripts/download_zk.sh zookeeper-bin
-          rm -rf hadoop-bin/share/doc
-          rm -rf zookeeper-bin/docs
+      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
       - name: Compilation
         run: |
           ccache -p
@@ -619,17 +453,7 @@ jobs:
         run: |
           ./run.sh pack_tools -j
           rm -rf pegasus-tools-*
-      - name: Tar files
-        run: |
-          mv thirdparty/hadoop-bin ./
-          mv thirdparty/zookeeper-bin ./
-          rm -rf thirdparty
-          tar -zcvhf release_jemalloc_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: release_jemalloc_artifact_${{ github.sha }}
-          path: release_jemalloc_builder.tar
+      - uses: "./.github/actions/upload_artifact/action.yaml"
 
   test_with_jemalloc:
     name: Test with jemallc
@@ -660,7 +484,7 @@ jobs:
           rm -f release_jemalloc_builder.tar
       - name: Unit Testing
         run: |
-          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
+          export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server
           ulimit -s unlimited
           ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}
 

diff --git a/.github/workflows/lint_and_test_go-client.yml b/.github/workflows/lint_and_test_go-client.yml
@@ -49,7 +49,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.14
+        go-version: 1.17
     - name: Run Test
       run: ./test.sh
       working-directory: ./go-client
@@ -60,5 +60,5 @@ jobs:
     - name: Lint
       uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.29
+        version: v1.55.2
         working-directory: ./go-client