[DEVOPS-3278] Move thirdparty builds off of amzn2 (#294)

While amzn2 does not go EOL until mid-2025, it is no longer supported on YBDB releases beyond 2.20. YBDB builds on master branch build on later OS, so are not compatible with older glibc. Thirdparty dependency builds on amzn2 are still supported on the 2.20 branch of this repo. Also fix python build issue on macos.
yugabyte · Nov 11, 2024 · fa66f97 · fa66f97
1 parent 5815946
commit fa66f97
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 51 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -96,45 +96,6 @@ jobs:
               --toolchain=llvm17
             architecture: x86_64
 
-          # ---------------------------------------------------------------------------------------
-          # Amazon Linux 2 -- the next oldest glibc after CentOS 7's EOL
-          # ---------------------------------------------------------------------------------------
-
-          # Clang 17
-          - name: amzn2-x86_64-clang17
-            runs_on: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_amazonlinux2_x86_64:v2024-09-13T18_52_53
-            build_thirdparty_args: >-
-              --toolchain=llvm17
-            architecture: x86_64
-
-          - name: amzn2-x86_64-clang17-full-lto
-            runs_on: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_amazonlinux2_x86_64:v2024-09-13T18_52_53
-            build_thirdparty_args: >-
-              --toolchain=llvm17
-              --lto=full
-            architecture: x86_64
-
-          # Clang 18
-          # TODO: enable ASAN/TSAN for Clang 18.
-          - name: amzn2-x86_64-clang18
-            runs_on: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_amazonlinux2_x86_64:v2024-09-13T18_52_53
-            build_thirdparty_args: >-
-              --toolchain=llvm18
-              --skip-sanitizers
-            architecture: x86_64
-
-          - name: amzn2-x86_64-clang18-full-lto
-            runs_on: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_amazonlinux2_x86_64:v2024-09-13T18_52_53
-            build_thirdparty_args: >-
-              --toolchain=llvm18
-              --lto=full
-              --skip-sanitizers
-            architecture: x86_64
-
           # ---------------------------------------------------------------------------------------
           # AlmaLinux 8
           # ---------------------------------------------------------------------------------------
@@ -168,8 +129,17 @@ jobs:
               --toolchain=llvm17
             architecture: x86_64
 
+          - name: almalinux8-x86_64-clang17-full-lto
+            runs_on: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2024-09-20T20_33_55
+            build_thirdparty_args: >-
+              --toolchain=llvm17
+              --lto=full
+            architecture: x86_64
+
           # Clang/LLVM 18
           # TODO: enable ASAN/TSAN for Clang 18.
+          # Need to debug build failures without the --skip-sanitizers flag.
           - name: almalinux8-x86_64-clang18
             runs_on: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
             docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2024-09-20T20_33_55
@@ -178,6 +148,15 @@ jobs:
               --skip-sanitizers
             architecture: x86_64
 
+          - name: almalinux8-x86_64-clang18-full-lto
+            runs_on: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2024-09-20T20_33_55
+            build_thirdparty_args: >-
+              --toolchain=llvm18
+              --lto=full
+              --skip-sanitizers
+            architecture: x86_64
+
           # ---------------------------------------------------------------------------------------
           # AlmaLinux 9
           # ---------------------------------------------------------------------------------------
@@ -222,34 +201,34 @@ jobs:
             architecture: arm64
 
           # ---------------------------------------------------------------------------------------
-          # aarch64 builds (Amazon Linux 2)
+          # aarch64 - almalinux8 builds
           # ---------------------------------------------------------------------------------------
 
-          - name: amzn2-aarch64-clang17
+          - name: almalinux8-aarch64-clang17
             runs_on: ubuntu-24.04-aarch64-4core-16gb
-            docker_image: yugabyteci/yb_build_infra_amazonlinux2_aarch64:v2024-09-13T18_54_13
+            docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06
             build_thirdparty_args: >-
               --toolchain=llvm17
               --skip-sanitizers
 
-          - name: amzn2-aarch64-clang17-full-lto
+          - name: almalinux8-aarch64-clang17-full-lto
             runs_on: ubuntu-24.04-aarch64-4core-16gb
-            docker_image: yugabyteci/yb_build_infra_amazonlinux2_aarch64:v2024-09-13T18_54_13
+            docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06
             build_thirdparty_args: >-
               --toolchain=llvm17
               --skip-sanitizers
               --lto=full
 
-          - name: amzn2-aarch64-clang18
+          - name: almalinux8-aarch64-clang18
             runs_on: ubuntu-24.04-aarch64-4core-16gb
-            docker_image: yugabyteci/yb_build_infra_amazonlinux2_aarch64:v2024-09-13T18_54_13
+            docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06
             build_thirdparty_args: >-
               --toolchain=llvm18
               --skip-sanitizers
 
-          - name: amzn2-aarch64-clang18-full-lto
+          - name: almalinux8-aarch64-clang18-full-lto
             runs_on: ubuntu-24.04-aarch64-4core-16gb
-            docker_image: yugabyteci/yb_build_infra_amazonlinux2_aarch64:v2024-09-13T18_54_13
+            docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06
             build_thirdparty_args: >-
               --toolchain=llvm18
               --skip-sanitizers
@@ -270,6 +249,17 @@ jobs:
           # in test code.
           fetch-depth: 0
 
+      # Python version really only matters in the non-docker cases (macos), but it should match
+      # the version used in the docker images, for a common requirements_frozen.txt.
+      # The linux images (from build-infra repo) should be updated to newer python and this version
+      # updated to match.
+      # This action puts python and python3.9 in path, but python3 is still latest version.
+      # See setting in .github/workflows/macos_build.sh
+      - name: PythonVer
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
       - name: Build
         run: .github/workflows/github_actions_build.sh
         env:

diff --git a/.github/workflows/macos_build.sh b/.github/workflows/macos_build.sh
@@ -6,4 +6,6 @@ brew install autoconf automake pkg-config shellcheck hub
 dirs=( /opt/yb-build/{thirdparty,brew,tmp} )
 sudo mkdir -p "${dirs[@]}"
 sudo chmod 777 "${dirs[@]}"
+
+export PYTHON="python3.9"
 ./build_and_release.sh
diff --git a/build_and_release.sh b/build_and_release.sh
@@ -158,7 +158,7 @@ tools_to_show_versions=(
   autoreconf
   cmake
   pkg-config
-  python3
+  "${PYTHON:-python3}"
 )
 
 if [[ $OSTYPE == darwin* ]]; then

diff --git a/build_thirdparty.sh b/build_thirdparty.sh
@@ -67,7 +67,8 @@ echo
 echo "Logging to ${log_path} (linked to ${link_path_list_str})"
 echo
 
-cmd=( python3 "${YB_THIRDPARTY_DIR}/python/yugabyte_db_thirdparty/yb_build_thirdparty_main.py" )
+cmd=( "${PYTHON:-python3}"
+      "${YB_THIRDPARTY_DIR}/python/yugabyte_db_thirdparty/yb_build_thirdparty_main.py" )
 if [[ ${#build_thirdparty_args[@]} -gt 0 ]]; then
   cmd+=( "${build_thirdparty_args[@]}" )
 fi

diff --git a/yb-thirdparty-common.sh b/yb-thirdparty-common.sh
@@ -43,7 +43,7 @@ compute_sha256sum() {
 
 activate_virtualenv() {
   if [[ ! -d $YB_THIRDPARTY_DIR/venv ]]; then
-    python3 -m venv "$YB_THIRDPARTY_DIR/venv"
+    "${PYTHON:-python3}" -m venv "$YB_THIRDPARTY_DIR/venv"
   fi
   set +u
   # shellcheck disable=SC1090