From fa66f9704082404c79deab0e1ee82af4f4e63300 Mon Sep 17 00:00:00 2001 From: Steve Varnau Date: Mon, 11 Nov 2024 08:10:12 -0800 Subject: [PATCH] [DEVOPS-3278] Move thirdparty builds off of amzn2 (#294) While amzn2 does not go EOL until mid-2025, it is no longer supported on YBDB releases beyond 2.20. YBDB builds on master branch build on later OS, so are not compatible with older glibc. Thirdparty dependency builds on amzn2 are still supported on the 2.20 branch of this repo. Also fix python build issue on macos. --- .github/workflows/ci.yml | 86 ++++++++++++++------------------ .github/workflows/macos_build.sh | 2 + build_and_release.sh | 2 +- build_thirdparty.sh | 3 +- yb-thirdparty-common.sh | 2 +- 5 files changed, 44 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03b02bbf..6bdc3c46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,45 +96,6 @@ jobs: --toolchain=llvm17 architecture: x86_64 - # --------------------------------------------------------------------------------------- - # Amazon Linux 2 -- the next oldest glibc after CentOS 7's EOL - # --------------------------------------------------------------------------------------- - - # Clang 17 - - name: amzn2-x86_64-clang17 - runs_on: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. - docker_image: yugabyteci/yb_build_infra_amazonlinux2_x86_64:v2024-09-13T18_52_53 - build_thirdparty_args: >- - --toolchain=llvm17 - architecture: x86_64 - - - name: amzn2-x86_64-clang17-full-lto - runs_on: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. - docker_image: yugabyteci/yb_build_infra_amazonlinux2_x86_64:v2024-09-13T18_52_53 - build_thirdparty_args: >- - --toolchain=llvm17 - --lto=full - architecture: x86_64 - - # Clang 18 - # TODO: enable ASAN/TSAN for Clang 18. - - name: amzn2-x86_64-clang18 - runs_on: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. - docker_image: yugabyteci/yb_build_infra_amazonlinux2_x86_64:v2024-09-13T18_52_53 - build_thirdparty_args: >- - --toolchain=llvm18 - --skip-sanitizers - architecture: x86_64 - - - name: amzn2-x86_64-clang18-full-lto - runs_on: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. - docker_image: yugabyteci/yb_build_infra_amazonlinux2_x86_64:v2024-09-13T18_52_53 - build_thirdparty_args: >- - --toolchain=llvm18 - --lto=full - --skip-sanitizers - architecture: x86_64 - # --------------------------------------------------------------------------------------- # AlmaLinux 8 # --------------------------------------------------------------------------------------- @@ -168,8 +129,17 @@ jobs: --toolchain=llvm17 architecture: x86_64 + - name: almalinux8-x86_64-clang17-full-lto + runs_on: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2024-09-20T20_33_55 + build_thirdparty_args: >- + --toolchain=llvm17 + --lto=full + architecture: x86_64 + # Clang/LLVM 18 # TODO: enable ASAN/TSAN for Clang 18. + # Need to debug build failures without the --skip-sanitizers flag. - name: almalinux8-x86_64-clang18 runs_on: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2024-09-20T20_33_55 @@ -178,6 +148,15 @@ jobs: --skip-sanitizers architecture: x86_64 + - name: almalinux8-x86_64-clang18-full-lto + runs_on: ubuntu-20.04 # Ubuntu 20.04 is for the top-level VM only. We use Docker in it. + docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2024-09-20T20_33_55 + build_thirdparty_args: >- + --toolchain=llvm18 + --lto=full + --skip-sanitizers + architecture: x86_64 + # --------------------------------------------------------------------------------------- # AlmaLinux 9 # --------------------------------------------------------------------------------------- @@ -222,34 +201,34 @@ jobs: architecture: arm64 # --------------------------------------------------------------------------------------- - # aarch64 builds (Amazon Linux 2) + # aarch64 - almalinux8 builds # --------------------------------------------------------------------------------------- - - name: amzn2-aarch64-clang17 + - name: almalinux8-aarch64-clang17 runs_on: ubuntu-24.04-aarch64-4core-16gb - docker_image: yugabyteci/yb_build_infra_amazonlinux2_aarch64:v2024-09-13T18_54_13 + docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06 build_thirdparty_args: >- --toolchain=llvm17 --skip-sanitizers - - name: amzn2-aarch64-clang17-full-lto + - name: almalinux8-aarch64-clang17-full-lto runs_on: ubuntu-24.04-aarch64-4core-16gb - docker_image: yugabyteci/yb_build_infra_amazonlinux2_aarch64:v2024-09-13T18_54_13 + docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06 build_thirdparty_args: >- --toolchain=llvm17 --skip-sanitizers --lto=full - - name: amzn2-aarch64-clang18 + - name: almalinux8-aarch64-clang18 runs_on: ubuntu-24.04-aarch64-4core-16gb - docker_image: yugabyteci/yb_build_infra_amazonlinux2_aarch64:v2024-09-13T18_54_13 + docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06 build_thirdparty_args: >- --toolchain=llvm18 --skip-sanitizers - - name: amzn2-aarch64-clang18-full-lto + - name: almalinux8-aarch64-clang18-full-lto runs_on: ubuntu-24.04-aarch64-4core-16gb - docker_image: yugabyteci/yb_build_infra_amazonlinux2_aarch64:v2024-09-13T18_54_13 + docker_image: yugabyteci/yb_build_infra_almalinux8_aarch64:v2024-09-20T23_59_06 build_thirdparty_args: >- --toolchain=llvm18 --skip-sanitizers @@ -270,6 +249,17 @@ jobs: # in test code. fetch-depth: 0 + # Python version really only matters in the non-docker cases (macos), but it should match + # the version used in the docker images, for a common requirements_frozen.txt. + # The linux images (from build-infra repo) should be updated to newer python and this version + # updated to match. + # This action puts python and python3.9 in path, but python3 is still latest version. + # See setting in .github/workflows/macos_build.sh + - name: PythonVer + uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Build run: .github/workflows/github_actions_build.sh env: diff --git a/.github/workflows/macos_build.sh b/.github/workflows/macos_build.sh index 20bdd719..a270c7cb 100755 --- a/.github/workflows/macos_build.sh +++ b/.github/workflows/macos_build.sh @@ -6,4 +6,6 @@ brew install autoconf automake pkg-config shellcheck hub dirs=( /opt/yb-build/{thirdparty,brew,tmp} ) sudo mkdir -p "${dirs[@]}" sudo chmod 777 "${dirs[@]}" + +export PYTHON="python3.9" ./build_and_release.sh diff --git a/build_and_release.sh b/build_and_release.sh index 712e7e92..d3584f65 100755 --- a/build_and_release.sh +++ b/build_and_release.sh @@ -158,7 +158,7 @@ tools_to_show_versions=( autoreconf cmake pkg-config - python3 + "${PYTHON:-python3}" ) if [[ $OSTYPE == darwin* ]]; then diff --git a/build_thirdparty.sh b/build_thirdparty.sh index c48bb19f..c30d74c1 100755 --- a/build_thirdparty.sh +++ b/build_thirdparty.sh @@ -67,7 +67,8 @@ echo echo "Logging to ${log_path} (linked to ${link_path_list_str})" echo -cmd=( python3 "${YB_THIRDPARTY_DIR}/python/yugabyte_db_thirdparty/yb_build_thirdparty_main.py" ) +cmd=( "${PYTHON:-python3}" + "${YB_THIRDPARTY_DIR}/python/yugabyte_db_thirdparty/yb_build_thirdparty_main.py" ) if [[ ${#build_thirdparty_args[@]} -gt 0 ]]; then cmd+=( "${build_thirdparty_args[@]}" ) fi diff --git a/yb-thirdparty-common.sh b/yb-thirdparty-common.sh index 24743183..7520a145 100644 --- a/yb-thirdparty-common.sh +++ b/yb-thirdparty-common.sh @@ -43,7 +43,7 @@ compute_sha256sum() { activate_virtualenv() { if [[ ! -d $YB_THIRDPARTY_DIR/venv ]]; then - python3 -m venv "$YB_THIRDPARTY_DIR/venv" + "${PYTHON:-python3}" -m venv "$YB_THIRDPARTY_DIR/venv" fi set +u # shellcheck disable=SC1090