[BACKPORT 2.20] Update OpenSSL to 3.0.8 (#255)

Context: yugabyte/yugabyte-db#16407
Original PR: #239, #248

.circleci/config.yml

@@ -46,15 +46,15 @@ workflows:
 
       - build:
           archive_name_suffix: centos7-aarch64-clang15
-          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2022-10-13T18_12_26
+          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55
           build_thirdparty_args: >-
             --toolchain=llvm15
             --expected-major-compiler-version=15
             --skip-sanitizers
 
       - build:
           archive_name_suffix: centos7-aarch64-clang15-full-lto
-          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2022-10-13T18_12_26
+          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55
           build_thirdparty_args: >-
             --toolchain=llvm15
             --expected-major-compiler-version=15
@@ -63,15 +63,15 @@ workflows:
 
       - build:
           archive_name_suffix: centos7-aarch64-clang16
-          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2022-10-13T18_12_26
+          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55
           build_thirdparty_args: >-
             --toolchain=llvm16
             --expected-major-compiler-version=16
             --skip-sanitizers
 
       - build:
           archive_name_suffix: centos7-aarch64-clang16-full-lto
-          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2022-10-13T18_12_26
+          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55
           build_thirdparty_args: >-
             --toolchain=llvm16
             --expected-major-compiler-version=16

.github/workflows/ci.yml

@@ -51,22 +51,22 @@ jobs:
           # ---------------------------------------------------------------------------------------
           - name: centos7-x86_64-gcc11
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --devtoolset=11
               --expected-major-compiler-version=11
 
           # Clang 15
           - name: centos7-x86_64-clang15
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
 
           - name: centos7-x86_64-clang15-full-lto
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
@@ -75,14 +75,14 @@ jobs:
           # Clang 16
           - name: centos7-x86_64-clang16
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --toolchain=llvm16
               --expected-major-compiler-version=16
 
           - name: centos7-x86_64-clang16-full-lto
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --toolchain=llvm16
               --expected-major-compiler-version=16
@@ -93,7 +93,7 @@ jobs:
           # ---------------------------------------------------------------------------------------
           - name: ubuntu2004-x86_64-clang15
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_ubuntu2004_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_ubuntu2004_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
@@ -104,7 +104,7 @@ jobs:
 
           - name: ubuntu2204-x86_64-gcc11
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2022-10-13T18_10_50
+            docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2023-10-25T03_33_00
             build_thirdparty_args: >-
               --compiler-prefix=/usr
               --compiler-family=gcc
@@ -113,7 +113,7 @@ jobs:
 
           - name: ubuntu2204-x86_64-clang15
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2022-10-13T18_10_50
+            docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2023-10-25T03_33_00
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
@@ -123,29 +123,29 @@ jobs:
           # ---------------------------------------------------------------------------------------
           - name: almalinux8-x86_64-gcc11
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --devtoolset=11
               --expected-major-compiler-version=11
 
           # Clang/LLVM 15
           - name: almalinux8-x86_64-clang15
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
 
           - name: almalinux8-x86_64-clang15-linuxbrew
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm15_linuxbrew
               --expected-major-compiler-version=15
 
           - name: almalinux8-x86_64-clang15-linuxbrew-full-lto
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm15_linuxbrew
               --expected-major-compiler-version=15
@@ -154,21 +154,21 @@ jobs:
           # Clang/LLVM 16
           - name: almalinux8-x86_64-clang16
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm16
               --expected-major-compiler-version=16
 
           - name: almalinux8-x86_64-clang16-linuxbrew
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm16_linuxbrew
               --expected-major-compiler-version=16
 
           - name: almalinux8-x86_64-clang16-linuxbrew-full-lto
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm16_linuxbrew
               --expected-major-compiler-version=16
@@ -181,14 +181,14 @@ jobs:
           # Clang/LLVM 16
           - name: almalinux9-x86_64-clang16
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux9_x86_64:v2023-04-07T00_19_48
+            docker_image: yugabyteci/yb_build_infra_almalinux9_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --toolchain=llvm16
               --expected-major-compiler-version=16
 
           - name: almalinux9-x86_64-gcc12
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux9_x86_64:v2023-04-07T00_19_48
+            docker_image: yugabyteci/yb_build_infra_almalinux9_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --devtoolset=12
               --expected-major-compiler-version=12

.github/workflows/linux_build.sh

@@ -8,6 +8,7 @@ export YB_SKIP_UPLOAD=${SKIP_UPLOAD:-}
 echo "Building in directory: $checkout_dir"
 docker run -t \
   --cap-add=SYS_PTRACE \
+  -u root \
   -e GITHUB_TOKEN \
   -e SNYK_TOKEN \
   -e YB_BUILD_THIRDPARTY_ARGS \

.github/workflows/macos_build.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-brew install autoconf automake pkg-config shellcheck
+brew install autoconf automake pkg-config shellcheck hub
 dirs=( /opt/yb-build/{thirdparty,brew,tmp} )
 sudo mkdir -p "${dirs[@]}"
 sudo chmod 777 "${dirs[@]}"

patches/openssl-fix-afalg-link-on-centos7.patch

@@ -0,0 +1,27 @@
+diff --git a/engines/e_afalg.c b/engines/e_afalg.c
+index 2c08cbb..f362d94 100644
+--- a/engines/e_afalg.c
++++ b/engines/e_afalg.c
+@@ -34,10 +34,22 @@
+ #  warning "AFALG ENGINE requires Kernel Headers >= 4.1.0"
+ #  warning "Skipping Compilation of AFALG engine"
+ # endif
++# ifndef OPENSSL_NO_DYNAMIC_ENGINE
++OPENSSL_EXPORT
++    int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns);
++OPENSSL_EXPORT
++    int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns)
++{
++    return 0;
++}
++
++IMPLEMENT_DYNAMIC_CHECK_FN()
++# else
+ void engine_load_afalg_int(void);
+ void engine_load_afalg_int(void)
+ {
+ }
++# endif
+ #else
+
+ # include <linux/if_alg.h>

python/build_definitions/cassandra_cpp_driver.py

@@ -22,7 +22,7 @@
 class CassandraCppDriverDependency(Dependency):
     def __init__(self) -> None:
         super(CassandraCppDriverDependency, self).__init__(
-                'cassandra-cpp-driver', '2.9.0-yb-13',
+                'cassandra-cpp-driver', '2.9.0-yb-14',
                 'https://github.com/yugabyte/cassandra-cpp-driver/archive/{0}.tar.gz',
                 BuildGroup.POTENTIALLY_INSTRUMENTED)
         self.copy_sources = False

python/build_definitions/openssl.py

@@ -34,10 +34,16 @@ class OpenSSLDependency(Dependency):
     def __init__(self) -> None:
         super(OpenSSLDependency, self).__init__(
             name='openssl',
-            version='1.1.1t',
+            version='3.0.8',
             url_pattern='https://www.openssl.org/source/openssl-{0}.tar.gz',
             build_group=BuildGroup.COMMON)
         self.copy_sources = True
+        # Patch fixes the following error on kernel versions < 4.1.0:
+        # ld.lld: error: version script assignment of 'global' to symbol 'bind_engine' failed:
+        # symbol not defined
+        # ld.lld: error: version script assignment of 'global' to symbol 'v_check' failed:
+        # symbol not defined
+        self.patches = ['openssl-fix-afalg-link-on-centos7.patch']
 
     def build(self, builder: BuilderInterface) -> None:
         common_configure_options = ['shared', 'no-tests']

python/yugabyte_db_thirdparty/util.py

@@ -367,7 +367,7 @@ def capture_all_output(
         cmd_line_str = shlex_join(args)
         if ex.returncode not in allowed_exit_codes:
             error_msg = f"Unexpected exit code {ex.returncode} from: {cmd_line_str} " \
-                        f"(expected one of { set(sorted(allowed_exit_codes | {0})) })"
+                        f"(expected one of {set(sorted(allowed_exit_codes | {0}))})"
             log(error_msg)
             log("Output from %s (stdout/stderr combined):", cmd_line_str)
             log(ex.stdout.decode('utf-8'))

requirements_frozen.txt

@@ -10,13 +10,13 @@ llvm-installer==1.3.6
 mypy==1.3.0
 mypy-extensions==1.0.0
 packaging==23.1
-pycodestyle==2.10.0
+pycodestyle==2.11.0
 requests==2.30.0
 ruamel.yaml==0.17.26
-ruamel.yaml.clib==0.2.7
+ruamel.yaml.clib==0.2.8
 sys-detection==1.3.0
 tomli==2.0.1
-typing-extensions==4.5.0
+typing-extensions==4.8.0
 urllib3==2.0.2
 websocket-client==1.5.1
 yugabyte-pycommon==1.9.15

thirdparty_src_checksums.txt

@@ -8,6 +8,7 @@ fc9f85fc030e233142908241af7a846e60630aa7388de9a5fafb1f3a26840854  boost-1.77.0.t
 71feeed900fbccca04a3b4f2f84a7c217186f28a940ed8b7ed4725986baf99fa  boost-1.81.0.tar.bz2
 7bc4b0257d4c15676a158dd30b665d85c1c6a590548e9f6288e2f1542ca6e05a  cassandra-cpp-driver-2.9.0-yb-12.tar.gz
 48dbb1e29028c2caeed6e6c6ec45a5deb0c18955f0292899e1994dd89defc3db  cassandra-cpp-driver-2.9.0-yb-13.tar.gz
+1e42551a7bf986be92d937a18be7208d38c5420550813b170969b08f1cb79058  cassandra-cpp-driver-2.9.0-yb-14.tar.gz
 e97dc472aae52197a4d5e0185eb8f9e04d7575d2dc2b12194ddc768e0f8a846d  cfe-7.1.0.tar.xz
 1ce0042c48ecea839ce67b87e9739cf18e7a5c2b3b9a36d177d00979609b6451  clang-tools-extra-7.1.0.tar.xz
 057bdac0581215b5ceb39edfd5bbef9eb79578f16a8908349f3066251fba88d8  compiler-rt-7.1.0.tar.xz
@@ -100,6 +101,7 @@ c48450d27524c2e5856997133e059e3cf9909241110a6e21ad278890ac425afc  lz4-r130.tar.g
 40dceb51a4f6a5275bde0e6bf20ef4b91bfc32ed57c0552e2e8e15463372b17a  openssl-1.1.1n.tar.gz
 9384a2b0570dd80358841464677115df785edb941c71211f75076d72fe6b438f  openssl-1.1.1o.tar.gz
 8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b  openssl-1.1.1t.tar.gz
+6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e  openssl-3.0.8.tar.gz
 0fdbefbdc2c154634728097e26de52a8210ed95cb032beb5f35da0a493cd5066  opentelemetry-cpp-1.9.0.tar.gz
 464bc2b348e674a1a03142e403cbccb01be8655b6de0f8bfe733ea31fcd421be  opentelemetry-proto-0.19.0.tar.gz
 05e28e5141c1962b1c9d8793cc9cfee8cd11bc24cea13fb9689ac3fc0a379bd3  protobuf-3.5.1-yb-1.tar.gz