[BACKPORT 2.18] Update OpenSSL to 3.0.8 (#246)

Context: yugabyte/yugabyte-db#16407 Original PR: #239, #248
yugabyte · Nov 14, 2023 · 1e1134f · 1e1134f
1 parent 13f9ef4
commit 1e1134f
Show file tree

Hide file tree

Showing 10 changed files with 55 additions and 19 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -46,15 +46,15 @@ workflows:
 
       - build:
           archive_name_suffix: centos7-aarch64-clang15
-          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2022-10-13T18_12_26
+          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55
           build_thirdparty_args: >-
             --toolchain=llvm15
             --expected-major-compiler-version=15
             --skip-sanitizers
 
       - build:
           archive_name_suffix: centos7-aarch64-clang15-full-lto
-          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2022-10-13T18_12_26
+          docker_image: yugabyteci/yb_build_infra_centos7_aarch64:v2023-10-25T03_32_55
           build_thirdparty_args: >-
             --toolchain=llvm15
             --expected-major-compiler-version=15

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -45,22 +45,22 @@ jobs:
           # ---------------------------------------------------------------------------------------
           - name: centos7-x86_64-gcc11
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --devtoolset=11
               --expected-major-compiler-version=11
 
           # Clang 15
           - name: centos7-x86_64-clang15
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
 
           - name: centos7-x86_64-clang15-full-lto
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_centos7_x86_64:v2023-10-25T03_33_02
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
@@ -71,7 +71,7 @@ jobs:
           # ---------------------------------------------------------------------------------------
           - name: ubuntu2004-x86_64-clang15
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_ubuntu2004_x86_64:v2022-10-13T18_10_48
+            docker_image: yugabyteci/yb_build_infra_ubuntu2004_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
@@ -82,7 +82,7 @@ jobs:
 
           - name: ubuntu2204-x86_64-gcc11
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2022-10-13T18_10_50
+            docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2023-10-25T03_33_00
             build_thirdparty_args: >-
               --compiler-prefix=/usr
               --compiler-family=gcc
@@ -91,7 +91,7 @@ jobs:
 
           - name: ubuntu2204-x86_64-clang15
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2022-10-13T18_10_50
+            docker_image: yugabyteci/yb_build_infra_ubuntu2204_x86_64:v2023-10-25T03_33_00
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
@@ -101,29 +101,29 @@ jobs:
           # ---------------------------------------------------------------------------------------
           - name: almalinux8-x86_64-gcc11
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --devtoolset=11
               --expected-major-compiler-version=11
 
           # Clang/LLVM 15
           - name: almalinux8-x86_64-clang15
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm15
               --expected-major-compiler-version=15
 
           - name: almalinux8-x86_64-clang15-linuxbrew
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm15_linuxbrew
               --expected-major-compiler-version=15
 
           - name: almalinux8-x86_64-clang15-linuxbrew-full-lto
             os: ubuntu-20.04  # Ubuntu 20.04 is for the top-level VM only. We use Docker in it.
-            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2022-10-13T18_10_49
+            docker_image: yugabyteci/yb_build_infra_almalinux8_x86_64:v2023-10-25T03_33_01
             build_thirdparty_args: >-
               --toolchain=llvm15_linuxbrew
               --expected-major-compiler-version=15

diff --git a/.github/workflows/linux_build.sh b/.github/workflows/linux_build.sh
@@ -8,6 +8,7 @@ export YB_SKIP_UPLOAD=${SKIP_UPLOAD:-}
 echo "Building in directory: $checkout_dir"
 docker run -t \
   --cap-add=SYS_PTRACE \
+  -u root \
   -e GITHUB_TOKEN \
   -e SNYK_TOKEN \
   -e YB_BUILD_THIRDPARTY_ARGS \

diff --git a/.github/workflows/macos_build.sh b/.github/workflows/macos_build.sh
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-brew install autoconf automake pkg-config shellcheck
+brew install autoconf automake pkg-config shellcheck hub
 dirs=( /opt/yb-build/{thirdparty,brew,tmp} )
 sudo mkdir -p "${dirs[@]}"
 sudo chmod 777 "${dirs[@]}"

diff --git a/patches/openssl-fix-afalg-link-on-centos7.patch b/patches/openssl-fix-afalg-link-on-centos7.patch
@@ -0,0 +1,27 @@
+diff --git a/engines/e_afalg.c b/engines/e_afalg.c
+index 2c08cbb..f362d94 100644
+--- a/engines/e_afalg.c
++++ b/engines/e_afalg.c
+@@ -34,10 +34,22 @@
+ #  warning "AFALG ENGINE requires Kernel Headers >= 4.1.0"
+ #  warning "Skipping Compilation of AFALG engine"
+ # endif
++# ifndef OPENSSL_NO_DYNAMIC_ENGINE
++OPENSSL_EXPORT
++    int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns);
++OPENSSL_EXPORT
++    int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns)
++{
++    return 0;
++}
++
++IMPLEMENT_DYNAMIC_CHECK_FN()
++# else
+ void engine_load_afalg_int(void);
+ void engine_load_afalg_int(void)
+ {
+ }
++# endif
+ #else
+
+ # include <linux/if_alg.h>
diff --git a/python/build_definitions/cassandra_cpp_driver.py b/python/build_definitions/cassandra_cpp_driver.py
@@ -22,7 +22,7 @@
 class CassandraCppDriverDependency(Dependency):
     def __init__(self) -> None:
         super(CassandraCppDriverDependency, self).__init__(
-                'cassandra-cpp-driver', '2.9.0-yb-13',
+                'cassandra-cpp-driver', '2.9.0-yb-14',
                 'https://github.com/yugabyte/cassandra-cpp-driver/archive/{0}.tar.gz',
                 BUILD_GROUP_INSTRUMENTED)
         self.copy_sources = False

diff --git a/python/build_definitions/openssl.py b/python/build_definitions/openssl.py
@@ -34,10 +34,16 @@ class OpenSSLDependency(Dependency):
     def __init__(self) -> None:
         super(OpenSSLDependency, self).__init__(
             name='openssl',
-            version='1.1.1o',
+            version='3.0.8',
             url_pattern='https://www.openssl.org/source/openssl-{0}.tar.gz',
             build_group=BUILD_GROUP_COMMON)
         self.copy_sources = True
+        # Patch fixes the following error on kernel versions < 4.1.0:
+        # ld.lld: error: version script assignment of 'global' to symbol 'bind_engine' failed:
+        # symbol not defined
+        # ld.lld: error: version script assignment of 'global' to symbol 'v_check' failed:
+        # symbol not defined
+        self.patches = ['openssl-fix-afalg-link-on-centos7.patch']
 
     def build(self, builder: BuilderInterface) -> None:
         common_configure_options = ['shared', 'no-tests']

diff --git a/python/yugabyte_db_thirdparty/util.py b/python/yugabyte_db_thirdparty/util.py
@@ -360,7 +360,7 @@ def capture_all_output(
         cmd_line_str = shlex_join(args)
         if ex.returncode not in allowed_exit_codes:
             error_msg = f"Unexpected exit code {ex.returncode} from: {cmd_line_str} " \
-                        f"(expected one of { set(sorted(allowed_exit_codes | {0})) })"
+                        f"(expected one of {set(sorted(allowed_exit_codes | {0}))})"
             log(error_msg)
             log("Output from %s (stdout/stderr combined):", cmd_line_str)
             log(ex.stdout.decode('utf-8'))

diff --git a/requirements_frozen.txt b/requirements_frozen.txt
@@ -10,14 +10,14 @@ llvm-installer==1.2.2
 mypy==0.971
 mypy-extensions==0.4.3
 packaging==21.3
-pycodestyle==2.9.1
+pycodestyle==2.11.0
 pyparsing==3.0.9
 requests==2.28.1
 ruamel.yaml==0.17.21
-ruamel.yaml.clib==0.2.6
+ruamel.yaml.clib==0.2.8
 sys-detection==1.3.0
 tomli==2.0.1
-typing-extensions==4.3.0
+typing-extensions==4.8.0
 urllib3==1.26.11
 websocket-client==1.3.3
 yugabyte-pycommon==1.9.15
diff --git a/thirdparty_src_checksums.txt b/thirdparty_src_checksums.txt
@@ -7,6 +7,7 @@ fc9f85fc030e233142908241af7a846e60630aa7388de9a5fafb1f3a26840854  boost-1.77.0.t
 8681f175d4bdb26c52222665793eef08490d7758529330f98d3b29dd0735bccc  boost-1.78.0.tar.bz2
 7bc4b0257d4c15676a158dd30b665d85c1c6a590548e9f6288e2f1542ca6e05a  cassandra-cpp-driver-2.9.0-yb-12.tar.gz
 48dbb1e29028c2caeed6e6c6ec45a5deb0c18955f0292899e1994dd89defc3db  cassandra-cpp-driver-2.9.0-yb-13.tar.gz
+1e42551a7bf986be92d937a18be7208d38c5420550813b170969b08f1cb79058  cassandra-cpp-driver-2.9.0-yb-14.tar.gz
 e97dc472aae52197a4d5e0185eb8f9e04d7575d2dc2b12194ddc768e0f8a846d  cfe-7.1.0.tar.xz
 1ce0042c48ecea839ce67b87e9739cf18e7a5c2b3b9a36d177d00979609b6451  clang-tools-extra-7.1.0.tar.xz
 057bdac0581215b5ceb39edfd5bbef9eb79578f16a8908349f3066251fba88d8  compiler-rt-7.1.0.tar.xz
@@ -85,6 +86,7 @@ c48450d27524c2e5856997133e059e3cf9909241110a6e21ad278890ac425afc  lz4-r130.tar.g
 0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1  openssl-1.1.1l.tar.gz
 40dceb51a4f6a5275bde0e6bf20ef4b91bfc32ed57c0552e2e8e15463372b17a  openssl-1.1.1n.tar.gz
 9384a2b0570dd80358841464677115df785edb941c71211f75076d72fe6b438f  openssl-1.1.1o.tar.gz
+6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e  openssl-3.0.8.tar.gz
 c28dba8782da2cfea1e11c61d335958c31a9c1bc553063546af9cbe98f204092  protobuf-3.5.1.tar.gz
 05e28e5141c1962b1c9d8793cc9cfee8cd11bc24cea13fb9689ac3fc0a379bd3  protobuf-3.5.1-yb-1.tar.gz
 ecff26c4f01c7904abff431b07b09f7c3837d3c2861cdc95ba6f0272d84a8e17  rapidjson-1.1.0-yb-1.zip