From f956945a2b170fd202dba364f56348348dfe901f Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Sat, 23 Dec 2023 05:07:17 +0530 Subject: [PATCH] [GLUTEN-4141][VL] Support Gluten Build in Debian 11 (#4142) Adds Support for building Gluten in Debian 11 --- .../backendsapi/velox/ListenerApiImpl.scala | 5 +- .../utils/SharedLibraryLoaderDebian11.scala | 51 +++++++++++++++++++ dev/package.sh | 9 ++++ dev/vcpkg/setup-build-depends.sh | 16 ++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 backends-velox/src/main/scala/io/glutenproject/utils/SharedLibraryLoaderDebian11.scala diff --git a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/ListenerApiImpl.scala b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/ListenerApiImpl.scala index c7efea87be2a..d00443bb6f4c 100644 --- a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/ListenerApiImpl.scala +++ b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/ListenerApiImpl.scala @@ -87,11 +87,14 @@ class ListenerApiImpl extends ListenerApi { new SharedLibraryLoaderCentos8 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("7")) { new SharedLibraryLoaderCentos7 + } else if (systemName.contains("Debian") && systemVersion.startsWith("11")) { + new SharedLibraryLoaderDebian11 } else { throw new GlutenException( "Found unsupported OS! Currently, Gluten's Velox backend" + " only supports Ubuntu 20.04/22.04, CentOS 7/8, " + - "Alibaba Cloud Linux 2/3 & Anolis 7/8, tencentos 3.2, RedHat 7/8.") + "Alibaba Cloud Linux 2/3 & Anolis 7/8, tencentos 3.2, RedHat 7/8, " + + "Debian 11.") } loader.loadLib(load) } diff --git a/backends-velox/src/main/scala/io/glutenproject/utils/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/io/glutenproject/utils/SharedLibraryLoaderDebian11.scala new file mode 100644 index 000000000000..4fd0e9999446 --- /dev/null +++ b/backends-velox/src/main/scala/io/glutenproject/utils/SharedLibraryLoaderDebian11.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.glutenproject.utils + +import io.glutenproject.vectorized.JniLibLoader + +class SharedLibraryLoaderDebian11 extends SharedLibraryLoader { + override def loadLib(loader: JniLibLoader): Unit = { + loader + .newTransaction() + .loadAndCreateLink("libicudata.so.67", "libicudata.so", false) + .loadAndCreateLink("libre2.so.9", "libre2.so", false) + .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) + .loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) + .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) + .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) + .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) + .loadAndCreateLink("libunwind.so.8", "libunwind.so", false) + .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) + .loadAndCreateLink("librtmp.so.1", "librtmp.so", false) + .loadAndCreateLink("libssh2.so.1", "libssh2.so", false) + .loadAndCreateLink("libpsl.so.5", "libpsl.so", false) + .loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) + .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) + .loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) + .loadAndCreateLink("libboost_context.so.1.74.0", "libboost_context.so", false) + .loadAndCreateLink("libboost_regex.so.1.74.0", "libboost_regex.so", false) + .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + .loadAndCreateLink("libglog.so.0", "libglog.so", false) + .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) + .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) + .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + .commit() + } +} diff --git a/dev/package.sh b/dev/package.sh index 4a831f6b6e96..7e6ff012ae58 100755 --- a/dev/package.sh +++ b/dev/package.sh @@ -37,6 +37,11 @@ function process_setup_centos_7 { cp /usr/local/lib/{libre2.so.10,libhdfs3.so.1,libboost_context.so.1.72.0,libboost_filesystem.so.1.72.0,libboost_program_options.so.1.72.0,libboost_system.so.1.72.0,libboost_thread.so.1.72.0,libboost_regex.so.1.72.0,libprotobuf.so.32} $THIRDPARTY_LIB/ } +function process_setup_debian_11 { + cp /usr/lib/x86_64-linux-gnu/{libre2.so.9,libthrift-0.13.0.so,libboost_context.so.1.74.0,libboost_regex.so.1.74.0,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.2,libglog.so.0,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.67,libicuuc.so.67,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap_r-2.4.so.2,liblber-2.4.so.2,libbrotlidec.so.1,libicudata.so.67,libsasl2.so.2,libbrotlicommon.so.1} $THIRDPARTY_LIB/ + cp /usr/local/lib/libprotobuf.so.32 $THIRDPARTY_LIB/ +} + if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then if [ "$VERSION" == "20.04" ]; then process_setup_ubuntu_2004 @@ -59,6 +64,10 @@ elif [ "$LINUX_OS" == "tencentos" ]; then if [ "$VERSION" == "3.2" ]; then process_setup_centos_8 fi +elif [ "$LINUX_OS" == "debian" ]; then + if [ "$VERSION" == "11" ]; then + process_setup_debian_11 + fi fi cd $THIRDPARTY_LIB/ jar cvf gluten-thirdparty-lib-$LINUX_OS-$VERSION.jar ./ diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh index 0bbc194be36a..4c357db16b13 100755 --- a/dev/vcpkg/setup-build-depends.sh +++ b/dev/vcpkg/setup-build-depends.sh @@ -117,6 +117,22 @@ install_tencentos_3.2() { install_centos_any_maven } +install_debian_11() { + apt-get -y install \ + wget curl tar zip unzip git apt-transport-https \ + build-essential ccache cmake ninja-build pkg-config autoconf autoconf-archive libtool \ + flex bison + + # Download the Eclipse Adoptium GPG key + wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor | tee /etc/apt/trusted.gpg.d/adoptium.gpg > /dev/null + + # Configure the Eclipse Adoptium repository + echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list + + # Install JDK + apt update && apt-get -y install temurin-8-jdk +} + ## Install function end ## Usage