Skip to content

Commit

Permalink
[GLUTEN-6571][VL] Add platform and arch subdirectory for base lib pac…
Browse files Browse the repository at this point in the history
…kage (apache#6942)

Closes apache#6571
  • Loading branch information
wForget authored and shamirchen committed Oct 14, 2024
1 parent b6826db commit f6e256a
Show file tree
Hide file tree
Showing 12 changed files with 317 additions and 219 deletions.
1 change: 1 addition & 0 deletions backends-velox/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@
<resources>
<resource>
<directory>${cpp.releases.dir}</directory>
<targetPath>org/apache/gluten/${platform}/${arch}</targetPath>
</resource>
<resource>
<directory>${resource.dir}</directory>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,8 @@ class VeloxListenerApi extends ListenerApi with Logging {
JniLibLoader.loadFromPath(libPath, false)
} else {
val baseLibName = conf.get(GlutenConfig.GLUTEN_LIB_NAME, "gluten")
loader.mapAndLoad(baseLibName, false)
loader.mapAndLoad(VeloxBackend.BACKEND_NAME, false)
loader.load(s"$baseLibPackage/${System.mapLibraryName(baseLibName)}", false)
loader.load(s"$baseLibPackage/${System.mapLibraryName(VeloxBackend.BACKEND_NAME)}", false)
}

// Initial native backend with configurations.
Expand All @@ -152,6 +152,18 @@ class VeloxListenerApi extends ListenerApi with Logging {
GlutenRowSplitter.setInstance(new VeloxRowSplitter())
}

private lazy val baseLibPackage: String = {
val osName = System.getProperty("os.name") match {
case n if n.contains("Linux") => "linux"
case n if n.contains("Mac") => "darwin"
case _ =>
// Default to linux
"linux"
}
val arch = System.getProperty("os.arch")
s"org/apache/gluten/$osName/$arch"
}

private def shutdown(): Unit = {
// TODO shutdown implementation in velox to release resources
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,26 +20,28 @@ import org.apache.gluten.vectorized.JniLibLoader

class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false)
loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false)
loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false)
loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false)
loader.loadAndCreateLink(
"libboost_program_options.so.1.84.0",
"libboost_program_options.so",
false)
loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false)
loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false)
loader.loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so", false)
loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so", false)
loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
loader.loadAndCreateLink("libre2.so.10", "libre2.so", false)
loader.loadAndCreateLink("libzstd.so.1", "libzstd.so", false)
loader.loadAndCreateLink("liblz4.so.1", "liblz4.so", false)
def loadAndCreateLink(libName: String, linkName: String): Unit = {
val mapLibName = System.mapLibraryName(libName)
loader.loadAndCreateLink(mapLibName, linkName, false)
}

loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so")
loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so")
loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so")
loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so")
loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so")
loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
loadAndCreateLink("libglog.so.0", "libglog.so")
loadAndCreateLink("libntlm.so.0", "libntlm.so")
loadAndCreateLink("libgsasl.so.7", "libgsasl.so")
loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so")
loadAndCreateLink("libre2.so.10", "libre2.so")
loadAndCreateLink("libzstd.so.1", "libzstd.so")
loadAndCreateLink("liblz4.so.1", "liblz4.so")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,30 +20,32 @@ import org.apache.gluten.vectorized.JniLibLoader

class SharedLibraryLoaderCentos8 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false)
loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false)
loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false)
loader.loadAndCreateLink("libicudata.so.60", "libicudata.so", false)
loader.loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false)
loader.loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false)
loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false)
loader.loadAndCreateLink(
"libboost_program_options.so.1.84.0",
"libboost_program_options.so",
false)
loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false)
loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false)
loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false)
loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so", false)
loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
loader.loadAndCreateLink("libglog.so.1", "libglog.so", false)
loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so", false)
loader.loadAndCreateLink("libidn.so.11", "libidn.so", false)
loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
loader.loadAndCreateLink("libre2.so.0", "libre2.so", false)
loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
def loadAndCreateLink(libName: String, linkName: String): Unit = {
val mapLibName = System.mapLibraryName(libName)
loader.loadAndCreateLink(mapLibName, linkName, false)
}

loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so")
loadAndCreateLink("libicudata.so.60", "libicudata.so")
loadAndCreateLink("libicuuc.so.60", "libicuuc.so")
loadAndCreateLink("libicui18n.so.60", "libicui18n.so")
loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so")
loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so")
loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so")
loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so")
loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
loadAndCreateLink("libglog.so.1", "libglog.so")
loadAndCreateLink("libdwarf.so.1", "libdwarf.so")
loadAndCreateLink("libidn.so.11", "libidn.so")
loadAndCreateLink("libntlm.so.0", "libntlm.so")
loadAndCreateLink("libgsasl.so.7", "libgsasl.so")
loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so")
loadAndCreateLink("libre2.so.0", "libre2.so")
loadAndCreateLink("libsodium.so.23", "libsodium.so")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,30 +20,32 @@ import org.apache.gluten.vectorized.JniLibLoader

class SharedLibraryLoaderCentos9 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false)
loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false)
loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false)
loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false)
loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false)
loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false)
loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false)
loader.loadAndCreateLink(
"libboost_program_options.so.1.84.0",
"libboost_program_options.so",
false)
loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false)
loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false)
loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false)
loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
loader.loadAndCreateLink("libglog.so.1", "libglog.so", false)
loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false)
loader.loadAndCreateLink("libidn.so.12", "libidn.so", false)
loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
loader.loadAndCreateLink("libre2.so.9", "libre2.so", false)
loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
def loadAndCreateLink(libName: String, linkName: String): Unit = {
val mapLibName = System.mapLibraryName(libName)
loader.loadAndCreateLink(mapLibName, linkName, false)
}

loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so")
loadAndCreateLink("libicudata.so.67", "libicudata.so")
loadAndCreateLink("libicuuc.so.67", "libicuuc.so")
loadAndCreateLink("libicui18n.so.67", "libicui18n.so")
loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so")
loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so")
loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so")
loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
loadAndCreateLink("libglog.so.1", "libglog.so")
loadAndCreateLink("libdwarf.so.0", "libdwarf.so")
loadAndCreateLink("libidn.so.12", "libidn.so")
loadAndCreateLink("libntlm.so.0", "libntlm.so")
loadAndCreateLink("libgsasl.so.7", "libgsasl.so")
loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so")
loadAndCreateLink("libre2.so.9", "libre2.so")
loadAndCreateLink("libsodium.so.23", "libsodium.so")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,30 +20,35 @@ import org.apache.gluten.vectorized.JniLibLoader

class SharedLibraryLoaderDebian11 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false)
loader.loadAndCreateLink("libre2.so.9", "libre2.so", false)
loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false)
loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false)
loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false)
loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false)
loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false)
loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false)
loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false)
loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false)
loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false)
loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false)
loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
def loadAndCreateLink(libName: String, linkName: String): Unit = {
val mapLibName = System.mapLibraryName(libName)
loader.loadAndCreateLink(mapLibName, linkName, false)
}

loadAndCreateLink("libicudata.so.67", "libicudata.so")
loadAndCreateLink("libre2.so.9", "libre2.so")
loadAndCreateLink("libicuuc.so.67", "libicuuc.so")
loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so")
loadAndCreateLink("libsasl2.so.2", "libsasl2.so")
loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so")
loadAndCreateLink("libicui18n.so.67", "libicui18n.so")
loadAndCreateLink("libunwind.so.8", "libunwind.so")
loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
loadAndCreateLink("librtmp.so.1", "librtmp.so")
loadAndCreateLink("libssh2.so.1", "libssh2.so")
loadAndCreateLink("libpsl.so.5", "libpsl.so")
loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so")
loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so")
loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so")
loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so")
loadAndCreateLink("libglog.so.0", "libglog.so")
loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
loadAndCreateLink("libsnappy.so.1", "libsnappy.so")
loadAndCreateLink("libcurl.so.4", "libcurl.so")
loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,36 +20,41 @@ import org.apache.gluten.vectorized.JniLibLoader

class SharedLibraryLoaderDebian12 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
loader.loadAndCreateLink("libcrypto.so.3", "libcrypto.so", false)
loader.loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so", false)
loader.loadAndCreateLink("libssl.so.3", "libssl.so", false)
loader.loadAndCreateLink("libicudata.so.72", "libicudata.so", false)
loader.loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so", false)
loader.loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so", false)
loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
loader.loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so", false)
loader.loadAndCreateLink("libicuuc.so.72", "libicuuc.so", false)
loader.loadAndCreateLink("libkrb5.so.3", "libkrb5.so", false)
loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so", false)
loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false)
loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so", false)
loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false)
loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
loader.loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so", false)
loader.loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so", false)
loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false)
loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false)
loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false)
loader.loadAndCreateLink("libglog.so.1", "libglog.so", false)
loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
def loadAndCreateLink(libName: String, linkName: String): Unit = {
val mapLibName = System.mapLibraryName(libName)
loader.loadAndCreateLink(mapLibName, linkName, false)
}

loadAndCreateLink("libcrypto.so.3", "libcrypto.so")
loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so")
loadAndCreateLink("libssl.so.3", "libssl.so")
loadAndCreateLink("libicudata.so.72", "libicudata.so")
loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so")
loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so")
loadAndCreateLink("libsnappy.so.1", "libsnappy.so")
loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so")
loadAndCreateLink("libicuuc.so.72", "libicuuc.so")
loadAndCreateLink("libkrb5.so.3", "libkrb5.so")
loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so")
loadAndCreateLink("libsasl2.so.2", "libsasl2.so")
loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so")
loadAndCreateLink("libicui18n.so.72", "libicui18n.so")
loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
loadAndCreateLink("libunwind.so.8", "libunwind.so")
loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
loadAndCreateLink("librtmp.so.1", "librtmp.so")
loadAndCreateLink("libssh2.so.1", "libssh2.so")
loadAndCreateLink("libpsl.so.5", "libpsl.so")
loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so")
loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so")
loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so")
loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so")
loadAndCreateLink("libglog.so.1", "libglog.so")
loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
loadAndCreateLink("libcurl.so.4", "libcurl.so")
loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so")
}
}
Loading

0 comments on commit f6e256a

Please sign in to comment.