WatsonX.Data download

yingsu00 · Feb 23, 2024 · 641e80c · 641e80c
1 parent 0d02c8f
commit 641e80c
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 4 deletions.
diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh
@@ -131,7 +131,16 @@ function install_hdfs_deps {
     sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt
     sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake
     # Dependencies for Hadoop testing
-    wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop
+    export HADOOP_VERSION=2.10.1
+        if [[ "${artifactory_user}" != "" ]] && [[ "${artifactory_token}" != "" ]]; then
+          mkdir -p hadoop
+          pushd hadoop
+          curl -u ${artifactory_user}:${artifactory_token} https://na.artifactory.swg-devops.com/artifactory/hyc-cpd-skywalker-team-lakehouse-on-prem-docker-local/hadoop/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz -o hadoop-${HADOOP_VERSION}.tar.gz
+          tar -xz --strip-components=1 -f hadoop-${HADOOP_VERSION}.tar.gz
+          popd
+        else
+          wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop
+        fi
     cp -a hadoop /usr/local/
   fi
   cmake_install

diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
@@ -56,12 +56,29 @@ if(VELOX_ENABLE_ARROW)
                                       ${THRIFT_INCLUDE_DIR})
   set_property(TARGET thrift PROPERTY IMPORTED_LOCATION ${THRIFT_LIB})
 
+  set(ARTIFACTORY_USER "")
+  set(ARTIFACTORY_TOKEN "")
+
+  if(DEFINED ENV{artifactory_user})
+    set(ARTIFACTORY_USER $ENV{artifactory_user})
+  endif()
+  if(DEFINED ENV{artifactory_token})
+    set(ARTIFACTORY_TOKEN $ENV{artifactory_token})
+  endif()
+
   set(VELOX_ARROW_BUILD_VERSION 15.0.0)
   set(VELOX_ARROW_BUILD_SHA256_CHECKSUM
       01dd3f70e85d9b5b933ec92c0db8a4ef504a5105f78d2d8622e84279fb45c25d)
-  set(VELOX_ARROW_SOURCE_URL
-      "https://archive.apache.org/dist/arrow/arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz"
-  )
+  if((NOT "${ARTIFACTORY_USER}" STREQUAL "") AND (NOT "${ARTIFACTORY_TOKEN}" STREQUAL ""))
+    set(VELOX_ARROW_SOURCE_URL
+            "https://na.artifactory.swg-devops.com/artifactory/hyc-cpd-skywalker-team-lakehouse-on-prem-docker-local/arrow/arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz"
+    )
+  else()
+    set(VELOX_ARROW_SOURCE_URL
+            "https://archive.apache.org/dist/arrow/arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz"
+    )
+  endif()
+
 
   resolve_dependency_url(ARROW)