From 641e80cc4e7c667491ca4e77d0e5a0f721a09f65 Mon Sep 17 00:00:00 2001 From: yingsu00 Date: Fri, 23 Feb 2024 18:25:01 +0800 Subject: [PATCH] WatsonX.Data download --- scripts/setup-adapters.sh | 11 ++++++++++- third_party/CMakeLists.txt | 23 ++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh index 675ff4c0291d..98bc1d7bf25a 100755 --- a/scripts/setup-adapters.sh +++ b/scripts/setup-adapters.sh @@ -131,7 +131,16 @@ function install_hdfs_deps { sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" ./CMakeLists.txt sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake # Dependencies for Hadoop testing - wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop + export HADOOP_VERSION=2.10.1 + if [[ "${artifactory_user}" != "" ]] && [[ "${artifactory_token}" != "" ]]; then + mkdir -p hadoop + pushd hadoop + curl -u ${artifactory_user}:${artifactory_token} https://na.artifactory.swg-devops.com/artifactory/hyc-cpd-skywalker-team-lakehouse-on-prem-docker-local/hadoop/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz -o hadoop-${HADOOP_VERSION}.tar.gz + tar -xz --strip-components=1 -f hadoop-${HADOOP_VERSION}.tar.gz + popd + else + wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop + fi cp -a hadoop /usr/local/ fi cmake_install diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index ce4c24dbee07..e9a52616870c 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -56,12 +56,29 @@ if(VELOX_ENABLE_ARROW) ${THRIFT_INCLUDE_DIR}) set_property(TARGET thrift PROPERTY IMPORTED_LOCATION ${THRIFT_LIB}) + set(ARTIFACTORY_USER "") + set(ARTIFACTORY_TOKEN "") + + if(DEFINED ENV{artifactory_user}) + set(ARTIFACTORY_USER $ENV{artifactory_user}) + endif() + if(DEFINED ENV{artifactory_token}) + set(ARTIFACTORY_TOKEN $ENV{artifactory_token}) + endif() + set(VELOX_ARROW_BUILD_VERSION 15.0.0) set(VELOX_ARROW_BUILD_SHA256_CHECKSUM 01dd3f70e85d9b5b933ec92c0db8a4ef504a5105f78d2d8622e84279fb45c25d) - set(VELOX_ARROW_SOURCE_URL - "https://archive.apache.org/dist/arrow/arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz" - ) + if((NOT "${ARTIFACTORY_USER}" STREQUAL "") AND (NOT "${ARTIFACTORY_TOKEN}" STREQUAL "")) + set(VELOX_ARROW_SOURCE_URL + "https://na.artifactory.swg-devops.com/artifactory/hyc-cpd-skywalker-team-lakehouse-on-prem-docker-local/arrow/arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz" + ) + else() + set(VELOX_ARROW_SOURCE_URL + "https://archive.apache.org/dist/arrow/arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz" + ) + endif() + resolve_dependency_url(ARROW)