From a1aaa32c6af15f9bc653b6ceeb81ac1767ef3f39 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 17 Mar 2021 17:34:09 +0000 Subject: [PATCH] Merged PR 18201: Install Boost in Azure pipelines Installing Boost manually in all workflows, because it has been recently removed from Azure/GitHub hosted runners. This should fix recent failures of Marian CI builds. --- CMakeLists.txt | 17 ++++++++++------- azure-pipelines.yml | 34 +++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 057e10a48..4ee339781 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -327,7 +327,7 @@ if(CUDA_FOUND) if(USE_STATIC_LIBS) set(EXT_LIBS ${EXT_LIBS} ${CUDA_curand_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_cusparse_LIBRARY}) set(CUDA_LIBS ${CUDA_curand_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_cusparse_LIBRARY}) - + find_library(CUDA_culibos_LIBRARY NAMES culibos PATHS ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ${CUDA_TOOLKIT_ROOT_DIR}/lib/x64) # The cuLIBOS library does not seem to exist in Windows CUDA toolkit installs if(CUDA_culibos_LIBRARY) @@ -504,8 +504,8 @@ if(USE_STATIC_LIBS) endif() # Find MPI -if(USE_MPI) - # 2.0 refers to MPI2 standard. OpenMPI is an implementation of that standard regardless of the specific OpenMPI version +if(USE_MPI) + # 2.0 refers to MPI2 standard. OpenMPI is an implementation of that standard regardless of the specific OpenMPI version # e.g. OpenMPI 1.10 implements MPI2 and will be found correctly. find_package(MPI 2.0 REQUIRED) if(MPI_FOUND) @@ -518,19 +518,22 @@ if(USE_MPI) endif(MPI_FOUND) endif(USE_MPI) -# TODO: move inside if(BOOST_COMPONENTS) -if(USE_STATIC_LIBS) - set(Boost_USE_STATIC_LIBS ON) -endif() ############################################################################### # Find Boost if required if(BOOST_COMPONENTS) + if(USE_STATIC_LIBS) + set(Boost_USE_STATIC_LIBS ON) + endif() + find_package(Boost COMPONENTS ${BOOST_COMPONENTS}) if(Boost_FOUND) include_directories(${Boost_INCLUDE_DIRS}) set(EXT_LIBS ${EXT_LIBS} ${Boost_LIBRARIES}) set(EXT_LIBS ${EXT_LIBS} ${ZLIB_LIBRARIES}) # hack for static compilation + if(MSVC) + add_definitions(-DBOOST_ALL_NO_LIB=1) # hack for missing date-time stub + endif() else(Boost_FOUND) message(SEND_ERROR "Cannot find Boost libraries. Terminating.") endif(Boost_FOUND) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ceb6475d5..a32a82884 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,6 +15,7 @@ pool: name: Azure Pipelines variables: + BOOST_ROOT_WINDOWS: "C:/hostedtoolcache/windows/Boost/1.72.0/x86_64" CUDA_PATH_WINDOWS: "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA" MKL_DIR: "$(Build.SourcesDirectory)/mkl" MKL_URL: "https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip" @@ -69,6 +70,14 @@ stages: # key: 'v0 | "$(VCPKG_PACKAGES)" | vcpkg | "$(Agent.OS)"' # path: $(VCPKG_DIR) + # Boost is no longer pre-installed on Azure/GitHub-hosted Windows runners + - pwsh: | + Write-Host "Downloading Boost to $(BOOST_ROOT_WINDOWS)" + $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/1.72.0/boost_1_72_0-msvc-14.2-64.exe" + C:\msys64\usr\bin\wget.exe -nv $Url -O "$(Pipeline.Workspace)/boost.exe" + Start-Process -Wait -FilePath "$(Pipeline.Workspace)/boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=$(BOOST_ROOT_WINDOWS)" + displayName: Download Boost + - pwsh: | git clone https://github.com/Microsoft/vcpkg.git $(VCPKG_DIR) cd $(VCPKG_DIR) @@ -121,9 +130,7 @@ stages: # Set envvars so that CMake can find the installed packages MKLROOT: $(MKL_DIR) CUDA_PATH: $(CUDA_PATH_WINDOWS)/v$(cuda_version) - # Boost is pre-installed on Azure/GitHub-hosted Windows runners - # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#boost - BOOST_ROOT: $(BOOST_ROOT_1_72_0) + BOOST_ROOT: $(BOOST_ROOT_WINDOWS) - script: | call "$(VS_PATH)/VC/Auxiliary/Build/vcvarsall.bat" x64 @@ -226,12 +233,18 @@ stages: - checkout: self submodules: true - # The following packages are already installed on Azure-hosted runners: build-essential openssl libssl-dev - # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev + # The following packages are already installed on Azure-hosted runners: build-essential openssl libssl-dev + # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev - bash: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler displayName: Install packages - # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html + # Boost is no longer pre-installed on Azure/GitHub-hosted runners + # TODO: check which Boost components are really needed and update the list + - bash: sudo apt-get install -y libboost-system-dev + displayName: Install Boost + condition: eq(variables.boost, true) + + # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html - bash: | wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" @@ -240,13 +253,11 @@ stages: displayName: Install MKL condition: eq(variables.cpu, true) - # The script simplifies installation of different versions of CUDA + # The script simplifies installation of different versions of CUDA - bash: ./scripts/ci/install_cuda_ubuntu.sh $(cuda) displayName: Install CUDA condition: eq(variables.gpu, true) - # Boost is already installed on Azure-hosted runners in a non-standard location - # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671 - bash: | mkdir -p build cd build @@ -260,9 +271,6 @@ stages: -DUSE_FBGEMM=$(cpu) \ -DUSE_SENTENCEPIECE=on \ -DUSE_STATIC_LIBS=$(static) \ - -DBOOST_ROOT=$BOOST_ROOT_1_72_0 \ - -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_72_0/include \ - -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_72_0/lib \ -DBoost_ARCHITECTURE=-x64 \ -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-$(cuda) displayName: Configure CMake @@ -346,7 +354,7 @@ stages: - checkout: self submodules: true - - bash: brew install openblas protobuf + - bash: brew install boost openblas openssl protobuf displayName: Install packages # Openblas location is exported explicitly because openblas is keg-only, which means it was not symlinked into /usr/local/.