Merge pull request #12 from anshuljainansja/Ubuntu184

Ubuntu 18.04 Installation and Testing Scripts

centos/centos-7.x/centos-7.6-hpc/install_gcc.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
 
-../../../common/install_gcc-9.2.sh
+MODULE_FILES_DIRECTORY=/usr/share/Modules/modulefiles
+
+mkdir -p ${MODULE_FILES_DIRECTORY}
+
+../../../common/install_gcc-9.2.sh ${MODULE_FILES_DIRECTORY}
 

centos/centos-7.x/centos-7.6-hpc/install_mellanoxofed.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 set -ex
 
-wget http://content.mellanox.com/ofed/MLNX_OFED-5.0-1.0.0.0/MLNX_OFED_LINUX-5.0-1.0.0.0-rhel7.6-x86_64.tgz
-tar zxvf MLNX_OFED_LINUX-5.0-1.0.0.0-rhel7.6-x86_64.tgz
+wget http://content.mellanox.com/ofed/MLNX_OFED-5.0-2.1.8.0/MLNX_OFED_LINUX-5.0-2.1.8.0-rhel7.6-x86_64.tgz
+tar zxvf MLNX_OFED_LINUX-5.0-2.1.8.0-rhel7.6-x86_64.tgz
 
 KERNEL=( $(rpm -q kernel | sed 's/kernel\-//g') )
 KERNEL=${KERNEL[-1]}
 yum -y install http://olcentgbl.trafficmanager.net/centos/7.6.1810/updates/x86_64/kernel-devel-${KERNEL}.rpm
-./MLNX_OFED_LINUX-5.0-1.0.0.0-rhel7.6-x86_64/mlnxofedinstall --kernel $KERNEL --kernel-sources /usr/src/kernels/${KERNEL} --add-kernel-support --skip-repo
+./MLNX_OFED_LINUX-5.0-2.1.8.0-rhel7.6-x86_64/mlnxofedinstall --kernel $KERNEL --kernel-sources /usr/src/kernels/${KERNEL} --add-kernel-support --skip-repo

centos/centos-7.x/centos-7.7-hpc/install_gcc.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
 
-../../../common/install_gcc-9.2.sh
+MODULE_FILES_DIRECTORY=/usr/share/Modules/modulefiles
 
+mkdir -p ${MODULE_FILES_DIRECTORY}
+
+../../../common/install_gcc-9.2.sh ${MODULE_FILES_DIRECTORY}

centos/centos-7.x/centos-7.7-hpc/install_mellanoxofed.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 set -ex
 
-wget http://content.mellanox.com/ofed/MLNX_OFED-5.0-1.0.0.0/MLNX_OFED_LINUX-5.0-1.0.0.0-rhel7.7-x86_64.tgz
-tar zxvf MLNX_OFED_LINUX-5.0-1.0.0.0-rhel7.7-x86_64.tgz
+wget http://content.mellanox.com/ofed/MLNX_OFED-5.0-2.1.8.0/MLNX_OFED_LINUX-5.0-2.1.8.0-rhel7.7-x86_64.tgz
+tar zxvf MLNX_OFED_LINUX-5.0-2.1.8.0-rhel7.7-x86_64.tgz
 
 KERNEL=( $(rpm -q kernel | sed 's/kernel\-//g') )
 KERNEL=${KERNEL[-1]}
 yum install -y kernel-devel-${KERNEL}
-./MLNX_OFED_LINUX-5.0-1.0.0.0-rhel7.7-x86_64/mlnxofedinstall --kernel $KERNEL --kernel-sources /usr/src/kernels/${KERNEL} --add-kernel-support --skip-repo
+./MLNX_OFED_LINUX-5.0-2.1.8.0-rhel7.7-x86_64/mlnxofedinstall --kernel $KERNEL --kernel-sources /usr/src/kernels/${KERNEL} --add-kernel-support --skip-repo
 

centos/centos-8.x/centos-8.1-hpc/install_gcc.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
 
-../../../common/install_gcc-9.2.sh
+MODULE_FILES_DIRECTORY=/usr/share/Modules/modulefiles
 
+mkdir -p ${MODULE_FILES_DIRECTORY}
+
+../../../common/install_gcc-9.2.sh ${MODULE_FILES_DIRECTORY}

centos/common/install_mpis.sh

@@ -42,9 +42,16 @@ cd ..
 # Install MVAPICH2-X 2.3rc3
 wget http://mvapich.cse.ohio-state.edu/download/mvapich/mv2x/2.3rc3/mofed5.0/mvapich2-x-advanced-xpmem-mofed5.0-gnu9.2.0-2.3rc3-1.el7.x86_64.rpm
 rpm -Uvh --nodeps mvapich2-x-advanced-xpmem-mofed5.0-gnu9.2.0-2.3rc3-1.el7.x86_64.rpm
-MV2X_PATH="/opt/mvapich2-x/gnu9.2.0/mofed5.0/advanced-xpmem/mpirun"
+MV2X_INSTALLATION_DIRECTORY="/opt/mvapich2-x"
+MV2X_PATH="${MV2X_INSTALLATION_DIRECTORY}/gnu9.2.0/mofed5.0/advanced-xpmem/mpirun"
 MV2X_VERSION="2.3rc3"
 
+# download and build benchmark for MVAPICH2-X 2.3rc3
+wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.6.2.tar.gz
+tar -xvf osu-micro-benchmarks-5.6.2.tar.gz
+cd osu-micro-benchmarks-5.6.2/
+./configure CC=${MV2X_PATH}/bin/mpicc CXX=${MV2X_PATH}/bin/mpicxx --prefix=${MV2X_INSTALLATION_DIRECTORY}/ && make -j$(nproc) && make install
+cd ..
 
 # Setup module files for MPIs
 mkdir -p /usr/share/Modules/modulefiles/mpi/

common/install_gcc-9.2.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
 set -x
 
-# Install gcc 9.2
+MODULE_FILES_DIRECTORY=$1
 
+# Install gcc 9.2
 wget ftp://gcc.gnu.org/pub/gcc/infrastructure/gmp-6.1.0.tar.bz2
 tar -xvf gmp-6.1.0.tar.bz2
 cd ./gmp-6.1.0
@@ -28,14 +29,12 @@ cd gcc-9.2.0
 ./configure --disable-multilib --prefix=/opt/gcc-9.2.0 && make -j$(nproc) && make install
 cd ..
 
-
 # create modulefile
-cat << EOF >> /usr/share/Modules/modulefiles/gcc-9.2.0
+cat << EOF >> ${MODULE_FILES_DIRECTORY}/gcc-9.2.0
 #%Module 1.0
 #
 #  GCC 9.2.0
 #
-
 prepend-path    PATH            /opt/gcc-9.2.0/bin
 prepend-path    LD_LIBRARY_PATH /opt/gcc-9.2.0/lib64
 setenv          CC              /opt/gcc-9.2.0/bin/gcc

tests/run-tests.sh

@@ -1,40 +1,108 @@
 #!/bin/bash
 source /etc/profile
-module use /usr/share/Modules/modulefiles
 
 GCC_VERSION="9.2.0"
 MKL_VERSION="2019.5.281"
 MOFED_VERSION="MLNX_OFED_LINUX-5.0-1.0.0.0"
-HPCX_PATH_76="/opt/hpcx-v2.6.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-redhat7.6-x86_64"
-HPCX_PATH_77="/opt/hpcx-v2.6.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-redhat7.7-x86_64"
-HPCX_PATH_81="/opt/hpcx-v2.6.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-redhat8.1-x86_64"
+HPCX_PATH_CENTOS_76="/opt/hpcx-v2.6.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-redhat7.6-x86_64"
+HPCX_PATH_CENTOS_77="/opt/hpcx-v2.6.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-redhat7.7-x86_64"
+HPCX_PATH_CENTOS_81="/opt/hpcx-v2.6.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-redhat8.1-x86_64"
+HPCX_PATH_UBUNTU_1804="/opt/hpcx-v2.6.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-ubuntu18.04-x86_64"
 
 IMPI2019_PATH="/opt/intel/compilers_and_libraries_2020.1.217"
 IMPI2018_PATH="/opt/intel/compilers_and_libraries_2018.5.274"
 
 MVAPICH2_PATH="/opt/mvapich2-2.3.3"
-MVAPICH2X_PATH="/opt/mvapich2-x/gnu9.2.0/mofed5.0/advanced-xpmem/mpirun"
+MVAPICH2X_INSTALLATION_DIRECTORY="/opt/mvapich2-x"
+MVAPICH2X_PATH="${MVAPICH2X_INSTALLATION_DIRECTORY}/gnu9.2.0/mofed5.0/advanced-xpmem/mpirun"
 
 OPENMPI_PATH="/opt/openmpi-4.0.3"
 
-distro=`cat /etc/redhat-release | awk '{print $4}'`
-if [ $distro == "7.6.1810" ]
+CENTOS_MODULE_FILES_ROOT="/usr/share/Modules/modulefiles"
+UBUNTU_MODULE_FILES_ROOT="/usr/share/modules/modulefiles"
+
+CHECK_HPCX=0
+CHECK_IMPI_2019=0
+CHECK_IMPI_2018=0
+CHECK_OMPI=0
+CHECK_MVAPICH2=0
+CHECK_MVAPICH2X=0
+CHECK_CUDA=0
+
+# Find distro
+find_distro() {
+    local os=`cat /etc/os-release | awk 'match($0, /^NAME="(.*)"/, result) { print result[1] }'`
+    if [[ $os == "CentOS Linux" ]]
+    then
+        local centos_distro=`find_centos_distro`
+        echo "${os} ${centos_distro}"
+    elif [[ $os == "Ubuntu" ]]
+    then
+        local ubuntu_distro=`find_ubuntu_distro`
+        echo "${os} ${ubuntu_distro}"
+    else
+        echo "*** Error - invalid distro!"
+        exit -1
+    fi
+}
+
+# Find CentOS distro
+find_centos_distro() {
+    echo `cat /etc/redhat-release | awk '{print $4}'`
+}
+
+# Find Ubuntu distro
+find_ubuntu_distro() {
+    echo `cat /etc/os-release | awk 'match($0, /^PRETTY_NAME="(.*)"/, result) { print result[1] }' | awk '{print $2}'`
+}
+
+distro=`find_distro`
+echo "Detected distro: ${distro}"
+
+if [[ $distro == "CentOS Linux 7.6.1810" ]]
 then
-    HPCX_PATH=${HPCX_PATH_76}
+    HPCX_PATH=${HPCX_PATH_CENTOS_76}
+    CHECK_HPCX=1
+    CHECK_IMPI_2019=1
     CHECK_IMPI_2018=1
-elif [ $distro == "7.7.1908" ]
+    CHECK_OMPI=1
+    CHECK_MVAPICH2=1
+    CHECK_MVAPICH2X=1
+    MODULE_FILES_ROOT=${CENTOS_MODULE_FILES_ROOT}
+elif [[ $distro == "CentOS Linux 7.7.1908" ]]
 then
-    HPCX_PATH=${HPCX_PATH_77}
+    HPCX_PATH=${HPCX_PATH_CENTOS_77}
+    CHECK_HPCX=1
+    CHECK_IMPI_2019=1
     CHECK_IMPI_2018=1
-elif [ $distro == "8.1.1911" ]
+    CHECK_OMPI=1
+    CHECK_MVAPICH2=1
+    CHECK_MVAPICH2X=1
+    MODULE_FILES_ROOT=${CENTOS_MODULE_FILES_ROOT}
+elif [[ $distro == "CentOS Linux 8.1.1911" ]]
+then
+    HPCX_PATH=${HPCX_PATH_CENTOS_81}
+    CHECK_HPCX=1
+    CHECK_IMPI_2019=1
+    CHECK_OMPI=1
+    CHECK_MVAPICH2=1
+    CHECK_MVAPICH2X=1
+    MODULE_FILES_ROOT=${CENTOS_MODULE_FILES_ROOT}
+elif [[ $distro == "Ubuntu 18.04.4" ]]
 then
-    HPCX_PATH=${HPCX_PATH_81}
-    CHECK_IMPI_2018=0
+    CHECK_HPCX=1
+    CHECK_IMPI_2019=1
+    CHECK_MVAPICH2=1
+    CHECK_CUDA=1
+    HPCX_PATH=${HPCX_PATH_UBUNTU_1804}
+    MODULE_FILES_ROOT=${UBUNTU_MODULE_FILES_ROOT}
 else
     echo "*** Error - invalid distro!"
     exit -1
 fi
 
+module use ${MODULE_FILES_ROOT}
+
 # check file is present
 check_exists() {
     ls $1
@@ -70,26 +138,14 @@ check_exit_code "IB device is listed" "IB device not found"
 ibstat | grep "LinkUp"
 check_exit_code "IB device state: LinkUp" "IB link not up"
 
-# verify MPI modulefiles
-check_exists "/usr/share/Modules/modulefiles/mpi/hpcx"
-
-if [ $CHECK_IMPI_2018 -eq 1 ]
-then
-    check_exists "/usr/share/Modules/modulefiles/mpi/impi"
-fi
-
-check_exists "/usr/share/Modules/modulefiles/mpi/impi-2019"
-check_exists "/usr/share/Modules/modulefiles/mpi/mvapich2"
-check_exists "/usr/share/Modules/modulefiles/mpi/mvapich2x"
-check_exists "/usr/share/Modules/modulefiles/mpi/openmpi"
-
-check_exists "/usr/share/Modules/modulefiles/gcc-${GCC_VERSION}"
+# verify GCC modulefile
+check_exists "${MODULE_FILES_ROOT}/gcc-${GCC_VERSION}"
 
 # verify AMD modulefiles
-check_exists "/usr/share/Modules/modulefiles/amd/fftw"
-check_exists "/usr/share/Modules/modulefiles/amd/libflame"
-check_exists "/usr/share/Modules/modulefiles/amd/blis"
-check_exists "/usr/share/Modules/modulefiles/amd/blis-mt"
+check_exists "${MODULE_FILES_ROOT}/amd/fftw"
+check_exists "${MODULE_FILES_ROOT}/amd/libflame"
+check_exists "${MODULE_FILES_ROOT}/amd/blis"
+check_exists "${MODULE_FILES_ROOT}/amd/blis-mt"
 
 # verify s/w package installations
 check_exists "/opt/gcc-${GCC_VERSION}/"
@@ -99,46 +155,78 @@ check_exists "/opt/amd/fftw/"
 check_exists "/opt/amd/libflame/"
 check_exists "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/"
 
-# verify mpi installations
+# verify mpi installations and their modulefiles
 module avail
 
 # hpcx
-module load mpi/hpcx
-mpirun -np 2 --map-by ppr:2:node -x UCX_TLS=rc ${HPCX_PATH}/ompi/tests/osu-micro-benchmarks-5.3.2/osu_latency
-check_exit_code "HPC-X" "Failed to run HPC-X"
-module unload mpi/hpcx
+if [ $CHECK_HPCX -eq 1 ]
+then
+    check_exists "${MODULE_FILES_ROOT}/mpi/hpcx"
+
+    module load mpi/hpcx
+    mpirun -np 2 --map-by ppr:2:node -x UCX_TLS=rc ${HPCX_PATH}/ompi/tests/osu-micro-benchmarks-5.3.2/osu_latency
+    check_exit_code "HPC-X" "Failed to run HPC-X"
+    module unload mpi/hpcx
+fi
 
 # impi 2019
-module load mpi/impi-2019
-mpiexec -np 2 -ppn 2 -env FI_PROVIDER=mlx -env UCX_TLS=rc -env I_MPI_SHM=0 ${IMPI2019_PATH}/linux/mpi/intel64/bin/IMB-MPI1 pingpong
-check_exit_code "Intel MPI 2019" "Failed to run Intel MPI 2019"
-module unload mpi/impi-2019
+if [ $CHECK_IMPI_2019 -eq 1 ]
+then
+    check_exists "${MODULE_FILES_ROOT}/mpi/impi-2019"
+
+    module load mpi/impi-2019
+    mpiexec -np 2 -ppn 2 -env FI_PROVIDER=verbs -env I_MPI_SHM=0 ${IMPI2019_PATH}/linux/mpi/intel64/bin/IMB-MPI1 pingpong
+    check_exit_code "Intel MPI 2019" "Failed to run Intel MPI 2019"
+    module unload mpi/impi-2019
+fi
 
-#impi 2018
+# impi 2018
 if [ $CHECK_IMPI_2018 -eq 1 ]
 then
+    check_exists "${MODULE_FILES_ROOT}/mpi/impi"
+
     module load mpi/impi
     mpiexec -np 2 -ppn 2 -env I_MPI_FABRICS=ofa ${IMPI2018_PATH}/linux/mpi/intel64/bin/IMB-MPI1 pingpong
     check_exit_code "Intel MPI 2018" "Failed to run Intel MPI 2018"
     module unload mpi/impi
 fi
 
 # mvapich2
-module load mpi/mvapich2
-mpiexec -np 2 -ppn 2 -env MV2_USE_SHARED_MEM=0  ${MVAPICH2_PATH}/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency
-check_exit_code "MVAPICH2" "Failed to run MVAPICH2"
-module unload mpi/mvapich2
+if [ $CHECK_MVAPICH2 -eq 1 ]
+then
+    check_exists "${MODULE_FILES_ROOT}/mpi/mvapich2"
+
+    module load mpi/mvapich2
+    mpiexec -np 2 -ppn 2 -env MV2_USE_SHARED_MEM=0  ${MVAPICH2_PATH}/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency
+    check_exit_code "MVAPICH2" "Failed to run MVAPICH2"
+    module unload mpi/mvapich2
+fi
 
 # mvapich2x
-module load mpi/mvapich2x
-mpiexec -np 2 -ppn 2 -env MV2_USE_SHARED_MEM=0  ${MVAPICH2_PATH}/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency
-check_exit_code "MVAPICH2X" "Failed to run MVAPICH2X"
-module unload mpi/mvapich2x
-
-# Note: no need to run MVAPICH2-x and OpenMPI, as these are already covered by MVAPICH2 and HPC-X runs
-# But make sure they are installed
-check_exists ${MVAPICH2X_PATH}
-check_exists ${OPENMPI_PATH}
+if [ $CHECK_MVAPICH2X -eq 1 ]
+then
+    check_exists "${MODULE_FILES_ROOT}/mpi/mvapich2x"
+    check_exists ${MVAPICH2X_PATH}
+
+    module load mpi/mvapich2x
+    mpiexec -np 2 -ppn 2 -env MV2_USE_SHARED_MEM=0  ${MVAPICH2X_INSTALLATION_DIRECTORY}/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency
+    check_exit_code "MVAPICH2X" "Failed to run MVAPICH2X"
+    module unload mpi/mvapich2x
+fi
+
+# Note: no need to run OpenMPI, as it is already covered by HPC-X runs, but make sure it is installed
+if [ $CHECK_OMPI -eq 1 ]
+then
+    check_exists "${MODULE_FILES_ROOT}/mpi/openmpi"
+    check_exists ${OPENMPI_PATH}
+fi
+
+# Check Cuda drivers by running Nvidia SMI
+if [ $CHECK_CUDA -eq 1 ]
+then
+    nvidia-smi
+    check_exit_code "Nvidia SMI - Cuda Drivers" "Failed to run Nvidia SMI - Cuda Drivers"
+fi
 
 echo "ALL OK!"
 

ubuntu/ubuntu-18.04-hpc/README.md

@@ -0,0 +1,25 @@
+# Ubuntu 18.04 HPC Image
+
+The Ubuntu 18.04 HPC Image includes optimizations and recommended configurations to deliver optimal performance,
+consistency, and reliability. This image consists of the following HPC tools and libraries:
+
+- Mellanox OFED
+- Pre-configured IPoIB (IP-over-InfiniBand)
+- Popular InfiniBand based MPI Libraries
+  - HPC-X
+  - IntelMPI
+  - MVAPICH2
+- Communication Runtimes
+  - Libfabric
+  - OpenUCX
+- Optimized librares
+  - AMD Blis
+  - AMD FFTW
+  - AMD Flame
+  - Intel MKL
+- GPU Drivers
+  - Nvidia GPU Driver
+
+Software packages are configured as environment modules. Users can select preferred MPI or software packages as follows:
+
+`module load <package-name>`