Benchmarks: Microbenchmark - Add MSCCL as basis of nccl-lat benchmark

microsoft · Dec 5, 2023 · a398837 · a398837
1 parent 254ea7f
commit a398837
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 1 deletion.
diff --git a/.gitmodules b/.gitmodules
@@ -21,3 +21,6 @@
 [submodule "third_party/gpu-burn"]
 	path = third_party/gpu-burn
 	url = https://github.com/wilicc/gpu-burn.git
+[submodule "msccl"]
+	path = msccl
+	url = https://github.com/Azure/msccl
diff --git a/dockerfile/cuda11.1.1.dockerfile b/dockerfile/cuda11.1.1.dockerfile
@@ -35,13 +35,15 @@ RUN apt-get update && \
     libavformat-dev \
     libavutil-dev \
     libcap2 \
+    libcurl4-openssl-dev \
     libnuma-dev \
     libpci-dev \
     libswresample-dev \
     libtinfo5 \
     libtool \
     lshw \
     net-tools \
+    nlohmann-json3-dev \
     openssh-client \
     openssh-server \
     pciutils \

diff --git a/dockerfile/cuda12.2.dockerfile b/dockerfile/cuda12.2.dockerfile
@@ -35,13 +35,15 @@ RUN apt-get update && \
     libavutil-dev \
     libboost-program-options-dev \
     libcap2 \
+    libcurl4-openssl-dev \
     libnuma-dev \
     libpci-dev \
     libswresample-dev \
     libtinfo5 \
     libtool \
     lshw \
     net-tools \
+    nlohmann-json3-dev \
     openssh-client \
     openssh-server \
     pciutils \

diff --git a/msccl b/msccl
diff --git a/superbench/config/azure_ndmv4.yaml b/superbench/config/azure_ndmv4.yaml
@@ -79,6 +79,9 @@ superbench:
         - name: mpi
           proc_num: 8
           node_num: 1
+          env:
+            LD_PRELOAD: '/usr/local/lib/msccl-executor-nccl/lib/libnccl.so:$LD_PRELOAD'
+            MSCCL_SCHEDULER: '/usr/local/lib/msccl-scheduler/lib/libmsccl-scheduler.so'
       parameters:
         maxbytes: 16M
         warmup_iters: 20

diff --git a/superbench/config/azure_ndv4.yaml b/superbench/config/azure_ndv4.yaml
@@ -85,6 +85,9 @@ superbench:
         - name: mpi
           proc_num: 8
           node_num: 1
+          env:
+            LD_PRELOAD: '/usr/local/lib/msccl-executor-nccl/lib/libnccl.so:$LD_PRELOAD'
+            MSCCL_SCHEDULER: '/usr/local/lib/msccl-scheduler/lib/libmsccl-scheduler.so'
       parameters:
         maxbytes: 16M
         warmup_iters: 20

diff --git a/third_party/Makefile b/third_party/Makefile
@@ -15,7 +15,7 @@ ROCBLAS_BRANCH ?= rocm-$(shell dpkg -l | grep 'rocm-dev ' | awk '{print $$3}' |
 
 # Build all targets.
 all: cuda rocm
-cuda: common cuda_cutlass cuda_bandwidthTest cuda_nccl_tests cuda_perftest gpcnet cuda_gpuburn
+cuda: common cuda_cutlass cuda_bandwidthTest cuda_nccl_tests cuda_perftest gpcnet cuda_gpuburn cuda_msccl
 rocm: common rocm_perftest rocm_rccl_tests rocm_rocblas rocm_bandwidthTest rocm_hipblaslt
 cpu: common cpu_perftest
 common: cpu_hpl cpu_stream fio
@@ -171,3 +171,26 @@ directx_amf_encoding_latency:
 		del vs_buildtools.exe && echo "Deleted vs_buildtools.exe" && \
 		"C:\temp\BuildTools\MSBuild\Current\Bin\MSBuild.exe" "AMF\amf\public\samples\CPPSamples_vs2019.sln" /t:EncoderLatency /p:Platform=x64 /p:Configuration=Release /p:OutDir="%SB_MICRO_PATH%\bin" \
 	)
+
+# Build MSCCL for CUDA
+cuda_msccl: sb_micro_path
+	ifneq (,$(wildcard msccl/executor/msccl-executor-nccl/Makefile))
+		cd ./msccl/executor/msccl-executor-nccl && \
+		make -j src.build && \
+		cd ../../..
+		mkdir -p $(SB_MICRO_PATH)/lib/msccl-executor-nccl && \
+		cp -r -v ./msccl/executor/msccl-executor-nccl/build/* $(SB_MICRO_PATH)/lib/msccl-executor-nccl/
+	endif
+	ifneq (,$(wildcard msccl/scheduler/msccl-scheduler/Makefile))
+		cd ./msccl/scheduler/msccl-scheduler && \
+		CXX=nvcc BIN_HOME=$(SB_MICRO_PATH)/lib/msccl-executor-nccl SRC_HOME=./msccl/executor/msccl-executor-nccl make -j && \
+		cd ../../..
+		mkdir -p $(SB_MICRO_PATH)/lib/msccl-scheduler && \
+		cp -r -v ./msccl/scheduler/msccl-scheduler/build/* $(SB_MICRO_PATH)/lib/msccl-scheduler/
+	endif
+	ifneq (,$(wildcard msccl/tests/msccl-tests-nccl/Makefile))
+		cd ./msccl/tests/msccl-tests-nccl && \
+		make MPI=1 MPI_HOME=$(MPI_HOME) NCCL_HOME=$(SB_MICRO_PATH)/lib/msccl-executor-nccl -j && cd ../../..
+		mkdir -p $(SB_MICRO_PATH)/bin/msccl-tests-nccl && \
+		cp -v ./msccl/tests/msccl-tests-nccl/build/* $(SB_MICRO_PATH)/bin/msccl-tests-nccl/
+	endif