Merge branch 'main' into yutji/data-diagnosis-fix

microsoft · Aug 13, 2024 · 531e575 · 531e575
2 parents 86a25ec + 9de841b
commit 531e575
Show file tree

Hide file tree

Showing 15 changed files with 369 additions and 224 deletions.
diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
@@ -18,7 +18,7 @@ jobs:
   docker:
     name: Docker build ${{ matrix.name }}
     runs-on: ${{ matrix.runner }}
-    timeout-minutes: 600
+    timeout-minutes: 1200
     permissions:
       contents: read
       packages: write
@@ -29,12 +29,12 @@ jobs:
           dockerfile: cuda12.4
           tags: superbench/main:cuda12.4
           runner: [self-hosted, rocm-build]
-          build_args: "NUM_MAKE_JOBS=32"
+          build_args: "NUM_MAKE_JOBS=8"
         - name: cuda12.2
           dockerfile: cuda12.2
           tags: superbench/main:cuda12.2
           runner: [self-hosted, rocm-build]
-          build_args: "NUM_MAKE_JOBS=64"
+          build_args: "NUM_MAKE_JOBS=8"
         - name: cuda11.1.1
           dockerfile: cuda11.1.1
           tags: superbench/main:cuda11.1.1,superbench/superbench:latest

diff --git a/CITATION.bib b/CITATION.bib
@@ -0,0 +1,12 @@
+@inproceedings {superbench,
+	author = {Yifan Xiong and Yuting Jiang and Ziyue Yang and Lei Qu and Guoshuai Zhao and Shuguang Liu and Dong Zhong and Boris Pinzur and Jie Zhang and Yang Wang and Jithin Jose and Hossein Pourreza and Jeff Baxter and Kushal Datta and Prabhat Ram and Luke Melton and Joe Chau and Peng Cheng and Yongqiang Xiong and Lidong Zhou},
+	title = {{SuperBench}: Improving Cloud {AI} Infrastructure Reliability with Proactive Validation},
+	booktitle = {2024 USENIX Annual Technical Conference (USENIX ATC 24)},
+	year = {2024},
+	isbn = {978-1-939133-41-0},
+	address = {Santa Clara, CA},
+	pages = {835--850},
+	url = {https://www.usenix.org/conference/atc24/presentation/xiong},
+	publisher = {USENIX Association},
+	month = jul
+}
diff --git a/README.md b/README.md
@@ -19,6 +19,25 @@ __SuperBench__ is a validation and profiling tool for AI infrastructure.
 
 ## _Check [aka.ms/superbench](https://aka.ms/superbench) for more details._
 
+## Citations
+
+To cite SuperBench in your publications:
+
+```bib
+@inproceedings {superbench,
+	author = {Yifan Xiong and Yuting Jiang and Ziyue Yang and Lei Qu and Guoshuai Zhao and Shuguang Liu and Dong Zhong and Boris Pinzur and Jie Zhang and Yang Wang and Jithin Jose and Hossein Pourreza and Jeff Baxter and Kushal Datta and Prabhat Ram and Luke Melton and Joe Chau and Peng Cheng and Yongqiang Xiong and Lidong Zhou},
+	title = {{SuperBench}: Improving Cloud {AI} Infrastructure Reliability with Proactive Validation},
+	booktitle = {2024 USENIX Annual Technical Conference (USENIX ATC 24)},
+	year = {2024},
+	isbn = {978-1-939133-41-0},
+	address = {Santa Clara, CA},
+	pages = {835--850},
+	url = {https://www.usenix.org/conference/atc24/presentation/xiong},
+	publisher = {USENIX Association},
+	month = jul
+}
+```
+
 ## Trademarks
 
 This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft

diff --git a/dockerfile/rocm5.7.x.dockerfile b/dockerfile/rocm5.7.x.dockerfile
@@ -133,7 +133,7 @@ RUN cd /tmp && \
 
 # Install RCCL
 RUN cd /opt/ &&  \
-    git clone https://github.com/ROCmSoftwarePlatform/rccl.git && \
+    git clone -b release/rocm-rel-5.7 https://github.com/ROCmSoftwarePlatform/rccl.git && \
     cd rccl && \
     mkdir build && \
     cd build && \
@@ -167,6 +167,17 @@ ADD third_party third_party
 # Apply patch
 RUN cd third_party/perftest && \
     git apply ../perftest_rocm6.patch
+
+# Update package index and install dependencies of ROCBLAS
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository universe && \
+    apt-get update && \
+    apt-get install -y --fix-missing \
+    make python3 python3-yaml python3-venv python3-joblib 'python3*-pip' libmsgpack-dev gfortran libomp-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
 RUN make RCCL_HOME=/opt/rccl/build/ ROCBLAS_BRANCH=release/rocm-rel-5.7.1.1 HIPBLASLT_BRANCH=release/rocm-rel-5.7 ROCM_VER=rocm-5.5.0 -C third_party rocm -o cpu_hpl -o cpu_stream -o megatron_lm
 
 ADD . .

diff --git a/dockerfile/rocm6.0.x.dockerfile b/dockerfile/rocm6.0.x.dockerfile
@@ -139,7 +139,7 @@ RUN cd /tmp && \
 
 # Install RCCL
 RUN cd /opt/ &&  \
-    git clone https://github.com/ROCmSoftwarePlatform/rccl.git && \
+    git clone -b release/rocm-rel-6.0 https://github.com/ROCmSoftwarePlatform/rccl.git && \
     cd rccl && \
     mkdir build && \
     cd build && \

diff --git a/docs/cli.md b/docs/cli.md
@@ -111,7 +111,7 @@ sb deploy [--docker-image]
 | `--docker-username`   | `None`                  | Docker registry username if authentication is needed.                             |
 | `--host-file` `-f`    | `None`                  | Path to Ansible inventory host file.                                              |
 | `--host-list` `-l`    | `None`                  | Comma separated host list.                                                        |
-| `--host-password`     | `None`                  | Host password or key passphase if needed.                                         |
+| `--host-password`     | `None`                  | Host password or key passphrase if needed.                                        |
 | `--host-username`     | `None`                  | Host username if needed.                                                          |
 | `--no-image-pull`     | `False`                 | Skip pull and use local Docker image.                                             |
 | `--output-dir`        | `None`                  | Path to output directory, outputs/{datetime} will be used if not specified.       |
@@ -373,7 +373,7 @@ sb run [--config-file]
 | `--get-info`             | `False`                 | Collect system info.                                                        |
 | `--host-file` `-f`       | `None`                  | Path to Ansible inventory host file.                                        |
 | `--host-list` `-l`       | `None`                  | Comma separated host list.                                                  |
-| `--host-password`        | `None`                  | Host password or key passphase if needed.                                   |
+| `--host-password`        | `None`                  | Host password or key passphrase if needed.                                  |
 | `--host-username`        | `None`                  | Host username if needed.                                                    |
 | `--no-docker`            | `False`                 | Run on host directly without Docker.                                        |
 | `--output-dir`           | `None`                  | Path to output directory, outputs/{datetime} will be used if not specified. |

diff --git a/docs/superbench-config.mdx b/docs/superbench-config.mdx
@@ -295,7 +295,7 @@ Enable current benchmark or not, can be overwritten by [`superbench.enable`](#su
 
 ### `timeout`
 
-Set the timeout value in seconds, the benchmarking will stop early if timeout is triggerred.
+Set the timeout value in seconds, the benchmarking will stop early if timeout is triggered.
 
 * default value: none
 
@@ -336,16 +336,16 @@ A list of models to run, only supported in model-benchmark.
 
 Parameters for benchmark to use, varying for different benchmarks.
 
-There have four common parameters for all benchmarks:
-* run_count: how many times do user want to run this benchmark, default value is 1.
+There are four common parameters for all benchmarks:
+* run_count: how many times does user want to run this benchmark, default value is 1.
 * duration: the elapsed time of benchmark in seconds. It can work for all model-benchmark. But for micro-benchmark, benchmark authors should consume it by themselves.
 * log_raw_data: log raw data into file instead of saving it into result object, default value is `False`.  Benchmarks who have large raw output may want to set it as `True`, such as `nccl-bw`/`rccl-bw`.
 * log_flushing: real-time log flushing, default value is `False`.
 
-For Model-Benchmark, there have some parameters that can control the elapsed time.
+For Model-Benchmark, there are some parameters that can control the elapsed time.
 * duration: the elapsed time of benchmark in seconds.
-* num_warmup: the number of warmup step, should be positive integer.
-* num_steps: the number of test step.
+* num_warmup: the number of warmup steps, should be positive integer.
+* num_steps: the number of test steps.
 
 If `duration > 0` and `num_steps > 0`, then benchmark will take the least as the elapsed time. Otherwise only one of them will take effect.
 
@@ -429,7 +429,7 @@ while `proc_num: 8, node_num: null` will run 32-GPU distributed training on all
 
 Command prefix to use in the mode, in Python formatted string.
 
-Available variables in formatted string includes:
+Available variables in formatted string include:
 + `proc_rank`
 + `proc_num`
 

diff --git a/setup.py b/setup.py
@@ -164,7 +164,7 @@ def run(self):
         'natsort>=7.1.1',
         'networkx>=2.5',
         'numpy>=1.19.2',
-        'omegaconf==2.0.6',
+        'omegaconf==2.3.0',
         'openpyxl>=3.0.7',
         'packaging>=21.0',
         'pandas>=1.1.5',
@@ -198,9 +198,9 @@ def run(self):
                 'pydocstyle>=5.1.1',
                 'pytest-cov>=2.11.1',
                 'pytest-subtests>=0.4.0',
-                'pytest>=6.2.2',
+                'pytest>=6.2.2, <=7.4.4',
                 'types-markdown',
-                'types-pkg_resources',
+                'types-setuptools',
                 'types-pyyaml',
                 'typing-extensions>=3.10',
                 'urllib3<2.0',

diff --git a/superbench/benchmarks/micro_benchmarks/cuda_common.cmake b/superbench/benchmarks/micro_benchmarks/cuda_common.cmake
@@ -33,6 +33,6 @@ if(NOT DEFINED NVCC_ARCHS_SUPPORTED)
       list(APPEND NVCC_ARCHS_SUPPORTED 86)
     endif()
     if (NOT CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.8)
-      list(APPEND NVCC_ARCHS_SUPPORTED 90)
+      list(APPEND NVCC_ARCHS_SUPPORTED 89 90)
     endif()
 endif()
diff --git a/superbench/benchmarks/micro_benchmarks/cuda_gemm_flops_performance.py b/superbench/benchmarks/micro_benchmarks/cuda_gemm_flops_performance.py
@@ -47,6 +47,8 @@ def __init__(self, name, parameters=''):
         # Skip FP64 for RTX Turing/Ampere and Tesla T4/GA10x due to very limited FP64 TFLOP rate
         self.__kernel_map[7.5] = {k: self.__kernel_map[7.0][k] for k in self.__kernel_map[7.0] if 'fp64' not in k}
         self.__kernel_map[8.6] = {k: self.__kernel_map[8.0][k] for k in self.__kernel_map[8.0] if 'fp64' not in k}
+        # Skip FP64 for Ada Lovelace L4/L40 due to no native CUDA/Tensor Cores
+        self.__kernel_map[8.9] = {k: self.__kernel_map[8.0][k] for k in self.__kernel_map[8.0] if 'fp64' not in k}
         # Skip INT4 for Hopper due to no native CUDA/Tensor Cores
         self.__kernel_map[9.0] = {k: self.__kernel_map[8.0][k] for k in self.__kernel_map[8.0] if 'int4_tc' not in k}
         self.__parse_logline = [

diff --git a/superbench/executor/executor.py b/superbench/executor/executor.py
@@ -71,13 +71,13 @@ def __get_enabled_benchmarks(self):
         Return:
             list: List of benchmarks which will be executed.
         """
-        if self._sb_config.superbench.enable:
+        if 'enable' in self._sb_config.superbench and self._sb_config.superbench.enable:
             if isinstance(self._sb_config.superbench.enable, str):
                 return [self._sb_config.superbench.enable]
             elif isinstance(self._sb_config.superbench.enable, (list, ListConfig)):
                 return list(self._sb_config.superbench.enable)
         # TODO: may exist order issue
-        return [k for k, v in self._sb_benchmarks.items() if v.enable]
+        return [k for k, v in self._sb_benchmarks.items() if 'enable' in v and v.enable]
 
     def __get_platform(self):
         """Detect runninng platform by environment."""
@@ -228,32 +228,37 @@ def exec(self):
                     logger.warning('Monitor can not support CPU platform.')
 
             benchmark_real_name = benchmark_name.split(':')[0]
-            for framework in benchmark_config.frameworks or [Framework.NONE.value]:
-                if benchmark_real_name == 'model-benchmarks' or (
-                    ':' not in benchmark_name and benchmark_name.endswith('_models')
-                ):
-                    for model in benchmark_config.models:
-                        full_name = f'{benchmark_name}/{framework}-{model}'
+            if 'frameworks' in benchmark_config:
+                for framework in benchmark_config.frameworks or [Framework.NONE.value]:
+                    if benchmark_real_name == 'model-benchmarks' or (
+                        ':' not in benchmark_name and benchmark_name.endswith('_models')
+                    ):
+                        for model in benchmark_config.models:
+                            full_name = f'{benchmark_name}/{framework}-{model}'
+                            logger.info('Executor is going to execute %s.', full_name)
+                            context = BenchmarkRegistry.create_benchmark_context(
+                                model,
+                                platform=self.__get_platform(),
+                                framework=Framework(framework.lower()),
+                                parameters=self.__get_arguments(
+                                    {} if 'parameters' not in benchmark_config else benchmark_config.parameters
+                                )
+                            )
+                            result = self.__exec_benchmark(full_name, context)
+                            benchmark_results.append(result)
+                    else:
+                        full_name = benchmark_name
                         logger.info('Executor is going to execute %s.', full_name)
                         context = BenchmarkRegistry.create_benchmark_context(
-                            model,
+                            benchmark_real_name,
                             platform=self.__get_platform(),
                             framework=Framework(framework.lower()),
-                            parameters=self.__get_arguments(benchmark_config.parameters)
+                            parameters=self.__get_arguments(
+                                {} if 'parameters' not in benchmark_config else benchmark_config.parameters
+                            )
                         )
                         result = self.__exec_benchmark(full_name, context)
                         benchmark_results.append(result)
-                else:
-                    full_name = benchmark_name
-                    logger.info('Executor is going to execute %s.', full_name)
-                    context = BenchmarkRegistry.create_benchmark_context(
-                        benchmark_real_name,
-                        platform=self.__get_platform(),
-                        framework=Framework(framework.lower()),
-                        parameters=self.__get_arguments(benchmark_config.parameters)
-                    )
-                    result = self.__exec_benchmark(full_name, context)
-                    benchmark_results.append(result)
 
             if monitor:
                 monitor.stop()