Set up CI

Copied from torchvision.

ghstack-source-id: 2158902438d9fef846d5fa1b93033135612879a9
Pull Request resolved: #87

.github/scripts/setup-env.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+# Prepare conda
+set +x && eval "$($(which conda) shell.bash hook)" && set -x
+
+# Setup the OS_TYPE environment variable that should be used for conditions involving the OS below.
+case $(uname) in
+  Linux)
+    OS_TYPE=linux
+    ;;
+  Darwin)
+    OS_TYPE=macos
+    ;;
+  MSYS*)
+    OS_TYPE=windows
+    ;;
+  *)
+    echo "Unknown OS type:" $(uname)
+    exit 1
+    ;;
+esac
+
+if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then
+  echo '::group::Uninstall system JPEG libraries on macOS'
+  # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG and PNG libraries
+  # installed by default that interfere with our build. We uninstall them here and use the one from conda below.
+  IMAGE_LIBS=$(brew list | grep -E "jpeg|png")
+  for lib in $IMAGE_LIBS; do
+    brew uninstall --ignore-dependencies --force "${lib}"
+  done
+  echo '::endgroup::'
+fi
+
+echo '::group::Create build environment'
+# See https://github.com/pytorch/vision/issues/7296 for ffmpeg
+conda create \
+  --name ci \
+  --quiet --yes \
+  python="${PYTHON_VERSION}" pip \
+  ninja cmake
+conda activate ci
+pip install --progress-bar=off --upgrade setuptools
+
+echo '::endgroup::'
+
+if [[ "${OS_TYPE}" == windows && "${GPU_ARCH_TYPE}" == cuda ]]; then
+  echo '::group::Install VisualStudio CUDA extensions on Windows'
+  if [[ "${VC_YEAR:-}" == "2022" ]]; then
+    TARGET_DIR="/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/MSBuild/Microsoft/VC/v170/BuildCustomizations"
+  else
+    TARGET_DIR="/c/Program Files (x86)/Microsoft Visual Studio/2019/BuildTools/MSBuild/Microsoft/VC/v160/BuildCustomizations"
+  fi
+  mkdir -p "${TARGET_DIR}"
+  cp -r "${CUDA_HOME}/MSBuildExtensions/"* "${TARGET_DIR}"
+  echo '::endgroup::'
+fi
+
+echo '::group::Install PyTorch'
+# TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`.
+if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
+  CHANNEL=test
+else
+  CHANNEL=nightly
+fi
+
+case $GPU_ARCH_TYPE in
+  cpu)
+    GPU_ARCH_ID="cpu"
+    ;;
+  cuda)
+    VERSION_WITHOUT_DOT=$(echo "${GPU_ARCH_VERSION}" | sed 's/\.//')
+    GPU_ARCH_ID="cu${VERSION_WITHOUT_DOT}"
+    ;;
+  *)
+    echo "Unknown GPU_ARCH_TYPE=${GPU_ARCH_TYPE}"
+    exit 1
+    ;;
+esac
+PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL}/${GPU_ARCH_ID}"
+pip install --progress-bar=off --pre torch --index-url="${PYTORCH_WHEEL_INDEX}"
+
+if [[ $GPU_ARCH_TYPE == 'cuda' ]]; then
+  python -c "import torch; exit(not torch.cuda.is_available())"
+fi
+echo '::endgroup::'
+
+echo '::group::Install third party dependencies prior to extension-cpp install'
+# Installing with `easy_install`, e.g. `python setup.py install` or `python setup.py develop`, has some quirks when
+# when pulling in third-party dependencies. For example:
+# - On Windows, we often hit an SSL error although `pip` can install just fine.
+# - It happily pulls in pre-releases, which can lead to more problems down the line.
+#   `pip` does not unless explicitly told to do so.
+# Thus, we use `easy_install` to extract the third-party dependencies here and install them upfront with `pip`.
+python setup.py egg_info
+# The requires.txt cannot be used with `pip install -r` directly. The requirements are listed at the top and the
+# optional dependencies come in non-standard syntax after a blank line. Thus, we just extract the header.
+sed -e '/^$/,$d' *.egg-info/requires.txt | tee requirements.txt
+pip install --progress-bar=off -r requirements.txt
+echo '::endgroup::'
+
+echo '::group::Install extension-cpp'
+python setup.py develop
+echo '::endgroup::'
+
+echo '::group::Collect environment information'
+conda list
+python -m torch.utils.collect_env
+echo '::endgroup::'

.github/scripts/unittest.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+./.github/scripts/setup-env.sh
+
+# Activate conda environment
+eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci
+
+echo '::group::Install testing utilities'
+pip install --progress-bar=off pytest pytest-mock pytest-cov expecttest numpy
+echo '::endgroup::'
+
+pytest test/ --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v

.github/workflows/tests.yml

@@ -0,0 +1,38 @@
+name: Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  unittests-linux:
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+        runner: ["linux.12xlarge"]
+        gpu-arch-type: ["cpu"]
+        include:
+          - python-version: 3.8
+            runner: linux.g5.4xlarge.nvidia.gpu
+            gpu-arch-type: cuda
+            gpu-arch-version: "11.8"
+      fail-fast: false
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    with:
+      repository: pytorch/extension-cpp
+      runner: ${{ matrix.runner }}
+      gpu-arch-type: ${{ matrix.gpu-arch-type }}
+      gpu-arch-version: ${{ matrix.gpu-arch-version }}
+      timeout: 120
+      script: |
+        set -euo pipefail
+
+        export PYTHON_VERSION=${{ matrix.python-version }}
+        export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
+        export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}
+
+        ./.github/scripts/unittest.sh

test/test_extension.py

@@ -62,6 +62,7 @@ def _opcheck(self, device):
     def test_opcheck_cpu(self):
         self._opcheck("cpu")
 
+    @unittest.skipIf(not torch.cuda.is_available(), "requires cuda")
     def test_opcheck_cuda(self):
         self._opcheck("cuda")