diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index a0bf14fffe8..d48d5815971 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -58,7 +58,7 @@ jobs: - name: Set up Python version uses: actions/setup-python@v4 with: - python-version: '3.7' + python-version: '3.8' - name: Install dependencies env: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 516e72df47b..bfd53f70547 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -56,7 +56,7 @@ jobs: - name: Set up Python version uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Install dependencies run: | brew install ccache pkg-config @@ -127,16 +127,16 @@ jobs: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' env: BUILD_CUDA_MODULE: OFF @@ -234,16 +234,16 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' env: OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 740ede78b16..f6dbcac232a 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python version uses: actions/setup-python@v4 with: - python-version: '3.7' + python-version: '3.8' - name: Install dependencies run: | pip install -U clang-format~=10.0.0 yapf==0.30.0 nbformat diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index 0c42b06e67b..16cce0b8272 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -19,6 +19,9 @@ concurrency: env: GCE_CLI_GHA_VERSION: '416.0.0' # Fixed to avoid dependency on API changes + BUILD_CUDA_MODULE: 'ON' + BUILD_PYTORCH_OPS: 'ON' + BUILD_TENSORFLOW_OPS: 'OFF' # Turn ON when cxx11_abi is same for TF and PyTorch jobs: build-wheel: @@ -27,16 +30,16 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' env: DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} PYTHON_VERSION: ${{ matrix.python_version }} @@ -53,22 +56,22 @@ jobs: # `docker/docker_build.sh xxx` command to execute locally. - name: Docker build run: | - if [ "${{ env.PYTHON_VERSION }}" = "3.7" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then - docker/docker_build.sh cuda_wheel_py37_dev - elif [ "${{ env.PYTHON_VERSION }}" = "3.8" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then + if [ "${{ env.PYTHON_VERSION }}" = "3.8" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py38_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.9" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py39_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.10" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py310_dev - elif [ "${{ env.PYTHON_VERSION }}" = "3.7" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then - docker/docker_build.sh cuda_wheel_py37 + elif [ "${{ env.PYTHON_VERSION }}" = "3.11" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then + docker/docker_build.sh cuda_wheel_py311_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.8" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then docker/docker_build.sh cuda_wheel_py38 elif [ "${{ env.PYTHON_VERSION }}" = "3.9" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then docker/docker_build.sh cuda_wheel_py39 elif [ "${{ env.PYTHON_VERSION }}" = "3.10" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then docker/docker_build.sh cuda_wheel_py310 + elif [ "${{ env.PYTHON_VERSION }}" = "3.11" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then + docker/docker_build.sh cuda_wheel_py311 fi PIP_PKG_NAME="$(basename ${GITHUB_WORKSPACE}/open3d-[0-9]*.whl)" PIP_CPU_PKG_NAME="$(basename ${GITHUB_WORKSPACE}/open3d_cpu*.whl)" @@ -114,16 +117,16 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' env: OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML steps: diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 66fc9aacb98..e1c4d0b8b44 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -35,8 +35,6 @@ jobs: env: BUILD_SHARED_LIBS: ${{ matrix.BUILD_SHARED_LIBS }} BUILD_CUDA_MODULE: OFF - BUILD_TENSORFLOW_OPS: ${{ matrix.MLOPS }} - BUILD_PYTORCH_OPS: ${{ matrix.MLOPS }} DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} OPEN3D_CPU_RENDERING: true steps: diff --git a/.github/workflows/vtk_packages.yml b/.github/workflows/vtk_packages.yml index bcde81876f0..30347cc2941 100644 --- a/.github/workflows/vtk_packages.yml +++ b/.github/workflows/vtk_packages.yml @@ -9,6 +9,7 @@ on: jobs: Linux: + # TODO: Convert to docker runs-on: ubuntu-18.04 steps: - name: Checkout source code diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 3e13d7854c1..66c7e5ba084 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -19,15 +19,11 @@ concurrency: cancel-in-progress: true env: - PIP_VER: "21.1.1" + PIP_VER: "23.2.1" WHEEL_VER: "0.38.4" STOOLS_VER: "67.3.2" - PYTEST_VER: "7.1.2" - PYTEST_RANDOMLY_VER: "3.8.0" - SCIPY_VER: "1.7.3" JEDI_VER: "0.17.2" # https://github.com/ipython/ipython/issues/12740 IDNA_VER: "2.8" # https://github.com/psf/requests/issues/5710 - TENSORBOARD_VER: "2.8" SRC_DIR: "D:\\a\\open3d\\open3d" BUILD_DIR: "C:\\Open3D\\build" NPROC: 2 @@ -101,7 +97,7 @@ jobs: - name: Set up Python version uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Config # Move build directory to C: https://github.com/actions/virtual-environments/issues/1341 @@ -246,16 +242,16 @@ jobs: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' steps: - name: Checkout source code @@ -351,16 +347,16 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' steps: - name: Checkout source code @@ -388,10 +384,7 @@ jobs: python -m pip install --upgrade pip==${{ env.PIP_VER }} ` wheel==${{ env.WHEEL_VER }} ` setuptools==${{ env.STOOLS_VER }} - python -m pip install -U pytest==${{ env.PYTEST_VER }} - python -m pip install -U pytest-randomly==${{ env.PYTEST_RANDOMLY_VER }} - python -m pip install -U scipy==${{ env.SCIPY_VER }} ` - tensorboard==${{ env.TENSORBOARD_VER }} + python -m pip install -U -r python/requirements_test.txt $py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") if (Test-Path -Path "pip_package") { $PIP_PKG_NAME=(Get-ChildItem pip_package\open3d*-$py_tag-*.whl).Name diff --git a/.gitignore b/.gitignore index cb1dbd7c1a6..8a041878cc6 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ docs/tutorial/**/*.ply docs/tutorial/**/*.pcd docs/tutorial/**/*.json docs/_out/ +docs/_build/ docs/python_api/ docs/python_example/ docs/conf.py diff --git a/3rdparty/cmake/FindPytorch.cmake b/3rdparty/cmake/FindPytorch.cmake index 202af7c461a..eb2a53e2ec5 100644 --- a/3rdparty/cmake/FindPytorch.cmake +++ b/3rdparty/cmake/FindPytorch.cmake @@ -97,18 +97,43 @@ foreach(lib ${TORCH_LIBRARIES}) endforeach(lib) # Check if the c++11 ABI is compatible on Linux -if(UNIX AND NOT APPLE AND ((Pytorch_CXX11_ABI AND (NOT GLIBCXX_USE_CXX11_ABI)) OR - (NOT Pytorch_CXX11_ABI AND GLIBCXX_USE_CXX11_ABI))) - if(Pytorch_CXX11_ABI) - set(NEEDED_ABI_FLAG "ON") +if(UNIX AND NOT APPLE) + if((Pytorch_CXX11_ABI AND (NOT GLIBCXX_USE_CXX11_ABI)) OR + (NOT Pytorch_CXX11_ABI AND GLIBCXX_USE_CXX11_ABI)) + if(Pytorch_CXX11_ABI) + set(NEEDED_ABI_FLAG "ON") + else() + set(NEEDED_ABI_FLAG "OFF") + endif() + message(FATAL_ERROR "PyTorch and Open3D ABI mismatch: ${Pytorch_CXX11_ABI} != ${GLIBCXX_USE_CXX11_ABI}.\n" + "Please use -DGLIBCXX_USE_CXX11_ABI=${NEEDED_ABI_FLAG} " + "in the cmake config command to change the Open3D ABI.") else() - set(NEEDED_ABI_FLAG "OFF") + message(STATUS "PyTorch matches Open3D ABI: ${Pytorch_CXX11_ABI} == ${GLIBCXX_USE_CXX11_ABI}") endif() - message(FATAL_ERROR "PyTorch and Open3D ABI mismatch: ${Pytorch_CXX11_ABI} != ${GLIBCXX_USE_CXX11_ABI}.\n" - "Please use -DGLIBCXX_USE_CXX11_ABI=${NEEDED_ABI_FLAG} " - "in the cmake config command to change the Open3D ABI.") -else() - message(STATUS "PyTorch matches Open3D ABI: ${Pytorch_CXX11_ABI} == ${GLIBCXX_USE_CXX11_ABI}") +endif() + +message(STATUS "Pytorch_VERSION: ${Pytorch_VERSION}, CUDAToolkit_VERSION: ${CUDAToolkit_VERSION}") +if (BUILD_PYTORCH_OPS AND BUILD_CUDA_MODULE AND CUDAToolkit_VERSION + VERSION_GREATER_EQUAL "11.0" AND Pytorch_VERSION VERSION_LESS + "1.9") + message(WARNING + "--------------------------------------------------------------------------------\n" + " \n" + " You are compiling PyTorch ops with CUDA 11 with PyTorch version < 1.9. This \n" + " configuration may have stability issues. See \n" + " https://github.com/isl-org/Open3D/issues/3324 and \n" + " https://github.com/pytorch/pytorch/issues/52663 for more information on this \n" + " problem. \n" + " \n" + " We recommend to compile PyTorch from source with compile flags \n" + " '-Xcompiler -fno-gnu-unique' \n" + " \n" + " or use the PyTorch wheels at \n" + " https://github.com/isl-org/open3d_downloads/releases/tag/torch1.8.2 \n" + " \n" + "--------------------------------------------------------------------------------\n" + ) endif() include(FindPackageHandleStandardArgs) diff --git a/3rdparty/cmake/FindTensorflow.cmake b/3rdparty/cmake/FindTensorflow.cmake index 815fa497cde..6dc3d3676a2 100644 --- a/3rdparty/cmake/FindTensorflow.cmake +++ b/3rdparty/cmake/FindTensorflow.cmake @@ -43,7 +43,7 @@ if(NOT Tensorflow_FOUND) # Get Tensorflow_FRAMEWORK_LIB find_library( Tensorflow_FRAMEWORK_LIB - NAMES tensorflow_framework libtensorflow_framework.so.2 + NAMES tensorflow_framework tensorflow_framework.2 libtensorflow_framework.so.2 PATHS "${Tensorflow_LIB_DIR}" NO_DEFAULT_PATH ) @@ -59,21 +59,23 @@ if (UNIX AND NOT APPLE) endif() # Check if the c++11 ABI is compatible -if(UNIX AND NOT APPLE AND ((Tensorflow_CXX11_ABI AND (NOT GLIBCXX_USE_CXX11_ABI)) OR - (NOT Tensorflow_CXX11_ABI AND GLIBCXX_USE_CXX11_ABI))) - if(TensorFlow_CXX11_ABI) - set(NEEDED_ABI_FLAG "ON") +if (UNIX AND NOT APPLE) + if(((Tensorflow_CXX11_ABI AND (NOT GLIBCXX_USE_CXX11_ABI)) OR + (NOT Tensorflow_CXX11_ABI AND GLIBCXX_USE_CXX11_ABI))) + if(TensorFlow_CXX11_ABI) + set(NEEDED_ABI_FLAG "ON") + else() + set(NEEDED_ABI_FLAG "OFF") + endif() + message(FATAL_ERROR "TensorFlow and Open3D ABI mismatch: ${Tensorflow_CXX11_ABI} != ${GLIBCXX_USE_CXX11_ABI}.\n" + "Please use -D GLIBCXX_USE_CXX11_ABI=${NEEDED_ABI_FLAG} " + "in the cmake config command to change the Open3D ABI.") else() - set(NEEDED_ABI_FLAG "OFF") + message(STATUS "TensorFlow matches Open3D ABI: ${Tensorflow_CXX11_ABI} == ${GLIBCXX_USE_CXX11_ABI}") endif() - message(FATAL_ERROR "TensorFlow and Open3D ABI mismatch: ${Tensorflow_CXX11_ABI} != ${GLIBCXX_USE_CXX11_ABI}.\n" - "Please use -D GLIBCXX_USE_CXX11_ABI=${NEEDED_ABI_FLAG} " - "in the cmake config command to change the Open3D ABI.") -else() - message(STATUS "TensorFlow matches Open3D ABI: ${Tensorflow_CXX11_ABI} == ${GLIBCXX_USE_CXX11_ABI}") endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args( Tensorflow DEFAULT_MSG Tensorflow_INCLUDE_DIR Tensorflow_LIB_DIR - Tensorflow_FRAMEWORK_LIB Tensorflow_DEFINITIONS) + Tensorflow_DEFINITIONS) diff --git a/3rdparty/pybind11/pybind11.cmake b/3rdparty/pybind11/pybind11.cmake index 5ed710aa58b..9a9c1b6df77 100644 --- a/3rdparty/pybind11/pybind11.cmake +++ b/3rdparty/pybind11/pybind11.cmake @@ -3,8 +3,8 @@ include(FetchContent) FetchContent_Declare( ext_pybind11 PREFIX pybind11 - URL https://github.com/pybind/pybind11/archive/refs/tags/v2.6.2.tar.gz - URL_HASH SHA256=8ff2fff22df038f5cd02cea8af56622bc67f5b64534f1b83b9f133b8366acff2 + URL https://github.com/pybind/pybind11/archive/refs/tags/v2.11.1.tar.gz + URL_HASH SHA256=d475978da0cdc2d43b73f30910786759d593a9d8ee05b1b6846d1eb16c6d2e0c DOWNLOAD_DIR "${OPEN3D_THIRD_PARTY_DOWNLOAD_DIR}/pybind11" ) diff --git a/3rdparty/vtk/CMakeLists.txt b/3rdparty/vtk/CMakeLists.txt index 7188cb76325..dd74c820c86 100644 --- a/3rdparty/vtk/CMakeLists.txt +++ b/3rdparty/vtk/CMakeLists.txt @@ -30,7 +30,7 @@ if(WIN32) endif() if (APPLE) -set (CMAKE_OSX_DEPLOYMENT_TARGET "10.15" CACHE STRING +set (CMAKE_OSX_DEPLOYMENT_TARGET "12.6" CACHE STRING "Minimum OS X deployment version" FORCE) endif() diff --git a/CHANGELOG.md b/CHANGELOG.md index 96bedc8af1f..15e4d6cf5ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ * Support msgpack versions without cmake * Support multi-threading in the RayCastingScene function to commit scene (PR #6051). * Fix some bad triangle generation in TriangleMesh::SimplifyQuadricDecimation +* Python 3.11 support. bump pybind11 v2.6.2 -> v2.11.1 ## 0.13 diff --git a/CMakeLists.txt b/CMakeLists.txt index e8c97dbf848..c3d64e4b95d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -290,11 +290,14 @@ endif() # Global flag to set CXX standard. # This does not affect 3rd party libraries. -if (BUILD_SYCL_MODULE) +# Tensorflow 2.9+ requires cxx_17, but MSVC 19.29 throws errors with C++17 +# enabled. +if (BUILD_SYCL_MODULE OR BUILD_TENSORFLOW_OPS) set(CMAKE_CXX_STANDARD 17) else() set(CMAKE_CXX_STANDARD 14) endif() +set(CMAKE_CXX_EXTENSIONS OFF) # Improved compatibility # FIXME: Remove this workaround once a fixed Visual Studio 16.10 version is released. if (BUILD_CUDA_MODULE diff --git a/cmake/Open3DPrintConfigurationSummary.cmake b/cmake/Open3DPrintConfigurationSummary.cmake index 9427d08c0c7..c8e8a891a8b 100644 --- a/cmake/Open3DPrintConfigurationSummary.cmake +++ b/cmake/Open3DPrintConfigurationSummary.cmake @@ -40,27 +40,6 @@ function(open3d_print_configuration_summary) open3d_aligned_print("Build Jupyter Extension" "${BUILD_JUPYTER_EXTENSION}") open3d_aligned_print("Build TensorFlow Ops" "${BUILD_TENSORFLOW_OPS}") open3d_aligned_print("Build PyTorch Ops" "${BUILD_PYTORCH_OPS}") - if (BUILD_PYTORCH_OPS AND BUILD_CUDA_MODULE AND CUDAToolkit_VERSION - VERSION_GREATER_EQUAL "11.0" AND Pytorch_VERSION VERSION_LESS - "1.9") - message(WARNING - "--------------------------------------------------------------------------------\n" - " \n" - " You are compiling PyTorch ops with CUDA 11 with PyTorch version < 1.9. This \n" - " configuration may have stability issues. See \n" - " https://github.com/isl-org/Open3D/issues/3324 and \n" - " https://github.com/pytorch/pytorch/issues/52663 for more information on this \n" - " problem. \n" - " \n" - " We recommend to compile PyTorch from source with compile flags \n" - " '-Xcompiler -fno-gnu-unique' \n" - " \n" - " or use the PyTorch wheels at \n" - " https://github.com/isl-org/open3d_downloads/releases/tag/torch1.8.2 \n" - " \n" - "--------------------------------------------------------------------------------\n" - ) - endif() open3d_aligned_print("Build Benchmarks" "${BUILD_BENCHMARKS}") open3d_aligned_print("Bundle Open3D-ML" "${BUNDLE_OPEN3D_ML}") if(GLIBCXX_USE_CXX11_ABI) diff --git a/cpp/open3d/ml/tensorflow/CMakeLists.txt b/cpp/open3d/ml/tensorflow/CMakeLists.txt index 756e76b07b5..cecd4152c49 100644 --- a/cpp/open3d/ml/tensorflow/CMakeLists.txt +++ b/cpp/open3d/ml/tensorflow/CMakeLists.txt @@ -136,6 +136,9 @@ set_target_properties(open3d_tf_ops PROPERTIES set_target_properties(open3d_tf_ops PROPERTIES PREFIX "") set_target_properties(open3d_tf_ops PROPERTIES DEBUG_POSTFIX "_debug") +# _GLIBCXX_USER_CXX11_ABI is set separately +list(REMOVE_ITEM Tensorflow_DEFINITIONS "_GLIBCXX_USE_CXX11_ABI=0" + "_GLIBCXX_USE_CXX11_ABI=1") target_compile_definitions(open3d_tf_ops PRIVATE "${Tensorflow_DEFINITIONS}") if (BUILD_CUDA_MODULE) diff --git a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvBackpropFilterOps.cpp b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvBackpropFilterOps.cpp index 3fa594026e2..06a4df2b463 100644 --- a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvBackpropFilterOps.cpp +++ b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvBackpropFilterOps.cpp @@ -120,8 +120,8 @@ REGISTER_OP("Open3DContinuousConvBackpropFilter") if (c->ValueKnown(c->Dim(filters_shape, i))) { int64_t n = c->Value(c->Dim(filters_shape, i)); if (n < 1) - return Status(error::INVALID_ARGUMENT, - "Each filter dimension must be >= 1"); + return absl::InvalidArgumentError( + "Each filter dimension must be >= 1"); } } diff --git a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvOps.cpp b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvOps.cpp index 154ed9e0813..a86f9dc7396 100644 --- a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvOps.cpp +++ b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvOps.cpp @@ -116,8 +116,8 @@ REGISTER_OP("Open3DContinuousConv") if (c->ValueKnown(c->Dim(filters_shape, i))) { int64_t n = c->Value(c->Dim(filters_shape, i)); if (n < 1) - return Status(error::INVALID_ARGUMENT, - "Each filter dimension must be >= 1"); + return absl::InvalidArgumentError( + "Each filter dimension must be >= 1"); } } diff --git a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeBackpropFilterOps.cpp b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeBackpropFilterOps.cpp index ae13c7fd12c..613224b562c 100644 --- a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeBackpropFilterOps.cpp +++ b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeBackpropFilterOps.cpp @@ -139,8 +139,8 @@ REGISTER_OP("Open3DContinuousConvTransposeBackpropFilter") if (c->ValueKnown(c->Dim(filters_shape, i))) { int64_t n = c->Value(c->Dim(filters_shape, i)); if (n < 1) - return Status(error::INVALID_ARGUMENT, - "Each filter dimension must be >= 1"); + return absl::InvalidArgumentError( + "Each filter dimension must be >= 1"); } } diff --git a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeOps.cpp b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeOps.cpp index 19a41bb40b3..b04c5b62afa 100644 --- a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeOps.cpp +++ b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeOps.cpp @@ -145,8 +145,8 @@ REGISTER_OP("Open3DContinuousConvTranspose") if (c->ValueKnown(c->Dim(filters_shape, i))) { int64_t n = c->Value(c->Dim(filters_shape, i)); if (n < 1) - return Status(error::INVALID_ARGUMENT, - "Each filter dimension must be >= 1"); + return absl::InvalidArgumentError( + "Each filter dimension must be >= 1"); } } diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 98d97c06f3a..dc6b1af2245 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -166,17 +166,16 @@ RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ && yarn --version # Build all -RUN if [ "${BUILD_PYTORCH_OPS}" = "ON" ] || [ "${BUILD_TENSORFLOW_OPS}" = "ON" ]; then \ - export GLIBCXX_USE_CXX11_ABI=OFF; \ - else \ - export GLIBCXX_USE_CXX11_ABI=ON; \ - fi \ - && if [ "${BUILD_SYCL_MODULE}" = "ON" ]; then \ +RUN \ + if [ "${BUILD_SYCL_MODULE}" = "ON" ]; then \ export CMAKE_CXX_COMPILER=icpx; \ export CMAKE_C_COMPILER=icx; \ + export GLIBCXX_USE_CXX11_ABI=ON; \ else \ export CMAKE_CXX_COMPILER=g++; \ export CMAKE_C_COMPILER=gcc; \ + # TODO: PyTorch still use old CXX ABI, remove this line when PyTorch is updated + export GLIBCXX_USE_CXX11_ABI=OFF; \ fi \ && mkdir build \ && cd build \ diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index 59d33503ab4..9b9fac01935 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -1,5 +1,5 @@ # FROM must be called before other ARGS except for ARG BASE_IMAGE -ARG BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 +ARG BASE_IMAGE=nvidia/cuda:11.7.1-cudnn8-devel-ubuntu18.04 FROM ${BASE_IMAGE} # Customizable build arguments from cuda.yml @@ -8,14 +8,18 @@ ARG CCACHE_TAR_NAME ARG CMAKE_VERSION ARG CCACHE_VERSION ARG PYTHON_VERSION +ARG BUILD_TENSORFLOW_OPS +ARG BUILD_PYTORCH_OPS # Forward all ARG to ENV # ci_utils.sh requires these environment variables ENV DEVELOPER_BUILD=${DEVELOPER_BUILD} ENV CCACHE_TAR_NAME=${CCACHE_TAR_NAME} ENV CMAKE_VERSION=${CMAKE_VERSION} -ARG CCACHE_VERSION=${CCACHE_VERSION} +ENV CCACHE_VERSION=${CCACHE_VERSION} ENV PYTHON_VERSION=${PYTHON_VERSION} +ENV BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS} +ENV BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS} # Prevent interactive inputs when installing packages ENV DEBIAN_FRONTEND=noninteractive @@ -125,8 +129,6 @@ WORKDIR /root/Open3D # Build python wheel RUN export NPROC=$(nproc) \ && export BUILD_SHARED_LIBS=OFF \ - && export BUILD_TENSORFLOW_OPS=ON \ - && export BUILD_PYTORCH_OPS=ON \ && source /root/Open3D/util/ci_utils.sh \ && build_pip_package build_azure_kinect build_jupyter diff --git a/docker/README.md b/docker/README.md index 89c5d34cabf..d14d9521c19 100644 --- a/docker/README.md +++ b/docker/README.md @@ -26,7 +26,7 @@ to install Nvidia Docker to run the CUDA container. To verify that the Nvidia Docker is working, run: ```bash -docker run --rm --gpus all nvidia/cuda:11.0-base nvidia-smi +docker run --rm --gpus all nvidia/cuda:11.7-base nvidia-smi ``` ### ARM64 Docker diff --git a/docker/docker_build.sh b/docker/docker_build.sh index a4e38cb84c3..b0b405d9de2 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -23,24 +23,24 @@ __usage_docker_build="USAGE: OPTION: # OpenBLAS AMD64 (Dockerfile.openblas) - openblas-amd64-py37-dev : OpenBLAS AMD64 3.7 wheel, developer mode openblas-amd64-py38-dev : OpenBLAS AMD64 3.8 wheel, developer mode openblas-amd64-py39-dev : OpenBLAS AMD64 3.9 wheel, developer mode openblas-amd64-py310-dev : OpenBLAS AMD64 3.10 wheel, developer mode - openblas-amd64-py37 : OpenBLAS AMD64 3.7 wheel, release mode + openblas-amd64-py311-dev : OpenBLAS AMD64 3.11 wheel, developer mode openblas-amd64-py38 : OpenBLAS AMD64 3.8 wheel, release mode openblas-amd64-py39 : OpenBLAS AMD64 3.9 wheel, release mode openblas-amd64-py310 : OpenBLAS AMD64 3.10 wheel, release mode + openblas-amd64-py311 : OpenBLAS AMD64 3.11 wheel, release mode # OpenBLAS ARM64 (Dockerfile.openblas) - openblas-arm64-py37-dev : OpenBLAS ARM64 3.7 wheel, developer mode openblas-arm64-py38-dev : OpenBLAS ARM64 3.8 wheel, developer mode openblas-arm64-py39-dev : OpenBLAS ARM64 3.9 wheel, developer mode openblas-arm64-py310-dev : OpenBLAS ARM64 3.10 wheel, developer mode - openblas-arm64-py37 : OpenBLAS ARM64 3.7 wheel, release mode + openblas-arm64-py311-dev : OpenBLAS ARM64 3.11 wheel, developer mode openblas-arm64-py38 : OpenBLAS ARM64 3.8 wheel, release mode openblas-arm64-py39 : OpenBLAS ARM64 3.9 wheel, release mode openblas-arm64-py310 : OpenBLAS ARM64 3.10 wheel, release mode + openblas-arm64-py311 : OpenBLAS ARM64 3.11 wheel, release mode # Ubuntu CPU CI (Dockerfile.ci) cpu-static : Ubuntu CPU static @@ -62,14 +62,14 @@ OPTION: 5-ml-focal : CUDA CI, 5-ml-focal, developer mode # CUDA wheels (Dockerfile.wheel) - cuda_wheel_py37_dev : CUDA Python 3.7 wheel, developer mode cuda_wheel_py38_dev : CUDA Python 3.8 wheel, developer mode cuda_wheel_py39_dev : CUDA Python 3.9 wheel, developer mode cuda_wheel_py310_dev : CUDA Python 3.10 wheel, developer mode - cuda_wheel_py37 : CUDA Python 3.7 wheel, release mode + cuda_wheel_py311_dev : CUDA Python 3.11 wheel, developer mode cuda_wheel_py38 : CUDA Python 3.8 wheel, release mode cuda_wheel_py39 : CUDA Python 3.9 wheel, release mode cuda_wheel_py310 : CUDA Python 3.10 wheel, release mode + cuda_wheel_py311 : CUDA Python 3.11 wheel, release mode " HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pwd)" @@ -78,6 +78,7 @@ HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pw CCACHE_VERSION=4.3 CMAKE_VERSION=cmake-3.20.6-linux-x86_64 CMAKE_VERSION_AARCH64=cmake-3.20.6-linux-aarch64 +CUDA_VERSION=11.7.1-cudnn8 print_usage_and_exit_docker_build() { echo "$__usage_docker_build" @@ -114,10 +115,7 @@ openblas_export_env() { print_usage_and_exit_docker_build fi - if [[ "py37" =~ ^($options)$ ]]; then - export PYTHON_VERSION=3.7 - export DOCKER_TAG=${DOCKER_TAG}-py37 - elif [[ "py38" =~ ^($options)$ ]]; then + if [[ "py38" =~ ^($options)$ ]]; then export PYTHON_VERSION=3.8 export DOCKER_TAG=${DOCKER_TAG}-py38 elif [[ "py39" =~ ^($options)$ ]]; then @@ -126,6 +124,9 @@ openblas_export_env() { elif [[ "py310" =~ ^($options)$ ]]; then export PYTHON_VERSION=3.10 export DOCKER_TAG=${DOCKER_TAG}-py310 + elif [[ "py311" =~ ^($options)$ ]]; then + export PYTHON_VERSION=3.11 + export DOCKER_TAG=${DOCKER_TAG}-py311 else echo "Invalid python version." print_usage_and_exit_docker_build @@ -168,19 +169,19 @@ openblas_build() { } cuda_wheel_build() { - BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 CCACHE_TAR_NAME=open3d-ubuntu-1804-cuda-ci-ccache options="$(echo "$@" | tr ' ' '|')" echo "[cuda_wheel_build()] options: ${options}" - if [[ "py37" =~ ^($options)$ ]]; then - PYTHON_VERSION=3.7 - elif [[ "py38" =~ ^($options)$ ]]; then + if [[ "py38" =~ ^($options)$ ]]; then PYTHON_VERSION=3.8 elif [[ "py39" =~ ^($options)$ ]]; then PYTHON_VERSION=3.9 elif [[ "py310" =~ ^($options)$ ]]; then PYTHON_VERSION=3.10 + elif [[ "py311" =~ ^($options)$ ]]; then + PYTHON_VERSION=3.11 else echo "Invalid python version." print_usage_and_exit_docker_build @@ -192,6 +193,8 @@ cuda_wheel_build() { fi echo "[cuda_wheel_build()] PYTHON_VERSION: ${PYTHON_VERSION}" echo "[cuda_wheel_build()] DEVELOPER_BUILD: ${DEVELOPER_BUILD}" + echo "[cuda_wheel_build()] BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS:?'env var must be set.'}" + echo "[cuda_wheel_build()] BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS:?'env var must be set.'}" pushd "${HOST_OPEN3D_ROOT}" docker build \ @@ -202,6 +205,8 @@ cuda_wheel_build() { --build-arg CMAKE_VERSION="${CMAKE_VERSION}" \ --build-arg CCACHE_VERSION="${CCACHE_VERSION}" \ --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ + --build-arg BUILD_TENSORFLOW_OPS="${BUILD_TENSORFLOW_OPS}" \ + --build-arg BUILD_PYTORCH_OPS="${BUILD_PYTORCH_OPS}" \ -t open3d-ci:wheel \ -f docker/Dockerfile.wheel . popd @@ -256,10 +261,10 @@ ci_build() { 2-bionic_export_env() { export DOCKER_TAG=open3d-ci:2-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-2-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=ON export BUILD_TENSORFLOW_OPS=OFF @@ -271,13 +276,14 @@ ci_build() { 3-ml-shared-bionic_export_env() { export DOCKER_TAG=open3d-ci:3-ml-shared-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -286,13 +292,14 @@ ci_build() { 3-ml-shared-bionic-release_export_env() { export DOCKER_TAG=open3d-ci:3-ml-shared-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -301,10 +308,10 @@ ci_build() { 4-shared-bionic_export_env() { export DOCKER_TAG=open3d-ci:4-shared-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-4-shared-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON export BUILD_TENSORFLOW_OPS=OFF @@ -316,10 +323,10 @@ ci_build() { 4-shared-bionic-release_export_env() { export DOCKER_TAG=open3d-ci:4-shared-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-4-shared-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON export BUILD_TENSORFLOW_OPS=OFF @@ -331,13 +338,14 @@ ci_build() { 5-ml-focal_export_env() { export DOCKER_TAG=open3d-ci:5-ml-focal - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-5-ml-focal - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=OFF export BUILD_SYCL_MODULE=OFF @@ -349,7 +357,7 @@ cpu-static_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -364,7 +372,7 @@ cpu-shared_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -379,10 +387,11 @@ cpu-shared-ml_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -394,7 +403,7 @@ cpu-shared-release_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -409,10 +418,11 @@ cpu-shared-ml-release_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -426,7 +436,7 @@ sycl-shared_export_env() { export BASE_IMAGE=intel/oneapi-basekit:2022.2-devel-ubuntu20.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-sycl - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -443,7 +453,7 @@ sycl-static_export_env() { export BASE_IMAGE=intel/oneapi-basekit:2022.2-devel-ubuntu20.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-sycl - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -460,10 +470,6 @@ function main() { echo "[$(basename $0)] building $1" case "$1" in # OpenBLAS AMD64 - openblas-amd64-py37-dev) - openblas_export_env amd64 py37 dev - openblas_build - ;; openblas-amd64-py38-dev) openblas_export_env amd64 py38 dev openblas_build @@ -476,8 +482,8 @@ function main() { openblas_export_env amd64 py310 dev openblas_build ;; - openblas-amd64-py37) - openblas_export_env amd64 py37 + openblas-amd64-py311-dev) + openblas_export_env amd64 py311 dev openblas_build ;; openblas-amd64-py38) @@ -492,12 +498,12 @@ function main() { openblas_export_env amd64 py310 openblas_build ;; - - # OpenBLAS ARM64 - openblas-arm64-py37-dev) - openblas_export_env arm64 py37 dev + openblas-amd64-py311) + openblas_export_env amd64 py311 openblas_build ;; + + # OpenBLAS ARM64 openblas-arm64-py38-dev) openblas_export_env arm64 py38 dev openblas_build @@ -510,8 +516,8 @@ function main() { openblas_export_env arm64 py310 dev openblas_build ;; - openblas-arm64-py37) - openblas_export_env arm64 py37 + openblas-arm64-py311-dev) + openblas_export_env arm64 py311 dev openblas_build ;; openblas-arm64-py38) @@ -526,6 +532,10 @@ function main() { openblas_export_env arm64 py310 openblas_build ;; + openblas-arm64-py311) + openblas_export_env arm64 py311 + openblas_build + ;; # CPU CI cpu-static) @@ -560,9 +570,6 @@ function main() { ;; # CUDA wheels - cuda_wheel_py37_dev) - cuda_wheel_build py37 dev - ;; cuda_wheel_py38_dev) cuda_wheel_build py38 dev ;; @@ -572,8 +579,8 @@ function main() { cuda_wheel_py310_dev) cuda_wheel_build py310 dev ;; - cuda_wheel_py37) - cuda_wheel_build py37 + cuda_wheel_py311_dev) + cuda_wheel_build py311 dev ;; cuda_wheel_py38) cuda_wheel_build py38 @@ -584,6 +591,9 @@ function main() { cuda_wheel_py310) cuda_wheel_build py310 ;; + cuda_wheel_py311) + cuda_wheel_build py311 + ;; # ML CIs 2-bionic) diff --git a/docker/docker_test.sh b/docker/docker_test.sh index 9993024a90f..877bbe1e5f8 100755 --- a/docker/docker_test.sh +++ b/docker/docker_test.sh @@ -16,24 +16,24 @@ __usage_docker_test="USAGE: OPTION: # OpenBLAS AMD64 (Dockerfile.openblas) - openblas-amd64-py37-dev : OpenBLAS AMD64 3.7 wheel, developer mode openblas-amd64-py38-dev : OpenBLAS AMD64 3.8 wheel, developer mode openblas-amd64-py39-dev : OpenBLAS AMD64 3.9 wheel, developer mode openblas-amd64-py310-dev : OpenBLAS AMD64 3.10 wheel, developer mode - openblas-amd64-py37 : OpenBLAS AMD64 3.7 wheel, release mode + openblas-amd64-py311-dev : OpenBLAS AMD64 3.11 wheel, developer mode openblas-amd64-py38 : OpenBLAS AMD64 3.8 wheel, release mode openblas-amd64-py39 : OpenBLAS AMD64 3.9 wheel, release mode openblas-amd64-py310 : OpenBLAS AMD64 3.10 wheel, release mode + openblas-amd64-py311 : OpenBLAS AMD64 3.11 wheel, release mode # OpenBLAS ARM64 (Dockerfile.openblas) - openblas-arm64-py37-dev : OpenBLAS ARM64 3.7 wheel, developer mode openblas-arm64-py38-dev : OpenBLAS ARM64 3.8 wheel, developer mode openblas-arm64-py39-dev : OpenBLAS ARM64 3.9 wheel, developer mode openblas-arm64-py310-dev : OpenBLAS ARM64 3.10 wheel, developer mode - openblas-arm64-py37 : OpenBLAS ARM64 3.7 wheel, release mode + openblas-arm64-py311-dev : OpenBLAS ARM64 3.11 wheel, developer mode openblas-arm64-py38 : OpenBLAS ARM64 3.8 wheel, release mode openblas-arm64-py39 : OpenBLAS ARM64 3.9 wheel, release mode openblas-arm64-py310 : OpenBLAS ARM64 3.10 wheel, release mode + openblas-arm64-py311 : OpenBLAS ARM64 3.11 wheel, release mode # Ubuntu CPU CI (Dockerfile.ci) cpu-static : Ubuntu CPU static @@ -208,11 +208,6 @@ echo "[$(basename $0)] building $1" source "${HOST_OPEN3D_ROOT}/docker/docker_build.sh" case "$1" in # OpenBLAS AMD64 -openblas-amd64-py37-dev) - openblas_export_env amd64 py37 dev - openblas_print_env - cpp_python_linking_uninstall_test - ;; openblas-amd64-py38-dev) openblas_export_env amd64 py38 dev openblas_print_env @@ -228,8 +223,8 @@ openblas-amd64-py310-dev) openblas_print_env cpp_python_linking_uninstall_test ;; -openblas-amd64-py37) - openblas_export_env amd64 py37 +openblas-amd64-py311-dev) + openblas_export_env amd64 py311 dev openblas_print_env cpp_python_linking_uninstall_test ;; @@ -248,13 +243,13 @@ openblas-amd64-py310) openblas_print_env cpp_python_linking_uninstall_test ;; - -# OpenBLAS ARM64 -openblas-arm64-py37-dev) - openblas_export_env arm64 py37 dev +openblas-amd64-py311) + openblas_export_env amd64 py311 openblas_print_env cpp_python_linking_uninstall_test ;; + +# OpenBLAS ARM64 openblas-arm64-py38-dev) openblas_export_env arm64 py38 dev openblas_print_env @@ -270,8 +265,8 @@ openblas-arm64-py310-dev) openblas_print_env cpp_python_linking_uninstall_test ;; -openblas-arm64-py37) - openblas_export_env arm64 py37 +openblas-arm64-py311-dev) + openblas_export_env arm64 py311 dev openblas_print_env cpp_python_linking_uninstall_test ;; @@ -290,6 +285,11 @@ openblas-arm64-py310) openblas_print_env cpp_python_linking_uninstall_test ;; +openblas-arm64-py311) + openblas_export_env arm64 py311 + openblas_print_env + cpp_python_linking_uninstall_test + ;; # CPU CI cpu-static) diff --git a/docs/arm.rst b/docs/arm.rst index dddb894987d..584d0d007ad 100644 --- a/docs/arm.rst +++ b/docs/arm.rst @@ -67,10 +67,10 @@ commands: cd docker - ./docker_build.sh openblas-arm64-py37 # Python 3.7 ./docker_build.sh openblas-arm64-py38 # Python 3.8 ./docker_build.sh openblas-arm64-py39 # Python 3.9 ./docker_build.sh openblas-arm64-py310 # Python 3.10 + ./docker_build.sh openblas-arm64-py311 # Python 3.11 After running ``docker_build.sh``, you shall see a ``.whl`` file generated the current directly on the host. Then simply install the ``.whl`` file by: diff --git a/docs/docker.in.rst b/docs/docker.in.rst index 25714d0f5d8..3647b99d48f 100644 --- a/docs/docker.in.rst +++ b/docs/docker.in.rst @@ -204,3 +204,6 @@ sample configuration options available are shown below. Run ``./docker_build.sh`` without arguments to get a list of all available build configurations. + +.. note:: You can control support for PyTorch and Tensorflow with environment variables: + `BUILD_PYTORCH_OPS=ON` and `BUILD_TENSORFLOW_OPS=ON` diff --git a/docs/getting_started.in.rst b/docs/getting_started.in.rst index 7bc974a87da..93fe8605a2c 100644 --- a/docs/getting_started.in.rst +++ b/docs/getting_started.in.rst @@ -68,28 +68,28 @@ version (``HEAD`` of ``master`` branch): :widths: auto * - Linux - - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ + - `Python 3.11 `__ * - Linux (CPU) - - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ + - `Python 3.11 `__ * - MacOS - - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ + - `Python 3.11 `__ * - Windows - - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ + - `Python 3.11 `__ Please use these links from the `latest version of this page `__ only. You can also diff --git a/python/README.rst b/python/README.rst index 64c5cb00c60..f55cf004b2a 100644 --- a/python/README.rst +++ b/python/README.rst @@ -44,10 +44,10 @@ The package has been tested on: With Python versions: -* 3.7 * 3.8 * 3.9 * 3.10 +* 3.11 Resources ====================== diff --git a/python/open3d/visualization/tensorboard_plugin/util.py b/python/open3d/visualization/tensorboard_plugin/util.py index b057ee1a5b0..8c8a5beda34 100644 --- a/python/open3d/visualization/tensorboard_plugin/util.py +++ b/python/open3d/visualization/tensorboard_plugin/util.py @@ -72,7 +72,7 @@ def release_read(self): try: self._readers -= 1 if not self._readers: - self._read_ready.notifyAll() + self._read_ready.notify_all() finally: self._read_ready.release() diff --git a/python/requirements_build.txt b/python/requirements_build.txt index 308feb49aeb..b9bb3b1f28a 100644 --- a/python/requirements_build.txt +++ b/python/requirements_build.txt @@ -1,3 +1,3 @@ -setuptools>=50.3.2 +setuptools>=67.3.2 wheel==0.38.4 yapf==0.30.0 diff --git a/python/requirements_test.txt b/python/requirements_test.txt index 9b3710ef1e4..4a2f418b6bb 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -1,6 +1,6 @@ pytest==7.1.2 pytest-randomly==3.8.0 -scipy==1.7.3 -tensorboard==2.8.0 +scipy==1.10.1 +tensorboard==2.13.0 oauthlib==3.2.2 certifi==2023.7.22 diff --git a/python/setup.py b/python/setup.py index 5322d230915..029c5717180 100644 --- a/python/setup.py +++ b/python/setup.py @@ -127,10 +127,10 @@ def finalize_options(self): "Programming Language :: C", "Programming Language :: C++", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Education", "Topic :: Multimedia :: Graphics :: 3D Modeling", "Topic :: Multimedia :: Graphics :: 3D Rendering", @@ -157,7 +157,7 @@ def finalize_options(self): setup_args = dict( name=name, version="@PROJECT_VERSION@", - python_requires=">=3.6", + python_requires=">=3.8", include_package_data=True, install_requires=install_requires, packages=find_packages(), diff --git a/python/test/core/test_core.py b/python/test/core/test_core.py index b0fac5c8f0c..47200dfe338 100644 --- a/python/test/core/test_core.py +++ b/python/test/core/test_core.py @@ -61,8 +61,8 @@ def to_numpy_dtype(dtype: o3c.Dtype): o3c.uint16: np.uint16, o3c.uint32: np.uint32, o3c.uint64: np.uint64, - o3c.bool8: np.bool8, # np.bool deprecated - o3c.bool: np.bool8, # o3c.bool is an alias for o3c.bool8 + o3c.bool8: np.bool_, + o3c.bool: np.bool_, # o3c.bool is an alias for o3c.bool8 } return conversions[dtype] @@ -163,7 +163,7 @@ def test_tensor_constructor(dtype, device): np.testing.assert_equal(np_t, o3_t.cpu().numpy()) # Boolean - np_t = np.array([True, False, True], dtype=np.bool8) + np_t = np.array([True, False, True], dtype=np.bool_) o3_t = o3c.Tensor([True, False, True], o3c.bool, device) np.testing.assert_equal(np_t, o3_t.cpu().numpy()) o3_t = o3c.Tensor(np_t, o3c.bool, device) @@ -801,7 +801,7 @@ def test_setitem(device): np.testing.assert_equal(o3_t.cpu().numpy(), np_t) # Scalar boolean set item - np_t = np.eye(4, dtype=np.bool8) + np_t = np.eye(4, dtype=np.bool_) o3_t = o3c.Tensor.eye(4, dtype=o3c.bool) np_t[2, 2] = False o3_t[2, 2] = False diff --git a/python/test/ml_ops/test_cconv_python.py b/python/test/ml_ops/test_cconv_python.py index 32a30aba4e7..2461d84b114 100644 --- a/python/test/ml_ops/test_cconv_python.py +++ b/python/test/ml_ops/test_cconv_python.py @@ -28,8 +28,8 @@ def test_compare_to_conv3d(dtype, filter_size, out_channels, in_channels, with_inp_importance, with_normalization): """Compares to the 3D convolution in tensorflow""" - import tensorflow as tf - import open3d.ml.tf as ml3d + tf = pytest.importorskip('tensorflow') + ml3d = pytest.importorskip('open3d.ml.tf') np.random.seed(0) conv_attrs = { @@ -123,8 +123,8 @@ def test_cconv_gradient(filter_size, out_channels, in_channels, with_inp_importance, with_neighbors_importance, with_individual_extent, with_normalization, align_corners, coordinate_mapping, interpolation): - import tensorflow as tf - import open3d.ml.tf as ml3d + tf = pytest.importorskip('tensorflow') + ml3d = pytest.importorskip('open3d.ml.tf') dtype = np.float64 np.random.seed(0) diff --git a/python/test/ml_ops/test_fixed_radius_search.py b/python/test/ml_ops/test_fixed_radius_search.py index 71ce5e1270a..f0ccf87eada 100644 --- a/python/test/ml_ops/test_fixed_radius_search.py +++ b/python/test/ml_ops/test_fixed_radius_search.py @@ -10,8 +10,10 @@ from scipy.spatial import cKDTree import pytest import mltest -import torch -import tensorflow as tf +if o3d._build_config['BUILD_PYTORCH_OPS']: + import torch +if o3d._build_config['BUILD_TENSORFLOW_OPS']: + import tensorflow as tf # skip all tests if the ml ops were not built pytestmark = mltest.default_marks diff --git a/python/test/ml_ops/test_knn_search.py b/python/test/ml_ops/test_knn_search.py index e4bdae38827..175835f6593 100644 --- a/python/test/ml_ops/test_knn_search.py +++ b/python/test/ml_ops/test_knn_search.py @@ -10,8 +10,10 @@ from scipy.spatial import cKDTree import pytest import mltest -import torch -import tensorflow as tf +if o3d._build_config['BUILD_PYTORCH_OPS']: + import torch +if o3d._build_config['BUILD_TENSORFLOW_OPS']: + import tensorflow as tf # skip all tests if the ml ops were not built pytestmark = mltest.default_marks diff --git a/python/test/ml_ops/test_radius_search.py b/python/test/ml_ops/test_radius_search.py index 6ce8da8b478..85b2af0ad0d 100644 --- a/python/test/ml_ops/test_radius_search.py +++ b/python/test/ml_ops/test_radius_search.py @@ -10,8 +10,10 @@ from scipy.spatial import cKDTree import pytest import mltest -import torch -import tensorflow as tf +if o3d._build_config['BUILD_PYTORCH_OPS']: + import torch +if o3d._build_config['BUILD_TENSORFLOW_OPS']: + import tensorflow as tf # skip all tests if the tf ops were not built and disable warnings caused by # tensorflow diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 80ca4469a14..3d9416d1f46 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -23,40 +23,20 @@ BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS:-ON} LOW_MEM_USAGE=${LOW_MEM_USAGE:-OFF} # Dependency versions: -# CUDA -if [[ $BUILD_TENSORFLOW_OPS == ON || $BUILD_PYTORCH_OPS == ON || - $UBUNTU_VERSION != bionic ]]; then - # CUDA version in sync with PyTorch and Tensorflow - CUDA_VERSION=("11-6" "11.6") - CUDNN_MAJOR_VERSION=8 - CUDNN_VERSION="8.4.1.50_cuda11.6" - GCC_MAX_VER=9 -else - # Without MLOps, ensure Open3D works with the lowest supported CUDA version - # Not available in Nvidia focal repos - CUDA_VERSION=("10-1" "10.1") - CUDNN_MAJOR_VERSION=8 - CUDNN_VERSION="8.0.5.39-1+cuda10.1" - GCC_MAX_VER=7 -fi +# CUDA: see docker/docker_build.sh # ML -TENSORFLOW_VER="2.8.4" -TENSORBOARD_VER="2.8.0" -TORCH_CPU_GLNX_VER="1.13.1+cpu" -TORCH_CUDA_GLNX_VER="1.13.1+cu116" -PYTHON_VER=$(python -c 'import sys; ver=f"{sys.version_info.major}{sys.version_info.minor}"; print(f"cp{ver}-cp{ver}{sys.abiflags}")' 2>/dev/null || true) -# TORCH_CUDA_GLNX_URL="https://github.com/isl-org/open3d_downloads/releases/download/torch1.8.2/torch-1.8.2-${PYTHON_VER}-linux_x86_64.whl" -TORCH_MACOS_VER="1.13.1" +TENSORFLOW_VER="2.13.0" +TORCH_VER="2.0.1" +TORCH_CPU_GLNX_VER="${TORCH_VER}+cpu" +TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu117" # match CUDA_VERSION in docker/docker_build.sh +TORCH_MACOS_VER="${TORCH_VER}" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python -PIP_VER="21.1.1" +PIP_VER="23.2.1" WHEEL_VER="0.38.4" STOOLS_VER="67.3.2" -PYTEST_VER="7.1.2" -PYTEST_RANDOMLY_VER="3.8.0" -SCIPY_VER="1.7.3" YAPF_VER="0.30.0" -PROTOBUF_VER="3.19.0" +PROTOBUF_VER="4.24.0" OPEN3D_INSTALL_DIR=~/open3d_install OPEN3D_SOURCE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pwd)" @@ -68,16 +48,15 @@ install_python_dependencies() { python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ setuptools=="$STOOLS_VER" if [[ "with-unit-test" =~ ^($options)$ ]]; then - python -m pip install -U scipy=="$SCIPY_VER" pytest=="$PYTEST_VER" \ - pytest-randomly=="$PYTEST_RANDOMLY_VER" + python -m pip install -U -r python/requirements_test.txt fi if [[ "with-cuda" =~ ^($options)$ ]]; then - TF_ARCH_NAME=tensorflow-gpu + TF_ARCH_NAME=tensorflow TF_ARCH_DISABLE_NAME=tensorflow-cpu TORCH_GLNX="torch==$TORCH_CUDA_GLNX_VER" else TF_ARCH_NAME=tensorflow-cpu - TF_ARCH_DISABLE_NAME=tensorflow-gpu + TF_ARCH_DISABLE_NAME=tensorflow TORCH_GLNX="torch==$TORCH_CPU_GLNX_VER" fi @@ -91,14 +70,14 @@ install_python_dependencies() { if [ "$BUILD_TENSORFLOW_OPS" == "ON" ]; then # TF happily installs both CPU and GPU versions at the same time, so remove the other python -m pip uninstall --yes "$TF_ARCH_DISABLE_NAME" - python -m pip install -U "$TF_ARCH_NAME"=="$TENSORFLOW_VER" + python -m pip install -U "$TF_ARCH_NAME"=="$TENSORFLOW_VER" # ML/requirements-tensorflow.txt fi - if [ "$BUILD_PYTORCH_OPS" == "ON" ]; then + if [ "$BUILD_PYTORCH_OPS" == "ON" ]; then # ML/requirements-torch.txt if [[ "$OSTYPE" == "linux-gnu"* ]]; then - python -m pip install -U "${TORCH_GLNX}" -f "$TORCH_REPO_URL" + python -m pip install -U "${TORCH_GLNX}" -f "$TORCH_REPO_URL" tensorboard elif [[ "$OSTYPE" == "darwin"* ]]; then - python -m pip install -U torch=="$TORCH_MACOS_VER" -f "$TORCH_REPO_URL" + python -m pip install -U torch=="$TORCH_MACOS_VER" -f "$TORCH_REPO_URL" tensorboard else echo "unknown OS $OSTYPE" exit 1 @@ -124,20 +103,17 @@ build_all() { mkdir -p build cd build - GLIBCXX_USE_CXX11_ABI=ON - if [ "$BUILD_PYTORCH_OPS" == ON ] || [ "$BUILD_TENSORFLOW_OPS" == ON ]; then - GLIBCXX_USE_CXX11_ABI=OFF - fi cmakeOptions=( - -DDEVELOPER_BUILD=$DEVELOPER_BUILD + -DDEVELOPER_BUILD="$DEVELOPER_BUILD" -DBUILD_SHARED_LIBS="$BUILD_SHARED_LIBS" -DCMAKE_BUILD_TYPE=Release -DBUILD_LIBREALSENSE=ON -DBUILD_CUDA_MODULE="$BUILD_CUDA_MODULE" -DBUILD_COMMON_CUDA_ARCHS=ON -DBUILD_COMMON_ISPC_ISAS=ON - -DGLIBCXX_USE_CXX11_ABI="$GLIBCXX_USE_CXX11_ABI" + # TODO: PyTorch still use old CXX ABI, remove this line when PyTorch is updated + -DGLIBCXX_USE_CXX11_ABI=OFF -DBUILD_TENSORFLOW_OPS="$BUILD_TENSORFLOW_OPS" -DBUILD_PYTORCH_OPS="$BUILD_PYTORCH_OPS" -DCMAKE_INSTALL_PREFIX="$OPEN3D_INSTALL_DIR" @@ -192,6 +168,13 @@ build_pip_package() { echo "Jupyter extension disabled in Python wheel." BUILD_JUPYTER_EXTENSION=OFF fi + CXX11_ABI=ON + if [ "$BUILD_TENSORFLOW_OPS" == "ON" ]; then + CXX11_ABI=$(python -c "import tensorflow as tf; print('ON' if tf.__cxx11_abi_flag__ else 'OFF')") + elif [ "$BUILD_PYTORCH_OPS" == "ON" ]; then + CXX11_ABI=$(python -c "import torch; print('ON' if torch._C._GLIBCXX_USE_CXX11_ABI else 'OFF')") + fi + echo Building with GLIBCXX_USE_CXX11_ABI="$CXX11_ABI" set -u echo @@ -202,10 +185,10 @@ build_pip_package() { "-DDEVELOPER_BUILD=$DEVELOPER_BUILD" "-DBUILD_COMMON_ISPC_ISAS=ON" "-DBUILD_AZURE_KINECT=$BUILD_AZURE_KINECT" - "-DBUILD_LIBREALSENSE=ON" - "-DGLIBCXX_USE_CXX11_ABI=OFF" - "-DBUILD_TENSORFLOW_OPS=ON" - "-DBUILD_PYTORCH_OPS=ON" + "-DBUILD_LIBREALSENSE=OFF" + "-DGLIBCXX_USE_CXX11_ABI=$CXX11_ABI" + "-DBUILD_TENSORFLOW_OPS=$BUILD_TENSORFLOW_OPS" + "-DBUILD_PYTORCH_OPS=$BUILD_PYTORCH_OPS" "-DBUILD_FILAMENT_FROM_SOURCE=$BUILD_FILAMENT_FROM_SOURCE" "-DBUILD_JUPYTER_EXTENSION=$BUILD_JUPYTER_EXTENSION" "-DCMAKE_INSTALL_PREFIX=$OPEN3D_INSTALL_DIR" @@ -218,7 +201,6 @@ build_pip_package() { cmake -DBUILD_CUDA_MODULE=OFF "${cmakeOptions[@]}" .. set +x # Echo commands off echo - make VERBOSE=1 -j"$NPROC" pybind open3d_tf_ops open3d_torch_ops echo "Packaging Open3D CPU pip package..." make VERBOSE=1 -j"$NPROC" pip-package @@ -303,13 +285,10 @@ test_wheel() { run_python_tests() { # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install -U pytest=="$PYTEST_VER" \ - pytest-randomly=="$PYTEST_RANDOMLY_VER" \ - scipy=="$SCIPY_VER" \ - tensorboard=="$TENSORBOARD_VER" + python -m pip install -U -r python/requirements_test.txt echo Add --randomly-seed=SEED to the test command to reproduce test order. pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/) - if [ "$BUILD_PYTORCH_OPS" == "OFF" ] || [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then + if [ "$BUILD_PYTORCH_OPS" == "OFF" ] && [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then echo Testing ML Ops disabled pytest_args+=(--ignore "$OPEN3D_SOURCE_ROOT"/python/test/ml_ops/) fi @@ -422,7 +401,8 @@ build_docs() { "-DBUILD_AZURE_KINECT=ON" "-DBUILD_LIBREALSENSE=ON" "-DGLIBCXX_USE_CXX11_ABI=OFF" - "-DBUILD_TENSORFLOW_OPS=ON" + # TODO: PyTorch still use old CXX ABI, re-enable Tensorflow when PyTorch is updated to use new ABI + "-DBUILD_TENSORFLOW_OPS=OFF" "-DBUILD_PYTORCH_OPS=ON" "-DBUILD_EXAMPLES=OFF" )