Python 3.12: Refactor out Distutils (#4392)

* CI: Unconstrain Some Upper Py Versions * Replace distutils * Update Docs * no `clean` commmand pypa/setuptools#2838 * Remove `SETUPTOOLS_USE_DISTUTILS` Old workaround for old mpi4py. * Update Doc String * Update CI * Update Tools/machines/lassen-llnl/install_v100_ml.sh Co-authored-by: Roelof Groenewald <[email protected]> --------- Co-authored-by: Roelof Groenewald <[email protected]>
ECP-WarpX · Nov 3, 2023 · 5b4dfdc · 5b4dfdc
1 parent f6cb58d
commit 5b4dfdc
Show file tree

Hide file tree

Showing 29 changed files with 72 additions and 54 deletions.
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -18,16 +18,12 @@ jobs:
     env:
       CXXFLAGS: "-Werror"
       CMAKE_GENERATOR: Ninja
-      # setuptools/mp4py work-around, see
-      #   https://github.com/mpi4py/mpi4py/pull/159
-      #   https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
-      SETUPTOOLS_USE_DISTUTILS: stdlib
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       name: Install Python
       with:
-        python-version: '3.10'
+        python-version: '3.x'
     - name: install dependencies
       run: |
         .github/workflows/dependencies/nvcc11-3.sh
@@ -75,7 +71,8 @@ jobs:
           -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
         cmake --build build_sp -j 2
 
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        python3 -m pip install --upgrade build packaging setuptools wheel
         export WARPX_MPI=ON
         export PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/
         python3 -m pip wheel .
@@ -118,13 +115,9 @@ jobs:
     name: [email protected] NVCC/NVC++ Release [tests]
     runs-on: ubuntu-20.04
     if: github.event.pull_request.draft == false
-    env:
-      # For NVHPC, Ninja is slower than the default:
-      #CMAKE_GENERATOR: Ninja
-      # setuptools/mp4py work-around, see
-      #   https://github.com/mpi4py/mpi4py/pull/159
-      #   https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
-      SETUPTOOLS_USE_DISTUTILS: stdlib
+    #env:
+    #  # For NVHPC, Ninja is slower than the default:
+    #  CMAKE_GENERATOR: Ninja
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies
@@ -175,7 +168,8 @@ jobs:
         #   https://github.com/mpi4py/mpi4py/issues/114
         #export CFLAGS="-noswitcherror"
 
-        #python3 -m pip install --upgrade pip setuptools wheel
+        #python3 -m pip install --upgrade pip
+        #python3 -m pip install --upgrade build packaging setuptools wheel
         #export WARPX_MPI=ON
         #export PYWARPX_LIB_DIR=$PWD/build/lib/site-packages/pywarpx/
         #python3 -m pip wheel .

diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml
@@ -41,7 +41,8 @@ jobs:
         export CXX=$(which icpc)
         export CC=$(which icc)
 
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        python3 -m pip install --upgrade build packaging setuptools wheel
 
         cmake -S . -B build_dp \
           -DCMAKE_VERBOSE_MAKEFILE=ON \
@@ -104,7 +105,7 @@ jobs:
         export CC=$(which icx)
 
         python3 -m pip install --upgrade pip
-        python3 -m pip install --upgrade setuptools wheel
+        python3 -m pip install --upgrade build packaging setuptools wheel
 
         cmake -S . -B build_sp         \
           -DCMAKE_CXX_FLAGS_RELEASE="-O1 -DNDEBUG" \
@@ -178,6 +179,6 @@ jobs:
 
      # Skip this as it will copy the binary artifacts and we are tight on disk space
      #   python3 -m pip install --upgrade pip
-     #   python3 -m pip install --upgrade setuptools wheel
+     #   python3 -m pip install --upgrade build packaging setuptools wheel
      #   PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel .
      #   python3 -m pip install *.whl
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -16,10 +16,6 @@ jobs:
       HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: TRUE
       # For macOS, Ninja is slower than the default:
       #CMAKE_GENERATOR: Ninja
-      # setuptools/mp4py work-around, see
-      #   https://github.com/mpi4py/mpi4py/pull/159
-      #   https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
-      SETUPTOOLS_USE_DISTUTILS: stdlib
     steps:
     - uses: actions/checkout@v3
     - name: Brew Cache
@@ -65,7 +61,7 @@ jobs:
     - name: build WarpX
       run: |
         python3 -m pip install --upgrade pip
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade build packaging setuptools wheel
 
         cmake -S . -B build_dp         \
           -DCMAKE_VERBOSE_MAKEFILE=ON  \

diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -167,10 +167,6 @@ jobs:
       CXX: clang++
       # On CI for this test, Ninja is slower than the default:
       #CMAKE_GENERATOR: Ninja
-      # setuptools/mp4py work-around, see
-      #   https://github.com/mpi4py/mpi4py/pull/159
-      #   https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
-      SETUPTOOLS_USE_DISTUTILS: stdlib
     steps:
     - uses: actions/checkout@v3
     - name: install dependencies
@@ -190,7 +186,8 @@ jobs:
           ccache-openmp-pyfull-
     - name: build WarpX
       run: |
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        python3 -m pip install --upgrade build packaging setuptools wheel
 
         export CXXFLAGS="-Werror -Wno-error=pass-failed"
 

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -15,7 +15,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.x'
     - name: CCache Cache
       uses: actions/cache@v3
       # - once stored under a key, they become immutable (even if local cache path content changes)
@@ -41,7 +41,9 @@ jobs:
         cmake --build build --config Debug --parallel 2
         if(!$?) { Exit $LASTEXITCODE }
 
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        if(!$?) { Exit $LASTEXITCODE }
+        python3 -m pip install --upgrade build packaging setuptools wheel
         if(!$?) { Exit $LASTEXITCODE }
         cmake --build build --config Debug --target install
         if(!$?) { Exit $LASTEXITCODE }
@@ -100,7 +102,9 @@ jobs:
         cmake --build build --config Release --target install
         if errorlevel 1 exit 1
 
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        if errorlevel 1 exit 1
+        python3 -m pip install --upgrade build packaging setuptools wheel
         if errorlevel 1 exit 1
         python3 -m pip install --upgrade -r requirements.txt
         if errorlevel 1 exit 1

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -127,7 +127,7 @@ option(WarpX_PYTHON_IPO
 )
 
 set(pyWarpX_VERSION_INFO "" CACHE STRING
-    "PEP-440 conformant version (set by distutils)")
+    "PEP-440 conformant version (set by setup.py)")
 
 # enforce consistency of dependent options
 if(WarpX_APP OR WarpX_PYTHON)

diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst
@@ -211,7 +211,8 @@ PICMI Python Bindings
 
    .. code-block:: bash
 
-      python3 -m pip install -U pip setuptools wheel
+      python3 -m pip install -U pip
+      python3 -m pip install -U build packaging setuptools wheel
       python3 -m pip install -U cmake
       python3 -m pip install -r requirements.txt
 

diff --git a/Docs/source/install/hpc/lawrencium.rst b/Docs/source/install/hpc/lawrencium.rst
@@ -80,6 +80,8 @@ Optionally, download and install Python packages for :ref:`PICMI <usage-picmi>`
    python3 -m venv $HOME/sw/v100/venvs/warpx
    source $HOME/sw/v100/venvs/warpx/bin/activate
    python3 -m pip install --upgrade pip
+   python3 -m pip install --upgrade build
+   python3 -m pip install --upgrade packaging
    python3 -m pip install --upgrade wheel
    python3 -m pip install --upgrade setuptools
    python3 -m pip install --upgrade cython

diff --git a/Docs/source/install/hpc/lxplus.rst b/Docs/source/install/hpc/lxplus.rst
@@ -147,7 +147,8 @@ Now, ensure Python tooling is up-to-date:
 
 .. code-block:: bash
 
-   python3 -m pip install -U pip setuptools wheel
+   python3 -m pip install -U pip
+   python3 -m pip install -U build packaging setuptools wheel
    python3 -m pip install -U cmake
 
 Then we compile WarpX as in the previous section (with or without CUDA) adding ``-DWarpX_PYTHON=ON`` and then we install it into our Python:

diff --git a/Docs/source/install/users.rst b/Docs/source/install/users.rst
@@ -109,12 +109,11 @@ Given that you have the :ref:`WarpX dependencies <install-dependencies>` install
 
 .. code-block:: bash
 
-   # optional:                                    --user
-   python3 -m pip install -U pip setuptools wheel
+   python3 -m pip install -U pip
+   python3 -m pip install -U build packaging setuptools wheel
    python3 -m pip install -U cmake
 
    python3 -m pip wheel -v git+https://github.com/ECP-WarpX/WarpX.git
-   # optional:                 --user
    python3 -m pip install *whl
 
 In the future, will publish pre-compiled binary packages on `PyPI <https://pypi.org/>`__ for faster installs.

diff --git a/Docs/source/maintenance/performance_tests.rst b/Docs/source/maintenance/performance_tests.rst
@@ -75,6 +75,7 @@ Then, in ``$AUTOMATED_PERF_TESTS``, create a file ``run_automated_performance_te
    # lines below this comment once, before submission.
    # The commented lines take too long for the job script.
    #python3 -m pip install --upgrade pip
+   #python3 -m pip install --upgrade build packaging setuptools wheel
    #python3 -m pip install --upgrade cython
    #python3 -m pip install --upgrade numpy
    #python3 -m pip install --upgrade markupsafe

diff --git a/Tools/machines/adastra-cines/install_dependencies.sh b/Tools/machines/adastra-cines/install_dependencies.sh
@@ -101,6 +101,8 @@ rm -rf ${SW_DIR}/venvs/warpx-adastra
 python3 -m venv ${SW_DIR}/venvs/warpx-adastra
 source ${SW_DIR}/venvs/warpx-adastra/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/crusher-olcf/install_dependencies.sh b/Tools/machines/crusher-olcf/install_dependencies.sh
@@ -86,6 +86,8 @@ rm -rf ${SW_DIR}/venvs/warpx-crusher
 python3 -m venv ${SW_DIR}/venvs/warpx-crusher
 source ${SW_DIR}/venvs/warpx-crusher/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh
@@ -86,6 +86,8 @@ rm -rf ${SW_DIR}/venvs/warpx-frontier
 python3 -m venv ${SW_DIR}/venvs/warpx-frontier
 source ${SW_DIR}/venvs/warpx-frontier/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh
@@ -116,6 +116,8 @@ rm -rf ${SW_DIR}/venvs/warpx-gpu
 python3 -m venv ${SW_DIR}/venvs/warpx-gpu
 source ${SW_DIR}/venvs/warpx-gpu/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh b/Tools/machines/karolina-it4i/install_cpu_dependencies.sh
@@ -120,6 +120,8 @@ rm -rf ${SW_DIR}/venvs/warpx-cpu
 python3 -m venv ${SW_DIR}/venvs/warpx-cpu
 source ${SW_DIR}/venvs/warpx-cpu/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh b/Tools/machines/karolina-it4i/install_gpu_dependencies.sh
@@ -120,6 +120,8 @@ rm -rf ${SW_DIR}/venvs/warpx-gpu
 python3 -m venv ${SW_DIR}/venvs/warpx-gpu
 source ${SW_DIR}/venvs/warpx-gpu/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh
@@ -117,6 +117,8 @@ python3 -m venv ${SW_DIR}/venvs/warpx-lassen
 source ${SW_DIR}/venvs/warpx-lassen/bin/activate
 python3 -m pip install --upgrade pip
 python3 -m pip cache purge
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 # Older version for h4py

diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
@@ -117,6 +117,8 @@ python3 -m venv ${SW_DIR}/venvs/warpx-lassen-toss3
 source ${SW_DIR}/venvs/warpx-lassen-toss3/bin/activate
 python3 -m pip install --upgrade pip
 python3 -m pip cache purge
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 # Older version for h4py

diff --git a/Tools/machines/lassen-llnl/install_v100_ml.sh b/Tools/machines/lassen-llnl/install_v100_ml.sh
@@ -64,6 +64,7 @@ rm -rf build
 cd -
 
 # optional: optimas dependencies (based on libEnsemble & ax->botorch->gpytorch->pytorch)
+# TODO: scikit-learn needs a BLAS hint
 #   commented because scikit-learn et al. compile > 2 hrs
 #   please run manually on a login node if needed
 #python3 -m pip install -r ${SRC_DIR}/warpx/Tools/optimas/requirements.txt
diff --git a/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh b/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh
@@ -85,6 +85,8 @@ source ${SW_DIR}/venvs/warpx/bin/activate
 python3 -m ensurepip --upgrade
 python3 -m pip cache purge
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/lumi-csc/install_dependencies.sh b/Tools/machines/lumi-csc/install_dependencies.sh
@@ -101,6 +101,8 @@ rm -rf ${SW_DIR}/venvs/warpx-lumi
 python3 -m venv ${SW_DIR}/venvs/warpx-lumi
 source ${SW_DIR}/venvs/warpx-lumi/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh
@@ -116,6 +116,8 @@ rm -rf ${SW_DIR}/venvs/warpx
 python3 -m venv ${SW_DIR}/venvs/warpx
 source ${SW_DIR}/venvs/warpx/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh
@@ -116,6 +116,8 @@ rm -rf ${SW_DIR}/venvs/warpx
 python3 -m venv ${SW_DIR}/venvs/warpx
 source ${SW_DIR}/venvs/warpx/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/quartz-llnl/install_dependencies.sh b/Tools/machines/quartz-llnl/install_dependencies.sh
@@ -99,6 +99,8 @@ python3 -m venv ${SW_DIR}/venvs/warpx-quartz
 source ${SW_DIR}/venvs/warpx-quartz/bin/activate
 python3 -m pip install --upgrade pip
 python3 -m pip cache purge
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/Tools/machines/summit-olcf/install_gpu_dependencies.sh b/Tools/machines/summit-olcf/install_gpu_dependencies.sh
@@ -99,6 +99,8 @@ rm -rf ${SW_DIR}/venvs/warpx-summit
 python3 -m venv ${SW_DIR}/venvs/warpx-summit
 source ${SW_DIR}/venvs/warpx-summit/bin/activate
 python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade packaging
 python3 -m pip install --upgrade wheel
 python3 -m pip install --upgrade setuptools
 python3 -m pip install --upgrade cython

diff --git a/pyproject.toml b/pyproject.toml
@@ -2,6 +2,7 @@
 requires = [
     "setuptools>=42",
     "wheel",
-    "cmake>=3.20.0,<4.0.0"
+    "cmake>=3.20.0,<4.0.0",
+    "packaging>=23",
 ]
 build-backend = "setuptools.build_meta"
diff --git a/run_test.sh b/run_test.sh
@@ -61,12 +61,9 @@ echo "cd $PWD"
 rm -rf py-venv
 python3 -m venv py-venv
 source py-venv/bin/activate
-python3 -m pip install --upgrade pip setuptools wheel
+python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade build packaging setuptools wheel
 python3 -m pip install --upgrade cmake
-# setuptools/mp4py work-around, see
-#   https://github.com/mpi4py/mpi4py/pull/159
-#   https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
-export SETUPTOOLS_USE_DISTUTILS="stdlib"
 python3 -m pip install --upgrade -r warpx/Regression/requirements.txt
 
 # Clone AMReX and warpx-data