diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..14e91f41 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,27 @@ +name: "Lint" +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3.5.2 + with: + submodules: 'recursive' + - name: Step Python 3.8.12 + uses: actions/setup-python@v4.6.0 + with: + python-version: '3.8.12' + - name: Install OpenMPI for gt4py + run: | + sudo apt-get install libopenmpi-dev + - name: Install Python packages + run: | + python -m pip install --upgrade pip setuptools wheel + pip install .[develop] + - name: Run lint via pre-commit + run: | + pre-commit run --all-files diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml new file mode 100644 index 00000000..c1dc9257 --- /dev/null +++ b/.github/workflows/unit_tests.yaml @@ -0,0 +1,30 @@ +name: "Unit tests" +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] + +jobs: + all: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3.5.2 + with: + submodules: 'recursive' + - name: Setup Python + uses: actions/setup-python@v4.6.0 + with: + python-version: '3.8.12' + - name: Install OpenMPI & Boost for gt4py + run: | + sudo apt-get install libopenmpi-dev libboost1.74-dev + - name: Install Python packages + run: | + python -m pip install --upgrade pip setuptools wheel + pip install .[test] + - name: Run serial-cpu tests + run: | + pytest -x tests + - name: Run parallel-cpu tests + run: | + pytest -x tests/mpi diff --git a/.gitignore b/.gitignore index 0a957698..65de9591 100644 --- a/.gitignore +++ b/.gitignore @@ -151,6 +151,10 @@ dmypy.json # GT4Py **/.gt_cache*/ +# Tests +.my_cache_path/* +.my_relocated_cache_path/* + # Run outputs plot_output/ profiling_results/ diff --git a/README.md b/README.md index 0faa83e6..3996801d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,21 @@ # NOAA/NASA Domain Specific Language middleware -Use `git clone --recurse-submodule` to pull all vetted versions of the submodules used by `ndsl` +NDSL is a middleware for climate and weather modelling developped conjointment by NOAA and NASA. The middleware brings together [GT4Py](https://github.com/GridTools/gt4py/) (the `cartesian` flavor), an ETH CSCS's stencil DSL, and [DaCE](https://github.com/spcl/dace/), an ETH SPCL's data flow framework, both developped for high-performance and portability. On top of those pillars, NDSL deploys a series of optimized APIs for common operations (Halo exchange, domain decomposition, MPI...) and a set of bespoke optimizations for the models targeted by the middleware. + +## Battery-included for FV-based models + +Historically NDSL was developed to port the FV3 dynamical core on the cube-sphere. Therefore, the middleware ships with ready-to-execute specilization for models based on cube-sphere grid and FV-based model in particular. + +## Quickstart + +NDSL submodules `gt4py` and `dace` to point to vetted versions, use `git clone --recurse-submodule`. + +NDSL is __NOT__ available on `pypi`. Installation of the package has to be local, via `pip install ./NDSL` (`-e` supported). The packages has a few options: + +- `ndsl[test]`: installs the test packages (based on `pytest`) +- `ndsl[develop]`: installs tools for development and tests. + +Tests are available via: + +- `pytest -x test`: running CPU serial tests (GPU as well if `cupy` is installed) +- `mpirun -np 6 pytest -x test/mpi`: running CPU parallel tests (GPU as well if `cupy` is installed) diff --git a/ndsl/utils.py b/ndsl/utils.py index 235b56ed..0d22330c 100644 --- a/ndsl/utils.py +++ b/ndsl/utils.py @@ -55,7 +55,7 @@ def is_c_contiguous(array: np.ndarray) -> bool: def ensure_contiguous(maybe_array: Union[np.ndarray, None]) -> None: if maybe_array is not None and not is_contiguous(maybe_array): - raise ValueError("ndarray is not contiguous") + raise BufferError("dlpack: buffer is not contiguous") def safe_assign_array(to_array: np.ndarray, from_array: np.ndarray): diff --git a/setup.py b/setup.py index 444e89a5..73ec210a 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ def local_pkg(name: str, relative_path: str) -> str: "Programming Language :: Python :: 3.9", ], install_requires=requirements, - extras_requires=extras_requires, + extras_require=extras_requires, name="ndsl", license="BSD license", packages=find_namespace_packages(include=["ndsl", "ndsl.*"]), diff --git a/tests/dsl/test_caches.py b/tests/dsl/test_caches.py index ec6d3aec..a7218b05 100644 --- a/tests/dsl/test_caches.py +++ b/tests/dsl/test_caches.py @@ -2,6 +2,7 @@ from gt4py.cartesian.gtscript import PARALLEL, Field, computation, interval from gt4py.storage import empty, ones +from ndsl.comm.mpi import MPI from ndsl.dsl.dace import orchestrate from ndsl.dsl.dace.dace_config import DaceConfig, DaCeOrchestration from ndsl.dsl.stencil import ( @@ -77,6 +78,9 @@ def __call__(self): pytest.param("dace:cpu"), ], ) +@pytest.mark.skipif( + MPI is not None, reason="relocatibility checked with a one-rank setup" +) def test_relocatability_orchestration(backend): import os import shutil @@ -133,6 +137,9 @@ def test_relocatability_orchestration(backend): pytest.param("dace:cpu"), ], ) +@pytest.mark.skipif( + MPI is not None, reason="relocatibility checked with a one-rank setup" +) def test_relocatability(backend: str): import os import shutil @@ -140,8 +147,6 @@ def test_relocatability(backend: str): import gt4py from gt4py.cartesian import config as gt_config - from ..mpi.mpi_comm import MPI - # Restore original dir name gt4py.cartesian.config.cache_settings["dir_name"] = os.environ.get( "GT_CACHE_DIR_NAME", f".gt_cache_{MPI.COMM_WORLD.Get_rank():06}"