From ec05939c205db12f9afe23a0d2dfb2687579e463 Mon Sep 17 00:00:00 2001 From: Florian Deconinck Date: Mon, 5 Feb 2024 12:33:18 -0500 Subject: [PATCH 1/7] Lint on PR/Merge --- .github/workflows/lint.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/lint.yaml diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..14e91f41 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,27 @@ +name: "Lint" +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3.5.2 + with: + submodules: 'recursive' + - name: Step Python 3.8.12 + uses: actions/setup-python@v4.6.0 + with: + python-version: '3.8.12' + - name: Install OpenMPI for gt4py + run: | + sudo apt-get install libopenmpi-dev + - name: Install Python packages + run: | + python -m pip install --upgrade pip setuptools wheel + pip install .[develop] + - name: Run lint via pre-commit + run: | + pre-commit run --all-files From 2300d4414c69f4528350ecec8dd127652b3cd32c Mon Sep 17 00:00:00 2001 From: Florian Deconinck Date: Mon, 5 Feb 2024 12:33:35 -0500 Subject: [PATCH 2/7] Hotfix: bad parameter naming on setup.py for extras --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 444e89a5..73ec210a 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ def local_pkg(name: str, relative_path: str) -> str: "Programming Language :: Python :: 3.9", ], install_requires=requirements, - extras_requires=extras_requires, + extras_require=extras_requires, name="ndsl", license="BSD license", packages=find_namespace_packages(include=["ndsl", "ndsl.*"]), From 30af5a9222c15636a0ba38b642753980357add96 Mon Sep 17 00:00:00 2001 From: Florian Deconinck Date: Mon, 5 Feb 2024 13:14:34 -0500 Subject: [PATCH 3/7] Unit tests, serial/parallel CPU --- .github/workflows/unit_tests.yaml | 31 +++++++++++++++++++++++++++++++ .gitignore | 4 ++++ ndsl/utils.py | 2 +- tests/dsl/test_caches.py | 9 +++++++-- 4 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/unit_tests.yaml diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml new file mode 100644 index 00000000..d6ff6e64 --- /dev/null +++ b/.github/workflows/unit_tests.yaml @@ -0,0 +1,31 @@ +name: "Unit tests" +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] + +jobs: + main_unit_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3.5.2 + with: + submodules: 'recursive' + - name: Setup Python + uses: actions/setup-python@v4.6.0 + with: + python-version: '3.8.12' + - name: Install OpenMPI & Boost for gt4py + run: | + sudo apt-get install libopenmpi-dev libboost1.74-dev + - name: Install Python packages + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -r requirements_dev.txt -c constraints.txt + - name: Run serial-cpu tests + run: | + pytest -x tests + - name: Run parallel-cpu tests + run: | + pytest -x tests/mpi + diff --git a/.gitignore b/.gitignore index 0a957698..65de9591 100644 --- a/.gitignore +++ b/.gitignore @@ -151,6 +151,10 @@ dmypy.json # GT4Py **/.gt_cache*/ +# Tests +.my_cache_path/* +.my_relocated_cache_path/* + # Run outputs plot_output/ profiling_results/ diff --git a/ndsl/utils.py b/ndsl/utils.py index 235b56ed..0d22330c 100644 --- a/ndsl/utils.py +++ b/ndsl/utils.py @@ -55,7 +55,7 @@ def is_c_contiguous(array: np.ndarray) -> bool: def ensure_contiguous(maybe_array: Union[np.ndarray, None]) -> None: if maybe_array is not None and not is_contiguous(maybe_array): - raise ValueError("ndarray is not contiguous") + raise BufferError("dlpack: buffer is not contiguous") def safe_assign_array(to_array: np.ndarray, from_array: np.ndarray): diff --git a/tests/dsl/test_caches.py b/tests/dsl/test_caches.py index ec6d3aec..04e919ff 100644 --- a/tests/dsl/test_caches.py +++ b/tests/dsl/test_caches.py @@ -10,6 +10,7 @@ StencilConfig, StencilFactory, ) +from ndsl.comm.mpi import MPI def _make_storage( @@ -77,6 +78,9 @@ def __call__(self): pytest.param("dace:cpu"), ], ) +@pytest.mark.skipif( + MPI is not None, reason="relocatibility checked with a one-rank setup" +) def test_relocatability_orchestration(backend): import os import shutil @@ -133,6 +137,9 @@ def test_relocatability_orchestration(backend): pytest.param("dace:cpu"), ], ) +@pytest.mark.skipif( + MPI is not None, reason="relocatibility checked with a one-rank setup" +) def test_relocatability(backend: str): import os import shutil @@ -140,8 +147,6 @@ def test_relocatability(backend: str): import gt4py from gt4py.cartesian import config as gt_config - from ..mpi.mpi_comm import MPI - # Restore original dir name gt4py.cartesian.config.cache_settings["dir_name"] = os.environ.get( "GT_CACHE_DIR_NAME", f".gt_cache_{MPI.COMM_WORLD.Get_rank():06}" From c4bb0f7149605a4f0300f4f44619ba839c6fee28 Mon Sep 17 00:00:00 2001 From: Florian Deconinck Date: Mon, 5 Feb 2024 13:16:31 -0500 Subject: [PATCH 4/7] README update --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0faa83e6..3996801d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,21 @@ # NOAA/NASA Domain Specific Language middleware -Use `git clone --recurse-submodule` to pull all vetted versions of the submodules used by `ndsl` +NDSL is a middleware for climate and weather modelling developped conjointment by NOAA and NASA. The middleware brings together [GT4Py](https://github.com/GridTools/gt4py/) (the `cartesian` flavor), an ETH CSCS's stencil DSL, and [DaCE](https://github.com/spcl/dace/), an ETH SPCL's data flow framework, both developped for high-performance and portability. On top of those pillars, NDSL deploys a series of optimized APIs for common operations (Halo exchange, domain decomposition, MPI...) and a set of bespoke optimizations for the models targeted by the middleware. + +## Battery-included for FV-based models + +Historically NDSL was developed to port the FV3 dynamical core on the cube-sphere. Therefore, the middleware ships with ready-to-execute specilization for models based on cube-sphere grid and FV-based model in particular. + +## Quickstart + +NDSL submodules `gt4py` and `dace` to point to vetted versions, use `git clone --recurse-submodule`. + +NDSL is __NOT__ available on `pypi`. Installation of the package has to be local, via `pip install ./NDSL` (`-e` supported). The packages has a few options: + +- `ndsl[test]`: installs the test packages (based on `pytest`) +- `ndsl[develop]`: installs tools for development and tests. + +Tests are available via: + +- `pytest -x test`: running CPU serial tests (GPU as well if `cupy` is installed) +- `mpirun -np 6 pytest -x test/mpi`: running CPU parallel tests (GPU as well if `cupy` is installed) From 450173c52d02bcdf92d76c82b1c8dc6781a98497 Mon Sep 17 00:00:00 2001 From: Florian Deconinck Date: Mon, 5 Feb 2024 14:19:35 -0500 Subject: [PATCH 5/7] Rename the job in unit tests --- .github/workflows/unit_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index d6ff6e64..fe237955 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -4,7 +4,7 @@ on: types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] jobs: - main_unit_tests: + all: runs-on: ubuntu-latest steps: - name: Checkout repository From ec39a3bd7ee90460dc220929121decc12ce1cb5b Mon Sep 17 00:00:00 2001 From: Florian Deconinck Date: Mon, 5 Feb 2024 14:20:09 -0500 Subject: [PATCH 6/7] Lint --- .github/workflows/unit_tests.yaml | 1 - tests/dsl/test_caches.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index fe237955..f0d3796e 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -28,4 +28,3 @@ jobs: - name: Run parallel-cpu tests run: | pytest -x tests/mpi - diff --git a/tests/dsl/test_caches.py b/tests/dsl/test_caches.py index 04e919ff..a7218b05 100644 --- a/tests/dsl/test_caches.py +++ b/tests/dsl/test_caches.py @@ -2,6 +2,7 @@ from gt4py.cartesian.gtscript import PARALLEL, Field, computation, interval from gt4py.storage import empty, ones +from ndsl.comm.mpi import MPI from ndsl.dsl.dace import orchestrate from ndsl.dsl.dace.dace_config import DaceConfig, DaCeOrchestration from ndsl.dsl.stencil import ( @@ -10,7 +11,6 @@ StencilConfig, StencilFactory, ) -from ndsl.comm.mpi import MPI def _make_storage( From 6b852611f1af7f2739e90e942fbfa6502e73ad9f Mon Sep 17 00:00:00 2001 From: Florian Deconinck Date: Mon, 5 Feb 2024 14:23:26 -0500 Subject: [PATCH 7/7] Bad pip install --- .github/workflows/unit_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index f0d3796e..c1dc9257 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -21,7 +21,7 @@ jobs: - name: Install Python packages run: | python -m pip install --upgrade pip setuptools wheel - pip install -r requirements_dev.txt -c constraints.txt + pip install .[test] - name: Run serial-cpu tests run: | pytest -x tests