Merge branch 'main' into rel-0.8.0

docs/history.md

@@ -4,13 +4,15 @@
 
 This is the first version which uses datatree directly from xarray. Thus, xarray is pinned to version >= 2024.10.0.
 
+* FIX: Convert volumes to_cfradial1 containing sweeps with different range and azimuth shapes, raise for different range bin sizes ({issue}`233`) by [@syedhamidali](https://github.com/syedhamidali), ({pull}`234`) by [@kmuehlbauer](https://github.com/kmuehlbauer).
+* FIX: Correctly handle 8bit/16bit, big-endian/little-endian in nexrad reader (PHI and ZDR) ({issue}`230`) by [@syedhamidali](https://github.com/syedhamidali), ({pull}`231`) by [@kmuehlbauer](https://github.com/kmuehlbauer).
+* ENH: Refactoring all xradar backends to use `from_dict` datatree constructor. Test for `_get_required_root`, `_get_subgroup`, and `_get_radar_calibration` were also added ({pull}`221`) by [@aladinor](https://github.com/aladinor)
 * ENH: Added pytests to the missing functions in the `test_xradar` and `test_iris` in order to increase codecov in ({pull}`228`) by [@syedhamidali](https://github.com/syedhamidali).
 * ENH: Updated Readme ({pull}`226`) by [@syedhamidali](https://github.com/syedhamidali).
 * ADD: Added new module `transform` for transforming CF1 data to CF2 and vice versa ({pull}`224`) by [@syedhamidali](https://github.com/syedhamidali).
 * Use DataTree from xarray and add xarray nightly run ({pull}`213`, {pull}`214`, {pull}`215`, {pull}`218`) by [@kmuehlbauer](https://github.com/kmuehlbauer).
 * ADD: Added new accessor `map_over_sweeps` for volume operations on DataTrees and a matching decorator ({pull}`203`) by [@syedhamidali](https://github.com/syedhamidali).
 
-
 ## 0.7.0 (2024-10-26)
 
 This is the last version which uses datatree from xarray-contrib/datatree. Thus, xarray is pinned to version 2024.9.0.

tests/io/test_io.py

@@ -109,7 +109,7 @@ def test_open_odim_datatree_sweep(odim_file, sweep):
         lswp = len([sweep])
     else:
         lswp = len(sweep)
-    assert len(dtree.groups[1:]) == lswp
+    assert len(dtree.match("sweep_*")) == lswp
 
 
 def test_open_odim_datatree(odim_file):
@@ -164,7 +164,7 @@ def test_open_odim_datatree(odim_file):
         200,
         200,
     ]
-    for i, grp in enumerate(dtree.groups[1:]):
+    for i, grp in enumerate(dtree.match("sweep_*")):
         ds = dtree[grp].ds
         assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]}
         assert set(ds.data_vars) & (
@@ -183,7 +183,7 @@ def test_open_odim_datatree(odim_file):
             "range",
         }
         assert np.round(ds.elevation.mean().values.item(), 1) == elevations[i]
-        assert ds.sweep_number.values == int(grp[7:])
+        assert ds.sweep_number.values == int(grp[6:])
 
 
 @pytest.mark.parametrize("first_dim", ["auto", "time"])
@@ -258,7 +258,7 @@ def test_open_gamic_datatree_sweep(gamic_file, sweep):
         lswp = len([sweep])
     else:
         lswp = len(sweep)
-    assert len(dtree.groups[1:]) == lswp
+    assert len(dtree.match("sweep_*")) == lswp
 
 
 def test_open_gamic_datatree(gamic_file):
@@ -319,7 +319,7 @@ def test_open_gamic_datatree(gamic_file):
         1000,
         1000,
     ]
-    for i, grp in enumerate(dtree.groups[1:]):
+    for i, grp in enumerate(dtree.match("sweep_*")):
         ds = dtree[grp].ds
         assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]}
         assert set(ds.data_vars) & (
@@ -545,7 +545,7 @@ def test_open_rainbow_datatree(rainbow_file):
     ]
     azimuths = [361] * 14
     ranges = [400] * 14
-    for i, grp in enumerate(dtree.groups[1:]):
+    for i, grp in enumerate(dtree.match("sweep_*")):
         ds = dtree[grp].ds
         assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]}
         assert set(ds.data_vars) & (
@@ -641,28 +641,27 @@ def test_open_iris_datatree(iris0_file):
     azimuths = [360] * 10
     ranges = [664] * 10
     i = 0
-    for grp in dtree.groups:
-        if grp.startswith("/sweep_"):
-            ds = dtree[grp].ds
-            assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]}
-            assert set(ds.data_vars) & (
-                sweep_dataset_vars | non_standard_sweep_dataset_vars
-            ) == set(moments)
-            assert set(ds.data_vars) & (required_sweep_metadata_vars) == set(
-                required_sweep_metadata_vars ^ {"azimuth", "elevation"}
-            )
-            assert set(ds.coords) == {
-                "azimuth",
-                "elevation",
-                "time",
-                "latitude",
-                "longitude",
-                "altitude",
-                "range",
-            }
-            assert np.round(ds.elevation.mean().values.item(), 1) == elevations[i]
-            assert ds.sweep_number == i
-            i += 1
+    for grp in dtree.match("sweep_*"):
+        ds = dtree[grp].ds
+        assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]}
+        assert set(ds.data_vars) & (
+            sweep_dataset_vars | non_standard_sweep_dataset_vars
+        ) == set(moments)
+        assert set(ds.data_vars) & (required_sweep_metadata_vars) == set(
+            required_sweep_metadata_vars ^ {"azimuth", "elevation"}
+        )
+        assert set(ds.coords) == {
+            "azimuth",
+            "elevation",
+            "time",
+            "latitude",
+            "longitude",
+            "altitude",
+            "range",
+        }
+        assert np.round(ds.elevation.mean().values.item(), 1) == elevations[i]
+        assert ds.sweep_number == i
+        i += 1
 
 
 def test_open_iris0_dataset(iris0_file):
@@ -879,36 +878,35 @@ def test_open_datamet_datatree(datamet_file):
     azimuths = [360] * 11
     ranges = [493, 493, 493, 664, 832, 832, 1000, 1000, 1332, 1332, 1332]
     i = 0
-    for grp in dtree.groups:
-        if grp.startswith("/sweep_"):
-            ds = dtree[grp].ds
-            assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]}
-            assert set(ds.data_vars) & (
-                sweep_dataset_vars | non_standard_sweep_dataset_vars
-            ) == set(moments)
-            assert set(ds.data_vars) & (required_sweep_metadata_vars) == set(
-                required_sweep_metadata_vars ^ {"azimuth", "elevation"}
-            )
-            assert set(ds.coords) == {
-                "azimuth",
-                "elevation",
-                "time",
-                "latitude",
-                "longitude",
-                "altitude",
-                "range",
-            }
-            assert np.round(ds.elevation.mean().values.item(), 1) == elevations[i]
-            assert ds.sweep_number == i
-            i += 1
+    for grp in dtree.match("sweep_*"):
+        ds = dtree[grp].ds
+        assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]}
+        assert set(ds.data_vars) & (
+            sweep_dataset_vars | non_standard_sweep_dataset_vars
+        ) == set(moments)
+        assert set(ds.data_vars) & (required_sweep_metadata_vars) == set(
+            required_sweep_metadata_vars ^ {"azimuth", "elevation"}
+        )
+        assert set(ds.coords) == {
+            "azimuth",
+            "elevation",
+            "time",
+            "latitude",
+            "longitude",
+            "altitude",
+            "range",
+        }
+        assert np.round(ds.elevation.mean().values.item(), 1) == elevations[i]
+        assert ds.sweep_number == i
+        i += 1
 
     # Try to reed single sweep
     dtree = open_datamet_datatree(datamet_file, sweep=1)
-    assert len(dtree.groups) == 2
+    assert len(dtree.groups) == 5
 
     # Try to read list of sweeps
     dtree = open_datamet_datatree(datamet_file, sweep=[1, 2])
-    assert len(dtree.groups) == 3
+    assert len(dtree.groups) == 6
 
 
 @pytest.mark.parametrize("first_dim", ["time", "auto"])
@@ -993,6 +991,7 @@ def test_cfradial_n_points_file(cfradial1n_file):
         assert ds.sweep_mode == "azimuth_surveillance"
 
 
+@pytest.mark.run(order=1)
 @pytest.mark.parametrize("sweep", ["sweep_0", 0, [0, 1], ["sweep_0", "sweep_1"]])
 @pytest.mark.parametrize(
     "nexradlevel2_files", ["nexradlevel2_gzfile", "nexradlevel2_bzfile"], indirect=True
@@ -1003,7 +1002,7 @@ def test_open_nexradlevel2_datatree_sweep(nexradlevel2_files, sweep):
         lswp = len([sweep])
     else:
         lswp = len(sweep)
-    assert len(dtree.groups[1:]) == lswp
+    assert len(dtree.match("sweep*")) == lswp
 
 
 @pytest.mark.parametrize(
@@ -1080,8 +1079,8 @@ def test_open_nexradlevel2_datatree(nexradlevel2_files):
         308,
         232,
     ]
-    assert len(dtree.groups[1:]) == 11
-    for i, grp in enumerate(dtree.groups[1:]):
+    assert len(dtree.groups[1:]) == 14
+    for i, grp in enumerate(dtree.match("sweep_*")):
         print(i)
         ds = dtree[grp].ds
         assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]}
@@ -1101,4 +1100,4 @@ def test_open_nexradlevel2_datatree(nexradlevel2_files):
             "range",
         }
         assert np.round(ds.elevation.mean().values.item(), 1) == elevations[i]
-        assert ds.sweep_number.values == int(grp[7:])
+        assert ds.sweep_number.values == int(grp[6:])

tests/test_util.py

@@ -11,6 +11,11 @@
 
 import xradar as xd
 from xradar import io, model, util
+from xradar.io.backends.common import (
+    _get_radar_calibration,
+    _get_required_root_dataset,
+    _get_subgroup,
+)
 
 
 @pytest.fixture(
@@ -424,3 +429,57 @@ def dummy_function(ds, refl="none"):
         sweep_0.dummy_field.values
     )  # Convert NaNs to zero for comparison
     assert np.all(np.isclose(non_nan_values, 0))
+
+
+def test_get_required_root_dataset():
+
+    filename = DATASETS.fetch("cor-main131125105503.RAW2049")
+    sweeps = [f"sweep_{i}" for i in range(10)]
+    ls_ds = [xr.open_dataset(filename, engine="iris", group=sweep) for sweep in sweeps]
+    root = _get_required_root_dataset(ls_ds, optional=True)
+    elevations = [
+        0.5,
+        1.0,
+        2.0,
+        3.0,
+        5.0,
+        7.0,
+        10.0,
+        15.0,
+        20.0,
+        30.0,
+    ]
+    assert len(root.variables) == 10
+    assert root.variables["time_coverage_start"] == "2013-11-25T10:55:04Z"
+    assert root.variables["time_coverage_end"] == "2013-11-25T10:59:24Z"
+    np.testing.assert_equal(
+        root.variables["sweep_fixed_angle"].values, np.array(elevations)
+    )
+    assert len(list(root.attrs.keys())) == 10
+    assert root.attrs["instrument_name"] == "Corozal, Radar"
+    assert root.attrs["scan_name"] == "SURV_HV_300 "
+    assert root.attrs["comment"] == "AEROCIVIL OPERATIONAL DUAL POLE SCAN"
+
+
+def test_get_radar_calibration():
+    filename = DATASETS.fetch("DWD-Vol-2_99999_20180601054047_00.h5")
+    sweeps = [f"sweep_{i}" for i in range(10)]
+    ls_ds = [xr.open_dataset(filename, engine="gamic", group=sweep) for sweep in sweeps]
+    subgroup = _get_radar_calibration(ls_ds, model.radar_calibration_subgroup)
+    assert len(subgroup.variables) == 6
+    assert subgroup["noise_power_h"] == "-3.8298"
+    assert subgroup["rx_loss_h"] == "3"
+    assert subgroup["ant_gain_v"] == "43"
+    assert subgroup["ant_gain_h"] == "43"
+
+
+def test_get_subgroup():
+    filename = DATASETS.fetch("71_20181220_060628.pvol.h5")
+    sweeps = [f"sweep_{i}" for i in range(10)]
+    ls_ds = [xr.open_dataset(filename, engine="odim", group=sweep) for sweep in sweeps]
+    subgroup = _get_subgroup(ls_ds, model.radar_parameters_subgroup)
+    assert len(subgroup.variables) == 3
+    assert list(subgroup.variables) == ["longitude", "latitude", "altitude"]
+    np.testing.assert_almost_equal(subgroup.longitude.values.item(), 151.20899963378906)
+    np.testing.assert_almost_equal(subgroup.latitude.values.item(), -33.700801849365234)
+    assert isinstance(subgroup.altitude.values.item(), float)

tests/transform/test_cfradial.py

@@ -2,39 +2,71 @@
 # Copyright (c) 2024, openradar developers.
 # Distributed under the MIT License. See LICENSE for more info.
 
+import pytest
 import xarray as xr
-from open_radar_data import DATASETS
+from xarray import MergeError
 
 import xradar as xd
 
 
-def test_to_cfradial1():
+def test_to_cfradial1(cfradial1_file):
     """Test the conversion from DataTree to CfRadial1 format."""
-    file = DATASETS.fetch("cfrad.20080604_002217_000_SPOL_v36_SUR.nc")
-    dtree = xd.io.open_cfradial1_datatree(file)
+    with xd.io.open_cfradial1_datatree(cfradial1_file) as dtree:
 
-    # Call the conversion function
-    ds_cf1 = xd.transform.to_cfradial1(dtree)
+        # Call the conversion function
+        ds_cf1 = xd.transform.to_cfradial1(dtree)
 
-    # Verify key attributes and data structures in the resulting dataset
-    assert isinstance(ds_cf1, xr.Dataset), "Output is not a valid xarray Dataset"
-    assert "Conventions" in ds_cf1.attrs and ds_cf1.attrs["Conventions"] == "Cf/Radial"
-    assert "sweep_mode" in ds_cf1.variables, "Missing sweep_mode in converted dataset"
-    assert ds_cf1.attrs["version"] == "1.2", "Incorrect CfRadial version"
+        # Verify key attributes and data structures in the resulting dataset
+        assert isinstance(ds_cf1, xr.Dataset), "Output is not a valid xarray Dataset"
+        assert (
+            "Conventions" in ds_cf1.attrs and ds_cf1.attrs["Conventions"] == "Cf/Radial"
+        )
+        assert (
+            "sweep_mode" in ds_cf1.variables
+        ), "Missing sweep_mode in converted dataset"
+        assert ds_cf1.attrs["version"] == "1.2", "Incorrect CfRadial version"
 
 
-def test_to_cfradial2():
+def test_to_cfradial2(cfradial1_file):
     """Test the conversion from CfRadial1 to CfRadial2 DataTree format."""
-    file = DATASETS.fetch("cfrad.20080604_002217_000_SPOL_v36_SUR.nc")
-    dtree = xd.io.open_cfradial1_datatree(file)
+    with xd.io.open_cfradial1_datatree(cfradial1_file) as dtree:
 
-    # Convert to CfRadial1 dataset first
-    ds_cf1 = xd.transform.to_cfradial1(dtree)
+        # Convert to CfRadial1 dataset first
+        ds_cf1 = xd.transform.to_cfradial1(dtree)
 
-    # Call the conversion back to CfRadial2
-    dtree_cf2 = xd.transform.to_cfradial2(ds_cf1)
+        # Call the conversion back to CfRadial2
+        dtree_cf2 = xd.transform.to_cfradial2(ds_cf1)
 
-    # Verify key attributes and data structures in the resulting datatree
-    assert isinstance(dtree_cf2, xr.DataTree), "Output is not a valid DataTree"
-    assert "radar_parameters" in dtree_cf2, "Missing radar_parameters in DataTree"
-    assert dtree_cf2.attrs == ds_cf1.attrs, "Attributes mismatch between formats"
+        # Verify key attributes and data structures in the resulting datatree
+        assert isinstance(dtree_cf2, xr.DataTree), "Output is not a valid DataTree"
+        assert "radar_parameters" in dtree_cf2, "Missing radar_parameters in DataTree"
+        assert dtree_cf2.attrs == ds_cf1.attrs, "Attributes mismatch between formats"
+
+
+def test_to_cfradial1_with_different_range_shapes(nexradlevel2_bzfile):
+    with xd.io.open_nexradlevel2_datatree(nexradlevel2_bzfile) as dtree:
+        ds_cf1 = xd.transform.to_cfradial1(dtree)
+        # Verify key attributes and data structures in the resulting dataset
+        assert isinstance(ds_cf1, xr.Dataset), "Output is not a valid xarray Dataset"
+        assert (
+            "Conventions" in ds_cf1.attrs and ds_cf1.attrs["Conventions"] == "Cf/Radial"
+        )
+        assert (
+            "sweep_mode" in ds_cf1.variables
+        ), "Missing sweep_mode in converted dataset"
+        assert ds_cf1.attrs["version"] == "1.2", "Incorrect CfRadial version"
+        assert ds_cf1.sizes.mapping == {"time": 5400, "range": 1832, "sweep": 11}
+
+        # Call the conversion back to CfRadial2
+        dtree_cf2 = xd.transform.to_cfradial2(ds_cf1)
+        # Verify key attributes and data structures in the resulting datatree
+        assert isinstance(dtree_cf2, xr.DataTree), "Output is not a valid DataTree"
+        # todo: this needs to be fixed in nexrad level2reader
+        # assert "radar_parameters" in dtree_cf2, "Missing radar_parameters in DataTree"
+        assert dtree_cf2.attrs == ds_cf1.attrs, "Attributes mismatch between formats"
+
+
+def test_to_cfradial1_error_with_different_range_bin_sizes(gamic_file):
+    with xd.io.open_gamic_datatree(gamic_file) as dtree:
+        with pytest.raises(MergeError):
+            xd.transform.to_cfradial1(dtree)