Skip to content

Commit

Permalink
Merge pull request #2976 from pnuu/bugfix-mersi-dtype
Browse files Browse the repository at this point in the history 
Fix dtype promotion in `mersi2_l1b` reader
  • Loading branch information
@pnuu
pnuu authored Nov 12, 2024
2 parents bd29bb1 + fd2cec6 commit a68c801
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 34 deletions.
2 changes: 1 addition & 1 deletion satpy/readers/mersi_l1b.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ def _mask_data(self, data, dataset_id, attrs):
attrs["_FillValue"] = fill_value
new_fill = data.dtype.type(fill_value)
else:
new_fill = np.nan
new_fill = np.float32(np.nan)
try:
# Due to a bug in the valid_range upper limit in the 10.8(24) and 12.0(25)
# in the HDF data, this is hardcoded here.
Expand Down
81 changes: 48 additions & 33 deletions satpy/tests/reader_tests/test_mersi_l1b.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,23 +31,27 @@ def _get_calibration(num_scans, ftype):
calibration = {
f"Calibration/{ftype}_Cal_Coeff":
xr.DataArray(
da.ones((19, 3), chunks=1024),
attrs={"Slope": np.array([1.] * 19), "Intercept": np.array([0.] * 19)},
da.ones((19, 3), chunks=1024, dtype=np.float32),
attrs={"Slope": np.array([1.] * 19, dtype=np.float32),
"Intercept": np.array([0.] * 19, dtype=np.float32)},
dims=("_bands", "_coeffs")),
"Calibration/Solar_Irradiance":
xr.DataArray(
da.ones((19, ), chunks=1024),
attrs={"Slope": np.array([1.] * 19), "Intercept": np.array([0.] * 19)},
da.ones((19, ), chunks=1024, dtype=np.float32),
attrs={"Slope": np.array([1.] * 19, dtype=np.float32),
"Intercept": np.array([0.] * 19, dtype=np.float32)},
dims=("_bands")),
"Calibration/Solar_Irradiance_LL":
xr.DataArray(
da.ones((1, ), chunks=1024),
attrs={"Slope": np.array([1.]), "Intercept": np.array([0.])},
da.ones((1, ), chunks=1024, dtype=np.float32),
attrs={"Slope": np.array([1.], dtype=np.float32),
"Intercept": np.array([0.], dtype=np.float32)},
dims=("_bands")),
"Calibration/IR_Cal_Coeff":
xr.DataArray(
da.ones((6, 4, num_scans), chunks=1024),
attrs={"Slope": np.array([1.] * 6), "Intercept": np.array([0.] * 6)},
da.ones((6, 4, num_scans), chunks=1024, dtype=np.float32),
attrs={"Slope": np.array([1.] * 6, dtype=np.float32),
"Intercept": np.array([0.] * 6, dtype=np.float32)},
dims=("_bands", "_coeffs", "_scans")),
}
return calibration
Expand All @@ -62,7 +66,7 @@ def _get_250m_data(num_scans, rows_per_scan, num_cols, filetype_info):

def_attrs = {fill_value_name: 65535,
"valid_range": [0, 4095],
"Slope": np.array([1.] * 1), "Intercept": np.array([0.] * 1)
"Slope": np.array([1.] * 1, dtype=np.float32), "Intercept": np.array([0.] * 1, dtype=np.float32)
}
nounits_attrs = {**def_attrs, **{"units": "NO"}}
radunits_attrs = {**def_attrs, **{"units": "mW/ (m2 cm-1 sr)"}}
Expand Down Expand Up @@ -116,7 +120,7 @@ def _get_500m_data(num_scans, rows_per_scan, num_cols):
da.ones((5, num_scans * rows_per_scan, num_cols), chunks=1024,
dtype=np.uint16),
attrs={
"Slope": np.array([1.] * 5), "Intercept": np.array([0.] * 5),
"Slope": np.array([1.] * 5, dtype=np.float32), "Intercept": np.array([0.] * 5, dtype=np.float32),
"FillValue": 65535,
"units": "NO",
"valid_range": [0, 4095],
Expand All @@ -128,7 +132,7 @@ def _get_500m_data(num_scans, rows_per_scan, num_cols):
da.ones((3, num_scans * rows_per_scan, num_cols), chunks=1024,
dtype=np.uint16),
attrs={
"Slope": np.array([1.] * 3), "Intercept": np.array([0.] * 3),
"Slope": np.array([1.] * 3, dtype=np.float32), "Intercept": np.array([0.] * 3, dtype=np.float32),
"FillValue": 65535,
"units": "mW/ (m2 cm-1 sr)",
"valid_range": [0, 25000],
Expand All @@ -150,56 +154,63 @@ def _get_1km_data(num_scans, rows_per_scan, num_cols, filetype_info):

data = {"Data/EV_1KM_LL":
xr.DataArray(da.ones((num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.uint16),
attrs={"Slope": np.array([1.]), "Intercept": np.array([0.]),
attrs={"Slope": np.array([1.], dtype=np.float32),
"Intercept": np.array([0.], dtype=np.float32),
"FillValue": 65535,
"units": "NO",
"valid_range": [0, 4095],
"long_name": b"1km Earth View Science Data"},
dims=("_rows", "_cols")),
f"{key_prefix}EV_1KM_RefSB":
xr.DataArray(da.ones((15, num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.uint16),
attrs={"Slope": np.array([1.] * 15), "Intercept": np.array([0.] * 15),
attrs={"Slope": np.array([1.] * 15, dtype=np.float32),
"Intercept": np.array([0.] * 15, dtype=np.float32),
fill_value_name: 65535,
"units": "NO",
"valid_range": [0, 4095],
"long_name": b"1km Earth View Science Data"},
dims=("_ref_bands", "_rows", "_cols")),
"Data/EV_1KM_Emissive":
xr.DataArray(da.ones((4, num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.uint16),
attrs={"Slope": np.array([1.] * 4), "Intercept": np.array([0.] * 4),
attrs={"Slope": np.array([1.] * 4, dtype=np.float32),
"Intercept": np.array([0.] * 4, dtype=np.float32),
"FillValue": 65535,
"units": "mW/ (m2 cm-1 sr)",
"valid_range": [0, 25000],
"long_name": b"1km Emissive Bands Earth View Science Data"},
dims=("_ir_bands", "_rows", "_cols")),
f"{key_prefix}EV_250_Aggr.1KM_RefSB":
xr.DataArray(da.ones((4, num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.uint16),
attrs={"Slope": np.array([1.] * 4), "Intercept": np.array([0.] * 4),
attrs={"Slope": np.array([1.] * 4, dtype=np.float32),
"Intercept": np.array([0.] * 4, dtype=np.float32),
fill_value_name: 65535,
"units": "NO",
"valid_range": [0, 4095],
"long_name": b"250m Reflective Bands Earth View Science Data Aggregated to 1 km"},
dims=("_ref250_bands", "_rows", "_cols")),
f"{key_prefix}EV_250_Aggr.1KM_Emissive":
xr.DataArray(da.ones((num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.uint16),
attrs={"Slope": np.array([1.]), "Intercept": np.array([0.]),
attrs={"Slope": np.array([1.], dtype=np.float32),
"Intercept": np.array([0.], dtype=np.float32),
fill_value_name: 65535,
"units": radunits,
"valid_range": [0, 4095],
"long_name": b"250m Emissive Bands Earth View Science Data Aggregated to 1 km"},
dims=("_rows", "_cols")) if is_mersi1 else
xr.DataArray(da.ones((2, num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.uint16),
attrs={"Slope": np.array([1.] * 2), "Intercept": np.array([0.] * 2),
attrs={"Slope": np.array([1.] * 2, dtype=np.float32),
"Intercept": np.array([0.] * 2, dtype=np.float32),
"FillValue": 65535,
"units": "mW/ (m2 cm-1 sr)",
"valid_range": [0, 4095],
"long_name": b"250m Emissive Bands Earth View Science Data Aggregated to 1 km"},
dims=("_ir250_bands", "_rows", "_cols")),
f"{key_prefix}SensorZenith":
xr.DataArray(
da.ones((num_scans * rows_per_scan, num_cols), chunks=1024),
da.ones((num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.int16),
attrs={
"Slope": np.array([.01] * 1), "Intercept": np.array([0.] * 1),
"Slope": np.array([.01] * 1, dtype=np.float32),
"Intercept": np.array([0.] * 1, dtype=np.float32),
"units": "degree",
"valid_range": [0, 28000],
},
Expand All @@ -212,7 +223,7 @@ def _get_250m_ll_data(num_scans, rows_per_scan, num_cols):
# Set some default attributes
def_attrs = {"FillValue": 65535,
"valid_range": [0, 4095],
"Slope": np.array([1.]), "Intercept": np.array([0.]),
"Slope": np.array([1.], dtype=np.float32), "Intercept": np.array([0.], dtype=np.float32),
"long_name": b"250m Earth View Science Data",
"units": "mW/ (m2 cm-1 sr)",
}
Expand All @@ -235,27 +246,27 @@ def _get_geo_data(num_scans, rows_per_scan, num_cols, prefix):
geo = {
prefix + "Longitude":
xr.DataArray(
da.ones((num_scans * rows_per_scan, num_cols), chunks=1024),
da.ones((num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.float64),
attrs={
"Slope": np.array([1.] * 1), "Intercept": np.array([0.] * 1),
"Slope": np.array([1.] * 1, dtype=np.float64), "Intercept": np.array([0.] * 1, dtype=np.float64),
"units": "degree",
"valid_range": [-90, 90],
},
dims=("_rows", "_cols")),
prefix + "Latitude":
xr.DataArray(
da.ones((num_scans * rows_per_scan, num_cols), chunks=1024),
da.ones((num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.float64),
attrs={
"Slope": np.array([1.] * 1), "Intercept": np.array([0.] * 1),
"Slope": np.array([1.] * 1, dtype=np.float64), "Intercept": np.array([0.] * 1, dtype=np.float64),
"units": "degree",
"valid_range": [-180, 180],
},
dims=("_rows", "_cols")),
prefix + "SensorZenith":
xr.DataArray(
da.ones((num_scans * rows_per_scan, num_cols), chunks=1024),
da.ones((num_scans * rows_per_scan, num_cols), chunks=1024, dtype=np.int16),
attrs={
"Slope": np.array([.01] * 1), "Intercept": np.array([0.] * 1),
"Slope": np.array([.01] * 1, dtype=np.float32), "Intercept": np.array([0.] * 1, dtype=np.float32),
"units": "degree",
"valid_range": [0, 28000],
},
Expand Down Expand Up @@ -288,13 +299,13 @@ def get_test_content(self, filename, filename_info, filetype_info):
"/attr/Observing Ending Time": "18:38:36.728",
}
fy3a_attrs = {
"/attr/VIR_Cal_Coeff": np.array([0.0, 1.0, 0.0] * 19),
"/attr/VIR_Cal_Coeff": np.array([0.0, 1.0, 0.0] * 19, dtype=np.float32),
}
fy3b_attrs = {
"/attr/VIS_Cal_Coeff": np.array([0.0, 1.0, 0.0] * 19),
"/attr/VIS_Cal_Coeff": np.array([0.0, 1.0, 0.0] * 19, dtype=np.float32),
}
fy3d_attrs = {
"/attr/Solar_Irradiance": np.array([1.0] * 19),
"/attr/Solar_Irradiance": np.array([1.0] * 19, dtype=np.float32),
}

global_attrs, ftype = self._set_sensor_attrs(global_attrs)
Expand Down Expand Up @@ -384,11 +395,11 @@ def _add_tbb_coefficients(self, global_attrs):
return

if "_1000" in self.filetype_info["file_type"]:
global_attrs["/attr/TBB_Trans_Coefficient_A"] = np.array([1.0] * 6)
global_attrs["/attr/TBB_Trans_Coefficient_B"] = np.array([0.0] * 6)
global_attrs["/attr/TBB_Trans_Coefficient_A"] = np.array([1.0] * 6, dtype=np.float32)
global_attrs["/attr/TBB_Trans_Coefficient_B"] = np.array([0.0] * 6, dtype=np.float32)
else:
global_attrs["/attr/TBB_Trans_Coefficient_A"] = np.array([0.0] * 6)
global_attrs["/attr/TBB_Trans_Coefficient_B"] = np.array([0.0] * 6)
global_attrs["/attr/TBB_Trans_Coefficient_A"] = np.array([0.0] * 6, dtype=np.float32)
global_attrs["/attr/TBB_Trans_Coefficient_B"] = np.array([0.0] * 6, dtype=np.float32)

@property
def _num_cols_for_file_type(self):
Expand Down Expand Up @@ -512,6 +523,10 @@ def test_all_resolutions(self):
_test_multi_resolutions(available_datasets, self.ir_250_bands, resolution, ir_num_results)

res = reader.load(self.bands_1000 + self.bands_250)
for i in res:
assert res[i].dtype == np.float32
assert res[i].values.dtype == np.float32

if resolution != "250":
assert len(res) == len(self.bands_1000 + self.bands_250)
else:
Expand Down

0 comments on commit a68c801

Please sign in to comment.