From f9a2d20c3752a6b907a85b13ec7ee1e4254ba7d2 Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Fri, 3 May 2024 08:42:18 +0200 Subject: [PATCH 1/4] Add support for GK-2B/GOCI2 data --- satpy/etc/composites/goci2.yaml | 148 +++++++++++++++++++++ satpy/etc/readers/goci2_l1_nc.yaml | 204 +++++++++++++++++++++++++++++ satpy/readers/goci2_l1_nc.py | 171 ++++++++++++++++++++++++ 3 files changed, 523 insertions(+) create mode 100644 satpy/etc/composites/goci2.yaml create mode 100644 satpy/etc/readers/goci2_l1_nc.yaml create mode 100644 satpy/readers/goci2_l1_nc.py diff --git a/satpy/etc/composites/goci2.yaml b/satpy/etc/composites/goci2.yaml new file mode 100644 index 0000000000..818c88c3a6 --- /dev/null +++ b/satpy/etc/composites/goci2.yaml @@ -0,0 +1,148 @@ +sensor_name: visir/goci2 + + +modifiers: + + rayleigh_corrected: + modifier: !!python/name:satpy.modifiers.PSPRayleighReflectance + atmosphere: us-standard + aerosol_type: rayleigh_only + prerequisites: + - name: 'L660' + modifiers: [sunz_corrected] + optional_prerequisites: + - satellite_azimuth_angle + - satellite_zenith_angle + - solar_azimuth_angle + - solar_zenith_angle + + rayleigh_corrected_marine_clean: + modifier: !!python/name:satpy.modifiers.PSPRayleighReflectance + atmosphere: us-standard + aerosol_type: marine_clean_aerosol + prerequisites: + - name: 'L660' + modifiers: [sunz_corrected] + optional_prerequisites: + - satellite_azimuth_angle + - satellite_zenith_angle + - solar_azimuth_angle + - solar_zenith_angle + + rayleigh_corrected_marine_tropical: + modifier: !!python/name:satpy.modifiers.PSPRayleighReflectance + atmosphere: tropical + aerosol_type: marine_tropical_aerosol + prerequisites: + - name: 'L660' + modifiers: [sunz_corrected] + optional_prerequisites: + - satellite_azimuth_angle + - satellite_zenith_angle + - solar_azimuth_angle + - solar_zenith_angle + + rayleigh_corrected_desert: + modifier: !!python/name:satpy.modifiers.PSPRayleighReflectance + atmosphere: tropical + aerosol_type: desert_aerosol + prerequisites: + - name: 'L660' + modifiers: [sunz_corrected] + optional_prerequisites: + - satellite_azimuth_angle + - satellite_zenith_angle + - solar_azimuth_angle + - solar_zenith_angle + + rayleigh_corrected_land: + modifier: !!python/name:satpy.modifiers.PSPRayleighReflectance + atmosphere: us-standard + aerosol_type: continental_average_aerosol + prerequisites: + - name: 'L660' + modifiers: [sunz_corrected] + optional_prerequisites: + - satellite_azimuth_angle + - satellite_zenith_angle + - solar_azimuth_angle + - solar_zenith_angle + + +composites: + true_color: + compositor: !!python/name:satpy.composites.GenericCompositor + prerequisites: + - name: 'L660' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + - name: 'L555' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + - name: 'L443' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + standard_name: true_color + + true_color_land: + compositor: !!python/name:satpy.composites.GenericCompositor + prerequisites: + - name: 'L660' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_land] + - name: 'L555' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_land] + - name: 'L443' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_land] + standard_name: true_color + + true_color_desert: + compositor: !!python/name:satpy.composites.GenericCompositor + prerequisites: + - name: 'L660' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_desert] + - name: 'L555' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_desert] + - name: 'L443' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_desert] + standard_name: true_color + + true_color_marine_clean: + compositor: !!python/name:satpy.composites.GenericCompositor + prerequisites: + - name: 'L660' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_marine_clean] + - name: 'L555' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_marine_clean] + - name: 'L443' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_marine_clean] + standard_name: true_color + + true_color_marine_tropical: + compositor: !!python/name:satpy.composites.GenericCompositor + prerequisites: + - name: 'L660' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_marine_tropical] + - name: 'L555' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_marine_tropical] + - name: 'L443' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected_marine_tropical] + standard_name: true_color + + true_color_raw: + compositor: !!python/name:satpy.composites.GenericCompositor + prerequisites: + - name: 'L660' + modifiers: [effective_solar_pathlength_corrected] + - name: 'L555' + modifiers: [effective_solar_pathlength_corrected] + - name: 'L443' + modifiers: [effective_solar_pathlength_corrected] + standard_name: true_color + + ocean_color: + compositor: !!python/name:satpy.composites.GenericCompositor + prerequisites: + - name: 'L660' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + - name: 'L555' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + - name: 'L443' + modifiers: [effective_solar_pathlength_corrected, rayleigh_corrected] + standard_name: ocean_color diff --git a/satpy/etc/readers/goci2_l1_nc.yaml b/satpy/etc/readers/goci2_l1_nc.yaml new file mode 100644 index 0000000000..0d4c0a7377 --- /dev/null +++ b/satpy/etc/readers/goci2_l1_nc.yaml @@ -0,0 +1,204 @@ +reader: + name: goci2_l1_nc + short_name: GOCI-II L1 NetCDF4 + long_name: GK-2B GOCI-II Level 1 products in netCDF4 format from NOSC + status: Beta + supports_fsspec: true + sensors: ['goci2'] + reader: !!python/name:satpy.readers.yaml_reader.GEOSegmentYAMLReader + # file pattern keys to sort files by with 'satpy.utils.group_files' + group_keys: ['start_time', 'platform_shortname', "slot"] + +file_types: + goci2_l1: + file_reader: !!python/name:satpy.readers.goci2_l1_nc.GOCI2L1NCFileHandler + file_patterns: + - '{platform:3s}_{sensor:5s}_{processing_level:3s}_{acquisition_date:%Y%m%d}_{acquisition_time:%H%M%S}_{coverage:2s}_S{slot:3d}_G{segment:3d}.nc' + - '{platform:3s}_{sensor:5s}_{processing_level:3s}_{acquisition_date:%Y%m%d}_{acquisition_time:%H%M%S}_{coverage:2s}_S{slot:3d}.nc' + + +datasets: +# --- Navigation Data --- + latitude: + name: latitude + file_type: goci2_l1 + file_key: latitude + standard_name: latitude + units: degrees_north + + longitude: + name: longitude + file_type: goci2_l1 + file_key: longitude + standard_name: longitude + units: degrees_east + +# --- Radiance Data --- + L380: + name: L380 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_380 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L412: + name: L412 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_412 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L443: + name: L443 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_443 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L490: + name: L490 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_490 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L510: + name: L510 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_510 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L555: + name: L555 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_555 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L620: + name: L620 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_620 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L660: + name: L660 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_660 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L680: + name: L680 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_680 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L709: + name: L709 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_709 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L745: + name: L745 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_745 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + L865: + name: L865 + wavelength: [0.450, 0.470, 0.490] + resolution: 250 + file_type: goci2_l1 + file_key: L_TOA_865 + coordinates: [longitude, latitude] + calibration: + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" diff --git a/satpy/readers/goci2_l1_nc.py b/satpy/readers/goci2_l1_nc.py new file mode 100644 index 0000000000..e640297aa2 --- /dev/null +++ b/satpy/readers/goci2_l1_nc.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2023 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Reader for GK-2B GOCI-II L1 products from NOSC. + +For more information about the data, see: + +The L1 data products from NOSC do not contain solar irradiance factors, which are necessary to transform +radiance to reflectance. The reader hardcodes these values based on calculation from `pyspectral`: +``` +from pyspectral.utils import convert2wavenumber, get_central_wave +from pyspectral.rsr_reader import RelativeSpectralResponse +from pyspectral.solar import SolarIrradianceSpectrum + +goci2 = RelativeSpectralResponse('GK-2B', 'goci2') +rsr, info = convert2wavenumber(goci2.rsr) +solar_irr = SolarIrradianceSpectrum(dlambda=0.0005, wavespace='wavenumber') +for band in goci2.band_names: + print(f"Solar Irradiance (GOCI2 band {band}) = {solar_irr.inband_solarirradiance(rsr[band]):12.6f}") +``` + +""" + +import logging +from datetime import datetime + +import xarray as xr + +from satpy.readers.netcdf_utils import NetCDF4FileHandler + +logger = logging.getLogger(__name__) + +GOCI2_SOLAR_IRRAD = {"L380": 15.389094, + "L412": 29.085551, + "L443": 37.377115, + "L490": 46.492590, + "L510": 48.745104, + "L555": 57.122588, + "L620": 65.075606, + "L660": 67.138780, + "L680": 68.981090, + "L709": 69.870426, + "L745": 70.849054, + "L865": 72.470692 + } + +GOCI2_SOLAR_IRRAD = {"L380": 1061.509391, + "L412": 1710.525606, + "L443": 1899.063039, + "L490": 1931.795505, + "L510": 1871.312024, + "L555": 1853.857524, + "L620": 1693.385039, + "L660": 1541.451305, + "L680": 1491.561883, + "L709": 1389.725683, + "L745": 1274.906171, + "L865": 971.089088} + +class GOCI2L1NCFileHandler(NetCDF4FileHandler): + """File handler for GOCI-II L1 official data in netCDF format.""" + + def __init__(self, filename, filename_info, filetype_info, mask_zeros=True): + """Initialize the reader.""" + super().__init__(filename, filename_info, filetype_info, auto_maskandscale=True) + + # By default, we mask nodata areas (zero values) near the edges of the extent + self.mask_zeros = mask_zeros + + self.attrs = self["/attrs"] + self.nc = self._merge_navigation_data(filetype_info["file_type"]) + + # Read metadata which are common to all datasets + self.nlines = self.nc.sizes["number_of_lines"] + self.ncols = self.nc.sizes["number_of_columns"] + + def _merge_navigation_data(self, filetype): + """Merge navigation data and geophysical data.""" + groups = ["geophysical_data", "navigation_data"] + return xr.merge([self[group] for group in groups]) + + @property + def start_time(self): + """Start timestamp of the dataset.""" + dt = self.attrs["observation_start_time"] + return datetime.strptime(dt, "%Y%m%d_%H%M%S") + + @property + def end_time(self): + """End timestamp of the dataset.""" + dt = self.attrs["observation_end_time"] + return datetime.strptime(dt, "%Y%m%d_%H%M%S") + + + def _calibrate(self, data, bname): + """Convert raw radiances into reflectance.""" + from pyorbital.astronomy import sun_earth_distance_correction + import numpy as np + + esd = sun_earth_distance_correction(self.start_time) + + factor = np.pi * esd * esd / GOCI2_SOLAR_IRRAD[bname] + + print(np.nanmax(data)) + print(factor, GOCI2_SOLAR_IRRAD[bname]) + + res = data * np.float32(factor) + + # Convert from 0-1 range to 0-100 + res = 100 * res + + res.attrs = data.attrs + print(np.nanmax(res)) + print("") + + res.attrs["units"] = "1" + res.attrs["long_name"] = "Bidirectional Reflectance" + res.attrs["standard_name"] = "toa_bidirectional_reflectance" + + return res + + def get_dataset(self, key, info): + """Load a dataset.""" + var = info["file_key"] + logger.debug("Reading in get_dataset %s.", var) + variable = self.nc[var] + + variable = variable.rename({"number_of_lines": "y", "number_of_columns": "x"}) + + # Some products may miss lon/lat standard_name, use name as base name if it is not already present + if variable.attrs.get("standard_name", None) is None: + variable.attrs.update({"standard_name": variable.name}) + variable.attrs.update({"platform_name": self.attrs['platform'], + "sensor": "goci2"}) + + # The data lists "0" as the valid minimum, but this is also used for fill values + # at the edge of the image extent. If required, filter these. + if self.mask_zeros: + variable = variable.where(variable != 0) + + # If required, convert raw radiances to reflectance + if "calibration" in key: + if key["calibration"] == "reflectance": + variable = self._calibrate(variable, info["name"]) + elif key["calibration"] is not "radiance": + raise ValueError(f"Calibration type {key['calibration']} not supported.") + + variable.attrs.update(key.to_dict()) + + variable.attrs["orbital_parameters"] = { + "satellite_nominal_longitude": self.attrs["sub_longitude"], + "satellite_nominal_latitude": 0., + "projection_longitude": self.attrs["longitude_of_projection_origin"], + "projection_latitude": self.attrs["latitude_of_projection_origin"], + "projection_altitude": self.attrs['perspective_point_height'] + } + return variable From d5113c87eb894484bf7852797522b3961868c930 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 06:56:32 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- satpy/readers/goci2_l1_nc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/readers/goci2_l1_nc.py b/satpy/readers/goci2_l1_nc.py index e640297aa2..8fd0b0eb37 100644 --- a/satpy/readers/goci2_l1_nc.py +++ b/satpy/readers/goci2_l1_nc.py @@ -104,15 +104,15 @@ def end_time(self): """End timestamp of the dataset.""" dt = self.attrs["observation_end_time"] return datetime.strptime(dt, "%Y%m%d_%H%M%S") - + def _calibrate(self, data, bname): """Convert raw radiances into reflectance.""" - from pyorbital.astronomy import sun_earth_distance_correction import numpy as np + from pyorbital.astronomy import sun_earth_distance_correction esd = sun_earth_distance_correction(self.start_time) - + factor = np.pi * esd * esd / GOCI2_SOLAR_IRRAD[bname] print(np.nanmax(data)) @@ -147,7 +147,7 @@ def get_dataset(self, key, info): variable.attrs.update({"platform_name": self.attrs['platform'], "sensor": "goci2"}) - # The data lists "0" as the valid minimum, but this is also used for fill values + # The data lists "0" as the valid minimum, but this is also used for fill values # at the edge of the image extent. If required, filter these. if self.mask_zeros: variable = variable.where(variable != 0) From f111bf4e49448b87b969eb603bbf963ee236317f Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Fri, 3 May 2024 09:27:46 +0200 Subject: [PATCH 3/4] Replace single with double quotes --- satpy/readers/goci2_l1_nc.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/satpy/readers/goci2_l1_nc.py b/satpy/readers/goci2_l1_nc.py index 8fd0b0eb37..5ea47c3427 100644 --- a/satpy/readers/goci2_l1_nc.py +++ b/satpy/readers/goci2_l1_nc.py @@ -26,9 +26,9 @@ from pyspectral.rsr_reader import RelativeSpectralResponse from pyspectral.solar import SolarIrradianceSpectrum -goci2 = RelativeSpectralResponse('GK-2B', 'goci2') +goci2 = RelativeSpectralResponse("GK-2B", "goci2") rsr, info = convert2wavenumber(goci2.rsr) -solar_irr = SolarIrradianceSpectrum(dlambda=0.0005, wavespace='wavenumber') +solar_irr = SolarIrradianceSpectrum(dlambda=0.0005, wavespace="wavenumber") for band in goci2.band_names: print(f"Solar Irradiance (GOCI2 band {band}) = {solar_irr.inband_solarirradiance(rsr[band]):12.6f}") ``` @@ -115,17 +115,12 @@ def _calibrate(self, data, bname): factor = np.pi * esd * esd / GOCI2_SOLAR_IRRAD[bname] - print(np.nanmax(data)) - print(factor, GOCI2_SOLAR_IRRAD[bname]) - res = data * np.float32(factor) # Convert from 0-1 range to 0-100 res = 100 * res res.attrs = data.attrs - print(np.nanmax(res)) - print("") res.attrs["units"] = "1" res.attrs["long_name"] = "Bidirectional Reflectance" @@ -144,7 +139,7 @@ def get_dataset(self, key, info): # Some products may miss lon/lat standard_name, use name as base name if it is not already present if variable.attrs.get("standard_name", None) is None: variable.attrs.update({"standard_name": variable.name}) - variable.attrs.update({"platform_name": self.attrs['platform'], + variable.attrs.update({"platform_name": self.attrs["platform"], "sensor": "goci2"}) # The data lists "0" as the valid minimum, but this is also used for fill values @@ -156,8 +151,8 @@ def get_dataset(self, key, info): if "calibration" in key: if key["calibration"] == "reflectance": variable = self._calibrate(variable, info["name"]) - elif key["calibration"] is not "radiance": - raise ValueError(f"Calibration type {key['calibration']} not supported.") + elif key["calibration"] != "radiance": + raise ValueError(f"Calibration type {key["calibration"]} not supported.") variable.attrs.update(key.to_dict()) @@ -166,6 +161,6 @@ def get_dataset(self, key, info): "satellite_nominal_latitude": 0., "projection_longitude": self.attrs["longitude_of_projection_origin"], "projection_latitude": self.attrs["latitude_of_projection_origin"], - "projection_altitude": self.attrs['perspective_point_height'] + "projection_altitude": self.attrs["perspective_point_height"] } return variable From 6527d45dcae81d4a30c0436dc91186360132ae88 Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Fri, 3 May 2024 11:38:47 +0200 Subject: [PATCH 4/4] Update calibration handler for GOCI2 --- satpy/readers/goci2_l1_nc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/readers/goci2_l1_nc.py b/satpy/readers/goci2_l1_nc.py index 5ea47c3427..ecca586c6d 100644 --- a/satpy/readers/goci2_l1_nc.py +++ b/satpy/readers/goci2_l1_nc.py @@ -149,10 +149,10 @@ def get_dataset(self, key, info): # If required, convert raw radiances to reflectance if "calibration" in key: - if key["calibration"] == "reflectance": + if key["calibration"].name == "reflectance": variable = self._calibrate(variable, info["name"]) - elif key["calibration"] != "radiance": - raise ValueError(f"Calibration type {key["calibration"]} not supported.") + elif key["calibration"].name != "radiance": + raise ValueError(f"Calibration type {key["calibration"].name} not supported.") variable.attrs.update(key.to_dict())