From 02aa8e6913f8f29472afc0ea68a10504ffa26ed0 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 May 2021 16:06:35 -0400 Subject: [PATCH 001/396] ENH: add support for de2 vefi --- pysatNASA/instruments/__init__.py | 2 +- pysatNASA/instruments/de2_vefi.py | 120 +++++++++++++++++++++++++++ pysatNASA/instruments/methods/de2.py | 3 +- 3 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 pysatNASA/instruments/de2_vefi.py diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index bc217be2..cc14e854 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -1,5 +1,5 @@ __all__ = ['cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', - 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', + 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'de2_vefi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', 'iss_fpmu', 'omni_hro', 'ses14_gold', diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py new file mode 100644 index 00000000..7f0f8037 --- /dev/null +++ b/pysatNASA/instruments/de2_vefi.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +"""Supports the Vector Electric Field Instrument (VEFI) on +Dynamics Explorer 2 (DE2). + +From CDAWeb: + + + + +References +---------- + + +Properties +---------- +platform + 'de2' +name + 'vefi' +inst_id + None Supported +tag + 'dca' or 'ac' + + +Warnings +-------- +- Currently no cleaning routine. + + +Authors +------- +J. Klenzing + +""" + +import datetime as dt +import functools +import warnings + +from pysat import logger +from pysat.instruments.methods import general as mm_gen +from pysatNASA.instruments.methods import de2 as mm_de2 +from pysatNASA.instruments.methods import cdaweb as cdw + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'de2' +name = 'vefi' +tags = {'dca': '500 ms cadence DC Averaged Electric Field data', + 'ac': '500ms cadence AC Electric Field data'} +inst_ids = {'': ['dca', 'ac']} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags}} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +def init(self): + """Initializes the Instrument object with instrument specific values. + + Runs once upon instantiation. + + """ + + logger.info(mm_de2.ackn_str) + self.acknowledgements = mm_de2.ackn_str + self.references = mm_de2.refs['vefi'] + return + + +def clean(self): + """Routine to return DE2 VEFI data cleaned to the specified level + + Note + ---- + 'clean' - Not specified + 'dusty' - Not specified + 'dirty' - Not specified + 'none' No cleaning applied, routine not called in this case. + + """ + warnings.warn('No cleaning routines available for DE2 VEFI') + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +datestr = '{year:04d}{month:02d}{day:02d}_v{version:02d}' +fname = 'de2_{tag:s}500ms_vefi_{datestr:s}.cdf' +supported_tags = {'': {tag: fname.format(tag=tag, datestr=datestr) + for tag in tags}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = cdw.load + +# Set the download routine +download_tags = {'': {tag: {'remote_dir': ''.join(('/pub/data/de/de2/', + 'electric_fields_vefi/', + tag, '500ms_vefi_cdaweb/', + '{year:4d}/')), + 'fname': fname.format(tag=tag, datestr=datestr)} + for tag in tags}} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/de2.py b/pysatNASA/instruments/methods/de2.py index 56b6c0e4..35ae1ae5 100644 --- a/pysatNASA/instruments/methods/de2.py +++ b/pysatNASA/instruments/methods/de2.py @@ -19,5 +19,6 @@ 'wats': ' '.join(('N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E.', 'Hedin, G. R. Carrignan, J. C. Maurer, The', 'Dynamics Explorer Wind and Temperature Spectrometer', - 'Space Sci. Instrum., v. 5, n. 4, p. 417, 1981.')) + 'Space Sci. Instrum., v. 5, n. 4, p. 417, 1981.')), + 'vefi': ' '.join(('TBD')) } From 69f2ad97f788de7ff3907601c95ab3e301212bae Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 May 2021 16:06:52 -0400 Subject: [PATCH 002/396] ENH: add support for de2 duct --- pysatNASA/instruments/de2_rpa.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 50a10839..8ba7df4c 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -75,13 +75,14 @@ platform = 'de2' name = 'rpa' -tags = {'': '2 sec cadence RPA data'} # this is the default cadence -inst_ids = {'': ['']} +tags = {'': '2 sec cadence RPA data', # this is the default cadence + 'duct': '16ms DUCT plasma density'} +inst_ids = {'': [tag for tag in tags]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags}} # ---------------------------------------------------------------------------- # Instrument methods @@ -96,7 +97,7 @@ def init(self): logger.info(mm_de2.ackn_str) self.acknowledgements = mm_de2.ackn_str - self.references = mm_de2.refs['lang'] + self.references = mm_de2.refs['rpa'] return @@ -122,8 +123,12 @@ def clean(self): # Use the default CDAWeb and pysat methods # Set the list_files routine -fname = 'de2_ion2s_rpa_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' -supported_tags = {'': {'': fname}} +datestr = '{year:04d}{month:02d}{day:02d}_v{version:02d}' +dataproduct = {'': 'ion2s', + 'duct': 'duct16ms'} +fname = 'de2_{dp:s}_rpa_{datestr:s}.cdf' +supported_tags = {'': {tag: fname.format(dp=dataproduct[tag], datestr=datestr) + for tag in tags}} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -131,10 +136,16 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_rpa', - '/ion2s_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} +download_tags = {'': {'': {'remote_dir': ''.join(('/pub/data/de/de2/plasma_rpa', + '/ion2s_cdaweb/{year:4d}/')), + 'fname': fname.format(dp=dataproduct[''], + datestr=datestr)}, + 'duct': {'remote_dir': ''.join(('/pub/data/de/de2/', + 'plasma_rpa/', + 'rpa16ms_cdaweb/', + '{year:4d}/')), + 'fname': fname.format(dp=dataproduct['duct'], + datestr=datestr)}}} download = functools.partial(cdw.download, supported_tags=download_tags) # Set the list_remote_files routine From cd04c6746e09b9aa87c9b54fd4678224d99ab4ad Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 May 2021 16:22:19 -0400 Subject: [PATCH 003/396] ENH: de2 idm --- pysatNASA/instruments/__init__.py | 2 +- pysatNASA/instruments/de2_idm.py | 110 +++++++++++++++++++++++++++ pysatNASA/instruments/methods/de2.py | 3 +- 3 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 pysatNASA/instruments/de2_idm.py diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index cc14e854..1d6dbd61 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -1,4 +1,4 @@ -__all__ = ['cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', +__all__ = ['cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_idm', 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'de2_vefi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', diff --git a/pysatNASA/instruments/de2_idm.py b/pysatNASA/instruments/de2_idm.py new file mode 100644 index 00000000..334e48be --- /dev/null +++ b/pysatNASA/instruments/de2_idm.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +"""Supports the Ion Drift Meter (IDM) instrument on +Dynamics Explorer 2 (DE2). + +From CDAWeb: + + +References +---------- + +Properties +---------- +platform + 'de2' +name + 'idm' +inst_id + None Supported +tag + None Supported + +Warnings +-------- +- Currently no cleaning routine. + +Authors +------- +J. Klenzing + +""" + +import datetime as dt +import functools +import warnings + +from pysat import logger +from pysat.instruments.methods import general as mm_gen +from pysatNASA.instruments.methods import de2 as mm_de2 +from pysatNASA.instruments.methods import cdaweb as cdw + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'de2' +name = 'idm' +tags = {'': '2 sec cadence IDM data'} # this is the default cadence +inst_ids = {'': ['']} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +def init(self): + """Initializes the Instrument object with instrument specific values. + + Runs once upon instantiation. + + """ + + logger.info(mm_de2.ackn_str) + self.acknowledgements = mm_de2.ackn_str + self.references = mm_de2.refs['idm'] + return + + +def clean(self): + """Routine to return DE2 IDM data cleaned to the specified level + + Note + ---- + 'clean' - Not specified + 'dusty' - Not specified + 'dirty' - Not specified + 'none' No cleaning applied, routine not called in this case. + + """ + warnings.warn('No cleaning routines available for DE2 IDM') + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = 'de2_vion250ms_idm_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' +supported_tags = {'': {'': fname}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = cdw.load + +# Set the download routine +basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_idm', + '/vion250ms_cdaweb/{year:4d}/')), + 'fname': fname} +download_tags = {'': {'': basic_tag}} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/de2.py b/pysatNASA/instruments/methods/de2.py index 35ae1ae5..3fa91fae 100644 --- a/pysatNASA/instruments/methods/de2.py +++ b/pysatNASA/instruments/methods/de2.py @@ -3,7 +3,8 @@ ackn_str = "The Dynamics Explorer 2 satellite data is provided through CDAWeb" -refs = {'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', +refs = {'idm': ' '.join(('TBD')), + 'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', 'Pinkus, and R. B. Kaplan, The Dynamics Explorer 2', 'Langmuir Probe (LANG), Space Sci. Instrum., v. 5,', 'n. 4, p. 493, 1981.')), From f5a957f35ef8d658313c4970d04a6b2aa9a1df5f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 9 Aug 2022 15:44:16 -0400 Subject: [PATCH 004/396] STY: update SEE data to xarray --- pysatNASA/instruments/timed_see.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 903058ab..95c3531b 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -17,15 +17,6 @@ None inst_id None supported -flatten_twod - If True, then two dimensional data is flattened across - columns. Name mangling is used to group data, first column - is 'name', last column is 'name_end'. In between numbers are - appended 'name_1', 'name_2', etc. All data for a given 2D array - may be accessed via, data.loc[:, 'item':'item_end'] - If False, then 2D data is stored as a series of DataFrames, - indexed by Epoch. data.loc[0, 'item'] - (default=True) Note ---- @@ -54,6 +45,7 @@ name = 'see' tags = {'': ''} inst_ids = {'': [tag for tag in tags.keys()]} +pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -104,7 +96,8 @@ def init(self): file_cadence=pds.DateOffset(months=1)) # Set the load routine -load = functools.partial(cdw.load, file_cadence=pds.DateOffset(months=1)) +load = functools.partial(cdw.load, pandas_format=pandas_format, + file_cadence=pds.DateOffset(months=1)) # Set the download routine basic_tag = {'remote_dir': ''.join(('/pub/data/timed/see/data/level3a_cdf', From f2cff288305979cdb7d2ac8fbd8ad8aed81b8f92 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 9 Aug 2022 15:45:57 -0400 Subject: [PATCH 005/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 870e128d..c1fd2255 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Maintenance * Reduce duplication of code in instrument modules * Include flake8 linting of docstrings and style in Github Actions + * Update TIMED SEE data to use xarray * Documentation * New logo added From 8d780f11a339a163d3f159e05091b4c7eaf5ebb6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 18 Oct 2022 18:17:34 -0400 Subject: [PATCH 006/396] ENH: add ace instruments --- pysatNASA/instruments/__init__.py | 3 +- pysatNASA/instruments/ace_epam.py | 87 ++++++++++++++++++++++++++++ pysatNASA/instruments/ace_mag.py | 86 +++++++++++++++++++++++++++ pysatNASA/instruments/ace_sis.py | 87 ++++++++++++++++++++++++++++ pysatNASA/instruments/ace_swepam.py | 87 ++++++++++++++++++++++++++++ pysatNASA/instruments/methods/ace.py | 12 ++++ 6 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 pysatNASA/instruments/ace_epam.py create mode 100644 pysatNASA/instruments/ace_mag.py create mode 100644 pysatNASA/instruments/ace_sis.py create mode 100644 pysatNASA/instruments/ace_swepam.py create mode 100644 pysatNASA/instruments/methods/ace.py diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index c3a8bfae..d97bcdcb 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -5,7 +5,8 @@ """ -__all__ = ['cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', +__all__ = ['ace_epam', 'ace_mag', 'ace_sis', 'ace_swepam', + 'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', diff --git a/pysatNASA/instruments/ace_epam.py b/pysatNASA/instruments/ace_epam.py new file mode 100644 index 00000000..f1d56483 --- /dev/null +++ b/pysatNASA/instruments/ace_epam.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +"""Module for the ACE EPAM instrument. + +References +---------- +???? + +Properties +---------- +platform + 'ace' +name + 'epam' +tag + None supported +inst_id + ['h1', 'h2', 'h3', 'k0', 'k1'] + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'epam' +tags = {'': ''} +inst_ids = {id: [''] for id in ['h1', 'h2', 'h3', 'k0', 'k1']} +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {'': dt.datetime(2022, 1, 1)} for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(('ac_{inst_id:s}_epm_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {} +for inst_id in inst_ids.keys(): + supported_tags[inst_id] = {'': fname.format(inst_id=inst_id)} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(cdw.load, pandas_format=pandas_format) + +# Set the download routine +remote_dir = '/pub/data/ace/epam/level_2_cdaweb/epm_{inst_id:s}/{{year:4d}}/' +download_tags = {} +for inst_id in inst_ids.keys(): + download_tags[inst_id] = {'': { + 'remote_dir': remote_dir.format(inst_id=inst_id), + 'fname': fname.format(inst_id=inst_id)}} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_mag.py b/pysatNASA/instruments/ace_mag.py new file mode 100644 index 00000000..24afad8c --- /dev/null +++ b/pysatNASA/instruments/ace_mag.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +"""Module for the ACE MAG instrument. + +References +---------- +???? + +Properties +---------- +platform + 'ace' +name + 'mag' +tag + None supported +inst_id + ['h0', 'h1', 'h2', 'h3', 'k0', 'k1', 'k2'] + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'mag' +tags = {'': ''} +inst_ids = {id: [''] for id in ['h0', 'h1', 'h2', 'h3', 'k0', 'k1', 'k2']} +pandas_format = False +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {'': dt.datetime(2022, 1, 1)} for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(('ac_{inst_id:s}_mfi_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {} +for inst_id in inst_ids.keys(): + supported_tags[inst_id] = {'': fname.format(inst_id=inst_id)} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(cdw.load, pandas_format=pandas_format) + +# Set the download routine +remote_dir = '/pub/data/ace/mag/level_2_cdaweb/mfi_{inst_id:s}/{{year:4d}}/' +download_tags = {} +for inst_id in inst_ids.keys(): + download_tags[inst_id] = {'': { + 'remote_dir': remote_dir.format(inst_id=inst_id), + 'fname': fname.format(inst_id=inst_id)}} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_sis.py b/pysatNASA/instruments/ace_sis.py new file mode 100644 index 00000000..d701daa1 --- /dev/null +++ b/pysatNASA/instruments/ace_sis.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +"""Module for the ACE SIS instrument. + +References +---------- +???? + +Properties +---------- +platform + 'ace' +name + 'mag' +tag + None supported +inst_id + ['h1', 'h2', 'k0'] + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'sis' +tags = {'': ''} +inst_ids = {id: [''] for id in ['h1', 'h2', 'k0']} +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {'': dt.datetime(2022, 1, 1)} for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(('ac_{inst_id:s}_sis_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {} +for inst_id in inst_ids.keys(): + supported_tags[inst_id] = {'': fname.format(inst_id=inst_id)} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(cdw.load, pandas_format=pandas_format) + +# Set the download routine +remote_dir = '/pub/data/ace/sis/level_2_cdaweb/sis_{inst_id:s}/{{year:4d}}/' +download_tags = {} +for inst_id in inst_ids.keys(): + download_tags[inst_id] = {'': { + 'remote_dir': remote_dir.format(inst_id=inst_id), + 'fname': fname.format(inst_id=inst_id)}} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_swepam.py b/pysatNASA/instruments/ace_swepam.py new file mode 100644 index 00000000..0ba1a663 --- /dev/null +++ b/pysatNASA/instruments/ace_swepam.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +"""Module for the ACE EPAM instrument. + +References +---------- +???? + +Properties +---------- +platform + 'ace' +name + 'swepam' +tag + None supported +inst_id + ['h0', 'h2', 'k0', 'k1'] + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'swepam' +tags = {'': ''} +inst_ids = {id: [''] for id in ['h0', 'h2', 'k0', 'k1']} +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {'': dt.datetime(2021, 1, 1)} for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(('ac_{inst_id:s}_swe_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {} +for inst_id in inst_ids.keys(): + supported_tags[inst_id] = {'': fname.format(inst_id=inst_id)} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(cdw.load, pandas_format=pandas_format) + +# Set the download routine +remote_dir = '/pub/data/ace/swepam/level_2_cdaweb/swe_{inst_id:s}/{{year:4d}}/' +download_tags = {} +for inst_id in inst_ids.keys(): + download_tags[inst_id] = {'': { + 'remote_dir': remote_dir.format(inst_id=inst_id), + 'fname': fname.format(inst_id=inst_id)}} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py new file mode 100644 index 00000000..5d5e7cfe --- /dev/null +++ b/pysatNASA/instruments/methods/ace.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for ACE data.""" + +ackn_str = ' '.join(("The ACE", + "(ACE) satellite data is provided through CDAWeb")) + +refs = {'mission': ' '.join(('???')), + 'epam': ' '.join(('???')), + 'mag': ' '.join(('???')), + 'sis': ' '.join(('???')), + 'swepam': ' '.join(('???')) + } From 70dfbd0b014e3ef4490308e528a5ccd1399855c1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 18 Oct 2022 18:18:28 -0400 Subject: [PATCH 007/396] DOC: update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3ec7e0d..f150f977 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Adds pysatCDF to optional requirements invoked via '[all]' option at installation * New Instruments * JPL GPS ROTI + * ACE EPAM + * ACE MAG + * ACE SIS + * ACE SWEPAM * Bug Fixes * Fixed a bug in metadata when loading GOLD Nmax data. * Fixed a bug in user feedback for `methods.cdaweb.download` From 7ec7ec27cd6cfc9f513147c6a4f2c6b03e0a7961 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 19 Oct 2022 11:49:28 -0400 Subject: [PATCH 008/396] STY: PEP8 --- pysatNASA/instruments/ace_epam.py | 2 +- pysatNASA/instruments/ace_mag.py | 2 +- pysatNASA/instruments/ace_sis.py | 2 +- pysatNASA/instruments/ace_swepam.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/ace_epam.py b/pysatNASA/instruments/ace_epam.py index f1d56483..fe3d11b7 100644 --- a/pysatNASA/instruments/ace_epam.py +++ b/pysatNASA/instruments/ace_epam.py @@ -27,8 +27,8 @@ from pysat.instruments.methods import general as mm_gen -from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- diff --git a/pysatNASA/instruments/ace_mag.py b/pysatNASA/instruments/ace_mag.py index 24afad8c..c942d836 100644 --- a/pysatNASA/instruments/ace_mag.py +++ b/pysatNASA/instruments/ace_mag.py @@ -27,8 +27,8 @@ from pysat.instruments.methods import general as mm_gen -from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- diff --git a/pysatNASA/instruments/ace_sis.py b/pysatNASA/instruments/ace_sis.py index d701daa1..578efcb6 100644 --- a/pysatNASA/instruments/ace_sis.py +++ b/pysatNASA/instruments/ace_sis.py @@ -27,8 +27,8 @@ from pysat.instruments.methods import general as mm_gen -from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- diff --git a/pysatNASA/instruments/ace_swepam.py b/pysatNASA/instruments/ace_swepam.py index 0ba1a663..f3a2ff66 100644 --- a/pysatNASA/instruments/ace_swepam.py +++ b/pysatNASA/instruments/ace_swepam.py @@ -27,8 +27,8 @@ from pysat.instruments.methods import general as mm_gen -from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- From 71a62be9e667efbbe9e6c9ee10493cbb99b88396 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 16:42:21 -0400 Subject: [PATCH 009/396] MAINT: migrate to pyproject.toml --- pyproject.toml | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 68 ----------------------------------------- setup.py | 17 ----------- tox.ini | 6 ++++ 4 files changed, 88 insertions(+), 85 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py create mode 100644 tox.ini diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..33f9ab53 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,82 @@ +[build-system] +requires = ["setuptools", "pip >= 10"] +build-backend = "setuptools.build_meta" + +[project] +name = "pysatNASA" +version = "0.0.4.1" +description = "pysat support for NASA Instruments" +readme = "README.md" +requires-python = ">=3.8" +license = {file = "LICENSE"} +authors = [ + {name = "Jeff Klenzing, et al", email = "pysat.developers@gmail.com"}, +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Topic :: Scientific/Engineering :: Astronomy", + "Topic :: Scientific/Engineering :: Physics", + "Topic :: Scientific/Engineering :: Atmospheric Science", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows" +] +keywords = [ + "pysat", + "ionosphere" +] +dependencies = [ + "netCDF4", + "requests", + "beautifulsoup4", + "lxml", + "cdflib >= 0.4.4", + "numpy", + "pandas", + "pysat >= 3.0.2", + "xarray" +] + +[project.optional-dependencies] +pysatcdf = ["pysatCDF"] +test = [ + "coveralls<3.3", + "pytest", + "pytest-cov", + "pytest-ordering" +] +doc = [ + "flake8", + "flake8-docstrings", + "hacking>=1.0", + "ipython", + "m2r2", + "numpydoc", + "sphinx", + "sphinx_rtd_theme" +] + +[project.urls] +Documentation = "https://pysatnasa.readthedocs.io/en/latest/" +Source = "https://github.com/pysat/pysatNASA" + +[tool.coverage.report] +omit = ["*/instruments/templates/"] + +[tool.pytest.ini_options] +addopts = "-vs --cov=pysatNASA" +markers = [ + "all_inst", + "download", + "no_download", + "load_options", + "first", + "second" +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 49912451..00000000 --- a/setup.cfg +++ /dev/null @@ -1,68 +0,0 @@ -[metadata] -name = pysatNASA -version = file: pysatNASA/version.txt -url = https://github.com/pysat/pysatNASA -author = Jeff Klenzing -author_email = jeffrey.klenzing@nasa.gov -description = 'pysat support for NASA Instruments' -keywords = - pysat - ionosphere -classifiers = - Development Status :: 3 - Alpha - Topic :: Scientific/Engineering :: Physics - Topic :: Scientific/Engineering :: Atmospheric Science - Intended Audience :: Science/Research - License :: OSI Approved :: BSD License - Natural Language :: English - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Operating System :: MacOS :: MacOS X - Operating System :: POSIX :: Linux -license_file = LICENSE -long_description = file: README.md -long_description_content_type = text/markdown - -[options] -python_requires = >= 3.5 -setup_requires = - setuptools >= 38.6 - pip >= 10 -include_package_data = True -zip_safe = False -packages = find: -install_requires = - netCDF4 - requests - beautifulsoup4 - lxml - cdflib - numpy - pandas - xarray - pysat - -[options.extras_require] -all = - pysatCDF - -[coverage:report] -omit = - */instruments/templates/ - -[flake8] -max-line-length = 80 -ignore = - D200 - D202 - W503 - -[tool:pytest] -markers = - all_inst: tests all instruments - download: tests for downloadable instruments - no_download: tests for instruments without download support - load_options: tests for instruments with additional options - first: first tests to run - second: second tests to run diff --git a/setup.py b/setup.py deleted file mode 100644 index 4c0bbd8e..00000000 --- a/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2020, Authors -# Full license can be found in License.md and AUTHORS.md -# ----------------------------------------------------------------------------- -"""Setup routines for pysatNASA. - -Note ----- -Package metadata stored in setup.cfg - -""" - -from setuptools import setup - -# Run setup -setup() diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..ccced987 --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[flake8] +max-line-length = 80 +ignore = + D200 + D202 + W503 From 03eefb013360669894c4006592c1f242d0223332 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 16:42:33 -0400 Subject: [PATCH 010/396] MAINT: one version --- pysatNASA/__init__.py | 9 +++------ pysatNASA/version.txt | 1 - 2 files changed, 3 insertions(+), 7 deletions(-) delete mode 100644 pysatNASA/version.txt diff --git a/pysatNASA/__init__.py b/pysatNASA/__init__.py index 1df92789..2f8f3370 100644 --- a/pysatNASA/__init__.py +++ b/pysatNASA/__init__.py @@ -6,13 +6,10 @@ """ -import os +import pkg_resources + from pysatNASA import constellations # noqa F401 from pysatNASA import instruments # noqa F401 # set version -here = os.path.abspath(os.path.dirname(__file__)) -version_filename = os.path.join(here, 'version.txt') -with open(version_filename, 'r') as version_file: - __version__ = version_file.read().strip() -del here, version_filename, version_file +__version__ = pkg_resources.get_distribution('pysatNASA').version diff --git a/pysatNASA/version.txt b/pysatNASA/version.txt deleted file mode 100644 index bcab45af..00000000 --- a/pysatNASA/version.txt +++ /dev/null @@ -1 +0,0 @@ -0.0.3 From 16c924a8479c55c079bc7a077f77554211028cd8 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 16:42:43 -0400 Subject: [PATCH 011/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3ec7e0d..84469a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Include flake8 linting of docstrings and style in Github Actions * Move OMNI HRO custom functions to a methods module * Deprecate OMNI HRO custom functions in instrument module + * Use pyproject.toml to manage setup * Documentation * New logo added From e71fce49b75769064360da1ba83730a5a9b7f695 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 16:42:55 -0400 Subject: [PATCH 012/396] TST: try new docs install --- .github/workflows/docs.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2f8cd626..fed56ae4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -25,8 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r test_requirements.txt - pip install -r requirements.txt + pip install .[doc] - name: Set up pysat run: | From 172c325d5ce0bb463f43694b29f0d03368e51c97 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:03:27 -0400 Subject: [PATCH 013/396] STY: docstring --- pysatNASA/instruments/ses14_gold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 05435fe2..e5f29ac8 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -68,7 +68,7 @@ def init(self): Runs once upon instantiation. Parameters - ----------- + ---------- self : pysat.Instrument Instrument class object From f6a774a3934192aad4c02e7913cd7b1f4bdad8cf Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:08:26 -0400 Subject: [PATCH 014/396] DOC/BUG: update docs --- README.md | 8 ++++---- docs/conf.py | 9 ++++----- docs/figures/{logo.png => pysatnasa_logo.png} | Bin docs/installation.rst | 13 ++++++------- docs/overview.rst | 2 +- pyproject.toml | 3 ++- 6 files changed, 17 insertions(+), 18 deletions(-) rename docs/figures/{logo.png => pysatnasa_logo.png} (100%) diff --git a/README.md b/README.md index f5782a5e..1fceb954 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
- pysatNASA + pysatNASA
# pysatNASA: pysat support for NASA Space Science instruments @@ -40,17 +40,17 @@ Currently, the main way to get pysatNASA is through github. git clone https://github.com/pysat/pysatNASA.git ``` -Change directories into the repository folder and run the setup.py file. For +Change directories into the repository folder and build the project. For a local install use the "--user" flag after "install". ``` cd pysatNASA/ -python setup.py install +pip install . ``` Note: pre-1.0.0 version ----------------------- -pysatNASA is currently in an initial development phase and requires pysat 3.0.0. +pysatNASA is currently in an initial development phase and requires pysat 3.0.4. # Using with pysat diff --git a/docs/conf.py b/docs/conf.py index adff053a..4a6c06f2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,6 +10,7 @@ import json import os +import pkg_resources import sys sys.path.insert(0, os.path.abspath('..')) @@ -62,9 +63,7 @@ # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. -doc_dir = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(doc_dir, "..", project, "version.txt"), "r") as fin: - version = fin.read().strip() +version = pkg_resources.get_distribution('pysatNASA').version release = '{:s}-alpha'.format(version) # Include alpha/beta/rc tags. # The language for content autogenerated by Sphinx. Refer to documentation @@ -72,7 +71,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -96,7 +95,7 @@ # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_logo = os.path.join(os.path.abspath('.'), 'figures', 'pysatnasa_logo.jpg') +html_logo = os.path.join(os.path.abspath('.'), 'figures', 'pysatnasa_logo.png') html_theme_options = {'logo_only': True} # Add any paths that contain custom static files (such as style sheets) here, diff --git a/docs/figures/logo.png b/docs/figures/pysatnasa_logo.png similarity index 100% rename from docs/figures/logo.png rename to docs/figures/pysatnasa_logo.png diff --git a/docs/installation.rst b/docs/installation.rst index c149522c..5975d708 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -40,24 +40,23 @@ Installation Options 2. Install pysatNASA: - Change directories into the repository folder and run the setup.py file. + Change directories into the repository folder and build the project. There are a few ways you can do this: A. Install on the system (root privileges required):: - sudo python setup.py install + sudo pip install . + B. Install at the user level:: - python setup.py install --user - C. Install with the intent to develop locally:: + pip install . + C. Install with the intent to develop locally:: - python setup.py develop --user -.. extras-require:: all - :setup.cfg: + pip install -e . .. _post-install: diff --git a/docs/overview.rst b/docs/overview.rst index 5cd631ad..4c8a8d4e 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -5,7 +5,7 @@ This is a library of ``pysat`` instrument modules and methods designed to suppor NASA instruments and missions archived at the Community Data Analysis Web portal. -.. image:: figures/pysatnasa_logo.jpg +.. image:: figures/pysatnasa_logo.png :width: 400px :align: center :alt: pysatNASA Logo, a blue planet with red orbiting python and the module name superimposed diff --git a/pyproject.toml b/pyproject.toml index 33f9ab53..591ff662 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "cdflib >= 0.4.4", "numpy", "pandas", - "pysat >= 3.0.2", + "pysat >= 3.0.4", "xarray" ] @@ -53,6 +53,7 @@ test = [ "pytest-ordering" ] doc = [ + "extras_require", "flake8", "flake8-docstrings", "hacking>=1.0", From 4f6bad4c5178a3de68f4f33830d9bfdac29d0bac Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:09:30 -0400 Subject: [PATCH 015/396] TST: update main job --- .github/workflows/main.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0ea9322..57b084e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,13 +31,12 @@ jobs: if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - pip install "xarray<2022.06" + pip install .[test,pysatcdf] - name: Install standard dependencies + if: ${{ matrix.numpy_ver == 'latest'}} run: | - pip install -r requirements.txt - pip install pysatCDF --no-binary=pysatCDF - pip install -r test_requirements.txt + pip install .[test,pysatcdf] - name: Set up pysat run: | @@ -51,8 +50,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: | - pytest -vs --cov=pysatNASA/ + run: pytest - name: Publish results to coveralls env: From 2e42767ebd457c2831b22219e530283179f21953 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:14:16 -0400 Subject: [PATCH 016/396] TST: remove pysatcdf --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 57b084e7..e532b1bb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,12 +31,12 @@ jobs: if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - pip install .[test,pysatcdf] + pip install .[test] - name: Install standard dependencies if: ${{ matrix.numpy_ver == 'latest'}} run: | - pip install .[test,pysatcdf] + pip install .[test] - name: Set up pysat run: | From ab26d38256ea5890f948caa380af29c8e4502ab2 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:15:36 -0400 Subject: [PATCH 017/396] MAINT: drop requirements files --- requirements.txt | 9 --------- test_requirements.txt | 13 ------------- 2 files changed, 22 deletions(-) delete mode 100644 requirements.txt delete mode 100644 test_requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a7141a59..00000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -netCDF4 -requests -beautifulsoup4 -lxml -cdflib>=0.4.4 -numpy -pandas -pysat>=3.0.2 -xarray diff --git a/test_requirements.txt b/test_requirements.txt deleted file mode 100644 index 6c5e6eff..00000000 --- a/test_requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -coveralls<3.3 -extras_require -flake8 -flake8-docstrings -hacking>=1.0 -ipython -m2r2 -numpydoc -pytest -pytest-cov -pytest-ordering -sphinx -sphinx_rtd_theme From 4ac41675d03ffab950c0cd45d4b07dad8937dbab Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:21:37 -0400 Subject: [PATCH 018/396] BUG: install flake8 --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e532b1bb..da6f8cfa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,12 +31,12 @@ jobs: if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - pip install .[test] + pip install .[doc,test] - name: Install standard dependencies if: ${{ matrix.numpy_ver == 'latest'}} run: | - pip install .[test] + pip install .[doc,test] - name: Set up pysat run: | From 64496cf6e250dee2991ce8e46316070ade32b490 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:29:46 -0400 Subject: [PATCH 019/396] STY: whitespace --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 591ff662..33571708 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dependencies = [ [project.optional-dependencies] pysatcdf = ["pysatCDF"] test = [ - "coveralls<3.3", + "coveralls < 3.3", "pytest", "pytest-cov", "pytest-ordering" @@ -56,7 +56,7 @@ doc = [ "extras_require", "flake8", "flake8-docstrings", - "hacking>=1.0", + "hacking >= 1.0", "ipython", "m2r2", "numpydoc", From 1301eeeec9e3606227624f5f2cfd39e148c56ec5 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:36:04 -0400 Subject: [PATCH 020/396] STY: group optional dependencies --- .github/workflows/main.yml | 4 ++-- pyproject.toml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index da6f8cfa..e532b1bb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,12 +31,12 @@ jobs: if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - pip install .[doc,test] + pip install .[test] - name: Install standard dependencies if: ${{ matrix.numpy_ver == 'latest'}} run: | - pip install .[doc,test] + pip install .[test] - name: Set up pysat run: | diff --git a/pyproject.toml b/pyproject.toml index 33571708..9148089c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,15 +48,15 @@ dependencies = [ pysatcdf = ["pysatCDF"] test = [ "coveralls < 3.3", + "flake8", + "flake8-docstrings", + "hacking >= 1.0", "pytest", "pytest-cov", "pytest-ordering" ] doc = [ "extras_require", - "flake8", - "flake8-docstrings", - "hacking >= 1.0", "ipython", "m2r2", "numpydoc", From 2788454288da08f1598b9783b394e3cd4be0169d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:36:41 -0400 Subject: [PATCH 021/396] DOC: show optional requirements --- docs/installation.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 5975d708..0431da34 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -58,6 +58,15 @@ Installation Options pip install -e . +.. extras-require:: pysatcdf + :pyproject: + +.. extras-require:: test + :pyproject: + +.. extras-require:: doc + :pyproject: + .. _post-install: Post Installation From 2a8b50c99fbd56e3075141e279edb6d17c646865 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:40:20 -0400 Subject: [PATCH 022/396] TST: improve nep29 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e532b1bb..09c22e1c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,7 +31,7 @@ jobs: if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - pip install .[test] + pip install --upgrade-strategy only-if-needed .[test] - name: Install standard dependencies if: ${{ matrix.numpy_ver == 'latest'}} From 4b2a31756e03918e2fc9958589ab7d27b1adf10e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 25 Oct 2022 17:40:29 -0400 Subject: [PATCH 023/396] TST: optional pysatcdf --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 09c22e1c..52735d1a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,6 +38,9 @@ jobs: run: | pip install .[test] + - name: Install pysatCDF + run: pip install pysatCDF --no-binary=pysatCDF + - name: Set up pysat run: | mkdir pysatData From cd7a86f170385024e4c14695c651b0b637735f89 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 25 Oct 2022 22:14:26 -0400 Subject: [PATCH 024/396] Update main.yml --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 52735d1a..4d00d2ac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,6 +31,8 @@ jobs: if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} + # Force install version compatible with NEP29 + pip install "pandas<1.5" pip install --upgrade-strategy only-if-needed .[test] - name: Install standard dependencies @@ -58,4 +60,4 @@ jobs: - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls --rcfile=setup.cfg --service=github + run: coveralls --rcfile=pyproject.toml --service=github From d48ff062177573271d554366f18b4f60628ad183 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 26 Oct 2022 10:48:12 -0400 Subject: [PATCH 025/396] STY: cleanup --- .github/workflows/main.yml | 1 + docs/installation.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0def20f7..b4f19747 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,6 +41,7 @@ jobs: pip install .[test] - name: Install pysatCDF + # Need custom install until pysatCDF is updated run: pip install pysatCDF --no-binary=pysatCDF - name: Set up pysat diff --git a/docs/installation.rst b/docs/installation.rst index 0431da34..7ba6d0ba 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -51,7 +51,7 @@ Installation Options B. Install at the user level:: - pip install . + pip install --user . C. Install with the intent to develop locally:: From e22664b9a002ef0e6f5a7cfe822eb7909ecf7360 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Oct 2022 11:02:16 -0400 Subject: [PATCH 026/396] BUG: optional pysatCDF --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b4f19747..fec0dc3f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,6 +41,8 @@ jobs: pip install .[test] - name: Install pysatCDF + # Optional package, continue with tests if install fails + continue-on-error: true # Need custom install until pysatCDF is updated run: pip install pysatCDF --no-binary=pysatCDF From 4472fa770e86d33f077f2040ac98e682d5262aa1 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 2 Nov 2022 17:15:33 -0400 Subject: [PATCH 027/396] STY: pep8 --- pysatNASA/instruments/de2_vefi.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index 7f0f8037..cdb5b252 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -4,9 +4,6 @@ From CDAWeb: - - - References ---------- From 62258551d1249ab3ac1660102984b6e54b3cb942 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 3 Nov 2022 09:26:33 -0400 Subject: [PATCH 028/396] STY: pep8 --- pysatNASA/instruments/de2_vefi.py | 45 +++++++------------------------ 1 file changed, 10 insertions(+), 35 deletions(-) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index cdb5b252..9f580ff3 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -1,12 +1,11 @@ -# -*- coding: utf-8 -*- -"""Supports the Vector Electric Field Instrument (VEFI) on -Dynamics Explorer 2 (DE2). +"""Module for the DE2 VEFI instrument. From CDAWeb: +TBA References ---------- - +TBD Properties ---------- @@ -33,12 +32,11 @@ import datetime as dt import functools -import warnings -from pysat import logger from pysat.instruments.methods import general as mm_gen -from pysatNASA.instruments.methods import de2 as mm_de2 from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import de2 as mm_de2 +from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- # Instrument attributes @@ -54,38 +52,15 @@ _test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags}} + # ---------------------------------------------------------------------------- # Instrument methods +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -def init(self): - """Initializes the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - logger.info(mm_de2.ackn_str) - self.acknowledgements = mm_de2.ackn_str - self.references = mm_de2.refs['vefi'] - return - - -def clean(self): - """Routine to return DE2 VEFI data cleaned to the specified level - - Note - ---- - 'clean' - Not specified - 'dusty' - Not specified - 'dirty' - Not specified - 'none' No cleaning applied, routine not called in this case. - - """ - warnings.warn('No cleaning routines available for DE2 VEFI') - - return - +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn # ---------------------------------------------------------------------------- # Instrument functions From 221dff912c8f3604a88d6cfe5d0da4bb170ab700 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 3 Nov 2022 10:46:06 -0400 Subject: [PATCH 029/396] BUG: typo --- pysatNASA/instruments/de2_rpa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 92140472..069cf28f 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -26,7 +26,7 @@ spectrum; and the concentration of H+, He+, O+, and Fe+, and of molecular ions near perigee. -It includes the DUCT portion of the high resolutiondata from the Dynamics +It includes the DUCT portion of the high resolution data from the Dynamics Explorer 2 (DE-2) Retarding Potential Analyzer (RPA) for the whole DE-2 mission time period in ASCII format. This version was generated at NSSDC from the PI-provided binary data (SPIO-00232). The DUCT files include RPA measurements From e9aa6e72d8fa063702f9fa939e0d9df262076a93 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 3 Nov 2022 10:46:15 -0400 Subject: [PATCH 030/396] ENH: de2 idm --- pysatNASA/instruments/__init__.py | 2 +- pysatNASA/instruments/de2_idm.py | 131 +++++++++++++++++++++++++++ pysatNASA/instruments/methods/de2.py | 3 +- 3 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 pysatNASA/instruments/de2_idm.py diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index f9772e8a..19856e4e 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -5,7 +5,7 @@ """ -__all__ = ['cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', +__all__ = ['cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_idm', 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'de2_vefi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', diff --git a/pysatNASA/instruments/de2_idm.py b/pysatNASA/instruments/de2_idm.py new file mode 100644 index 00000000..e1d79413 --- /dev/null +++ b/pysatNASA/instruments/de2_idm.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +"""The DE2 NACS instrument. + +Supports the Neutral Atmosphere Composition Spectrometer (NACS) instrument +on Dynamics Explorer 2 (DE2). + +From CDAWeb: + +The Neutral Atmosphere Composition Spectrometer (NACS) was designed to obtain +in situ measurements of the neutral atmospheric composition and to study the +variations of the neutral atmosphere in response to energy coupled into it from +the magnetosphere. Because temperature enhancements, large-scale circulation +cells, and wave propagation are produced by energy input (each of which +posseses a specific signature in composition variation), the measurements +permitted the study of the partition, flow, and deposition of energy from the +magnetosphere. Specifically, the investigation objective was to characterize +the composition of the neutral atmosphere with particular emphasis on +variability in constituent densities driven by interactions in the atmosphere, +ionosphere, and magnetosphere system. The quadrupole mass spectrometer used was +nearly identical to those flown on the AE-C, -D, and -E missions. The electron- +impact ion source was used in a closed mode. Atmospheric particles entered an +antechamber through a knife-edged orifice, where they were thermalized to the +instrument temperature. The ions with the selected charge-to-mass ratios had +stable trajectories through the hyperbolic electric field, exited the analyzer, +and entered the detection system. An off-axis beryllium-copper dynode +multiplier operating at a gain of 2.E6 provided an output pulse of electrons +for each ion arrival. The detector output had a pulse rate proportional to the +neutral density in the ion source of the selected mass. The instrument also +included two baffles that scanned across the input orifice for optional +measurement of the zonal and vertical components of the neutral wind. The mass +select system provided for 256 mass values between 0 and 51 atomic mass units +(u) or each 0.2 u. It was possible to call any one of these mass numbers into +each of eight 0.016-s intervals. This sequence was repeated each 0.128 s. + +This data set includes daily files of the PI-provided DE-2 NACS 1-second data +and corresponding orbit parameters. The data set was generated at NSSDC from +the original PI-provided data and software (SPTH-00010) and from the +orbit/attitude database and software that is part of the DE-2 UA data set +(SPIO-00174). The original NACS data were provided by the PI team in a highly +compressed VAX/VMS binary format on magnetic tapes. The data set covers the +whole DE-2 mission time period. Each data point is an average over the normally +8 measurements per second. Densities and relative errors are provided for +atomic oxygen (O), molecular nitrogen (N2), helium (He), atomic nitrogen (N), +and argon (Ar). The data quality is generally quite good below 500 km, but +deteriorates towards higher altitudes as oxygen and molecular nitrogen approach +their background values (which could only be determined from infrequent +spinning orbits) and the count rate for Ar becomes very low. The difference +between minimum (background) and maximum count rate for atomic nitrogen +(estimated from mass 30) was so small that results are generally poor. Data +were lost between 12 March 1982 and 31 March 1982 when the counter overflowed. + + +References +---------- +G. R. Carrignan, B. P. Block, J. C. Maurer, A. E. Hedin, C. A. Reber, +N. W. Spencer, "The neutral mass spectrometer on Dynamics Explorer B", +Space Sci. Instrum., 5, 429-441, 1981. + +Properties +---------- +platform + 'de2' +name + 'idm' +inst_id + None Supported +tag + None Supported + +Warnings +-------- +- Currently no cleaning routine. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import de2 as mm_de2 +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'de2' +name = 'idm' +tags = {'': '1 s cadence Neutral Atmosphere Composition Spectrometer data'} +inst_ids = {'': ['']} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_de2, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = 'de2_vion250ms_idm_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' +supported_tags = {'': {'': fname}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Use the default CDAWeb method +load = cdw.load + +# Support download routine +basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_idm', + '/vion250ms_cdaweb/{year:4d}/')), + 'fname': fname} +download_tags = {'': {'': basic_tag}} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Support listing files currently on CDAWeb +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/de2.py b/pysatNASA/instruments/methods/de2.py index 126cd112..39d41f42 100644 --- a/pysatNASA/instruments/methods/de2.py +++ b/pysatNASA/instruments/methods/de2.py @@ -3,7 +3,8 @@ ackn_str = "The Dynamics Explorer 2 satellite data is provided through CDAWeb" -refs = {'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', +refs = {'idm': ' '.join(('TBD')), + 'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', 'Pinkus, and R. B. Kaplan, The Dynamics Explorer 2', 'Langmuir Probe (LANG), Space Sci. Instrum., v. 5,', 'n. 4, p. 493, 1981.')), From 16b21006b9cfaf5dfe6c40dc69872fda492003bc Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 4 Nov 2022 08:41:02 -0400 Subject: [PATCH 031/396] BUG: update idm load --- pysatNASA/instruments/de2_idm.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/de2_idm.py b/pysatNASA/instruments/de2_idm.py index e1d79413..697e9969 100644 --- a/pysatNASA/instruments/de2_idm.py +++ b/pysatNASA/instruments/de2_idm.py @@ -89,6 +89,7 @@ name = 'idm' tags = {'': '1 s cadence Neutral Atmosphere Composition Spectrometer data'} inst_ids = {'': ['']} +pandas_format=False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -117,7 +118,21 @@ supported_tags=supported_tags) # Use the default CDAWeb method -load = cdw.load +load = functools.partial(cdw.load, pandas_format=pandas_format) + +def preprocess(self): + """Apply DE2 IDM default attributes. + + Note + ---- + Corrects the epoch names + + """ + + self.data = self.data.rename({'record0': 'Epoch_velZ', + 'record1': 'Epoch_velY'}) + return + # Support download routine basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_idm', From 7e35ffc30d39af93d590673175f7e0429aab4685 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 4 Nov 2022 08:51:58 -0400 Subject: [PATCH 032/396] STY: pep8 --- pysatNASA/instruments/de2_idm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/de2_idm.py b/pysatNASA/instruments/de2_idm.py index 697e9969..790d2656 100644 --- a/pysatNASA/instruments/de2_idm.py +++ b/pysatNASA/instruments/de2_idm.py @@ -89,7 +89,7 @@ name = 'idm' tags = {'': '1 s cadence Neutral Atmosphere Composition Spectrometer data'} inst_ids = {'': ['']} -pandas_format=False +pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -120,6 +120,7 @@ # Use the default CDAWeb method load = functools.partial(cdw.load, pandas_format=pandas_format) + def preprocess(self): """Apply DE2 IDM default attributes. From 349c59b85ad034a403af41e6ef7f040d029cc4dd Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 4 Nov 2022 08:58:35 -0400 Subject: [PATCH 033/396] DOC: VEFI refs --- pysatNASA/instruments/de2_vefi.py | 23 ++++++++++++++++------- pysatNASA/instruments/methods/de2.py | 5 ++++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index 9f580ff3..0643dbe4 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -1,11 +1,24 @@ """Module for the DE2 VEFI instrument. -From CDAWeb: -TBA +From CDAWeb (adpated): +This directory gathers data for the VEFI instrument that flew on the DE 2 spacecraft +which was launched on 3 August 1981 into an elliptical orbit with an altitude range of +300 km to 1000 km and re-entered the atmosphere on 19 February 1983. + +dca (NSSDC ID: 81-070B-02C) + +This data set contains the averaged (2 samples per second) DC electric fields in +spacecraft coordinates and orbit information in ASCII format. + +ac (NSSDC ID: 81-070B-02E) + +This data set contains the averaged AC electric field data (1 or 2 points per +second) and orbit information. References ---------- -TBD +Maynard, N. C., E. A. Bielecki, H. G. Burdick, Instrumentation for vector +electric field measurements from DE-B, Space Sci. Instrum., 5, 523, 1981. Properties ---------- @@ -24,10 +37,6 @@ - Currently no cleaning routine. -Authors -------- -J. Klenzing - """ import datetime as dt diff --git a/pysatNASA/instruments/methods/de2.py b/pysatNASA/instruments/methods/de2.py index 39d41f42..50e70a2e 100644 --- a/pysatNASA/instruments/methods/de2.py +++ b/pysatNASA/instruments/methods/de2.py @@ -21,5 +21,8 @@ 'Hedin, G. R. Carrignan, J. C. Maurer, The', 'Dynamics Explorer Wind and Temperature Spectrometer', 'Space Sci. Instrum., v. 5, n. 4, p. 417, 1981.')), - 'vefi': ' '.join(('TBD')) + 'vefi': ' '.join(('Maynard, N. C., E. A. Bielecki, H. G. Burdick,', + 'Instrumentation for vector electric field', + 'measurements from DE-B, Space Sci. Instrum., 5,', + '523, 1981.')) } From 3f25a7563650883d1ebd413245fae3c791720501 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 4 Nov 2022 09:02:36 -0400 Subject: [PATCH 034/396] DOC: IDM records --- pysatNASA/instruments/de2_idm.py | 65 +++++++--------------------- pysatNASA/instruments/methods/de2.py | 5 ++- 2 files changed, 20 insertions(+), 50 deletions(-) diff --git a/pysatNASA/instruments/de2_idm.py b/pysatNASA/instruments/de2_idm.py index 790d2656..7e878e55 100644 --- a/pysatNASA/instruments/de2_idm.py +++ b/pysatNASA/instruments/de2_idm.py @@ -1,60 +1,27 @@ # -*- coding: utf-8 -*- -"""The DE2 NACS instrument. +"""The DE2 IDM instrument. -Supports the Neutral Atmosphere Composition Spectrometer (NACS) instrument +Supports the Ion Drift Meter (IDM) instrument on Dynamics Explorer 2 (DE2). -From CDAWeb: - -The Neutral Atmosphere Composition Spectrometer (NACS) was designed to obtain -in situ measurements of the neutral atmospheric composition and to study the -variations of the neutral atmosphere in response to energy coupled into it from -the magnetosphere. Because temperature enhancements, large-scale circulation -cells, and wave propagation are produced by energy input (each of which -posseses a specific signature in composition variation), the measurements -permitted the study of the partition, flow, and deposition of energy from the -magnetosphere. Specifically, the investigation objective was to characterize -the composition of the neutral atmosphere with particular emphasis on -variability in constituent densities driven by interactions in the atmosphere, -ionosphere, and magnetosphere system. The quadrupole mass spectrometer used was -nearly identical to those flown on the AE-C, -D, and -E missions. The electron- -impact ion source was used in a closed mode. Atmospheric particles entered an -antechamber through a knife-edged orifice, where they were thermalized to the -instrument temperature. The ions with the selected charge-to-mass ratios had -stable trajectories through the hyperbolic electric field, exited the analyzer, -and entered the detection system. An off-axis beryllium-copper dynode -multiplier operating at a gain of 2.E6 provided an output pulse of electrons -for each ion arrival. The detector output had a pulse rate proportional to the -neutral density in the ion source of the selected mass. The instrument also -included two baffles that scanned across the input orifice for optional -measurement of the zonal and vertical components of the neutral wind. The mass -select system provided for 256 mass values between 0 and 51 atomic mass units -(u) or each 0.2 u. It was possible to call any one of these mass numbers into -each of eight 0.016-s intervals. This sequence was repeated each 0.128 s. - -This data set includes daily files of the PI-provided DE-2 NACS 1-second data -and corresponding orbit parameters. The data set was generated at NSSDC from -the original PI-provided data and software (SPTH-00010) and from the -orbit/attitude database and software that is part of the DE-2 UA data set -(SPIO-00174). The original NACS data were provided by the PI team in a highly -compressed VAX/VMS binary format on magnetic tapes. The data set covers the -whole DE-2 mission time period. Each data point is an average over the normally -8 measurements per second. Densities and relative errors are provided for -atomic oxygen (O), molecular nitrogen (N2), helium (He), atomic nitrogen (N), -and argon (Ar). The data quality is generally quite good below 500 km, but -deteriorates towards higher altitudes as oxygen and molecular nitrogen approach -their background values (which could only be determined from infrequent -spinning orbits) and the count rate for Ar becomes very low. The difference -between minimum (background) and maximum count rate for atomic nitrogen -(estimated from mass 30) was so small that results are generally poor. Data -were lost between 12 March 1982 and 31 March 1982 when the counter overflowed. +From CDAWeb (adapted): + +This directory gathers data for the IDM instrument that flew on the DE 2 spacecraft +which was launched on 3 August 1981 into an elliptical orbit with an altitude range +of 300 km to 1000 km and re-entered the atmosphere on 19 February 1983. + +(NSSDC ID: 81-070B-06D) +This data set provides the cross track ion drift with a time resolution of +125 milli-seconds for the time period from Aug 15, 1981 to Feb 16, 1983. The +ASCII version was generated at NSSDC from the originally PI-submitted binary +data. References ---------- -G. R. Carrignan, B. P. Block, J. C. Maurer, A. E. Hedin, C. A. Reber, -N. W. Spencer, "The neutral mass spectrometer on Dynamics Explorer B", -Space Sci. Instrum., 5, 429-441, 1981. +Heelis, R. A., W. B. Hanson, C. R. Lippincott, D. R. Zuccaro, L. L. Harmon, +B. J. Holt, J. E. Doherty, R. A. Power, The ion drift meter for Dynamics +Explorer-B, Space Sci. Instrum., 5, 511, 1981. Properties ---------- diff --git a/pysatNASA/instruments/methods/de2.py b/pysatNASA/instruments/methods/de2.py index 50e70a2e..90edf2a7 100644 --- a/pysatNASA/instruments/methods/de2.py +++ b/pysatNASA/instruments/methods/de2.py @@ -3,7 +3,10 @@ ackn_str = "The Dynamics Explorer 2 satellite data is provided through CDAWeb" -refs = {'idm': ' '.join(('TBD')), +refs = {'idm': ' '.join(('Heelis, R. A., W. B. Hanson, C. R. Lippincott, D. R.', + 'Zuccaro, L. L. Harmon, B. J. Holt, J. E. Doherty, R.', + 'A. Power, The ion drift meter for Dynamics', + 'Explorer-B, Space Sci. Instrum., 5, 511, 1981.')), 'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', 'Pinkus, and R. B. Kaplan, The Dynamics Explorer 2', 'Langmuir Probe (LANG), Space Sci. Instrum., v. 5,', From 514bc012085484199fd856a8741677f08b00c466 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 4 Nov 2022 09:18:44 -0400 Subject: [PATCH 035/396] DOC: update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfb01196..e75fba19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +## [0.1.0] - 202X-XX-XX +* New Instruments + * DE2 Ion Drift Meter + * DE2 RPA DUCT (high rate plasma density) + * DE2 Vector Electric Field Instrument + ## [0.0.4] - 2022-11-11 * Update instrument tests with new test class * Support xarray datasets through cdflib From c247687da158f046c0252fdd4c48c2491afb4ea2 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 4 Nov 2022 09:37:58 -0400 Subject: [PATCH 036/396] STY: pep8 --- pysatNASA/instruments/de2_idm.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/de2_idm.py b/pysatNASA/instruments/de2_idm.py index 7e878e55..f121e32f 100644 --- a/pysatNASA/instruments/de2_idm.py +++ b/pysatNASA/instruments/de2_idm.py @@ -6,9 +6,10 @@ From CDAWeb (adapted): -This directory gathers data for the IDM instrument that flew on the DE 2 spacecraft -which was launched on 3 August 1981 into an elliptical orbit with an altitude range -of 300 km to 1000 km and re-entered the atmosphere on 19 February 1983. +This directory gathers data for the IDM instrument that flew on the DE 2 +spacecraft which was launched on 3 August 1981 into an elliptical orbit +with an altitude range of 300 km to 1000 km and re-entered the atmosphere on +19 February 1983. (NSSDC ID: 81-070B-06D) This data set provides the cross track ion drift with a time resolution of From 713f530d09a8c6b0ab350f227c2eeb02c5210b52 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 4 Nov 2022 09:38:28 -0400 Subject: [PATCH 037/396] STY: pep8 --- pysatNASA/instruments/de2_vefi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index 0643dbe4..becfcd38 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -1,9 +1,10 @@ """Module for the DE2 VEFI instrument. From CDAWeb (adpated): -This directory gathers data for the VEFI instrument that flew on the DE 2 spacecraft -which was launched on 3 August 1981 into an elliptical orbit with an altitude range of -300 km to 1000 km and re-entered the atmosphere on 19 February 1983. +This directory gathers data for the VEFI instrument that flew on the DE 2 +spacecraft which was launched on 3 August 1981 into an elliptical orbit with +an altitude range of 300 km to 1000 km and re-entered the atmosphere on +19 February 1983. dca (NSSDC ID: 81-070B-02C) From 071adc0c6e5986002c471933af09329b3478d0bb Mon Sep 17 00:00:00 2001 From: landsito Date: Sat, 12 Nov 2022 01:18:01 -0700 Subject: [PATCH 038/396] Adding timed_guvi instrument to support on L1C's intensity data from both sensors spectrograph and imaging and for both resolutions high and low. Sensor type is handled by inst_id and resolution by tag_id. Adding some cases for year/day combinations not considered in cdaweb and needed by timed_guvi formats --- docs/supported_instruments.rst | 8 + pysatNASA/instruments/__init__.py | 2 +- pysatNASA/instruments/methods/cdaweb.py | 37 ++- pysatNASA/instruments/timed_guvi.py | 353 ++++++++++++++++++++++++ 4 files changed, 392 insertions(+), 8 deletions(-) create mode 100644 pysatNASA/instruments/timed_guvi.py diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 054bc982..acf1e3e1 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -130,6 +130,14 @@ SES14 GOLD .. automodule:: pysatNASA.instruments.ses14_gold :members: +.. _timed_guvi: + +TIMED GUVI +---------- + +.. automodule:: pysatNASA.instruments.timed_guvi + :members: + .. _timed_saber: TIMED SABER diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index c3a8bfae..052013ba 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -10,7 +10,7 @@ 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', - 'timed_saber', 'timed_see'] + 'timed_guvi', 'timed_saber', 'timed_see'] for inst in __all__: exec("from pysatNASA.instruments import {x}".format(x=inst)) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index c71115f2..526b260e 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -457,12 +457,21 @@ def download(date_array, tag='', inst_id='', supported_tags=None, # Download only requested files that exist remotely for date, fname in remote_files.items(): # Format files for specific dates and download location - formatted_remote_dir = remote_dir.format(year=date.year, - month=date.month, - day=date.day, - hour=date.hour, - min=date.minute, - sec=date.second) + # Year and day found in remote_dir: day is assumed to be day of year + if 'day' in remote_dir and 'month' not in remote_dir: + doy = date.timetuple().tm_yday + formatted_remote_dir = remote_dir.format(year=date.year, + day=doy, + hour=date.hour, + min=date.minute, + sec=date.second) + else: + formatted_remote_dir = remote_dir.format(year=date.year, + month=date.month, + day=date.day, + hour=date.hour, + min=date.minute, + sec=date.second) remote_path = '/'.join((remote_url.strip('/'), formatted_remote_dir.strip('/'), fname)) @@ -599,17 +608,31 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, stop = dt.datetime.now() if (stop is None) else stop if 'year' in search_dir['keys']: + # Year, month found: assume monthly cadence of directory format if 'month' in search_dir['keys']: search_times = pds.date_range(start, stop + pds.DateOffset(months=1), freq='M') + # Year, day found: assume daily cadence of directory format + elif 'day' in search_dir['keys']: + search_times = pds.date_range(start, + stop + pds.DateOffset(days=1), + freq='D') + # Year found alone: assume yearly cadence of directory format else: search_times = pds.date_range(start, stop + pds.DateOffset(years=1), freq='Y') + url_list = [] for time in search_times: - subdir = format_dir.format(year=time.year, month=time.month) + if 'month' in search_dir['keys']: + subdir = format_dir.format(year=time.year, month=time.month) + elif 'day' in search_dir['keys']: + subdir = format_dir.format(year=time.year, + day=time.timetuple().tm_yday) + else: + subdir = format_dir.format(year=time.year) url_list.append('/'.join((remote_url, subdir))) try: for top_url in url_list: diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py new file mode 100644 index 00000000..6067294d --- /dev/null +++ b/pysatNASA/instruments/timed_guvi.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- +"""Supports the GUVI instrument on TIMED. + +Downloads data from the NASA Coordinated Data +Analysis Web (CDAWeb). + +Supports two options for loading that may be +specified at instantiation. + +Properties +---------- +platform + 'timed' +name + 'guvi' +tag + 'low' + 'high' +inst_id + 'imaging' + 'spectrograph' + +Note +---- +No `inst_id` required. + +Warnings +-------- +Currently no cleaning routine. + + +Example +------- +:: + + import pysat + guvi = pysat.Instrument(platform='timed', name='guvi', inst_id='imaging', tag='low') + guvi.download(dt.datetime(2005, 6, 28), dt.datetime(2005, 6, 29)) + guvi.load(yr=2005, doy=171) + +""" + +import datetime as dt +import functools +import numpy as np +import warnings +import xarray as xr + +import pysat +from pysat.instruments.methods import general as mm_gen +from pysat import logger + +from pysatNASA.instruments.methods import cdaweb as cdw + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'timed' +name = 'guvi' +tags = {'high': 'Level 1C high resolution data', + 'low': 'Level 1C low resolution data'} +inst_ids = {'imaging': list(tags.keys()), + 'spectrograph': list(tags.keys())} + +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'imaging': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}, + 'spectrograph': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}} + +# ---------------------------------------------------------------------------- +# Instrument methods + +def init(self): + """Initialize the Instrument object with references and acknowledgements.""" + + rules_url = 'http://guvitimed.jhuapl.edu/home_guvi-datausage' + ackn_str = ' '.join(('Please see the Rules of the Road at', rules_url)) + logger.info(ackn_str) + self.acknowledgements = ackn_str + self.references = ' '.join(('Paxton,L. J., Christensen, A. B., Humm,', + 'D. C., Ogorzalek, B. S., Pardoe, C. T.,', + 'Monison, D., Weiss, M. B., Cram, W.,', + 'Lew, P. H., Mabry, D. J., Goldstena,', + 'J. O., Gary, A., Persons, D. F., Harold,', + 'M. J., Alvarez, E. B., ErcoF, C. J.,', + 'Strickland, D. J., Meng, C.-I.', + 'Global ultraviolet imager (GUVI): Measuring', + 'composition and energy inputs for the NASA', + 'Thermosphere Ionosphere Mesosphere Energetics', + 'and Dynamics (TIMED) mission.', + 'Optical spectroscopic techniques and', + 'instrumentation for atmospheric and', + 'space research III. Vol. 3756.', + 'International Society for Optics', + 'and Photonics, 1999.')) + + return + + +def clean(self): + """Routine to return TIMED GUVI data cleaned to the specified level. + + Note + ---- + No cleaning currently available. + + """ + warnings.warn('no cleaning routines available for TIMED GUVI data') + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods +# Set the list_files routine + +fname = ''.join(('TIMED_GUVI_L1C{res:s}-disk-{mode:s}_{{year:04d}}{{day:03d}}', + '{{hour:02d}}{{minute:02d}}{{second:02d}}', + '-?????????????_REV??????_', + 'Av{{version:02d}}-??r{{revision:03d}}.nc')) + +supported_tags = {inst: {tag: fname.format(res = "-2" if 'low' in tag else "", + mode = "IMG" if 'imag' in inst else "SPECT") + for tag in tags.keys()} + for inst in inst_ids} + +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags,) + +# Set the download routine +url = ''.join(('/pub/data/timed/guvi/levels_v13/level1c/{mode:s}/', + '{{year:4d}}/{{day:03d}}/')) + +supported_urls = {inst: {tag: url.format(mode=inst) + for tag in tags.keys()} + for inst in inst_ids} + +download_tags = {inst: {tag: {'remote_dir': supported_urls[inst][tag], + 'fname': supported_tags[inst][tag]} + for tag in tags.keys()} + for inst in inst_ids.keys()} + +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial( + cdw.list_remote_files, supported_tags=download_tags) + + +# Set the load routine +def load(fnames, tag='', inst_id=''): + """Load TIMED GUVI data into `xarray.DataSet` and `pysat.Meta` objects. + + This routine is called as needed by pysat. It is not intended + for direct user interaction. + + Parameters + ---------- + fnames : array-like + iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + inst_id : str + Satellite ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + + Returns + ------- + data : xr.DataSet + A xarray DataSet with data prepared for the pysat.Instrument + meta : pysat.Meta + Metadata formatted for a pysat.Instrument object. + + Note + ---- + Any additional keyword arguments passed to pysat.Instrument + upon instantiation are passed along to this routine. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'guvi', inst_id='imaging', tag='high') + inst.load(2005, 179) + + """ + labels = {'units': ('units', str), 'name': ('long_name', str), + 'notes': ('notes', str), 'desc': ('desc', str), + 'plot': ('plot_label', str), 'axis': ('axis', str), + 'scale': ('scale', str), + 'min_val': ('value_min', float), + 'max_val': ('value_max', float), + 'fill_val': ('fill', float)} + meta = pysat.Meta(labels=labels) + + def get_dt_objects(dataset, tag): + dts = [] + for i, year in enumerate(dataset['YEAR_%s' % tag].data): + idt = dt.datetime(year, 1, 1) + dt.timedelta( + days=float(dataset['DOY_%s' % tag].data[i] - 1), + seconds=float(dataset['TIME_%s' % tag].data[i])) + dts.append(idt) + return dts + + # Dimensions for time variables + # night/day along, cross and time are the same imaging low, high, spectrograph low, high + # imaging high possess extra dims DayAur + # spectrograph low possess extra dims GAIMDay, GAIMNight + dims = ['time'] + if 'imag' in inst_id: + dims = dims + ['time_auroral'] + elif 'spect' in inst_id: + if 'low' in tag: + dims = dims + ['time_gaim_day', 'time_gaim_night'] + + # Separate and concatenate into datasets + inners = {dim: None for dim in dims} + + for path in fnames: + data = xr.open_dataset(path, chunks='auto') + + # Collect datetime objects + day_dts = np.array(get_dt_objects(data, "DAY")) + night_dts = np.array(get_dt_objects(data, "NIGHT")) + if 'imag' in inst_id: + aur_dts = get_dt_objects(data, "DAY_AURORAL") + elif 'spect' in inst_id: + if 'low' in tag: + gaimday_dts = get_dt_objects(data, "GAIM_DAY") + gaimnight_dts = get_dt_objects(data, "GAIM_NIGHT") + + # Drop out redundant time variables + data = data.drop_vars(["YEAR_DAY", "DOY_DAY", "TIME_DAY", "TIME_EPOCH_DAY", + "YEAR_NIGHT", "DOY_NIGHT", + "TIME_NIGHT", "TIME_EPOCH_NIGHT"]) + if 'imag' in inst_id: + data = data.drop_vars(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", + "TIME_DAY_AURORAL", "TIME_EPOCH_DAY_AURORAL"]) + elif 'spect' in inst_id: + if 'low' in tag: + data = data.drop_vars(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", + "TIME_GAIM_DAY", "TIME_GAIM_NIGHT", + "YEAR_GAIM_NIGHT", "DOY_GAIM_NIGHT"]) + + if day_dts.size != night_dts.size: + raise Exception("nAlongDay & nAlongNight have different dimensions") + + if np.any(day_dts != night_dts): + raise Exception("time along day and night are different") + + # Renaming along/cross dimensions + data = data.rename_dims({"nAlongDay":"nAlong", "nAlongNight":"nAlong"}) + + # 'nCross' dimension only in imaging + if 'imag' in inst_id: + + if data.nCrossDay.size != data.nCrossNight.size: + raise Exception("nCrossDay/Night have different dimensions") + + data = data.rename_dims({"nCrossDay":"nCross", + "nCrossNight":"nCross"}) + + # 'nAlong' will be renamed as 'time' to follow pysat standards + data = data.swap_dims({"nAlong": "time"}) + if 'imag' in inst_id: + data = data.swap_dims({"nAlongDayAur": "time_auroral"}) + elif 'spect' in inst_id: + if 'low' in tag: + data = data.swap_dims({"nAlongGAIMDay": "time_gaim_day", + "nAlongGAIMNight": "time_gaim_night"}) + + # Update time variables + # night_dts and day_dts are the same temporal array + data = data.assign(time=night_dts) + if 'imag' in inst_id: + data = data.assign(time_auroral=aur_dts) + elif 'spect' in inst_id: + if 'low' in tag: + data = data.assign(time_gaim_day=gaimday_dts, + time_gaim_night=gaimnight_dts) + + # Separate into inner datasets + inner_keys = {dim: [] for dim in dims} + + for dim in dims: + for key in data.keys(): + if dim in data[key].dims: + inner_keys[dim].append(key) + + jnners = {dim: data.get(inner_keys[dim]) for dim in dims} + + # Add 'single_var's into 'time' dataset to keep track + sv_keys = [v.name for v in data.values() if 'single_var' in v.dims] + singlevar_set = data.get(sv_keys) + jnners['time'] = xr.merge([jnners['time'], singlevar_set]) + + # Concatenate along corresponding dimension + if inners['time'] is None: + inners = jnners + else: + inners = {dim : xr.concat([inners[dim], jnners[dim]], dim=dim) + for dim in dims } + + data = xr.merge([inners[dim] for dim in dims]) + + # Set up coordinates + coords = ['PIERCEPOINT_NIGHT_LATITUDE', + 'PIERCEPOINT_NIGHT_LONGITUDE', + 'PIERCEPOINT_NIGHT_ALTITUDE', + 'PIERCEPOINT_NIGHT_SZA', + 'PIERCEPOINT_DAY_LATITUDE', + 'PIERCEPOINT_DAY_LONGITUDE', + 'PIERCEPOINT_DAY_ALTITUDE', + 'PIERCEPOINT_DAY_SZA'] + + if 'imag' in inst_id: + coords = coords + ['PIERCEPOINT_DAY_LATITUDE_AURORAL', + 'PIERCEPOINT_DAY_LONGITUDE_AURORAL', + 'PIERCEPOINT_DAY_ALTITUDE_AURORAL', + 'PIERCEPOINT_DAY_SZA_AURORAL'] + elif 'spect' in inst_id: + coords = coords + ['PIERCEPOINT_NIGHT_ZENITH_ANGLE', + 'PIERCEPOINT_NIGHT_SAZIMUTH', + 'PIERCEPOINT_DAY_ZENITH_ANGLE', + 'PIERCEPOINT_DAY_SAZIMUTH'] + + data = data.set_coords(coords) + + # Set 'nchan' and 'nCross' as coordinates + if 'imag' in inst_id: + coords = {"nchan": ["121.6nm", "130.4nm", "135.6nm", + "LBHshort", "LBHlong"], + "nchanAur": ["121.6nm", "130.4nm", "135.6nm", + "LBHshort", "LBHlong"], + "nCross": data.nCross.data, + "nCrossDayAur": data.nCrossDayAur.data} + elif 'spect' in inst_id: + coords = {"nchan": ["121.6nm", "130.4nm", "135.6nm", + "LBHshort", "LBHlong","?"]} + + data = data.assign_coords(coords=coords) + + # Sort + data = data.sortby("time") + + return data, meta From 8e10ca57100c2ef79736cbb1c42cdbbf40bcc36d Mon Sep 17 00:00:00 2001 From: landsito Date: Sat, 12 Nov 2022 01:25:27 -0700 Subject: [PATCH 039/396] Fixing some comments --- pysatNASA/instruments/timed_guvi.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 6067294d..e2f6a453 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -22,13 +22,11 @@ Note ---- -No `inst_id` required. Warnings -------- Currently no cleaning routine. - Example ------- :: From d22262b6a1b2a1024e5ffc00651bd6287878d0d3 Mon Sep 17 00:00:00 2001 From: landsito Date: Sat, 12 Nov 2022 01:51:00 -0700 Subject: [PATCH 040/396] Updating CHANGELOG and .zenodo to include TIMED-GUVI platform addition --- .zenodo.json | 5 +++++ CHANGELOG.md | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index a5bcff9a..1052a817 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -27,6 +27,11 @@ { "name": "Spence, Carey", "orcid": "0000-0001-8340-5625" + }, + { + "affilitation":"University of Colorado at Boulder", + "name": "Navarro, Luis", + "orcid": "0000-0002-6362-6575" } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a1d903..e8de552e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +## [0.0.5] - 2022-11-12 +* New Instruments + * TIMED GUVI +* Add TIMED GUVI platform to support L1C intensity datasets. + * Type of sensor source handled by inst_id with options of + spectrograph, imaging + * Resolution of dataset handled by tag with + low, high +* Bug Fixes + * CDAWEB improved to consider cases of + remote directory structured by DOY instead of month/day combination + cadence of directory format consider yearly, monthly and daily cases. +* Documentation + * Added TIMED-GUVI platform + ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class * Support xarray datasets through cdflib From 97d38913613381528f773e4e4b111bb4fb13a116 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 25 Nov 2022 14:32:56 -0500 Subject: [PATCH 041/396] STY: rename as L2 --- pysatNASA/instruments/__init__.py | 2 +- .../instruments/{ace_epam.py => ace_epam-l2.py} | 6 +++++- .../instruments/{ace_mag.py => ace_mag-l2.py} | 6 +++++- .../instruments/{ace_sis.py => ace_sis-l2.py} | 6 +++++- .../{ace_swepam.py => ace_swepam-l2.py} | 6 +++++- test.nc | Bin 0 -> 11313 bytes 6 files changed, 21 insertions(+), 5 deletions(-) rename pysatNASA/instruments/{ace_epam.py => ace_epam-l2.py} (96%) rename pysatNASA/instruments/{ace_mag.py => ace_mag-l2.py} (96%) rename pysatNASA/instruments/{ace_sis.py => ace_sis-l2.py} (96%) rename pysatNASA/instruments/{ace_swepam.py => ace_swepam-l2.py} (96%) create mode 100644 test.nc diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index d97bcdcb..b5781bfd 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -5,7 +5,7 @@ """ -__all__ = ['ace_epam', 'ace_mag', 'ace_sis', 'ace_swepam', +__all__ = ['ace_epam-l2', 'ace_mag-l2', 'ace_sis-l2', 'ace_swepam-l2', 'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'formosat1_ivm', diff --git a/pysatNASA/instruments/ace_epam.py b/pysatNASA/instruments/ace_epam-l2.py similarity index 96% rename from pysatNASA/instruments/ace_epam.py rename to pysatNASA/instruments/ace_epam-l2.py index fe3d11b7..e55ad25d 100644 --- a/pysatNASA/instruments/ace_epam.py +++ b/pysatNASA/instruments/ace_epam-l2.py @@ -10,12 +10,16 @@ platform 'ace' name - 'epam' + 'epam-l2' tag None supported inst_id ['h1', 'h2', 'h3', 'k0', 'k1'] +Note +---- +Level 1 ACE data is maintained at pysatSpaceWeather. + Warnings -------- - The cleaning parameters for the instrument are still under development. diff --git a/pysatNASA/instruments/ace_mag.py b/pysatNASA/instruments/ace_mag-l2.py similarity index 96% rename from pysatNASA/instruments/ace_mag.py rename to pysatNASA/instruments/ace_mag-l2.py index c942d836..cfb30fc6 100644 --- a/pysatNASA/instruments/ace_mag.py +++ b/pysatNASA/instruments/ace_mag-l2.py @@ -10,12 +10,16 @@ platform 'ace' name - 'mag' + 'mag-l2' tag None supported inst_id ['h0', 'h1', 'h2', 'h3', 'k0', 'k1', 'k2'] +Note +---- +Level 1 ACE data is maintained at pysatSpaceWeather. + Warnings -------- - The cleaning parameters for the instrument are still under development. diff --git a/pysatNASA/instruments/ace_sis.py b/pysatNASA/instruments/ace_sis-l2.py similarity index 96% rename from pysatNASA/instruments/ace_sis.py rename to pysatNASA/instruments/ace_sis-l2.py index 578efcb6..a80705fa 100644 --- a/pysatNASA/instruments/ace_sis.py +++ b/pysatNASA/instruments/ace_sis-l2.py @@ -10,12 +10,16 @@ platform 'ace' name - 'mag' + 'sis-l2' tag None supported inst_id ['h1', 'h2', 'k0'] +Note +---- +Level 1 ACE data is maintained at pysatSpaceWeather. + Warnings -------- - The cleaning parameters for the instrument are still under development. diff --git a/pysatNASA/instruments/ace_swepam.py b/pysatNASA/instruments/ace_swepam-l2.py similarity index 96% rename from pysatNASA/instruments/ace_swepam.py rename to pysatNASA/instruments/ace_swepam-l2.py index f3a2ff66..d7abf954 100644 --- a/pysatNASA/instruments/ace_swepam.py +++ b/pysatNASA/instruments/ace_swepam-l2.py @@ -10,12 +10,16 @@ platform 'ace' name - 'swepam' + 'swepam-l2' tag None supported inst_id ['h0', 'h2', 'k0', 'k1'] +Note +---- +Level 1 ACE data is maintained at pysatSpaceWeather. + Warnings -------- - The cleaning parameters for the instrument are still under development. diff --git a/test.nc b/test.nc new file mode 100644 index 0000000000000000000000000000000000000000..66b36379d3516860a2ffd2f3d97409d794debfda GIT binary patch literal 11313 zcmeHNZ;Tto6@P2rrI+Ncxuhk9YXU<;%O&v{`~H+9ssNwYK09)J)~!iMB;qXg-entm z*IDl!2_G8TL_{T(`hf~nm7x4lRiIU+RDi1H3lgmgmC6Sy0;p2|q`IUZXh9VP+C-u6 z&FnayQ)sFKKBU>X^Ulu9Z{Ezj_vX#)oB38w&-N`FSOye&dY}vX`Kx?<7=&a@(K&jcl5bTx3`AJw@xvQxPd^x$YCooXy~Y*WC7p9C7jHH`AwIwSxq z0^H>ytk$k6^m-7LH#sQl9o_ZvzsAcs9ead2yb`l(gBOCfSAcA#VSLcXQA-KcCs5sj z#zoW}>rVeVYalI2z(ZP{bb*6crVBx`J8{|rw>3Qs6xi4c;D2wp_dR-IO{Pwn;7~r9HR^zFtOur zTgHD5ho~qa*c5&G?w8>hrr|Cz*1sPpyb3>~D(eB3V085Pf#1Q4)XH9fVF>iT|+={#rmY_)27Zj8um!@6!AiwgKcRJ{gKHJutW;><7|(>EPE z#>)FF=2^b!)of>mHC^9weA}!e(5xT2&+^!`>#^D)$86Zu7PAgE>#k>cL*p@KG|j5T zjH(@Hd8g%jvyC_#i?Ol9$m#nZ8cQbFU~{SZjtZiYh$KZ=0!Rn{DmXqs5P-FT zXEH54lxNh=2vqp|f8@ZPZQuF9e|O$LOa5C7w$u)JP9-!_Y9_1_u**I#?YuvCv#6-S z{EuGv>q6)K$)N33?N*Y1Ub%Xq^Zw*oLE|Fowo^yuh4cPT?PYI`2;f z&igL~02$BSa(sW_ygy2`&=XIt`NC7IA6C<$L`fVty!+q{H^F0=hAqOazIpzsjW9sH zKm;8@ytVFyq;LSETb;r$&p!NB7^bO*2=byo@$PdI@U@^yKW3b`?t}r|)a;<}_mAA3 z-vwDr!+XM#WgI-rF96SBdC|kkpddoFD{d(80-fAAau)oT3aoCm~1OjT_gz z4;S!-`q7Qii9pHK(7hpyFdHhvyY(JNWIUf*HXXD!k7mF)KR;y=0^MnB{f zak>Xv2Dw(t_){2gNHXL=$bnCc1M2)IvIm+kU8CWe--CbYLdq~OkM{4h=*xMn=eK8y zMt-tbG193*2J^c)zL?scK^Vu=u0i6(tsD)P<6gv7ik4rkO^@n!!>%EUJ}J0C0*0xJ zwZ!{UPC1*;6m%rd0r;RE(PawRLfLLutc9nJ7E9pqq#mc6KQx5wRIM{254-c{m#M+~P|8@#i;LBpLL zNulmKGZj3~4Hg?two_zWizH144}>MnOMrwv)~r?)+vAjO>S(xV7?5A8`xJdm2C@F&GjC;YlRz|RB+ z_-vT9Oy8>4ZQo)v-6uXeg1y|8jC{>)6QF z;nDc5$t~>8a+;(}&2p*+UI_6QLp3qBbz~R`Bh?W>wVO|G+~IP2b2*)%4@wF$=3@zT zhIq}tCWAC#q){XK73w-w#_(}@Ayi|+axBjzWfQcCDm5-SJe(LFP9_t#N>wVjGj`Ri X(|+En Date: Fri, 25 Nov 2022 14:46:51 -0500 Subject: [PATCH 042/396] DOC: references --- pysatNASA/instruments/ace_epam-l2.py | 12 ++++++--- pysatNASA/instruments/ace_mag-l2.py | 12 ++++++--- pysatNASA/instruments/ace_sis-l2.py | 10 +++++--- pysatNASA/instruments/ace_swepam-l2.py | 12 ++++++--- pysatNASA/instruments/methods/ace.py | 34 ++++++++++++++++++++------ 5 files changed, 61 insertions(+), 19 deletions(-) diff --git a/pysatNASA/instruments/ace_epam-l2.py b/pysatNASA/instruments/ace_epam-l2.py index e55ad25d..30e731ff 100644 --- a/pysatNASA/instruments/ace_epam-l2.py +++ b/pysatNASA/instruments/ace_epam-l2.py @@ -1,9 +1,13 @@ # -*- coding: utf-8 -*- -"""Module for the ACE EPAM instrument. +"""Module for the Advanced Composition Explorer (ACE) EPAM instrument. References ---------- -???? +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Gold, R., Krimigis, S., Hawkins, S. et al. Electron, Proton, and Alpha Monitor + on the Advanced Composition Explorer spacecraft. Space Science Reviews 86, + 541–562 (1998). https://doi.org/10.1023/A:1005088115759 Properties ---------- @@ -18,7 +22,9 @@ Note ---- -Level 1 ACE data is maintained at pysatSpaceWeather. +- Level 1 ACE data is maintained at pysatSpaceWeather. +- Release notes at + https://cdaweb.gsfc.nasa.gov/pub/data/ace/epam/epam_level2_release_notes.txt Warnings -------- diff --git a/pysatNASA/instruments/ace_mag-l2.py b/pysatNASA/instruments/ace_mag-l2.py index cfb30fc6..23be0260 100644 --- a/pysatNASA/instruments/ace_mag-l2.py +++ b/pysatNASA/instruments/ace_mag-l2.py @@ -1,9 +1,13 @@ # -*- coding: utf-8 -*- -"""Module for the ACE MAG instrument. +"""Module for the Advanced Composition Explorer (ACE) MAG instrument. References ---------- -???? +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Smith, C., L'Heureux, J., Ness, N. et al. The ACE Magnetic Fields Experiment. + Space Science Reviews 86, 613–632 (1998). + https://doi.org/10.1023/A:1005092216668 Properties ---------- @@ -18,7 +22,9 @@ Note ---- -Level 1 ACE data is maintained at pysatSpaceWeather. +- Level 1 ACE data is maintained at pysatSpaceWeather. +- Release notes at + https://cdaweb.gsfc.nasa.gov/pub/data/ace/mag/mag_level2_release_notes.txt Warnings -------- diff --git a/pysatNASA/instruments/ace_sis-l2.py b/pysatNASA/instruments/ace_sis-l2.py index a80705fa..1c46827e 100644 --- a/pysatNASA/instruments/ace_sis-l2.py +++ b/pysatNASA/instruments/ace_sis-l2.py @@ -1,9 +1,13 @@ # -*- coding: utf-8 -*- -"""Module for the ACE SIS instrument. +"""Module for the Advanced Composition Explorer (ACE) SIS instrument. References ---------- -???? +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Stone, E., Cohen, C., Cook, W. et al. The Solar Isotope Spectrometer for the + Advanced Composition Explorer. Space Science Reviews 86, 357–408 (1998). + https://doi.org/10.1023/A:1005027929871 Properties ---------- @@ -18,7 +22,7 @@ Note ---- -Level 1 ACE data is maintained at pysatSpaceWeather. +- Level 1 ACE data is maintained at pysatSpaceWeather. Warnings -------- diff --git a/pysatNASA/instruments/ace_swepam-l2.py b/pysatNASA/instruments/ace_swepam-l2.py index d7abf954..34a9f903 100644 --- a/pysatNASA/instruments/ace_swepam-l2.py +++ b/pysatNASA/instruments/ace_swepam-l2.py @@ -1,9 +1,13 @@ # -*- coding: utf-8 -*- -"""Module for the ACE EPAM instrument. +"""Module for the Advanced Composition Explorer (ACE) EPAM instrument. References ---------- -???? +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- McComas, D., Bame, S., Barker, P. et al. Solar Wind Electron Proton Alpha + Monitor (SWEPAM) for the Advanced Composition Explorer. Space Science Reviews + 86, 563–612 (1998). https://doi.org/10.1023/A:1005040232597 Properties ---------- @@ -18,7 +22,9 @@ Note ---- -Level 1 ACE data is maintained at pysatSpaceWeather. +- Level 1 ACE data is maintained at pysatSpaceWeather. +- Release notes at + https://cdaweb.gsfc.nasa.gov/pub/data/ace/swepam/swepam_level2_release_notes.txt Warnings -------- diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index 5d5e7cfe..341fc993 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -1,12 +1,32 @@ # -*- coding: utf-8 -*- """Provides non-instrument specific routines for ACE data.""" -ackn_str = ' '.join(("The ACE", - "(ACE) satellite data is provided through CDAWeb")) +ackn_str = ' '.join(("Please acknowledge the NASA National Space Science Data", + "Center, the Space Physics Data Facility, and the ACE", + "Principal Investigator, Edward C. Stone of the", + "California Institute of Technology, for usage of ACE", + "data from this site in publications and presentations.")) -refs = {'mission': ' '.join(('???')), - 'epam': ' '.join(('???')), - 'mag': ' '.join(('???')), - 'sis': ' '.join(('???')), - 'swepam': ' '.join(('???')) +refs = {'mission': ' '.join(('Stone, E., Frandsen, A., Mewaldt, R. et al.', + 'The Advanced Composition Explorer. Space Science', + 'Reviews 86, 1–22 (1998).', + 'https://doi.org/10.1023/A:1005082526237')), + 'epam': ' '.join(('Gold, R., Krimigis, S., Hawkins, S. et al.', + 'Electron, Proton, and Alpha Monitor on the Advanced', + 'Composition Explorer spacecraft. Space Science', + 'Reviews 86, 541–562 (1998).', + 'https://doi.org/10.1023/A:1005088115759')), + 'mag': ' '.join(("Smith, C., L'Heureux, J., Ness, N. et al. The ACE", + "Magnetic Fields Experiment. Space Science Reviews", + "86, 613–632 (1998).", + "https://doi.org/10.1023/A:1005092216668")), + 'sis': ' '.join(('Stone, E., Cohen, C., Cook, W. et al. The Solar', + 'Isotope Spectrometer for the Advanced Composition', + 'Explorer. Space Science Reviews 86, 357–408 (1998).', + 'https://doi.org/10.1023/A:1005027929871')), + 'swepam': ' '.join(('McComas, D., Bame, S., Barker, P. et al. Solar', + 'Wind Electron Proton Alpha Monitor (SWEPAM) for', + 'the Advanced Composition Explorer. Space Science', + 'Reviews 86, 563–612 (1998).', + 'https://doi.org/10.1023/A:1005040232597')) } From 408870c1bc166e8fd7e11903a74b0c4a4a4aca25 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 25 Nov 2022 14:48:16 -0500 Subject: [PATCH 043/396] STY: underscore --- pysatNASA/instruments/__init__.py | 2 +- pysatNASA/instruments/{ace_epam-l2.py => ace_epam_l2.py} | 2 +- pysatNASA/instruments/{ace_mag-l2.py => ace_mag_l2.py} | 2 +- pysatNASA/instruments/{ace_sis-l2.py => ace_sis_l2.py} | 2 +- pysatNASA/instruments/{ace_swepam-l2.py => ace_swepam_l2.py} | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename pysatNASA/instruments/{ace_epam-l2.py => ace_epam_l2.py} (99%) rename pysatNASA/instruments/{ace_mag-l2.py => ace_mag_l2.py} (99%) rename pysatNASA/instruments/{ace_sis-l2.py => ace_sis_l2.py} (99%) rename pysatNASA/instruments/{ace_swepam-l2.py => ace_swepam_l2.py} (99%) diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index b5781bfd..0a5d5031 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -5,7 +5,7 @@ """ -__all__ = ['ace_epam-l2', 'ace_mag-l2', 'ace_sis-l2', 'ace_swepam-l2', +__all__ = ['ace_epam_l2', 'ace_mag_l2', 'ace_sis_l2', 'ace_swepam_l2', 'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'formosat1_ivm', diff --git a/pysatNASA/instruments/ace_epam-l2.py b/pysatNASA/instruments/ace_epam_l2.py similarity index 99% rename from pysatNASA/instruments/ace_epam-l2.py rename to pysatNASA/instruments/ace_epam_l2.py index 30e731ff..83e9aa47 100644 --- a/pysatNASA/instruments/ace_epam-l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -14,7 +14,7 @@ platform 'ace' name - 'epam-l2' + 'epam_l2' tag None supported inst_id diff --git a/pysatNASA/instruments/ace_mag-l2.py b/pysatNASA/instruments/ace_mag_l2.py similarity index 99% rename from pysatNASA/instruments/ace_mag-l2.py rename to pysatNASA/instruments/ace_mag_l2.py index 23be0260..e448fb88 100644 --- a/pysatNASA/instruments/ace_mag-l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -14,7 +14,7 @@ platform 'ace' name - 'mag-l2' + 'mag_l2' tag None supported inst_id diff --git a/pysatNASA/instruments/ace_sis-l2.py b/pysatNASA/instruments/ace_sis_l2.py similarity index 99% rename from pysatNASA/instruments/ace_sis-l2.py rename to pysatNASA/instruments/ace_sis_l2.py index 1c46827e..b901aa97 100644 --- a/pysatNASA/instruments/ace_sis-l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -14,7 +14,7 @@ platform 'ace' name - 'sis-l2' + 'sis_l2' tag None supported inst_id diff --git a/pysatNASA/instruments/ace_swepam-l2.py b/pysatNASA/instruments/ace_swepam_l2.py similarity index 99% rename from pysatNASA/instruments/ace_swepam-l2.py rename to pysatNASA/instruments/ace_swepam_l2.py index 34a9f903..869d49c8 100644 --- a/pysatNASA/instruments/ace_swepam-l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -14,7 +14,7 @@ platform 'ace' name - 'swepam-l2' + 'swepam_l2' tag None supported inst_id From 7c1b6d79172357367b7697a700eda2820e30d889 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 25 Nov 2022 14:50:47 -0500 Subject: [PATCH 044/396] DOC: update docs for ACE --- docs/supported_instruments.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 054bc982..941a761b 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -1,6 +1,38 @@ Supported Instruments ===================== +.. _ace_epam: + +ACE EPAM +-------- + +.. automodule:: pysatNASA.instruments.ace_epam + :members: + +.. _ace_mag: + +ACE MAG +------- + +.. automodule:: pysatNASA.instruments.ace_mag + :members: + +.. _ace_epam: + +ACE SIS +------- + +.. automodule:: pysatNASA.instruments.ace_sis + :members: + +.. _ace_mag: + +ACE SWEPAM +---------- + +.. automodule:: pysatNASA.instruments.ace_swepam + :members: + .. _cnofs_ivm: C/NOFS IVM From efed9d3675710d0adba16a7bbd74600c3fa9b39f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 25 Nov 2022 14:52:23 -0500 Subject: [PATCH 045/396] DOC: update changelog --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ec548e0..6ed0b54f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +## [0.1.0] - 2022-XX-XX +* New Instruments +* ACE EPAM +* ACE MAG +* ACE SIS +* ACE SWEPAM + ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class * Support xarray datasets through cdflib @@ -9,10 +16,6 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Adds pysatCDF to optional requirements invoked via '[all]' option at installation * New Instruments * JPL GPS ROTI - * ACE EPAM - * ACE MAG - * ACE SIS - * ACE SWEPAM * Bug Fixes * Fixed a bug in metadata when loading GOLD Nmax data. * Fixed a bug in user feedback for `methods.cdaweb.download` From f2ae665a64f0589543198c57901972b3bb3d35bb Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 25 Nov 2022 14:58:06 -0500 Subject: [PATCH 046/396] BUG: fix docs --- docs/supported_instruments.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 941a761b..943e9a1c 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -6,7 +6,7 @@ Supported Instruments ACE EPAM -------- -.. automodule:: pysatNASA.instruments.ace_epam +.. automodule:: pysatNASA.instruments.ace_epam_l2 :members: .. _ace_mag: @@ -14,23 +14,23 @@ ACE EPAM ACE MAG ------- -.. automodule:: pysatNASA.instruments.ace_mag +.. automodule:: pysatNASA.instruments.ace_mag_l2 :members: -.. _ace_epam: +.. _ace_sis: ACE SIS ------- -.. automodule:: pysatNASA.instruments.ace_sis +.. automodule:: pysatNASA.instruments.ace_sis_l2 :members: -.. _ace_mag: +.. _ace_swepam: ACE SWEPAM ---------- -.. automodule:: pysatNASA.instruments.ace_swepam +.. automodule:: pysatNASA.instruments.ace_swepam_l2 :members: .. _cnofs_ivm: From 92157081f23ee37a288b3e44cfbc221879d0528d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 25 Nov 2022 18:56:47 -0500 Subject: [PATCH 047/396] Update ace_epam_l2.py --- pysatNASA/instruments/ace_epam_l2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index 83e9aa47..2125f486 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -45,7 +45,7 @@ # Instrument attributes platform = 'ace' -name = 'epam' +name = 'epam_l2' tags = {'': ''} inst_ids = {id: [''] for id in ['h1', 'h2', 'h3', 'k0', 'k1']} pandas_format = False From 336b5a8592eb2563c7b85e6770b9d71c208c61da Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 25 Nov 2022 18:57:05 -0500 Subject: [PATCH 048/396] Update ace_mag_l2.py --- pysatNASA/instruments/ace_mag_l2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index e448fb88..90047aae 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -45,7 +45,7 @@ # Instrument attributes platform = 'ace' -name = 'mag' +name = 'mag_l2' tags = {'': ''} inst_ids = {id: [''] for id in ['h0', 'h1', 'h2', 'h3', 'k0', 'k1', 'k2']} pandas_format = False From 7befaa295645dd464989233e6fced304bc847439 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 25 Nov 2022 18:57:19 -0500 Subject: [PATCH 049/396] Update ace_sis_l2.py --- pysatNASA/instruments/ace_sis_l2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index b901aa97..71d58c4c 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -43,7 +43,7 @@ # Instrument attributes platform = 'ace' -name = 'sis' +name = 'sis_l2' tags = {'': ''} inst_ids = {id: [''] for id in ['h1', 'h2', 'k0']} pandas_format = False From 4080db41f156c62813fb2fc2dd4227e1a56a04b3 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 25 Nov 2022 18:57:35 -0500 Subject: [PATCH 050/396] Update ace_swepam_l2.py --- pysatNASA/instruments/ace_swepam_l2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index 869d49c8..4e018e62 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -45,7 +45,7 @@ # Instrument attributes platform = 'ace' -name = 'swepam' +name = 'swepam_l2' tags = {'': ''} inst_ids = {id: [''] for id in ['h0', 'h2', 'k0', 'k1']} pandas_format = False From 8097458bf5492b9058a105f023df31d6a2e9c805 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 25 Nov 2022 19:07:23 -0500 Subject: [PATCH 051/396] Update ace.py --- pysatNASA/instruments/methods/ace.py | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index 341fc993..709b74b6 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -11,22 +11,22 @@ 'The Advanced Composition Explorer. Space Science', 'Reviews 86, 1–22 (1998).', 'https://doi.org/10.1023/A:1005082526237')), - 'epam': ' '.join(('Gold, R., Krimigis, S., Hawkins, S. et al.', - 'Electron, Proton, and Alpha Monitor on the Advanced', - 'Composition Explorer spacecraft. Space Science', - 'Reviews 86, 541–562 (1998).', - 'https://doi.org/10.1023/A:1005088115759')), - 'mag': ' '.join(("Smith, C., L'Heureux, J., Ness, N. et al. The ACE", - "Magnetic Fields Experiment. Space Science Reviews", - "86, 613–632 (1998).", - "https://doi.org/10.1023/A:1005092216668")), - 'sis': ' '.join(('Stone, E., Cohen, C., Cook, W. et al. The Solar', - 'Isotope Spectrometer for the Advanced Composition', - 'Explorer. Space Science Reviews 86, 357–408 (1998).', - 'https://doi.org/10.1023/A:1005027929871')), - 'swepam': ' '.join(('McComas, D., Bame, S., Barker, P. et al. Solar', - 'Wind Electron Proton Alpha Monitor (SWEPAM) for', - 'the Advanced Composition Explorer. Space Science', - 'Reviews 86, 563–612 (1998).', - 'https://doi.org/10.1023/A:1005040232597')) + 'epam_l2': ' '.join(('Gold, R., Krimigis, S., Hawkins, S. et al.', + 'Electron, Proton, and Alpha Monitor on the', + 'Advanced Composition Explorer spacecraft.', + 'Space Science Reviews 86, 541–562 (1998).', + 'https://doi.org/10.1023/A:1005088115759')), + 'mag_l2': ' '.join(("Smith, C., L'Heureux, J., Ness, N. et al. The ACE", + "Magnetic Fields Experiment. Space Science Reviews", + "86, 613–632 (1998).", + "https://doi.org/10.1023/A:1005092216668")), + 'sis_l2': ' '.join(('Stone, E., Cohen, C., Cook, W. et al. The Solar', + 'Isotope Spectrometer for the Advanced Composition', + 'Explorer. Space Science Reviews 86, 357–408', + '(1998). https://doi.org/10.1023/A:1005027929871')), + 'swepam_l2': ' '.join(('McComas, D., Bame, S., Barker, P. et al. Solar', + 'Wind Electron Proton Alpha Monitor (SWEPAM)', + 'for the Advanced Composition Explorer. Space', + 'Science Reviews 86, 563–612 (1998).', + 'https://doi.org/10.1023/A:1005040232597')) } From d8e30bc413f46bf30b64f205cbbfaca92ee5a130 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:32:55 -0700 Subject: [PATCH 052/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index e2f6a453..f130d110 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -32,7 +32,8 @@ :: import pysat - guvi = pysat.Instrument(platform='timed', name='guvi', inst_id='imaging', tag='low') + guvi = pysat.Instrument(platform='timed', name='guvi', inst_id='imaging', + tag='low') guvi.download(dt.datetime(2005, 6, 28), dt.datetime(2005, 6, 29)) guvi.load(yr=2005, doy=171) From 3fbe3105a44be7c13428598b4a8bc050fa4a7bba Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:33:27 -0700 Subject: [PATCH 053/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index f130d110..ccc04e0b 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -66,8 +66,9 @@ # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'imaging': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}, - 'spectrograph': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}} +_test_dates = { + 'imaging': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}, + 'spectrograph': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods From ba470661559d6434da6d272af2cd310f5365086a Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:33:37 -0700 Subject: [PATCH 054/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index ccc04e0b..456d069d 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -124,10 +124,11 @@ def clean(self): '-?????????????_REV??????_', 'Av{{version:02d}}-??r{{revision:03d}}.nc')) -supported_tags = {inst: {tag: fname.format(res = "-2" if 'low' in tag else "", - mode = "IMG" if 'imag' in inst else "SPECT") - for tag in tags.keys()} - for inst in inst_ids} +res = {'low': '-2', 'high': ''} +mode = {'imaging': 'IMG', 'spectrograph': 'SPECT'} +supported_tags = {inst: {tag: fname.format(res=res[tag], mode=mode[inst]) + for tag in tags.keys()} + for inst in inst_ids} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags,) From cd03e186d1400be7c935c81242b913b39638a9aa Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:34:07 -0700 Subject: [PATCH 055/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 456d069d..b6af0ff5 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -138,8 +138,8 @@ def clean(self): '{{year:4d}}/{{day:03d}}/')) supported_urls = {inst: {tag: url.format(mode=inst) - for tag in tags.keys()} - for inst in inst_ids} + for tag in tags.keys()} + for inst in inst_ids}``` download_tags = {inst: {tag: {'remote_dir': supported_urls[inst][tag], 'fname': supported_tags[inst][tag]} From d95fdf182c44cb254b97656a7f991390b92fbae4 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:34:16 -0700 Subject: [PATCH 056/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index b6af0ff5..9ba7a7ae 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -265,8 +265,8 @@ def get_dt_objects(dataset, tag): if data.nCrossDay.size != data.nCrossNight.size: raise Exception("nCrossDay/Night have different dimensions") - data = data.rename_dims({"nCrossDay":"nCross", - "nCrossNight":"nCross"}) + data = data.rename_dims({"nCrossDay": "nCross", + "nCrossNight": "nCross"}) # 'nAlong' will be renamed as 'time' to follow pysat standards data = data.swap_dims({"nAlong": "time"}) From 6cea25163d07f99d1b91765eba0ccc5ba9be9019 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:34:39 -0700 Subject: [PATCH 057/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 9ba7a7ae..3da2ef8d 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -278,7 +278,7 @@ def get_dt_objects(dataset, tag): "nAlongGAIMNight": "time_gaim_night"}) # Update time variables - # night_dts and day_dts are the same temporal array + # night_dts and day_dts are the same temporal array data = data.assign(time=night_dts) if 'imag' in inst_id: data = data.assign(time_auroral=aur_dts) From 20f025f32773beda6495492699ca8cb16cf524ed Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:34:46 -0700 Subject: [PATCH 058/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 3da2ef8d..0cc2ee74 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -306,8 +306,8 @@ def get_dt_objects(dataset, tag): if inners['time'] is None: inners = jnners else: - inners = {dim : xr.concat([inners[dim], jnners[dim]], dim=dim) - for dim in dims } + inners = {dim: xr.concat([inners[dim], jnners[dim]], dim=dim) + for dim in dims} data = xr.merge([inners[dim] for dim in dims]) From bc45c262fc728a04bd9b1c22570e9e44bcffa4f6 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:34:53 -0700 Subject: [PATCH 059/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 0cc2ee74..8498c3f3 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -344,7 +344,7 @@ def get_dt_objects(dataset, tag): "nCrossDayAur": data.nCrossDayAur.data} elif 'spect' in inst_id: coords = {"nchan": ["121.6nm", "130.4nm", "135.6nm", - "LBHshort", "LBHlong","?"]} + "LBHshort", "LBHlong", "?"]} data = data.assign_coords(coords=coords) From ab7fcfd7bcb305dd3c70ae8532b0d63a027e63b6 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:35:14 -0700 Subject: [PATCH 060/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 8498c3f3..e86ffa88 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -35,7 +35,7 @@ guvi = pysat.Instrument(platform='timed', name='guvi', inst_id='imaging', tag='low') guvi.download(dt.datetime(2005, 6, 28), dt.datetime(2005, 6, 29)) - guvi.load(yr=2005, doy=171) + guvi.load(date=dt.datetime(2005, 6, 28)) """ From 880c6833f326fc6e16dd6287b949d9b9f6c0f7ed Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 29 Nov 2022 10:17:09 -0500 Subject: [PATCH 061/396] Update pysatNASA/instruments/timed_guvi.py --- pysatNASA/instruments/timed_guvi.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index e86ffa88..55daff21 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -20,9 +20,6 @@ 'imaging' 'spectrograph' -Note ----- - Warnings -------- Currently no cleaning routine. From 8fc14557f7f93ad61c0d71b8c8f6140775935f56 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 8 Dec 2022 18:36:46 -0700 Subject: [PATCH 062/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 55daff21..c7f6684a 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -140,8 +140,8 @@ def clean(self): download_tags = {inst: {tag: {'remote_dir': supported_urls[inst][tag], 'fname': supported_tags[inst][tag]} - for tag in tags.keys()} - for inst in inst_ids.keys()} + for tag in tags.keys()} + for inst in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) From ab89c9cb06b3cb0c1e6aaad710fd8d83e4404290 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 8 Dec 2022 18:36:59 -0700 Subject: [PATCH 063/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index c7f6684a..65000170 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -208,7 +208,8 @@ def get_dt_objects(dataset, tag): return dts # Dimensions for time variables - # night/day along, cross and time are the same imaging low, high, spectrograph low, high + # night/day along, cross and time are the same imaging low, high, + # spectrograph low, high # imaging high possess extra dims DayAur # spectrograph low possess extra dims GAIMDay, GAIMNight dims = ['time'] From 97451e705abf0bdb00765c9b5a0616eff9e703a9 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 8 Dec 2022 18:37:10 -0700 Subject: [PATCH 064/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 65000170..0021d97f 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -236,8 +236,8 @@ def get_dt_objects(dataset, tag): gaimnight_dts = get_dt_objects(data, "GAIM_NIGHT") # Drop out redundant time variables - data = data.drop_vars(["YEAR_DAY", "DOY_DAY", "TIME_DAY", "TIME_EPOCH_DAY", - "YEAR_NIGHT", "DOY_NIGHT", + data = data.drop_vars(["YEAR_DAY", "DOY_DAY", "TIME_DAY", + "TIME_EPOCH_DAY", "YEAR_NIGHT", "DOY_NIGHT" "TIME_NIGHT", "TIME_EPOCH_NIGHT"]) if 'imag' in inst_id: data = data.drop_vars(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", From 92238a4fdcdf35c31c4744925b10b9a454458247 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 8 Dec 2022 18:37:18 -0700 Subject: [PATCH 065/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 0021d97f..96918046 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -241,7 +241,8 @@ def get_dt_objects(dataset, tag): "TIME_NIGHT", "TIME_EPOCH_NIGHT"]) if 'imag' in inst_id: data = data.drop_vars(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", - "TIME_DAY_AURORAL", "TIME_EPOCH_DAY_AURORAL"]) + "TIME_DAY_AURORAL", + "TIME_EPOCH_DAY_AURORAL"]) elif 'spect' in inst_id: if 'low' in tag: data = data.drop_vars(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", From 465b1b86179cfcc6548f56741b5f3f9c10184279 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 8 Dec 2022 18:37:29 -0700 Subject: [PATCH 066/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 96918046..dbb3efad 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -256,7 +256,8 @@ def get_dt_objects(dataset, tag): raise Exception("time along day and night are different") # Renaming along/cross dimensions - data = data.rename_dims({"nAlongDay":"nAlong", "nAlongNight":"nAlong"}) + data = data.rename_dims({"nAlongDay": "nAlong", + "nAlongNight": "nAlong"}) # 'nCross' dimension only in imaging if 'imag' in inst_id: From 94b50f202cc2b0850a54c0fa31f86a943564c603 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 8 Dec 2022 18:37:39 -0700 Subject: [PATCH 067/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index dbb3efad..ae689429 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -259,7 +259,7 @@ def get_dt_objects(dataset, tag): data = data.rename_dims({"nAlongDay": "nAlong", "nAlongNight": "nAlong"}) - # 'nCross' dimension only in imaging + # 'nCross' dimension only in imaging if 'imag' in inst_id: if data.nCrossDay.size != data.nCrossNight.size: From f9843fee77b5f0e0869553e21132eccd646a0780 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 12 Dec 2022 12:54:12 -0500 Subject: [PATCH 068/396] Update pysatNASA/instruments/timed_guvi.py --- pysatNASA/instruments/timed_guvi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index ae689429..bf75f3b9 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -136,7 +136,7 @@ def clean(self): supported_urls = {inst: {tag: url.format(mode=inst) for tag in tags.keys()} - for inst in inst_ids}``` + for inst in inst_ids} download_tags = {inst: {tag: {'remote_dir': supported_urls[inst][tag], 'fname': supported_tags[inst][tag]} From 632430bb2f262f0542ebcbc0dfa1c65ac05b5a5c Mon Sep 17 00:00:00 2001 From: Luis Navarro Date: Tue, 20 Dec 2022 17:34:27 -0700 Subject: [PATCH 069/396] removing whitespaces --- pysatNASA/instruments/timed_guvi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index bf75f3b9..f263afd3 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -70,6 +70,7 @@ # ---------------------------------------------------------------------------- # Instrument methods + def init(self): """Initialize the Instrument object with references and acknowledgements.""" From e8ae7428832dee9bfeec26d4564cdb895b705fbd Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 20 Dec 2022 19:51:35 -0500 Subject: [PATCH 070/396] Update pysatNASA/instruments/timed_guvi.py --- pysatNASA/instruments/timed_guvi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index f263afd3..aa54323a 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -29,7 +29,7 @@ :: import pysat - guvi = pysat.Instrument(platform='timed', name='guvi', inst_id='imaging', + guvi = pysat.Instrument(platform='timed', name='guvi', inst_id='imaging', tag='low') guvi.download(dt.datetime(2005, 6, 28), dt.datetime(2005, 6, 29)) guvi.load(date=dt.datetime(2005, 6, 28)) From 06b15042fea45024625d627816226b9208137544 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 21 Dec 2022 11:20:20 -0500 Subject: [PATCH 071/396] BUG: typo --- pysatNASA/instruments/timed_guvi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index aa54323a..5171a8ff 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -238,7 +238,7 @@ def get_dt_objects(dataset, tag): # Drop out redundant time variables data = data.drop_vars(["YEAR_DAY", "DOY_DAY", "TIME_DAY", - "TIME_EPOCH_DAY", "YEAR_NIGHT", "DOY_NIGHT" + "TIME_EPOCH_DAY", "YEAR_NIGHT", "DOY_NIGHT", "TIME_NIGHT", "TIME_EPOCH_NIGHT"]) if 'imag' in inst_id: data = data.drop_vars(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", From 4caf6c41364b820e5f62d9c53ba3ea8698147627 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 21 Dec 2022 12:57:51 -0500 Subject: [PATCH 072/396] STY: version cap on numpy --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5bcb0ece..35ab70f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ requests beautifulsoup4 lxml cdflib>=0.4.4 -numpy +numpy<1.24 pandas pysat>=3.0.4 xarray<2022.11 From f956100318d80e2ac2b5a8b535544fc3402e9174 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 21 Dec 2022 14:55:19 -0500 Subject: [PATCH 073/396] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a1d903..38bd7555 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +## [0.0.5] - 2023-XX-XX +* Maintenance + * Added a version cap for numpy (required by pysatCDF, revisit before release) + ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class * Support xarray datasets through cdflib From 8fed3bcfd4fb10af31949c9f9b16be466318145f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 21 Dec 2022 15:07:19 -0500 Subject: [PATCH 074/396] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38bd7555..8de2459c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [0.0.5] - 2023-XX-XX * Maintenance - * Added a version cap for numpy (required by pysatCDF, revisit before release) + * Added a version cap for numpy (required for cdf interface, revisit before release) ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 34c6a7c8ac4f8fdc811cea318aff2637320df0b3 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 29 Dec 2022 13:38:55 -0500 Subject: [PATCH 075/396] MAINT: standardize actions --- .github/pull_request_template.md | 11 +++++++---- .github/workflows/docs.yml | 4 ++-- .github/workflows/main.yml | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5331eb30..1a454bc5 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,6 @@ # Description -Addresses # (issue) +Addresses #(issue) Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required @@ -25,9 +25,9 @@ your test configuration - Test A - Test B -## Test Configuration -* Operating system: [Os Type] -* Version number: [Python 2.9] +**Test Configuration**: +* Operating system: Hal +* Version number: Python 3.X * Any details about your local setup that are relevant # Checklist: @@ -43,3 +43,6 @@ your test configuration - [ ] Any dependent changes have been merged and published in downstream modules - [ ] Add a note to ``CHANGELOG.md``, summarizing the changes - [ ] Update zenodo.json file for new code contributors + +If this is a release PR, replace the first item of the above checklist with the release +checklist on the wiki: https://github.com/pysat/pysat/wiki/Checklist-for-Release diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dffe178e..311ef320 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,11 +8,11 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ["ubuntu-latest"] strategy: fail-fast: false matrix: - python-version: [3.9] + python-version: ["3.9"] name: Documentation tests steps: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index be920387..755a8b4a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,8 +56,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: | - pytest -vs --cov=pysatNASA/ + run: pytest -vs --cov=pysatNASA/ - name: Publish results to coveralls env: From c5f5e920ba59c2708138457de2045e0f5ba2a476 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 29 Dec 2022 13:40:20 -0500 Subject: [PATCH 076/396] DOC: update contributing --- CONTRIBUTING.md | 55 ++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5e9b2a04..1e325724 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,33 +78,44 @@ To set up `pysatNASA` for local development: 4. When you're done making changes, run all the checks to ensure that nothing - is broken on your local system, as well as check for flake8 compliance: + is broken on your local system: - ``` - pytest -vs --flake8 pysatNASA - ``` + ``` + pytest PACKAGENAME + ``` + +5. You should also check for flake8 style compliance: + + ``` + flake8 . --count --select=D,E,F,H,W --show-source --statistics + ``` -5. Update/add documentation (in ``docs``), if relevant + Note that pysat uses the `flake-docstrings` and `hacking` packages to ensure + standards in docstring formatting. -6. Add your name to the .zenodo.json file as an author -7. Commit your changes: - ``` - git add . - git commit -m "AAA: Brief description of your changes" - ``` - Where AAA is a standard shorthand for the type of change (eg, BUG or DOC). - `pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html), - see the discussion there for a full list of this shorthand notation. +6. Update/add documentation (in ``docs``), if relevant -8. Once you are happy with the local changes, push to Github: - ``` - git push origin name-of-your-bugfix-or-feature - ``` - Note that each push will trigger the Continuous Integration workflow. +7. Add your name to the .zenodo.json file as an author + +8. Commit your changes: + ``` + git add . + git commit -m "AAA: Brief description of your changes" + ``` + Where AAA is a standard shorthand for the type of change (eg, BUG or DOC). + `pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html), + see the discussion there for a full list of this shorthand notation. + +9. Once you are happy with the local changes, push to Github: + ``` + git push origin name-of-your-bugfix-or-feature + ``` + Note that each push will trigger the Continuous Integration workflow. -9. Submit a pull request through the GitHub website. Pull requests should be - made to the ``develop`` branch. +10. Submit a pull request through the GitHub website. Pull requests should be + made to the ``develop`` branch. Note that automated tests will be run on + github actions, but these must be initialized by a member of the pysat team. Pull Request Guidelines ----------------------- @@ -160,3 +171,5 @@ These include: * Block and inline comments should use proper English grammar and punctuation with the exception of single sentences in a block, which may then omit the final period +* When casting is necessary, use `np.int64` and `np.float64` to ensure operating + system agnosticism From ec006e092f080b9d494ec23c00b6a377a45c55e5 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 29 Dec 2022 13:40:35 -0500 Subject: [PATCH 077/396] TST: remove pandas cap --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 755a8b4a..c2a3fbfc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,7 +36,7 @@ jobs: run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} # Need to force a version of pandas compliant with NEP29 - pip install "pandas<1.5" + # pip install "pandas<1.5" - name: Install standard dependencies run: | From 3d93184ba81c399d6d74450a614d753358a351f0 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 29 Dec 2022 13:41:16 -0500 Subject: [PATCH 078/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8de2459c..478d975f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [0.0.5] - 2023-XX-XX * Maintenance * Added a version cap for numpy (required for cdf interface, revisit before release) + * Updated actions and templates based on pysatEcosystem docs. ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 2c2d6b6e1bdf9719a65846f54a288e8a70923908 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 29 Dec 2022 14:06:12 -0500 Subject: [PATCH 079/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 478d975f..abf01911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Maintenance * Added a version cap for numpy (required for cdf interface, revisit before release) * Updated actions and templates based on pysatEcosystem docs. + * Remove pandas cap on NEP29 tests ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 7a350091e794d728574fbab0c2fd75655e432fab Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 29 Dec 2022 14:06:44 -0500 Subject: [PATCH 080/396] MAINT: remove pandas cap --- .github/workflows/main.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c2a3fbfc..cc04a773 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,10 +33,7 @@ jobs: - name: Install NEP29 dependencies if: ${{ matrix.numpy_ver != 'latest'}} - run: | - pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - # Need to force a version of pandas compliant with NEP29 - # pip install "pandas<1.5" + run: pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - name: Install standard dependencies run: | From 21fb4ba1accbdba50f8b1c32b4283b6579009ea2 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:56:29 -0500 Subject: [PATCH 081/396] added cdas download and list remote files functions --- pysatNASA/instruments/methods/cdaweb.py | 179 ++++++++++++++++++++++-- 1 file changed, 171 insertions(+), 8 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index c71115f2..2d5f2bd4 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -16,6 +16,7 @@ import xarray as xr from bs4 import BeautifulSoup +from cdasws import CdasWs import pysat from pysat.instruments.methods import general @@ -31,6 +32,35 @@ auto_CDF = libCDF +def try_inst_dict(inst_id, tag, supported_tags): + """Check that the inst_id and tag combination is valid. + + Parameters + ---------- + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be + pre-set with functools.partial then assigned to new instrument code. + (default=None) + + Returns + ------- + inst_dict : dict or str + dictionary containing file location in spdf archive, or dataset ID for + cdasws + """ + try: + inst_dict = supported_tags[inst_id][tag] + except KeyError: + raise ValueError('inst_id / tag combo unknown.') + + return inst_dict + + def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), flatten_twod=True, pandas_format=True, epoch_name='Epoch', meta_processor=None, meta_translation=None, drop_meta_labels=None, @@ -439,10 +469,7 @@ def download(date_array, tag='', inst_id='', supported_tags=None, """ - try: - inst_dict = supported_tags[inst_id][tag] - except KeyError: - raise ValueError('inst_id / tag combo unknown.') + inst_dict = try_inst_dict(inst_id, tag, supported_tags) # Naming scheme for files on the CDAWeb server remote_dir = inst_dict['remote_dir'] @@ -488,6 +515,76 @@ def download(date_array, tag='', inst_id='', supported_tags=None, return +def cdas_download(date_array, tag='', inst_id='', supported_tags=None, + data_path=None): + """Download NASA CDAWeb CDF data using cdasws. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + date_array : array-like + Array of datetimes to download data for. Provided by pysat. + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be pre-set with + functools.partial then assigned to new instrument code. + (default=None) + data_path : str or NoneType + Path to data directory. If None is specified, the value previously + set in Instrument.files.data_path is used. (default=None) + + Examples + -------- + :: + + # download support added to cnofs_vefi.py using code below + fn = 'cnofs_vefi_bfield_1sec_{year:4d}{month:02d}{day:02d}_v05.cdf' + dc_b_tag = {'remote_dir': ''.join(('/pub/data/cnofs/vefi/bfield_1sec', + '/{year:4d}/')), + 'fname': fn} + supported_tags = {'dc_b': dc_b_tag} + + download = functools.partial(nasa_cdaweb.download, + supported_tags=supported_tags) + + """ + + start = date_array[0] + stop = date_array[-1] + remote_files = cdas_list_remote_files(tag=tag, inst_id=inst_id, + start=start, stop=stop, + supported_tags=supported_tags) + + for file in remote_files: + + fname = file.split('/')[-1] + saved_local_fname = os.path.join(data_path, fname) + + # Perform download + logger.info(' '.join(('Attempting to download file: ', + file))) + try: + with requests.get(file) as req: + if req.status_code != 404: + with open(saved_local_fname, 'wb') as open_f: + open_f.write(req.content) + logger.info('Successfully downloaded {:}.'.format( + saved_local_fname)) + else: + logger.info(' '.join(('File: "', file, + '" is not available'))) + except requests.exceptions.RequestException as exception: + logger.info(' '.join((str(exception), '- File: "', file, + '" Is not available'))) + return + + def list_remote_files(tag='', inst_id='', start=None, stop=None, remote_url='https://cdaweb.gsfc.nasa.gov', supported_tags=None, two_digit_year_break=None, @@ -551,10 +648,7 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, """ - try: - inst_dict = supported_tags[inst_id][tag] - except KeyError: - raise ValueError('inst_id / tag combo unknown.') + inst_dict = try_inst_dict(inst_id, tag, supported_tags) # Naming scheme for files on the CDAWeb server format_str = '/'.join((inst_dict['remote_dir'].strip('/'), @@ -654,3 +748,72 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, stored_list = stored_list[mask] return stored_list + + +def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, + supported_tags=None): + """Return a list of every file for chosen remote data. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + start : dt.datetime or NoneType + Starting time for file list. A None value will start with the first + file found. + (default=None) + stop : dt.datetime or NoneType + Ending time for the file list. A None value will stop with the last + file found. + (default=None) + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be + pre-set with functools.partial then assigned to new instrument code. + (default=None) + + Returns + ------- + file_list : list + A list containing the verified available files + + Examples + -------- + :: + + fname = 'cnofs_vefi_bfield_1sec_{year:04d}{month:02d}{day:02d}_v05.cdf' + supported_tags = {'dc_b': fname} + list_remote_files = \ + functools.partial(nasa_cdaweb.list_remote_files, + supported_tags=supported_tags) + + fname = 'cnofs_cindi_ivm_500ms_{year:4d}{month:02d}{day:02d}_v01.cdf' + supported_tags = {'': fname} + list_remote_files = \ + functools.partial(cdw.list_remote_files, + supported_tags=supported_tags) + + """ + cdas = CdasWs() + dataset = try_inst_dict(inst_id, tag, supported_tags) + + if start is None and stop is None: + # Use the topmost directory without variables + start = cdas.get_inventory(identifier=dataset)[0].start + stop = cdas.get_inventory(identifier=dataset)[-1].end + + if start == stop: + stop = start + dt.timedelta(days=1) + + if isinstance(start, pds._libs.tslibs.timestamps.Timestamp): + start = start.tz_localize('utc') + stop = stop.tz_localize('utc') + og_files = cdas.get_original_files(dataset=dataset, start=start, end=stop) + file_list = [f['Name'] for f in og_files[1]] + + return file_list From afdc5f681b345fc70b861a6f016cd6fc7091286e Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:57:09 -0500 Subject: [PATCH 082/396] modified compatible instruments to use the cdasws download and list remore files functinos --- pysatNASA/instruments/cnofs_ivm.py | 9 ++-- pysatNASA/instruments/cnofs_plp.py | 8 ++-- pysatNASA/instruments/cnofs_vefi.py | 8 ++-- pysatNASA/instruments/de2_lang.py | 9 ++-- pysatNASA/instruments/de2_nacs.py | 9 ++-- pysatNASA/instruments/de2_rpa.py | 9 ++-- pysatNASA/instruments/de2_wats.py | 9 ++-- pysatNASA/instruments/formosat1_ivm.py | 9 ++-- pysatNASA/instruments/icon_euv.py | 8 ++-- pysatNASA/instruments/icon_fuv.py | 18 ++++---- pysatNASA/instruments/icon_ivm.py | 9 ++-- pysatNASA/instruments/iss_fpmu.py | 9 ++-- pysatNASA/instruments/omni_hro.py | 58 +++----------------------- pysatNASA/instruments/ses14_gold.py | 10 ++--- pysatNASA/instruments/timed_saber.py | 9 ++-- pysatNASA/instruments/timed_see.py | 9 ++-- 16 files changed, 58 insertions(+), 142 deletions(-) diff --git a/pysatNASA/instruments/cnofs_ivm.py b/pysatNASA/instruments/cnofs_ivm.py index a7693648..d52858c8 100644 --- a/pysatNASA/instruments/cnofs_ivm.py +++ b/pysatNASA/instruments/cnofs_ivm.py @@ -245,12 +245,9 @@ def clean(self): # Set the load routine load = cdw.load -# Set the download routine -basic_tag = {'remote_dir': '/pub/data/cnofs/cindi/ivm_500ms_cdf/{year:4d}/', - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'CNOFS_CINDI_IVM_500MS'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/cnofs_plp.py b/pysatNASA/instruments/cnofs_plp.py index fb43ab9e..f112aa51 100644 --- a/pysatNASA/instruments/cnofs_plp.py +++ b/pysatNASA/instruments/cnofs_plp.py @@ -119,11 +119,9 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': '/pub/data/cnofs/plp/plasma_1sec/{year:4d}/', - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'CNOFS_PLP_PLASMA_1SEC'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py index 78d54e36..aff71c13 100644 --- a/pysatNASA/instruments/cnofs_vefi.py +++ b/pysatNASA/instruments/cnofs_vefi.py @@ -122,11 +122,9 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': '/pub/data/cnofs/vefi/bfield_1sec/{year:4d}/', - 'fname': fname} -download_tags = {'': {'dc_b': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_lang.py b/pysatNASA/instruments/de2_lang.py index 07430357..1bd35536 100644 --- a/pysatNASA/instruments/de2_lang.py +++ b/pysatNASA/instruments/de2_lang.py @@ -97,12 +97,9 @@ load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_lang', - '/plasma500ms_lang_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_PLASMA500MS_LANG'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_nacs.py b/pysatNASA/instruments/de2_nacs.py index b03212fe..75af6b13 100644 --- a/pysatNASA/instruments/de2_nacs.py +++ b/pysatNASA/instruments/de2_nacs.py @@ -120,12 +120,9 @@ load = cdw.load # Support download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/neutral_gas_nacs', - '/neutral1s_nacs_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_NEUTRAL1S_NACS'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Support listing files currently on CDAWeb -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 912dd66f..3e1bd6f0 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -106,12 +106,9 @@ load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_rpa', - '/ion2s_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_ION2S_RPA'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_wats.py b/pysatNASA/instruments/de2_wats.py index 1345841a..f586ce0b 100644 --- a/pysatNASA/instruments/de2_wats.py +++ b/pysatNASA/instruments/de2_wats.py @@ -117,12 +117,9 @@ load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/neutral_gas_wats', - '/wind2s_wats_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_WIND2S_WATS'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/formosat1_ivm.py b/pysatNASA/instruments/formosat1_ivm.py index 24be65bd..e770abb6 100644 --- a/pysatNASA/instruments/formosat1_ivm.py +++ b/pysatNASA/instruments/formosat1_ivm.py @@ -92,12 +92,9 @@ def init(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/formosat-rocsat/formosat-1', - '/ipei/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'RS_K0_IPEI'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/icon_euv.py b/pysatNASA/instruments/icon_euv.py index 7a5be9e4..c16e8e7c 100644 --- a/pysatNASA/instruments/icon_euv.py +++ b/pysatNASA/instruments/icon_euv.py @@ -127,13 +127,11 @@ def clean(self): supported_tags=supported_tags) # Set the download routine -basic_tag = {'remote_dir': '/pub/data/icon/l2/l2-6_euv/{year:04d}/', - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'ICON_L2-6_EUV'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py index 19b3cae5..964399d0 100644 --- a/pysatNASA/instruments/icon_fuv.py +++ b/pysatNASA/instruments/icon_fuv.py @@ -118,17 +118,19 @@ def clean(self): list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) -# Set the download routine -basic_tag24 = {'remote_dir': '/pub/data/icon/l2/l2-4_fuv_day/{year:04d}/', - 'fname': fname24} -basic_tag25 = {'remote_dir': '/pub/data/icon/l2/l2-5_fuv_night/{year:04d}/', - 'fname': fname25} -download_tags = {'': {'day': basic_tag24, 'night': basic_tag25}} +# old Set the download routine +# basic_tag24 = {'remote_dir': '/pub/data/icon/l2/l2-4_fuv_day/{year:04d}/', +# 'fname': fname24} +# basic_tag25 = {'remote_dir': '/pub/data/icon/l2/l2-5_fuv_night/{year:04d}/', +# 'fname': fname25} +# download_tags = {'': {'day': basic_tag24, 'night': basic_tag25}} +download_tags = {'': {'day': 'ICON_L2-5_FUV_DAY', + 'night': 'ICON_L2-5_FUV_NIGHT'}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/icon_ivm.py b/pysatNASA/instruments/icon_ivm.py index e16ca5b0..6df533a0 100644 --- a/pysatNASA/instruments/icon_ivm.py +++ b/pysatNASA/instruments/icon_ivm.py @@ -192,13 +192,12 @@ def clean(self): # Set the download routine dirstr = '/pub/data/icon/l2/l2-7_ivm-{id:s}/{{year:4d}}/' -download_tags = {id: {'': {'remote_dir': dirstr.format(id=id), - 'fname': supported_tags[id]['']}} - for id in ['a', 'b']} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'a': {'': 'ICON_L2-7_IVM-A'}, 'b': {'': 'ICON_L2-7_IVM-B'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/iss_fpmu.py b/pysatNASA/instruments/iss_fpmu.py index b85c6aa5..3fc44673 100644 --- a/pysatNASA/instruments/iss_fpmu.py +++ b/pysatNASA/instruments/iss_fpmu.py @@ -115,12 +115,9 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/international_space_station_iss', - '/sp_fpmu/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'ISS_SP_FPMU'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/omni_hro.py b/pysatNASA/instruments/omni_hro.py index d709c1b5..aad7d05d 100644 --- a/pysatNASA/instruments/omni_hro.py +++ b/pysatNASA/instruments/omni_hro.py @@ -46,7 +46,6 @@ from pysat.instruments.methods import general as mm_gen from pysat import logger -from pysat.utils import time as pysat_time from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import omni as mm_omni @@ -140,59 +139,12 @@ def clean(self): file_cadence=pds.DateOffset(months=1)) # Set the list_remote_files routine -remote_dir = '/pub/data/omni/omni_cdaweb/hro_{tag:s}/{{year:4d}}/' -download_tags = {inst_id: {tag: {'remote_dir': remote_dir.format(tag=tag), - 'fname': supported_tags[inst_id][tag]} - for tag in tags.keys()} - for inst_id in inst_ids.keys()} -list_remote_files = functools.partial(cdw.list_remote_files, - supported_tags=download_tags) - - -# Set the download routine -def download(date_array, tag, inst_id, data_path, update_files=False): - """Download OMNI HRO data from CDAWeb. - - Parameters - ---------- - date_array : array-like - Sequence of dates for which files will be downloaded. - tag : str - Denotes type of file to load. - inst_id : str - Specifies the satellite ID for a constellation. - data_path : str - Path to data directory. - update_files : bool - Re-download data for files that already exist if True (default=False) - - Raises - ------ - IOError - If a problem is encountered connecting to the gateway or retrieving - data from the repository. - - Warnings - -------- - Only able to download current forecast data, not archived forecasts. - - Note - ---- - Called by pysat. Not intended for direct use by user. - - """ - - # Set the download tags - - # Adjust the date_array for monthly downloads - if date_array.freq != 'MS': - date_array = pysat_time.create_date_range( - dt.datetime(date_array[0].year, date_array[0].month, 1), - date_array[-1], freq='MS') +download_tags = {'': {'1min': 'OMNI_HRO_1MIN', '5min': 'OMNI_HRO_5MIN'}} +download = functools.partial(cdw.cdas_download, + supported_tags=download_tags) - cdw.download(date_array, tag=tag, inst_id=inst_id, - supported_tags=download_tags, data_path=data_path) - return +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) # Set the load routine diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 5a89413f..14c6f9e9 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -104,15 +104,11 @@ def init(self): supported_tags=supported_tags) # Set the download routine -download_tags = {inst_id: - {tag: {'remote_dir': ''.join(('/pub/data/gold/level2/', tag, - '/{year:4d}/')), - 'fname': supported_tags[''][tag]} - for tag in tags.keys()} for inst_id in inst_ids.keys()} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'nmax': 'GOLD_L2_NMAX'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index 38f82b91..6a758da9 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -117,12 +117,9 @@ def init(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/timed/saber/level2a_cdf', - '/{year:4d}/{month:02d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'TIMED_L2A_SABER'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 903058ab..afe7c04a 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -107,12 +107,9 @@ def init(self): load = functools.partial(cdw.load, file_cadence=pds.DateOffset(months=1)) # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/timed/see/data/level3a_cdf', - '/{year:4d}/{month:02d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'TIMED_L3A_SEE'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) From beb96cd6d307d651484e0267bff1e290ead3d402 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 11 Jan 2023 13:00:02 -0500 Subject: [PATCH 083/396] updated tests for new cdasws download and list remote files. This includes changing the instrument used to test non cdasws methods and modifying existing tests to be compatible with new contents in the inst_dict --- pysatNASA/tests/test_methods_cdaweb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index deb25417..e4813e87 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -16,7 +16,7 @@ class TestCDAWeb(object): def setup_method(self): """Set up the unit test environment for each method.""" - self.download_tags = pysatNASA.instruments.cnofs_plp.download_tags + self.download_tags = pysatNASA.instruments.icon_mighti.download_tags self.kwargs = {'tag': None, 'inst_id': None} return @@ -31,7 +31,7 @@ def test_remote_file_list_connection_error_append(self): with pytest.raises(Exception) as excinfo: # Giving a bad remote_site address yields similar ConnectionError - cdw.list_remote_files(tag='', inst_id='', + cdw.list_remote_files(tag='los_wind_green', inst_id='a', supported_tags=self.download_tags, remote_url='https://bad/path') From ee093286c47e0754b0c0f95b79446cd4005a2d64 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 11 Jan 2023 13:00:15 -0500 Subject: [PATCH 084/396] updated changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a1d903..80fb30ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +## [0.0.5] - 2023-XX-XX +* Added cdaweb methods that can use cdasws to get the remote file list + ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class * Support xarray datasets through cdflib From b8fd3a34de0175cd896bab2c81dfc9b52696b6b5 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 11 Jan 2023 16:14:01 -0500 Subject: [PATCH 085/396] added cdasws requirement --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 49912451..6cf21798 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,6 +33,7 @@ include_package_data = True zip_safe = False packages = find: install_requires = + cdasws netCDF4 requests beautifulsoup4 From d7b6686f06205cb693b5cf8d164c63fe89d85483 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Thu, 12 Jan 2023 12:08:03 -0500 Subject: [PATCH 086/396] added cdasws to requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 35ab70f2..76cad30a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ numpy<1.24 pandas pysat>=3.0.4 xarray<2022.11 +cdasws From 9c36a65a9153ee4abbce4493346c75db571b749b Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 18 Jan 2023 14:02:02 -0500 Subject: [PATCH 087/396] updated .gitignore for vscode --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6db464db..c8350919 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,6 @@ custom_lint.sh # IDEs .idea/ + +# vscode +.vscode From a1889e4635c42e769bb89c852bd36e53f92bb613 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 17:05:26 -0500 Subject: [PATCH 088/396] ENH: added DMSP SSUSI instrument Added a DMSP SSUSI instrument, currently only handling one of the available data products from this instrument. --- pysatNASA/instruments/__init__.py | 3 +- pysatNASA/instruments/dmsp_ssusi.py | 225 ++++++++++++++++++++++ pysatNASA/instruments/methods/__init__.py | 1 + pysatNASA/instruments/methods/dmsp.py | 16 ++ 4 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 pysatNASA/instruments/dmsp_ssusi.py create mode 100644 pysatNASA/instruments/methods/dmsp.py diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index c3a8bfae..a2f55597 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -4,10 +4,11 @@ Each instrument is contained within a subpackage of this set. """ +from pysatNASA.instruments import methods # noqa F401 __all__ = ['cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', - 'formosat1_ivm', + 'dmsp_ssusi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', 'timed_saber', 'timed_see'] diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py new file mode 100644 index 00000000..a5b95900 --- /dev/null +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8 -*- +"""Module for the DMSP SSUSI instrument. + +Supports the Special Sensor Ultraviolet Spectrographic Imager (SSUSI) +instrument on Defense Meteorological Satellite Program (DMSP). + +From JHU APL: + +SSUSI was designed for the DMSP Block 5D-3 satellites. These satellites are +placed into nearly polar, sun-synchronous orbits at an altitude of about 850 km. +SSUSI is a remote-sensing instrument which measures ultraviolet (UV) emissions +in five different wavelength bands from the Earth's upper atmosphere. SSUSI is +mounted on a nadir-looking panel of the satellite. The multicolor images from +SSUSI cover the visible Earth disk from horizon to horizon and the anti-sunward +limb up to an altitude of approximately 520 km. + +The UV images and the derived environmental data provide the Air Force Weather +Agency (Offutt Air Force Base, Bellevue, NE) with near real-time information +that can be utilized in a number of applications, such as maintenance of high +frequency (HF) communication links and related systems and assessment of the +environmental hazard to astronauts on the Space Station. + +References +---------- +Larry J. Paxton, Daniel Morrison, Yongliang Zhang, Hyosub Kil, Brian Wolven, +Bernard S. Ogorzalek, David C. Humm, and Ching-I. Meng "Validation of remote +sensing products produced by the Special Sensor Ultraviolet Scanning Imager +(SSUSI): a far UV-imaging spectrograph on DMSP F-16", Proc. SPIE 4485, Optical +Spectroscopic Techniques, Remote Sensing, and Instrumentation for Atmospheric +and Space Research IV, (30 January 2002); doi:10.1117/12.454268 + +Properties +---------- +platform + 'dmsp' +name + 'ssusi' +tag + 'edr-aurora' +inst_id + 'f16', 'f17', 'f18', 'f19' + + +Warnings +-------- +- Currently no cleaning routine. + + +""" + +import datetime as dt +import functools +import numpy as np +import pandas as pds +import xarray as xr + +from pysat.instruments.methods import general as mm_gen +from pysat.utils.io import load_netcdf + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import dmsp as mm_dmsp + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'dmsp' +name = 'ssusi' +tags = {'edr-aurora': ''.join(['Electron energy flux and mean energy, auroral', + ' boundaries, identified discrete auroral arcs,', + ' hemispheric power, and magnetic field lines ', + 'traced to 4 Earth radii'])} +inst_ids = {sat_id: list(tags.keys()) + for sat_id in ['f16', 'f17', 'f18', 'f19']} + +pandas_format = False +multi_file_day = True + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {inst_id: {tag: dt.datetime(2015, 1, 1) for tag in tags.keys()} + for inst_id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_dmsp, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(['dmsp{inst_id:s}_ssusi_{tag:s}_{{year:04d}}{{day:03d}}T', + '{{hour:02d}}{{minute:02d}}{{second:02d}}-???????T??????-REV', + '?????_vA{{version:1d}}.?.?r{{cycle:03d}}.nc']) +supported_tags = {sat_id: {tag: fname.format(tag=tag, inst_id=sat_id) + for tag in tags.keys()} + for sat_id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + + +# Set the load routine +def load(fnames, tag='', inst_id=''): + """Load DMSP SSUSI data and meta data. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + tag : str + Tag name used to identify particular data set to be loaded (default='') + inst_id : str + DMSP satellite ID (default='') + + Returns + ------- + data : pds.DataFrame or xr.Dataset + Data to be assigned to the pysat.Instrument.data object. + mdata : pysat.Meta + Pysat Meta data for each data variable. + + Examples + -------- + :: + + inst = pysat.Instrument('dmsp', 'ssusi', tag='edr-aurora', + inst_id='f16') + inst.load(2006, 1) + + """ + # Define the input variables + labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} + + # CDAWeb stores these files in the NetCDF format instead of the CDF format + single_data = list() + for fname in fnames: + # There are multiple files per day, with time as a variable rather + # than a dimension or coordinate. Additionally, no coordinates + # are assigned. + subday_data, mdata = load_netcdf(fname, epoch_name='TIME', + epoch_unit='s', labels=labels, + pandas_format=pandas_format) + single_data.append(subday_data) + + # After loading all the data, determine which dimensions need to be expanded + combo_dims = {dim: max([sdata.dims[dim] for sdata in single_data]) + for dim in subday_data.dims.keys()} + + # Expand the data so that all dimensions are the same shape + for i, sdata in enumerate(single_data): + # Determine which dimensions need to be updated + fix_dims = [dim for dim in sdata.dims.keys() + if sdata.dims[dim] < combo_dims[dim]] + + new_data = {} + update_new = False + for dvar in sdata.data_vars.keys(): + # See if any dimensions need to be updated + update_dims = list(set(sdata[dvar].dims) & set(fix_dims)) + + # Save the old data as is, or pad it to have the right dims + if len(update_dims) > 0: + update_new = True + new_shape = list(sdata[dvar].values.shape) + old_slice = [slice(0, ns) for ns in new_shape] + + for dim in update_dims: + idim = list(sdata[dvar].dims).index(dim) + new_shape[idim] = combo_dims[dim] + + # Set the new data for output + new_dat = np.full(shape=new_shape, fill_value=mdata[ + dvar, mdata.labels.fill_val]) + new_dat[tuple(old_slice)] = sdata[dvar].values + new_data[dvar] = (sdata[dvar].dims, new_dat) + else: + new_data[dvar] = sdata[dvar] + + # Calculate the time for this data file + ftime = dt.datetime.strptime( + "{:4d}-{:03d}".format( + sdata['YEAR'].values.astype(int), + sdata['DOY'].values.astype(int)), '%Y-%j') + ( + pds.to_datetime(sdata['time'].values).to_pydatetime() + - dt.datetime(1970, 1, 1)) + + # Get the updated dataset + ndata = xr.Dataset(new_data) if update_new else sdata + ndata['time'] = ftime + + # Assign a datetime variable, making indexing possible + single_data[i] = ndata.assign_coords( + {'time': ndata['time']}).expand_dims(dim='time') + + # Combine all the data, indexing along time + data = xr.combine_by_coords(single_data) + + # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by + # removing 'TIME', once possible + + return data, mdata + + +# Set the download routine +basic_tag = {'remote_dir': ''.join(('/pub/data/dmsp/dmsp{inst_id:s}/ssusi/', + '/data/{tag:s}/{{year:4d}}/{{day:03d}}/')), + 'fname': fname} +download_tags = { + sat_id: {tag: {btag: basic_tag[btag].format(tag=tag, inst_id=sat_id) + for btag in basic_tag.keys()} for tag in tags.keys()} + for sat_id in inst_ids.keys()} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/__init__.py b/pysatNASA/instruments/methods/__init__.py index 5bde9a41..7c8a3c93 100644 --- a/pysatNASA/instruments/methods/__init__.py +++ b/pysatNASA/instruments/methods/__init__.py @@ -4,6 +4,7 @@ from pysatNASA.instruments.methods import cdaweb # noqa F401 from pysatNASA.instruments.methods import cnofs # noqa F401 from pysatNASA.instruments.methods import de2 # noqa F401 +from pysatNASA.instruments.methods import dmsp # noqa F401 from pysatNASA.instruments.methods import general # noqa F401 from pysatNASA.instruments.methods import icon # noqa F401 from pysatNASA.instruments.methods import omni # noqa F401 diff --git a/pysatNASA/instruments/methods/dmsp.py b/pysatNASA/instruments/methods/dmsp.py new file mode 100644 index 00000000..b38392db --- /dev/null +++ b/pysatNASA/instruments/methods/dmsp.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for the DMSP data.""" + +ackn_str = "".join(["This Defense Meteorological Satellite Program (DMSP) ", + "satellite data is provided through CDAWeb"]) + +refs = {'ssusi': ' '.join(('Larry J. Paxton, Daniel Morrison, Yongliang Zhang,', + ' Hyosub Kil, Brian Wolven, Bernard S. Ogorzalek, ', + 'David C. Humm, and Ching-I. Meng "Validation of ', + 'remote sensing products produced by the Special ', + 'Sensor Ultraviolet Scanning Imager (SSUSI): a far ', + 'UV-imaging spectrograph on DMSP F-16", Proc. SPIE ', + '4485, Optical Spectroscopic Techniques, Remote ', + 'Sensing, and Instrumentation for Atmospheric and ', + 'Space Research IV, (30 January 2002); ', + 'doi:10.1117/12.454268'))} From 07f20907bb6b3c8e2ad8c887f3bf5eeb0a689920 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 17:05:51 -0500 Subject: [PATCH 089/396] TST: added DMSP method unit tests Added unit tests for the DMSP methods. --- pysatNASA/tests/test_methods_dmsp.py | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 pysatNASA/tests/test_methods_dmsp.py diff --git a/pysatNASA/tests/test_methods_dmsp.py b/pysatNASA/tests/test_methods_dmsp.py new file mode 100644 index 00000000..86075330 --- /dev/null +++ b/pysatNASA/tests/test_methods_dmsp.py @@ -0,0 +1,31 @@ +"""Unit tests for the DMSP methods.""" + +from pysatNASA.instruments.methods import dmsp + + +class TestDMSPMethods(object): + """Unit tests for `pysat.instruments.methods.dmsp`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['ssusi'] + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names + return + + def test_ack(self): + """Test that the acknowledgements reference the correct platform.""" + + assert dmsp.find('Defense Meteorological Satellite Program') >= 0 + return + + def test_ref(self): + """Test that all DMSP instruments have references.""" + + for name in self.names: + assert name in dmsp.refs.keys(), "No reference for {:}".format(name) + return From 7777b154defc3a9a9717a5d5cd840a657afab52c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 17:06:27 -0500 Subject: [PATCH 090/396] BUG: updated download directory structure Allow downloads to be performed from directories with the YYYY/DDD structure. --- pysatNASA/instruments/methods/cdaweb.py | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index c71115f2..5ec711ac 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -398,7 +398,7 @@ def load_xarray(fnames, tag='', inst_id='', def download(date_array, tag='', inst_id='', supported_tags=None, remote_url='https://cdaweb.gsfc.nasa.gov', data_path=None): - """Download NASA CDAWeb CDF data. + """Download NASA CDAWeb data. This routine is intended to be used by pysat instrument modules supporting a particular NASA CDAWeb dataset. @@ -427,7 +427,7 @@ def download(date_array, tag='', inst_id='', supported_tags=None, -------- :: - # download support added to cnofs_vefi.py using code below + # Download support added to cnofs_vefi.py using code below fn = 'cnofs_vefi_bfield_1sec_{year:4d}{month:02d}{day:02d}_v05.cdf' dc_b_tag = {'remote_dir': ''.join(('/pub/data/cnofs/vefi/bfield_1sec', '/{year:4d}/')), @@ -599,18 +599,27 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, stop = dt.datetime.now() if (stop is None) else stop if 'year' in search_dir['keys']: + url_list = [] if 'month' in search_dir['keys']: search_times = pds.date_range(start, stop + pds.DateOffset(months=1), freq='M') + for time in search_times: + subdir = format_dir.format(year=time.year, month=time.month) + url_list.append('/'.join((remote_url, subdir))) else: - search_times = pds.date_range(start, - stop + pds.DateOffset(years=1), - freq='Y') - url_list = [] - for time in search_times: - subdir = format_dir.format(year=time.year, month=time.month) - url_list.append('/'.join((remote_url, subdir))) + if 'day' in search_dir['keys']: + search_times = pds.date_range(start, stop + + pds.DateOffset(days=1), + freq='D') + else: + search_times = pds.date_range(start, stop + + pds.DateOffset(years=1), + freq='Y') + for time in search_times: + doy = int(time.strftime('%j')) + subdir = format_dir.format(year=time.year, day=doy) + url_list.append('/'.join((remote_url, subdir))) try: for top_url in url_list: for level in range(n_layers + 1): From 67c71b000880816577cad433930ef9a5300f0818 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 17:06:56 -0500 Subject: [PATCH 091/396] DOC: added DMSP SSUSI to the docs Added the new instrument to the list of supported instruments. --- docs/supported_instruments.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 054bc982..10d3962a 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -57,6 +57,14 @@ DE2 WATS .. automodule:: pysatNASA.instruments.de2_wats :members: +.. _dmsp_ssusi: + +DMSP SSUSI +---------- + +.. automodule:: pysatNASA.instruments.dmsp_ssusi + :members: + .. _formosat1_ivm: FORMOSAT-1 IVM From c081a09c56b9a31b8f9671ed78ca6260bcbc713c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 17:07:11 -0500 Subject: [PATCH 092/396] DOC: updated changelog Added a description of the changes to the changelog. --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8de2459c..d5335347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). ## [0.0.5] - 2023-XX-XX +* New Instruments + * DMSP SSUSI EDR-Aurora data +* Bug Fixes + * Updated CDAWeb routines to allow for data stored by year/day-of-year * Maintenance * Added a version cap for numpy (required for cdf interface, revisit before release) From d78467354a6c8315df2301cbe51a4dc46ec97087 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 17:16:57 -0500 Subject: [PATCH 093/396] DOC: improve example Improved the example, ensuring there's data available. --- pysatNASA/instruments/dmsp_ssusi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index a5b95900..85552008 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -133,7 +133,7 @@ def load(fnames, tag='', inst_id=''): :: inst = pysat.Instrument('dmsp', 'ssusi', tag='edr-aurora', - inst_id='f16') + inst_id='f15') inst.load(2006, 1) """ From 26ff03a2cfc31e092cf989ef66f382bbecffb75d Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 17:18:28 -0500 Subject: [PATCH 094/396] STY: updated import order Updated the import order to be alphabetical. --- pysatNASA/instruments/dmsp_ssusi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index 85552008..baf9bfb6 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -58,8 +58,8 @@ from pysat.utils.io import load_netcdf from pysatNASA.instruments.methods import cdaweb as cdw -from pysatNASA.instruments.methods import general as mm_nasa from pysatNASA.instruments.methods import dmsp as mm_dmsp +from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- # Instrument attributes From b533aa142f602081b14850026d5464b19c64deef Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 17:21:37 -0500 Subject: [PATCH 095/396] REV: fixed a bad update I had the satellite right the first time. --- pysatNASA/instruments/dmsp_ssusi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index baf9bfb6..a16acd69 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -133,7 +133,7 @@ def load(fnames, tag='', inst_id=''): :: inst = pysat.Instrument('dmsp', 'ssusi', tag='edr-aurora', - inst_id='f15') + inst_id='f16') inst.load(2006, 1) """ From 42590bc3d84c3f31bf0139d3c3ba2f5d9e6c4d5a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 17:44:12 -0500 Subject: [PATCH 096/396] BUG: updated unit test Fixed the attribute being tested from the DMSP instrument module. --- pysatNASA/tests/test_methods_dmsp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_methods_dmsp.py b/pysatNASA/tests/test_methods_dmsp.py index 86075330..1bef74a4 100644 --- a/pysatNASA/tests/test_methods_dmsp.py +++ b/pysatNASA/tests/test_methods_dmsp.py @@ -20,7 +20,8 @@ def teardown_method(self): def test_ack(self): """Test that the acknowledgements reference the correct platform.""" - assert dmsp.find('Defense Meteorological Satellite Program') >= 0 + assert dmsp.ackn_str.find( + 'Defense Meteorological Satellite Program') >= 0 return def test_ref(self): From 9898c621802e564b2a13921e4955a8bb96000765 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:52:16 -0500 Subject: [PATCH 097/396] ENH: added a TIMED GUVI instrument Added a TIMED GUVI instrument for the EDR Auroral data (L2). --- pysatNASA/instruments/__init__.py | 2 +- pysatNASA/instruments/timed_guvi.py | 113 ++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 pysatNASA/instruments/timed_guvi.py diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index a2f55597..a00b1ae8 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -11,7 +11,7 @@ 'dmsp_ssusi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', - 'timed_saber', 'timed_see'] + 'timed_guvi', 'timed_saber', 'timed_see'] for inst in __all__: exec("from pysatNASA.instruments import {x}".format(x=inst)) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py new file mode 100644 index 00000000..6dda4c95 --- /dev/null +++ b/pysatNASA/instruments/timed_guvi.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +"""Module for the TIMED GUVI instrument. + +Supports the Global UltraViolet Imager (GUVI) instrument on the Thermosphere +Ionosphere Mesosphere Energetics Dynamics (TIMED) satellite. + +From JHU APL: + +The Global Ultraviolet Imager (GUVI) is one of four instruments that constitute +the TIMED spacecraft, the first mission of the NASA Solar Connections program. +The TIMED spacecraft is being built by Johns Hopkins University Applied Physics +Laboratory and GUVI is a joint collaboration between JHU/APL and the Aerospace +Corporation. TIMED will be used to study the energetics and dynamics of the +Mesosphere and lower Thermosphere between an altitude of approximately 60 to 180 +kilometers. + +References +---------- +Larry J. Paxton, Andrew B. Christensen, David C. Humm, Bernard S. Ogorzalek, C. +Thompson Pardoe, Daniel Morrison, Michele B. Weiss, W. Crain, Patricia H. Lew, +Dan J. Mabry, John O. Goldsten, Stephen A. Gary, David F. Persons, Mark J. +Harold, E. Brian Alvarez, Carl J. Ercol, Douglas J. Strickland, and Ching-I. +Meng "Global ultraviolet imager (GUVI): measuring composition and energy inputs +for the NASA Thermosphere Ionosphere Mesosphere Energetics and Dynamics (TIMED) +mission", Proc. SPIE 3756, Optical Spectroscopic Techniques and Instrumentation +for Atmospheric and Space Research III, (20 October 1999); +https://doi.org/10.1117/12.366380 + +Properties +---------- +platform + 'timed' +name + 'guvi' +tag + 'edr-aur' +inst_id + None supported + + +Warnings +-------- +- Currently no cleaning routine. + + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import jhuapl +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import timed as mm_timed + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'timed' +name = 'ssusi' +tags = {'edr-aur': ''.join(['Auroral disk imaging mode'])} +inst_ids = {'': list(tags.keys())} + +pandas_format = False +multi_file_day = True + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {inst_id: {tag: dt.datetime(2003, 1, 1) for tag in tags.keys()} + for inst_id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_timed, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(['TIMED_GUVI_L2B_{tag:s}_IMB_{{year:04d}}{{day:03d}}', + '{{hour:02d}}{{minute:02d}}{{second:02d}}-?????????????-REV', + '??????_Av{{version:2d}}-01r{{cycle:03d}}.nc']) +supported_tags = {inst_id: {tag: fname.format(tag=tag) for tag in tags.keys()} + for inst_id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(jhuapl.load_edr_aurora, pandas_format=pandas_format) + +# Set the download routine +basic_tag = {'remote_dir': ''.join(('/pub/data/dmsp/dmsp{inst_id:s}/ssusi/', + '/data/{tag:s}/{{year:4d}}/{{day:03d}}/')), + 'fname': fname} +download_tags = { + sat_id: {tag: {btag: basic_tag[btag].format(tag=tag, inst_id=sat_id) + for btag in basic_tag.keys()} for tag in tags.keys()} + for sat_id in inst_ids.keys()} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) From e63cc2b90773a30dd3466f0ef5e9c34fd2a85ee2 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:53:04 -0500 Subject: [PATCH 098/396] ENH: added a methods module for TIMED Added a methods module for TIMED, with all the rules URLs, references, and acknowledgements. --- pysatNASA/instruments/methods/timed.py | 33 ++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 pysatNASA/instruments/methods/timed.py diff --git a/pysatNASA/instruments/methods/timed.py b/pysatNASA/instruments/methods/timed.py new file mode 100644 index 00000000..e6d4e2fa --- /dev/null +++ b/pysatNASA/instruments/methods/timed.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for the TIMED data.""" + +rules_url = {'guvi': 'http://guvitimed.jhuapl.edu/home_guvi-datausage', + 'saber': 'https://saber.gats-inc.com/data_services.php', + 'see': 'https://www.timed.jhuapl.edu/WWW/scripts/mdc_rules.pl'} + +ackn_str = "".join(["This Thermosphere Ionosphere Mesosphere Energetics ", + "Dynamics (TIMED) satellite data is provided through ", + "CDAWeb. Please see the Rules of the Road at {:s}"]) + +refs = {'guvi': ''.join(['Larry J. Paxton, Andrew B. Christensen, David C. ', + 'Humm, Bernard S. Ogorzalek, C. Thompson Pardoe, ', + 'Daniel Morrison, Michele B. Weiss, W. Crain, ', + 'Patricia H. Lew, Dan J. Mabry, John O. Goldsten, ', + 'Stephen A. Gary, David F. Persons, Mark J. Harold, ', + 'E. Brian Alvarez, Carl J. Ercol, Douglas J. ', + 'Strickland, and Ching-I. Meng "Global ultraviolet ', + 'imager (GUVI): measuring composition and energy ', + 'inputs for the NASA Thermosphere Ionosphere ', + 'Mesosphere Energetics and Dynamics (TIMED) mission",', + 'Proc. SPIE 3756, Optical Spectroscopic Techniques ', + 'and Instrumentation for Atmospheric and Space ', + 'Research III, (20 October 1999); ', + 'doi:10.1117/12.366380']), + 'saber': '', + 'see': ' '.join(('Woods, T. N., Eparvier, F. G., Bailey,', + 'S. M., Chamberlin, P. C., Lean, J.,', + 'Rottman, G. J., Solomon, S. C., Tobiska,', + 'W. K., and Woodraska, D. L. (2005),', + 'Solar EUV Experiment (SEE): Mission', + 'overview and first results, J. Geophys.', + 'Res., 110, A01312, doi:10.1029/2004JA010765.'))} From 0de411c59a6f9ba6b62bbe939768b7a64404c4e5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:53:33 -0500 Subject: [PATCH 099/396] ENH: updated general init Updated the general init function to allow for rules of the road URLs. --- pysatNASA/instruments/methods/general.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index 60ead57f..013e8c7c 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -23,6 +23,10 @@ def init(self, module, name): # Set acknowledgements self.acknowledgements = getattr(module, 'ackn_str') + + if hasattr(module, 'rules_url'): + self.acknowledgements.format(getattr(module, 'rules_url')[name]) + pysat.logger.info(self.acknowledgements) # Set references From 4af03cda350d58f850e62b5a95792088ab7f68d0 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:54:17 -0500 Subject: [PATCH 100/396] ENH: added a JHU APL methods module Added a JHU APL methods module to hold routines needed for their common data formats. --- pysatNASA/instruments/methods/jhuapl.py | 114 ++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 pysatNASA/instruments/methods/jhuapl.py diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py new file mode 100644 index 00000000..7b4b5e03 --- /dev/null +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +"""Module for data sets created by JHU APL.""" + +import datetime as dt +import numpy as np +import pandas as pds +import xarray as xr + +from pysat.utils.io import load_netcdf + + +def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): + """Load JHU APL EDR Aurora data and meta data. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + tag : str + Tag name used to identify particular data set to be loaded (default='') + inst_id : str + Instrument ID name used to identify different instrument carriers + (default='') + pandas_format : bool + False for xarray format, True for pandas (default=False) + + Returns + ------- + data : pds.DataFrame or xr.Dataset + Data to be assigned to the pysat.Instrument.data object. + mdata : pysat.Meta + Pysat Meta data for each data variable. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'guvi', tag='edr-aur') + inst.load(2003, 1) + + """ + # Define the input variables + labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} + + # CDAWeb stores these files in the NetCDF format instead of the CDF format + single_data = list() + for fname in fnames: + # There are multiple files per day, with time as a variable rather + # than a dimension or coordinate. Additionally, no coordinates + # are assigned. + subday_data, mdata = load_netcdf(fname, epoch_name='TIME', + epoch_unit='s', labels=labels, + pandas_format=pandas_format) + single_data.append(subday_data) + + # After loading all the data, determine which dimensions need to be expanded + combo_dims = {dim: max([sdata.dims[dim] for sdata in single_data]) + for dim in subday_data.dims.keys()} + + # Expand the data so that all dimensions are the same shape + for i, sdata in enumerate(single_data): + # Determine which dimensions need to be updated + fix_dims = [dim for dim in sdata.dims.keys() + if sdata.dims[dim] < combo_dims[dim]] + + new_data = {} + update_new = False + for dvar in sdata.data_vars.keys(): + # See if any dimensions need to be updated + update_dims = list(set(sdata[dvar].dims) & set(fix_dims)) + + # Save the old data as is, or pad it to have the right dims + if len(update_dims) > 0: + update_new = True + new_shape = list(sdata[dvar].values.shape) + old_slice = [slice(0, ns) for ns in new_shape] + + for dim in update_dims: + idim = list(sdata[dvar].dims).index(dim) + new_shape[idim] = combo_dims[dim] + + # Set the new data for output + new_dat = np.full(shape=new_shape, fill_value=mdata[ + dvar, mdata.labels.fill_val]) + new_dat[tuple(old_slice)] = sdata[dvar].values + new_data[dvar] = (sdata[dvar].dims, new_dat) + else: + new_data[dvar] = sdata[dvar] + + # Calculate the time for this data file. The pysat `load_netcdf` routine + # converts the 'TIME' parameter (seconds of day) into datetime using + # the UNIX epoch as the date offset + ftime = dt.datetime.strptime( + "{:4d}-{:03d}".format( + sdata['YEAR'].values.astype(int), + sdata['DOY'].values.astype(int)), '%Y-%j') + ( + pds.to_datetime(sdata['time'].values).to_pydatetime() + - dt.datetime(1970, 1, 1)) + + # Get the updated dataset + ndata = xr.Dataset(new_data) if update_new else sdata + ndata['time'] = ftime + + # Assign a datetime variable, making indexing possible + single_data[i] = ndata.assign_coords( + {'time': ndata['time']}).expand_dims(dim='time') + + # Combine all the data, indexing along time + data = xr.combine_by_coords(single_data) + + # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by + # removing 'TIME', once possible + + return data, mdata From 983a41b129a8d47a0820ca727fad6685d75a6144 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:55:02 -0500 Subject: [PATCH 101/396] ENH: updated TIMED SEE/SABER Updated the SEE and SABER instruments to use the new timed methods module. --- pysatNASA/instruments/timed_saber.py | 21 ++------------------- pysatNASA/instruments/timed_see.py | 27 ++------------------------- 2 files changed, 4 insertions(+), 44 deletions(-) diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index 38f82b91..c2858b3c 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -50,10 +50,10 @@ # CDAWeb methods prewritten for pysat from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import timed as mm_timed # ---------------------------------------------------------------------------- # Instrument attributes @@ -78,28 +78,11 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - rules_url = 'https://saber.gats-inc.com/data_services.php' - ackn_str = ' '.join(('Please see the Rules of the Road at', rules_url)) - - logger.info(ackn_str) - self.acknowledgements = ackn_str - self.references = '' - - return - +init = functools.partial(mm_nasa.init, module=mm_timed, name=name) # No cleaning, use standard warning function instead clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 903058ab..32080254 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -42,10 +42,10 @@ import pandas as pds from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import timed as mm_timed # ---------------------------------------------------------------------------- # Instrument attributes @@ -63,34 +63,11 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - rules_url = 'https://www.timed.jhuapl.edu/WWW/scripts/mdc_rules.pl' - ackn_str = ' '.join(('Please see the Rules of the Road at', rules_url)) - logger.info(ackn_str) - self.acknowledgements = ackn_str - self.references = ' '.join(('Woods, T. N., Eparvier, F. G., Bailey,', - 'S. M., Chamberlin, P. C., Lean, J.,', - 'Rottman, G. J., Solomon, S. C., Tobiska,', - 'W. K., and Woodraska, D. L. (2005),', - 'Solar EUV Experiment (SEE): Mission', - 'overview and first results, J. Geophys.', - 'Res., 110, A01312,', - 'doi:10.1029/2004JA010765.')) - - return - +init = functools.partial(mm_nasa.init, module=mm_timed, name=name) # No cleaning, use standard warning function instead clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # From 3277e735b3259d2173174aa9702b94e6a5ff350e Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:56:15 -0500 Subject: [PATCH 102/396] STY: renamed gold to ses14 Renamed the gold module ses14 to maintain a consistent nomenclature for the platform modules. --- pysatNASA/instruments/methods/{gold.py => ses14.py} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename pysatNASA/instruments/methods/{gold.py => ses14.py} (76%) diff --git a/pysatNASA/instruments/methods/gold.py b/pysatNASA/instruments/methods/ses14.py similarity index 76% rename from pysatNASA/instruments/methods/gold.py rename to pysatNASA/instruments/methods/ses14.py index 9ac51312..4b12786d 100644 --- a/pysatNASA/instruments/methods/gold.py +++ b/pysatNASA/instruments/methods/ses14.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Provides non-instrument specific routines for GOLD data.""" +"""Provides non-instrument specific routines for SES14 instruments.""" ack_str = ' '.join(('This is a data product from the NASA Global-scale', 'Observations of the Limb and Disk (GOLD) mission, an', @@ -18,7 +18,7 @@ 'Investigator Dr. Richard Eastes and the party', 'directly responsible for the data product and the', 'NASA Explorers Project Office.')) -ref_str = ' '.join(('Eastes, R.W., McClintock, W.E., Burns, A.G. et al.', - 'The Global-Scale Observations of the Limb and Disk', - '(GOLD) Mission. Space Sci Rev 212, 383–408 (2017).', - 'https://doi.org/10.1007/s11214-017-0392-2')) +refs = {'gold': ' '.join(('Eastes, R.W., McClintock, W.E., Burns, A.G. et', + 'al., The Global-Scale Observations of the Limb', + 'and Disk (GOLD) Mission. Space Sci Rev 212,', + '383–408 (2017). doi:10.1007/s11214-017-0392-2'))} From c1db73b218b0aaf2dc827ff23902d110f501eebc Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:57:02 -0500 Subject: [PATCH 103/396] ENH: use common JHU APL load function Update DMSP SSUSI to use the common JHU APL loading function for imager data. --- pysatNASA/instruments/dmsp_ssusi.py | 107 +--------------------------- 1 file changed, 2 insertions(+), 105 deletions(-) diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index a16acd69..b3d8e295 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -50,15 +50,12 @@ import datetime as dt import functools -import numpy as np -import pandas as pds -import xarray as xr from pysat.instruments.methods import general as mm_gen -from pysat.utils.io import load_netcdf from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import dmsp as mm_dmsp +from pysatNASA.instruments.methods import jhuapl from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- @@ -107,108 +104,8 @@ list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) - # Set the load routine -def load(fnames, tag='', inst_id=''): - """Load DMSP SSUSI data and meta data. - - Parameters - ---------- - fnames : array-like - Iterable of filename strings, full path, to data files to be loaded. - tag : str - Tag name used to identify particular data set to be loaded (default='') - inst_id : str - DMSP satellite ID (default='') - - Returns - ------- - data : pds.DataFrame or xr.Dataset - Data to be assigned to the pysat.Instrument.data object. - mdata : pysat.Meta - Pysat Meta data for each data variable. - - Examples - -------- - :: - - inst = pysat.Instrument('dmsp', 'ssusi', tag='edr-aurora', - inst_id='f16') - inst.load(2006, 1) - - """ - # Define the input variables - labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} - - # CDAWeb stores these files in the NetCDF format instead of the CDF format - single_data = list() - for fname in fnames: - # There are multiple files per day, with time as a variable rather - # than a dimension or coordinate. Additionally, no coordinates - # are assigned. - subday_data, mdata = load_netcdf(fname, epoch_name='TIME', - epoch_unit='s', labels=labels, - pandas_format=pandas_format) - single_data.append(subday_data) - - # After loading all the data, determine which dimensions need to be expanded - combo_dims = {dim: max([sdata.dims[dim] for sdata in single_data]) - for dim in subday_data.dims.keys()} - - # Expand the data so that all dimensions are the same shape - for i, sdata in enumerate(single_data): - # Determine which dimensions need to be updated - fix_dims = [dim for dim in sdata.dims.keys() - if sdata.dims[dim] < combo_dims[dim]] - - new_data = {} - update_new = False - for dvar in sdata.data_vars.keys(): - # See if any dimensions need to be updated - update_dims = list(set(sdata[dvar].dims) & set(fix_dims)) - - # Save the old data as is, or pad it to have the right dims - if len(update_dims) > 0: - update_new = True - new_shape = list(sdata[dvar].values.shape) - old_slice = [slice(0, ns) for ns in new_shape] - - for dim in update_dims: - idim = list(sdata[dvar].dims).index(dim) - new_shape[idim] = combo_dims[dim] - - # Set the new data for output - new_dat = np.full(shape=new_shape, fill_value=mdata[ - dvar, mdata.labels.fill_val]) - new_dat[tuple(old_slice)] = sdata[dvar].values - new_data[dvar] = (sdata[dvar].dims, new_dat) - else: - new_data[dvar] = sdata[dvar] - - # Calculate the time for this data file - ftime = dt.datetime.strptime( - "{:4d}-{:03d}".format( - sdata['YEAR'].values.astype(int), - sdata['DOY'].values.astype(int)), '%Y-%j') + ( - pds.to_datetime(sdata['time'].values).to_pydatetime() - - dt.datetime(1970, 1, 1)) - - # Get the updated dataset - ndata = xr.Dataset(new_data) if update_new else sdata - ndata['time'] = ftime - - # Assign a datetime variable, making indexing possible - single_data[i] = ndata.assign_coords( - {'time': ndata['time']}).expand_dims(dim='time') - - # Combine all the data, indexing along time - data = xr.combine_by_coords(single_data) - - # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by - # removing 'TIME', once possible - - return data, mdata - +load = functools.partial(jhuapl.load_edr_aurora, pandas_format=pandas_format) # Set the download routine basic_tag = {'remote_dir': ''.join(('/pub/data/dmsp/dmsp{inst_id:s}/ssusi/', From 1d5ace0e8021f0c1e6621b24f6fceba524afe48b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:57:43 -0500 Subject: [PATCH 104/396] BUG: added new and missing imports Added the new and missing module imports. --- pysatNASA/instruments/methods/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysatNASA/instruments/methods/__init__.py b/pysatNASA/instruments/methods/__init__.py index 7c8a3c93..8542fcbc 100644 --- a/pysatNASA/instruments/methods/__init__.py +++ b/pysatNASA/instruments/methods/__init__.py @@ -6,5 +6,9 @@ from pysatNASA.instruments.methods import de2 # noqa F401 from pysatNASA.instruments.methods import dmsp # noqa F401 from pysatNASA.instruments.methods import general # noqa F401 +from pysatNASA.instruments.methods import gps # noqa F401 from pysatNASA.instruments.methods import icon # noqa F401 +from pysatNASA.instruments.methods import jhuapl # noqa F401 from pysatNASA.instruments.methods import omni # noqa F401 +from pysatNASA.instruments.methods import ses14 # noqa F401 +from pysatNASA.instruments.methods import timed # noqa F401 From ade00fbd68ae9d05f9b2d95b2b60f98ae6fed6a6 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:58:08 -0500 Subject: [PATCH 105/396] ENH: use general init function Updated GOLD to use the common init function. --- pysatNASA/instruments/ses14_gold.py | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 5a89413f..b021b21b 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -36,7 +36,6 @@ import numpy as np from pysat.instruments.methods import general as ps_gen -from pysat import logger from pysat.utils.io import load_netcdf from pysatNASA.instruments.methods import cdaweb as cdw @@ -61,35 +60,11 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - Parameters - ----------- - self : pysat.Instrument - Instrument class object - - """ - - logger.info(mm_gold.ack_str) - logger.warning(' '.join(('Time stamps may be non-unique because Channel A', - 'and B are different instruments. An upgrade to', - 'the pysat.Constellation object is required to', - 'solve this issue. See pysat issue #614 for more', - 'info.'))) - self.acknowledgements = mm_gold.ack_str - self.references = mm_gold.ref_str - - return - +init = functools.partial(mm_nasa.init, module=mm_gold, name=name) # No cleaning, use standard warning function instead clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # From eb76501be29a4e49b76e8421cbe7673039149796 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:58:45 -0500 Subject: [PATCH 106/396] TST: generalized the platform methods tests Generalized the platform methods test class and applied it to all existing platform modules. --- pysatNASA/tests/test_methods_dmsp.py | 32 ------ pysatNASA/tests/test_methods_platform.py | 129 +++++++++++++++++++++++ 2 files changed, 129 insertions(+), 32 deletions(-) delete mode 100644 pysatNASA/tests/test_methods_dmsp.py create mode 100644 pysatNASA/tests/test_methods_platform.py diff --git a/pysatNASA/tests/test_methods_dmsp.py b/pysatNASA/tests/test_methods_dmsp.py deleted file mode 100644 index 1bef74a4..00000000 --- a/pysatNASA/tests/test_methods_dmsp.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Unit tests for the DMSP methods.""" - -from pysatNASA.instruments.methods import dmsp - - -class TestDMSPMethods(object): - """Unit tests for `pysat.instruments.methods.dmsp`.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - self.names = ['ssusi'] - return - - def teardown_method(self): - """Clean up the unit test environment after each method.""" - - del self.names - return - - def test_ack(self): - """Test that the acknowledgements reference the correct platform.""" - - assert dmsp.ackn_str.find( - 'Defense Meteorological Satellite Program') >= 0 - return - - def test_ref(self): - """Test that all DMSP instruments have references.""" - - for name in self.names: - assert name in dmsp.refs.keys(), "No reference for {:}".format(name) - return diff --git a/pysatNASA/tests/test_methods_platform.py b/pysatNASA/tests/test_methods_platform.py new file mode 100644 index 00000000..a1cd6611 --- /dev/null +++ b/pysatNASA/tests/test_methods_platform.py @@ -0,0 +1,129 @@ +"""Unit tests for the common NASA platform method attributes.""" + + +from pysatNASA.instruments import methods + + +class TestTIMEDMethods(object): + """Unit tests for `pysat.instruments.methods.timed`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['see', 'saber', 'guvi'] + self.module = methods.timed + self.platform_str = '(TIMED)' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + def test_ack(self): + """Test that the acknowledgements reference the correct platform.""" + + assert self.module.ackn_str.find(self.platform_str) >= 0 + return + + def test_rules(self): + """Test that the rules of the road exist for each instrument.""" + + if hasattr(self.module, "rules_url"): + for name in self.names: + assert name in self.module.rules_url.keys( + ), "No rules URL for {:}".format(name) + return + + def test_ref(self): + """Test that all instruments have references.""" + + for name in self.names: + assert name in self.module.refs.keys( + ), "No reference for {:}".format(name) + return + + +class TestDMSPMethods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.dmsp`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['ssusi'] + self.module = methods.dmsp + self.platform_str = '(DMSP)' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestCNOFSMethods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.cnofs`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['ivm', 'plp', 'vefi'] + self.module = methods.cnofs + self.platform_str = '(C/NOFS)' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestDE2Methods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.de2`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['lang', 'nacs', 'rpa', 'wats'] + self.module = methods.de2 + self.platform_str = 'Dynamics Explorer 2' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestSES14Methods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.ses14`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['gold'] + self.module = methods.ses14 + self.platform_str = 'Global-scale Observations of the Limb and Disk' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestGPSMethods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.gps`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['roti15min_jpl'] + self.module = methods.gps + self.platform_str = 'GPS Total Electron Content' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return From d8471a7d9588507ef9b4e3372afbff63bd44c869 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:59:03 -0500 Subject: [PATCH 107/396] DOC: updated supported instruments Added TIMED GUVI to the documentation. --- docs/supported_instruments.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 10d3962a..f359c5e6 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -138,6 +138,14 @@ SES14 GOLD .. automodule:: pysatNASA.instruments.ses14_gold :members: +.. _timed_guvi: + +TIMED GUVI +---------- + +.. automodule:: pysatNASA.instruments.timed_guvi + :members: + .. _timed_saber: TIMED SABER From 6a92fcd79bf3920e839356d8409d9338559dc3f3 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 18:59:19 -0500 Subject: [PATCH 108/396] DOC: updated changelog Added a summary of the changes to the log. --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5335347..d31f235d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,17 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [0.0.5] - 2023-XX-XX * New Instruments * DMSP SSUSI EDR-Aurora data + * TIMED GUVI EDR-Aur data * Bug Fixes * Updated CDAWeb routines to allow for data stored by year/day-of-year + * Added missing sub-module imports +* Enhancements + * Updated platform methods to follow a consistent style and work with the + general `init` function + * Added unit tests for the different platform method attributes * Maintenance - * Added a version cap for numpy (required for cdf interface, revisit before release) + * Added a version cap for numpy (required for cdf interface, revisit before + release) ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From cef15dda7af2ec0b048f81c5d582e4d754f29b68 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 19:16:30 -0500 Subject: [PATCH 109/396] BUG: fixed remote dir and filename Fixed the formatting of the remote directory, filename, and instrument `name` attribute. --- pysatNASA/instruments/timed_guvi.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 6dda4c95..f745ff8a 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -59,7 +59,7 @@ # Instrument attributes platform = 'timed' -name = 'ssusi' +name = 'guvi' tags = {'edr-aur': ''.join(['Auroral disk imaging mode'])} inst_ids = {'': list(tags.keys())} @@ -88,9 +88,9 @@ # Use the default CDAWeb and pysat methods # Set the list_files routine -fname = ''.join(['TIMED_GUVI_L2B_{tag:s}_IMB_{{year:04d}}{{day:03d}}', - '{{hour:02d}}{{minute:02d}}{{second:02d}}-?????????????-REV', - '??????_Av{{version:2d}}-01r{{cycle:03d}}.nc']) +fname = ''.join(['TIMED_GUVI_L2B-{tag:s}-IMG_{{year:04d}}{{day:03d}}', + '{{hour:02d}}{{minute:02d}}{{second:02d}}-?????????????_REV', + '??????_Av{{version:2d}}-??r{{cycle:03d}}.nc']) supported_tags = {inst_id: {tag: fname.format(tag=tag) for tag in tags.keys()} for inst_id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -99,8 +99,9 @@ load = functools.partial(jhuapl.load_edr_aurora, pandas_format=pandas_format) # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/dmsp/dmsp{inst_id:s}/ssusi/', - '/data/{tag:s}/{{year:4d}}/{{day:03d}}/')), +basic_tag = {'remote_dir': ''.join(('/data/timed/guvi/levels_v13/level2b/', + 'imaging/{tag:s}/{{year:4d}}/', + '{{day:03d}}/')), 'fname': fname} download_tags = { sat_id: {tag: {btag: basic_tag[btag].format(tag=tag, inst_id=sat_id) From 7a881e673bc10c057c86b55aee7cc77613dc4acb Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 19:21:46 -0500 Subject: [PATCH 110/396] STY: fixed import order Fixed the import order to be alphabetical. --- pysatNASA/instruments/dmsp_ssusi.py | 2 +- pysatNASA/instruments/timed_guvi.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index b3d8e295..a0bee192 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -55,8 +55,8 @@ from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import dmsp as mm_dmsp -from pysatNASA.instruments.methods import jhuapl from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import jhuapl # ---------------------------------------------------------------------------- # Instrument attributes diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index f745ff8a..5c376076 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -51,8 +51,8 @@ from pysat.instruments.methods import general as mm_gen from pysatNASA.instruments.methods import cdaweb as cdw -from pysatNASA.instruments.methods import jhuapl from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import jhuapl from pysatNASA.instruments.methods import timed as mm_timed # ---------------------------------------------------------------------------- From 5d55a3039be70adde97f86bacb05b9b7aa07d9d4 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 19:41:30 -0500 Subject: [PATCH 111/396] BUG: updated gold methods import Updated the GOLD methods import to SES14 in the GOLD instrument file. --- pysatNASA/instruments/ses14_gold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index b021b21b..0337bfa5 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -40,7 +40,7 @@ from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa -from pysatNASA.instruments.methods import gold as mm_gold +from pysatNASA.instruments.methods import ses14 as mm_gold # ---------------------------------------------------------------------------- # Instrument attributes From 083a4b0658131e3b6d250bcd22a6c249faf9be01 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 24 Feb 2023 19:48:46 -0500 Subject: [PATCH 112/396] BUG: updated SES14 attrs Updated the SES14 attribute names to conform with the new standards. --- pysatNASA/instruments/methods/ses14.py | 34 +++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pysatNASA/instruments/methods/ses14.py b/pysatNASA/instruments/methods/ses14.py index 4b12786d..5307d592 100644 --- a/pysatNASA/instruments/methods/ses14.py +++ b/pysatNASA/instruments/methods/ses14.py @@ -1,23 +1,23 @@ # -*- coding: utf-8 -*- """Provides non-instrument specific routines for SES14 instruments.""" -ack_str = ' '.join(('This is a data product from the NASA Global-scale', - 'Observations of the Limb and Disk (GOLD) mission, an', - 'Heliophysics Explorer mission of opportunity launched', - 'in January 2018.\n Responsibility of the mission', - 'science falls to the Principal Investigator, Dr.', - 'Richard Eastes at University of Colorado/LASP.\n', - 'Validation of the L1B data products falls to the', - 'instrument lead investigators/scientists.\n* EUV', - 'Dr. Bill McClintock\nValidation of the L2 data', - 'products falls to Computational Physics, Inc.\n* Dr.', - 'Jerry Lumpe\n (https://gold.cs.ucf.edu/).\nOverall', - 'validation of the products is overseen by the GOLD', - 'Project Scientist Dr. Alan Burns.\nUsers of these', - 'data should contact and acknowledge the Principal', - 'Investigator Dr. Richard Eastes and the party', - 'directly responsible for the data product and the', - 'NASA Explorers Project Office.')) +ackn_str = ' '.join(('This is a data product from the NASA Global-scale', + 'Observations of the Limb and Disk (GOLD) mission, an', + 'Heliophysics Explorer mission of opportunity launched', + 'in January 2018.\n Responsibility of the mission', + 'science falls to the Principal Investigator, Dr.', + 'Richard Eastes at University of Colorado/LASP.\n', + 'Validation of the L1B data products falls to the', + 'instrument lead investigators/scientists.\n* EUV', + 'Dr. Bill McClintock\nValidation of the L2 data', + 'products falls to Computational Physics, Inc.\n* Dr.', + 'Jerry Lumpe\n (https://gold.cs.ucf.edu/).\nOverall', + 'validation of the products is overseen by the GOLD', + 'Project Scientist Dr. Alan Burns.\nUsers of these', + 'data should contact and acknowledge the Principal', + 'Investigator Dr. Richard Eastes and the party', + 'directly responsible for the data product and the', + 'NASA Explorers Project Office.')) refs = {'gold': ' '.join(('Eastes, R.W., McClintock, W.E., Burns, A.G. et', 'al., The Global-Scale Observations of the Limb', 'and Disk (GOLD) Mission. Space Sci Rev 212,', From 1abf73a1f0cf8fdd2d4494e74156ca3d6e7a37f7 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 27 Feb 2023 12:55:18 -0500 Subject: [PATCH 113/396] BUG: fixed spaces Removed the extra space when joining. --- pysatNASA/instruments/methods/dmsp.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pysatNASA/instruments/methods/dmsp.py b/pysatNASA/instruments/methods/dmsp.py index b38392db..3f930c0b 100644 --- a/pysatNASA/instruments/methods/dmsp.py +++ b/pysatNASA/instruments/methods/dmsp.py @@ -4,13 +4,13 @@ ackn_str = "".join(["This Defense Meteorological Satellite Program (DMSP) ", "satellite data is provided through CDAWeb"]) -refs = {'ssusi': ' '.join(('Larry J. Paxton, Daniel Morrison, Yongliang Zhang,', - ' Hyosub Kil, Brian Wolven, Bernard S. Ogorzalek, ', - 'David C. Humm, and Ching-I. Meng "Validation of ', - 'remote sensing products produced by the Special ', - 'Sensor Ultraviolet Scanning Imager (SSUSI): a far ', - 'UV-imaging spectrograph on DMSP F-16", Proc. SPIE ', - '4485, Optical Spectroscopic Techniques, Remote ', - 'Sensing, and Instrumentation for Atmospheric and ', - 'Space Research IV, (30 January 2002); ', - 'doi:10.1117/12.454268'))} +refs = {'ssusi': ''.join(('Larry J. Paxton, Daniel Morrison, Yongliang Zhang,', + ' Hyosub Kil, Brian Wolven, Bernard S. Ogorzalek, ', + 'David C. Humm, and Ching-I. Meng "Validation of ', + 'remote sensing products produced by the Special ', + 'Sensor Ultraviolet Scanning Imager (SSUSI): a far ', + 'UV-imaging spectrograph on DMSP F-16", Proc. SPIE ', + '4485, Optical Spectroscopic Techniques, Remote ', + 'Sensing, and Instrumentation for Atmospheric and ', + 'Space Research IV, (30 January 2002); ', + 'doi:10.1117/12.454268'))} From d682efd2fa1b812e9bf97755773da26604084903 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 27 Feb 2023 14:55:59 -0500 Subject: [PATCH 114/396] BUG: remove test.nc --- test.nc | Bin 11313 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test.nc diff --git a/test.nc b/test.nc deleted file mode 100644 index 66b36379d3516860a2ffd2f3d97409d794debfda..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11313 zcmeHNZ;Tto6@P2rrI+Ncxuhk9YXU<;%O&v{`~H+9ssNwYK09)J)~!iMB;qXg-entm z*IDl!2_G8TL_{T(`hf~nm7x4lRiIU+RDi1H3lgmgmC6Sy0;p2|q`IUZXh9VP+C-u6 z&FnayQ)sFKKBU>X^Ulu9Z{Ezj_vX#)oB38w&-N`FSOye&dY}vX`Kx?<7=&a@(K&jcl5bTx3`AJw@xvQxPd^x$YCooXy~Y*WC7p9C7jHH`AwIwSxq z0^H>ytk$k6^m-7LH#sQl9o_ZvzsAcs9ead2yb`l(gBOCfSAcA#VSLcXQA-KcCs5sj z#zoW}>rVeVYalI2z(ZP{bb*6crVBx`J8{|rw>3Qs6xi4c;D2wp_dR-IO{Pwn;7~r9HR^zFtOur zTgHD5ho~qa*c5&G?w8>hrr|Cz*1sPpyb3>~D(eB3V085Pf#1Q4)XH9fVF>iT|+={#rmY_)27Zj8um!@6!AiwgKcRJ{gKHJutW;><7|(>EPE z#>)FF=2^b!)of>mHC^9weA}!e(5xT2&+^!`>#^D)$86Zu7PAgE>#k>cL*p@KG|j5T zjH(@Hd8g%jvyC_#i?Ol9$m#nZ8cQbFU~{SZjtZiYh$KZ=0!Rn{DmXqs5P-FT zXEH54lxNh=2vqp|f8@ZPZQuF9e|O$LOa5C7w$u)JP9-!_Y9_1_u**I#?YuvCv#6-S z{EuGv>q6)K$)N33?N*Y1Ub%Xq^Zw*oLE|Fowo^yuh4cPT?PYI`2;f z&igL~02$BSa(sW_ygy2`&=XIt`NC7IA6C<$L`fVty!+q{H^F0=hAqOazIpzsjW9sH zKm;8@ytVFyq;LSETb;r$&p!NB7^bO*2=byo@$PdI@U@^yKW3b`?t}r|)a;<}_mAA3 z-vwDr!+XM#WgI-rF96SBdC|kkpddoFD{d(80-fAAau)oT3aoCm~1OjT_gz z4;S!-`q7Qii9pHK(7hpyFdHhvyY(JNWIUf*HXXD!k7mF)KR;y=0^MnB{f zak>Xv2Dw(t_){2gNHXL=$bnCc1M2)IvIm+kU8CWe--CbYLdq~OkM{4h=*xMn=eK8y zMt-tbG193*2J^c)zL?scK^Vu=u0i6(tsD)P<6gv7ik4rkO^@n!!>%EUJ}J0C0*0xJ zwZ!{UPC1*;6m%rd0r;RE(PawRLfLLutc9nJ7E9pqq#mc6KQx5wRIM{254-c{m#M+~P|8@#i;LBpLL zNulmKGZj3~4Hg?two_zWizH144}>MnOMrwv)~r?)+vAjO>S(xV7?5A8`xJdm2C@F&GjC;YlRz|RB+ z_-vT9Oy8>4ZQo)v-6uXeg1y|8jC{>)6QF z;nDc5$t~>8a+;(}&2p*+UI_6QLp3qBbz~R`Bh?W>wVO|G+~IP2b2*)%4@wF$=3@zT zhIq}tCWAC#q){XK73w-w#_(}@Ayi|+axBjzWfQcCDm5-SJe(LFP9_t#N>wVjGj`Ri X(|+En Date: Mon, 27 Feb 2023 15:12:28 -0500 Subject: [PATCH 115/396] STY: generalize and consistency --- pysatNASA/instruments/ace_epam_l2.py | 7 +++---- pysatNASA/instruments/ace_mag_l2.py | 7 +++---- pysatNASA/instruments/ace_sis_l2.py | 6 ++---- pysatNASA/instruments/ace_swepam_l2.py | 6 ++---- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index 2125f486..f036a8e1 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -47,7 +47,8 @@ platform = 'ace' name = 'epam_l2' tags = {'': ''} -inst_ids = {id: [''] for id in ['h1', 'h2', 'h3', 'k0', 'k1']} +inst_ids = {id: [tag for tag in tags.keys()] + for id in ['h1', 'h2', 'h3', 'k0', 'k1']} pandas_format = False # ---------------------------------------------------------------------------- @@ -74,9 +75,7 @@ # Set the list_files routine fname = ''.join(('ac_{inst_id:s}_epm_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {} -for inst_id in inst_ids.keys(): - supported_tags[inst_id] = {'': fname.format(inst_id=inst_id)} +supported_tags = {id: {'': fname.format(inst_id=id)} for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index 90047aae..c3b91624 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -47,7 +47,8 @@ platform = 'ace' name = 'mag_l2' tags = {'': ''} -inst_ids = {id: [''] for id in ['h0', 'h1', 'h2', 'h3', 'k0', 'k1', 'k2']} +inst_ids = {id: [tag for tag in tags.keys()] + for id in ['h0', 'h1', 'h2', 'h3', 'k0', 'k1', 'k2']} pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -73,9 +74,7 @@ # Set the list_files routine fname = ''.join(('ac_{inst_id:s}_mfi_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {} -for inst_id in inst_ids.keys(): - supported_tags[inst_id] = {'': fname.format(inst_id=inst_id)} +supported_tags = {id: {'': fname.format(inst_id=id)} for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index 71d58c4c..9e5ace3e 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -45,7 +45,7 @@ platform = 'ace' name = 'sis_l2' tags = {'': ''} -inst_ids = {id: [''] for id in ['h1', 'h2', 'k0']} +inst_ids = {id: [tag for tag in tags.keys()] for id in ['h1', 'h2', 'k0']} pandas_format = False # ---------------------------------------------------------------------------- @@ -72,9 +72,7 @@ # Set the list_files routine fname = ''.join(('ac_{inst_id:s}_sis_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {} -for inst_id in inst_ids.keys(): - supported_tags[inst_id] = {'': fname.format(inst_id=inst_id)} +supported_tags = {id: {'': fname.format(inst_id=id)} for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index 4e018e62..11a3d661 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -47,7 +47,7 @@ platform = 'ace' name = 'swepam_l2' tags = {'': ''} -inst_ids = {id: [''] for id in ['h0', 'h2', 'k0', 'k1']} +inst_ids = {id: [tag for tag in tags.keys()] for id in ['h0', 'h2', 'k0', 'k1']} pandas_format = False # ---------------------------------------------------------------------------- @@ -74,9 +74,7 @@ # Set the list_files routine fname = ''.join(('ac_{inst_id:s}_swe_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {} -for inst_id in inst_ids.keys(): - supported_tags[inst_id] = {'': fname.format(inst_id=inst_id)} +supported_tags = {id: {'': fname.format(inst_id=id)} for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) From 13154bd04f0a654956f1885ec1e1888357dffcc6 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 27 Feb 2023 22:26:41 -0500 Subject: [PATCH 116/396] DOC: updated docstring Added a note about expected warnings to the docstring. --- pysatNASA/instruments/methods/jhuapl.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 7b4b5e03..e12bdade 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -31,6 +31,11 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): mdata : pysat.Meta Pysat Meta data for each data variable. + Note + ---- + Logger warning 'Epoch label: TIME is not a dimension.' is raised due to + the data format and pysat file expectations. + Examples -------- :: From 8d63b05016172dd5a4a00f446c93bd565001f35e Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 27 Feb 2023 22:31:54 -0500 Subject: [PATCH 117/396] BUG: fixed spaces Removed the extra space when joining. --- pysatNASA/instruments/methods/dmsp.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pysatNASA/instruments/methods/dmsp.py b/pysatNASA/instruments/methods/dmsp.py index b38392db..3f930c0b 100644 --- a/pysatNASA/instruments/methods/dmsp.py +++ b/pysatNASA/instruments/methods/dmsp.py @@ -4,13 +4,13 @@ ackn_str = "".join(["This Defense Meteorological Satellite Program (DMSP) ", "satellite data is provided through CDAWeb"]) -refs = {'ssusi': ' '.join(('Larry J. Paxton, Daniel Morrison, Yongliang Zhang,', - ' Hyosub Kil, Brian Wolven, Bernard S. Ogorzalek, ', - 'David C. Humm, and Ching-I. Meng "Validation of ', - 'remote sensing products produced by the Special ', - 'Sensor Ultraviolet Scanning Imager (SSUSI): a far ', - 'UV-imaging spectrograph on DMSP F-16", Proc. SPIE ', - '4485, Optical Spectroscopic Techniques, Remote ', - 'Sensing, and Instrumentation for Atmospheric and ', - 'Space Research IV, (30 January 2002); ', - 'doi:10.1117/12.454268'))} +refs = {'ssusi': ''.join(('Larry J. Paxton, Daniel Morrison, Yongliang Zhang,', + ' Hyosub Kil, Brian Wolven, Bernard S. Ogorzalek, ', + 'David C. Humm, and Ching-I. Meng "Validation of ', + 'remote sensing products produced by the Special ', + 'Sensor Ultraviolet Scanning Imager (SSUSI): a far ', + 'UV-imaging spectrograph on DMSP F-16", Proc. SPIE ', + '4485, Optical Spectroscopic Techniques, Remote ', + 'Sensing, and Instrumentation for Atmospheric and ', + 'Space Research IV, (30 January 2002); ', + 'doi:10.1117/12.454268'))} From 1df5f8ae116c7133a001f2a8da020aab8469a9a2 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 28 Feb 2023 10:24:54 -0500 Subject: [PATCH 118/396] STY: loops --- pysatNASA/instruments/ace_epam_l2.py | 12 ++++++------ pysatNASA/instruments/ace_mag_l2.py | 12 ++++++------ pysatNASA/instruments/ace_sis_l2.py | 12 ++++++------ pysatNASA/instruments/ace_swepam_l2.py | 12 ++++++------ 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index f036a8e1..5b269544 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -75,7 +75,8 @@ # Set the list_files routine fname = ''.join(('ac_{inst_id:s}_epm_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {id: {'': fname.format(inst_id=id)} for id in inst_ids.keys()} +supported_tags = {id: {tag: fname.format(inst_id=id) for tag in tags.keys()} + for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -84,11 +85,10 @@ # Set the download routine remote_dir = '/pub/data/ace/epam/level_2_cdaweb/epm_{inst_id:s}/{{year:4d}}/' -download_tags = {} -for inst_id in inst_ids.keys(): - download_tags[inst_id] = {'': { - 'remote_dir': remote_dir.format(inst_id=inst_id), - 'fname': fname.format(inst_id=inst_id)}} +download_tags = {id: {tag: {'remote_dir': remote_dir.format(inst_id=id), + 'fname': fname.format(inst_id=id)} + for tag in tags.keys()} + for id in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) # Set the list_remote_files routine diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index c3b91624..6595fb3c 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -74,7 +74,8 @@ # Set the list_files routine fname = ''.join(('ac_{inst_id:s}_mfi_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {id: {'': fname.format(inst_id=id)} for id in inst_ids.keys()} +supported_tags = {id: {tag: fname.format(inst_id=id) for tag in tags.keys()} + for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -83,11 +84,10 @@ # Set the download routine remote_dir = '/pub/data/ace/mag/level_2_cdaweb/mfi_{inst_id:s}/{{year:4d}}/' -download_tags = {} -for inst_id in inst_ids.keys(): - download_tags[inst_id] = {'': { - 'remote_dir': remote_dir.format(inst_id=inst_id), - 'fname': fname.format(inst_id=inst_id)}} +download_tags = {id: {tag: {'remote_dir': remote_dir.format(inst_id=id), + 'fname': fname.format(inst_id=id)} + for tag in tags.keys()} + for id in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) # Set the list_remote_files routine diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index 9e5ace3e..b8972068 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -72,7 +72,8 @@ # Set the list_files routine fname = ''.join(('ac_{inst_id:s}_sis_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {id: {'': fname.format(inst_id=id)} for id in inst_ids.keys()} +supported_tags = {id: {tag: fname.format(inst_id=id) for tag in tags.keys()} + for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -81,11 +82,10 @@ # Set the download routine remote_dir = '/pub/data/ace/sis/level_2_cdaweb/sis_{inst_id:s}/{{year:4d}}/' -download_tags = {} -for inst_id in inst_ids.keys(): - download_tags[inst_id] = {'': { - 'remote_dir': remote_dir.format(inst_id=inst_id), - 'fname': fname.format(inst_id=inst_id)}} +download_tags = {id: {tag: {'remote_dir': remote_dir.format(inst_id=id), + 'fname': fname.format(inst_id=id)} + for tag in tags.keys()} + for id in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) # Set the list_remote_files routine diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index 11a3d661..308e156f 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -74,7 +74,8 @@ # Set the list_files routine fname = ''.join(('ac_{inst_id:s}_swe_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {id: {'': fname.format(inst_id=id)} for id in inst_ids.keys()} +supported_tags = {id: {tag: fname.format(inst_id=id) for tag in tags.keys()} + for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -83,11 +84,10 @@ # Set the download routine remote_dir = '/pub/data/ace/swepam/level_2_cdaweb/swe_{inst_id:s}/{{year:4d}}/' -download_tags = {} -for inst_id in inst_ids.keys(): - download_tags[inst_id] = {'': { - 'remote_dir': remote_dir.format(inst_id=inst_id), - 'fname': fname.format(inst_id=inst_id)}} +download_tags = {id: {tag: {'remote_dir': remote_dir.format(inst_id=id), + 'fname': fname.format(inst_id=id)} + for tag in tags.keys()} + for id in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) # Set the list_remote_files routine From d22aaa3e70eca4cba2c9c8a323191b8cb3ada4d9 Mon Sep 17 00:00:00 2001 From: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 1 Mar 2023 14:17:45 -0500 Subject: [PATCH 119/396] fixed fuv_day instrument name in download tags Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/icon_fuv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py index 964399d0..921001bf 100644 --- a/pysatNASA/instruments/icon_fuv.py +++ b/pysatNASA/instruments/icon_fuv.py @@ -124,7 +124,7 @@ def clean(self): # basic_tag25 = {'remote_dir': '/pub/data/icon/l2/l2-5_fuv_night/{year:04d}/', # 'fname': fname25} # download_tags = {'': {'day': basic_tag24, 'night': basic_tag25}} -download_tags = {'': {'day': 'ICON_L2-5_FUV_DAY', +download_tags = {'': {'day': 'ICON_L2-4_FUV_DAY', 'night': 'ICON_L2-5_FUV_NIGHT'}} download = functools.partial(cdw.cdas_download, supported_tags=download_tags) From 673092442dd5d226c0f758832d3348c5fa125eab Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 1 Mar 2023 16:49:34 -0500 Subject: [PATCH 120/396] STY: update tag / inst_id --- pysatNASA/instruments/ace_epam_l2.py | 31 ++++++++++++++--------- pysatNASA/instruments/ace_mag_l2.py | 35 +++++++++++++++++--------- pysatNASA/instruments/ace_sis_l2.py | 28 +++++++++++++-------- pysatNASA/instruments/ace_swepam_l2.py | 30 ++++++++++++++-------- 4 files changed, 78 insertions(+), 46 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index 5b269544..90527612 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -16,9 +16,9 @@ name 'epam_l2' tag - None supported + 'base' or 'key' inst_id - ['h1', 'h2', 'h3', 'k0', 'k1'] + '12sec', '5min', '1hr' Note ---- @@ -46,15 +46,18 @@ platform = 'ace' name = 'epam_l2' -tags = {'': ''} -inst_ids = {id: [tag for tag in tags.keys()] - for id in ['h1', 'h2', 'h3', 'k0', 'k1']} +tags = {'base': 'ACE/EPAM Solar Energetic Particle Base Data', + 'key': 'ACE/EPAM Solar Energetic Particle Key Parameters'} +inst_ids = {'12sec': ['base'], + '5min': ['key', 'base'], + '1hr': ['key', 'base']} pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {id: {'': dt.datetime(2022, 1, 1)} for id in inst_ids.keys()} +_test_dates = {id: {tag: dt.datetime(2022, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} # ---------------------------------------------------------------------------- # Instrument methods @@ -73,9 +76,13 @@ # Use the default CDAWeb and pysat methods # Set the list_files routine -fname = ''.join(('ac_{inst_id:s}_epm_{{year:4d}}{{month:02d}}{{day:02d}}_', +strid = {'12sec': {'base': 'h3'}, + '5min': {'base': 'h1', 'key': 'k0'}, + '1hr': {'base': 'h2', 'key': 'k1'}} +fname = ''.join(('ac_{sid:s}_epm_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {id: {tag: fname.format(inst_id=id) for tag in tags.keys()} +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -84,10 +91,10 @@ load = functools.partial(cdw.load, pandas_format=pandas_format) # Set the download routine -remote_dir = '/pub/data/ace/epam/level_2_cdaweb/epm_{inst_id:s}/{{year:4d}}/' -download_tags = {id: {tag: {'remote_dir': remote_dir.format(inst_id=id), - 'fname': fname.format(inst_id=id)} - for tag in tags.keys()} +remote_dir = '/pub/data/ace/epam/level_2_cdaweb/epm_{sid:s}/{{year:4d}}/' +download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]), + 'fname': fname.format(sid=strid[id][tag])} + for tag in inst_ids[id]} for id in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index 6595fb3c..39a743b4 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -16,9 +16,9 @@ name 'mag_l2' tag - None supported + 'base' or 'key' inst_id - ['h0', 'h1', 'h2', 'h3', 'k0', 'k1', 'k2'] + '1sec', '16sec', '4min', '5min', '1hr' Note ---- @@ -46,14 +46,19 @@ platform = 'ace' name = 'mag_l2' -tags = {'': ''} -inst_ids = {id: [tag for tag in tags.keys()] - for id in ['h0', 'h1', 'h2', 'h3', 'k0', 'k1', 'k2']} +tags = {'base': 'ACE Magnetic Field Base Data', + 'key': 'ACE Magnetic Field Key Parameters'} +inst_ids = {'1sec': ['base'], + '16sec': ['base', 'key'], + '4min': ['base'], + '5min': ['key'], + '1hr': ['base', 'key']} pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {id: {'': dt.datetime(2022, 1, 1)} for id in inst_ids.keys()} +_test_dates = {id: {tag: dt.datetime(2022, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} # ---------------------------------------------------------------------------- # Instrument methods @@ -72,9 +77,15 @@ # Use the default CDAWeb and pysat methods # Set the list_files routine -fname = ''.join(('ac_{inst_id:s}_mfi_{{year:4d}}{{month:02d}}{{day:02d}}_', +strid = {'1sec': {'base': 'h3'}, + '16sec': {'base': 'h0', 'key': 'k1'}, + '4min': {'base': 'h1'}, + '5min': {'key': 'k0'}, + '1hr': {'base': 'h2', 'key': 'k2'}} +fname = ''.join(('ac_{sid:s}_mfi_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {id: {tag: fname.format(inst_id=id) for tag in tags.keys()} +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -83,10 +94,10 @@ load = functools.partial(cdw.load, pandas_format=pandas_format) # Set the download routine -remote_dir = '/pub/data/ace/mag/level_2_cdaweb/mfi_{inst_id:s}/{{year:4d}}/' -download_tags = {id: {tag: {'remote_dir': remote_dir.format(inst_id=id), - 'fname': fname.format(inst_id=id)} - for tag in tags.keys()} +remote_dir = '/pub/data/ace/mag/level_2_cdaweb/mfi_{sid:s}/{{year:4d}}/' +download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]), + 'fname': fname.format(sid=strid[id][tag])} + for tag in inst_ids[id]} for id in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index b8972068..98fa4f0c 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -16,9 +16,9 @@ name 'sis_l2' tag - None supported + 'base' or 'key' inst_id - ['h1', 'h2', 'k0'] + '256sec' or '1hr' Note ---- @@ -44,14 +44,17 @@ platform = 'ace' name = 'sis_l2' -tags = {'': ''} -inst_ids = {id: [tag for tag in tags.keys()] for id in ['h1', 'h2', 'k0']} +tags = {'base': 'ACE/SIS Solar Isotope Spectrometer Base Data', + 'key': 'ACE/SIS Solar Isotope Spectrometer Key Parameters'} +inst_ids = {'256sec': ['base'], + '1hr': ['base', 'key']} pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {id: {'': dt.datetime(2022, 1, 1)} for id in inst_ids.keys()} +_test_dates = {id: {tag: dt.datetime(2022, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} # ---------------------------------------------------------------------------- # Instrument methods @@ -70,9 +73,12 @@ # Use the default CDAWeb and pysat methods # Set the list_files routine -fname = ''.join(('ac_{inst_id:s}_sis_{{year:4d}}{{month:02d}}{{day:02d}}_', +strid = {'256sec': {'base': 'h1'}, + '1hr': {'base': 'h2', 'key': 'k0'}} +fname = ''.join(('ac_{sid:s}_sis_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {id: {tag: fname.format(inst_id=id) for tag in tags.keys()} +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -81,10 +87,10 @@ load = functools.partial(cdw.load, pandas_format=pandas_format) # Set the download routine -remote_dir = '/pub/data/ace/sis/level_2_cdaweb/sis_{inst_id:s}/{{year:4d}}/' -download_tags = {id: {tag: {'remote_dir': remote_dir.format(inst_id=id), - 'fname': fname.format(inst_id=id)} - for tag in tags.keys()} +remote_dir = '/pub/data/ace/sis/level_2_cdaweb/sis_{sid:s}/{{year:4d}}/' +download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]), + 'fname': fname.format(sid=strid[id][tag])} + for tag in inst_ids[id]} for id in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index 308e156f..7c3454b6 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -16,9 +16,9 @@ name 'swepam_l2' tag - None supported + 'base' or 'key' inst_id - ['h0', 'h2', 'k0', 'k1'] + '64sec', '5min', '1hr' Note ---- @@ -46,14 +46,18 @@ platform = 'ace' name = 'swepam_l2' -tags = {'': ''} -inst_ids = {id: [tag for tag in tags.keys()] for id in ['h0', 'h2', 'k0', 'k1']} +tags = {'base': 'ACE/SWEPAM Solar Wind Experiment Base Data', + 'key': 'ACE/SWEPAM Solar Wind Experiment Key Parameters'} +inst_ids = {'64sec': ['base'], + '5min': ['key'], + '1hr': ['key', 'base']} pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {id: {'': dt.datetime(2021, 1, 1)} for id in inst_ids.keys()} +_test_dates = {id: {tag: dt.datetime(2021, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} # ---------------------------------------------------------------------------- # Instrument methods @@ -72,9 +76,13 @@ # Use the default CDAWeb and pysat methods # Set the list_files routine -fname = ''.join(('ac_{inst_id:s}_swe_{{year:4d}}{{month:02d}}{{day:02d}}_', +strid = {'64sec': {'base': 'h0'}, + '5min': {'key': 'k0'}, + '1hr': {'base': 'h2', 'key': 'k1'}} +fname = ''.join(('ac_{sid:s}_swe_{{year:4d}}{{month:02d}}{{day:02d}}_', 'v{{version:02d}}.cdf')) -supported_tags = {id: {tag: fname.format(inst_id=id) for tag in tags.keys()} +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} for id in inst_ids.keys()} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -83,10 +91,10 @@ load = functools.partial(cdw.load, pandas_format=pandas_format) # Set the download routine -remote_dir = '/pub/data/ace/swepam/level_2_cdaweb/swe_{inst_id:s}/{{year:4d}}/' -download_tags = {id: {tag: {'remote_dir': remote_dir.format(inst_id=id), - 'fname': fname.format(inst_id=id)} - for tag in tags.keys()} +remote_dir = '/pub/data/ace/swepam/level_2_cdaweb/swe_{sid:s}/{{year:4d}}/' +download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]), + 'fname': fname.format(sid=strid[id][tag])} + for tag in inst_ids[id]} for id in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) From ca80de850aad89cbc84b975bc3d2de770fd395ce Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 2 Mar 2023 13:54:30 -0500 Subject: [PATCH 121/396] STY: update refs --- pysatNASA/instruments/ace_epam_l2.py | 16 ++++++++-------- pysatNASA/instruments/ace_mag_l2.py | 16 ++++++++-------- pysatNASA/instruments/ace_sis_l2.py | 16 ++++++++-------- pysatNASA/instruments/ace_swepam_l2.py | 16 ++++++++-------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index 90527612..4f0f0652 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -1,14 +1,6 @@ # -*- coding: utf-8 -*- """Module for the Advanced Composition Explorer (ACE) EPAM instrument. -References ----------- -- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. - Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 -- Gold, R., Krimigis, S., Hawkins, S. et al. Electron, Proton, and Alpha Monitor - on the Advanced Composition Explorer spacecraft. Space Science Reviews 86, - 541–562 (1998). https://doi.org/10.1023/A:1005088115759 - Properties ---------- platform @@ -20,6 +12,14 @@ inst_id '12sec', '5min', '1hr' +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Gold, R., Krimigis, S., Hawkins, S. et al. Electron, Proton, and Alpha Monitor + on the Advanced Composition Explorer spacecraft. Space Science Reviews 86, + 541–562 (1998). https://doi.org/10.1023/A:1005088115759 + Note ---- - Level 1 ACE data is maintained at pysatSpaceWeather. diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index 39a743b4..931fa7f0 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -1,14 +1,6 @@ # -*- coding: utf-8 -*- """Module for the Advanced Composition Explorer (ACE) MAG instrument. -References ----------- -- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. - Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 -- Smith, C., L'Heureux, J., Ness, N. et al. The ACE Magnetic Fields Experiment. - Space Science Reviews 86, 613–632 (1998). - https://doi.org/10.1023/A:1005092216668 - Properties ---------- platform @@ -20,6 +12,14 @@ inst_id '1sec', '16sec', '4min', '5min', '1hr' +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Smith, C., L'Heureux, J., Ness, N. et al. The ACE Magnetic Fields Experiment. + Space Science Reviews 86, 613–632 (1998). + https://doi.org/10.1023/A:1005092216668 + Note ---- - Level 1 ACE data is maintained at pysatSpaceWeather. diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index 98fa4f0c..8c36bdf1 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -1,14 +1,6 @@ # -*- coding: utf-8 -*- """Module for the Advanced Composition Explorer (ACE) SIS instrument. -References ----------- -- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. - Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 -- Stone, E., Cohen, C., Cook, W. et al. The Solar Isotope Spectrometer for the - Advanced Composition Explorer. Space Science Reviews 86, 357–408 (1998). - https://doi.org/10.1023/A:1005027929871 - Properties ---------- platform @@ -20,6 +12,14 @@ inst_id '256sec' or '1hr' +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Stone, E., Cohen, C., Cook, W. et al. The Solar Isotope Spectrometer for the + Advanced Composition Explorer. Space Science Reviews 86, 357–408 (1998). + https://doi.org/10.1023/A:1005027929871 + Note ---- - Level 1 ACE data is maintained at pysatSpaceWeather. diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index 7c3454b6..9e09722e 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -1,14 +1,6 @@ # -*- coding: utf-8 -*- """Module for the Advanced Composition Explorer (ACE) EPAM instrument. -References ----------- -- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. - Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 -- McComas, D., Bame, S., Barker, P. et al. Solar Wind Electron Proton Alpha - Monitor (SWEPAM) for the Advanced Composition Explorer. Space Science Reviews - 86, 563–612 (1998). https://doi.org/10.1023/A:1005040232597 - Properties ---------- platform @@ -20,6 +12,14 @@ inst_id '64sec', '5min', '1hr' +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- McComas, D., Bame, S., Barker, P. et al. Solar Wind Electron Proton Alpha + Monitor (SWEPAM) for the Advanced Composition Explorer. Space Science Reviews + 86, 563–612 (1998). https://doi.org/10.1023/A:1005040232597 + Note ---- - Level 1 ACE data is maintained at pysatSpaceWeather. From d26d67006d0edfec961ab538c5474720cb395e94 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 2 Mar 2023 13:59:22 -0500 Subject: [PATCH 122/396] DOC/STY: refs after properties --- pysatNASA/instruments/cnofs_ivm.py | 24 +++++++++++++----------- pysatNASA/instruments/cnofs_plp.py | 14 ++++++++------ pysatNASA/instruments/cnofs_vefi.py | 14 ++++++++------ pysatNASA/instruments/de2_lang.py | 13 +++++++------ pysatNASA/instruments/de2_nacs.py | 14 ++++++++------ pysatNASA/instruments/de2_rpa.py | 13 ++++++++----- pysatNASA/instruments/de2_wats.py | 14 ++++++++------ pysatNASA/instruments/dmsp_ssusi.py | 18 ++++++++++-------- pysatNASA/instruments/jpl_gps.py | 22 ++++++++++++---------- 9 files changed, 82 insertions(+), 64 deletions(-) diff --git a/pysatNASA/instruments/cnofs_ivm.py b/pysatNASA/instruments/cnofs_ivm.py index a7693648..29d8a2a4 100644 --- a/pysatNASA/instruments/cnofs_ivm.py +++ b/pysatNASA/instruments/cnofs_ivm.py @@ -16,6 +16,19 @@ motion of the satellite the angle is converted into ion motion along two orthogonal directions, perpendicular to the satellite track. + +Properties +---------- +platform + 'cnofs' +name + 'ivm' +tag + None supported +inst_id + None supported + + References ---------- A brief discussion of the C/NOFS mission and instruments can be found at @@ -34,17 +47,6 @@ A08324, doi:10.1029/2009JA015051. -Properties ----------- -platform - 'cnofs' -name - 'ivm' -tag - None supported -inst_id - None supported - Warnings -------- - The sampling rate of the instrument changes on July 29th, 2010. diff --git a/pysatNASA/instruments/cnofs_plp.py b/pysatNASA/instruments/cnofs_plp.py index fb43ab9e..e4c3ebfc 100644 --- a/pysatNASA/instruments/cnofs_plp.py +++ b/pysatNASA/instruments/cnofs_plp.py @@ -29,12 +29,6 @@ The data is PRELIMINARY, and as such, is intended for BROWSE PURPOSES ONLY. -References ----------- -A brief discussion of the C/NOFS mission and instruments can be found at -de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast -scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, -doi:10.1016/j.jastp.2004.07.030. Properties ---------- @@ -48,6 +42,14 @@ None supported +References +---------- +A brief discussion of the C/NOFS mission and instruments can be found at +de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast +scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, +doi:10.1016/j.jastp.2004.07.030. + + Warnings -------- - The data are PRELIMINARY, and as such, are intended for BROWSE PURPOSES ONLY. diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py index 78d54e36..fcf39383 100644 --- a/pysatNASA/instruments/cnofs_vefi.py +++ b/pysatNASA/instruments/cnofs_vefi.py @@ -26,12 +26,6 @@ The data is PRELIMINARY, and as such, is intended for BROWSE PURPOSES ONLY. -References ----------- -A brief discussion of the C/NOFS mission and instruments can be found at -de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast -scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, -doi:10.1016/j.jastp.2004.07.030. Properties ---------- @@ -45,6 +39,14 @@ None supported +References +---------- +A brief discussion of the C/NOFS mission and instruments can be found at +de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast +scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, +doi:10.1016/j.jastp.2004.07.030. + + Note ---- - tag = 'dc_b': 1 second DC magnetometer data diff --git a/pysatNASA/instruments/de2_lang.py b/pysatNASA/instruments/de2_lang.py index 07430357..ce5740ac 100644 --- a/pysatNASA/instruments/de2_lang.py +++ b/pysatNASA/instruments/de2_lang.py @@ -25,12 +25,6 @@ or correct the inflight processed data. Time resolution was 0.5 seconds. -References ----------- -J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H. Pinkus, and R. B. Kaplan, -"The Dynamics Explorer 2 Langmuir Probe (LANG)", Space Sci. Instrum., 5, -493-502, 1981. - Properties ---------- platform @@ -43,6 +37,13 @@ None Supported +References +---------- +J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H. Pinkus, and R. B. Kaplan, +"The Dynamics Explorer 2 Langmuir Probe (LANG)", Space Sci. Instrum., 5, +493-502, 1981. + + Warnings -------- - Currently no cleaning routine. diff --git a/pysatNASA/instruments/de2_nacs.py b/pysatNASA/instruments/de2_nacs.py index b03212fe..00b6c30d 100644 --- a/pysatNASA/instruments/de2_nacs.py +++ b/pysatNASA/instruments/de2_nacs.py @@ -50,12 +50,6 @@ were lost between 12 March 1982 and 31 March 1982 when the counter overflowed. -References ----------- -G. R. Carrignan, B. P. Block, J. C. Maurer, A. E. Hedin, C. A. Reber, -N. W. Spencer, "The neutral mass spectrometer on Dynamics Explorer B", -Space Sci. Instrum., 5, 429-441, 1981. - Properties ---------- platform @@ -67,6 +61,14 @@ tag None Supported + +References +---------- +G. R. Carrignan, B. P. Block, J. C. Maurer, A. E. Hedin, C. A. Reber, +N. W. Spencer, "The neutral mass spectrometer on Dynamics Explorer B", +Space Sci. Instrum., 5, 429-441, 1981. + + Warnings -------- - Currently no cleaning routine. diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 912dd66f..68ad5b57 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -36,11 +36,6 @@ files that was submitted by the PI team in June of 1995. The revised RPA data include a correction to the spacecraft potential. -References ----------- -W. B. Hanson, R. A. Heelis, R. A. Power, C. R. Lippincott, D. R. Zuccaro, -B. J. Holt, L. H. Harmon, and S. Sanatani, “The retarding potential analyzer -for dynamics explorer-B,” Space Sci. Instrum. 5, 503–510 (1981). Properties ---------- @@ -53,6 +48,14 @@ tag None Supported + +References +---------- +W. B. Hanson, R. A. Heelis, R. A. Power, C. R. Lippincott, D. R. Zuccaro, +B. J. Holt, L. H. Harmon, and S. Sanatani, “The retarding potential analyzer +for dynamics explorer-B,” Space Sci. Instrum. 5, 503–510 (1981). + + Warnings -------- - Currently no cleaning routine. diff --git a/pysatNASA/instruments/de2_wats.py b/pysatNASA/instruments/de2_wats.py index 1345841a..d7567e4e 100644 --- a/pysatNASA/instruments/de2_wats.py +++ b/pysatNASA/instruments/de2_wats.py @@ -47,12 +47,6 @@ about the processing done at NSSDC is given in WATS_NSSDC_PRO_DE.DOC. -References ----------- -N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E. Hedin, G. R. Carrignan, -J. C. Maurer, "The Dynamics Explorer Wind and Temperature Spectrometer", -Space Sci. Instrum., 5, 417-428, 1981. - Properties ---------- platform @@ -64,6 +58,14 @@ tag None Supported + +References +---------- +N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E. Hedin, G. R. Carrignan, +J. C. Maurer, "The Dynamics Explorer Wind and Temperature Spectrometer", +Space Sci. Instrum., 5, 417-428, 1981. + + Warnings -------- - Currently no cleaning routine. diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index a16acd69..e717c0ff 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -20,14 +20,6 @@ frequency (HF) communication links and related systems and assessment of the environmental hazard to astronauts on the Space Station. -References ----------- -Larry J. Paxton, Daniel Morrison, Yongliang Zhang, Hyosub Kil, Brian Wolven, -Bernard S. Ogorzalek, David C. Humm, and Ching-I. Meng "Validation of remote -sensing products produced by the Special Sensor Ultraviolet Scanning Imager -(SSUSI): a far UV-imaging spectrograph on DMSP F-16", Proc. SPIE 4485, Optical -Spectroscopic Techniques, Remote Sensing, and Instrumentation for Atmospheric -and Space Research IV, (30 January 2002); doi:10.1117/12.454268 Properties ---------- @@ -41,6 +33,16 @@ 'f16', 'f17', 'f18', 'f19' +References +---------- +Larry J. Paxton, Daniel Morrison, Yongliang Zhang, Hyosub Kil, Brian Wolven, +Bernard S. Ogorzalek, David C. Humm, and Ching-I. Meng "Validation of remote +sensing products produced by the Special Sensor Ultraviolet Scanning Imager +(SSUSI): a far UV-imaging spectrograph on DMSP F-16", Proc. SPIE 4485, Optical +Spectroscopic Techniques, Remote Sensing, and Instrumentation for Atmospheric +and Space Research IV, (30 January 2002); doi:10.1117/12.454268 + + Warnings -------- - Currently no cleaning routine. diff --git a/pysatNASA/instruments/jpl_gps.py b/pysatNASA/instruments/jpl_gps.py index ddbe66cf..4bce888a 100644 --- a/pysatNASA/instruments/jpl_gps.py +++ b/pysatNASA/instruments/jpl_gps.py @@ -16,16 +16,6 @@ the global network of International GNSS Service and the regional network of Continuous Operating Reference Station (CORS). -References ----------- -Pi, X., A. J. Mannucci, U. J. Lindqwister, and C. M. Ho, Monitoring of global -ionospheric irregularities using the worldwide GPS network, Geophys. Res. -Lett., 24, 2283, 1997. - -Pi, X., F. J. Meyer, K. Chotoo, Anthony Freeman, R. G. Caton, and C. T. -Bridgwood, Impact of ionospheric scintillation on Spaceborne SAR observations -studied using GNSS, Proc. ION-GNSS, pp.1998-2006, 2012. - Properties ---------- @@ -38,6 +28,18 @@ inst_id None supported + +References +---------- +Pi, X., A. J. Mannucci, U. J. Lindqwister, and C. M. Ho, Monitoring of global +ionospheric irregularities using the worldwide GPS network, Geophys. Res. +Lett., 24, 2283, 1997. + +Pi, X., F. J. Meyer, K. Chotoo, Anthony Freeman, R. G. Caton, and C. T. +Bridgwood, Impact of ionospheric scintillation on Spaceborne SAR observations +studied using GNSS, Proc. ION-GNSS, pp.1998-2006, 2012. + + Warnings -------- - The cleaning parameters for the instrument are still under development. From 2c383be1b0283925c668e189ae9baf15f3d494ca Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 2 Mar 2023 13:59:36 -0500 Subject: [PATCH 123/396] DOC/BUG: update module description --- pysatNASA/instruments/formosat1_ivm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/formosat1_ivm.py b/pysatNASA/instruments/formosat1_ivm.py index 24be65bd..48444e33 100644 --- a/pysatNASA/instruments/formosat1_ivm.py +++ b/pysatNASA/instruments/formosat1_ivm.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Module for the ICON EUV instrument. +"""Module for the Formosat-1 IVM instrument. Supports the Ion Velocity Meter (IVM) onboard the Formosat-1 (formerly ROCSAT-1) mission. Downloads data from the NASA Coordinated Data Analysis From 1225a855df82dc08c06eac91587a823f58d6b96f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 2 Mar 2023 14:00:27 -0500 Subject: [PATCH 124/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5335347..de9dc7ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated CDAWeb routines to allow for data stored by year/day-of-year * Maintenance * Added a version cap for numpy (required for cdf interface, revisit before release) + * Updated dosctring style for consistency ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 9350170182ece7fde38cedc57b874e1652b85f10 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 2 Mar 2023 16:26:40 -0500 Subject: [PATCH 125/396] BUG: meta translation --- pysatNASA/instruments/ace_epam_l2.py | 6 +++++- pysatNASA/instruments/ace_mag_l2.py | 6 +++++- pysatNASA/instruments/ace_sis_l2.py | 6 +++++- pysatNASA/instruments/ace_swepam_l2.py | 6 +++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index 4f0f0652..140e049d 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -88,7 +88,11 @@ supported_tags=supported_tags) # Set the load routine -load = functools.partial(cdw.load, pandas_format=pandas_format) +meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} +load = functools.partial(cdw.load, pandas_format=pandas_format, + meta_translation=meta_translation) # Set the download routine remote_dir = '/pub/data/ace/epam/level_2_cdaweb/epm_{sid:s}/{{year:4d}}/' diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index 931fa7f0..c40c0cca 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -91,7 +91,11 @@ supported_tags=supported_tags) # Set the load routine -load = functools.partial(cdw.load, pandas_format=pandas_format) +meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} +load = functools.partial(cdw.load, pandas_format=pandas_format, + meta_translation=meta_translation) # Set the download routine remote_dir = '/pub/data/ace/mag/level_2_cdaweb/mfi_{sid:s}/{{year:4d}}/' diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index 8c36bdf1..88261497 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -84,7 +84,11 @@ supported_tags=supported_tags) # Set the load routine -load = functools.partial(cdw.load, pandas_format=pandas_format) +meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} +load = functools.partial(cdw.load, pandas_format=pandas_format, + meta_translation=meta_translation) # Set the download routine remote_dir = '/pub/data/ace/sis/level_2_cdaweb/sis_{sid:s}/{{year:4d}}/' diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index 9e09722e..e2c99033 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -88,7 +88,11 @@ supported_tags=supported_tags) # Set the load routine -load = functools.partial(cdw.load, pandas_format=pandas_format) +meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} +load = functools.partial(cdw.load, pandas_format=pandas_format, + meta_translation=meta_translation) # Set the download routine remote_dir = '/pub/data/ace/swepam/level_2_cdaweb/swe_{sid:s}/{{year:4d}}/' From 046292aff3b14d2e553556e579b46c1b9ccb0bea Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 2 Mar 2023 17:22:49 -0500 Subject: [PATCH 126/396] DOC: update docstrings --- pysatNASA/instruments/cnofs_ivm.py | 16 ++++++++-------- pysatNASA/instruments/cnofs_plp.py | 14 +++++++------- pysatNASA/instruments/cnofs_vefi.py | 16 ++++++++-------- pysatNASA/instruments/de2_lang.py | 11 +++++------ pysatNASA/instruments/de2_nacs.py | 10 +++++----- pysatNASA/instruments/de2_rpa.py | 10 +++++----- pysatNASA/instruments/de2_wats.py | 10 +++++----- pysatNASA/instruments/dmsp_ssusi.py | 11 +++++------ pysatNASA/instruments/jpl_gps.py | 10 +++++----- 9 files changed, 53 insertions(+), 55 deletions(-) diff --git a/pysatNASA/instruments/cnofs_ivm.py b/pysatNASA/instruments/cnofs_ivm.py index 29d8a2a4..cee3dc68 100644 --- a/pysatNASA/instruments/cnofs_ivm.py +++ b/pysatNASA/instruments/cnofs_ivm.py @@ -29,6 +29,14 @@ None supported +Warnings +-------- +- The sampling rate of the instrument changes on July 29th, 2010. + The rate is attached to the instrument object as .sample_rate. + +- The cleaning parameters for the instrument are still under development. + + References ---------- A brief discussion of the C/NOFS mission and instruments can be found at @@ -46,14 +54,6 @@ duskside topside ionosphere with CINDI and DMSP, J. Geophys. Res.,115, A08324, doi:10.1029/2009JA015051. - -Warnings --------- -- The sampling rate of the instrument changes on July 29th, 2010. - The rate is attached to the instrument object as .sample_rate. - -- The cleaning parameters for the instrument are still under development. - """ import datetime as dt diff --git a/pysatNASA/instruments/cnofs_plp.py b/pysatNASA/instruments/cnofs_plp.py index e4c3ebfc..f885db38 100644 --- a/pysatNASA/instruments/cnofs_plp.py +++ b/pysatNASA/instruments/cnofs_plp.py @@ -42,6 +42,13 @@ None supported +Warnings +-------- +- The data are PRELIMINARY, and as such, are intended for BROWSE PURPOSES ONLY. +- Currently no cleaning routine. +- Module not written by PLP team. + + References ---------- A brief discussion of the C/NOFS mission and instruments can be found at @@ -49,13 +56,6 @@ scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, doi:10.1016/j.jastp.2004.07.030. - -Warnings --------- -- The data are PRELIMINARY, and as such, are intended for BROWSE PURPOSES ONLY. -- Currently no cleaning routine. -- Module not written by PLP team. - """ import datetime as dt diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py index fcf39383..6ad7c6ab 100644 --- a/pysatNASA/instruments/cnofs_vefi.py +++ b/pysatNASA/instruments/cnofs_vefi.py @@ -39,14 +39,6 @@ None supported -References ----------- -A brief discussion of the C/NOFS mission and instruments can be found at -de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast -scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, -doi:10.1016/j.jastp.2004.07.030. - - Note ---- - tag = 'dc_b': 1 second DC magnetometer data @@ -58,6 +50,14 @@ - Limited cleaning routine. - Module not written by VEFI team. + +References +---------- +A brief discussion of the C/NOFS mission and instruments can be found at +de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast +scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, +doi:10.1016/j.jastp.2004.07.030. + """ import datetime as dt diff --git a/pysatNASA/instruments/de2_lang.py b/pysatNASA/instruments/de2_lang.py index ce5740ac..3643c566 100644 --- a/pysatNASA/instruments/de2_lang.py +++ b/pysatNASA/instruments/de2_lang.py @@ -37,18 +37,17 @@ None Supported +Warnings +-------- +- Currently no cleaning routine. + + References ---------- J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H. Pinkus, and R. B. Kaplan, "The Dynamics Explorer 2 Langmuir Probe (LANG)", Space Sci. Instrum., 5, 493-502, 1981. - -Warnings --------- -- Currently no cleaning routine. - - """ import datetime as dt diff --git a/pysatNASA/instruments/de2_nacs.py b/pysatNASA/instruments/de2_nacs.py index 00b6c30d..a586c81c 100644 --- a/pysatNASA/instruments/de2_nacs.py +++ b/pysatNASA/instruments/de2_nacs.py @@ -62,17 +62,17 @@ None Supported +Warnings +-------- +- Currently no cleaning routine. + + References ---------- G. R. Carrignan, B. P. Block, J. C. Maurer, A. E. Hedin, C. A. Reber, N. W. Spencer, "The neutral mass spectrometer on Dynamics Explorer B", Space Sci. Instrum., 5, 429-441, 1981. - -Warnings --------- -- Currently no cleaning routine. - """ import datetime as dt diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 68ad5b57..e817f7a8 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -49,17 +49,17 @@ None Supported +Warnings +-------- +- Currently no cleaning routine. + + References ---------- W. B. Hanson, R. A. Heelis, R. A. Power, C. R. Lippincott, D. R. Zuccaro, B. J. Holt, L. H. Harmon, and S. Sanatani, “The retarding potential analyzer for dynamics explorer-B,” Space Sci. Instrum. 5, 503–510 (1981). - -Warnings --------- -- Currently no cleaning routine. - """ import datetime as dt diff --git a/pysatNASA/instruments/de2_wats.py b/pysatNASA/instruments/de2_wats.py index d7567e4e..4fa5fe61 100644 --- a/pysatNASA/instruments/de2_wats.py +++ b/pysatNASA/instruments/de2_wats.py @@ -59,17 +59,17 @@ None Supported +Warnings +-------- +- Currently no cleaning routine. + + References ---------- N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E. Hedin, G. R. Carrignan, J. C. Maurer, "The Dynamics Explorer Wind and Temperature Spectrometer", Space Sci. Instrum., 5, 417-428, 1981. - -Warnings --------- -- Currently no cleaning routine. - """ import datetime as dt diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index e717c0ff..60973d81 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -33,6 +33,11 @@ 'f16', 'f17', 'f18', 'f19' +Warnings +-------- +- Currently no cleaning routine. + + References ---------- Larry J. Paxton, Daniel Morrison, Yongliang Zhang, Hyosub Kil, Brian Wolven, @@ -42,12 +47,6 @@ Spectroscopic Techniques, Remote Sensing, and Instrumentation for Atmospheric and Space Research IV, (30 January 2002); doi:10.1117/12.454268 - -Warnings --------- -- Currently no cleaning routine. - - """ import datetime as dt diff --git a/pysatNASA/instruments/jpl_gps.py b/pysatNASA/instruments/jpl_gps.py index 4bce888a..63787548 100644 --- a/pysatNASA/instruments/jpl_gps.py +++ b/pysatNASA/instruments/jpl_gps.py @@ -29,6 +29,11 @@ None supported +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + + References ---------- Pi, X., A. J. Mannucci, U. J. Lindqwister, and C. M. Ho, Monitoring of global @@ -39,11 +44,6 @@ Bridgwood, Impact of ionospheric scintillation on Spaceborne SAR observations studied using GNSS, Proc. ION-GNSS, pp.1998-2006, 2012. - -Warnings --------- -- The cleaning parameters for the instrument are still under development. - """ import datetime as dt From c580167229ea05675c32e7ceae57d2b59a5db4c3 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:33:02 -0500 Subject: [PATCH 127/396] improved documentation for cdasws functions --- pysatNASA/instruments/methods/cdaweb.py | 41 ++++++++++++------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 2d5f2bd4..9528acdf 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -539,19 +539,18 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, Path to data directory. If None is specified, the value previously set in Instrument.files.data_path is used. (default=None) + Note + ---- + Supported tags for this function use the cdaweb dataset naming convention. + You can find the dataset names on cdaweb or you can use cdasws. + Examples -------- :: - # download support added to cnofs_vefi.py using code below - fn = 'cnofs_vefi_bfield_1sec_{year:4d}{month:02d}{day:02d}_v05.cdf' - dc_b_tag = {'remote_dir': ''.join(('/pub/data/cnofs/vefi/bfield_1sec', - '/{year:4d}/')), - 'fname': fn} - supported_tags = {'dc_b': dc_b_tag} - - download = functools.partial(nasa_cdaweb.download, - supported_tags=supported_tags) + download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} + download = functools.partial(cdw.cdas_download, + supported_tags=download_tags) """ @@ -782,22 +781,21 @@ def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, file_list : list A list containing the verified available files + Note + ---- + Supported tags for this function use the cdaweb dataset naming convention. + You can find the dataset names on cdaweb or you can use cdasws. + Examples -------- :: + download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} + list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) - fname = 'cnofs_vefi_bfield_1sec_{year:04d}{month:02d}{day:02d}_v05.cdf' - supported_tags = {'dc_b': fname} - list_remote_files = \ - functools.partial(nasa_cdaweb.list_remote_files, - supported_tags=supported_tags) - - fname = 'cnofs_cindi_ivm_500ms_{year:4d}{month:02d}{day:02d}_v01.cdf' - supported_tags = {'': fname} - list_remote_files = \ - functools.partial(cdw.list_remote_files, - supported_tags=supported_tags) - + download_tags = {'': {'': 'CNOFS_CINDI_IVM_500MS'}} + list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) """ cdas = CdasWs() dataset = try_inst_dict(inst_id, tag, supported_tags) @@ -813,6 +811,7 @@ def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, if isinstance(start, pds._libs.tslibs.timestamps.Timestamp): start = start.tz_localize('utc') stop = stop.tz_localize('utc') + og_files = cdas.get_original_files(dataset=dataset, start=start, end=stop) file_list = [f['Name'] for f in og_files[1]] From 59738bb8816375185e2a71b7747b0deef194ec59 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:33:25 -0500 Subject: [PATCH 128/396] added tests for cdasws date handling --- pysatNASA/tests/test_methods_cdaweb.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index e4813e87..8bd7cbba 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -1,6 +1,7 @@ """Unit tests for the cdaweb instrument methods.""" import datetime as dt +import pandas as pds import requests import pytest @@ -80,11 +81,17 @@ def test_bad_kwarg_list_remote_files(self, bad_key, bad_val, err_msg): assert str(excinfo.value).find(err_msg) >= 0 return - def test_remote_file_list_all(self): - """Test that remote_file_list works if start/stop dates unspecified.""" + @pytest.mark.parametrize("start, stop", + [(None, None), + (dt.datetime(2009, 1, 1), + dt.datetime(2009, 1, 1)), + (pds.Timestamp(2009, 1, 1), + pds.Timestamp(2009, 1, 2))]) + def test_remote_file_list_all(self, start, stop): + """Test that remote_file_list works for all start and stop cases.""" self.module = pysatNASA.instruments.cnofs_plp self.test_inst = pysat.Instrument(inst_module=self.module) - files = self.test_inst.remote_file_list() + files = self.test_inst.remote_file_list(start, stop) assert len(files) > 0 return From 57b9ec7d136928053495a89f942c97c517feec6b Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 3 Mar 2023 15:14:15 -0500 Subject: [PATCH 129/396] TST: remove version cap for xarray --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 35ab70f2..ab69e36c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ cdflib>=0.4.4 numpy<1.24 pandas pysat>=3.0.4 -xarray<2022.11 +xarray From 583a6241d5cae0780741f854308dbc1d60956b00 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 3 Mar 2023 15:36:15 -0500 Subject: [PATCH 130/396] TST: remove version cap for numpy --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ab69e36c..f591b306 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ requests beautifulsoup4 lxml cdflib>=0.4.4 -numpy<1.24 +numpy pandas pysat>=3.0.4 xarray From c2ccf5f5afe94b4aee88d7c3d38d9e17ec53bb52 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 3 Mar 2023 16:34:51 -0500 Subject: [PATCH 131/396] TST: restore numpy cap --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f591b306..ab69e36c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ requests beautifulsoup4 lxml cdflib>=0.4.4 -numpy +numpy<1.24 pandas pysat>=3.0.4 xarray From e4180380c5ff513b59d553388441ec6861546186 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 11:02:48 -0500 Subject: [PATCH 132/396] DOC: pysatCDF info --- README.md | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d04d1045..4b5cb99e 100644 --- a/README.md +++ b/README.md @@ -22,15 +22,15 @@ pysatNASA uses common Python modules, as well as modules developed by and for the Space Physics community. This module officially supports Python 3.8+. -| Common modules | Community modules | -| ---------------- | ----------------- | -| beautifulsoup4 | cdflib | -| lxml | pysat>=3.0.4 | -| netCDF4 | | -| numpy | | -| pandas | | -| requests | | -| xarray<2022.11 | | +| Common modules | Community modules | Optional Modules | +| ---------------- | ----------------- |------------------| +| beautifulsoup4 | cdflib | pysatCDF | +| lxml | pysat>=3.0.4 | | +| netCDF4 | | | +| numpy | | | +| pandas | | | +| requests | | | +| xarray<2022.11 | | | ## GitHub Installation @@ -70,3 +70,18 @@ import pysat pysat.utils.registry.register(['pysatNASA.instruments.icon_ivm']) ivm = pysat.Instrument('icon', 'ivm', inst_id='a') ``` + +# CDF Integration +For data products stored as CDF files, this package can use either `cdflib` or +`pysatCDF`. Note that `cdflib` is a pure python package and more readily +deployable across systems, whereas `pysatCDF` interfaces with the fortran. +This is a faster approach for loading data, but may not install on all systems. +Therefore, `pysatCDF` is optional rather than required. + +You can specify which load routine to use via the optional `use_cdflib` kwarg. +If no kwarg is specified, `pysatNASA` will default to `pysatCDF` if it is +successfully installed, and default to `cdflib` otherwise. + +``` +ivm = pysat.Instrument('cnofs', 'ivm', use_cdflib=True) +``` From 69a2b9c56d8526a6cfc161aad209a87a043c8b66 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 11:02:57 -0500 Subject: [PATCH 133/396] DOC: update readme --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4b5cb99e..c1e591a7 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,12 @@ Python 3.8+. | requests | | | | xarray<2022.11 | | | -## GitHub Installation +## PyPi Installation +``` +pip install pysatNASA +``` -Currently, the main way to get pysatNASA is through github. +## GitHub Installation ``` git clone https://github.com/pysat/pysatNASA.git @@ -51,6 +54,7 @@ python setup.py install Note: pre-1.0.0 version ----------------------- pysatNASA is currently in an initial development phase and requires pysat 3.0.4. +Feedback and contributions are appreciated. # Using with pysat @@ -62,7 +66,9 @@ from pysatNASA.instruments import icon_ivm ivm = pysat.Instrument(inst_module=icon_ivm, inst_id='a') ``` -Another way to use the instruments in an external repository is to register the instruments. This only needs to be done the first time you load an instrument. Afterward, pysat will identify them using the `platform` and `name` keywords. +Another way to use the instruments in an external repository is to register the +instruments. This only needs to be done the first time you load an instrument. +Afterward, pysat will identify them using the `platform` and `name` keywords. ``` import pysat From 71d681a06eecc3210e3f287375e5f65d36bced4d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 11:03:07 -0500 Subject: [PATCH 134/396] STY: update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 34 ++++++++++------------- .github/ISSUE_TEMPLATE/feature_request.md | 24 +++++++++++----- .github/ISSUE_TEMPLATE/question.md | 19 +++++++++++++ 3 files changed, 50 insertions(+), 27 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/question.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 23d236d6..57640182 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,35 +1,29 @@ --- name: Bug report -about: Create a report to help us improve +about: Create a report to report a problem that needs to be fixed +labels: bug +title: "BUG: " --- -**Describe the bug** -A clear and concise description of what the bug is. +# Description +A clear and concise description of what the bug is, including a description +of what you expected the outcome to be. -**To Reproduce** +# To Reproduce this bug: Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -or +Consider including images or test files to help others reproduce the bug and +solve the problem. -``` -# test code here -``` - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Version [e.g. Python 3.7] +## Test configuration + - OS: [e.g. Hal] + - Version [e.g. Python 3.47] - Other details about your setup that could be relevant -**Additional context** -Add any other context about the problem here. +# Additional context +Add any other context about the problem here, including expected behaviour. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 066b2d92..d02da2ef 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,17 +1,27 @@ --- name: Feature request about: Suggest an idea for this project +title: "ENH: " +labels: enhancement --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +# Description +A clear and concise description of the new feature or behaviour you would like. -**Describe the solution you'd like** +## Potential impact + +- Is the feature related to an existing problem? +- How critical is this feature to your workflow? +- How wide of an impact to you anticipate this enhancement having? +- Would this break any existing functionality? + +## Potential solution(s) A clear and concise description of what you want to happen. -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +# Alternatives +A clear description of any alternative solutions or features you've considered. -**Additional context** -Add any other context or screenshots about the feature request here. +# Additional context +Add any other context or screenshots about the feature request here, potentially +including your operational configuration. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 00000000..da43edc7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,19 @@ +--- +name: Question +about: A question about this project +title: "QUEST: " +labels: question + +--- + +# Description +A clear and concise summary of your query + +## Example code (optional) +If relevant, include sample code, images, or files so that others can understand +the full context of your question. + +## Configuration + - OS: [e.g. Hal] + - Version: [e.g. Python 3.47] + - Other details about your setup that could be relevant From f47994d2defbcdec31321488cff8e4c503b66ec1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 11:04:41 -0500 Subject: [PATCH 135/396] STY: update workflows --- .github/workflows/main.yml | 4 ++-- .github/workflows/pysat_rc.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cc04a773..9bf89254 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,13 +10,13 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.9", "3.10"] numpy_ver: [latest] include: - python-version: "3.8" numpy_ver: "1.20" - os: ubuntu-latest + os: "ubuntu-latest" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 5268f2c2..a72b5600 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.10"] name: Python ${{ matrix.python-version }} on ${{ matrix.os }} From 78d872aec9b716d147c5f7a4bb1c6807be2d22ab Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 11:05:22 -0500 Subject: [PATCH 136/396] STY: update PR template --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1a454bc5..e1d2dbe5 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,7 +6,7 @@ Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. Please see ``CONTRIBUTING.md`` for more guidelines. -## Type of change +# Type of change Please delete options that are not relevant. From ba844a8f461c182ec2a0218cb8850a990fe77933 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 11:08:55 -0500 Subject: [PATCH 137/396] STY: update docs config --- docs/conf.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c3398628..2a4a7a62 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,11 +52,12 @@ master_doc = 'index' # General information about the project. -zenodo = json.loads(open('../.zenodo.json').read()) project = 'pysatNASA' -author = ', '.join([creator['name'] for creator in zenodo['creators']]) -manual_copyright = ', '.join(['2021', author]) title = '{:s} Documentation'.format(project) +zenodo = json.loads(open('../.zenodo.json').read()) +author = ', '.join([creator['name'] for creator in zenodo['creators']]) +manual_copyright = ', '.join(['2023', author]) +category = 'Space Physics' description = 'Tools for NASA CDAWeb instruments.' # The version info for the project you're documenting, acts as replacement for @@ -148,8 +149,8 @@ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) -texinfo_documents = [(master_doc, project, title, author, project, description, - 'Space Physics')] +texinfo_documents = [(master_doc, project, title, author, project, + description, category)] # -- Options for Epub output ---------------------------------------------- From 5eab7d21488efb57ededc595516ed5a365316b9a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 11:11:37 -0500 Subject: [PATCH 138/396] DOC: update copyright --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index c9a10064..666d8f5d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2020, pysat +Copyright (c) 2023, pysat All rights reserved. Redistribution and use in source and binary forms, with or without From b4c5523898df704cbec7497eb774de6c0b3f75be Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 11:12:35 -0500 Subject: [PATCH 139/396] DOC: update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d04d1045..40b6c31b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Python 3.8+. | numpy | | | pandas | | | requests | | -| xarray<2022.11 | | +| xarray | | ## GitHub Installation From 90404bffb90655097be378a4fb5eb627882193b8 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 11:13:04 -0500 Subject: [PATCH 140/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de9dc7ab..abf4619b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Maintenance * Added a version cap for numpy (required for cdf interface, revisit before release) * Updated dosctring style for consistency + * Removed version cap for xarray ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 27920393a3f8187bc74695fcdb7c633a5b0137a1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 12:14:34 -0500 Subject: [PATCH 141/396] STY: add clean --- pysatNASA/instruments/ace_epam_l2.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index 140e049d..b452a687 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -34,6 +34,7 @@ import datetime as dt import functools +import numpy as np from pysat.instruments.methods import general as mm_gen @@ -66,13 +67,29 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # + +def clean(self): + """Clean ACE EPAM data to the specified level. + + Note + ---- + Basic cleaning to replace fill values with NaN + + """ + + for key in self.variables: + if key != 'time': + fill = self.meta[key, self.meta.labels.fill_val] + # Replace fill with nan + self[key] = self.data[key].where(key != fill) + self.meta[key, self.meta.labels.fill_val] = np.nan + return + + # Use the default CDAWeb and pysat methods # Set the list_files routine From dc49e5b7969e9492b6610ef9d732467cd53f1df3 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 13:28:08 -0500 Subject: [PATCH 142/396] STY: strings --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9bf89254..b9ea9d65 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.9", "3.10"] - numpy_ver: [latest] + numpy_ver: ["latest"] include: - python-version: "3.8" numpy_ver: "1.20" From 3ac28b0f7b81905f5e571fc093c666705da64d0a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 6 Mar 2023 13:31:10 -0500 Subject: [PATCH 143/396] STY: pipe --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b9ea9d65..0cd854e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,8 @@ jobs: - name: Install NEP29 dependencies if: ${{ matrix.numpy_ver != 'latest'}} - run: pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} + run: | + pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - name: Install standard dependencies run: | From f3790b7bc8a1e388bc6e49cdf369146870d00c73 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 6 Mar 2023 15:46:32 -0500 Subject: [PATCH 144/396] Update CONTRIBUTING.md Co-authored-by: Angeline Burrell --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e325724..45943ed4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,7 +81,7 @@ To set up `pysatNASA` for local development: is broken on your local system: ``` - pytest PACKAGENAME + pytest pysatNASA ``` 5. You should also check for flake8 style compliance: From 04b7ca005f6d95458540b1d8509f6c318db6216a Mon Sep 17 00:00:00 2001 From: jklenzing Date: Tue, 7 Mar 2023 11:13:42 -0500 Subject: [PATCH 145/396] ENH: standard clean routine --- pysatNASA/instruments/ace_epam_l2.py | 20 ++------------------ pysatNASA/instruments/ace_mag_l2.py | 5 ++--- pysatNASA/instruments/ace_sis_l2.py | 5 ++--- pysatNASA/instruments/ace_swepam_l2.py | 4 ++-- pysatNASA/instruments/methods/ace.py | 21 +++++++++++++++++++++ 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index b452a687..2d905334 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -67,29 +67,13 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) +# Use default ace clean +clean = mm_ace.clean # ---------------------------------------------------------------------------- # Instrument functions # -def clean(self): - """Clean ACE EPAM data to the specified level. - - Note - ---- - Basic cleaning to replace fill values with NaN - - """ - - for key in self.variables: - if key != 'time': - fill = self.meta[key, self.meta.labels.fill_val] - # Replace fill with nan - self[key] = self.data[key].where(key != fill) - self.meta[key, self.meta.labels.fill_val] = np.nan - return - - # Use the default CDAWeb and pysat methods # Set the list_files routine diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index c40c0cca..ebc9d72e 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -67,9 +67,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn - +# Use default ace clean +clean = mm_ace.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index 88261497..eb74ef39 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -63,9 +63,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn - +# Use default ace clean +clean = mm_ace.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index e2c99033..e55f2cda 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -66,8 +66,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default ace clean +clean = mm_ace.clean # ---------------------------------------------------------------------------- diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index 709b74b6..c1349740 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- """Provides non-instrument specific routines for ACE data.""" +import numpy as np + + ackn_str = ' '.join(("Please acknowledge the NASA National Space Science Data", "Center, the Space Physics Data Facility, and the ACE", "Principal Investigator, Edward C. Stone of the", @@ -30,3 +33,21 @@ 'Science Reviews 86, 563–612 (1998).', 'https://doi.org/10.1023/A:1005040232597')) } + + +def clean(self): + """Clean ACE data to the specified level. + + Note + ---- + Basic cleaning to replace fill values with NaN + + """ + + for key in self.variables: + if key != 'time': + fill = self.meta[key, self.meta.labels.fill_val] + # Replace fill with nan + self[key] = self.data[key].where(key != fill) + self.meta[key] = {self.meta.labels.fill_val: np.nan} + return From 255d02c6c8279d2e12000cdffc01943119c892e9 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Tue, 7 Mar 2023 11:19:05 -0500 Subject: [PATCH 146/396] BUG: unused import --- pysatNASA/instruments/ace_epam_l2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index 2d905334..b4e0f6cb 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -34,7 +34,6 @@ import datetime as dt import functools -import numpy as np from pysat.instruments.methods import general as mm_gen From 553b27352617c2208ddc7c084d5c510ef7775428 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 9 Mar 2023 09:56:45 -0500 Subject: [PATCH 147/396] TST: check latest RC --- .github/workflows/pip_rc_install.yml | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/pip_rc_install.yml diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml new file mode 100644 index 00000000..f5bdd267 --- /dev/null +++ b/.github/workflows/pip_rc_install.yml @@ -0,0 +1,29 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Test with latest pysat RC + +on: [push] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.10"] + + name: Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install standard dependencies + run: pip install -r requirements.txt + + - name: Install pysatNASA RC + run: pip install --no-deps -i https://test.pypi.org/simple/ pysatNASA From 5d3391789e4b30e1f3d6b52bc86a1ec200775ab6 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 9 Mar 2023 10:33:57 -0500 Subject: [PATCH 148/396] DOC: rename test --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index f5bdd267..7e855162 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Test with latest pysat RC +name: Test install of latest RC from pip on: [push] From f4ccd25f88808b3228c3c4d1b2030155387066e0 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 9 Mar 2023 10:34:58 -0500 Subject: [PATCH 149/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f601b3cb..211caf84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Remove pandas cap on NEP29 tests * Updated dosctring style for consistency * Removed version cap for xarray + * Added manual workflow to check that latest RC is installable through test pip ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 33d5d06e05390c24fbf95c0e95f0bd18e56e3864 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 9 Mar 2023 10:35:30 -0500 Subject: [PATCH 150/396] TST: set to workflow dispatch --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 7e855162..8062aa7f 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -3,7 +3,7 @@ name: Test install of latest RC from pip -on: [push] +on: [workflow_dispatch] jobs: build: From 163068bb3e20237628e43fd7a416563aa9847d3d Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 9 Mar 2023 10:43:46 -0500 Subject: [PATCH 151/396] DOC: clarify RC workflows --- .github/workflows/pip_rc_install.yml | 3 ++- .github/workflows/pysat_rc.yml | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 8062aa7f..4973e2b3 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -1,4 +1,5 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# This workflow will install Python dependencies and the latest RC of pysatNASA from test pypi. +# This test should be manually run before a pysatNASA RC is officially approved and versioned. # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Test install of latest RC from pip diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index a72b5600..e9c61f6e 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -1,4 +1,6 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# This workflow will install Python dependencies and the latest RC of pysat from test pypi. +# All unit tests for pysatNASA will be run using the pysat RC. +# This test should be manually run before a pysat RC is officially approved and versioned. # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Test with latest pysat RC From d934c0612d703e2a601bc318708fcf3b6db07f5f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 9 Mar 2023 12:48:07 -0500 Subject: [PATCH 152/396] Update .github/workflows/pip_rc_install.yml Co-authored-by: Angeline Burrell --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 4973e2b3..45d835e5 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10"] + python-version: ["3.10"] # Keep this version at the highest supported Python version name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} From bcdd777da1fa7b1fdb9c60edce918e5b6d650240 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 9 Mar 2023 12:57:31 -0500 Subject: [PATCH 153/396] DOC: improve actions comments --- .github/workflows/docs.yml | 4 ++-- .github/workflows/pysat_rc.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 311ef320..b178a061 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,4 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# This workflow will install Python dependencies and check the sphinx build, links in the docs, and the readability of the zenodo file # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Documentation Check @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9"] + python-version: ["3.10"] # Keep this version at the highest supported Python version name: Documentation tests steps: diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index e9c61f6e..c73bcf07 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10"] + python-version: ["3.10"] # Keep this version at the highest supported Python version name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} From f7d23923e9cc3f128fd9e223ba70e2e8af40e5e2 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:09:28 -0700 Subject: [PATCH 154/396] Update pysatNASA/instruments/timed_guvi.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 5171a8ff..da6444b1 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -53,10 +53,10 @@ platform = 'timed' name = 'guvi' -tags = {'high': 'Level 1C high resolution data', - 'low': 'Level 1C low resolution data'} -inst_ids = {'imaging': list(tags.keys()), - 'spectrograph': list(tags.keys())} +tags = {'imaging': 'Level 1C imaging data', + 'spectrograph': 'Level 1C spectrograph data'} +inst_ids = {'high_res': list(tags.keys()), + 'low_res': list(tags.keys())} pandas_format = False From 3586f0d6911576e201d8f831e392d38a9de3a9b6 Mon Sep 17 00:00:00 2001 From: land Date: Thu, 9 Mar 2023 17:09:04 -0700 Subject: [PATCH 155/396] Swapped instruments and tags. Also renaming inst_ids to high_res/low_res following pysat team suggestions. --- pysatNASA/instruments/timed_guvi.py | 66 ++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index da6444b1..9b006683 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -14,11 +14,11 @@ name 'guvi' tag - 'low' - 'high' -inst_id 'imaging' 'spectrograph' +inst_id + 'high_res' + 'low_res' Warnings -------- @@ -64,8 +64,8 @@ # Instrument test attributes _test_dates = { - 'imaging': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}, - 'spectrograph': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}} + 'high_res': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}, + 'low_res': {tag: dt.datetime(2005, 6, 28) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods @@ -122,9 +122,9 @@ def clean(self): '-?????????????_REV??????_', 'Av{{version:02d}}-??r{{revision:03d}}.nc')) -res = {'low': '-2', 'high': ''} +res = {'low_res': '-2', 'high_res': ''} mode = {'imaging': 'IMG', 'spectrograph': 'SPECT'} -supported_tags = {inst: {tag: fname.format(res=res[tag], mode=mode[inst]) +supported_tags = {inst: {tag: fname.format(res=res[inst], mode=mode[tag]) for tag in tags.keys()} for inst in inst_ids} @@ -135,7 +135,7 @@ def clean(self): url = ''.join(('/pub/data/timed/guvi/levels_v13/level1c/{mode:s}/', '{{year:4d}}/{{day:03d}}/')) -supported_urls = {inst: {tag: url.format(mode=inst) +supported_urls = {inst: {tag: url.format(mode=tag) for tag in tags.keys()} for inst in inst_ids} @@ -186,7 +186,7 @@ def load(fnames, tag='', inst_id=''): -------- :: - inst = pysat.Instrument('timed', 'guvi', inst_id='imaging', tag='high') + inst = pysat.Instrument('timed', 'guvi', inst_id='high_res', tag='imaging') inst.load(2005, 179) """ @@ -209,15 +209,15 @@ def get_dt_objects(dataset, tag): return dts # Dimensions for time variables - # night/day along, cross and time are the same imaging low, high, - # spectrograph low, high - # imaging high possess extra dims DayAur - # spectrograph low possess extra dims GAIMDay, GAIMNight + # night/day along, cross and time variables are within + # imaging low_res/high_res and spectrograph high_res/low_res + # imaging high_res also has dims DayAur + # spectrograph low_res also has dims GAIMDay, GAIMNight dims = ['time'] - if 'imag' in inst_id: + if 'imag' in tag: dims = dims + ['time_auroral'] - elif 'spect' in inst_id: - if 'low' in tag: + elif 'spect' in tag: + if 'low' in inst_id: dims = dims + ['time_gaim_day', 'time_gaim_night'] # Separate and concatenate into datasets @@ -229,10 +229,10 @@ def get_dt_objects(dataset, tag): # Collect datetime objects day_dts = np.array(get_dt_objects(data, "DAY")) night_dts = np.array(get_dt_objects(data, "NIGHT")) - if 'imag' in inst_id: + if 'imag' in tag: aur_dts = get_dt_objects(data, "DAY_AURORAL") - elif 'spect' in inst_id: - if 'low' in tag: + elif 'spect' in tag: + if 'low' in inst_id: gaimday_dts = get_dt_objects(data, "GAIM_DAY") gaimnight_dts = get_dt_objects(data, "GAIM_NIGHT") @@ -240,12 +240,12 @@ def get_dt_objects(dataset, tag): data = data.drop_vars(["YEAR_DAY", "DOY_DAY", "TIME_DAY", "TIME_EPOCH_DAY", "YEAR_NIGHT", "DOY_NIGHT", "TIME_NIGHT", "TIME_EPOCH_NIGHT"]) - if 'imag' in inst_id: + if 'imag' in tag: data = data.drop_vars(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", "TIME_DAY_AURORAL", "TIME_EPOCH_DAY_AURORAL"]) - elif 'spect' in inst_id: - if 'low' in tag: + elif 'spect' in tag: + if 'low' in inst_id: data = data.drop_vars(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", "TIME_GAIM_DAY", "TIME_GAIM_NIGHT", "YEAR_GAIM_NIGHT", "DOY_GAIM_NIGHT"]) @@ -261,7 +261,7 @@ def get_dt_objects(dataset, tag): "nAlongNight": "nAlong"}) # 'nCross' dimension only in imaging - if 'imag' in inst_id: + if 'imag' in tag: if data.nCrossDay.size != data.nCrossNight.size: raise Exception("nCrossDay/Night have different dimensions") @@ -271,20 +271,20 @@ def get_dt_objects(dataset, tag): # 'nAlong' will be renamed as 'time' to follow pysat standards data = data.swap_dims({"nAlong": "time"}) - if 'imag' in inst_id: + if 'imag' in tag: data = data.swap_dims({"nAlongDayAur": "time_auroral"}) - elif 'spect' in inst_id: - if 'low' in tag: + elif 'spect' in tag: + if 'low' in inst_id: data = data.swap_dims({"nAlongGAIMDay": "time_gaim_day", "nAlongGAIMNight": "time_gaim_night"}) # Update time variables # night_dts and day_dts are the same temporal array data = data.assign(time=night_dts) - if 'imag' in inst_id: + if 'imag' in tag: data = data.assign(time_auroral=aur_dts) - elif 'spect' in inst_id: - if 'low' in tag: + elif 'spect' in tag: + if 'low' in inst_id: data = data.assign(time_gaim_day=gaimday_dts, time_gaim_night=gaimnight_dts) @@ -322,12 +322,12 @@ def get_dt_objects(dataset, tag): 'PIERCEPOINT_DAY_ALTITUDE', 'PIERCEPOINT_DAY_SZA'] - if 'imag' in inst_id: + if 'imag' in tag: coords = coords + ['PIERCEPOINT_DAY_LATITUDE_AURORAL', 'PIERCEPOINT_DAY_LONGITUDE_AURORAL', 'PIERCEPOINT_DAY_ALTITUDE_AURORAL', 'PIERCEPOINT_DAY_SZA_AURORAL'] - elif 'spect' in inst_id: + elif 'spect' in tag: coords = coords + ['PIERCEPOINT_NIGHT_ZENITH_ANGLE', 'PIERCEPOINT_NIGHT_SAZIMUTH', 'PIERCEPOINT_DAY_ZENITH_ANGLE', @@ -336,14 +336,14 @@ def get_dt_objects(dataset, tag): data = data.set_coords(coords) # Set 'nchan' and 'nCross' as coordinates - if 'imag' in inst_id: + if 'imag' in tag: coords = {"nchan": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong"], "nchanAur": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong"], "nCross": data.nCross.data, "nCrossDayAur": data.nCrossDayAur.data} - elif 'spect' in inst_id: + elif 'spect' in tag: coords = {"nchan": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong", "?"]} From be82857c2b764da5ee462f28fbcac733fdba2cbc Mon Sep 17 00:00:00 2001 From: land Date: Fri, 10 Mar 2023 12:23:41 -0700 Subject: [PATCH 156/396] Adding sdr prefix to tags --- pysatNASA/instruments/timed_guvi.py | 47 +++++++++++++++-------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 9b006683..9b01b8f1 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -14,8 +14,8 @@ name 'guvi' tag - 'imaging' - 'spectrograph' + 'sdr-imaging' + 'sdr-spectrograph' inst_id 'high_res' 'low_res' @@ -29,8 +29,8 @@ :: import pysat - guvi = pysat.Instrument(platform='timed', name='guvi', inst_id='imaging', - tag='low') + guvi = pysat.Instrument(platform='timed', name='guvi', + inst_id='sdr-imaging', tag='low_res') guvi.download(dt.datetime(2005, 6, 28), dt.datetime(2005, 6, 29)) guvi.load(date=dt.datetime(2005, 6, 28)) @@ -53,8 +53,8 @@ platform = 'timed' name = 'guvi' -tags = {'imaging': 'Level 1C imaging data', - 'spectrograph': 'Level 1C spectrograph data'} +tags = {'sdr-imaging': 'Level 1C imaging data', + 'sdr-spectrograph': 'Level 1C spectrograph data'} inst_ids = {'high_res': list(tags.keys()), 'low_res': list(tags.keys())} @@ -123,7 +123,7 @@ def clean(self): 'Av{{version:02d}}-??r{{revision:03d}}.nc')) res = {'low_res': '-2', 'high_res': ''} -mode = {'imaging': 'IMG', 'spectrograph': 'SPECT'} +mode = {'sdr-imaging': 'IMG', 'sdr-spectrograph': 'SPECT'} supported_tags = {inst: {tag: fname.format(res=res[inst], mode=mode[tag]) for tag in tags.keys()} for inst in inst_ids} @@ -186,7 +186,8 @@ def load(fnames, tag='', inst_id=''): -------- :: - inst = pysat.Instrument('timed', 'guvi', inst_id='high_res', tag='imaging') + inst = pysat.Instrument('timed', 'guvi', + inst_id='high_res', tag='sdr-imaging') inst.load(2005, 179) """ @@ -214,9 +215,9 @@ def get_dt_objects(dataset, tag): # imaging high_res also has dims DayAur # spectrograph low_res also has dims GAIMDay, GAIMNight dims = ['time'] - if 'imag' in tag: + if 'sdr-imag' in tag: dims = dims + ['time_auroral'] - elif 'spect' in tag: + elif 'sdr-spect' in tag: if 'low' in inst_id: dims = dims + ['time_gaim_day', 'time_gaim_night'] @@ -229,9 +230,9 @@ def get_dt_objects(dataset, tag): # Collect datetime objects day_dts = np.array(get_dt_objects(data, "DAY")) night_dts = np.array(get_dt_objects(data, "NIGHT")) - if 'imag' in tag: + if 'sdr-imag' in tag: aur_dts = get_dt_objects(data, "DAY_AURORAL") - elif 'spect' in tag: + elif 'sdr-spect' in tag: if 'low' in inst_id: gaimday_dts = get_dt_objects(data, "GAIM_DAY") gaimnight_dts = get_dt_objects(data, "GAIM_NIGHT") @@ -240,11 +241,11 @@ def get_dt_objects(dataset, tag): data = data.drop_vars(["YEAR_DAY", "DOY_DAY", "TIME_DAY", "TIME_EPOCH_DAY", "YEAR_NIGHT", "DOY_NIGHT", "TIME_NIGHT", "TIME_EPOCH_NIGHT"]) - if 'imag' in tag: + if 'sdr-imag' in tag: data = data.drop_vars(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", "TIME_DAY_AURORAL", "TIME_EPOCH_DAY_AURORAL"]) - elif 'spect' in tag: + elif 'sdr-spect' in tag: if 'low' in inst_id: data = data.drop_vars(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", "TIME_GAIM_DAY", "TIME_GAIM_NIGHT", @@ -261,7 +262,7 @@ def get_dt_objects(dataset, tag): "nAlongNight": "nAlong"}) # 'nCross' dimension only in imaging - if 'imag' in tag: + if 'sdr-imag' in tag: if data.nCrossDay.size != data.nCrossNight.size: raise Exception("nCrossDay/Night have different dimensions") @@ -271,9 +272,9 @@ def get_dt_objects(dataset, tag): # 'nAlong' will be renamed as 'time' to follow pysat standards data = data.swap_dims({"nAlong": "time"}) - if 'imag' in tag: + if 'sdr-imag' in tag: data = data.swap_dims({"nAlongDayAur": "time_auroral"}) - elif 'spect' in tag: + elif 'sdr-spect' in tag: if 'low' in inst_id: data = data.swap_dims({"nAlongGAIMDay": "time_gaim_day", "nAlongGAIMNight": "time_gaim_night"}) @@ -281,9 +282,9 @@ def get_dt_objects(dataset, tag): # Update time variables # night_dts and day_dts are the same temporal array data = data.assign(time=night_dts) - if 'imag' in tag: + if 'sdr-imag' in tag: data = data.assign(time_auroral=aur_dts) - elif 'spect' in tag: + elif 'sdr-spect' in tag: if 'low' in inst_id: data = data.assign(time_gaim_day=gaimday_dts, time_gaim_night=gaimnight_dts) @@ -322,12 +323,12 @@ def get_dt_objects(dataset, tag): 'PIERCEPOINT_DAY_ALTITUDE', 'PIERCEPOINT_DAY_SZA'] - if 'imag' in tag: + if 'sdr-imag' in tag: coords = coords + ['PIERCEPOINT_DAY_LATITUDE_AURORAL', 'PIERCEPOINT_DAY_LONGITUDE_AURORAL', 'PIERCEPOINT_DAY_ALTITUDE_AURORAL', 'PIERCEPOINT_DAY_SZA_AURORAL'] - elif 'spect' in tag: + elif 'sdr-spect' in tag: coords = coords + ['PIERCEPOINT_NIGHT_ZENITH_ANGLE', 'PIERCEPOINT_NIGHT_SAZIMUTH', 'PIERCEPOINT_DAY_ZENITH_ANGLE', @@ -336,14 +337,14 @@ def get_dt_objects(dataset, tag): data = data.set_coords(coords) # Set 'nchan' and 'nCross' as coordinates - if 'imag' in tag: + if 'sdr-imag' in tag: coords = {"nchan": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong"], "nchanAur": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong"], "nCross": data.nCross.data, "nCrossDayAur": data.nCrossDayAur.data} - elif 'spect' in tag: + elif 'sdr-spect' in tag: coords = {"nchan": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong", "?"]} From 4da322681ff46eec274f7cb2e737312838caff2f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 14 Mar 2023 15:36:39 -0400 Subject: [PATCH 157/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysatNASA/instruments/ace_mag_l2.py | 1 + pysatNASA/instruments/ace_swepam_l2.py | 1 - pysatNASA/instruments/methods/ace.py | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index ebc9d72e..fcd07f38 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -54,6 +54,7 @@ '5min': ['key'], '1hr': ['base', 'key']} pandas_format = False + # ---------------------------------------------------------------------------- # Instrument test attributes diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index e55f2cda..987a7048 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -69,7 +69,6 @@ # Use default ace clean clean = mm_ace.clean - # ---------------------------------------------------------------------------- # Instrument functions # diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index c1349740..91f0a3a0 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -47,7 +47,9 @@ def clean(self): for key in self.variables: if key != 'time': fill = self.meta[key, self.meta.labels.fill_val] + # Replace fill with nan - self[key] = self.data[key].where(key != fill) + fill_mask = self[key] == fill + self[key] = self.data[key].where(~fill_mask) self.meta[key] = {self.meta.labels.fill_val: np.nan} return From 29983ebb55eda2986a92bd5c172736b65181bccd Mon Sep 17 00:00:00 2001 From: land Date: Tue, 14 Mar 2023 14:45:04 -0600 Subject: [PATCH 158/396] Fixed bug in supported_urls for new tags --- pysatNASA/instruments/timed_guvi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 9b01b8f1..706896e8 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -135,7 +135,7 @@ def clean(self): url = ''.join(('/pub/data/timed/guvi/levels_v13/level1c/{mode:s}/', '{{year:4d}}/{{day:03d}}/')) -supported_urls = {inst: {tag: url.format(mode=tag) +supported_urls = {inst: {tag: url.format(mode=tag.split('-')[1]) for tag in tags.keys()} for inst in inst_ids} From 37d50f0518730c7d52920afbb51164e2738dd88c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 15 Mar 2023 09:57:06 -0400 Subject: [PATCH 159/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysatNASA/instruments/ace_mag_l2.py | 1 - pysatNASA/instruments/ace_swepam_l2.py | 1 - pysatNASA/instruments/methods/ace.py | 1 - 3 files changed, 3 deletions(-) diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index fcd07f38..6e43be50 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -64,7 +64,6 @@ # ---------------------------------------------------------------------------- # Instrument methods - # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index 987a7048..d570cf7a 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -62,7 +62,6 @@ # ---------------------------------------------------------------------------- # Instrument methods - # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index 91f0a3a0..69e70e7e 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -3,7 +3,6 @@ import numpy as np - ackn_str = ' '.join(("Please acknowledge the NASA National Space Science Data", "Center, the Space Physics Data Facility, and the ACE", "Principal Investigator, Edward C. Stone of the", From 87517461301aa15cfb88ebce133152e359376e60 Mon Sep 17 00:00:00 2001 From: land Date: Wed, 15 Mar 2023 18:14:21 -0600 Subject: [PATCH 160/396] Adding tlimb and tdisk. --- pysatNASA/instruments/ses14_gold.py | 68 +++++++++++++++-------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 5a89413f..62385dd9 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -11,6 +11,8 @@ 'gold' tag 'nmax' + 'tlimb' + 'tdisk' Warnings -------- @@ -33,7 +35,9 @@ import datetime as dt import functools +import pysat import numpy as np +import xarray as xr from pysat.instruments.methods import general as ps_gen from pysat import logger @@ -48,15 +52,19 @@ platform = 'ses14' name = 'gold' -tags = {'nmax': 'Level 2 Nmax data for the GOLD instrument'} -inst_ids = {'': ['nmax']} +tags = {'nmax': 'Level 2 Nmax data for the GOLD instrument', + 'tlimb': 'Level 2 Tlimb data for the GOLD instrument', + 'tdisk': 'Level 2 Tdisk data for the GOLD instrument'} +inst_ids = {'': ['nmax','tlimb','tdisk']} pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'nmax': dt.datetime(2020, 1, 1)}} +_test_dates = {'': {'nmax': dt.datetime(2020, 1, 1)}, + 'tlimb': dt.datetime(2020, 1, 1), + 'tdisk': dt.datetime(2020, 1, 1)} # ---------------------------------------------------------------------------- # Instrument methods @@ -168,36 +176,30 @@ def load(fnames, tag='', inst_id=''): 'min_val': ('Valid_Min', np.float64), 'max_val': ('Valid_Max', np.float64), 'fill_val': ('fill', np.float64)} + meta = pysat.Meta(labels=labels) - # Generate custom meta translation table. When left unspecified the default - # table handles the multiple values for fill. We must recreate that - # functionality in our table. The targets for meta_translation should - # map to values in `labels` above. - meta_translation = {'FIELDNAM': 'plot', 'LABLAXIS': 'axis', - 'ScaleTyp': 'scale', 'VALIDMIN': 'Valid_Min', - 'Valid_Min': 'Valid_Min', 'VALIDMAX': 'Valid_Max', - 'Valid_Max': 'Valid_Max', '_FillValue': 'fill', - 'FillVal': 'fill', 'TIME_BASE': 'time_base'} - - data, meta = load_netcdf(fnames, pandas_format=pandas_format, - epoch_name='nscans', labels=labels, - meta_translation=meta_translation, - drop_meta_labels='FILLVAL') - - if tag == 'nmax': - # Add time coordinate from scan_start_time - data['time'] = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") - for val in data['scan_start_time'].values] - - # Update coordinates with dimensional data - data = data.assign_coords({'nlats': data['nlats'], - 'nlons': data['nlons'], - 'nmask': data['nmask'], - 'channel': data['channel'], - 'hemisphere': data['hemisphere']}) - meta['time'] = {meta.labels.notes: 'Converted from scan_start_time'} - meta['nlats'] = {meta.labels.notes: 'Index for latitude values'} - meta['nlons'] = {meta.labels.notes: 'Index for longitude values'} - meta['nmask'] = {meta.labels.notes: 'Index for mask values'} + datos = [] + + for path in fnames: + + data = xr.load_dataset(path) + + if tag in ['nmax', 'tlimb', 'tdisk']: + # Add time coordinate from scan_start_time + data['time'] = ('nscans', + [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") + for val in data['scan_start_time'].values]) + data = data.swap_dims({'nscans': 'time'}) + + # Update coordinates with dimensional data + data = data.assign_coords({'nlats': data['nlats'], + 'nlons': data['nlons'], + 'nmask': data['nmask'], + 'channel': data['channel'], + 'hemisphere': data['hemisphere']}) + + datos.append(data) + + data = xr.concat(datos, dim='time') return data, meta From 461c6433589acbf52a1fae8542d2b8e7e999d7ea Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 16 Mar 2023 13:40:52 -0400 Subject: [PATCH 161/396] STY: pep8 --- pysatNASA/instruments/timed_guvi.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index f0968689..f0b53652 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -61,12 +61,10 @@ import datetime as dt import functools import numpy as np -import warnings import xarray as xr import pysat from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa @@ -91,7 +89,7 @@ # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {iid: {tag: dt.datetime(2005, 6, 28) for tag in inst_ids[iid] +_test_dates = {iid: {tag: dt.datetime(2005, 6, 28) for tag in inst_ids[iid]} for iid in inst_ids.keys()} # ---------------------------------------------------------------------------- @@ -114,7 +112,7 @@ '??????_Av{{version:02d}}-??r{{revision:03d}}.nc')) file_lvl = {'low_res': 'L1C-2-disk', 'high_res': 'L1C-disk', '': 'L2B'} mode = {'sdr-imaging': '-IMG', 'sdr-spectrograph': '-SPECT', - 'edr-aur': '-edr-aur-IMG'} + 'edr-aur': '-edr-aur-IMG'} supported_tags = {inst_id: {tag: fname.format(lvl=file_lvl[inst_id], mode=mode[tag]) for tag in tags.keys()} @@ -129,7 +127,8 @@ url_mode = {tag: 'imaging' if tag == 'edr-aur' else tag.split('-')[1] for tag in tags.keys()} download_tags = { - sat_id: {tag: {'remote_dir': url.format(lvl=url_lvl[tag], url_mode[tag]) + sat_id: {tag: {'remote_dir': url.format(lvl=url_lvl[tag], + mode=url_mode[tag]), 'fname': fname.format(lvl=file_lvl[sat_id], mode=mode[tag])} for tag in tags.keys()} for sat_id in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) @@ -237,7 +236,8 @@ def get_dt_objects(dataset, tag): "YEAR_GAIM_NIGHT", "DOY_GAIM_NIGHT"]) if day_dts.size != night_dts.size: - raise Exception("nAlongDay & nAlongNight have different dimensions") + raise Exception(" ".join(("nAlongDay & nAlongNight have", + "different dimensions"))) if np.any(day_dts != night_dts): raise Exception("time along day and night are different") @@ -262,7 +262,8 @@ def get_dt_objects(dataset, tag): elif 'sdr-spect' in tag: if 'low' in inst_id: data = data.swap_dims({"nAlongGAIMDay": "time_gaim_day", - "nAlongGAIMNight": "time_gaim_night"}) + "nAlongGAIMNight": + "time_gaim_night"}) # Update time variables # night_dts and day_dts are the same temporal array @@ -294,7 +295,7 @@ def get_dt_objects(dataset, tag): inners = jnners else: inners = {dim: xr.concat([inners[dim], jnners[dim]], dim=dim) - for dim in dims} + for dim in dims} data = xr.merge([inners[dim] for dim in dims]) From 862e61e474232a82c541fb6a7eea11d725bec247 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 17 Mar 2023 10:01:48 -0400 Subject: [PATCH 162/396] BUG: fix monthly cadence --- pysatNASA/instruments/methods/cdaweb.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 8e02a7cb..b068684a 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -9,6 +9,7 @@ import cdflib import datetime as dt +import numpy as np import os import pandas as pds import requests @@ -110,6 +111,7 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id, epoch_name=epoch_name, + file_cadence=file_cadence, meta_processor=meta_processor, meta_translation=meta_translation, drop_meta_labels=drop_meta_labels) @@ -224,6 +226,7 @@ def load_pandas(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), def load_xarray(fnames, tag='', inst_id='', + file_cadence=dt.timedelta(days=1), labels={'units': ('units', str), 'name': ('long_name', str), 'notes': ('notes', str), 'desc': ('desc', str), 'plot': ('plot_label', str), 'axis': ('axis', str), @@ -243,6 +246,11 @@ def load_xarray(fnames, tag='', inst_id='', Data product tag (default='') inst_id : str Instrument ID (default='') + file_cadence : dt.timedelta or pds.DateOffset + pysat assumes a daily file cadence, but some instrument data files + contain longer periods of time. This parameter allows the specification + of regular file cadences greater than or equal to a day (e.g., weekly, + monthly, or yearly). (default=dt.timedelta(days=1)) labels : dict Dict where keys are the label attribute names and the values are tuples that have the label values and value types in that order. @@ -308,7 +316,14 @@ def load_xarray(fnames, tag='', inst_id='', # metadata for pysat using some assumptions. Depending upon your needs # the resulting pandas DataFrame may need modification. ldata = [] - for lfname in fnames: + + # Find unique files for monthly / yearly cadence. + # Arbitrary timestamp needed for comparison. + t0 = dt.datetime(2009, 1, 1) + if (t0 + file_cadence) > (t0 + dt.timedelta(days=1)): + lfnames = list(np.unique([fname[:-11] for fname in fnames])) + + for lfname in lfnames: temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True) ldata.append(temp_data) From e5f51fd81f7201c06bc623c003ce1f421f43afa8 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Fri, 17 Mar 2023 15:01:11 -0400 Subject: [PATCH 163/396] cdas_list_remote_files can now output a pandas series for download_updated_files compatiblity --- pysatNASA/instruments/methods/cdaweb.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 9528acdf..c63c6e5b 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -558,7 +558,8 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, stop = date_array[-1] remote_files = cdas_list_remote_files(tag=tag, inst_id=inst_id, start=start, stop=stop, - supported_tags=supported_tags) + supported_tags=supported_tags, + series_out=False) for file in remote_files: @@ -750,7 +751,7 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, - supported_tags=None): + supported_tags=None, series_out=True): """Return a list of every file for chosen remote data. This routine is intended to be used by pysat instrument modules supporting @@ -775,6 +776,9 @@ def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, a dict with 'remote_dir', 'fname'. Inteded to be pre-set with functools.partial then assigned to new instrument code. (default=None) + series_out : bool + boolean to determine output type. True for pandas series, and False for + list. Returns ------- @@ -813,6 +817,12 @@ def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, stop = stop.tz_localize('utc') og_files = cdas.get_original_files(dataset=dataset, start=start, end=stop) - file_list = [f['Name'] for f in og_files[1]] + + if series_out: + name_list = [os.path.basename(f['Name']) for f in og_files[1]] + t_stamp = [pds.Timestamp(f['StartTime'][:10]) for f in og_files[1]] + file_list = pds.Series(data=name_list, index=t_stamp) + else: + file_list = [f['Name'] for f in og_files[1]] return file_list From 2c9cfec4b806cdf5dd141ec918652fa3c4a71dc4 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Fri, 17 Mar 2023 15:01:50 -0400 Subject: [PATCH 164/396] added test for new cdas_list_remote_files keyward argument --- pysatNASA/tests/test_methods_cdaweb.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index 8bd7cbba..1b593f5b 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -95,3 +95,18 @@ def test_remote_file_list_all(self, start, stop): files = self.test_inst.remote_file_list(start, stop) assert len(files) > 0 return + + @pytest.mark.parametrize("series_out", [(True), (False)]) + def test_cdas_remote_files(self, series_out): + """Test that cdas_list_remote_files can return pandas series.""" + start = dt.datetime(2009, 1, 1) + stop = dt.datetime(2009, 1, 2) + self.module = pysatNASA.instruments.cnofs_plp + self.test_inst = pysat.Instrument(inst_module=self.module) + files = self.test_inst.remote_file_list(start, stop, + series_out=series_out) + if series_out is True: + assert isinstance(files, pds.Series) + else: + assert isinstance(files, list) + return From 0d4c7eb5eaf1cbf0b59ec84ebeceea2ab36614b8 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 17 Mar 2023 15:47:39 -0400 Subject: [PATCH 165/396] ENH: created new JHU APL functions Created new functions to streamline the JHU APL file loading. --- pysatNASA/instruments/methods/jhuapl.py | 363 +++++++++++++++++++++--- 1 file changed, 318 insertions(+), 45 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index e12bdade..8c71de95 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -9,6 +9,132 @@ from pysat.utils.io import load_netcdf +def build_dtimes(data, var, epoch=None, epoch_var='time'): + """Build datetime objects from standard JPL time variables. + + Parameters + ---------- + data : xr.Dataset + Xarray dataset with time variables + var : str + Common string to identify desired year, day of year, and seconds of day + epoch : dt.datetime or NoneType + Epoch to subtract from data or NoneType to get seconds of day from + `data` (default=None) + epoch_var : str + Epoch variable containing time data that seconds of day will be + obtained from if `epoch` != None (default='time') + + Returns + ------- + dtimes : list-like + List of datetime objects + + """ + ykey = 'YEAR{:s}'.format(var) + dkey = 'DOY{:s}'.format(var) + skey = 'TIME{:s}'.format(var) + + if epoch is None: + hours = [int(np.floor(sec / 3600.0)) for sec in data[skey].values] + mins = [int(np.floor((sec - hours[i] * 3600) / 60.0)) + for i, sec in enumerate(data[skey].values)] + secs = [int(np.floor((sec - hours[i] * 3600 - mins[i] * 60))) + for i, sec in enumerate(data[skey].values)] + dtimes = [ + dt.datetime.strptime( + "{:4d}-{:03d}-{:02d}-{:02d}-{:02d}-{:06.0f}".format( + int(data[ykey].values[i]), int(data[dkey].values[i]), + hours[i], mins[i], secs[i], + (sec - hours[i] * 3600 - mins[i] * 60 - secs[i]) * 1.0e6), + '%Y-%j-%H-%M-%S-%f') + for i, sec in enumerate(data[skey].values)] + else: + dtimes = [ + dt.datetime.strptime("{:4d}-{:03d}".format( + int(data[ykey].values[i]), int(data[dkey].values[i])), '%Y-%j') + + (pds.to_datetime(etime).to_pydatetime() - epoch) + for i, etime in enumerate(data[epoch_var].values)] + + return dtimes + + +def expand_coords(data_list, mdata, dims_equal=False): + """Ensure that dimensions do not vary from file to file. + + Parameters + ---------- + data_list : list-like + List of xr.Dataset objects with the same dimensions and variables + mdata : pysat.Meta + Metadata for the data in `data_list` + dims_equal : bool + Assert that all xr.Dataset objects have the same dimensions if True + (default=False) + + Returns + ------- + out_list : list-like + List of xr.Dataset objects with the same dimensions and variables, + now with dimensions that all have the same values and data padded + when needed. + + """ + # Get a list of all the dimensions + if dims_equal: + dims = list(data_list[0].dims.keys()) if len(data_list) > 0 else [] + else: + dims = list() + for sdata in data_list: + if len(dims) == 0: + dims = list(sdata.dims.keys()) + else: + for dim in list(sdata.dims.keys()): + if dim not in dims: + dims.append(dim) + + # After loading all the data, determine which dimensions may need to be + # expanded, as they could differ in dimensions from file to file + combo_dims = {dim: max([sdata.dims[dim] for sdata in data_list + if dim in sdata.dims]) for dim in dims} + + # Expand the data so that all dimensions are the same shape + out_list = list() + for i, sdata in enumerate(data_list): + # Determine which dimensions need to be updated + fix_dims = [dim for dim in sdata.dims.keys() + if sdata.dims[dim] < combo_dims[dim]] + + new_data = {} + update_new = False + for dvar in sdata.data_vars.keys(): + # See if any dimensions need to be updated + update_dims = list(set(sdata[dvar].dims) & set(fix_dims)) + + # Save the old data as is, or pad it to have the right dims + if len(update_dims) > 0: + update_new = True + new_shape = list(sdata[dvar].values.shape) + old_slice = [slice(0, ns) for ns in new_shape] + + for dim in update_dims: + idim = list(sdata[dvar].dims).index(dim) + new_shape[idim] = combo_dims[dim] + + # Set the new data for output + new_dat = np.full(shape=new_shape, fill_value=mdata[ + dvar, mdata.labels.fill_val]) + new_dat[tuple(old_slice)] = sdata[dvar].values + new_data[dvar] = (sdata[dvar].dims, new_dat) + else: + new_data[dvar] = sdata[dvar] + + # Get the updated dataset + out_list.append(xr.Dataset(new_data) if update_new else sdata) + + return out_list + + def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): """Load JHU APL EDR Aurora data and meta data. @@ -53,44 +179,8 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): # There are multiple files per day, with time as a variable rather # than a dimension or coordinate. Additionally, no coordinates # are assigned. - subday_data, mdata = load_netcdf(fname, epoch_name='TIME', - epoch_unit='s', labels=labels, - pandas_format=pandas_format) - single_data.append(subday_data) - - # After loading all the data, determine which dimensions need to be expanded - combo_dims = {dim: max([sdata.dims[dim] for sdata in single_data]) - for dim in subday_data.dims.keys()} - - # Expand the data so that all dimensions are the same shape - for i, sdata in enumerate(single_data): - # Determine which dimensions need to be updated - fix_dims = [dim for dim in sdata.dims.keys() - if sdata.dims[dim] < combo_dims[dim]] - - new_data = {} - update_new = False - for dvar in sdata.data_vars.keys(): - # See if any dimensions need to be updated - update_dims = list(set(sdata[dvar].dims) & set(fix_dims)) - - # Save the old data as is, or pad it to have the right dims - if len(update_dims) > 0: - update_new = True - new_shape = list(sdata[dvar].values.shape) - old_slice = [slice(0, ns) for ns in new_shape] - - for dim in update_dims: - idim = list(sdata[dvar].dims).index(dim) - new_shape[idim] = combo_dims[dim] - - # Set the new data for output - new_dat = np.full(shape=new_shape, fill_value=mdata[ - dvar, mdata.labels.fill_val]) - new_dat[tuple(old_slice)] = sdata[dvar].values - new_data[dvar] = (sdata[dvar].dims, new_dat) - else: - new_data[dvar] = sdata[dvar] + sdata, mdata = load_netcdf(fname, epoch_name='TIME', epoch_unit='s', + labels=labels, pandas_format=pandas_format) # Calculate the time for this data file. The pysat `load_netcdf` routine # converts the 'TIME' parameter (seconds of day) into datetime using @@ -102,18 +192,201 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): pds.to_datetime(sdata['time'].values).to_pydatetime() - dt.datetime(1970, 1, 1)) - # Get the updated dataset - ndata = xr.Dataset(new_data) if update_new else sdata - ndata['time'] = ftime - # Assign a datetime variable, making indexing possible - single_data[i] = ndata.assign_coords( - {'time': ndata['time']}).expand_dims(dim='time') + sdata['time'] = ftime + sdata = sdata.assign_coords( + {'time': sdata['time']}).expand_dims(dim='time') + + # Save the data in the file list + single_data.append(sdata) + + # Update the meta data + # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by + # removing 'TIME', once possible + for var in mdata.keys(): + # Update the fill value, using information from the global header + mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} + + # After loading all the data, determine which dimensions need to be + # expanded. Pad the data so that all dimensions are the same shape + single_data = expand_coords(single_data, mdata, dims_equal=True) # Combine all the data, indexing along time data = xr.combine_by_coords(single_data) + return data, mdata + + +def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, + combine_times=False): + """Load JHU APL SDR data and meta data. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + tag : str + Tag name used to identify particular data set to be loaded (default='') + inst_id : str + Instrument ID name used to identify different instrument carriers + (default='') + pandas_format : bool + False for xarray format, True for pandas (default=False) + combine_times : bool + For SDR data, optionally combine the different datetime coordinates + into a single time coordinate (default=False) + + Returns + ------- + data : pds.DataFrame or xr.Dataset + Data to be assigned to the pysat.Instrument.data object. + mdata : pysat.Meta + Pysat Meta data for each data variable. + + Note + ---- + Logger warning 'Epoch label: TIME is not a dimension.' is raised due to + the data format and pysat file expectations. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'guvi', tag='edr-aur') + inst.load(2003, 1) + + """ + # Define the input variables and working variables + labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} + load_time = 'TIME_DAY' + time_vars = ['YEAR_DAY', 'DOY_DAY', 'TIME_EPOCH_DAY', 'YEAR_NIGHT', + 'DOY_NIGHT', 'TIME_NIGHT', 'TIME_EPOCH_NIGHT'] + coords = ['PIERCEPOINT_NIGHT_LATITUDE', 'PIERCEPOINT_NIGHT_LONGITUDE', + 'PIERCEPOINT_NIGHT_ALTITUDE', 'PIERCEPOINT_NIGHT_SZA', + 'PIERCEPOINT_DAY_LATITUDE', 'PIERCEPOINT_DAY_LONGITUDE', + 'PIERCEPOINT_DAY_ALTITUDE', 'PIERCEPOINT_DAY_SZA'] + time_dims = ['time'] + + if tag == 'sdr-imaging': + time_vars.extend(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", + "TIME_DAY_AURORAL", "TIME_EPOCH_DAY_AURORAL"]) + coords.extend(['PIERCEPOINT_DAY_LATITUDE_AURORAL', + 'PIERCEPOINT_DAY_LONGITUDE_AURORAL', + 'PIERCEPOINT_DAY_ALTITUDE_AURORAL', + 'PIERCEPOINT_DAY_SZA_AURORAL']) + time_dims.append('time_auroral') + elif tag == 'sdr-spectrograph': + time_vars.extend(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", "TIME_GAIM_DAY", + "TIME_GAIM_NIGHT", "YEAR_GAIM_NIGHT", + "DOY_GAIM_NIGHT"]) + coords.extend(['PIERCEPOINT_NIGHT_ZENITH_ANGLE', + 'PIERCEPOINT_NIGHT_SAZIMUTH', + 'PIERCEPOINT_DAY_ZENITH_ANGLE', + 'PIERCEPOINT_DAY_SAZIMUTH']) + time_dims.extend(['time_gaim_day', 'time_gaim_night']) + + # CDAWeb stores these files in the NetCDF format instead of the CDF format + inners = None + for fname in fnames: + # There are multiple files per day, with time as a variable rather + # than a dimension or coordinate. Additionally, no coordinates + # are assigned. + sdata, mdata = load_netcdf(fname, epoch_name=load_time, epoch_unit='s', + labels=labels, pandas_format=pandas_format) + + # Calculate the time for this data file. The pysat `load_netcdf` routine + # converts the 'TIME' parameter (seconds of day) into datetime using + # the UNIX epoch as the date offset + ftime = build_dtimes(sdata, '_DAY', dt.datetime(1970, 1, 1)) + + # Ensure identical day and night dimensions + if sdata.dims['nAlongDay'] != sdata.dims['nAlongNight']: + raise ValueError('Along-track day and night dimensions differ') + + if sdata.dims['nCrossDay'] != sdata.dims['nCrossNight']: + raise ValueError('Cross-track day and night dimensions differ') + + # Combine identical dimensions and rename 'nAlong' to 'time' + sdata = sdata.rename_dims({'nAlongDay': 'nAlong', + 'nAlongNight': 'nAlong', + 'nCrossDay': 'nCross', + 'nCrossNight': 'nCross'}) + + if tag == 'sdr-imaging': + sdata = sdata.rename_dims({'nAlongDayAur': 'time_auroral'}) + sdata = sdata.assign(time_auroral=build_dtimes( + sdata, '_DAY_AURORAL')) + elif tag == 'sdr-spectrograph' and inst_id == 'low_res': + sdata = sdata.rename_dims({'nAlongGAIMDay': 'time_gaim_day', + 'nAlongGAIMNight': 'time_gaim_night'}) + sdata = sdata.assign(time_gaim_day=build_dtimes( + sdata, '_GAIM_DAY'), time_gaim_night=build_dtimes( + sdata, '_GAIM_NIGHT')) + + # Test that day and night times are consistent to the nearest second + for i, ntime in enumerate(build_dtimes(sdata, '_NIGHT')): + if abs(ntime - ftime[i]).total_seconds() > 1.0: + raise ValueError('Day and night times differ') + + # Remove redundant time variables and rname the 'nAlong' dimension + sdata = sdata.drop_vars(time_vars).swap_dims({'nAlong': 'time'}) + + # Assign time as a coordinate for combining files indexing + sdata['time'] = ftime + + # Separate into inner datasets + inner_keys = {dim: [key for key in sdata.keys() + if dim in sdata[key].dims] for dim in time_dims} + inner_dat = {dim: sdata.get(inner_keys[dim]) for dim in time_dims} + + # Add 'single_var's into 'time' dataset to keep track + sv_keys = [val.name for val in sdata.values() + if 'single_var' in val.dims] + singlevar_set = sdata.get(sv_keys) + inner_dat['time'] = xr.merge([inner_dat['time'], singlevar_set]) + + # Concatenate along desired dimension with previous files' data + if inners is None: + # No previous data, assign the data separated by dimension + inners = dict(inner_dat) + else: + # Concatenate with existing data + inners = {dim: xr.concat([inners[dim], inner_dat[dim]], dim=dim) + for dim in time_dims} + + # Update the meta data # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by - # removing 'TIME', once possible + # removing dimensions and time, once possible + for var in mdata.keys(): + # Update the fill value, using information from the global header + mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} + + # Combine all time dimensions + if combine_times: + data_list = expand_coords([inners[dim] if dim == 'time' else + inners[dim].rename_dims({dim: 'time'}) + for dim in time_dims], mdata, + dims_equal=False) + else: + data_list = [inners[dim] for dim in time_dims] + + # Combine all the data, indexing along time + data = xr.merge(data_list) + + # Set additional coordinates + data = data.set_coords(coords).assign_coords({'time': data['time']}) + if tag == 'sdr-imaging': + data = data.assign_coords( + {'nchan': ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong"], + "nchanAur": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", + "LBHlong"], + "nCross": sdata.nCross.data, + "nCrossDayAur": sdata.nCrossDayAur.data}) + elif tag == 'sdr-spectrograph': + data = data.assign_coords({"nchan": ["121.6nm", "130.4nm", "135.6nm", + "LBHshort", "LBHlong", "?"]}) + + # Ensure the data is ordered correctly + data = data.sortby('time') return data, mdata From de20e90acd4f8b50875eb2f2c9c7fa6044b01e36 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 17 Mar 2023 15:50:27 -0400 Subject: [PATCH 166/396] STY: streamlined GUVI loading Streamlined GUVI file loading, added a custom keyword argument that will combine the different time indices by padding the data. --- pysatNASA/instruments/timed_guvi.py | 185 +++------------------------- 1 file changed, 20 insertions(+), 165 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index f0b53652..294e0586 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -5,7 +5,7 @@ Ionosphere Mesosphere Energetics Dynamics (TIMED) satellite data from the NASA Coordinated Data Analysis Web (CDAWeb). -From JHU APL (L2 Data): +From JHU APL: The Global Ultraviolet Imager (GUVI) is one of four instruments that constitute the TIMED spacecraft, the first mission of the NASA Solar Connections program. @@ -84,7 +84,7 @@ 'low_res': ['sdr-imaging', 'sdr-spectrograph']} pandas_format = False -multi_file_day = True # This is only true for EDR-AUR +multi_file_day = True # ---------------------------------------------------------------------------- # Instrument test attributes @@ -124,13 +124,13 @@ '{{year:4d}}/{{day:03d}}/')) url_lvl = {'sdr-imaging': 'level1c', 'sdr-spectrograph': 'level1c', 'edr-aur': 'level2b'} -url_mode = {tag: 'imaging' if tag == 'edr-aur' else tag.split('-')[1] +url_mode = {tag: 'imaging/edr-aur' if tag == 'edr-aur' else tag.split('-')[1] for tag in tags.keys()} -download_tags = { - sat_id: {tag: {'remote_dir': url.format(lvl=url_lvl[tag], - mode=url_mode[tag]), - 'fname': fname.format(lvl=file_lvl[sat_id], mode=mode[tag])} - for tag in tags.keys()} for sat_id in inst_ids.keys()} +download_tags = {iid: {tag: {'remote_dir': url.format(lvl=url_lvl[tag], + mode=url_mode[tag]), + 'fname': fname.format(lvl=file_lvl[iid], + mode=mode[tag])} + for tag in tags.keys()} for iid in inst_ids.keys()} download = functools.partial(cdw.download, supported_tags=download_tags) # Set the list_remote_files routine @@ -139,7 +139,7 @@ # Set the load routine -def load(fnames, tag='', inst_id=''): +def load(fnames, tag='', inst_id='', combine_times=False): """Load TIMED GUVI data into `xarray.DataSet` and `pysat.Meta` objects. This routine is called as needed by pysat. It is not intended @@ -156,6 +156,9 @@ def load(fnames, tag='', inst_id=''): inst_id : str Satellite ID used to identify particular data set to be loaded. This input is nominally provided by pysat itself. + combine_times : bool + For SDR data, optionally combine the different datetime coordinates + into a single time coordinate (default=False) Returns ------- @@ -164,6 +167,11 @@ def load(fnames, tag='', inst_id=''): meta : pysat.Meta Metadata formatted for a pysat.Instrument object. + Raises + ------ + ValueError + If temporal dimensions are not consistent + Note ---- Any additional keyword arguments passed to pysat.Instrument @@ -182,161 +190,8 @@ def load(fnames, tag='', inst_id=''): data, meta = jhuapl.load_edr_aurora(fnames, tag, inst_id, pandas_format=pandas_format) else: - meta = pysat.Meta() - - def get_dt_objects(dataset, tag): - dts = [] - for i, year in enumerate(dataset['YEAR_%s' % tag].data): - idt = dt.datetime(year, 1, 1) + dt.timedelta( - days=float(dataset['DOY_%s' % tag].data[i] - 1), - seconds=float(dataset['TIME_%s' % tag].data[i])) - dts.append(idt) - return dts - - # Dimensions for time variables - # night/day along, cross and time variables are within - # imaging low_res/high_res and spectrograph high_res/low_res - # imaging high_res also has dims DayAur - # spectrograph low_res also has dims GAIMDay, GAIMNight - dims = ['time'] - if 'sdr-imag' in tag: - dims = dims + ['time_auroral'] - elif 'sdr-spect' in tag: - if 'low' in inst_id: - dims = dims + ['time_gaim_day', 'time_gaim_night'] - - # Separate and concatenate into datasets - inners = {dim: None for dim in dims} - - for path in fnames: - data = xr.open_dataset(path, chunks='auto') - - # Collect datetime objects - day_dts = np.array(get_dt_objects(data, "DAY")) - night_dts = np.array(get_dt_objects(data, "NIGHT")) - if 'sdr-imag' in tag: - aur_dts = get_dt_objects(data, "DAY_AURORAL") - elif 'sdr-spect' in tag: - if 'low' in inst_id: - gaimday_dts = get_dt_objects(data, "GAIM_DAY") - gaimnight_dts = get_dt_objects(data, "GAIM_NIGHT") - - # Drop out redundant time variables - data = data.drop_vars(["YEAR_DAY", "DOY_DAY", "TIME_DAY", - "TIME_EPOCH_DAY", "YEAR_NIGHT", "DOY_NIGHT", - "TIME_NIGHT", "TIME_EPOCH_NIGHT"]) - if 'sdr-imag' in tag: - data = data.drop_vars(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", - "TIME_DAY_AURORAL", - "TIME_EPOCH_DAY_AURORAL"]) - elif 'sdr-spect' in tag: - if 'low' in inst_id: - data = data.drop_vars(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", - "TIME_GAIM_DAY", "TIME_GAIM_NIGHT", - "YEAR_GAIM_NIGHT", "DOY_GAIM_NIGHT"]) - - if day_dts.size != night_dts.size: - raise Exception(" ".join(("nAlongDay & nAlongNight have", - "different dimensions"))) - - if np.any(day_dts != night_dts): - raise Exception("time along day and night are different") - - # Renaming along/cross dimensions - data = data.rename_dims({"nAlongDay": "nAlong", - "nAlongNight": "nAlong"}) - - # 'nCross' dimension only in imaging - if 'sdr-imag' in tag: - - if data.nCrossDay.size != data.nCrossNight.size: - raise Exception("nCrossDay/Night have different dimensions") - - data = data.rename_dims({"nCrossDay": "nCross", - "nCrossNight": "nCross"}) - - # 'nAlong' will be renamed as 'time' to follow pysat standards - data = data.swap_dims({"nAlong": "time"}) - if 'sdr-imag' in tag: - data = data.swap_dims({"nAlongDayAur": "time_auroral"}) - elif 'sdr-spect' in tag: - if 'low' in inst_id: - data = data.swap_dims({"nAlongGAIMDay": "time_gaim_day", - "nAlongGAIMNight": - "time_gaim_night"}) - - # Update time variables - # night_dts and day_dts are the same temporal array - data = data.assign(time=night_dts) - if 'sdr-imag' in tag: - data = data.assign(time_auroral=aur_dts) - elif 'sdr-spect' in tag: - if 'low' in inst_id: - data = data.assign(time_gaim_day=gaimday_dts, - time_gaim_night=gaimnight_dts) - - # Separate into inner datasets - inner_keys = {dim: [] for dim in dims} - - for dim in dims: - for key in data.keys(): - if dim in data[key].dims: - inner_keys[dim].append(key) - - jnners = {dim: data.get(inner_keys[dim]) for dim in dims} - - # Add 'single_var's into 'time' dataset to keep track - sv_keys = [v.name for v in data.values() if 'single_var' in v.dims] - singlevar_set = data.get(sv_keys) - jnners['time'] = xr.merge([jnners['time'], singlevar_set]) - - # Concatenate along corresponding dimension - if inners['time'] is None: - inners = jnners - else: - inners = {dim: xr.concat([inners[dim], jnners[dim]], dim=dim) - for dim in dims} - - data = xr.merge([inners[dim] for dim in dims]) - - # Set up coordinates - coords = ['PIERCEPOINT_NIGHT_LATITUDE', - 'PIERCEPOINT_NIGHT_LONGITUDE', - 'PIERCEPOINT_NIGHT_ALTITUDE', - 'PIERCEPOINT_NIGHT_SZA', - 'PIERCEPOINT_DAY_LATITUDE', - 'PIERCEPOINT_DAY_LONGITUDE', - 'PIERCEPOINT_DAY_ALTITUDE', - 'PIERCEPOINT_DAY_SZA'] - - if 'sdr-imag' in tag: - coords = coords + ['PIERCEPOINT_DAY_LATITUDE_AURORAL', - 'PIERCEPOINT_DAY_LONGITUDE_AURORAL', - 'PIERCEPOINT_DAY_ALTITUDE_AURORAL', - 'PIERCEPOINT_DAY_SZA_AURORAL'] - elif 'sdr-spect' in tag: - coords = coords + ['PIERCEPOINT_NIGHT_ZENITH_ANGLE', - 'PIERCEPOINT_NIGHT_SAZIMUTH', - 'PIERCEPOINT_DAY_ZENITH_ANGLE', - 'PIERCEPOINT_DAY_SAZIMUTH'] - - data = data.set_coords(coords) - - # Set 'nchan' and 'nCross' as coordinates - if 'sdr-imag' in tag: - coords = {"nchan": ["121.6nm", "130.4nm", "135.6nm", - "LBHshort", "LBHlong"], - "nchanAur": ["121.6nm", "130.4nm", "135.6nm", - "LBHshort", "LBHlong"], - "nCross": data.nCross.data, - "nCrossDayAur": data.nCrossDayAur.data} - elif 'sdr-spect' in tag: - coords = {"nchan": ["121.6nm", "130.4nm", "135.6nm", - "LBHshort", "LBHlong", "?"]} - - data = data.assign_coords(coords=coords) - - # Sort - data = data.sortby("time") + data, meta = jhuapl.load_sdr_aurora(fnames, tag, inst_id, + pandas_format=pandas_format, + combine_times=combine_times) return data, meta From 761fb63244f458173b345f4ce5fe4cb7a10f38c2 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 17 Mar 2023 15:55:21 -0400 Subject: [PATCH 167/396] DOC: fixed institution name Fixed the name of the institution being referenced. --- pysatNASA/instruments/methods/jhuapl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 8c71de95..7a29ee03 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -10,7 +10,7 @@ def build_dtimes(data, var, epoch=None, epoch_var='time'): - """Build datetime objects from standard JPL time variables. + """Build datetime objects from standard JHU APL time variables. Parameters ---------- From 300047dda032795fb0b6aa12d395ed19f8600975 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 17 Mar 2023 15:55:43 -0400 Subject: [PATCH 168/396] STY: removed unused imports Removed imports not needed after streamlining the code. --- pysatNASA/instruments/timed_guvi.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 294e0586..f3d8a10e 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -60,10 +60,7 @@ import datetime as dt import functools -import numpy as np -import xarray as xr -import pysat from pysat.instruments.methods import general as mm_gen from pysatNASA.instruments.methods import cdaweb as cdw From 5af470d0c9565ec2f35aee9a0dc061d06b21b44b Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 17 Mar 2023 16:29:45 -0400 Subject: [PATCH 169/396] BUG: fix dimensions for spectrograph data Fixed the dimension renaming for spectrograph data. --- pysatNASA/instruments/methods/jhuapl.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 7a29ee03..f8705b9a 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -266,6 +266,7 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, 'PIERCEPOINT_DAY_LATITUDE', 'PIERCEPOINT_DAY_LONGITUDE', 'PIERCEPOINT_DAY_ALTITUDE', 'PIERCEPOINT_DAY_SZA'] time_dims = ['time'] + rename_dims = {'nAlongDay': 'nAlong', 'nAlongNight': 'nAlong'} if tag == 'sdr-imaging': time_vars.extend(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", @@ -275,6 +276,9 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, 'PIERCEPOINT_DAY_ALTITUDE_AURORAL', 'PIERCEPOINT_DAY_SZA_AURORAL']) time_dims.append('time_auroral') + rename_dims['nCrossDay'] = 'nCross' + rename_dims['nCrossNight'] = 'nCross' + rename_dims['nAlongDayAur'] = 'time_auroral' elif tag == 'sdr-spectrograph': time_vars.extend(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", "TIME_GAIM_DAY", "TIME_GAIM_NIGHT", "YEAR_GAIM_NIGHT", @@ -284,6 +288,8 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, 'PIERCEPOINT_DAY_ZENITH_ANGLE', 'PIERCEPOINT_DAY_SAZIMUTH']) time_dims.extend(['time_gaim_day', 'time_gaim_night']) + rename_dims['nAlongGAIMDay'] = 'time_gaim_day' + rename_dims['nAlongGAIMNight'] = 'time_gaim_night' # CDAWeb stores these files in the NetCDF format instead of the CDF format inners = None @@ -303,22 +309,17 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, if sdata.dims['nAlongDay'] != sdata.dims['nAlongNight']: raise ValueError('Along-track day and night dimensions differ') - if sdata.dims['nCrossDay'] != sdata.dims['nCrossNight']: - raise ValueError('Cross-track day and night dimensions differ') + if 'nCrossDay' in rename_dims.keys(): + if sdata.dims['nCrossDay'] != sdata.dims['nCrossNight']: + raise ValueError('Cross-track day and night dimensions differ') # Combine identical dimensions and rename 'nAlong' to 'time' - sdata = sdata.rename_dims({'nAlongDay': 'nAlong', - 'nAlongNight': 'nAlong', - 'nCrossDay': 'nCross', - 'nCrossNight': 'nCross'}) + sdata = sdata.rename_dims(rename_dims) if tag == 'sdr-imaging': - sdata = sdata.rename_dims({'nAlongDayAur': 'time_auroral'}) - sdata = sdata.assign(time_auroral=build_dtimes( - sdata, '_DAY_AURORAL')) + sdata = sdata.assign(time_auroral=build_dtimes(sdata, + '_DAY_AURORAL')) elif tag == 'sdr-spectrograph' and inst_id == 'low_res': - sdata = sdata.rename_dims({'nAlongGAIMDay': 'time_gaim_day', - 'nAlongGAIMNight': 'time_gaim_night'}) sdata = sdata.assign(time_gaim_day=build_dtimes( sdata, '_GAIM_DAY'), time_gaim_night=build_dtimes( sdata, '_GAIM_NIGHT')) From 36433c6033994001f7259e52551f467cff17ed13 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 17 Mar 2023 17:10:16 -0400 Subject: [PATCH 170/396] BUG: added inst_id logic Added inst_id logic for the GUVI spectrograph data. --- pysatNASA/instruments/methods/jhuapl.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index f8705b9a..e11971ea 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -280,16 +280,18 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, rename_dims['nCrossNight'] = 'nCross' rename_dims['nAlongDayAur'] = 'time_auroral' elif tag == 'sdr-spectrograph': - time_vars.extend(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", "TIME_GAIM_DAY", - "TIME_GAIM_NIGHT", "YEAR_GAIM_NIGHT", - "DOY_GAIM_NIGHT"]) coords.extend(['PIERCEPOINT_NIGHT_ZENITH_ANGLE', 'PIERCEPOINT_NIGHT_SAZIMUTH', 'PIERCEPOINT_DAY_ZENITH_ANGLE', 'PIERCEPOINT_DAY_SAZIMUTH']) - time_dims.extend(['time_gaim_day', 'time_gaim_night']) - rename_dims['nAlongGAIMDay'] = 'time_gaim_day' - rename_dims['nAlongGAIMNight'] = 'time_gaim_night' + + if inst_id == 'low_res': + time_vars.extend(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", "TIME_GAIM_DAY", + "TIME_GAIM_NIGHT", "YEAR_GAIM_NIGHT", + "DOY_GAIM_NIGHT"]) + time_dims.extend(['time_gaim_day', 'time_gaim_night']) + rename_dims['nAlongGAIMDay'] = 'time_gaim_day' + rename_dims['nAlongGAIMNight'] = 'time_gaim_night' # CDAWeb stores these files in the NetCDF format instead of the CDF format inners = None From acd78c91a4a5ace15b0cb901a57253a56967e751 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 20 Mar 2023 13:07:43 -0400 Subject: [PATCH 171/396] TST: added custom load options to test keys Added the 'combine_times' load kwarg to the load tests. --- pysatNASA/instruments/timed_guvi.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index f3d8a10e..26daf9e8 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -88,6 +88,9 @@ _test_dates = {iid: {tag: dt.datetime(2005, 6, 28) for tag in inst_ids[iid]} for iid in inst_ids.keys()} +_test_load_opt = {iid: {tag: [{'combine_times': True}, {'combine_times': False}] + for tag in inst_ids[iid]} for iid in ['high_res', + 'low_res']} # ---------------------------------------------------------------------------- # Instrument methods From 941c58e8f7b068e586cb7c9e158300ae3c4e8e68 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Tue, 21 Mar 2023 14:20:17 -0400 Subject: [PATCH 172/396] fixed cdas_get_remote files error when stop=None --- pysatNASA/instruments/methods/cdaweb.py | 5 +++-- pysatNASA/tests/test_methods_cdaweb.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 641d8643..f4e216ef 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -817,8 +817,9 @@ def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, # Use the topmost directory without variables start = cdas.get_inventory(identifier=dataset)[0].start stop = cdas.get_inventory(identifier=dataset)[-1].end - - if start == stop: + elif stop is None: + stop = start + dt.timedelta(days=1) + elif start == stop: stop = start + dt.timedelta(days=1) if isinstance(start, pds._libs.tslibs.timestamps.Timestamp): diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index 1b593f5b..51139469 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -83,6 +83,7 @@ def test_bad_kwarg_list_remote_files(self, bad_key, bad_val, err_msg): @pytest.mark.parametrize("start, stop", [(None, None), + (dt.datetime(2009, 1, 1), None), (dt.datetime(2009, 1, 1), dt.datetime(2009, 1, 1)), (pds.Timestamp(2009, 1, 1), From bc0f1788a5f19a461061f392ae9b8243817b0fb1 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Tue, 21 Mar 2023 14:27:31 -0400 Subject: [PATCH 173/396] added detailed information for getting the cdasws data set names --- pysatNASA/instruments/methods/cdaweb.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index f4e216ef..9b34959f 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -544,6 +544,31 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, Supported tags for this function use the cdaweb dataset naming convention. You can find the dataset names on cdaweb or you can use cdasws. + Starting from scratch using cdasws + :: + from cdasws import CdasWs + cads = CdasWs() + + # Get list of available observatories/platforms. + cdas.get_observatories() + + # Once your observatory is located, list the available instruments. + cdas.get_instruments(observatory=‘observatory_name’) + + # Now list the available data sets for one instrument. + cdas.get_datasets(observatory=‘observatory_name’, + instrument=‘instrument_name’) + + # You can also list all of the datasets for an observatory. + cdas.get_datasets(observatory=‘observatory_name’) + + Alternatively + :: + Visit https://cdaweb.gsfc.nasa.gov/ + Select the observatory you want from the list and press submit. + The following page will have a list of the data sets. + The bolded names are in the format that cdasws uses. + Examples -------- :: From c10da280f316de1ef521e6216bcad4b9ce2c8f25 Mon Sep 17 00:00:00 2001 From: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> Date: Tue, 21 Mar 2023 14:29:00 -0400 Subject: [PATCH 174/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- CHANGELOG.md | 2 +- pysatNASA/instruments/icon_fuv.py | 7 +------ pysatNASA/instruments/methods/cdaweb.py | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adeaf5b8..b581be2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). ## [0.0.5] - 2023-XX-XX -* Added cdaweb methods that can use cdasws to get the remote file list +* Added CDAWeb methods that can use cdasws to get the remote file list * New Instruments * DMSP SSUSI EDR-Aurora data * Bug Fixes diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py index 921001bf..c5333ee9 100644 --- a/pysatNASA/instruments/icon_fuv.py +++ b/pysatNASA/instruments/icon_fuv.py @@ -118,12 +118,7 @@ def clean(self): list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) -# old Set the download routine -# basic_tag24 = {'remote_dir': '/pub/data/icon/l2/l2-4_fuv_day/{year:04d}/', -# 'fname': fname24} -# basic_tag25 = {'remote_dir': '/pub/data/icon/l2/l2-5_fuv_night/{year:04d}/', -# 'fname': fname25} -# download_tags = {'': {'day': basic_tag24, 'night': basic_tag25}} +# Set the download routine download_tags = {'': {'day': 'ICON_L2-4_FUV_DAY', 'night': 'ICON_L2-5_FUV_NIGHT'}} diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 9b34959f..7c867ca1 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -542,7 +542,7 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, Note ---- Supported tags for this function use the cdaweb dataset naming convention. - You can find the dataset names on cdaweb or you can use cdasws. + You can find the data set names on CDAWeb or you can use cdasws. Starting from scratch using cdasws :: @@ -572,7 +572,7 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, Examples -------- :: - # download support added to cnofs_vefi.py using code below + # Download support added to cnofs_vefi.py using code below download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} download = functools.partial(cdw.cdas_download, supported_tags=download_tags) From 637d82227e27a3803ee20b243cf84b97cb00e42d Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Tue, 28 Mar 2023 21:21:34 -0600 Subject: [PATCH 175/396] Update ses14_gold.py Added o2den. Working version for multiple files. --- pysatNASA/instruments/ses14_gold.py | 109 ++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 30 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 62385dd9..9757743b 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -13,6 +13,7 @@ 'nmax' 'tlimb' 'tdisk' + 'o2den' Warnings -------- @@ -35,12 +36,11 @@ import datetime as dt import functools -import pysat import numpy as np import xarray as xr from pysat.instruments.methods import general as ps_gen -from pysat import logger +from pysat import logger, Meta from pysat.utils.io import load_netcdf from pysatNASA.instruments.methods import cdaweb as cdw @@ -54,17 +54,20 @@ name = 'gold' tags = {'nmax': 'Level 2 Nmax data for the GOLD instrument', 'tlimb': 'Level 2 Tlimb data for the GOLD instrument', - 'tdisk': 'Level 2 Tdisk data for the GOLD instrument'} -inst_ids = {'': ['nmax','tlimb','tdisk']} + 'tdisk': 'Level 2 Tdisk data for the GOLD instrument', + 'o2den': 'Level 2 O2den data for the GOLD instrument', + } +inst_ids = {'': ['nmax', 'tlimb', 'tdisk','o2den']} pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'nmax': dt.datetime(2020, 1, 1)}, +_test_dates = {'': {'nmax': dt.datetime(2020, 1, 1), 'tlimb': dt.datetime(2020, 1, 1), - 'tdisk': dt.datetime(2020, 1, 1)} + 'tdisk': dt.datetime(2020, 1, 1), + 'o2den': dt.datetime(2020, 1, 1)}} # ---------------------------------------------------------------------------- # Instrument methods @@ -176,30 +179,76 @@ def load(fnames, tag='', inst_id=''): 'min_val': ('Valid_Min', np.float64), 'max_val': ('Valid_Max', np.float64), 'fill_val': ('fill', np.float64)} - meta = pysat.Meta(labels=labels) - datos = [] - - for path in fnames: - - data = xr.load_dataset(path) - - if tag in ['nmax', 'tlimb', 'tdisk']: - # Add time coordinate from scan_start_time - data['time'] = ('nscans', - [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") - for val in data['scan_start_time'].values]) - data = data.swap_dims({'nscans': 'time'}) - - # Update coordinates with dimensional data - data = data.assign_coords({'nlats': data['nlats'], - 'nlons': data['nlons'], - 'nmask': data['nmask'], - 'channel': data['channel'], - 'hemisphere': data['hemisphere']}) - - datos.append(data) - - data = xr.concat(datos, dim='time') + # Generate custom meta translation table. When left unspecified the default + # table handles the multiple values for fill. We must recreate that + # functionality in our table. The targets for meta_translation should + # map to values in `labels` above. + meta_translation = {'FIELDNAM': 'plot', 'LABLAXIS': 'axis', + 'ScaleTyp': 'scale', 'VALIDMIN': 'Valid_Min', + 'Valid_Min': 'Valid_Min', 'VALIDMAX': 'Valid_Max', + 'Valid_Max': 'Valid_Max', '_FillValue': 'fill', + 'FillVal': 'fill', 'TIME_BASE': 'time_base'} + + if tag in ['nmax', 'tdisk', 'tlimb']: + epoch_name = 'nscans' + + elif tag == 'o2den': + epoch_name = 'nevents' + + try: + data, meta = load_netcdf(fnames, pandas_format=pandas_format, + epoch_name=epoch_name, labels=labels, + meta_translation=meta_translation, + drop_meta_labels='FILLVAL') + except: + meta = Meta(labels=labels) + data = xr.open_mfdataset(fnames, concat_dim=epoch_name, + combine='nested') + + # Renaming epoch variable + data = data.swap_dims({epoch_name: 'time'}) + + if tag in ['nmax', 'tdisk', 'tlimb']: + # Add time coordinate from scan_start_time + data['time'] = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") + for val in data['scan_start_time'].values] + + # Update coordinates with dimensional data + data = data.assign_coords({'nlats': data['nlats'], + 'nlons': data['nlons'], + 'nmask': data['nmask'], + 'channel': data['channel'], + 'hemisphere': data['hemisphere']}) + meta['time'] = {meta.labels.notes: 'Converted from scan_start_time'} + meta['nlats'] = {meta.labels.notes: 'Index for latitude values'} + meta['nlons'] = {meta.labels.notes: 'Index for longitude values'} + meta['nmask'] = {meta.labels.notes: 'Index for mask values'} + + elif tag == 'o2den': + + # Removing extra variables + if len(data['zret'].dims) > 1: + data['zret'] = data['zret'].isel(time=0) + data['zdat'] = data['zdat'].isel(time=0) + + # Add time coordinate from utc_time + data['time'] = [dt.datetime.strptime(str(val), + "b'%Y-%m-%dT%H:%M:%S.%fZ'") + for val in data['time_utc'].values] + + # Add retrieval altitude values and data tangent altitude values + data = data.swap_dims({"nzret": "zret", "nzdat": "zdat"}) + + # Update coordinates with dimensional data + data = data.assign_coords({'zret': data['zret'], + 'zdat': data['zdat'], + 'n_wavelength': data['n_wavelength'], + 'channel': data['channel']}) + meta['time'] = {meta.labels.notes: 'Converted from time_utc'} + meta['zret'] = {meta.labels.notes: ''.join(('Index for retrieval', + ' altitude values'))} + meta['zdat'] = {meta.labels.notes: ''.joing(('Index for data tangent', + ' altitude values'))} return data, meta From 9f9268fa113d79878eb79cc580ce0585adf2fd5e Mon Sep 17 00:00:00 2001 From: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:06:14 -0400 Subject: [PATCH 176/396] Apply suggestions from code review Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/methods/cdaweb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 6901d5a6..eebf2029 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -556,7 +556,7 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, Starting from scratch using cdasws :: from cdasws import CdasWs - cads = CdasWs() + cdas = CdasWs() # Get list of available observatories/platforms. cdas.get_observatories() @@ -820,8 +820,8 @@ def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, pre-set with functools.partial then assigned to new instrument code. (default=None) series_out : bool - boolean to determine output type. True for pandas series, and False for - list. + boolean to determine output type. True for pandas series of file names, + and False for a list of the full web address. Returns ------- From 5c32ed14fd0bd84ad16f83cc18b14caa705108cb Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:40:06 -0400 Subject: [PATCH 177/396] latest ace and icon instruments now use cdasws --- pysatNASA/instruments/ace_epam_l2.py | 13 ++++----- pysatNASA/instruments/ace_mag_l2.py | 15 +++++----- pysatNASA/instruments/ace_sis_l2.py | 12 ++++---- pysatNASA/instruments/ace_swepam_l2.py | 13 ++++----- pysatNASA/instruments/icon_mighti.py | 37 ++++++++++--------------- pysatNASA/instruments/methods/cdaweb.py | 2 +- 6 files changed, 41 insertions(+), 51 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index b4e0f6cb..670ed3ad 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -95,13 +95,12 @@ meta_translation=meta_translation) # Set the download routine -remote_dir = '/pub/data/ace/epam/level_2_cdaweb/epm_{sid:s}/{{year:4d}}/' -download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]), - 'fname': fname.format(sid=strid[id][tag])} - for tag in inst_ids[id]} - for id in inst_ids.keys()} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'12sec': {'base': 'AC_H3_EPM'}, + '5min': {'base': 'AC_H1_EPM', 'key': 'AC_K0_EPM'}, + '1hr': {'base': 'AC_H2_EPM', 'key': 'AC_K1_EPM'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index 6e43be50..8b5c4ea6 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -97,13 +97,14 @@ meta_translation=meta_translation) # Set the download routine -remote_dir = '/pub/data/ace/mag/level_2_cdaweb/mfi_{sid:s}/{{year:4d}}/' -download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]), - 'fname': fname.format(sid=strid[id][tag])} - for tag in inst_ids[id]} - for id in inst_ids.keys()} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'1sec': {'base': 'AC_H3_MFI'}, + '16sec': {'base': 'AC_H0_MFI', 'key': 'AC_K1_MFI'}, + '4min': {'base': 'AC_H1_MFI'}, + '5min': {'key': 'AC_K0_MFI'}, + '1hr': {'base': 'AC_H2_MFI', 'key': 'AC_K2_MFI'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index eb74ef39..f69a4308 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -90,13 +90,11 @@ meta_translation=meta_translation) # Set the download routine -remote_dir = '/pub/data/ace/sis/level_2_cdaweb/sis_{sid:s}/{{year:4d}}/' -download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]), - 'fname': fname.format(sid=strid[id][tag])} - for tag in inst_ids[id]} - for id in inst_ids.keys()} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'256sec': {'base': 'AC_H1_SIS'}, + '1hr': {'base': 'AC_H2_SIS', 'key': 'AC_K0_SIS'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index d570cf7a..a4a5ae20 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -93,13 +93,12 @@ meta_translation=meta_translation) # Set the download routine -remote_dir = '/pub/data/ace/swepam/level_2_cdaweb/swe_{sid:s}/{{year:4d}}/' -download_tags = {id: {tag: {'remote_dir': remote_dir.format(sid=strid[id][tag]), - 'fname': fname.format(sid=strid[id][tag])} - for tag in inst_ids[id]} - for id in inst_ids.keys()} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'64sec': {'base': 'AC_H0_SWE'}, + '5min': {'key': 'AC_K0_SWE'}, + '1hr': {'base': 'AC_H2_SWE', 'key': 'AC_K1_SWE'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/icon_mighti.py b/pysatNASA/instruments/icon_mighti.py index 006a9f8a..d2f11ab1 100644 --- a/pysatNASA/instruments/icon_mighti.py +++ b/pysatNASA/instruments/icon_mighti.py @@ -226,30 +226,23 @@ def _clean_vars(var_list, flag, min_level): supported_tags=supported_tags) # Set the download routine -dirstr1 = '/pub/data/icon/l2/l2-1_mighti-{{id:s}}_los-wind-{color:s}/' -dirstr2 = '/pub/data/icon/l2/l2-2_mighti_vector-wind-{color:s}/' -dirstr3 = '/pub/data/icon/l2/l2-3_mighti-{id:s}_temperature/' -dirnames = {'los_wind_green': dirstr1.format(color='green'), - 'los_wind_red': dirstr1.format(color='red'), - 'vector_wind_green': dirstr2.format(color='green'), - 'vector_wind_red': dirstr2.format(color='red'), - 'temperature': dirstr3} - -download_tags = {} -for inst_id in supported_tags.keys(): - download_tags[inst_id] = {} - for tag in supported_tags[inst_id].keys(): - fname = supported_tags[inst_id][tag] - - download_tags[inst_id][tag] = { - 'remote_dir': ''.join((dirnames[tag].format(id=inst_id), - '{year:04d}/')), - 'fname': fname} - -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'vector': + {'vector_wind_green': 'ICON_L2-2_MIGHTI_VECTOR-WIND-GREEN', + 'vector_wind_red': 'ICON_L2-2_MIGHTI_VECTOR-WIND-RED'}, + 'a': + {'los_wind_green': 'ICON_L2-1_MIGHTI-A_LOS-WIND-GREEN', + 'los_wind_red': 'ICON_L2-1_MIGHTI-A_LOS-WIND-RED', + 'temperature': 'ICON_L2-3_MIGHTI-A_TEMPERATURE'}, + 'b': + {'los_wind_green': 'ICON_L2-1_MIGHTI-B_LOS-WIND-GREEN', + 'los_wind_red': 'ICON_L2-1_MIGHTI-B_LOS-WIND-RED', + 'temperature': 'ICON_L2-3_MIGHTI-B_TEMPERATURE'}} + + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index eebf2029..d1969bf8 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -820,7 +820,7 @@ def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, pre-set with functools.partial then assigned to new instrument code. (default=None) series_out : bool - boolean to determine output type. True for pandas series of file names, + boolean to determine output type. True for pandas series of file names, and False for a list of the full web address. Returns From 1a57834815201ecbc9d0d3fcc995c9375e470a14 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:40:53 -0400 Subject: [PATCH 178/396] trailing white space --- pysatNASA/tests/test_methods_cdaweb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index 51139469..f601517d 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -17,7 +17,7 @@ class TestCDAWeb(object): def setup_method(self): """Set up the unit test environment for each method.""" - self.download_tags = pysatNASA.instruments.icon_mighti.download_tags + self.download_tags = pysatNASA.instruments.timed_guvi.download_tags self.kwargs = {'tag': None, 'inst_id': None} return @@ -32,7 +32,7 @@ def test_remote_file_list_connection_error_append(self): with pytest.raises(Exception) as excinfo: # Giving a bad remote_site address yields similar ConnectionError - cdw.list_remote_files(tag='los_wind_green', inst_id='a', + cdw.list_remote_files(tag='sdr-imaging', inst_id='high_res', supported_tags=self.download_tags, remote_url='https://bad/path') From 111c241009a7afcfb501d0bcfd2375c7f480fb21 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Mar 2023 11:15:37 -0400 Subject: [PATCH 179/396] DOC: add info about expected warnings --- pysatNASA/instruments/timed_see.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 72d5972c..aa60379b 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -21,6 +21,8 @@ Note ---- - no tag required +- cdflib load routine raises ISTP Compliance Warnings for severa variables. + This is due to how the Epoch is listed in the original files. Warnings -------- From 0191466e71efb05c70cc13dc86d310e6da5fa166 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Mar 2023 11:39:30 -0400 Subject: [PATCH 180/396] BUG: lfnames not always defined --- pysatNASA/instruments/methods/cdaweb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 39b22eeb..36899ed8 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -352,6 +352,8 @@ def load_xarray(fnames, tag='', inst_id='', t0 = dt.datetime(2009, 1, 1) if (t0 + file_cadence) > (t0 + dt.timedelta(days=1)): lfnames = list(np.unique([fname[:-11] for fname in fnames])) + else: + lfnames = fnames for lfname in lfnames: temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True) @@ -441,6 +443,7 @@ def load_xarray(fnames, tag='', inst_id='', return data, meta +# TODO(#103): Include support to unzip / untar files after download. def download(date_array, tag='', inst_id='', supported_tags=None, remote_url='https://cdaweb.gsfc.nasa.gov', data_path=None): """Download NASA CDAWeb data. From b1fd19891af50793eee7f686208cb62abf68c9c2 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Mar 2023 11:39:41 -0400 Subject: [PATCH 181/396] DOC: add todo statements --- pysatNASA/instruments/timed_saber.py | 2 ++ pysatNASA/instruments/timed_see.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index 6a758da9..b3c8027e 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -105,6 +105,8 @@ def init(self): # # Use the default CDAWeb and pysat methods +# TODO(#104): Switch to netCDF4 files once unzip (#103) is supported. + # Set the list_files routine fname = ''.join(('timed_l2a_saber_{year:04d}{month:02d}{day:02d}', '{hour:02d}{minute:02d}_v{version:02d}-{revision:02d}-', diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index aa60379b..e3547e9f 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -90,6 +90,8 @@ def init(self): # # Use the default CDAWeb and pysat methods +# TODO(#104): Switch to netCDF4 files once unzip (#103) is supported. + # Set the list_files routine fname = 'timed_l3a_see_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' supported_tags = {'': {'': fname}} From d4b98268f25a3756247460f98b3fb3763113b7f1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Mar 2023 13:40:18 -0400 Subject: [PATCH 182/396] STY: fix metadata --- pysatNASA/instruments/timed_see.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index e3547e9f..2f1a7567 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -35,7 +35,7 @@ import pandas as pds from pysat.instruments.methods import general as mm_gen -from pysat import logger +import pysat from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa @@ -67,7 +67,7 @@ def init(self): rules_url = 'https://www.timed.jhuapl.edu/WWW/scripts/mdc_rules.pl' ackn_str = ' '.join(('Please see the Rules of the Road at', rules_url)) - logger.info(ackn_str) + pysat.logger.info(ackn_str) self.acknowledgements = ackn_str self.references = ' '.join(('Woods, T. N., Eparvier, F. G., Bailey,', 'S. M., Chamberlin, P. C., Lean, J.,', @@ -100,8 +100,15 @@ def init(self): file_cadence=pds.DateOffset(months=1)) # Set the load routine +meta = pysat.Meta() +meta_translation = {'CATDESC': meta.labels.desc, + 'FILLVAL': meta.labels.fill_val, + 'VALIDMIN': meta.labels.min_val, + 'VALIDMAX': meta.labels.max_val, + 'VAR_NOTES': meta.labels.notes} load = functools.partial(cdw.load, pandas_format=pandas_format, - file_cadence=pds.DateOffset(months=1)) + file_cadence=pds.DateOffset(months=1), + meta_translation=meta_translation) # Set the download routine download_tags = {'': {'': 'TIMED_L3A_SEE'}} From f736bdd04d684b2522a42973163b9f8b30cad62c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Mar 2023 13:45:54 -0400 Subject: [PATCH 183/396] STY: update default labels for xarray --- pysatNASA/instruments/methods/cdaweb.py | 12 +++++------- pysatNASA/instruments/timed_see.py | 9 +-------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 36899ed8..4ec9dab6 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -257,13 +257,11 @@ def load_pandas(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), def load_xarray(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), - labels={'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'plot': ('plot_label', str), 'axis': ('axis', str), - 'scale': ('scale', str), - 'min_val': ('value_min', float), - 'max_val': ('value_max', float), - 'fill_val': ('fill', float)}, + labels={'units': ('Units', str), 'name': ('Long_Name', str), + 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), + 'min_val': ('ValidMin', float), + 'max_val': ('ValidMax', float), + 'fill_val': ('FillVal', float)}, epoch_name='Epoch', meta_processor=None, meta_translation=None, drop_meta_labels=None): """Load NASA CDAWeb CDF files into an xarray Dataset. diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 2f1a7567..75a79a48 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -100,15 +100,8 @@ def init(self): file_cadence=pds.DateOffset(months=1)) # Set the load routine -meta = pysat.Meta() -meta_translation = {'CATDESC': meta.labels.desc, - 'FILLVAL': meta.labels.fill_val, - 'VALIDMIN': meta.labels.min_val, - 'VALIDMAX': meta.labels.max_val, - 'VAR_NOTES': meta.labels.notes} load = functools.partial(cdw.load, pandas_format=pandas_format, - file_cadence=pds.DateOffset(months=1), - meta_translation=meta_translation) + file_cadence=pds.DateOffset(months=1)) # Set the download routine download_tags = {'': {'': 'TIMED_L3A_SEE'}} From e79a963a1f08643d46f7d7a78f7513d2c93d8bbd Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Mar 2023 13:46:50 -0400 Subject: [PATCH 184/396] STY: pep8 --- pysatNASA/instruments/timed_see.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 75a79a48..1b0d957a 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -34,8 +34,8 @@ import functools import pandas as pds -from pysat.instruments.methods import general as mm_gen import pysat +from pysat.instruments.methods import general as mm_gen from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa From caecca454569fd5dfd3b212573a5a051465c423f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Mar 2023 14:01:14 -0400 Subject: [PATCH 185/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98d3978c..36c969d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). spectrograph, imaging * Resolution of dataset handled by tag with low, high +* xarray support for TIMED SEE * Added CDAWeb methods that can use cdasws to get the remote file list * Bug Fixes * Updated CDAWeb routines to allow for data stored by year/day-of-year From d832db7b24533ff2ed10a1d5c1698beb17aa7713 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Mar 2023 14:11:59 -0400 Subject: [PATCH 186/396] MAINT: delete duplicate instrument --- CHANGELOG.md | 1 - pysatNASA/instruments/__init__.py | 2 +- pysatNASA/instruments/de2_idm.py | 115 --------------------------- pysatNASA/instruments/methods/de2.py | 12 +-- 4 files changed, 7 insertions(+), 123 deletions(-) delete mode 100644 pysatNASA/instruments/de2_idm.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bbb1300b..76aa1196 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ This project adheres to [Semantic Versioning](https://semver.org/). * ACE MAG * ACE SIS * ACE SWEPAM - * DE2 Ion Drift Meter * DE2 RPA DUCT (high rate plasma density) * DE2 Vector Electric Field Instrument * DMSP SSUSI EDR-Aurora data diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index 8d572d00..81a6abe0 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -8,7 +8,7 @@ __all__ = ['ace_epam_l2', 'ace_mag_l2', 'ace_sis_l2', 'ace_swepam_l2', 'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', - 'de2_idm', 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'de2_vefi', + 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'de2_vefi', 'dmsp_ssusi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', diff --git a/pysatNASA/instruments/de2_idm.py b/pysatNASA/instruments/de2_idm.py deleted file mode 100644 index f121e32f..00000000 --- a/pysatNASA/instruments/de2_idm.py +++ /dev/null @@ -1,115 +0,0 @@ -# -*- coding: utf-8 -*- -"""The DE2 IDM instrument. - -Supports the Ion Drift Meter (IDM) instrument -on Dynamics Explorer 2 (DE2). - -From CDAWeb (adapted): - -This directory gathers data for the IDM instrument that flew on the DE 2 -spacecraft which was launched on 3 August 1981 into an elliptical orbit -with an altitude range of 300 km to 1000 km and re-entered the atmosphere on -19 February 1983. - -(NSSDC ID: 81-070B-06D) -This data set provides the cross track ion drift with a time resolution of -125 milli-seconds for the time period from Aug 15, 1981 to Feb 16, 1983. The -ASCII version was generated at NSSDC from the originally PI-submitted binary -data. - - -References ----------- -Heelis, R. A., W. B. Hanson, C. R. Lippincott, D. R. Zuccaro, L. L. Harmon, -B. J. Holt, J. E. Doherty, R. A. Power, The ion drift meter for Dynamics -Explorer-B, Space Sci. Instrum., 5, 511, 1981. - -Properties ----------- -platform - 'de2' -name - 'idm' -inst_id - None Supported -tag - None Supported - -Warnings --------- -- Currently no cleaning routine. - -""" - -import datetime as dt -import functools - -from pysat.instruments.methods import general as mm_gen - -from pysatNASA.instruments.methods import cdaweb as cdw -from pysatNASA.instruments.methods import de2 as mm_de2 -from pysatNASA.instruments.methods import general as mm_nasa - -# ---------------------------------------------------------------------------- -# Instrument attributes - -platform = 'de2' -name = 'idm' -tags = {'': '1 s cadence Neutral Atmosphere Composition Spectrometer data'} -inst_ids = {'': ['']} -pandas_format = False - -# ---------------------------------------------------------------------------- -# Instrument test attributes - -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} - -# ---------------------------------------------------------------------------- -# Instrument methods - - -# Use standard init routine -init = functools.partial(mm_nasa.init, module=mm_de2, name=name) - -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn - -# ---------------------------------------------------------------------------- -# Instrument functions -# -# Use the default CDAWeb and pysat methods - -# Set the list_files routine -fname = 'de2_vion250ms_idm_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' -supported_tags = {'': {'': fname}} -list_files = functools.partial(mm_gen.list_files, - supported_tags=supported_tags) - -# Use the default CDAWeb method -load = functools.partial(cdw.load, pandas_format=pandas_format) - - -def preprocess(self): - """Apply DE2 IDM default attributes. - - Note - ---- - Corrects the epoch names - - """ - - self.data = self.data.rename({'record0': 'Epoch_velZ', - 'record1': 'Epoch_velY'}) - return - - -# Support download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_idm', - '/vion250ms_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) - -# Support listing files currently on CDAWeb -list_remote_files = functools.partial(cdw.list_remote_files, - supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/de2.py b/pysatNASA/instruments/methods/de2.py index 90edf2a7..4a293e5e 100644 --- a/pysatNASA/instruments/methods/de2.py +++ b/pysatNASA/instruments/methods/de2.py @@ -3,11 +3,7 @@ ackn_str = "The Dynamics Explorer 2 satellite data is provided through CDAWeb" -refs = {'idm': ' '.join(('Heelis, R. A., W. B. Hanson, C. R. Lippincott, D. R.', - 'Zuccaro, L. L. Harmon, B. J. Holt, J. E. Doherty, R.', - 'A. Power, The ion drift meter for Dynamics', - 'Explorer-B, Space Sci. Instrum., 5, 511, 1981.')), - 'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', +refs = {'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', 'Pinkus, and R. B. Kaplan, The Dynamics Explorer 2', 'Langmuir Probe (LANG), Space Sci. Instrum., v. 5,', 'n. 4, p. 493, 1981.')), @@ -19,7 +15,11 @@ 'Lippincott, D. R. Zuccaro, B. J. Holt, L. H. Harmon,', 'and S. Sanatani, The retarding potential analyzer', 'for dynamics explorer-B, Space Sci. Instrum. 5,', - '503–510 (1981).')), + '503–510 (1981).\n', + 'Heelis, R. A., W. B. Hanson, C. R. Lippincott, D. R.', + 'Zuccaro, L. L. Harmon, B. J. Holt, J. E. Doherty, R.', + 'A. Power, The ion drift meter for Dynamics', + 'Explorer-B, Space Sci. Instrum., 5, 511, 1981.')), 'wats': ' '.join(('N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E.', 'Hedin, G. R. Carrignan, J. C. Maurer, The', 'Dynamics Explorer Wind and Temperature Spectrometer', From 28d76abfcdabeafc7b4a3978e2d78a378f6266e6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 30 Mar 2023 17:02:50 -0400 Subject: [PATCH 187/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysatNASA/instruments/timed_see.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 1b0d957a..e764b456 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -21,7 +21,7 @@ Note ---- - no tag required -- cdflib load routine raises ISTP Compliance Warnings for severa variables. +- cdflib load routine raises ISTP Compliance Warnings for several variables. This is due to how the Epoch is listed in the original files. Warnings From e43aad60d247707b30e9a541d108fde163fbc818 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 31 Mar 2023 12:30:59 -0400 Subject: [PATCH 188/396] STY: update vefi --- pysatNASA/instruments/de2_vefi.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index becfcd38..aff5b56e 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -89,14 +89,10 @@ load = cdw.load # Set the download routine -download_tags = {'': {tag: {'remote_dir': ''.join(('/pub/data/de/de2/', - 'electric_fields_vefi/', - tag, '500ms_vefi_cdaweb/', - '{year:4d}/')), - 'fname': fname.format(tag=tag, datestr=datestr)} - for tag in tags}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'ac': 'DE2_AC500MS_VEFI', + 'dca': 'DE2_DCA500MS_VEFI'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) From aa2420a65e832e967325e56284207f68be800f1b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 31 Mar 2023 13:06:20 -0400 Subject: [PATCH 189/396] BUG: only include non-standard options Only add non-standard options to the `_test_load_opt` dictionary. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/instruments/timed_guvi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 26daf9e8..36b8111b 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -88,7 +88,7 @@ _test_dates = {iid: {tag: dt.datetime(2005, 6, 28) for tag in inst_ids[iid]} for iid in inst_ids.keys()} -_test_load_opt = {iid: {tag: [{'combine_times': True}, {'combine_times': False}] +_test_load_opt = {iid: {tag: {'combine_times': True} for tag in inst_ids[iid]} for iid in ['high_res', 'low_res']} From 65ab870fb5f8f8629060f3c99b97e2e7b6b0bf7a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 31 Mar 2023 14:17:04 -0400 Subject: [PATCH 190/396] BUG: monotonic increasing nmax --- pysatNASA/instruments/ses14_gold.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 14c6f9e9..b5d98e6c 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -184,6 +184,7 @@ def load(fnames, tag='', inst_id=''): # Add time coordinate from scan_start_time data['time'] = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") for val in data['scan_start_time'].values] + data = data.sortby('time') # Update coordinates with dimensional data data = data.assign_coords({'nlats': data['nlats'], From 8db0eefbf84943b5d4259468ed14e2284f7209cd Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 31 Mar 2023 14:17:54 -0400 Subject: [PATCH 191/396] DOC: changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98d3978c..5ad85f60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added CDAWeb methods that can use cdasws to get the remote file list * Bug Fixes * Updated CDAWeb routines to allow for data stored by year/day-of-year + * Added a sortby statement to GOLD nmax to sort scans by time. * Documentation * Added TIMED-GUVI platform * Maintenance From 167f521444b552b354e1ed3271a0950cf218a124 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 31 Mar 2023 14:21:07 -0400 Subject: [PATCH 192/396] REV: revert changes to develop --- CHANGELOG.md | 1 - pysatNASA/instruments/ses14_gold.py | 1 - 2 files changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ad85f60..98d3978c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,6 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added CDAWeb methods that can use cdasws to get the remote file list * Bug Fixes * Updated CDAWeb routines to allow for data stored by year/day-of-year - * Added a sortby statement to GOLD nmax to sort scans by time. * Documentation * Added TIMED-GUVI platform * Maintenance diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index b5d98e6c..14c6f9e9 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -184,7 +184,6 @@ def load(fnames, tag='', inst_id=''): # Add time coordinate from scan_start_time data['time'] = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") for val in data['scan_start_time'].values] - data = data.sortby('time') # Update coordinates with dimensional data data = data.assign_coords({'nlats': data['nlats'], From 191630acf7b67b6dbe9aec7ed5b1848b652c0a05 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 31 Mar 2023 14:28:54 -0400 Subject: [PATCH 193/396] DOC: update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ad85f60..ef51f96d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added CDAWeb methods that can use cdasws to get the remote file list * Bug Fixes * Updated CDAWeb routines to allow for data stored by year/day-of-year - * Added a sortby statement to GOLD nmax to sort scans by time. + * Updated GOLD nmax to sort scans by time. * Documentation * Added TIMED-GUVI platform * Maintenance From f5b41e5ab57d0fb5928666dd1f17dc1d5e0c7073 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 31 Mar 2023 15:17:35 -0400 Subject: [PATCH 194/396] Update ses14_gold.py --- pysatNASA/instruments/ses14_gold.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index b5d98e6c..516f72d5 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -185,6 +185,7 @@ def load(fnames, tag='', inst_id=''): data['time'] = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") for val in data['scan_start_time'].values] data = data.sortby('time') + data = data.sortby('time') # Update coordinates with dimensional data data = data.assign_coords({'nlats': data['nlats'], From a44cdda9e3c2666df106c4170b1f8f95b86156d5 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Sun, 2 Apr 2023 10:06:40 -0400 Subject: [PATCH 195/396] STY: pep8 --- pysatNASA/instruments/timed_see.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index a16934d7..1980443e 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -34,7 +34,6 @@ import functools import pandas as pds -import pysat from pysat.instruments.methods import general as mm_gen from pysatNASA.instruments.methods import cdaweb as cdw From 2f8c3d4135702b56a71209544064a57804917ea7 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 11:13:35 -0400 Subject: [PATCH 196/396] ENH: add fpi --- pysatNASA/instruments/__init__.py | 2 +- pysatNASA/instruments/de2_fpi.py | 107 +++++++++++++++++++++++++++ pysatNASA/instruments/methods/de2.py | 5 +- 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 pysatNASA/instruments/de2_fpi.py diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index 81a6abe0..3cc7d241 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -7,7 +7,7 @@ from pysatNASA.instruments import methods # noqa F401 __all__ = ['ace_epam_l2', 'ace_mag_l2', 'ace_sis_l2', 'ace_swepam_l2', - 'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', + 'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_fpi', 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'de2_vefi', 'dmsp_ssusi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', diff --git a/pysatNASA/instruments/de2_fpi.py b/pysatNASA/instruments/de2_fpi.py new file mode 100644 index 00000000..ad5e064d --- /dev/null +++ b/pysatNASA/instruments/de2_fpi.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +"""The DE2 FPI instrument. + +Supports the Fabry-Perot Interferometer (FPI) instrument on Dynamics Explorer 2 +(DE2). + +From CDAWeb: + +The Fabry-Perot Interferometer (FPI) was a high-resolution remote sensing +instrument designed to measure the thermospheric temperature, meridional wind, +and density of the following metastable atoms: atomic oxygen (singlet S and D) +and the 2P state of ionic atomic oxygen. The FPI performed a wavelength analysis +on the light detected from the thermospheric emission features by spatially +scanning the interference fringe plane with a multichannel array detector. The +wavelength analysis characterized the Doppler line profile of the emitting +species. A sequential altitude scan performed by a commandable horizon scan +mirror provided a cross-sectional view of the thermodynamic and dynamic state of +the thermosphere below the DE 2 orbit. The information obtained from this +investigation was used to study the dynamic response of the thermosphere to the +energy sources caused by magnetospheric electric fields and the absorption of +solar ultraviolet light in the thermosphere. The instrument was based on the +visible airglow experiment (VAE) used in the AE program. The addition of a +scanning mirror, the Fabry-Perot etalon, an image plane detector, and a +calibration lamp were the principal differences. Interference filters isolated +lines at (in Angstroms) 5577, 6300, 7320, 5896, and 5200. The FPI had a field of +view of 0.53 deg (half-cone angle). More details are found in P. B. Hays et al., +Space Sci. Instrum., v. 5, n. 4, p. 395, 1981. From February 16, 1982 to +September 11, 1982 the DE satellite was inverted and the FPI measured galactic +emissions. + +Properties +---------- +platform + 'de2' +name + 'fpi' +inst_id + None Supported +tag + None Supported + + +Warnings +-------- +- Currently no cleaning routine. + + +References +---------- +Hays, P B, Killeen, T L, and Kennedy, B C. "Fabry-Perot interferometer on +Dynamics Explorer". Space Sci. Instrum., 5, 395-416, 1981. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import de2 as mm_de2 +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'de2' +name = 'fpi' +tags = {'': '8 s cadence Fabry-Perot Interferometer data'} +inst_ids = {'': ['']} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_de2, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = 'de2_neutral8s_fpi_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' +supported_tags = {'': {'': fname}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Use the default CDAWeb method +load = cdw.load + +# Support download routine +download_tags = {'': {'': 'DE2_NEUTRAL8S_FPI'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Support listing files currently on CDAWeb +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/de2.py b/pysatNASA/instruments/methods/de2.py index 4a293e5e..8e44575d 100644 --- a/pysatNASA/instruments/methods/de2.py +++ b/pysatNASA/instruments/methods/de2.py @@ -3,7 +3,10 @@ ackn_str = "The Dynamics Explorer 2 satellite data is provided through CDAWeb" -refs = {'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', +refs = {'fpi': ' '.join(('Hays, P B, Killeen, T L, and Kennedy, B C.', + '"Fabry-Perot interferometer on Dynamics Explorer".', + 'Space Sci. Instrum., v. 5, p. 395-416, 1981.')), + 'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', 'Pinkus, and R. B. Kaplan, The Dynamics Explorer 2', 'Langmuir Probe (LANG), Space Sci. Instrum., v. 5,', 'n. 4, p. 493, 1981.')), From 606514c4c84e6bb9b464c2be2645fd5e6e79bfd7 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 11:15:30 -0400 Subject: [PATCH 197/396] ENH: remove duct --- pysatNASA/instruments/de2_rpa.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 9b0e862f..2a408f9e 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -76,8 +76,7 @@ platform = 'de2' name = 'rpa' -tags = {'': '2 sec cadence RPA data', # this is the default cadence - 'duct': '16ms DUCT plasma density'} +tags = {'': '2 sec cadence RPA data'} inst_ids = {'': [tag for tag in tags]} # ---------------------------------------------------------------------------- @@ -102,8 +101,7 @@ # Set the list_files routine datestr = '{year:04d}{month:02d}{day:02d}_v{version:02d}' -dataproduct = {'': 'ion2s', - 'duct': 'duct16ms'} +dataproduct = {'': 'ion2s'} fname = 'de2_{dp:s}_rpa_{datestr:s}.cdf' supported_tags = {'': {tag: fname.format(dp=dataproduct[tag], datestr=datestr) for tag in tags}} @@ -114,8 +112,7 @@ load = cdw.load # Set the download routine -download_tags = {'': {'': 'DE2_ION2S_RPA', - 'duct': 'DE2_DUCT16MS_RPA'}} +download_tags = {'': {'': 'DE2_ION2S_RPA'}} download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine From ee58e452f3b13ece2ee5c1ac727ffd306c1810b7 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 11:51:52 -0400 Subject: [PATCH 198/396] ENH: add mag --- pysatNASA/instruments/de2_vefi.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index aff5b56e..de4a304d 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -53,9 +53,10 @@ platform = 'de2' name = 'vefi' -tags = {'dca': '500 ms cadence DC Averaged Electric Field data', - 'ac': '500ms cadence AC Electric Field data'} -inst_ids = {'': ['dca', 'ac']} +tags = {'': '62 ms combination of Electric Field and Magnetometer', + 'dca': '500 ms cadence DC Averaged Electric Field data', + 'ac': '500 ms cadence AC Electric Field data'} +inst_ids = {'': [tag for tag in tags]} # ---------------------------------------------------------------------------- # Instrument test attributes @@ -79,8 +80,11 @@ # Set the list_files routine datestr = '{year:04d}{month:02d}{day:02d}_v{version:02d}' -fname = 'de2_{tag:s}500ms_vefi_{datestr:s}.cdf' -supported_tags = {'': {tag: fname.format(tag=tag, datestr=datestr) +fid = {'': '62ms_vefimagb', + 'ac': 'ac500ms_vefi', + 'dca': 'dca500ms_vefi'} +fname = 'de2_{fid:s}_{datestr:s}.cdf' +supported_tags = {'': {tag: fname.format(fid=fid[tag], datestr=datestr) for tag in tags}} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -89,7 +93,8 @@ load = cdw.load # Set the download routine -download_tags = {'': {'ac': 'DE2_AC500MS_VEFI', +download_tags = {'': {'': 'DE2_62MS_VEFIMAGB', + 'ac': 'DE2_AC500MS_VEFI', 'dca': 'DE2_DCA500MS_VEFI'}} download = functools.partial(cdw.cdas_download, supported_tags=download_tags) From 5c789929dd1eaf7559ff48d70fd72fe75144b957 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 12:16:10 -0400 Subject: [PATCH 199/396] DOC: update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58d8263b..b98930da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * ACE MAG * ACE SIS * ACE SWEPAM - * DE2 RPA DUCT (high rate plasma density) - * DE2 Vector Electric Field Instrument + * DE2 Fabry-Perot Interferometer (FPI) + * DE2 Vector Electric Field Instrument (VEFI) and magnetometer * DMSP SSUSI EDR-Aurora data * TIMED GUVI * Add TIMED GUVI platform to support L1C intensity datasets. From ad6eadfaf0e381f2db4df659b2fe4281f5441a42 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 12:27:48 -0400 Subject: [PATCH 200/396] MAINT: remove numpy cap --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0b485ab5..a1dafc41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ requests beautifulsoup4 lxml cdflib>=0.4.4 -numpy<1.24 +numpy pandas pysat>=3.0.4 cdasws From b2345f278473fea725436a7607894d2979693361 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 12:27:59 -0400 Subject: [PATCH 201/396] MAINT: update NEP29 versions --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0cd854e5..b89a54cf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: numpy_ver: ["latest"] include: - python-version: "3.8" - numpy_ver: "1.20" + numpy_ver: "1.21" os: "ubuntu-latest" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} From d68732068fde8cebe2750e973a1496ce5f94facc Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 12:28:38 -0400 Subject: [PATCH 202/396] TST: skip pysatCDF tests if not installed --- pysatNASA/tests/test_instruments.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 07e850f4..cca52c6b 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -16,6 +16,15 @@ # Import the test classes from pysat from pysat.tests.classes import cls_instrument_library as clslib +try: + import pysatCDF # noqa: F401 + # If this successfully imports, tests need to be run with both pysatCDF + # and cdflib + cdflib_only = False +except ImportError: + # pysatCDF is not present, standard tests default to cdflib. + cdflib_only = True + # Tell the standard tests which instruments to run each test on. # Need to return instrument list for custom tests. @@ -42,6 +51,7 @@ class TestInstruments(clslib.InstLibTests): @pytest.mark.second @pytest.mark.parametrize("inst_dict", instruments['cdf']) + @pytest.skipif(cdflib_only) def test_load_cdflib(self, inst_dict): """Test that instruments load at each cleaning level. From ff900978ad362480ff21b5bd529ef76b627c25b1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 12:28:53 -0400 Subject: [PATCH 203/396] TST: skip pysatCDF install on latest numpy --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b89a54cf..7291779f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,11 +35,11 @@ jobs: if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} + pip install pysatCDF --no-binary=pysatCDF - name: Install standard dependencies run: | pip install -r requirements.txt - pip install pysatCDF --no-binary=pysatCDF pip install -r test_requirements.txt - name: Set up pysat From 3d1962f52cb56a81fab31154fb1682b1eb72450b Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 12:30:28 -0400 Subject: [PATCH 204/396] DOC: update changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bf7fd0f..0a151816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,9 +26,9 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added unit tests for the different platform method attributes * xarray support for TIMED SEE * Maintenance - * Added a version cap for numpy (required for cdf interface, revisit before - release) - * Updated actions and templates based on pysatEcosystem docs. + * Removed duplicate tests if pysatCDF not isntalled + * Only test pysatCDF on GitHub Actions for older numpy versions + * Updated actions and templates based on pysatEcosystem docs * Remove pandas cap on NEP29 tests * Updated dosctring style for consistency * Removed version cap for xarray From 1ca1c31fa13d0fe4d53c44f5188ac0dfaf002c58 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 12:35:33 -0400 Subject: [PATCH 205/396] DOC: update info on pysatCDF --- README.md | 3 ++- pysatNASA/instruments/methods/cdaweb.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c7999c92..34a9e84a 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,8 @@ For data products stored as CDF files, this package can use either `cdflib` or `pysatCDF`. Note that `cdflib` is a pure python package and more readily deployable across systems, whereas `pysatCDF` interfaces with the fortran. This is a faster approach for loading data, but may not install on all systems. -Therefore, `pysatCDF` is optional rather than required. +There are known issues with `numpy`>=1.24. Therefore, `pysatCDF` is optional +rather than required. You can specify which load routine to use via the optional `use_cdflib` kwarg. If no kwarg is specified, `pysatNASA` will default to `pysatCDF` if it is diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 4ec9dab6..acfbc92a 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -124,8 +124,10 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), Note ---- - This routine is intended to be used by pysat instrument modules supporting - a particular NASA CDAWeb dataset + - This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset + - pysatCDF (as of v0.3.2) does not support numpy>=1.24. Load errors may + arise. See https://github.com/pysat/pysatCDF/issues/46 """ From 0006f3ee5057fe7859e7ed1e48a3bd89bb63c63a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 13:00:42 -0400 Subject: [PATCH 206/396] BUG: pytest skip --- pysatNASA/tests/test_instruments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index cca52c6b..d4736b72 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -51,7 +51,7 @@ class TestInstruments(clslib.InstLibTests): @pytest.mark.second @pytest.mark.parametrize("inst_dict", instruments['cdf']) - @pytest.skipif(cdflib_only) + @pytest.mark.skipif(cdflib_only) def test_load_cdflib(self, inst_dict): """Test that instruments load at each cleaning level. From 766241cc0617a373999eb492f5820eef793bf32a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 13:28:29 -0400 Subject: [PATCH 207/396] BUG: reason for skipif --- pysatNASA/tests/test_instruments.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index d4736b72..1a7dc59a 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -51,7 +51,9 @@ class TestInstruments(clslib.InstLibTests): @pytest.mark.second @pytest.mark.parametrize("inst_dict", instruments['cdf']) - @pytest.mark.skipif(cdflib_only) + @pytest.mark.skipif(cdflib_only, + reason=" ".join(("Additional load tests not required", + "when pysatCDF not installed"))) def test_load_cdflib(self, inst_dict): """Test that instruments load at each cleaning level. From b698e2885f0ba5045a54796d07d2ade9a9dcf4a4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 13:28:50 -0400 Subject: [PATCH 208/396] STY: only run extra tests for pandas --- pysatNASA/tests/test_instruments.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 1a7dc59a..25776e41 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -36,7 +36,9 @@ for inst in instruments['download']: fname = inst['inst_module'].supported_tags[inst['inst_id']][inst['tag']] if '.cdf' in fname: - instruments['cdf'].append(inst) + temp_inst, _ = clslib.initialize_test_inst_and_date(inst) + if temp_inst.pandas_format: + instruments['cdf'].append(inst) class TestInstruments(clslib.InstLibTests): From 4b3f7ef628612a2b642c0b4d73daa9cc6f70f685 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 16:30:47 -0400 Subject: [PATCH 209/396] DOC: update supported instruments --- docs/supported_instruments.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index d0ec68f5..1a8174e5 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -57,6 +57,14 @@ C/NOFS VEFI .. automodule:: pysatNASA.instruments.cnofs_vefi :members: +.. _de2_fpi: + +DE2 FPI +-------- + +.. automodule:: pysatNASA.instruments.de2_fpi + :members: + .. _de2_lang: DE2 LANG @@ -89,6 +97,14 @@ DE2 WATS .. automodule:: pysatNASA.instruments.de2_wats :members: +.. _de2_vefi: + +DE2 VEFI +-------- + +.. automodule:: pysatNASA.instruments.de2_vefi + :members: + .. _dmsp_ssusi: DMSP SSUSI From f75a6ec5584648ec1ef8c276661e945283802cd9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 16:47:28 -0400 Subject: [PATCH 210/396] DOC: update docstring --- pysatNASA/constellations/icon.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pysatNASA/constellations/icon.py b/pysatNASA/constellations/icon.py index 3df198a0..f65ca1d6 100644 --- a/pysatNASA/constellations/icon.py +++ b/pysatNASA/constellations/icon.py @@ -1,4 +1,11 @@ -"""Creates a constellation from NASA the ICON satellite platform.""" +"""Creates a constellation from NASA the ICON satellite platform. + +Includes the core instruments without the line of sight winds. + +Note that IVM A and B are nominally never active at the same time. This +constellation requires a bug-fix in pysat 3.1.0. + +""" import pysat From cb7e42c49079d629fa00a34d4ba4af428ba08796 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 16:48:17 -0400 Subject: [PATCH 211/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a151816..eae91750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Documentation * Added TIMED-GUVI platform * Added missing sub-module imports + * Added discussion of ICON constellation to docstrings, including caveats * Enhancements * Updated platform methods to follow a consistent style and work with the general `init` function From aadd5f789030c6cfd94a3f190027d609c2dc9f9c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 16:51:00 -0400 Subject: [PATCH 212/396] ENH: update DE2 constellation --- pysatNASA/constellations/de2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pysatNASA/constellations/de2.py b/pysatNASA/constellations/de2.py index eca99b6a..aa2883b3 100644 --- a/pysatNASA/constellations/de2.py +++ b/pysatNASA/constellations/de2.py @@ -5,9 +5,11 @@ from pysatNASA import instruments +fpi = pysat.Instrument(inst_module=instruments.de2_fpi) lang = pysat.Instrument(inst_module=instruments.de2_lang) nacs = pysat.Instrument(inst_module=instruments.de2_nacs) rpa = pysat.Instrument(inst_module=instruments.de2_rpa) wats = pysat.Instrument(inst_module=instruments.de2_wats) +vefi = pysat.Instrument(inst_module=instruments.de2_vefi) -instruments = [lang, nacs, rpa, wats] +instruments = [fpi, lang, nacs, rpa, wats, vefi] From 85362a557e006c8fbf04276ff06dfce1f2a1d48d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 3 Apr 2023 16:51:08 -0400 Subject: [PATCH 213/396] DOC: update docs --- docs/supported_constellations.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/supported_constellations.rst b/docs/supported_constellations.rst index f5ebf71f..d7e7eaa3 100644 --- a/docs/supported_constellations.rst +++ b/docs/supported_constellations.rst @@ -6,10 +6,12 @@ DE2 The Dynamics Explorer 2 spacecraft. Includes the instruments +- :ref:`de2_fpi` - :ref:`de2_lang` - :ref:`de2_nacs` - :ref:`de2_rpa` - :ref:`de2_wats` +- :ref:`de2_vefi` .. automodule:: pysatNASA.constellations.de2 :members: From 84c608a7d07125263a6c3aaafc4b3fc58b20b99f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 3 Apr 2023 19:45:40 -0400 Subject: [PATCH 214/396] Update de2_vefi.py --- pysatNASA/instruments/de2_vefi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index de4a304d..41863ba4 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -90,7 +90,7 @@ supported_tags=supported_tags) # Set the load routine -load = cdw.load +load = functools.partial(cdw.load, use_cdflib=True) # Set the download routine download_tags = {'': {'': 'DE2_62MS_VEFIMAGB', From 2e9d634be100812d21c06e8eb002bc3402e2e3ce Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 Apr 2023 09:42:20 -0400 Subject: [PATCH 215/396] DOC: add comment --- pysatNASA/instruments/de2_vefi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index 41863ba4..45bec0ad 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -90,6 +90,8 @@ supported_tags=supported_tags) # Set the load routine +# Forcing use of cdflib as default since pysatCDF has a known issue with vefi +# data. See pysat/pysatCDF#48 load = functools.partial(cdw.load, use_cdflib=True) # Set the download routine From dc80273298bfcb652aa4c7635bbb7334ed729e61 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 Apr 2023 09:43:44 -0400 Subject: [PATCH 216/396] STY: alphabetical order --- docs/supported_instruments.rst | 12 ++++++------ pysatNASA/instruments/__init__.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 1a8174e5..99bbe094 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -89,20 +89,20 @@ DE2 RPA .. automodule:: pysatNASA.instruments.de2_rpa :members: -.. _de2_wats: +.. _de2_vefi: -DE2 WATS +DE2 VEFI -------- -.. automodule:: pysatNASA.instruments.de2_wats +.. automodule:: pysatNASA.instruments.de2_vefi :members: -.. _de2_vefi: +.. _de2_wats: -DE2 VEFI +DE2 WATS -------- -.. automodule:: pysatNASA.instruments.de2_vefi +.. automodule:: pysatNASA.instruments.de2_wats :members: .. _dmsp_ssusi: diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index 3cc7d241..1b028313 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -8,7 +8,7 @@ __all__ = ['ace_epam_l2', 'ace_mag_l2', 'ace_sis_l2', 'ace_swepam_l2', 'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_fpi', - 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', 'de2_vefi', + 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_vefi', 'de2_wats', 'dmsp_ssusi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', From ef7152861189cb16bb91ff6761b28b5fcf96bbaf Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 Apr 2023 11:27:30 -0400 Subject: [PATCH 217/396] DOC: update constellation info --- pysatNASA/constellations/de2.py | 19 ++++++++++++++++++- pysatNASA/constellations/icon.py | 15 ++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/pysatNASA/constellations/de2.py b/pysatNASA/constellations/de2.py index eca99b6a..01a45624 100644 --- a/pysatNASA/constellations/de2.py +++ b/pysatNASA/constellations/de2.py @@ -1,4 +1,21 @@ -"""Creates a constellation from the NASA DE2 satellite platform.""" +"""Creates a constellation from the NASA DE2 satellite platform. + +Includes the core supported instruments. + + +Examples +-------- +:: + + import pysat + import pysatNASA + + de2 = pysat.Constellation(const_module=pysatNASA.constellations.de2) + + de2.load(1983, 1) + + +""" import pysat diff --git a/pysatNASA/constellations/icon.py b/pysatNASA/constellations/icon.py index f65ca1d6..42cd6462 100644 --- a/pysatNASA/constellations/icon.py +++ b/pysatNASA/constellations/icon.py @@ -3,7 +3,20 @@ Includes the core instruments without the line of sight winds. Note that IVM A and B are nominally never active at the same time. This -constellation requires a bug-fix in pysat 3.1.0. +constellation should be initialized with `common_index=False`. This forgoes +the pysat check that ensures all instruments load data. + +Examples +-------- +:: + + import pysat + import pysatNASA + + icon = pysat.Constellation(const_module=pysatNASA.constellations.icon, + common_index=False) + + icon.load(2020, 1) """ From 00a52b33e07194082bed9a1a9184fef4add95f47 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 Apr 2023 13:31:06 -0400 Subject: [PATCH 218/396] DOC: remove duct refs --- pysatNASA/instruments/de2_rpa.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 2a408f9e..c9161118 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -26,15 +26,10 @@ spectrum; and the concentration of H+, He+, O+, and Fe+, and of molecular ions near perigee. -It includes the DUCT portion of the high resolution data from the Dynamics -Explorer 2 (DE-2) Retarding Potential Analyzer (RPA) for the whole DE-2 mission -time period in ASCII format. This version was generated at NSSDC from the -PI-provided binary data (SPIO-00232). The DUCT files include RPA measurements -ofthe total ion concentration every 64 times per second. Due to a failure in -the instrument memory system RPA data are not available from 81317 06:26:40 UT -to 82057 13:16:00 UT. This data set is based on the revised version of the RPA -files that was submitted by the PI team in June of 1995. The revised RPA data -include a correction to the spacecraft potential. +Due to a failure in the instrument memory system RPA data are not available +from 81317 06:26:40 UT to 82057 13:16:00 UT. This data set is based on the +revised version of the RPA files that was submitted by the PI team in June of +1995. The revised RPA data include a correction to the spacecraft potential. Properties From 158073710cdb4ce2dd30135c6094d0957418eeb1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 Apr 2023 16:36:35 -0400 Subject: [PATCH 219/396] BUG: fix uniqueness --- pysatNASA/instruments/ses14_gold.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index e597aa87..d6c45d9f 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -156,9 +156,16 @@ def load(fnames, tag='', inst_id=''): drop_meta_labels='FILLVAL') if tag == 'nmax': - # Add time coordinate from scan_start_time - data['time'] = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") - for val in data['scan_start_time'].values] + # Add time coordinate from scan_start_time. + time = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") + for val in data['scan_start_time'].values] + + # Add a delta of 1 microsecond for channel B. + delta_time = [1 if ch == b'CHB' else 0 for ch in data['channel'].values] + data['time'] = [time[i] + dt.timedelta(microseconds=delta_time[i]) + for i in range(0, len(time))] + + # Sort times to ensure monotonic increase. data = data.sortby('time') # Update coordinates with dimensional data From 2e3ba2131598664704a24d0b9f93d9cc35f7b207 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 Apr 2023 16:39:28 -0400 Subject: [PATCH 220/396] DOC: update docstrings --- pysatNASA/instruments/ses14_gold.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index d6c45d9f..7da4c7df 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -14,9 +14,13 @@ Warnings -------- -- The cleaning parameters for the instrument are still under development. -- strict_time_flag must be set to False +The cleaning parameters for the instrument are still under development. +Note +---- +In roughly 0.3% of daily files, Channel A and Channel B scans begin at the same +time. One microsecond is added to Channel B to ensure uniqueness in the xarray +index. The nominal scan rate for each channel is every 30 minutes. Examples -------- @@ -24,8 +28,7 @@ import datetime as dt import pysat - nmax = pysat.Instrument(platform='ses14', name='gold', tag='nmax' - strict_time_flag=False) + nmax = pysat.Instrument(platform='ses14', name='gold', tag='nmax') nmax.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) nmax.load(2020, 1) From 9f75981377ace753903ec17776cb6534239ba823 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 4 Apr 2023 16:39:35 -0400 Subject: [PATCH 221/396] DOC: udpate changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0995c8fa..31fee1af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Bug Fixes * Updated CDAWeb routines to allow for data stored by year/day-of-year * Updated GOLD nmax to sort scans by time. + * Added 1 usec to GOLD nmax channel B times to ensure uniqueness * Documentation * Added TIMED-GUVI platform * Added missing sub-module imports From b2fcea3a56d889e3beca1d088a3dc1c10548994c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 12:36:52 -0400 Subject: [PATCH 222/396] ENH: add IGS --- pysatNASA/instruments/__init__.py | 2 +- pysatNASA/instruments/igs_gps.py | 121 +++++++++++++++++++++++++++ pysatNASA/instruments/methods/igs.py | 45 ++++++++++ 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 pysatNASA/instruments/igs_gps.py create mode 100644 pysatNASA/instruments/methods/igs.py diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index 1b028313..5407b031 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -11,7 +11,7 @@ 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_vefi', 'de2_wats', 'dmsp_ssusi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', - 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', + 'igs_gps', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', 'timed_guvi', 'timed_saber', 'timed_see'] for inst in __all__: diff --git a/pysatNASA/instruments/igs_gps.py b/pysatNASA/instruments/igs_gps.py new file mode 100644 index 00000000..d5290d20 --- /dev/null +++ b/pysatNASA/instruments/igs_gps.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +"""Module for the IGS GPS data products. + +Supports GPS data produced from International GNSS Service Total Electron +Content (TEC). + +From CDAWeb (modified): + +This directory contains the GPS Total Electron Content (TEC) data produced by +the International Global Navigation Satellite Systems Service (IGS) Ionosphere +Working Group and by the Analysis Centers that have contributed to the IGS data +including CODE of the University of Bern (Switzerland), ESA of the European +Space Operations Center (ESOC) in Darmstadt (Germany), JPL of the Jet Propulsion +Laboratory, Pasadena (USA), and UPC of the University Politechnical Catalonia in +Barcelona (Spain). The IGS data are a computed as a weighted mean of the data +from the four analysis centers. + +Properties +---------- +platform + 'igs' +name + 'gps' +tag + ['15min', '1hr', '2hr'] +inst_id + None supported + + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + + +References +---------- +M. Hernández-Pajares, J.M. Juan, J. Sanz, R. Orus, A. Garcia-Rigo, J. Feltens, +A. Komjathy, S.C. Schaer, and A. Krankowski, The IGS VTEC maps: a reliable +source of ionospheric information since 1998Journal of Geodesy (2009) 83:263–275 +doi:10.1007/s00190-008-0266-1 + +Feltens, J., M. Angling, N. Jackson‐Booth, N. Jakowski, M. Hoque, M. +Hernández‐Pajares, A. Aragón‐Àngel, R. Orús, and R. Zandbergen (2011), +Comparative testing of four ionospheric models driven with GPS measurements, +Radio Sci., 46, RS0D12, doi:10.1029/2010RS004584 + +Peng Chen, Hang Liu, Yongchao Ma, Naiquan Zheng, Accuracy and consistency of +different global ionospheric maps released by IGS ionosphere associate analysis +centers, Advances in Space Research, Volume 65, Issue 1, 2020, Pages 163-174, +doi:10.1016/j.asr.2019.09.042. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import igs as mm_igs + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'igs' +name = 'gps' +tags = {'tec': 'Total Electron Content', + 'roti': 'Rate of Change in TEC'} +# tags = {'15min': '15 min cadence TEC', +# '1hr': '1 hour cadence TEC', +# '2hr': '2 hour cadence TEC'} +inst_ids = {'15min': ['tec', 'roti'], + '1hr': ['tec'], + '2hr': ['tec']} +pandas_format = False +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {jj: {kk: dt.datetime(2013, 1, 1) for kk in inst_ids[jj]} + for jj in inst_ids.keys()} +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_igs, name=name) + + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +cdas_labels = {'15min': {'tec': 'GPS_TEC15MIN_IGS', + 'roti': 'GPS_ROTI15MIN_JPL'}, + '1hr': {'tec': 'GPS_TEC1HR_IGS'}, + '2hr': {'tec': 'GPS_TEC2HR_IGS'}} + +date_ver = '{year:4d}{month:02d}{day:02d}_v{version:02d}' +fname = '{cdas:s}_{date_ver:s}.cdf' + +supported_tags = {id: {tag: fname.format(cdas=cdas_labels[id][tag].lower(), + date_ver=date_ver) + for tag in inst_ids[id]} for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(cdw.load, pandas_format=pandas_format) + +# Set the download routine +download = functools.partial(cdw.cdas_download, supported_tags=cdas_labels) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=cdas_labels) diff --git a/pysatNASA/instruments/methods/igs.py b/pysatNASA/instruments/methods/igs.py new file mode 100644 index 00000000..63ad4a5d --- /dev/null +++ b/pysatNASA/instruments/methods/igs.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for JPL ROTI data.""" + +ackn_str = ' '.join(("The GPS Total Electron Content (TEC) data", + "produced by the International Global Navigation", + "Satellite Systems Service (IGS) Ionosphere Working", + "Group is provided through CDAWeb")) + +refs = {'mission': ' '.join(('Feltens J, Schaer S (1998) IGS Products for the', + 'Ionosphere, IGS Position Paper. In:', + 'Proceedings of the IGS analysis centers', + 'workshop, ESOC, Darmstadt, Germany,', + 'pp 225–232, 9–11 February')), + 'gps': {'tec': ' '.join(('M. Hernández-Pajares, J.M. Juan, J. Sanz, R.', + 'Orus, A. Garcia-Rigo, J. Feltens, A.', + 'Komjathy, S.C. Schaer, and A. Krankowski,', + 'The IGS VTEC maps: a reliable source of', + 'ionospheric information since 1998, Journal', + 'of Geodesy (2009) 83:263–275', + 'doi:10.1007/s00190-008-0266-1.\n', + 'Feltens, J., M. Angling, N. Jackson‐Booth,', + 'N. Jakowski, M. Hoque, M. Hernández‐Pajares,', + 'A. Aragón‐Àngel, R. Orús, and R. Zandbergen', + '(2011), Comparative testing of four', + 'ionospheric models driven with GPS', + 'measurements, Radio Sci., 46, RS0D12,', + 'doi:10.1029/2010RS004584.\n', + 'Peng Chen, Hang Liu, Yongchao Ma, Naiquan', + 'Zheng, Accuracy and consistency of different', + 'global ionospheric maps released by IGS', + 'ionosphere associate analysis centers,', + 'Advances in Space Research, Volume 65, Issue', + '1, 2020, Pages 163-174,', + 'doi:10.1016/j.asr.2019.09.042.\n')), + 'roti': ' '.join(('Pi, X., A. J. Mannucci, U. J.', + 'Lindqwister, and C. M. Ho, Monitoring of', + 'global ionospheric irregularities using', + 'the worldwide GPS network, Geophys. Res.', + 'Lett., 24, 2283, 1997.\n', + 'Pi, X., F. J. Meyer, K. Chotoo, Anthony', + 'Freeman, R. G. Caton, and C. T. Bridgwood,', + 'Impact of ionospheric scintillation on', + 'Spaceborne SAR observations studied using', + 'GNSS, Proc. ION-GNSS, pp.1998-2006,', + '2012.'))}} From 0248719bf4ae13c2088744b46c711d5631be6f37 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 12:37:13 -0400 Subject: [PATCH 223/396] ENH: allow tag-level refs --- pysatNASA/instruments/methods/general.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index 013e8c7c..aca19ff1 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -31,10 +31,16 @@ def init(self, module, name): # Set references refs = getattr(module, 'refs') + try: + # See if there is a tag level reference + inst_refs = refs[name][self.tag] + except TypeError: + # No tag-level ref, use name-levele + inst_refs = refs[name] if 'mission' in refs.keys(): - self.references = '\n'.join((refs['mission'], refs[name])) + self.references = '\n'.join((refs['mission'], inst_refs)) else: - self.references = refs[name] + self.references = inst_refs return From 8e8b251bd832a91fe0ec140cdfeeeb8f5141634f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 12:43:59 -0400 Subject: [PATCH 224/396] DEP: deprectae jpl gps roti --- pysatNASA/instruments/jpl_gps.py | 18 +++++++++++++----- pysatNASA/instruments/methods/gps.py | 8 +++++++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pysatNASA/instruments/jpl_gps.py b/pysatNASA/instruments/jpl_gps.py index 63787548..43414b0b 100644 --- a/pysatNASA/instruments/jpl_gps.py +++ b/pysatNASA/instruments/jpl_gps.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- """Module for the JPL GPS data products. +.. deprecated:: 0.0.5 + This module is now included in igs_gps.py. + This instrument will be removed in 0.1.0+ to reduce redundancy. + + Supports ROTI data produced at JPL from International GNSS Service Total Electron Content (TEC) @@ -48,6 +53,7 @@ import datetime as dt import functools +import warnings import pysat from pysat.instruments.methods import general as mm_gen @@ -80,6 +86,10 @@ def init(self): """ + warnings.warn(" ".join(["The instrument module `jpl_gps` has", + "been deprecated and will be removed in 0.1.0+."]), + DeprecationWarning, stacklevel=2) + pysat.logger.info('') self.acknowledgements = mm_gps.ackn_str self.references = '\n'.join((mm_gps.refs['mission'], @@ -106,11 +116,9 @@ def init(self): load = functools.partial(cdw.load, pandas_format=pandas_format) # Set the download routine -basic_tag = {'remote_dir': '/pub/data/gps/roti15min_jpl/{year:4d}/', - 'fname': fname} -download_tags = {'': {'roti': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'roti': 'GPS_ROTI15MIN_JPL'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/methods/gps.py b/pysatNASA/instruments/methods/gps.py index bb11168e..684ab544 100644 --- a/pysatNASA/instruments/methods/gps.py +++ b/pysatNASA/instruments/methods/gps.py @@ -1,5 +1,11 @@ # -*- coding: utf-8 -*- -"""Provides non-instrument specific routines for JPL ROTI data.""" +"""Provides non-instrument specific routines for JPL ROTI data. + +.. deprecated:: 0.0.5 + This module is now included in `methods.igs`. + This instrument will be removed in 0.1.0+ to reduce redundancy. + +""" ackn_str = ' '.join(("The GPS Total Electron Content (TEC) data", "produced by the International Global Navigation", From bd818a4e9af68f185bedebf19848f884256a35b0 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 12:44:14 -0400 Subject: [PATCH 225/396] DOC: update docstrings --- pysatNASA/instruments/igs_gps.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/igs_gps.py b/pysatNASA/instruments/igs_gps.py index d5290d20..e9e17bcc 100644 --- a/pysatNASA/instruments/igs_gps.py +++ b/pysatNASA/instruments/igs_gps.py @@ -15,6 +15,18 @@ Barcelona (Spain). The IGS data are a computed as a weighted mean of the data from the four analysis centers. +The rate of TEC index (ROTI) characterizes TEC fluctuations observed along +receiver-to-satellite line of sight links over a 5-minute interval. +The measurement is obtained by processing GNSS dual-frequency phase data and +computing the standard deviation of the rate of TEC change over that interval +after removing its background variation trend. + +ROTI data are provided as global maps using a 2.5 x 5 degree (geographic +latitude x longitude) grid. The median ROTI value is calculated in each bin. +GNSS data contributing to the ROTI computation are primarily collected from +the global network of International GNSS Service and the regional network of +Continuous Operating Reference Station (CORS). + Properties ---------- platform @@ -22,9 +34,9 @@ name 'gps' tag - ['15min', '1hr', '2hr'] + ['tec', 'roti'] inst_id - None supported + ['15min', '1hr', '2hr'] Warnings From 7cac0fabc8109453cf03e6770326ed8795f05d8e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 12:47:55 -0400 Subject: [PATCH 226/396] DOC: update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0995c8fa..ad952162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * DE2 Fabry-Perot Interferometer (FPI) * DE2 Vector Electric Field Instrument (VEFI) and magnetometer * DMSP SSUSI EDR-Aurora data + * IGS GPS (TEC and ROTI) * TIMED GUVI * Add TIMED GUVI platform to support L1C intensity datasets. * Type of sensor source handled by inst_id with options of @@ -30,6 +31,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). general `init` function * Added unit tests for the different platform method attributes * xarray support for TIMED SEE +* Deprecations + * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps * Maintenance * Removed duplicate tests if pysatCDF not isntalled * Only test pysatCDF on GitHub Actions for older numpy versions From 3c82579ecd8a7c85e65f94c610e5b28544fd04cb Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 12:54:31 -0400 Subject: [PATCH 227/396] TST: test deprecation --- pysatNASA/tests/test_instruments.py | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 25776e41..b19380bd 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -8,6 +8,7 @@ import warnings +import pysat import pytest # Make sure to import your instrument library here @@ -15,6 +16,7 @@ # Import the test classes from pysat from pysat.tests.classes import cls_instrument_library as clslib +from pysat.utils import testing try: import pysatCDF # noqa: F401 @@ -93,3 +95,48 @@ def test_load_cdflib(self, inst_dict): pytest.skip("Download data not available.") return + + +class TestDeprecation(object): + """Unit test for deprecation warnings.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + + warnings.simplefilter("always", DeprecationWarning) + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + return + + @pytest.mark.parametrize("inst_module,tag", [('jpl_gps', 'roti')]) + def test_deprecated_instruments(self, inst_module, tag): + """Check that instantiating old instruments raises a DeprecationWarning. + + Parameters + ---------- + inst_module : str + name of deprecated module. + tag : str + tag of depracted instrument. + + """ + + with warnings.catch_warnings(record=True) as war: + pysat.Instrument(inst_module=getattr(pysatNASA.instruments, + inst_module), + tag=tag, use_header=True) + + warn_msgs = [" ".join(["The instrument module", + "`{:}`".format(inst_module), + "has been deprecated and will be removed", + "in 0.1.0+."])] + + # Ensure the minimum number of warnings were raised. + assert len(war) >= len(warn_msgs) + + # Test the warning messages, ensuring each attribute is present. + testing.eval_warnings(war, warn_msgs) + return From c4eddf445ccf2648d28420c5360e10c958f77043 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 13:42:46 -0400 Subject: [PATCH 228/396] BUG: fix concat for xarray --- pysatNASA/instruments/methods/cdaweb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index acfbc92a..480c708a 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -359,9 +359,9 @@ def load_xarray(fnames, tag='', inst_id='', temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True) ldata.append(temp_data) - # Combine individual files together + # Combine individual files together, concat along epoch if len(ldata) > 0: - data = xr.combine_by_coords(ldata) + data = xr.combine_nested(ldata, epoch_name) all_vars = io.xarray_all_vars(data) From 9e431fe0bccffde114b6628d3df6cc42f64159c6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 13:43:03 -0400 Subject: [PATCH 229/396] ENH: add drop_dims kwarg --- pysatNASA/instruments/methods/cdaweb.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 480c708a..02407a85 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -64,8 +64,8 @@ def try_inst_dict(inst_id, tag, supported_tags): def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), flatten_twod=True, pandas_format=True, epoch_name='Epoch', - meta_processor=None, meta_translation=None, drop_meta_labels=None, - use_cdflib=None): + drop_dims=None, meta_processor=None, meta_translation=None, + drop_meta_labels=None, use_cdflib=None): """Load NASA CDAWeb CDF files. Parameters @@ -93,6 +93,10 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), specified by `epoch_origin` with units specified by `epoch_unit`. This epoch variable will be converted to a `DatetimeIndex` for consistency across pysat instruments. (default='Epoch') + drop_dims : list or NoneType + List of variable dimensions that should be dropped. Applied + to data as loaded from the file. Used only from xarray Dataset. + (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -143,6 +147,7 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id, epoch_name=epoch_name, + drop_dims=drop_dims, file_cadence=file_cadence, meta_processor=meta_processor, meta_translation=meta_translation, @@ -264,7 +269,7 @@ def load_xarray(fnames, tag='', inst_id='', 'min_val': ('ValidMin', float), 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)}, - epoch_name='Epoch', meta_processor=None, + epoch_name='Epoch', drop_dims=None, meta_processor=None, meta_translation=None, drop_meta_labels=None): """Load NASA CDAWeb CDF files into an xarray Dataset. @@ -295,6 +300,9 @@ def load_xarray(fnames, tag='', inst_id='', specified by `epoch_origin` with units specified by `epoch_unit`. This epoch variable will be converted to a `DatetimeIndex` for consistency across pysat instruments. (default='Epoch') + drop_dims : list or NoneType + List of variable dimensions that should be dropped. Applied + to data as loaded from the file. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -357,6 +365,8 @@ def load_xarray(fnames, tag='', inst_id='', for lfname in lfnames: temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True) + if drop_dims: + temp_data = temp_data.drop_dims(drop_dims) ldata.append(temp_data) # Combine individual files together, concat along epoch From 3515c84061d69b02803c0e52db0b0c1b97689642 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 13:43:13 -0400 Subject: [PATCH 230/396] ENH: saber xarray --- pysatNASA/instruments/timed_saber.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index 16357673..70256b2d 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -68,7 +68,7 @@ # Set to False to specify using xarray (not using pandas) # Set to True if data will be returned via a pandas DataFrame -pandas_format = True +pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -99,11 +99,12 @@ supported_tags=supported_tags) # Set the load routine -load = cdw.load +load = functools.partial(cdw.load, pandas_format=pandas_format, + drop_dims='record0') # Set the download routine download_tags = {'': {'': 'TIMED_L2A_SABER'}} -download = functools.partial(cdw.cdas_download, supported_tags=download_tags) +download = functools.partial(cdw.download, supported_tags=download_tags) # Set the list_remote_files routine list_remote_files = functools.partial(cdw.cdas_list_remote_files, From d2d5249d057cc819b0f866fa0ae795b9dd26763b Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 13:43:37 -0400 Subject: [PATCH 231/396] DOC: update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31fee1af..d65150e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated platform methods to follow a consistent style and work with the general `init` function * Added unit tests for the different platform method attributes - * xarray support for TIMED SEE + * xarray support for TIMED SABER and SEE * Maintenance * Removed duplicate tests if pysatCDF not isntalled * Only test pysatCDF on GitHub Actions for older numpy versions From 6a64f9ca6a2c0f63a27550568fbe558bddefb207 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 13:48:08 -0400 Subject: [PATCH 232/396] DOC: update supported instruments --- docs/supported_instruments.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 99bbe094..0980414e 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -154,6 +154,14 @@ ICON MIGHTI .. automodule:: pysatNASA.instruments.icon_mighti :members: +.. _igs_gps: + +IGS GPS +------- + +.. automodule:: pysatNASA.instruments.igs_gps + :members: + .. _iss_fpmu: ISS FPMU From 1c257671639fa4d22688979626a8bf19c87e82de Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 13:57:32 -0400 Subject: [PATCH 233/396] DOC: update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d65150e5..689178f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated CDAWeb routines to allow for data stored by year/day-of-year * Updated GOLD nmax to sort scans by time. * Added 1 usec to GOLD nmax channel B times to ensure uniqueness + * Fixed multi-file loads for cdf xarray datasets. * Documentation * Added TIMED-GUVI platform * Added missing sub-module imports @@ -31,6 +32,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). general `init` function * Added unit tests for the different platform method attributes * xarray support for TIMED SABER and SEE + * Added `drop_dims` kwarg to `load_xarray` interface so that orphan dims can + be removed before attempting to merge. * Maintenance * Removed duplicate tests if pysatCDF not isntalled * Only test pysatCDF on GitHub Actions for older numpy versions From 9faf534865a7e858e13fa79c71ec807ca2cbfabe Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 14:05:29 -0400 Subject: [PATCH 234/396] Merge branch 'develop' into sty/pyproject --- .github/ISSUE_TEMPLATE/bug_report.md | 34 +- .github/ISSUE_TEMPLATE/feature_request.md | 24 +- .github/ISSUE_TEMPLATE/question.md | 19 ++ .github/pull_request_template.md | 13 +- .github/workflows/docs.yml | 6 +- .github/workflows/main.yml | 21 +- .github/workflows/pip_rc_install.yml | 30 ++ .github/workflows/pysat_rc.yml | 51 +++ .gitignore | 3 + .zenodo.json | 5 + CHANGELOG.md | 46 ++- CONTRIBUTING.md | 55 +-- LICENSE | 2 +- README.md | 46 ++- docs/conf.py | 11 +- docs/installation.rst | 2 +- docs/supported_constellations.rst | 2 + docs/supported_instruments.rst | 64 ++++ pysatNASA/constellations/de2.py | 23 +- pysatNASA/constellations/icon.py | 22 +- pysatNASA/instruments/__init__.py | 10 +- pysatNASA/instruments/ace_epam_l2.py | 106 ++++++ pysatNASA/instruments/ace_mag_l2.py | 110 ++++++ pysatNASA/instruments/ace_sis_l2.py | 100 ++++++ pysatNASA/instruments/ace_swepam_l2.py | 104 ++++++ pysatNASA/instruments/cnofs_ivm.py | 45 ++- pysatNASA/instruments/cnofs_plp.py | 22 +- pysatNASA/instruments/cnofs_vefi.py | 22 +- pysatNASA/instruments/de2_fpi.py | 107 ++++++ pysatNASA/instruments/de2_lang.py | 21 +- pysatNASA/instruments/de2_nacs.py | 23 +- pysatNASA/instruments/de2_rpa.py | 48 ++- pysatNASA/instruments/de2_vefi.py | 105 ++++++ pysatNASA/instruments/de2_wats.py | 23 +- pysatNASA/instruments/dmsp_ssusi.py | 123 +++++++ pysatNASA/instruments/formosat1_ivm.py | 11 +- pysatNASA/instruments/icon_euv.py | 27 +- pysatNASA/instruments/icon_fuv.py | 34 +- pysatNASA/instruments/icon_ivm.py | 29 +- pysatNASA/instruments/icon_mighti.py | 56 ++- pysatNASA/instruments/iss_fpmu.py | 9 +- pysatNASA/instruments/jpl_gps.py | 22 +- pysatNASA/instruments/methods/__init__.py | 5 + pysatNASA/instruments/methods/ace.py | 54 +++ pysatNASA/instruments/methods/cdaweb.py | 326 +++++++++++++++--- pysatNASA/instruments/methods/de2.py | 17 +- pysatNASA/instruments/methods/dmsp.py | 16 + pysatNASA/instruments/methods/general.py | 4 + pysatNASA/instruments/methods/gold.py | 24 -- pysatNASA/instruments/methods/jhuapl.py | 395 ++++++++++++++++++++++ pysatNASA/instruments/methods/ses14.py | 24 ++ pysatNASA/instruments/methods/timed.py | 33 ++ pysatNASA/instruments/omni_hro.py | 58 +--- pysatNASA/instruments/ses14_gold.py | 78 ++--- pysatNASA/instruments/timed_guvi.py | 197 +++++++++++ pysatNASA/instruments/timed_saber.py | 44 +-- pysatNASA/instruments/timed_see.py | 53 +-- pysatNASA/tests/test_instruments.py | 16 +- pysatNASA/tests/test_methods_cdaweb.py | 37 +- pysatNASA/tests/test_methods_platform.py | 129 +++++++ pysatNASA/tests/test_omni_hro.py | 15 +- requirements.txt | 10 + tox.ini => setup.cfg | 0 63 files changed, 2603 insertions(+), 568 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/workflows/pip_rc_install.yml create mode 100644 .github/workflows/pysat_rc.yml create mode 100644 pysatNASA/instruments/ace_epam_l2.py create mode 100644 pysatNASA/instruments/ace_mag_l2.py create mode 100644 pysatNASA/instruments/ace_sis_l2.py create mode 100644 pysatNASA/instruments/ace_swepam_l2.py create mode 100644 pysatNASA/instruments/de2_fpi.py create mode 100644 pysatNASA/instruments/de2_vefi.py create mode 100644 pysatNASA/instruments/dmsp_ssusi.py create mode 100644 pysatNASA/instruments/methods/ace.py create mode 100644 pysatNASA/instruments/methods/dmsp.py delete mode 100644 pysatNASA/instruments/methods/gold.py create mode 100644 pysatNASA/instruments/methods/jhuapl.py create mode 100644 pysatNASA/instruments/methods/ses14.py create mode 100644 pysatNASA/instruments/methods/timed.py create mode 100644 pysatNASA/instruments/timed_guvi.py create mode 100644 pysatNASA/tests/test_methods_platform.py create mode 100644 requirements.txt rename tox.ini => setup.cfg (100%) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 23d236d6..57640182 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,35 +1,29 @@ --- name: Bug report -about: Create a report to help us improve +about: Create a report to report a problem that needs to be fixed +labels: bug +title: "BUG: " --- -**Describe the bug** -A clear and concise description of what the bug is. +# Description +A clear and concise description of what the bug is, including a description +of what you expected the outcome to be. -**To Reproduce** +# To Reproduce this bug: Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -or +Consider including images or test files to help others reproduce the bug and +solve the problem. -``` -# test code here -``` - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Version [e.g. Python 3.7] +## Test configuration + - OS: [e.g. Hal] + - Version [e.g. Python 3.47] - Other details about your setup that could be relevant -**Additional context** -Add any other context about the problem here. +# Additional context +Add any other context about the problem here, including expected behaviour. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 066b2d92..d02da2ef 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,17 +1,27 @@ --- name: Feature request about: Suggest an idea for this project +title: "ENH: " +labels: enhancement --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +# Description +A clear and concise description of the new feature or behaviour you would like. -**Describe the solution you'd like** +## Potential impact + +- Is the feature related to an existing problem? +- How critical is this feature to your workflow? +- How wide of an impact to you anticipate this enhancement having? +- Would this break any existing functionality? + +## Potential solution(s) A clear and concise description of what you want to happen. -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +# Alternatives +A clear description of any alternative solutions or features you've considered. -**Additional context** -Add any other context or screenshots about the feature request here. +# Additional context +Add any other context or screenshots about the feature request here, potentially +including your operational configuration. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 00000000..da43edc7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,19 @@ +--- +name: Question +about: A question about this project +title: "QUEST: " +labels: question + +--- + +# Description +A clear and concise summary of your query + +## Example code (optional) +If relevant, include sample code, images, or files so that others can understand +the full context of your question. + +## Configuration + - OS: [e.g. Hal] + - Version: [e.g. Python 3.47] + - Other details about your setup that could be relevant diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5331eb30..e1d2dbe5 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,12 @@ # Description -Addresses # (issue) +Addresses #(issue) Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. Please see ``CONTRIBUTING.md`` for more guidelines. -## Type of change +# Type of change Please delete options that are not relevant. @@ -25,9 +25,9 @@ your test configuration - Test A - Test B -## Test Configuration -* Operating system: [Os Type] -* Version number: [Python 2.9] +**Test Configuration**: +* Operating system: Hal +* Version number: Python 3.X * Any details about your local setup that are relevant # Checklist: @@ -43,3 +43,6 @@ your test configuration - [ ] Any dependent changes have been merged and published in downstream modules - [ ] Add a note to ``CHANGELOG.md``, summarizing the changes - [ ] Update zenodo.json file for new code contributors + +If this is a release PR, replace the first item of the above checklist with the release +checklist on the wiki: https://github.com/pysat/pysat/wiki/Checklist-for-Release diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4baa1b71..fe6d6ca4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,4 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# This workflow will install Python dependencies and check the sphinx build, links in the docs, and the readability of the zenodo file # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Documentation Check @@ -8,11 +8,11 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ["ubuntu-latest"] strategy: fail-fast: false matrix: - python-version: [3.9] + python-version: ["3.10"] # Keep this version at the highest supported Python version name: Documentation tests steps: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fec0dc3f..80643508 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,13 +10,13 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.9", "3.10"] - numpy_ver: [latest] + numpy_ver: ["latest"] include: - python-version: "3.8" - numpy_ver: "1.20" - os: ubuntu-latest + numpy_ver: "1.21" + os: "ubuntu-latest" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} @@ -27,25 +27,22 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Reinstall fortran on MacOS + if: ${{ matrix.os == 'macos-latest' }} + run: brew reinstall gcc + - name: Install NEP29 dependencies if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - # Force install version compatible with NEP29 - pip install "pandas<1.5" pip install --upgrade-strategy only-if-needed .[test] + pip install pysatCDF --no-binary=pysatCDF - name: Install standard dependencies if: ${{ matrix.numpy_ver == 'latest'}} run: | pip install .[test] - - name: Install pysatCDF - # Optional package, continue with tests if install fails - continue-on-error: true - # Need custom install until pysatCDF is updated - run: pip install pysatCDF --no-binary=pysatCDF - - name: Set up pysat run: | mkdir pysatData diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml new file mode 100644 index 00000000..45d835e5 --- /dev/null +++ b/.github/workflows/pip_rc_install.yml @@ -0,0 +1,30 @@ +# This workflow will install Python dependencies and the latest RC of pysatNASA from test pypi. +# This test should be manually run before a pysatNASA RC is officially approved and versioned. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Test install of latest RC from pip + +on: [workflow_dispatch] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.10"] # Keep this version at the highest supported Python version + + name: Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install standard dependencies + run: pip install -r requirements.txt + + - name: Install pysatNASA RC + run: pip install --no-deps -i https://test.pypi.org/simple/ pysatNASA diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml new file mode 100644 index 00000000..c73bcf07 --- /dev/null +++ b/.github/workflows/pysat_rc.yml @@ -0,0 +1,51 @@ +# This workflow will install Python dependencies and the latest RC of pysat from test pypi. +# All unit tests for pysatNASA will be run using the pysat RC. +# This test should be manually run before a pysat RC is officially approved and versioned. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Test with latest pysat RC + +on: [workflow_dispatch] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.10"] # Keep this version at the highest supported Python version + + name: Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Reinstall fortran on MacOS + if: ${{ matrix.os == 'macos-latest' }} + run: brew reinstall gcc + + - name: Install pysat RC + run: pip install --no-deps -i https://test.pypi.org/simple/ pysat + + - name: Install standard dependencies + run: | + pip install -r requirements.txt + pip install pysatCDF --no-binary=pysatCDF + pip install -r test_requirements.txt + + - name: Set up pysat + run: | + mkdir pysatData + python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" + + - name: Test with pytest + run: pytest -vs --cov=pysatNASA/ + + - name: Publish results to coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: coveralls --rcfile=setup.cfg --service=github diff --git a/.gitignore b/.gitignore index 6db464db..c8350919 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,6 @@ custom_lint.sh # IDEs .idea/ + +# vscode +.vscode diff --git a/.zenodo.json b/.zenodo.json index a5bcff9a..1052a817 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -27,6 +27,11 @@ { "name": "Spence, Carey", "orcid": "0000-0001-8340-5625" + }, + { + "affilitation":"University of Colorado at Boulder", + "name": "Navarro, Luis", + "orcid": "0000-0002-6362-6575" } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index e1761132..d06311f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,46 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -## [0.0.4] - 2022-XX-XX +## [0.X.X] - 2023-XX-XX +* New Instruments + * ACE EPAM + * ACE MAG + * ACE SIS + * ACE SWEPAM + * DE2 Fabry-Perot Interferometer (FPI) + * DE2 Vector Electric Field Instrument (VEFI) and magnetometer + * DMSP SSUSI EDR-Aurora data + * TIMED GUVI +* Add TIMED GUVI platform to support L1C intensity datasets. + * Type of sensor source handled by inst_id with options of + spectrograph, imaging + * Resolution of dataset handled by tag with + low, high +* Added CDAWeb methods that can use cdasws to get the remote file list +* Bug Fixes + * Updated CDAWeb routines to allow for data stored by year/day-of-year + * Updated GOLD nmax to sort scans by time. + * Added 1 usec to GOLD nmax channel B times to ensure uniqueness +* Documentation + * Added TIMED-GUVI platform + * Added missing sub-module imports + * Added discussion of ICON constellation to docstrings, including caveats +* Enhancements + * Updated platform methods to follow a consistent style and work with the + general `init` function + * Added unit tests for the different platform method attributes + * xarray support for TIMED SEE +* Maintenance + * Removed duplicate tests if pysatCDF not isntalled + * Only test pysatCDF on GitHub Actions for older numpy versions + * Updated actions and templates based on pysatEcosystem docs + * Remove pandas cap on NEP29 tests + * Updated dosctring style for consistency + * Removed version cap for xarray + * Added manual workflow to check that latest RC is installable through test pip + * Use pyproject.toml to manage setup + +## [0.0.4] - 2022-11-07 * Update instrument tests with new test class * Support xarray datasets through cdflib * Preferentially loads data into pandas using pysatCDF if installed @@ -22,7 +61,10 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Move OMNI HRO custom functions to a methods module * Deprecate OMNI HRO custom functions in instrument module * Update GitHub actions to the latest versions - * Use pyproject.toml to manage setup + * Added downstream test to test code with pysat RC + * Remove deprecated `convert_timestamp_to_datetime` calls + * Remove deprecated pandas syntax + * Added version cap for xarray 2022.11 * Documentation * New logo added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5e9b2a04..45943ed4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,33 +78,44 @@ To set up `pysatNASA` for local development: 4. When you're done making changes, run all the checks to ensure that nothing - is broken on your local system, as well as check for flake8 compliance: + is broken on your local system: - ``` - pytest -vs --flake8 pysatNASA - ``` + ``` + pytest pysatNASA + ``` + +5. You should also check for flake8 style compliance: + + ``` + flake8 . --count --select=D,E,F,H,W --show-source --statistics + ``` -5. Update/add documentation (in ``docs``), if relevant + Note that pysat uses the `flake-docstrings` and `hacking` packages to ensure + standards in docstring formatting. -6. Add your name to the .zenodo.json file as an author -7. Commit your changes: - ``` - git add . - git commit -m "AAA: Brief description of your changes" - ``` - Where AAA is a standard shorthand for the type of change (eg, BUG or DOC). - `pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html), - see the discussion there for a full list of this shorthand notation. +6. Update/add documentation (in ``docs``), if relevant -8. Once you are happy with the local changes, push to Github: - ``` - git push origin name-of-your-bugfix-or-feature - ``` - Note that each push will trigger the Continuous Integration workflow. +7. Add your name to the .zenodo.json file as an author + +8. Commit your changes: + ``` + git add . + git commit -m "AAA: Brief description of your changes" + ``` + Where AAA is a standard shorthand for the type of change (eg, BUG or DOC). + `pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html), + see the discussion there for a full list of this shorthand notation. + +9. Once you are happy with the local changes, push to Github: + ``` + git push origin name-of-your-bugfix-or-feature + ``` + Note that each push will trigger the Continuous Integration workflow. -9. Submit a pull request through the GitHub website. Pull requests should be - made to the ``develop`` branch. +10. Submit a pull request through the GitHub website. Pull requests should be + made to the ``develop`` branch. Note that automated tests will be run on + github actions, but these must be initialized by a member of the pysat team. Pull Request Guidelines ----------------------- @@ -160,3 +171,5 @@ These include: * Block and inline comments should use proper English grammar and punctuation with the exception of single sentences in a block, which may then omit the final period +* When casting is necessary, use `np.int64` and `np.float64` to ensure operating + system agnosticism diff --git a/LICENSE b/LICENSE index c9a10064..666d8f5d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2020, pysat +Copyright (c) 2023, pysat All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index 1fceb954..9bdce9c5 100644 --- a/README.md +++ b/README.md @@ -22,20 +22,23 @@ pysatNASA uses common Python modules, as well as modules developed by and for the Space Physics community. This module officially supports Python 3.8+. -| Common modules | Community modules | -| ---------------- | ----------------- | -| beautifulsoup4 | cdflib | -| lxml | pysat>=3.0.4 | -| netCDF4 | | -| numpy | | -| pandas | | -| requests | | -| xarray | | +| Common modules | Community modules | Optional Modules | +| ---------------- | ----------------- |------------------| +| beautifulsoup4 | cdflib | pysatCDF | +| lxml | pysat>=3.0.4 | | +| netCDF4 | | | +| numpy<1.24 | | | +| pandas | | | +| requests | | | +| xarray | | | + +## PyPi Installation +``` +pip install pysatNASA +``` ## GitHub Installation -Currently, the main way to get pysatNASA is through github. - ``` git clone https://github.com/pysat/pysatNASA.git ``` @@ -51,6 +54,7 @@ pip install . Note: pre-1.0.0 version ----------------------- pysatNASA is currently in an initial development phase and requires pysat 3.0.4. +Feedback and contributions are appreciated. # Using with pysat @@ -62,7 +66,9 @@ from pysatNASA.instruments import icon_ivm ivm = pysat.Instrument(inst_module=icon_ivm, inst_id='a') ``` -Another way to use the instruments in an external repository is to register the instruments. This only needs to be done the first time you load an instrument. Afterward, pysat will identify them using the `platform` and `name` keywords. +Another way to use the instruments in an external repository is to register the +instruments. This only needs to be done the first time you load an instrument. +Afterward, pysat will identify them using the `platform` and `name` keywords. ``` import pysat @@ -70,3 +76,19 @@ import pysat pysat.utils.registry.register(['pysatNASA.instruments.icon_ivm']) ivm = pysat.Instrument('icon', 'ivm', inst_id='a') ``` + +# CDF Integration +For data products stored as CDF files, this package can use either `cdflib` or +`pysatCDF`. Note that `cdflib` is a pure python package and more readily +deployable across systems, whereas `pysatCDF` interfaces with the fortran. +This is a faster approach for loading data, but may not install on all systems. +There are known issues with `numpy`>=1.24. Therefore, `pysatCDF` is optional +rather than required. + +You can specify which load routine to use via the optional `use_cdflib` kwarg. +If no kwarg is specified, `pysatNASA` will default to `pysatCDF` if it is +successfully installed, and default to `cdflib` otherwise. + +``` +ivm = pysat.Instrument('cnofs', 'ivm', use_cdflib=True) +``` diff --git a/docs/conf.py b/docs/conf.py index 4a6c06f2..43245c9d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,11 +53,12 @@ master_doc = 'index' # General information about the project. -zenodo = json.loads(open('../.zenodo.json').read()) project = 'pysatNASA' -author = ', '.join([creator['name'] for creator in zenodo['creators']]) -manual_copyright = ', '.join(['2021', author]) title = '{:s} Documentation'.format(project) +zenodo = json.loads(open('../.zenodo.json').read()) +author = ', '.join([creator['name'] for creator in zenodo['creators']]) +manual_copyright = ', '.join(['2023', author]) +category = 'Space Physics' description = 'Tools for NASA CDAWeb instruments.' # The version info for the project you're documenting, acts as replacement for @@ -147,8 +148,8 @@ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) -texinfo_documents = [(master_doc, project, title, author, project, description, - 'Space Physics')] +texinfo_documents = [(master_doc, project, title, author, project, + description, category)] # -- Options for Epub output ---------------------------------------------- diff --git a/docs/installation.rst b/docs/installation.rst index 7ba6d0ba..ae8eedda 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -25,7 +25,7 @@ Python 3.8+ and pysat 3.0.4+. numpy pandas requests - xarray + xarray<2022.11 ================== ================= diff --git a/docs/supported_constellations.rst b/docs/supported_constellations.rst index f5ebf71f..d7e7eaa3 100644 --- a/docs/supported_constellations.rst +++ b/docs/supported_constellations.rst @@ -6,10 +6,12 @@ DE2 The Dynamics Explorer 2 spacecraft. Includes the instruments +- :ref:`de2_fpi` - :ref:`de2_lang` - :ref:`de2_nacs` - :ref:`de2_rpa` - :ref:`de2_wats` +- :ref:`de2_vefi` .. automodule:: pysatNASA.constellations.de2 :members: diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 054bc982..99bbe094 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -1,6 +1,38 @@ Supported Instruments ===================== +.. _ace_epam: + +ACE EPAM +-------- + +.. automodule:: pysatNASA.instruments.ace_epam_l2 + :members: + +.. _ace_mag: + +ACE MAG +------- + +.. automodule:: pysatNASA.instruments.ace_mag_l2 + :members: + +.. _ace_sis: + +ACE SIS +------- + +.. automodule:: pysatNASA.instruments.ace_sis_l2 + :members: + +.. _ace_swepam: + +ACE SWEPAM +---------- + +.. automodule:: pysatNASA.instruments.ace_swepam_l2 + :members: + .. _cnofs_ivm: C/NOFS IVM @@ -25,6 +57,14 @@ C/NOFS VEFI .. automodule:: pysatNASA.instruments.cnofs_vefi :members: +.. _de2_fpi: + +DE2 FPI +-------- + +.. automodule:: pysatNASA.instruments.de2_fpi + :members: + .. _de2_lang: DE2 LANG @@ -49,6 +89,14 @@ DE2 RPA .. automodule:: pysatNASA.instruments.de2_rpa :members: +.. _de2_vefi: + +DE2 VEFI +-------- + +.. automodule:: pysatNASA.instruments.de2_vefi + :members: + .. _de2_wats: DE2 WATS @@ -57,6 +105,14 @@ DE2 WATS .. automodule:: pysatNASA.instruments.de2_wats :members: +.. _dmsp_ssusi: + +DMSP SSUSI +---------- + +.. automodule:: pysatNASA.instruments.dmsp_ssusi + :members: + .. _formosat1_ivm: FORMOSAT-1 IVM @@ -130,6 +186,14 @@ SES14 GOLD .. automodule:: pysatNASA.instruments.ses14_gold :members: +.. _timed_guvi: + +TIMED GUVI +---------- + +.. automodule:: pysatNASA.instruments.timed_guvi + :members: + .. _timed_saber: TIMED SABER diff --git a/pysatNASA/constellations/de2.py b/pysatNASA/constellations/de2.py index eca99b6a..2a5ad572 100644 --- a/pysatNASA/constellations/de2.py +++ b/pysatNASA/constellations/de2.py @@ -1,13 +1,32 @@ -"""Creates a constellation from the NASA DE2 satellite platform.""" +"""Creates a constellation from the NASA DE2 satellite platform. + +Includes the core supported instruments. + + +Examples +-------- +:: + + import pysat + import pysatNASA + + de2 = pysat.Constellation(const_module=pysatNASA.constellations.de2) + + de2.load(1983, 1) + + +""" import pysat from pysatNASA import instruments +fpi = pysat.Instrument(inst_module=instruments.de2_fpi) lang = pysat.Instrument(inst_module=instruments.de2_lang) nacs = pysat.Instrument(inst_module=instruments.de2_nacs) rpa = pysat.Instrument(inst_module=instruments.de2_rpa) wats = pysat.Instrument(inst_module=instruments.de2_wats) +vefi = pysat.Instrument(inst_module=instruments.de2_vefi) -instruments = [lang, nacs, rpa, wats] +instruments = [fpi, lang, nacs, rpa, wats, vefi] diff --git a/pysatNASA/constellations/icon.py b/pysatNASA/constellations/icon.py index 3df198a0..42cd6462 100644 --- a/pysatNASA/constellations/icon.py +++ b/pysatNASA/constellations/icon.py @@ -1,4 +1,24 @@ -"""Creates a constellation from NASA the ICON satellite platform.""" +"""Creates a constellation from NASA the ICON satellite platform. + +Includes the core instruments without the line of sight winds. + +Note that IVM A and B are nominally never active at the same time. This +constellation should be initialized with `common_index=False`. This forgoes +the pysat check that ensures all instruments load data. + +Examples +-------- +:: + + import pysat + import pysatNASA + + icon = pysat.Constellation(const_module=pysatNASA.constellations.icon, + common_index=False) + + icon.load(2020, 1) + +""" import pysat diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index c3a8bfae..1b028313 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -4,13 +4,15 @@ Each instrument is contained within a subpackage of this set. """ +from pysatNASA.instruments import methods # noqa F401 -__all__ = ['cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', - 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_wats', - 'formosat1_ivm', +__all__ = ['ace_epam_l2', 'ace_mag_l2', 'ace_sis_l2', 'ace_swepam_l2', + 'cnofs_ivm', 'cnofs_plp', 'cnofs_vefi', 'de2_fpi', + 'de2_lang', 'de2_nacs', 'de2_rpa', 'de2_vefi', 'de2_wats', + 'dmsp_ssusi', 'formosat1_ivm', 'icon_euv', 'icon_fuv', 'icon_ivm', 'icon_mighti', 'iss_fpmu', 'jpl_gps', 'omni_hro', 'ses14_gold', - 'timed_saber', 'timed_see'] + 'timed_guvi', 'timed_saber', 'timed_see'] for inst in __all__: exec("from pysatNASA.instruments import {x}".format(x=inst)) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py new file mode 100644 index 00000000..670ed3ad --- /dev/null +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +"""Module for the Advanced Composition Explorer (ACE) EPAM instrument. + +Properties +---------- +platform + 'ace' +name + 'epam_l2' +tag + 'base' or 'key' +inst_id + '12sec', '5min', '1hr' + +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Gold, R., Krimigis, S., Hawkins, S. et al. Electron, Proton, and Alpha Monitor + on the Advanced Composition Explorer spacecraft. Space Science Reviews 86, + 541–562 (1998). https://doi.org/10.1023/A:1005088115759 + +Note +---- +- Level 1 ACE data is maintained at pysatSpaceWeather. +- Release notes at + https://cdaweb.gsfc.nasa.gov/pub/data/ace/epam/epam_level2_release_notes.txt + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'epam_l2' +tags = {'base': 'ACE/EPAM Solar Energetic Particle Base Data', + 'key': 'ACE/EPAM Solar Energetic Particle Key Parameters'} +inst_ids = {'12sec': ['base'], + '5min': ['key', 'base'], + '1hr': ['key', 'base']} +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {tag: dt.datetime(2022, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# Use default ace clean +clean = mm_ace.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# + +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +strid = {'12sec': {'base': 'h3'}, + '5min': {'base': 'h1', 'key': 'k0'}, + '1hr': {'base': 'h2', 'key': 'k1'}} +fname = ''.join(('ac_{sid:s}_epm_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} + for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} +load = functools.partial(cdw.load, pandas_format=pandas_format, + meta_translation=meta_translation) + +# Set the download routine +download_tags = {'12sec': {'base': 'AC_H3_EPM'}, + '5min': {'base': 'AC_H1_EPM', 'key': 'AC_K0_EPM'}, + '1hr': {'base': 'AC_H2_EPM', 'key': 'AC_K1_EPM'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py new file mode 100644 index 00000000..8b5c4ea6 --- /dev/null +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +"""Module for the Advanced Composition Explorer (ACE) MAG instrument. + +Properties +---------- +platform + 'ace' +name + 'mag_l2' +tag + 'base' or 'key' +inst_id + '1sec', '16sec', '4min', '5min', '1hr' + +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Smith, C., L'Heureux, J., Ness, N. et al. The ACE Magnetic Fields Experiment. + Space Science Reviews 86, 613–632 (1998). + https://doi.org/10.1023/A:1005092216668 + +Note +---- +- Level 1 ACE data is maintained at pysatSpaceWeather. +- Release notes at + https://cdaweb.gsfc.nasa.gov/pub/data/ace/mag/mag_level2_release_notes.txt + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'mag_l2' +tags = {'base': 'ACE Magnetic Field Base Data', + 'key': 'ACE Magnetic Field Key Parameters'} +inst_ids = {'1sec': ['base'], + '16sec': ['base', 'key'], + '4min': ['base'], + '5min': ['key'], + '1hr': ['base', 'key']} +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {tag: dt.datetime(2022, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# Use default ace clean +clean = mm_ace.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +strid = {'1sec': {'base': 'h3'}, + '16sec': {'base': 'h0', 'key': 'k1'}, + '4min': {'base': 'h1'}, + '5min': {'key': 'k0'}, + '1hr': {'base': 'h2', 'key': 'k2'}} +fname = ''.join(('ac_{sid:s}_mfi_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} + for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} +load = functools.partial(cdw.load, pandas_format=pandas_format, + meta_translation=meta_translation) + +# Set the download routine +download_tags = {'1sec': {'base': 'AC_H3_MFI'}, + '16sec': {'base': 'AC_H0_MFI', 'key': 'AC_K1_MFI'}, + '4min': {'base': 'AC_H1_MFI'}, + '5min': {'key': 'AC_K0_MFI'}, + '1hr': {'base': 'AC_H2_MFI', 'key': 'AC_K2_MFI'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py new file mode 100644 index 00000000..f69a4308 --- /dev/null +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +"""Module for the Advanced Composition Explorer (ACE) SIS instrument. + +Properties +---------- +platform + 'ace' +name + 'sis_l2' +tag + 'base' or 'key' +inst_id + '256sec' or '1hr' + +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- Stone, E., Cohen, C., Cook, W. et al. The Solar Isotope Spectrometer for the + Advanced Composition Explorer. Space Science Reviews 86, 357–408 (1998). + https://doi.org/10.1023/A:1005027929871 + +Note +---- +- Level 1 ACE data is maintained at pysatSpaceWeather. + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'sis_l2' +tags = {'base': 'ACE/SIS Solar Isotope Spectrometer Base Data', + 'key': 'ACE/SIS Solar Isotope Spectrometer Key Parameters'} +inst_ids = {'256sec': ['base'], + '1hr': ['base', 'key']} +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {tag: dt.datetime(2022, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# Use default ace clean +clean = mm_ace.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +strid = {'256sec': {'base': 'h1'}, + '1hr': {'base': 'h2', 'key': 'k0'}} +fname = ''.join(('ac_{sid:s}_sis_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} + for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} +load = functools.partial(cdw.load, pandas_format=pandas_format, + meta_translation=meta_translation) + +# Set the download routine +download_tags = {'256sec': {'base': 'AC_H1_SIS'}, + '1hr': {'base': 'AC_H2_SIS', 'key': 'AC_K0_SIS'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py new file mode 100644 index 00000000..a4a5ae20 --- /dev/null +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +"""Module for the Advanced Composition Explorer (ACE) EPAM instrument. + +Properties +---------- +platform + 'ace' +name + 'swepam_l2' +tag + 'base' or 'key' +inst_id + '64sec', '5min', '1hr' + +References +---------- +- Stone, E., Frandsen, A., Mewaldt, R. et al. The Advanced Composition Explorer. + Space Science Reviews 86, 1–22 (1998). https://doi.org/10.1023/A:1005082526237 +- McComas, D., Bame, S., Barker, P. et al. Solar Wind Electron Proton Alpha + Monitor (SWEPAM) for the Advanced Composition Explorer. Space Science Reviews + 86, 563–612 (1998). https://doi.org/10.1023/A:1005040232597 + +Note +---- +- Level 1 ACE data is maintained at pysatSpaceWeather. +- Release notes at + https://cdaweb.gsfc.nasa.gov/pub/data/ace/swepam/swepam_level2_release_notes.txt + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import ace as mm_ace +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ace' +name = 'swepam_l2' +tags = {'base': 'ACE/SWEPAM Solar Wind Experiment Base Data', + 'key': 'ACE/SWEPAM Solar Wind Experiment Key Parameters'} +inst_ids = {'64sec': ['base'], + '5min': ['key'], + '1hr': ['key', 'base']} +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {id: {tag: dt.datetime(2021, 1, 1) for tag in inst_ids[id]} + for id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +# Use default ace clean +clean = mm_ace.clean + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +strid = {'64sec': {'base': 'h0'}, + '5min': {'key': 'k0'}, + '1hr': {'base': 'h2', 'key': 'k1'}} +fname = ''.join(('ac_{sid:s}_swe_{{year:4d}}{{month:02d}}{{day:02d}}_', + 'v{{version:02d}}.cdf')) +supported_tags = {id: {tag: fname.format(sid=strid[id][tag]) + for tag in inst_ids[id]} + for id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} +load = functools.partial(cdw.load, pandas_format=pandas_format, + meta_translation=meta_translation) + +# Set the download routine +download_tags = {'64sec': {'base': 'AC_H0_SWE'}, + '5min': {'key': 'AC_K0_SWE'}, + '1hr': {'base': 'AC_H2_SWE', 'key': 'AC_K1_SWE'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/cnofs_ivm.py b/pysatNASA/instruments/cnofs_ivm.py index a7693648..59dbf57b 100644 --- a/pysatNASA/instruments/cnofs_ivm.py +++ b/pysatNASA/instruments/cnofs_ivm.py @@ -16,23 +16,6 @@ motion of the satellite the angle is converted into ion motion along two orthogonal directions, perpendicular to the satellite track. -References ----------- -A brief discussion of the C/NOFS mission and instruments can be found at -de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast -scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, -doi:10.1016/j.jastp.2004.07.030. - -Discussion of cleaning parameters for ion drifts can be found in: -Burrell, Angeline G., Equatorial topside magnetic field-aligned ion drifts -at solar minimum, The University of Texas at Dallas, ProQuest -Dissertations Publishing, 2012. 3507604. - -Discussion of cleaning parameters for ion temperature can be found in: -Hairston, M. R., W. R. Coley, and R. A. Heelis (2010), Mapping the -duskside topside ionosphere with CINDI and DMSP, J. Geophys. Res.,115, -A08324, doi:10.1029/2009JA015051. - Properties ---------- @@ -45,6 +28,7 @@ inst_id None supported + Warnings -------- - The sampling rate of the instrument changes on July 29th, 2010. @@ -52,6 +36,24 @@ - The cleaning parameters for the instrument are still under development. + +References +---------- +A brief discussion of the C/NOFS mission and instruments can be found at +de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast +scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, +doi:10.1016/j.jastp.2004.07.030. + +Discussion of cleaning parameters for ion drifts can be found in: +Burrell, Angeline G., Equatorial topside magnetic field-aligned ion drifts +at solar minimum, The University of Texas at Dallas, ProQuest +Dissertations Publishing, 2012. 3507604. + +Discussion of cleaning parameters for ion temperature can be found in: +Hairston, M. R., W. R. Coley, and R. A. Heelis (2010), Mapping the +duskside topside ionosphere with CINDI and DMSP, J. Geophys. Res.,115, +A08324, doi:10.1029/2009JA015051. + """ import datetime as dt @@ -245,12 +247,9 @@ def clean(self): # Set the load routine load = cdw.load -# Set the download routine -basic_tag = {'remote_dir': '/pub/data/cnofs/cindi/ivm_500ms_cdf/{year:4d}/', - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'CNOFS_CINDI_IVM_500MS'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/cnofs_plp.py b/pysatNASA/instruments/cnofs_plp.py index fb43ab9e..71eb1064 100644 --- a/pysatNASA/instruments/cnofs_plp.py +++ b/pysatNASA/instruments/cnofs_plp.py @@ -29,12 +29,6 @@ The data is PRELIMINARY, and as such, is intended for BROWSE PURPOSES ONLY. -References ----------- -A brief discussion of the C/NOFS mission and instruments can be found at -de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast -scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, -doi:10.1016/j.jastp.2004.07.030. Properties ---------- @@ -54,6 +48,14 @@ - Currently no cleaning routine. - Module not written by PLP team. + +References +---------- +A brief discussion of the C/NOFS mission and instruments can be found at +de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast +scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, +doi:10.1016/j.jastp.2004.07.030. + """ import datetime as dt @@ -119,11 +121,9 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': '/pub/data/cnofs/plp/plasma_1sec/{year:4d}/', - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'CNOFS_PLP_PLASMA_1SEC'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py index 78d54e36..f2e4bf24 100644 --- a/pysatNASA/instruments/cnofs_vefi.py +++ b/pysatNASA/instruments/cnofs_vefi.py @@ -26,12 +26,6 @@ The data is PRELIMINARY, and as such, is intended for BROWSE PURPOSES ONLY. -References ----------- -A brief discussion of the C/NOFS mission and instruments can be found at -de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast -scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, -doi:10.1016/j.jastp.2004.07.030. Properties ---------- @@ -56,6 +50,14 @@ - Limited cleaning routine. - Module not written by VEFI team. + +References +---------- +A brief discussion of the C/NOFS mission and instruments can be found at +de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast +scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591, +doi:10.1016/j.jastp.2004.07.030. + """ import datetime as dt @@ -122,11 +124,9 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': '/pub/data/cnofs/vefi/bfield_1sec/{year:4d}/', - 'fname': fname} -download_tags = {'': {'dc_b': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_fpi.py b/pysatNASA/instruments/de2_fpi.py new file mode 100644 index 00000000..ad5e064d --- /dev/null +++ b/pysatNASA/instruments/de2_fpi.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +"""The DE2 FPI instrument. + +Supports the Fabry-Perot Interferometer (FPI) instrument on Dynamics Explorer 2 +(DE2). + +From CDAWeb: + +The Fabry-Perot Interferometer (FPI) was a high-resolution remote sensing +instrument designed to measure the thermospheric temperature, meridional wind, +and density of the following metastable atoms: atomic oxygen (singlet S and D) +and the 2P state of ionic atomic oxygen. The FPI performed a wavelength analysis +on the light detected from the thermospheric emission features by spatially +scanning the interference fringe plane with a multichannel array detector. The +wavelength analysis characterized the Doppler line profile of the emitting +species. A sequential altitude scan performed by a commandable horizon scan +mirror provided a cross-sectional view of the thermodynamic and dynamic state of +the thermosphere below the DE 2 orbit. The information obtained from this +investigation was used to study the dynamic response of the thermosphere to the +energy sources caused by magnetospheric electric fields and the absorption of +solar ultraviolet light in the thermosphere. The instrument was based on the +visible airglow experiment (VAE) used in the AE program. The addition of a +scanning mirror, the Fabry-Perot etalon, an image plane detector, and a +calibration lamp were the principal differences. Interference filters isolated +lines at (in Angstroms) 5577, 6300, 7320, 5896, and 5200. The FPI had a field of +view of 0.53 deg (half-cone angle). More details are found in P. B. Hays et al., +Space Sci. Instrum., v. 5, n. 4, p. 395, 1981. From February 16, 1982 to +September 11, 1982 the DE satellite was inverted and the FPI measured galactic +emissions. + +Properties +---------- +platform + 'de2' +name + 'fpi' +inst_id + None Supported +tag + None Supported + + +Warnings +-------- +- Currently no cleaning routine. + + +References +---------- +Hays, P B, Killeen, T L, and Kennedy, B C. "Fabry-Perot interferometer on +Dynamics Explorer". Space Sci. Instrum., 5, 395-416, 1981. + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import de2 as mm_de2 +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'de2' +name = 'fpi' +tags = {'': '8 s cadence Fabry-Perot Interferometer data'} +inst_ids = {'': ['']} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_de2, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = 'de2_neutral8s_fpi_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' +supported_tags = {'': {'': fname}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Use the default CDAWeb method +load = cdw.load + +# Support download routine +download_tags = {'': {'': 'DE2_NEUTRAL8S_FPI'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Support listing files currently on CDAWeb +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_lang.py b/pysatNASA/instruments/de2_lang.py index 07430357..5afeb803 100644 --- a/pysatNASA/instruments/de2_lang.py +++ b/pysatNASA/instruments/de2_lang.py @@ -25,12 +25,6 @@ or correct the inflight processed data. Time resolution was 0.5 seconds. -References ----------- -J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H. Pinkus, and R. B. Kaplan, -"The Dynamics Explorer 2 Langmuir Probe (LANG)", Space Sci. Instrum., 5, -493-502, 1981. - Properties ---------- platform @@ -48,6 +42,12 @@ - Currently no cleaning routine. +References +---------- +J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H. Pinkus, and R. B. Kaplan, +"The Dynamics Explorer 2 Langmuir Probe (LANG)", Space Sci. Instrum., 5, +493-502, 1981. + """ import datetime as dt @@ -97,12 +97,9 @@ load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_lang', - '/plasma500ms_lang_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_PLASMA500MS_LANG'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_nacs.py b/pysatNASA/instruments/de2_nacs.py index b03212fe..e6b9fa3a 100644 --- a/pysatNASA/instruments/de2_nacs.py +++ b/pysatNASA/instruments/de2_nacs.py @@ -50,12 +50,6 @@ were lost between 12 March 1982 and 31 March 1982 when the counter overflowed. -References ----------- -G. R. Carrignan, B. P. Block, J. C. Maurer, A. E. Hedin, C. A. Reber, -N. W. Spencer, "The neutral mass spectrometer on Dynamics Explorer B", -Space Sci. Instrum., 5, 429-441, 1981. - Properties ---------- platform @@ -67,10 +61,18 @@ tag None Supported + Warnings -------- - Currently no cleaning routine. + +References +---------- +G. R. Carrignan, B. P. Block, J. C. Maurer, A. E. Hedin, C. A. Reber, +N. W. Spencer, "The neutral mass spectrometer on Dynamics Explorer B", +Space Sci. Instrum., 5, 429-441, 1981. + """ import datetime as dt @@ -120,12 +122,9 @@ load = cdw.load # Support download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/neutral_gas_nacs', - '/neutral1s_nacs_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_NEUTRAL1S_NACS'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Support listing files currently on CDAWeb -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 912dd66f..c9161118 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -26,21 +26,11 @@ spectrum; and the concentration of H+, He+, O+, and Fe+, and of molecular ions near perigee. -It includes the DUCT portion of the high resolutiondata from the Dynamics -Explorer 2 (DE-2) Retarding Potential Analyzer (RPA) for the whole DE-2 mission -time period in ASCII format. This version was generated at NSSDC from the -PI-provided binary data (SPIO-00232). The DUCT files include RPA measurements -ofthe total ion concentration every 64 times per second. Due to a failure in -the instrument memory system RPA data are not available from 81317 06:26:40 UT -to 82057 13:16:00 UT. This data set is based on the revised version of the RPA -files that was submitted by the PI team in June of 1995. The revised RPA data -include a correction to the spacecraft potential. +Due to a failure in the instrument memory system RPA data are not available +from 81317 06:26:40 UT to 82057 13:16:00 UT. This data set is based on the +revised version of the RPA files that was submitted by the PI team in June of +1995. The revised RPA data include a correction to the spacecraft potential. -References ----------- -W. B. Hanson, R. A. Heelis, R. A. Power, C. R. Lippincott, D. R. Zuccaro, -B. J. Holt, L. H. Harmon, and S. Sanatani, “The retarding potential analyzer -for dynamics explorer-B,” Space Sci. Instrum. 5, 503–510 (1981). Properties ---------- @@ -53,10 +43,18 @@ tag None Supported + Warnings -------- - Currently no cleaning routine. + +References +---------- +W. B. Hanson, R. A. Heelis, R. A. Power, C. R. Lippincott, D. R. Zuccaro, +B. J. Holt, L. H. Harmon, and S. Sanatani, “The retarding potential analyzer +for dynamics explorer-B,” Space Sci. Instrum. 5, 503–510 (1981). + """ import datetime as dt @@ -73,13 +71,13 @@ platform = 'de2' name = 'rpa' -tags = {'': '2 sec cadence RPA data'} # this is the default cadence -inst_ids = {'': ['']} +tags = {'': '2 sec cadence RPA data'} +inst_ids = {'': [tag for tag in tags]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags}} # ---------------------------------------------------------------------------- # Instrument methods @@ -97,8 +95,11 @@ # Use the default CDAWeb and pysat methods # Set the list_files routine -fname = 'de2_ion2s_rpa_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' -supported_tags = {'': {'': fname}} +datestr = '{year:04d}{month:02d}{day:02d}_v{version:02d}' +dataproduct = {'': 'ion2s'} +fname = 'de2_{dp:s}_rpa_{datestr:s}.cdf' +supported_tags = {'': {tag: fname.format(dp=dataproduct[tag], datestr=datestr) + for tag in tags}} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) @@ -106,12 +107,9 @@ load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/plasma_rpa', - '/ion2s_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_ION2S_RPA'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py new file mode 100644 index 00000000..45bec0ad --- /dev/null +++ b/pysatNASA/instruments/de2_vefi.py @@ -0,0 +1,105 @@ +"""Module for the DE2 VEFI instrument. + +From CDAWeb (adpated): +This directory gathers data for the VEFI instrument that flew on the DE 2 +spacecraft which was launched on 3 August 1981 into an elliptical orbit with +an altitude range of 300 km to 1000 km and re-entered the atmosphere on +19 February 1983. + +dca (NSSDC ID: 81-070B-02C) + +This data set contains the averaged (2 samples per second) DC electric fields in +spacecraft coordinates and orbit information in ASCII format. + +ac (NSSDC ID: 81-070B-02E) + +This data set contains the averaged AC electric field data (1 or 2 points per +second) and orbit information. + +References +---------- +Maynard, N. C., E. A. Bielecki, H. G. Burdick, Instrumentation for vector +electric field measurements from DE-B, Space Sci. Instrum., 5, 523, 1981. + +Properties +---------- +platform + 'de2' +name + 'vefi' +inst_id + None Supported +tag + 'dca' or 'ac' + + +Warnings +-------- +- Currently no cleaning routine. + + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import de2 as mm_de2 +from pysatNASA.instruments.methods import general as mm_nasa + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'de2' +name = 'vefi' +tags = {'': '62 ms combination of Electric Field and Magnetometer', + 'dca': '500 ms cadence DC Averaged Electric Field data', + 'ac': '500 ms cadence AC Electric Field data'} +inst_ids = {'': [tag for tag in tags]} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags}} + + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_de2, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +datestr = '{year:04d}{month:02d}{day:02d}_v{version:02d}' +fid = {'': '62ms_vefimagb', + 'ac': 'ac500ms_vefi', + 'dca': 'dca500ms_vefi'} +fname = 'de2_{fid:s}_{datestr:s}.cdf' +supported_tags = {'': {tag: fname.format(fid=fid[tag], datestr=datestr) + for tag in tags}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +# Forcing use of cdflib as default since pysatCDF has a known issue with vefi +# data. See pysat/pysatCDF#48 +load = functools.partial(cdw.load, use_cdflib=True) + +# Set the download routine +download_tags = {'': {'': 'DE2_62MS_VEFIMAGB', + 'ac': 'DE2_AC500MS_VEFI', + 'dca': 'DE2_DCA500MS_VEFI'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/de2_wats.py b/pysatNASA/instruments/de2_wats.py index 1345841a..cdcd7ef2 100644 --- a/pysatNASA/instruments/de2_wats.py +++ b/pysatNASA/instruments/de2_wats.py @@ -47,12 +47,6 @@ about the processing done at NSSDC is given in WATS_NSSDC_PRO_DE.DOC. -References ----------- -N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E. Hedin, G. R. Carrignan, -J. C. Maurer, "The Dynamics Explorer Wind and Temperature Spectrometer", -Space Sci. Instrum., 5, 417-428, 1981. - Properties ---------- platform @@ -64,10 +58,18 @@ tag None Supported + Warnings -------- - Currently no cleaning routine. + +References +---------- +N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E. Hedin, G. R. Carrignan, +J. C. Maurer, "The Dynamics Explorer Wind and Temperature Spectrometer", +Space Sci. Instrum., 5, 417-428, 1981. + """ import datetime as dt @@ -117,12 +119,9 @@ load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/de/de2/neutral_gas_wats', - '/wind2s_wats_cdaweb/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'DE2_WIND2S_WATS'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py new file mode 100644 index 00000000..65f3fa4b --- /dev/null +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +"""Module for the DMSP SSUSI instrument. + +Supports the Special Sensor Ultraviolet Spectrographic Imager (SSUSI) +instrument on Defense Meteorological Satellite Program (DMSP). + +From JHU APL: + +SSUSI was designed for the DMSP Block 5D-3 satellites. These satellites are +placed into nearly polar, sun-synchronous orbits at an altitude of about 850 km. +SSUSI is a remote-sensing instrument which measures ultraviolet (UV) emissions +in five different wavelength bands from the Earth's upper atmosphere. SSUSI is +mounted on a nadir-looking panel of the satellite. The multicolor images from +SSUSI cover the visible Earth disk from horizon to horizon and the anti-sunward +limb up to an altitude of approximately 520 km. + +The UV images and the derived environmental data provide the Air Force Weather +Agency (Offutt Air Force Base, Bellevue, NE) with near real-time information +that can be utilized in a number of applications, such as maintenance of high +frequency (HF) communication links and related systems and assessment of the +environmental hazard to astronauts on the Space Station. + + +Properties +---------- +platform + 'dmsp' +name + 'ssusi' +tag + 'edr-aurora' +inst_id + 'f16', 'f17', 'f18', 'f19' + + +Warnings +-------- +- Currently no cleaning routine. + + +References +---------- +Larry J. Paxton, Daniel Morrison, Yongliang Zhang, Hyosub Kil, Brian Wolven, +Bernard S. Ogorzalek, David C. Humm, and Ching-I. Meng "Validation of remote +sensing products produced by the Special Sensor Ultraviolet Scanning Imager +(SSUSI): a far UV-imaging spectrograph on DMSP F-16", Proc. SPIE 4485, Optical +Spectroscopic Techniques, Remote Sensing, and Instrumentation for Atmospheric +and Space Research IV, (30 January 2002); doi:10.1117/12.454268 + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import dmsp as mm_dmsp +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import jhuapl + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'dmsp' +name = 'ssusi' +tags = {'edr-aurora': ''.join(['Electron energy flux and mean energy, auroral', + ' boundaries, identified discrete auroral arcs,', + ' hemispheric power, and magnetic field lines ', + 'traced to 4 Earth radii'])} +inst_ids = {sat_id: list(tags.keys()) + for sat_id in ['f16', 'f17', 'f18', 'f19']} + +pandas_format = False +multi_file_day = True + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {inst_id: {tag: dt.datetime(2015, 1, 1) for tag in tags.keys()} + for inst_id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_dmsp, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(['dmsp{inst_id:s}_ssusi_{tag:s}_{{year:04d}}{{day:03d}}T', + '{{hour:02d}}{{minute:02d}}{{second:02d}}-???????T??????-REV', + '?????_vA{{version:1d}}.?.?r{{cycle:03d}}.nc']) +supported_tags = {sat_id: {tag: fname.format(tag=tag, inst_id=sat_id) + for tag in tags.keys()} + for sat_id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags) + +# Set the load routine +load = functools.partial(jhuapl.load_edr_aurora, pandas_format=pandas_format) + +# Set the download routine +basic_tag = {'remote_dir': ''.join(('/pub/data/dmsp/dmsp{inst_id:s}/ssusi/', + '/data/{tag:s}/{{year:4d}}/{{day:03d}}/')), + 'fname': fname} +download_tags = { + sat_id: {tag: {btag: basic_tag[btag].format(tag=tag, inst_id=sat_id) + for btag in basic_tag.keys()} for tag in tags.keys()} + for sat_id in inst_ids.keys()} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/instruments/formosat1_ivm.py b/pysatNASA/instruments/formosat1_ivm.py index 24be65bd..6eb49a92 100644 --- a/pysatNASA/instruments/formosat1_ivm.py +++ b/pysatNASA/instruments/formosat1_ivm.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Module for the ICON EUV instrument. +"""Module for the Formosat-1 IVM instrument. Supports the Ion Velocity Meter (IVM) onboard the Formosat-1 (formerly ROCSAT-1) mission. Downloads data from the NASA Coordinated Data Analysis @@ -92,12 +92,9 @@ def init(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/formosat-rocsat/formosat-1', - '/ipei/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'RS_K0_IPEI'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/icon_euv.py b/pysatNASA/instruments/icon_euv.py index 5599856b..c16e8e7c 100644 --- a/pysatNASA/instruments/icon_euv.py +++ b/pysatNASA/instruments/icon_euv.py @@ -78,13 +78,12 @@ def preprocess(self, keep_original_names=False): Parameters ---------- - keep_original_names : boolean + keep_original_names : bool if True then the names as given in the netCDF ICON file will be used as is. If False, a preamble is removed. (default=False) """ - mm_gen.convert_timestamp_to_datetime(self, sec_mult=1.0e-3) if not keep_original_names: mm_gen.remove_leading_text(self, target='ICON_L26_') return @@ -128,13 +127,11 @@ def clean(self): supported_tags=supported_tags) # Set the download routine -basic_tag = {'remote_dir': '/pub/data/icon/l2/l2-6_euv/{year:04d}/', - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'ICON_L2-6_EUV'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) @@ -148,7 +145,7 @@ def filter_metadata(meta_dict): Returns ------- - dict + meta_dict : dict Filtered EUV metadata """ @@ -169,7 +166,7 @@ def filter_metadata(meta_dict): return meta_dict -def load(fnames, tag=None, inst_id=None, keep_original_names=False): +def load(fnames, tag='', inst_id='', keep_original_names=False): """Load ICON EUV data into `xarray.Dataset` object and `pysat.Meta` objects. This routine is called as needed by pysat. It is not intended @@ -180,13 +177,13 @@ def load(fnames, tag=None, inst_id=None, keep_original_names=False): fnames : array-like Iterable of filename strings, full path, to data files to be loaded. This input is nominally provided by pysat itself. - tag : string + tag : str Tag name used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. (default=None) - inst_id : string - Satellite ID used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. (default=None) - keep_original_names : boolean + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + keep_original_names : bool If True then the names as given in the netCDF ICON file will be used as is. If False, a preamble is removed. (default=False) diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py index 6e74f63f..c5333ee9 100644 --- a/pysatNASA/instruments/icon_fuv.py +++ b/pysatNASA/instruments/icon_fuv.py @@ -79,13 +79,12 @@ def preprocess(self, keep_original_names=False): Parameters ---------- - keep_original_names : boolean + keep_original_names : bool if True then the names as given in the netCDF ICON file will be used as is. If False, a preamble is removed. (default=False) """ - mm_gen.convert_timestamp_to_datetime(self, sec_mult=1.0e-3) if not keep_original_names: mm_icon.remove_preamble(self) return @@ -120,16 +119,13 @@ def clean(self): supported_tags=supported_tags) # Set the download routine -basic_tag24 = {'remote_dir': '/pub/data/icon/l2/l2-4_fuv_day/{year:04d}/', - 'fname': fname24} -basic_tag25 = {'remote_dir': '/pub/data/icon/l2/l2-5_fuv_night/{year:04d}/', - 'fname': fname25} -download_tags = {'': {'day': basic_tag24, 'night': basic_tag25}} +download_tags = {'': {'day': 'ICON_L2-4_FUV_DAY', + 'night': 'ICON_L2-5_FUV_NIGHT'}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) @@ -143,7 +139,7 @@ def filter_metadata(meta_dict): Returns ------- - dict + meta_dict : dict Filtered FUV metadata """ @@ -169,7 +165,7 @@ def filter_metadata(meta_dict): return meta_dict -def load(fnames, tag=None, inst_id=None, keep_original_names=False): +def load(fnames, tag='', inst_id='', keep_original_names=False): """Load ICON FUV data into xarray.Dataset object and pysat.Meta objects. This routine is called as needed by pysat. It is not intended @@ -180,15 +176,15 @@ def load(fnames, tag=None, inst_id=None, keep_original_names=False): fnames : array-like iterable of filename strings, full path, to data files to be loaded. This input is nominally provided by pysat itself. - tag : string - tag name used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. - inst_id : string - Satellite ID used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. - keep_original_names : boolean + tag : str + Tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + keep_original_names : bool if True then the names as given in the netCDF ICON file - will be used as is. If False, a preamble is removed. + will be used as is. If False, a preamble is removed. (default=False) Returns ------- diff --git a/pysatNASA/instruments/icon_ivm.py b/pysatNASA/instruments/icon_ivm.py index e9a55a58..6df533a0 100644 --- a/pysatNASA/instruments/icon_ivm.py +++ b/pysatNASA/instruments/icon_ivm.py @@ -87,7 +87,7 @@ def preprocess(self, keep_original_names=False): Parameters ---------- - keep_original_names : boolean + keep_original_names : bool if True then the names as given in the netCDF ICON file will be used as is. If False, a preamble is removed. (default=False) @@ -192,13 +192,12 @@ def clean(self): # Set the download routine dirstr = '/pub/data/icon/l2/l2-7_ivm-{id:s}/{{year:4d}}/' -download_tags = {id: {'': {'remote_dir': dirstr.format(id=id), - 'fname': supported_tags[id]['']}} - for id in ['a', 'b']} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'a': {'': 'ICON_L2-7_IVM-A'}, 'b': {'': 'ICON_L2-7_IVM-B'}} + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) @@ -229,7 +228,7 @@ def filter_metadata(meta_dict): return meta_dict -def load(fnames, tag=None, inst_id=None, keep_original_names=False): +def load(fnames, tag='', inst_id='', keep_original_names=False): """Load ICON IVM data into `pandas.DataFrame` and `pysat.Meta` objects. This routine is called as needed by pysat. It is not intended @@ -240,15 +239,15 @@ def load(fnames, tag=None, inst_id=None, keep_original_names=False): fnames : array-like iterable of filename strings, full path, to data files to be loaded. This input is nominally provided by pysat itself. - tag : string - tag name used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. - inst_id : string - Satellite ID used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. - keep_original_names : boolean + tag : str + Tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + keep_original_names : bool if True then the names as given in the netCDF ICON file - will be used as is. If False, a preamble is removed. + will be used as is. If False, a preamble is removed. (default=False) Returns ------- diff --git a/pysatNASA/instruments/icon_mighti.py b/pysatNASA/instruments/icon_mighti.py index 584fd3d4..d2f11ab1 100644 --- a/pysatNASA/instruments/icon_mighti.py +++ b/pysatNASA/instruments/icon_mighti.py @@ -99,13 +99,12 @@ def preprocess(self, keep_original_names=False): Parameters ---------- - keep_original_names : boolean + keep_original_names : bool if True then the names as given in the netCDF ICON file will be used as is. If False, a preamble is removed. (default=False) """ - mm_gen.convert_timestamp_to_datetime(self, sec_mult=1.0e-3) if not keep_original_names: mm_icon.remove_preamble(self) return @@ -227,30 +226,23 @@ def _clean_vars(var_list, flag, min_level): supported_tags=supported_tags) # Set the download routine -dirstr1 = '/pub/data/icon/l2/l2-1_mighti-{{id:s}}_los-wind-{color:s}/' -dirstr2 = '/pub/data/icon/l2/l2-2_mighti_vector-wind-{color:s}/' -dirstr3 = '/pub/data/icon/l2/l2-3_mighti-{id:s}_temperature/' -dirnames = {'los_wind_green': dirstr1.format(color='green'), - 'los_wind_red': dirstr1.format(color='red'), - 'vector_wind_green': dirstr2.format(color='green'), - 'vector_wind_red': dirstr2.format(color='red'), - 'temperature': dirstr3} - -download_tags = {} -for inst_id in supported_tags.keys(): - download_tags[inst_id] = {} - for tag in supported_tags[inst_id].keys(): - fname = supported_tags[inst_id][tag] - - download_tags[inst_id][tag] = { - 'remote_dir': ''.join((dirnames[tag].format(id=inst_id), - '{year:04d}/')), - 'fname': fname} - -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'vector': + {'vector_wind_green': 'ICON_L2-2_MIGHTI_VECTOR-WIND-GREEN', + 'vector_wind_red': 'ICON_L2-2_MIGHTI_VECTOR-WIND-RED'}, + 'a': + {'los_wind_green': 'ICON_L2-1_MIGHTI-A_LOS-WIND-GREEN', + 'los_wind_red': 'ICON_L2-1_MIGHTI-A_LOS-WIND-RED', + 'temperature': 'ICON_L2-3_MIGHTI-A_TEMPERATURE'}, + 'b': + {'los_wind_green': 'ICON_L2-1_MIGHTI-B_LOS-WIND-GREEN', + 'los_wind_red': 'ICON_L2-1_MIGHTI-B_LOS-WIND-RED', + 'temperature': 'ICON_L2-3_MIGHTI-B_TEMPERATURE'}} + + +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) @@ -264,7 +256,7 @@ def filter_metadata(meta_dict): Returns ------- - dict + meta_dict : dict Filtered FUV metadata """ @@ -283,7 +275,7 @@ def filter_metadata(meta_dict): return meta_dict -def load(fnames, tag=None, inst_id=None, keep_original_names=False): +def load(fnames, tag='', inst_id='', keep_original_names=False): """Load ICON MIGHTI data into `xarray.Dataset` and `pysat.Meta` objects. This routine is called as needed by pysat. It is not intended @@ -294,12 +286,12 @@ def load(fnames, tag=None, inst_id=None, keep_original_names=False): fnames : array-like iterable of filename strings, full path, to data files to be loaded. This input is nominally provided by pysat itself. - tag : str or NoneType - tag name used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. (default=None) - inst_id : str or NoneType - Satellite ID used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. (default=None) + tag : str + Tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') keep_original_names : bool if True then the names as given in the netCDF ICON file will be used as is. If False, a preamble is removed. (default=False) diff --git a/pysatNASA/instruments/iss_fpmu.py b/pysatNASA/instruments/iss_fpmu.py index b85c6aa5..3fc44673 100644 --- a/pysatNASA/instruments/iss_fpmu.py +++ b/pysatNASA/instruments/iss_fpmu.py @@ -115,12 +115,9 @@ def clean(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/international_space_station_iss', - '/sp_fpmu/{year:4d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'ISS_SP_FPMU'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/jpl_gps.py b/pysatNASA/instruments/jpl_gps.py index ddbe66cf..63787548 100644 --- a/pysatNASA/instruments/jpl_gps.py +++ b/pysatNASA/instruments/jpl_gps.py @@ -16,16 +16,6 @@ the global network of International GNSS Service and the regional network of Continuous Operating Reference Station (CORS). -References ----------- -Pi, X., A. J. Mannucci, U. J. Lindqwister, and C. M. Ho, Monitoring of global -ionospheric irregularities using the worldwide GPS network, Geophys. Res. -Lett., 24, 2283, 1997. - -Pi, X., F. J. Meyer, K. Chotoo, Anthony Freeman, R. G. Caton, and C. T. -Bridgwood, Impact of ionospheric scintillation on Spaceborne SAR observations -studied using GNSS, Proc. ION-GNSS, pp.1998-2006, 2012. - Properties ---------- @@ -38,10 +28,22 @@ inst_id None supported + Warnings -------- - The cleaning parameters for the instrument are still under development. + +References +---------- +Pi, X., A. J. Mannucci, U. J. Lindqwister, and C. M. Ho, Monitoring of global +ionospheric irregularities using the worldwide GPS network, Geophys. Res. +Lett., 24, 2283, 1997. + +Pi, X., F. J. Meyer, K. Chotoo, Anthony Freeman, R. G. Caton, and C. T. +Bridgwood, Impact of ionospheric scintillation on Spaceborne SAR observations +studied using GNSS, Proc. ION-GNSS, pp.1998-2006, 2012. + """ import datetime as dt diff --git a/pysatNASA/instruments/methods/__init__.py b/pysatNASA/instruments/methods/__init__.py index 5bde9a41..8542fcbc 100644 --- a/pysatNASA/instruments/methods/__init__.py +++ b/pysatNASA/instruments/methods/__init__.py @@ -4,6 +4,11 @@ from pysatNASA.instruments.methods import cdaweb # noqa F401 from pysatNASA.instruments.methods import cnofs # noqa F401 from pysatNASA.instruments.methods import de2 # noqa F401 +from pysatNASA.instruments.methods import dmsp # noqa F401 from pysatNASA.instruments.methods import general # noqa F401 +from pysatNASA.instruments.methods import gps # noqa F401 from pysatNASA.instruments.methods import icon # noqa F401 +from pysatNASA.instruments.methods import jhuapl # noqa F401 from pysatNASA.instruments.methods import omni # noqa F401 +from pysatNASA.instruments.methods import ses14 # noqa F401 +from pysatNASA.instruments.methods import timed # noqa F401 diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py new file mode 100644 index 00000000..69e70e7e --- /dev/null +++ b/pysatNASA/instruments/methods/ace.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for ACE data.""" + +import numpy as np + +ackn_str = ' '.join(("Please acknowledge the NASA National Space Science Data", + "Center, the Space Physics Data Facility, and the ACE", + "Principal Investigator, Edward C. Stone of the", + "California Institute of Technology, for usage of ACE", + "data from this site in publications and presentations.")) + +refs = {'mission': ' '.join(('Stone, E., Frandsen, A., Mewaldt, R. et al.', + 'The Advanced Composition Explorer. Space Science', + 'Reviews 86, 1–22 (1998).', + 'https://doi.org/10.1023/A:1005082526237')), + 'epam_l2': ' '.join(('Gold, R., Krimigis, S., Hawkins, S. et al.', + 'Electron, Proton, and Alpha Monitor on the', + 'Advanced Composition Explorer spacecraft.', + 'Space Science Reviews 86, 541–562 (1998).', + 'https://doi.org/10.1023/A:1005088115759')), + 'mag_l2': ' '.join(("Smith, C., L'Heureux, J., Ness, N. et al. The ACE", + "Magnetic Fields Experiment. Space Science Reviews", + "86, 613–632 (1998).", + "https://doi.org/10.1023/A:1005092216668")), + 'sis_l2': ' '.join(('Stone, E., Cohen, C., Cook, W. et al. The Solar', + 'Isotope Spectrometer for the Advanced Composition', + 'Explorer. Space Science Reviews 86, 357–408', + '(1998). https://doi.org/10.1023/A:1005027929871')), + 'swepam_l2': ' '.join(('McComas, D., Bame, S., Barker, P. et al. Solar', + 'Wind Electron Proton Alpha Monitor (SWEPAM)', + 'for the Advanced Composition Explorer. Space', + 'Science Reviews 86, 563–612 (1998).', + 'https://doi.org/10.1023/A:1005040232597')) + } + + +def clean(self): + """Clean ACE data to the specified level. + + Note + ---- + Basic cleaning to replace fill values with NaN + + """ + + for key in self.variables: + if key != 'time': + fill = self.meta[key, self.meta.labels.fill_val] + + # Replace fill with nan + fill_mask = self[key] == fill + self[key] = self.data[key].where(~fill_mask) + self.meta[key] = {self.meta.labels.fill_val: np.nan} + return diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index cb2f3986..acfbc92a 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -9,6 +9,7 @@ import cdflib import datetime as dt +import numpy as np import os import pandas as pds import requests @@ -16,6 +17,7 @@ import xarray as xr from bs4 import BeautifulSoup +from cdasws import CdasWs import pysat from pysat.instruments.methods import general @@ -31,6 +33,35 @@ auto_CDF = libCDF +def try_inst_dict(inst_id, tag, supported_tags): + """Check that the inst_id and tag combination is valid. + + Parameters + ---------- + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be + pre-set with functools.partial then assigned to new instrument code. + (default=None) + + Returns + ------- + inst_dict : dict or str + dictionary containing file location in spdf archive, or dataset ID for + cdasws + """ + try: + inst_dict = supported_tags[inst_id][tag] + except KeyError: + raise ValueError('inst_id / tag combo unknown.') + + return inst_dict + + def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), flatten_twod=True, pandas_format=True, epoch_name='Epoch', meta_processor=None, meta_translation=None, drop_meta_labels=None, @@ -42,9 +73,9 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), fnames : pandas.Series Series of filenames tag : str - tag or None (default='') + Data product tag (default='') inst_id : str - satellite id or None (default='') + Instrument ID (default='') file_cadence : dt.timedelta or pds.DateOffset pysat assumes a daily file cadence, but some instrument data files contain longer periods of time. This parameter allows the specification @@ -93,8 +124,10 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), Note ---- - This routine is intended to be used by pysat instrument modules supporting - a particular NASA CDAWeb dataset + - This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset + - pysatCDF (as of v0.3.2) does not support numpy>=1.24. Load errors may + arise. See https://github.com/pysat/pysatCDF/issues/46 """ @@ -110,6 +143,7 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id, epoch_name=epoch_name, + file_cadence=file_cadence, meta_processor=meta_processor, meta_translation=meta_translation, drop_meta_labels=drop_meta_labels) @@ -125,9 +159,9 @@ def load_pandas(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), fnames : pandas.Series Series of filenames tag : str - tag or None (default='') + Data product tag (default='') inst_id : str - satellite id or None (default='') + Instrument ID (default='') file_cadence : dt.timedelta or pds.DateOffset pysat assumes a daily file cadence, but some instrument data files contain longer periods of time. This parameter allows the specification @@ -224,13 +258,12 @@ def load_pandas(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), def load_xarray(fnames, tag='', inst_id='', - labels={'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'plot': ('plot_label', str), 'axis': ('axis', str), - 'scale': ('scale', str), - 'min_val': ('value_min', float), - 'max_val': ('value_max', float), - 'fill_val': ('fill', float)}, + file_cadence=dt.timedelta(days=1), + labels={'units': ('Units', str), 'name': ('Long_Name', str), + 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), + 'min_val': ('ValidMin', float), + 'max_val': ('ValidMax', float), + 'fill_val': ('FillVal', float)}, epoch_name='Epoch', meta_processor=None, meta_translation=None, drop_meta_labels=None): """Load NASA CDAWeb CDF files into an xarray Dataset. @@ -240,9 +273,14 @@ def load_xarray(fnames, tag='', inst_id='', fnames : pandas.Series Series of filenames tag : str - tag or (default='') + Data product tag (default='') inst_id : str - satellite id (default='') + Instrument ID (default='') + file_cadence : dt.timedelta or pds.DateOffset + pysat assumes a daily file cadence, but some instrument data files + contain longer periods of time. This parameter allows the specification + of regular file cadences greater than or equal to a day (e.g., weekly, + monthly, or yearly). (default=dt.timedelta(days=1)) labels : dict Dict where keys are the label attribute names and the values are tuples that have the label values and value types in that order. @@ -308,7 +346,16 @@ def load_xarray(fnames, tag='', inst_id='', # metadata for pysat using some assumptions. Depending upon your needs # the resulting pandas DataFrame may need modification. ldata = [] - for lfname in fnames: + + # Find unique files for monthly / yearly cadence. + # Arbitrary timestamp needed for comparison. + t0 = dt.datetime(2009, 1, 1) + if (t0 + file_cadence) > (t0 + dt.timedelta(days=1)): + lfnames = list(np.unique([fname[:-11] for fname in fnames])) + else: + lfnames = fnames + + for lfname in lfnames: temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True) ldata.append(temp_data) @@ -396,9 +443,10 @@ def load_xarray(fnames, tag='', inst_id='', return data, meta +# TODO(#103): Include support to unzip / untar files after download. def download(date_array, tag='', inst_id='', supported_tags=None, remote_url='https://cdaweb.gsfc.nasa.gov', data_path=None): - """Download NASA CDAWeb CDF data. + """Download NASA CDAWeb data. This routine is intended to be used by pysat instrument modules supporting a particular NASA CDAWeb dataset. @@ -408,9 +456,9 @@ def download(date_array, tag='', inst_id='', supported_tags=None, date_array : array-like Array of datetimes to download data for. Provided by pysat. tag : str - tag or None (default='') + Data product tag (default='') inst_id : str - satellite id or None (default='') + Instrument ID (default='') supported_tags : dict dict of dicts. Keys are supported tag names for download. Value is a dict with 'remote_dir', 'fname'. Inteded to be pre-set with @@ -427,7 +475,7 @@ def download(date_array, tag='', inst_id='', supported_tags=None, -------- :: - # download support added to cnofs_vefi.py using code below + # Download support added to cnofs_vefi.py using code below fn = 'cnofs_vefi_bfield_1sec_{year:4d}{month:02d}{day:02d}_v05.cdf' dc_b_tag = {'remote_dir': ''.join(('/pub/data/cnofs/vefi/bfield_1sec', '/{year:4d}/')), @@ -439,10 +487,7 @@ def download(date_array, tag='', inst_id='', supported_tags=None, """ - try: - inst_dict = supported_tags[inst_id][tag] - except KeyError: - raise ValueError('inst_id / tag combo unknown.') + inst_dict = try_inst_dict(inst_id, tag, supported_tags) # Naming scheme for files on the CDAWeb server remote_dir = inst_dict['remote_dir'] @@ -455,14 +500,23 @@ def download(date_array, tag='', inst_id='', supported_tags=None, stop=date_array[-1]) # Download only requested files that exist remotely - for date, fname in remote_files.iteritems(): + for date, fname in remote_files.items(): # Format files for specific dates and download location - formatted_remote_dir = remote_dir.format(year=date.year, - month=date.month, - day=date.day, - hour=date.hour, - min=date.minute, - sec=date.second) + # Year and day found in remote_dir: day is assumed to be day of year + if 'day' in remote_dir and 'month' not in remote_dir: + doy = date.timetuple().tm_yday + formatted_remote_dir = remote_dir.format(year=date.year, + day=doy, + hour=date.hour, + min=date.minute, + sec=date.second) + else: + formatted_remote_dir = remote_dir.format(year=date.year, + month=date.month, + day=date.day, + hour=date.hour, + min=date.minute, + sec=date.second) remote_path = '/'.join((remote_url.strip('/'), formatted_remote_dir.strip('/'), fname)) @@ -488,7 +542,102 @@ def download(date_array, tag='', inst_id='', supported_tags=None, return -def list_remote_files(tag=None, inst_id=None, start=None, stop=None, +def cdas_download(date_array, tag='', inst_id='', supported_tags=None, + data_path=None): + """Download NASA CDAWeb CDF data using cdasws. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + date_array : array-like + Array of datetimes to download data for. Provided by pysat. + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be pre-set with + functools.partial then assigned to new instrument code. + (default=None) + data_path : str or NoneType + Path to data directory. If None is specified, the value previously + set in Instrument.files.data_path is used. (default=None) + + Note + ---- + Supported tags for this function use the cdaweb dataset naming convention. + You can find the data set names on CDAWeb or you can use cdasws. + + Starting from scratch using cdasws + :: + from cdasws import CdasWs + cdas = CdasWs() + + # Get list of available observatories/platforms. + cdas.get_observatories() + + # Once your observatory is located, list the available instruments. + cdas.get_instruments(observatory=‘observatory_name’) + + # Now list the available data sets for one instrument. + cdas.get_datasets(observatory=‘observatory_name’, + instrument=‘instrument_name’) + + # You can also list all of the datasets for an observatory. + cdas.get_datasets(observatory=‘observatory_name’) + + Alternatively + :: + Visit https://cdaweb.gsfc.nasa.gov/ + Select the observatory you want from the list and press submit. + The following page will have a list of the data sets. + The bolded names are in the format that cdasws uses. + + Examples + -------- + :: + # Download support added to cnofs_vefi.py using code below + download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} + download = functools.partial(cdw.cdas_download, + supported_tags=download_tags) + + """ + + start = date_array[0] + stop = date_array[-1] + remote_files = cdas_list_remote_files(tag=tag, inst_id=inst_id, + start=start, stop=stop, + supported_tags=supported_tags, + series_out=False) + + for file in remote_files: + + fname = file.split('/')[-1] + saved_local_fname = os.path.join(data_path, fname) + + # Perform download + logger.info(' '.join(('Attempting to download file: ', + file))) + try: + with requests.get(file) as req: + if req.status_code != 404: + with open(saved_local_fname, 'wb') as open_f: + open_f.write(req.content) + logger.info('Successfully downloaded {:}.'.format( + saved_local_fname)) + else: + logger.info(' '.join(('File: "', file, + '" is not available'))) + except requests.exceptions.RequestException as exception: + logger.info(' '.join((str(exception), '- File: "', file, + '" Is not available'))) + return + + +def list_remote_files(tag='', inst_id='', start=None, stop=None, remote_url='https://cdaweb.gsfc.nasa.gov', supported_tags=None, two_digit_year_break=None, delimiter=None): @@ -500,11 +649,9 @@ def list_remote_files(tag=None, inst_id=None, start=None, stop=None, Parameters ---------- tag : str - Denotes type of file to load. Accepted types are . - (default='') + Data product tag (default='') inst_id : str - Specifies the satellite ID for a constellation. - (default='') + Instrument ID (default='') start : dt.datetime or NoneType Starting time for file list. A None value will start with the first file found. @@ -553,10 +700,7 @@ def list_remote_files(tag=None, inst_id=None, start=None, stop=None, """ - try: - inst_dict = supported_tags[inst_id][tag] - except KeyError: - raise ValueError('inst_id / tag combo unknown.') + inst_dict = try_inst_dict(inst_id, tag, supported_tags) # Naming scheme for files on the CDAWeb server format_str = '/'.join((inst_dict['remote_dir'].strip('/'), @@ -601,18 +745,27 @@ def list_remote_files(tag=None, inst_id=None, start=None, stop=None, stop = dt.datetime.now() if (stop is None) else stop if 'year' in search_dir['keys']: + url_list = [] if 'month' in search_dir['keys']: search_times = pds.date_range(start, stop + pds.DateOffset(months=1), freq='M') + for time in search_times: + subdir = format_dir.format(year=time.year, month=time.month) + url_list.append('/'.join((remote_url, subdir))) else: - search_times = pds.date_range(start, - stop + pds.DateOffset(years=1), - freq='Y') - url_list = [] - for time in search_times: - subdir = format_dir.format(year=time.year, month=time.month) - url_list.append('/'.join((remote_url, subdir))) + if 'day' in search_dir['keys']: + search_times = pds.date_range(start, stop + + pds.DateOffset(days=1), + freq='D') + else: + search_times = pds.date_range(start, stop + + pds.DateOffset(years=1), + freq='Y') + for time in search_times: + doy = int(time.strftime('%j')) + subdir = format_dir.format(year=time.year, day=doy) + url_list.append('/'.join((remote_url, subdir))) try: for top_url in url_list: for level in range(n_layers + 1): @@ -656,3 +809,82 @@ def list_remote_files(tag=None, inst_id=None, start=None, stop=None, stored_list = stored_list[mask] return stored_list + + +def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, + supported_tags=None, series_out=True): + """Return a list of every file for chosen remote data. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + start : dt.datetime or NoneType + Starting time for file list. A None value will start with the first + file found. + (default=None) + stop : dt.datetime or NoneType + Ending time for the file list. A None value will stop with the last + file found. + (default=None) + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be + pre-set with functools.partial then assigned to new instrument code. + (default=None) + series_out : bool + boolean to determine output type. True for pandas series of file names, + and False for a list of the full web address. + + Returns + ------- + file_list : list + A list containing the verified available files + + Note + ---- + Supported tags for this function use the cdaweb dataset naming convention. + You can find the dataset names on cdaweb or you can use cdasws. + + Examples + -------- + :: + download_tags = {'': {'dc_b': 'CNOFS_VEFI_BFIELD_1SEC'}} + list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) + + download_tags = {'': {'': 'CNOFS_CINDI_IVM_500MS'}} + list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) + """ + cdas = CdasWs() + dataset = try_inst_dict(inst_id, tag, supported_tags) + + if start is None and stop is None: + # Use the topmost directory without variables + start = cdas.get_inventory(identifier=dataset)[0].start + stop = cdas.get_inventory(identifier=dataset)[-1].end + elif stop is None: + stop = start + dt.timedelta(days=1) + elif start == stop: + stop = start + dt.timedelta(days=1) + + if isinstance(start, pds._libs.tslibs.timestamps.Timestamp): + start = start.tz_localize('utc') + stop = stop.tz_localize('utc') + + og_files = cdas.get_original_files(dataset=dataset, start=start, end=stop) + + if series_out: + name_list = [os.path.basename(f['Name']) for f in og_files[1]] + t_stamp = [pds.Timestamp(f['StartTime'][:10]) for f in og_files[1]] + file_list = pds.Series(data=name_list, index=t_stamp) + else: + file_list = [f['Name'] for f in og_files[1]] + + return file_list diff --git a/pysatNASA/instruments/methods/de2.py b/pysatNASA/instruments/methods/de2.py index ea33e23b..8e44575d 100644 --- a/pysatNASA/instruments/methods/de2.py +++ b/pysatNASA/instruments/methods/de2.py @@ -3,7 +3,10 @@ ackn_str = "The Dynamics Explorer 2 satellite data is provided through CDAWeb" -refs = {'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', +refs = {'fpi': ' '.join(('Hays, P B, Killeen, T L, and Kennedy, B C.', + '"Fabry-Perot interferometer on Dynamics Explorer".', + 'Space Sci. Instrum., v. 5, p. 395-416, 1981.')), + 'lang': ' '.join(('J. P. Krehbiel, L. H. Brace, R. F. Theis, W. H.', 'Pinkus, and R. B. Kaplan, The Dynamics Explorer 2', 'Langmuir Probe (LANG), Space Sci. Instrum., v. 5,', 'n. 4, p. 493, 1981.')), @@ -15,9 +18,17 @@ 'Lippincott, D. R. Zuccaro, B. J. Holt, L. H. Harmon,', 'and S. Sanatani, The retarding potential analyzer', 'for dynamics explorer-B, Space Sci. Instrum. 5,', - '503–510 (1981).')), + '503–510 (1981).\n', + 'Heelis, R. A., W. B. Hanson, C. R. Lippincott, D. R.', + 'Zuccaro, L. L. Harmon, B. J. Holt, J. E. Doherty, R.', + 'A. Power, The ion drift meter for Dynamics', + 'Explorer-B, Space Sci. Instrum., 5, 511, 1981.')), 'wats': ' '.join(('N. W. Spencer, L. E. Wharton, H. B. Niemann, A. E.', 'Hedin, G. R. Carrignan, J. C. Maurer, The', 'Dynamics Explorer Wind and Temperature Spectrometer', - 'Space Sci. Instrum., v. 5, n. 4, p. 417, 1981.')) + 'Space Sci. Instrum., v. 5, n. 4, p. 417, 1981.')), + 'vefi': ' '.join(('Maynard, N. C., E. A. Bielecki, H. G. Burdick,', + 'Instrumentation for vector electric field', + 'measurements from DE-B, Space Sci. Instrum., 5,', + '523, 1981.')) } diff --git a/pysatNASA/instruments/methods/dmsp.py b/pysatNASA/instruments/methods/dmsp.py new file mode 100644 index 00000000..3f930c0b --- /dev/null +++ b/pysatNASA/instruments/methods/dmsp.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for the DMSP data.""" + +ackn_str = "".join(["This Defense Meteorological Satellite Program (DMSP) ", + "satellite data is provided through CDAWeb"]) + +refs = {'ssusi': ''.join(('Larry J. Paxton, Daniel Morrison, Yongliang Zhang,', + ' Hyosub Kil, Brian Wolven, Bernard S. Ogorzalek, ', + 'David C. Humm, and Ching-I. Meng "Validation of ', + 'remote sensing products produced by the Special ', + 'Sensor Ultraviolet Scanning Imager (SSUSI): a far ', + 'UV-imaging spectrograph on DMSP F-16", Proc. SPIE ', + '4485, Optical Spectroscopic Techniques, Remote ', + 'Sensing, and Instrumentation for Atmospheric and ', + 'Space Research IV, (30 January 2002); ', + 'doi:10.1117/12.454268'))} diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index 60ead57f..013e8c7c 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -23,6 +23,10 @@ def init(self, module, name): # Set acknowledgements self.acknowledgements = getattr(module, 'ackn_str') + + if hasattr(module, 'rules_url'): + self.acknowledgements.format(getattr(module, 'rules_url')[name]) + pysat.logger.info(self.acknowledgements) # Set references diff --git a/pysatNASA/instruments/methods/gold.py b/pysatNASA/instruments/methods/gold.py deleted file mode 100644 index 9ac51312..00000000 --- a/pysatNASA/instruments/methods/gold.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -"""Provides non-instrument specific routines for GOLD data.""" - -ack_str = ' '.join(('This is a data product from the NASA Global-scale', - 'Observations of the Limb and Disk (GOLD) mission, an', - 'Heliophysics Explorer mission of opportunity launched', - 'in January 2018.\n Responsibility of the mission', - 'science falls to the Principal Investigator, Dr.', - 'Richard Eastes at University of Colorado/LASP.\n', - 'Validation of the L1B data products falls to the', - 'instrument lead investigators/scientists.\n* EUV', - 'Dr. Bill McClintock\nValidation of the L2 data', - 'products falls to Computational Physics, Inc.\n* Dr.', - 'Jerry Lumpe\n (https://gold.cs.ucf.edu/).\nOverall', - 'validation of the products is overseen by the GOLD', - 'Project Scientist Dr. Alan Burns.\nUsers of these', - 'data should contact and acknowledge the Principal', - 'Investigator Dr. Richard Eastes and the party', - 'directly responsible for the data product and the', - 'NASA Explorers Project Office.')) -ref_str = ' '.join(('Eastes, R.W., McClintock, W.E., Burns, A.G. et al.', - 'The Global-Scale Observations of the Limb and Disk', - '(GOLD) Mission. Space Sci Rev 212, 383–408 (2017).', - 'https://doi.org/10.1007/s11214-017-0392-2')) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py new file mode 100644 index 00000000..e11971ea --- /dev/null +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +"""Module for data sets created by JHU APL.""" + +import datetime as dt +import numpy as np +import pandas as pds +import xarray as xr + +from pysat.utils.io import load_netcdf + + +def build_dtimes(data, var, epoch=None, epoch_var='time'): + """Build datetime objects from standard JHU APL time variables. + + Parameters + ---------- + data : xr.Dataset + Xarray dataset with time variables + var : str + Common string to identify desired year, day of year, and seconds of day + epoch : dt.datetime or NoneType + Epoch to subtract from data or NoneType to get seconds of day from + `data` (default=None) + epoch_var : str + Epoch variable containing time data that seconds of day will be + obtained from if `epoch` != None (default='time') + + Returns + ------- + dtimes : list-like + List of datetime objects + + """ + ykey = 'YEAR{:s}'.format(var) + dkey = 'DOY{:s}'.format(var) + skey = 'TIME{:s}'.format(var) + + if epoch is None: + hours = [int(np.floor(sec / 3600.0)) for sec in data[skey].values] + mins = [int(np.floor((sec - hours[i] * 3600) / 60.0)) + for i, sec in enumerate(data[skey].values)] + secs = [int(np.floor((sec - hours[i] * 3600 - mins[i] * 60))) + for i, sec in enumerate(data[skey].values)] + dtimes = [ + dt.datetime.strptime( + "{:4d}-{:03d}-{:02d}-{:02d}-{:02d}-{:06.0f}".format( + int(data[ykey].values[i]), int(data[dkey].values[i]), + hours[i], mins[i], secs[i], + (sec - hours[i] * 3600 - mins[i] * 60 - secs[i]) * 1.0e6), + '%Y-%j-%H-%M-%S-%f') + for i, sec in enumerate(data[skey].values)] + else: + dtimes = [ + dt.datetime.strptime("{:4d}-{:03d}".format( + int(data[ykey].values[i]), int(data[dkey].values[i])), '%Y-%j') + + (pds.to_datetime(etime).to_pydatetime() - epoch) + for i, etime in enumerate(data[epoch_var].values)] + + return dtimes + + +def expand_coords(data_list, mdata, dims_equal=False): + """Ensure that dimensions do not vary from file to file. + + Parameters + ---------- + data_list : list-like + List of xr.Dataset objects with the same dimensions and variables + mdata : pysat.Meta + Metadata for the data in `data_list` + dims_equal : bool + Assert that all xr.Dataset objects have the same dimensions if True + (default=False) + + Returns + ------- + out_list : list-like + List of xr.Dataset objects with the same dimensions and variables, + now with dimensions that all have the same values and data padded + when needed. + + """ + # Get a list of all the dimensions + if dims_equal: + dims = list(data_list[0].dims.keys()) if len(data_list) > 0 else [] + else: + dims = list() + for sdata in data_list: + if len(dims) == 0: + dims = list(sdata.dims.keys()) + else: + for dim in list(sdata.dims.keys()): + if dim not in dims: + dims.append(dim) + + # After loading all the data, determine which dimensions may need to be + # expanded, as they could differ in dimensions from file to file + combo_dims = {dim: max([sdata.dims[dim] for sdata in data_list + if dim in sdata.dims]) for dim in dims} + + # Expand the data so that all dimensions are the same shape + out_list = list() + for i, sdata in enumerate(data_list): + # Determine which dimensions need to be updated + fix_dims = [dim for dim in sdata.dims.keys() + if sdata.dims[dim] < combo_dims[dim]] + + new_data = {} + update_new = False + for dvar in sdata.data_vars.keys(): + # See if any dimensions need to be updated + update_dims = list(set(sdata[dvar].dims) & set(fix_dims)) + + # Save the old data as is, or pad it to have the right dims + if len(update_dims) > 0: + update_new = True + new_shape = list(sdata[dvar].values.shape) + old_slice = [slice(0, ns) for ns in new_shape] + + for dim in update_dims: + idim = list(sdata[dvar].dims).index(dim) + new_shape[idim] = combo_dims[dim] + + # Set the new data for output + new_dat = np.full(shape=new_shape, fill_value=mdata[ + dvar, mdata.labels.fill_val]) + new_dat[tuple(old_slice)] = sdata[dvar].values + new_data[dvar] = (sdata[dvar].dims, new_dat) + else: + new_data[dvar] = sdata[dvar] + + # Get the updated dataset + out_list.append(xr.Dataset(new_data) if update_new else sdata) + + return out_list + + +def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): + """Load JHU APL EDR Aurora data and meta data. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + tag : str + Tag name used to identify particular data set to be loaded (default='') + inst_id : str + Instrument ID name used to identify different instrument carriers + (default='') + pandas_format : bool + False for xarray format, True for pandas (default=False) + + Returns + ------- + data : pds.DataFrame or xr.Dataset + Data to be assigned to the pysat.Instrument.data object. + mdata : pysat.Meta + Pysat Meta data for each data variable. + + Note + ---- + Logger warning 'Epoch label: TIME is not a dimension.' is raised due to + the data format and pysat file expectations. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'guvi', tag='edr-aur') + inst.load(2003, 1) + + """ + # Define the input variables + labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} + + # CDAWeb stores these files in the NetCDF format instead of the CDF format + single_data = list() + for fname in fnames: + # There are multiple files per day, with time as a variable rather + # than a dimension or coordinate. Additionally, no coordinates + # are assigned. + sdata, mdata = load_netcdf(fname, epoch_name='TIME', epoch_unit='s', + labels=labels, pandas_format=pandas_format) + + # Calculate the time for this data file. The pysat `load_netcdf` routine + # converts the 'TIME' parameter (seconds of day) into datetime using + # the UNIX epoch as the date offset + ftime = dt.datetime.strptime( + "{:4d}-{:03d}".format( + sdata['YEAR'].values.astype(int), + sdata['DOY'].values.astype(int)), '%Y-%j') + ( + pds.to_datetime(sdata['time'].values).to_pydatetime() + - dt.datetime(1970, 1, 1)) + + # Assign a datetime variable, making indexing possible + sdata['time'] = ftime + sdata = sdata.assign_coords( + {'time': sdata['time']}).expand_dims(dim='time') + + # Save the data in the file list + single_data.append(sdata) + + # Update the meta data + # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by + # removing 'TIME', once possible + for var in mdata.keys(): + # Update the fill value, using information from the global header + mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} + + # After loading all the data, determine which dimensions need to be + # expanded. Pad the data so that all dimensions are the same shape + single_data = expand_coords(single_data, mdata, dims_equal=True) + + # Combine all the data, indexing along time + data = xr.combine_by_coords(single_data) + + return data, mdata + + +def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, + combine_times=False): + """Load JHU APL SDR data and meta data. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + tag : str + Tag name used to identify particular data set to be loaded (default='') + inst_id : str + Instrument ID name used to identify different instrument carriers + (default='') + pandas_format : bool + False for xarray format, True for pandas (default=False) + combine_times : bool + For SDR data, optionally combine the different datetime coordinates + into a single time coordinate (default=False) + + Returns + ------- + data : pds.DataFrame or xr.Dataset + Data to be assigned to the pysat.Instrument.data object. + mdata : pysat.Meta + Pysat Meta data for each data variable. + + Note + ---- + Logger warning 'Epoch label: TIME is not a dimension.' is raised due to + the data format and pysat file expectations. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'guvi', tag='edr-aur') + inst.load(2003, 1) + + """ + # Define the input variables and working variables + labels = {'units': ('UNITS', str), 'desc': ('TITLE', str)} + load_time = 'TIME_DAY' + time_vars = ['YEAR_DAY', 'DOY_DAY', 'TIME_EPOCH_DAY', 'YEAR_NIGHT', + 'DOY_NIGHT', 'TIME_NIGHT', 'TIME_EPOCH_NIGHT'] + coords = ['PIERCEPOINT_NIGHT_LATITUDE', 'PIERCEPOINT_NIGHT_LONGITUDE', + 'PIERCEPOINT_NIGHT_ALTITUDE', 'PIERCEPOINT_NIGHT_SZA', + 'PIERCEPOINT_DAY_LATITUDE', 'PIERCEPOINT_DAY_LONGITUDE', + 'PIERCEPOINT_DAY_ALTITUDE', 'PIERCEPOINT_DAY_SZA'] + time_dims = ['time'] + rename_dims = {'nAlongDay': 'nAlong', 'nAlongNight': 'nAlong'} + + if tag == 'sdr-imaging': + time_vars.extend(["YEAR_DAY_AURORAL", "DOY_DAY_AURORAL", + "TIME_DAY_AURORAL", "TIME_EPOCH_DAY_AURORAL"]) + coords.extend(['PIERCEPOINT_DAY_LATITUDE_AURORAL', + 'PIERCEPOINT_DAY_LONGITUDE_AURORAL', + 'PIERCEPOINT_DAY_ALTITUDE_AURORAL', + 'PIERCEPOINT_DAY_SZA_AURORAL']) + time_dims.append('time_auroral') + rename_dims['nCrossDay'] = 'nCross' + rename_dims['nCrossNight'] = 'nCross' + rename_dims['nAlongDayAur'] = 'time_auroral' + elif tag == 'sdr-spectrograph': + coords.extend(['PIERCEPOINT_NIGHT_ZENITH_ANGLE', + 'PIERCEPOINT_NIGHT_SAZIMUTH', + 'PIERCEPOINT_DAY_ZENITH_ANGLE', + 'PIERCEPOINT_DAY_SAZIMUTH']) + + if inst_id == 'low_res': + time_vars.extend(["YEAR_GAIM_DAY", "DOY_GAIM_DAY", "TIME_GAIM_DAY", + "TIME_GAIM_NIGHT", "YEAR_GAIM_NIGHT", + "DOY_GAIM_NIGHT"]) + time_dims.extend(['time_gaim_day', 'time_gaim_night']) + rename_dims['nAlongGAIMDay'] = 'time_gaim_day' + rename_dims['nAlongGAIMNight'] = 'time_gaim_night' + + # CDAWeb stores these files in the NetCDF format instead of the CDF format + inners = None + for fname in fnames: + # There are multiple files per day, with time as a variable rather + # than a dimension or coordinate. Additionally, no coordinates + # are assigned. + sdata, mdata = load_netcdf(fname, epoch_name=load_time, epoch_unit='s', + labels=labels, pandas_format=pandas_format) + + # Calculate the time for this data file. The pysat `load_netcdf` routine + # converts the 'TIME' parameter (seconds of day) into datetime using + # the UNIX epoch as the date offset + ftime = build_dtimes(sdata, '_DAY', dt.datetime(1970, 1, 1)) + + # Ensure identical day and night dimensions + if sdata.dims['nAlongDay'] != sdata.dims['nAlongNight']: + raise ValueError('Along-track day and night dimensions differ') + + if 'nCrossDay' in rename_dims.keys(): + if sdata.dims['nCrossDay'] != sdata.dims['nCrossNight']: + raise ValueError('Cross-track day and night dimensions differ') + + # Combine identical dimensions and rename 'nAlong' to 'time' + sdata = sdata.rename_dims(rename_dims) + + if tag == 'sdr-imaging': + sdata = sdata.assign(time_auroral=build_dtimes(sdata, + '_DAY_AURORAL')) + elif tag == 'sdr-spectrograph' and inst_id == 'low_res': + sdata = sdata.assign(time_gaim_day=build_dtimes( + sdata, '_GAIM_DAY'), time_gaim_night=build_dtimes( + sdata, '_GAIM_NIGHT')) + + # Test that day and night times are consistent to the nearest second + for i, ntime in enumerate(build_dtimes(sdata, '_NIGHT')): + if abs(ntime - ftime[i]).total_seconds() > 1.0: + raise ValueError('Day and night times differ') + + # Remove redundant time variables and rname the 'nAlong' dimension + sdata = sdata.drop_vars(time_vars).swap_dims({'nAlong': 'time'}) + + # Assign time as a coordinate for combining files indexing + sdata['time'] = ftime + + # Separate into inner datasets + inner_keys = {dim: [key for key in sdata.keys() + if dim in sdata[key].dims] for dim in time_dims} + inner_dat = {dim: sdata.get(inner_keys[dim]) for dim in time_dims} + + # Add 'single_var's into 'time' dataset to keep track + sv_keys = [val.name for val in sdata.values() + if 'single_var' in val.dims] + singlevar_set = sdata.get(sv_keys) + inner_dat['time'] = xr.merge([inner_dat['time'], singlevar_set]) + + # Concatenate along desired dimension with previous files' data + if inners is None: + # No previous data, assign the data separated by dimension + inners = dict(inner_dat) + else: + # Concatenate with existing data + inners = {dim: xr.concat([inners[dim], inner_dat[dim]], dim=dim) + for dim in time_dims} + + # Update the meta data + # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by + # removing dimensions and time, once possible + for var in mdata.keys(): + # Update the fill value, using information from the global header + mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} + + # Combine all time dimensions + if combine_times: + data_list = expand_coords([inners[dim] if dim == 'time' else + inners[dim].rename_dims({dim: 'time'}) + for dim in time_dims], mdata, + dims_equal=False) + else: + data_list = [inners[dim] for dim in time_dims] + + # Combine all the data, indexing along time + data = xr.merge(data_list) + + # Set additional coordinates + data = data.set_coords(coords).assign_coords({'time': data['time']}) + if tag == 'sdr-imaging': + data = data.assign_coords( + {'nchan': ["121.6nm", "130.4nm", "135.6nm", "LBHshort", "LBHlong"], + "nchanAur": ["121.6nm", "130.4nm", "135.6nm", "LBHshort", + "LBHlong"], + "nCross": sdata.nCross.data, + "nCrossDayAur": sdata.nCrossDayAur.data}) + elif tag == 'sdr-spectrograph': + data = data.assign_coords({"nchan": ["121.6nm", "130.4nm", "135.6nm", + "LBHshort", "LBHlong", "?"]}) + + # Ensure the data is ordered correctly + data = data.sortby('time') + + return data, mdata diff --git a/pysatNASA/instruments/methods/ses14.py b/pysatNASA/instruments/methods/ses14.py new file mode 100644 index 00000000..5307d592 --- /dev/null +++ b/pysatNASA/instruments/methods/ses14.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for SES14 instruments.""" + +ackn_str = ' '.join(('This is a data product from the NASA Global-scale', + 'Observations of the Limb and Disk (GOLD) mission, an', + 'Heliophysics Explorer mission of opportunity launched', + 'in January 2018.\n Responsibility of the mission', + 'science falls to the Principal Investigator, Dr.', + 'Richard Eastes at University of Colorado/LASP.\n', + 'Validation of the L1B data products falls to the', + 'instrument lead investigators/scientists.\n* EUV', + 'Dr. Bill McClintock\nValidation of the L2 data', + 'products falls to Computational Physics, Inc.\n* Dr.', + 'Jerry Lumpe\n (https://gold.cs.ucf.edu/).\nOverall', + 'validation of the products is overseen by the GOLD', + 'Project Scientist Dr. Alan Burns.\nUsers of these', + 'data should contact and acknowledge the Principal', + 'Investigator Dr. Richard Eastes and the party', + 'directly responsible for the data product and the', + 'NASA Explorers Project Office.')) +refs = {'gold': ' '.join(('Eastes, R.W., McClintock, W.E., Burns, A.G. et', + 'al., The Global-Scale Observations of the Limb', + 'and Disk (GOLD) Mission. Space Sci Rev 212,', + '383–408 (2017). doi:10.1007/s11214-017-0392-2'))} diff --git a/pysatNASA/instruments/methods/timed.py b/pysatNASA/instruments/methods/timed.py new file mode 100644 index 00000000..e6d4e2fa --- /dev/null +++ b/pysatNASA/instruments/methods/timed.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for the TIMED data.""" + +rules_url = {'guvi': 'http://guvitimed.jhuapl.edu/home_guvi-datausage', + 'saber': 'https://saber.gats-inc.com/data_services.php', + 'see': 'https://www.timed.jhuapl.edu/WWW/scripts/mdc_rules.pl'} + +ackn_str = "".join(["This Thermosphere Ionosphere Mesosphere Energetics ", + "Dynamics (TIMED) satellite data is provided through ", + "CDAWeb. Please see the Rules of the Road at {:s}"]) + +refs = {'guvi': ''.join(['Larry J. Paxton, Andrew B. Christensen, David C. ', + 'Humm, Bernard S. Ogorzalek, C. Thompson Pardoe, ', + 'Daniel Morrison, Michele B. Weiss, W. Crain, ', + 'Patricia H. Lew, Dan J. Mabry, John O. Goldsten, ', + 'Stephen A. Gary, David F. Persons, Mark J. Harold, ', + 'E. Brian Alvarez, Carl J. Ercol, Douglas J. ', + 'Strickland, and Ching-I. Meng "Global ultraviolet ', + 'imager (GUVI): measuring composition and energy ', + 'inputs for the NASA Thermosphere Ionosphere ', + 'Mesosphere Energetics and Dynamics (TIMED) mission",', + 'Proc. SPIE 3756, Optical Spectroscopic Techniques ', + 'and Instrumentation for Atmospheric and Space ', + 'Research III, (20 October 1999); ', + 'doi:10.1117/12.366380']), + 'saber': '', + 'see': ' '.join(('Woods, T. N., Eparvier, F. G., Bailey,', + 'S. M., Chamberlin, P. C., Lean, J.,', + 'Rottman, G. J., Solomon, S. C., Tobiska,', + 'W. K., and Woodraska, D. L. (2005),', + 'Solar EUV Experiment (SEE): Mission', + 'overview and first results, J. Geophys.', + 'Res., 110, A01312, doi:10.1029/2004JA010765.'))} diff --git a/pysatNASA/instruments/omni_hro.py b/pysatNASA/instruments/omni_hro.py index d709c1b5..aad7d05d 100644 --- a/pysatNASA/instruments/omni_hro.py +++ b/pysatNASA/instruments/omni_hro.py @@ -46,7 +46,6 @@ from pysat.instruments.methods import general as mm_gen from pysat import logger -from pysat.utils import time as pysat_time from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import omni as mm_omni @@ -140,59 +139,12 @@ def clean(self): file_cadence=pds.DateOffset(months=1)) # Set the list_remote_files routine -remote_dir = '/pub/data/omni/omni_cdaweb/hro_{tag:s}/{{year:4d}}/' -download_tags = {inst_id: {tag: {'remote_dir': remote_dir.format(tag=tag), - 'fname': supported_tags[inst_id][tag]} - for tag in tags.keys()} - for inst_id in inst_ids.keys()} -list_remote_files = functools.partial(cdw.list_remote_files, - supported_tags=download_tags) - - -# Set the download routine -def download(date_array, tag, inst_id, data_path, update_files=False): - """Download OMNI HRO data from CDAWeb. - - Parameters - ---------- - date_array : array-like - Sequence of dates for which files will be downloaded. - tag : str - Denotes type of file to load. - inst_id : str - Specifies the satellite ID for a constellation. - data_path : str - Path to data directory. - update_files : bool - Re-download data for files that already exist if True (default=False) - - Raises - ------ - IOError - If a problem is encountered connecting to the gateway or retrieving - data from the repository. - - Warnings - -------- - Only able to download current forecast data, not archived forecasts. - - Note - ---- - Called by pysat. Not intended for direct use by user. - - """ - - # Set the download tags - - # Adjust the date_array for monthly downloads - if date_array.freq != 'MS': - date_array = pysat_time.create_date_range( - dt.datetime(date_array[0].year, date_array[0].month, 1), - date_array[-1], freq='MS') +download_tags = {'': {'1min': 'OMNI_HRO_1MIN', '5min': 'OMNI_HRO_5MIN'}} +download = functools.partial(cdw.cdas_download, + supported_tags=download_tags) - cdw.download(date_array, tag=tag, inst_id=inst_id, - supported_tags=download_tags, data_path=data_path) - return +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) # Set the load routine diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index e5f29ac8..7da4c7df 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -14,9 +14,13 @@ Warnings -------- -- The cleaning parameters for the instrument are still under development. -- strict_time_flag must be set to False +The cleaning parameters for the instrument are still under development. +Note +---- +In roughly 0.3% of daily files, Channel A and Channel B scans begin at the same +time. One microsecond is added to Channel B to ensure uniqueness in the xarray +index. The nominal scan rate for each channel is every 30 minutes. Examples -------- @@ -24,8 +28,7 @@ import datetime as dt import pysat - nmax = pysat.Instrument(platform='ses14', name='gold', tag='nmax' - strict_time_flag=False) + nmax = pysat.Instrument(platform='ses14', name='gold', tag='nmax') nmax.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) nmax.load(2020, 1) @@ -36,12 +39,11 @@ import numpy as np from pysat.instruments.methods import general as ps_gen -from pysat import logger from pysat.utils.io import load_netcdf from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa -from pysatNASA.instruments.methods import gold as mm_gold +from pysatNASA.instruments.methods import ses14 as mm_gold # ---------------------------------------------------------------------------- # Instrument attributes @@ -61,35 +63,11 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - Parameters - ---------- - self : pysat.Instrument - Instrument class object - - """ - - logger.info(mm_gold.ack_str) - logger.warning(' '.join(('Time stamps may be non-unique because Channel A', - 'and B are different instruments. An upgrade to', - 'the pysat.Constellation object is required to', - 'solve this issue. See pysat issue #614 for more', - 'info.'))) - self.acknowledgements = mm_gold.ack_str - self.references = mm_gold.ref_str - - return - +init = functools.partial(mm_nasa.init, module=mm_gold, name=name) # No cleaning, use standard warning function instead clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # @@ -104,19 +82,15 @@ def init(self): supported_tags=supported_tags) # Set the download routine -download_tags = {inst_id: - {tag: {'remote_dir': ''.join(('/pub/data/gold/level2/', tag, - '/{year:4d}/')), - 'fname': supported_tags[''][tag]} - for tag in tags.keys()} for inst_id in inst_ids.keys()} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'nmax': 'GOLD_L2_NMAX'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) -def load(fnames, tag=None, inst_id=None): +def load(fnames, tag='', inst_id=''): """Load GOLD NMAX data into `xarray.Dataset` and `pysat.Meta` objects. This routine is called as needed by pysat. It is not intended @@ -127,12 +101,12 @@ def load(fnames, tag=None, inst_id=None): fnames : array-like iterable of filename strings, full path, to data files to be loaded. This input is nominally provided by pysat itself. - tag : string - tag name used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. - inst_id : string - Satellite ID used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. + tag : str + Tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') **kwargs : extra keywords Passthrough for additional keyword arguments specified when instantiating an Instrument object. These additional keywords @@ -185,9 +159,17 @@ def load(fnames, tag=None, inst_id=None): drop_meta_labels='FILLVAL') if tag == 'nmax': - # Add time coordinate from scan_start_time - data['time'] = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") - for val in data['scan_start_time'].values] + # Add time coordinate from scan_start_time. + time = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%SZ'") + for val in data['scan_start_time'].values] + + # Add a delta of 1 microsecond for channel B. + delta_time = [1 if ch == b'CHB' else 0 for ch in data['channel'].values] + data['time'] = [time[i] + dt.timedelta(microseconds=delta_time[i]) + for i in range(0, len(time))] + + # Sort times to ensure monotonic increase. + data = data.sortby('time') # Update coordinates with dimensional data data = data.assign_coords({'nlats': data['nlats'], diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py new file mode 100644 index 00000000..36b8111b --- /dev/null +++ b/pysatNASA/instruments/timed_guvi.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +"""Module for the TIMED GUVI instrument. + +Supports the Global UltraViolet Imager (GUVI) instrument on the Thermosphere +Ionosphere Mesosphere Energetics Dynamics (TIMED) satellite data from the +NASA Coordinated Data Analysis Web (CDAWeb). + +From JHU APL: + +The Global Ultraviolet Imager (GUVI) is one of four instruments that constitute +the TIMED spacecraft, the first mission of the NASA Solar Connections program. +The TIMED spacecraft is being built by Johns Hopkins University Applied Physics +Laboratory and GUVI is a joint collaboration between JHU/APL and the Aerospace +Corporation. TIMED will be used to study the energetics and dynamics of the +Mesosphere and lower Thermosphere between an altitude of approximately 60 to 180 +kilometers. + +References +---------- +Larry J. Paxton, Andrew B. Christensen, David C. Humm, Bernard S. Ogorzalek, C. +Thompson Pardoe, Daniel Morrison, Michele B. Weiss, W. Crain, Patricia H. Lew, +Dan J. Mabry, John O. Goldsten, Stephen A. Gary, David F. Persons, Mark J. +Harold, E. Brian Alvarez, Carl J. Ercol, Douglas J. Strickland, and Ching-I. +Meng "Global ultraviolet imager (GUVI): measuring composition and energy inputs +for the NASA Thermosphere Ionosphere Mesosphere Energetics and Dynamics (TIMED) +mission", Proc. SPIE 3756, Optical Spectroscopic Techniques and Instrumentation +for Atmospheric and Space Research III, (20 October 1999); +https://doi.org/10.1117/12.366380 + +Properties +---------- +platform + 'timed' +name + 'guvi' +tag + 'edr-aur' + 'sdr-imaging' + 'sdr-spectrograph' +inst_id + '' + 'high_res' + 'low_res' + +Warnings +-------- +- Currently no cleaning routine. + +Example +------- +:: + + import pysat + guvi = pysat.Instrument(platform='timed', name='guvi', + inst_id='sdr-imaging', tag='low_res') + guvi.download(dt.datetime(2005, 6, 28), dt.datetime(2005, 6, 29)) + guvi.load(date=dt.datetime(2005, 6, 28)) + +""" + +import datetime as dt +import functools + +from pysat.instruments.methods import general as mm_gen + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import jhuapl +from pysatNASA.instruments.methods import timed as mm_timed + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'timed' +name = 'guvi' +tags = {'edr-aur': 'Level 2 Auroral disk imaging mode', + 'sdr-imaging': 'Level 1C imaging data', + 'sdr-spectrograph': 'Level 1C spectrograph data'} +inst_ids = {'': ['edr-aur'], + 'high_res': ['sdr-imaging', 'sdr-spectrograph'], + 'low_res': ['sdr-imaging', 'sdr-spectrograph']} + +pandas_format = False +multi_file_day = True + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {iid: {tag: dt.datetime(2005, 6, 28) for tag in inst_ids[iid]} + for iid in inst_ids.keys()} +_test_load_opt = {iid: {tag: {'combine_times': True} + for tag in inst_ids[iid]} for iid in ['high_res', + 'low_res']} + +# ---------------------------------------------------------------------------- +# Instrument methods + +# Use standard init routine +init = functools.partial(mm_nasa.init, module=mm_timed, name=name) + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(('TIMED_GUVI_{lvl:s}{mode:s}_{{year:04d}}{{day:03d}}', + '{{hour:02d}}{{minute:02d}}{{second:02d}}-?????????????_REV', + '??????_Av{{version:02d}}-??r{{revision:03d}}.nc')) +file_lvl = {'low_res': 'L1C-2-disk', 'high_res': 'L1C-disk', '': 'L2B'} +mode = {'sdr-imaging': '-IMG', 'sdr-spectrograph': '-SPECT', + 'edr-aur': '-edr-aur-IMG'} +supported_tags = {inst_id: {tag: fname.format(lvl=file_lvl[inst_id], + mode=mode[tag]) + for tag in tags.keys()} + for inst_id in inst_ids.keys()} +list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) + +# Set the download routine +url = ''.join(('/pub/data/timed/guvi/levels_v13/{lvl:s}/{mode:s}/', + '{{year:4d}}/{{day:03d}}/')) +url_lvl = {'sdr-imaging': 'level1c', 'sdr-spectrograph': 'level1c', + 'edr-aur': 'level2b'} +url_mode = {tag: 'imaging/edr-aur' if tag == 'edr-aur' else tag.split('-')[1] + for tag in tags.keys()} +download_tags = {iid: {tag: {'remote_dir': url.format(lvl=url_lvl[tag], + mode=url_mode[tag]), + 'fname': fname.format(lvl=file_lvl[iid], + mode=mode[tag])} + for tag in tags.keys()} for iid in inst_ids.keys()} +download = functools.partial(cdw.download, supported_tags=download_tags) + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.list_remote_files, + supported_tags=download_tags) + + +# Set the load routine +def load(fnames, tag='', inst_id='', combine_times=False): + """Load TIMED GUVI data into `xarray.DataSet` and `pysat.Meta` objects. + + This routine is called as needed by pysat. It is not intended + for direct user interaction. + + Parameters + ---------- + fnames : array-like + iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + inst_id : str + Satellite ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + combine_times : bool + For SDR data, optionally combine the different datetime coordinates + into a single time coordinate (default=False) + + Returns + ------- + data : xr.DataSet + A xarray DataSet with data prepared for the pysat.Instrument + meta : pysat.Meta + Metadata formatted for a pysat.Instrument object. + + Raises + ------ + ValueError + If temporal dimensions are not consistent + + Note + ---- + Any additional keyword arguments passed to pysat.Instrument + upon instantiation are passed along to this routine. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'guvi', + inst_id='high_res', tag='sdr-imaging') + inst.load(2005, 179) + + """ + if tag == 'edr-aur': + data, meta = jhuapl.load_edr_aurora(fnames, tag, inst_id, + pandas_format=pandas_format) + else: + data, meta = jhuapl.load_sdr_aurora(fnames, tag, inst_id, + pandas_format=pandas_format, + combine_times=combine_times) + + return data, meta diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index 038bc3cc..16357673 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -7,17 +7,19 @@ Properties ---------- -platform : string +platform : str 'timed' -name : string +name : str 'saber' -tag : string +tag : str None supported -inst_id : string +inst_id : str None supported Note ---- +Note on Temperature Errors: https://saber.gats-inc.com/temp_errors.php + SABER "Rules of the Road" for DATA USE Users of SABER data are asked to respect the following guidelines @@ -36,9 +38,9 @@ - Pre-prints of publications and conference abstracts should be widely distributed to interested parties within the mission and related projects. + Warnings -------- -- Note on Temperature Errors: http://saber.gats-inc.com/temp_errors.php - No cleaning routine """ @@ -48,10 +50,10 @@ # CDAWeb methods prewritten for pysat from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import timed as mm_timed # ---------------------------------------------------------------------------- # Instrument attributes @@ -76,33 +78,18 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - rules_url = 'https://saber.gats-inc.com/data_services.php' - ackn_str = ' '.join(('Please see the Rules of the Road at', rules_url)) - - logger.info(ackn_str) - self.acknowledgements = ackn_str - self.references = '' - - return - +init = functools.partial(mm_nasa.init, module=mm_timed, name=name) # No cleaning, use standard warning function instead clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # # Use the default CDAWeb and pysat methods +# TODO(#104): Switch to netCDF4 files once unzip (#103) is supported. + # Set the list_files routine fname = ''.join(('timed_l2a_saber_{year:04d}{month:02d}{day:02d}', '{hour:02d}{minute:02d}_v{version:02d}-{revision:02d}-', @@ -115,12 +102,9 @@ def init(self): load = cdw.load # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/timed/saber/level2a_cdf', - '/{year:4d}/{month:02d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'TIMED_L2A_SABER'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 903058ab..1980443e 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -17,19 +17,12 @@ None inst_id None supported -flatten_twod - If True, then two dimensional data is flattened across - columns. Name mangling is used to group data, first column - is 'name', last column is 'name_end'. In between numbers are - appended 'name_1', 'name_2', etc. All data for a given 2D array - may be accessed via, data.loc[:, 'item':'item_end'] - If False, then 2D data is stored as a series of DataFrames, - indexed by Epoch. data.loc[0, 'item'] - (default=True) Note ---- - no tag required +- cdflib load routine raises ISTP Compliance Warnings for several variables. + This is due to how the Epoch is listed in the original files. Warnings -------- @@ -42,10 +35,10 @@ import pandas as pds from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import timed as mm_timed # ---------------------------------------------------------------------------- # Instrument attributes @@ -54,6 +47,7 @@ name = 'see' tags = {'': ''} inst_ids = {'': [tag for tag in tags.keys()]} +pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -63,39 +57,18 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - rules_url = 'https://www.timed.jhuapl.edu/WWW/scripts/mdc_rules.pl' - ackn_str = ' '.join(('Please see the Rules of the Road at', rules_url)) - logger.info(ackn_str) - self.acknowledgements = ackn_str - self.references = ' '.join(('Woods, T. N., Eparvier, F. G., Bailey,', - 'S. M., Chamberlin, P. C., Lean, J.,', - 'Rottman, G. J., Solomon, S. C., Tobiska,', - 'W. K., and Woodraska, D. L. (2005),', - 'Solar EUV Experiment (SEE): Mission', - 'overview and first results, J. Geophys.', - 'Res., 110, A01312,', - 'doi:10.1029/2004JA010765.')) - - return - +init = functools.partial(mm_nasa.init, module=mm_timed, name=name) # No cleaning, use standard warning function instead clean = mm_nasa.clean_warn - # ---------------------------------------------------------------------------- # Instrument functions # # Use the default CDAWeb and pysat methods +# TODO(#104): Switch to netCDF4 files once unzip (#103) is supported. + # Set the list_files routine fname = 'timed_l3a_see_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf' supported_tags = {'': {'': fname}} @@ -104,15 +77,13 @@ def init(self): file_cadence=pds.DateOffset(months=1)) # Set the load routine -load = functools.partial(cdw.load, file_cadence=pds.DateOffset(months=1)) +load = functools.partial(cdw.load, pandas_format=pandas_format, + file_cadence=pds.DateOffset(months=1)) # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/timed/see/data/level3a_cdf', - '/{year:4d}/{month:02d}/')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'TIMED_L3A_SEE'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 07e850f4..25776e41 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -16,6 +16,15 @@ # Import the test classes from pysat from pysat.tests.classes import cls_instrument_library as clslib +try: + import pysatCDF # noqa: F401 + # If this successfully imports, tests need to be run with both pysatCDF + # and cdflib + cdflib_only = False +except ImportError: + # pysatCDF is not present, standard tests default to cdflib. + cdflib_only = True + # Tell the standard tests which instruments to run each test on. # Need to return instrument list for custom tests. @@ -27,7 +36,9 @@ for inst in instruments['download']: fname = inst['inst_module'].supported_tags[inst['inst_id']][inst['tag']] if '.cdf' in fname: - instruments['cdf'].append(inst) + temp_inst, _ = clslib.initialize_test_inst_and_date(inst) + if temp_inst.pandas_format: + instruments['cdf'].append(inst) class TestInstruments(clslib.InstLibTests): @@ -42,6 +53,9 @@ class TestInstruments(clslib.InstLibTests): @pytest.mark.second @pytest.mark.parametrize("inst_dict", instruments['cdf']) + @pytest.mark.skipif(cdflib_only, + reason=" ".join(("Additional load tests not required", + "when pysatCDF not installed"))) def test_load_cdflib(self, inst_dict): """Test that instruments load at each cleaning level. diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index cae3c747..f601517d 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -1,6 +1,7 @@ """Unit tests for the cdaweb instrument methods.""" import datetime as dt +import pandas as pds import requests import pytest @@ -13,14 +14,14 @@ class TestCDAWeb(object): """Unit tests for `pysat.instrument.methods.cdaweb`.""" - def setup(self): + def setup_method(self): """Set up the unit test environment for each method.""" - self.download_tags = pysatNASA.instruments.cnofs_plp.download_tags + self.download_tags = pysatNASA.instruments.timed_guvi.download_tags self.kwargs = {'tag': None, 'inst_id': None} return - def teardown(self): + def teardown_method(self): """Clean up the unit test environment after each method.""" del self.download_tags, self.kwargs @@ -31,7 +32,7 @@ def test_remote_file_list_connection_error_append(self): with pytest.raises(Exception) as excinfo: # Giving a bad remote_site address yields similar ConnectionError - cdw.list_remote_files(tag='', inst_id='', + cdw.list_remote_files(tag='sdr-imaging', inst_id='high_res', supported_tags=self.download_tags, remote_url='https://bad/path') @@ -80,11 +81,33 @@ def test_bad_kwarg_list_remote_files(self, bad_key, bad_val, err_msg): assert str(excinfo.value).find(err_msg) >= 0 return - def test_remote_file_list_all(self): - """Test that remote_file_list works if start/stop dates unspecified.""" + @pytest.mark.parametrize("start, stop", + [(None, None), + (dt.datetime(2009, 1, 1), None), + (dt.datetime(2009, 1, 1), + dt.datetime(2009, 1, 1)), + (pds.Timestamp(2009, 1, 1), + pds.Timestamp(2009, 1, 2))]) + def test_remote_file_list_all(self, start, stop): + """Test that remote_file_list works for all start and stop cases.""" self.module = pysatNASA.instruments.cnofs_plp self.test_inst = pysat.Instrument(inst_module=self.module) - files = self.test_inst.remote_file_list() + files = self.test_inst.remote_file_list(start, stop) assert len(files) > 0 return + + @pytest.mark.parametrize("series_out", [(True), (False)]) + def test_cdas_remote_files(self, series_out): + """Test that cdas_list_remote_files can return pandas series.""" + start = dt.datetime(2009, 1, 1) + stop = dt.datetime(2009, 1, 2) + self.module = pysatNASA.instruments.cnofs_plp + self.test_inst = pysat.Instrument(inst_module=self.module) + files = self.test_inst.remote_file_list(start, stop, + series_out=series_out) + if series_out is True: + assert isinstance(files, pds.Series) + else: + assert isinstance(files, list) + return diff --git a/pysatNASA/tests/test_methods_platform.py b/pysatNASA/tests/test_methods_platform.py new file mode 100644 index 00000000..a1cd6611 --- /dev/null +++ b/pysatNASA/tests/test_methods_platform.py @@ -0,0 +1,129 @@ +"""Unit tests for the common NASA platform method attributes.""" + + +from pysatNASA.instruments import methods + + +class TestTIMEDMethods(object): + """Unit tests for `pysat.instruments.methods.timed`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['see', 'saber', 'guvi'] + self.module = methods.timed + self.platform_str = '(TIMED)' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + def test_ack(self): + """Test that the acknowledgements reference the correct platform.""" + + assert self.module.ackn_str.find(self.platform_str) >= 0 + return + + def test_rules(self): + """Test that the rules of the road exist for each instrument.""" + + if hasattr(self.module, "rules_url"): + for name in self.names: + assert name in self.module.rules_url.keys( + ), "No rules URL for {:}".format(name) + return + + def test_ref(self): + """Test that all instruments have references.""" + + for name in self.names: + assert name in self.module.refs.keys( + ), "No reference for {:}".format(name) + return + + +class TestDMSPMethods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.dmsp`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['ssusi'] + self.module = methods.dmsp + self.platform_str = '(DMSP)' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestCNOFSMethods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.cnofs`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['ivm', 'plp', 'vefi'] + self.module = methods.cnofs + self.platform_str = '(C/NOFS)' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestDE2Methods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.de2`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['lang', 'nacs', 'rpa', 'wats'] + self.module = methods.de2 + self.platform_str = 'Dynamics Explorer 2' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestSES14Methods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.ses14`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['gold'] + self.module = methods.ses14 + self.platform_str = 'Global-scale Observations of the Limb and Disk' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return + + +class TestGPSMethods(TestTIMEDMethods): + """Unit tests for `pysat.instruments.methods.gps`.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.names = ['roti15min_jpl'] + self.module = methods.gps + self.platform_str = 'GPS Total Electron Content' + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.names, self.module, self.platform_str + return diff --git a/pysatNASA/tests/test_omni_hro.py b/pysatNASA/tests/test_omni_hro.py index 5a3c3f92..b3032024 100644 --- a/pysatNASA/tests/test_omni_hro.py +++ b/pysatNASA/tests/test_omni_hro.py @@ -15,7 +15,7 @@ class TestOMNICustom(object): """Unit tests for `pysat.instrument.methods.omni`.""" - def setup(self): + def setup_method(self): """Set up the unit test environment for each method.""" # Load a test instrument @@ -25,10 +25,9 @@ def setup(self): self.test_inst.load(2009, 1) # Recast time in minutes rather than seconds - self.test_inst.data.index = \ - pds.Series([t + dt.timedelta(seconds=(60 - i)) - + dt.timedelta(minutes=i) - for i, t in enumerate(self.test_inst.data.index)]) + self.test_inst.data.index = pds.Series( + [t + dt.timedelta(seconds=(60 - i)) + dt.timedelta(minutes=i) + for i, t in enumerate(self.test_inst.data.index)]) # Add IMF data self.test_inst['BX_GSM'] = pds.Series([3.17384966, 5.98685138, @@ -63,7 +62,7 @@ def setup(self): return - def teardown(self): + def teardown_method(self): """Clean up the unit test environment after each method.""" del self.test_inst @@ -168,7 +167,7 @@ def test_dayside_recon(self): class TestDeprecation(object): """Unit tests for deprecation warnings in `pysat.instrument.omni_hro`.""" - def setup(self): + def setup_method(self): """Set up the unit test environment for each method.""" # Use an empty instrument to test redirect @@ -178,7 +177,7 @@ def setup(self): return - def teardown(self): + def teardown_method(self): """Clean up test environment after each method.""" del self.test_inst diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..a1dafc41 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +netCDF4 +requests +beautifulsoup4 +lxml +cdflib>=0.4.4 +numpy +pandas +pysat>=3.0.4 +cdasws +xarray diff --git a/tox.ini b/setup.cfg similarity index 100% rename from tox.ini rename to setup.cfg From e38bcd5b45bba1ae560c1a1867a3b8894391f9ca Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 14:09:37 -0400 Subject: [PATCH 235/396] STY: clean up reqs --- pyproject.toml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9148089c..5c14a2a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pysatNASA" -version = "0.0.4.1" +version = "0.0.5" description = "pysat support for NASA Instruments" readme = "README.md" requires-python = ">=3.8" @@ -33,14 +33,15 @@ keywords = [ "ionosphere" ] dependencies = [ - "netCDF4", - "requests", "beautifulsoup4", - "lxml", + "cdasws", "cdflib >= 0.4.4", + "lxml", + "netCDF4", "numpy", "pandas", "pysat >= 3.0.4", + "requests", "xarray" ] From fc14e6207e57e67b006f16f53e8f094c8ae7f394 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 14:09:49 -0400 Subject: [PATCH 236/396] STY: restore backup requirement lists --- requirements.txt | 8 ++++---- test_requirements.txt | 13 +++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 test_requirements.txt diff --git a/requirements.txt b/requirements.txt index a1dafc41..6aedff4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -netCDF4 -requests beautifulsoup4 -lxml +cdasws cdflib>=0.4.4 +lxml +netCDF4 numpy pandas pysat>=3.0.4 -cdasws +requests xarray diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..6c5e6eff --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,13 @@ +coveralls<3.3 +extras_require +flake8 +flake8-docstrings +hacking>=1.0 +ipython +m2r2 +numpydoc +pytest +pytest-cov +pytest-ordering +sphinx +sphinx_rtd_theme From fb2af9b59cb570303db19b23ca469f8ee93430b1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 16:00:59 -0400 Subject: [PATCH 237/396] BUG: use cdas download --- pysatNASA/instruments/timed_saber.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index 70256b2d..d1e6aa0c 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -104,7 +104,7 @@ # Set the download routine download_tags = {'': {'': 'TIMED_L2A_SABER'}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine list_remote_files = functools.partial(cdw.cdas_list_remote_files, From 1e7e65a5f6852c33ecec8f44b8e31d179d310331 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 6 Apr 2023 16:08:04 -0400 Subject: [PATCH 238/396] MAINT: delete version --- pysatNASA/version.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 pysatNASA/version.txt diff --git a/pysatNASA/version.txt b/pysatNASA/version.txt deleted file mode 100644 index 81340c7e..00000000 --- a/pysatNASA/version.txt +++ /dev/null @@ -1 +0,0 @@ -0.0.4 From 09371ef98682c838679020761c5878326c924bb9 Mon Sep 17 00:00:00 2001 From: landsito Date: Fri, 7 Apr 2023 01:40:26 -0600 Subject: [PATCH 239/396] Using combine_by_coords from pysat develop branch --- pysatNASA/instruments/ses14_gold.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 9757743b..d32e5206 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -196,18 +196,11 @@ def load(fnames, tag='', inst_id=''): elif tag == 'o2den': epoch_name = 'nevents' - try: - data, meta = load_netcdf(fnames, pandas_format=pandas_format, - epoch_name=epoch_name, labels=labels, - meta_translation=meta_translation, - drop_meta_labels='FILLVAL') - except: - meta = Meta(labels=labels) - data = xr.open_mfdataset(fnames, concat_dim=epoch_name, - combine='nested') - - # Renaming epoch variable - data = data.swap_dims({epoch_name: 'time'}) + data, meta = load_netcdf(fnames, pandas_format=pandas_format, + epoch_name=epoch_name, labels=labels, + meta_translation=meta_translation, + combine_by_coords=False, + drop_meta_labels='FILLVAL') if tag in ['nmax', 'tdisk', 'tlimb']: # Add time coordinate from scan_start_time @@ -248,7 +241,7 @@ def load(fnames, tag='', inst_id=''): meta['time'] = {meta.labels.notes: 'Converted from time_utc'} meta['zret'] = {meta.labels.notes: ''.join(('Index for retrieval', ' altitude values'))} - meta['zdat'] = {meta.labels.notes: ''.joing(('Index for data tangent', + meta['zdat'] = {meta.labels.notes: ''.join(('Index for data tangent', ' altitude values'))} return data, meta From 0fd46ee3c55e702c39c2c3809e9fef91f53ddce1 Mon Sep 17 00:00:00 2001 From: landsito Date: Fri, 7 Apr 2023 02:02:24 -0600 Subject: [PATCH 240/396] Updating CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c069d3..6c524d13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). spectrograph, imaging * Resolution of dataset handled by tag with low, high +* New Tags + * Added tlimb, tdisk, o2den tags into SES14-GOLD platform. * Bug Fixes * Updated CDAWeb routines to allow for data stored by year/day-of-year * Documentation From 4180100439a6a072f8fb50db438e13d3dbc210df Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 7 Apr 2023 22:43:27 -0400 Subject: [PATCH 241/396] MAINT: update meta defaults for icon --- pysatNASA/instruments/icon_euv.py | 5 +++-- pysatNASA/instruments/icon_fuv.py | 5 +++-- pysatNASA/instruments/icon_ivm.py | 5 +++-- pysatNASA/instruments/icon_mighti.py | 5 +++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pysatNASA/instruments/icon_euv.py b/pysatNASA/instruments/icon_euv.py index c16e8e7c..258db2ab 100644 --- a/pysatNASA/instruments/icon_euv.py +++ b/pysatNASA/instruments/icon_euv.py @@ -212,8 +212,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): """ labels = {'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)} + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))} meta_translation = {'FieldNam': 'plot'} diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py index c5333ee9..49bfc652 100644 --- a/pysatNASA/instruments/icon_fuv.py +++ b/pysatNASA/instruments/icon_fuv.py @@ -208,8 +208,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): """ labels = {'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)} + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))} meta_translation = {'FieldNam': 'plot', 'LablAxis': 'axis', 'FIELDNAM': 'plot', 'LABLAXIS': 'axis', diff --git a/pysatNASA/instruments/icon_ivm.py b/pysatNASA/instruments/icon_ivm.py index 6df533a0..034b63e0 100644 --- a/pysatNASA/instruments/icon_ivm.py +++ b/pysatNASA/instruments/icon_ivm.py @@ -272,8 +272,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): labels = {'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)} + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))} meta_translation = {'FieldNam': 'plot', 'LablAxis': 'axis', 'ScaleTyp': 'scale', diff --git a/pysatNASA/instruments/icon_mighti.py b/pysatNASA/instruments/icon_mighti.py index d2f11ab1..b195fcfd 100644 --- a/pysatNASA/instruments/icon_mighti.py +++ b/pysatNASA/instruments/icon_mighti.py @@ -318,8 +318,9 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): """ labels = {'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)} + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))} meta_translation = {'FieldNam': 'plot', 'LablAxis': 'axis', 'FIELDNAM': 'plot', 'LABLAXIS': 'axis', From cc13bdc94e3a1efa38c40eff0fb2d7908aeb2e9d Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 7 Apr 2023 22:43:39 -0400 Subject: [PATCH 242/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31fee1af..067a8e46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated dosctring style for consistency * Removed version cap for xarray * Added manual workflow to check that latest RC is installable through test pip + * Update meta label type for ICON instruments ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 847377e8a81329f9731c3633dd0dd13784662a53 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 7 Apr 2023 22:58:52 -0400 Subject: [PATCH 243/396] MAINT: update defaults for default load --- pysatNASA/instruments/methods/_cdf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pysatNASA/instruments/methods/_cdf.py b/pysatNASA/instruments/methods/_cdf.py index 29071ff8..ca2a6155 100644 --- a/pysatNASA/instruments/methods/_cdf.py +++ b/pysatNASA/instruments/methods/_cdf.py @@ -360,9 +360,9 @@ def load_variables(self): def to_pysat(self, flatten_twod=True, labels={'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), - 'fill_val': ('FillVal', float)}): + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))}): """Export loaded CDF data into data, meta for pysat module. Parameters @@ -382,9 +382,9 @@ def to_pysat(self, flatten_twod=True, that order. (default={'units': ('units', str), 'name': ('long_name', str), 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', float), - 'max_val': ('value_max', float) - 'fill_val': ('fill', float)}) + 'min_val': ('value_min', (int, float)), + 'max_val': ('value_max', (int, float)) + 'fill_val': ('fill', (int, float))}) Returns ------- From cfc24a508e0032a06641e69782d2987744dee9b2 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 7 Apr 2023 22:59:06 -0400 Subject: [PATCH 244/396] DOC: update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 067a8e46..60b95485 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated dosctring style for consistency * Removed version cap for xarray * Added manual workflow to check that latest RC is installable through test pip - * Update meta label type for ICON instruments + * Update meta label type for instruments ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From e76235fbda06020a3058a0d3199fcd941f307af6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 10 Apr 2023 12:03:51 -0400 Subject: [PATCH 245/396] MAINT: pysat compatibility --- pysatNASA/instruments/ses14_gold.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index c025e7bc..5bd8b525 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -41,6 +41,7 @@ import functools import numpy as np +import pysat from pysat.instruments.methods import general as ps_gen from pysat.utils.io import load_netcdf @@ -169,11 +170,20 @@ def load(fnames, tag='', inst_id=''): elif tag == 'o2den': epoch_name = 'nevents' - data, meta = load_netcdf(fnames, pandas_format=pandas_format, - epoch_name=epoch_name, labels=labels, - meta_translation=meta_translation, - combine_by_coords=False, - drop_meta_labels='FILLVAL') + try: + data, meta = load_netcdf(fnames, pandas_format=pandas_format, + epoch_name=epoch_name, labels=labels, + meta_translation=meta_translation, + combine_by_coords=False, + drop_meta_labels='FILLVAL') + except TypeError: + pysat.logger.warn(' '.join(("Loading multiple days of data may error.", + "Upgrade to pysat 3.1.0 or higher to", + "resolve this issue."))) + data, meta = load_netcdf(fnames, pandas_format=pandas_format, + epoch_name=epoch_name, labels=labels, + meta_translation=meta_translation, + drop_meta_labels='FILLVAL') if tag in ['nmax', 'tdisk', 'tlimb']: # Add time coordinate from scan_start_time From 84147ae1cb6a769112a27e8fdadd6d59a46b06c1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 10 Apr 2023 12:04:06 -0400 Subject: [PATCH 246/396] STY: cdasws implementation --- pysatNASA/instruments/ses14_gold.py | 90 +++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 5bd8b525..29877e75 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -92,13 +92,95 @@ list_files = functools.partial(ps_gen.list_files, supported_tags=supported_tags) +# Set download tags. Note that tlimb uses the general implementation, while +# other tags use the cdasws implementation. +download_tags = {'': {'tlimb': {'remote_dir': ''.join(('/pub/data/gold/', + 'level2/tlimb', + '/{year:4d}/')), + 'fname': supported_tags['']['tlimb']}, + 'nmax': 'GOLD_L2_NMAX', + 'o2den': 'GOLD_L2_O2DEN', + 'tdisk': 'GOLD_L2_TDISK'}} + + # Set the download routine -download_tags = {'': {'nmax': 'GOLD_L2_NMAX'}} -download = functools.partial(cdw.cdas_download, supported_tags=download_tags) +def download(date_array, tag='', inst_id='', data_path=None): + """Download NASA GOLD data. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + date_array : array-like + Array of datetimes to download data for. Provided by pysat. + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + data_path : str or NoneType + Path to data directory. If None is specified, the value previously + set in Instrument.files.data_path is used. (default=None) + + """ + + if tag == 'tlimb': + cdw.download(date_array, tag=tag, inst_id=inst_id, + supported_tags=download_tags, data_path=data_path) + else: + cdw.cdas_download(date_array, tag=tag, inst_id=inst_id, + supported_tags=download_tags, data_path=data_path) + # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.cdas_list_remote_files, - supported_tags=download_tags) +def list_remote_files(tag='', inst_id='', start=None, stop=None, + series_out=True): + """Return a list of every file for chosen remote data. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + start : dt.datetime or NoneType + Starting time for file list. A None value will start with the first + file found. + (default=None) + stop : dt.datetime or NoneType + Ending time for the file list. A None value will stop with the last + file found. + (default=None) + supported_tags : dict + dict of dicts. Keys are supported tag names for download. Value is + a dict with 'remote_dir', 'fname'. Inteded to be + pre-set with functools.partial then assigned to new instrument code. + (default=None) + series_out : bool + boolean to determine output type. True for pandas series of file names, + and False for a list of the full web address. + (default=True) + + Returns + ------- + file_list : list + A list containing the verified available files + + """ + + if tag == 'tlimb': + file_list = cdw.list_remote_files(tag=tag, inst_id=inst_id, + start=start, stop=stop, + supported_tags=download_tags) + else: + file_list = cdw.cdas_list_remote_files(tag=tag, inst_id=inst_id, + start=start, stop=stop, + supported_tags=download_tags, + series_out=series_out) + return file_list def load(fnames, tag='', inst_id=''): From 56e34fd18282e8750790bb6fd164109fe7ce1f78 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 10 Apr 2023 12:04:44 -0400 Subject: [PATCH 247/396] STY: style --- pysatNASA/instruments/ses14_gold.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 29877e75..6369495a 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -57,8 +57,7 @@ tags = {'nmax': 'Level 2 Nmax data for the GOLD instrument', 'tlimb': 'Level 2 Tlimb data for the GOLD instrument', 'tdisk': 'Level 2 Tdisk data for the GOLD instrument', - 'o2den': 'Level 2 O2den data for the GOLD instrument', - } + 'o2den': 'Level 2 O2den data for the GOLD instrument'} inst_ids = {'': ['nmax', 'tlimb', 'tdisk', 'o2den']} pandas_format = False @@ -66,10 +65,7 @@ # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'nmax': dt.datetime(2020, 1, 1), - 'tlimb': dt.datetime(2020, 1, 1), - 'tdisk': dt.datetime(2020, 1, 1), - 'o2den': dt.datetime(2020, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(2020, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods From 4b3db663766eabc7861fa54cdeaf348d457e1f9e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 10 Apr 2023 12:08:42 -0400 Subject: [PATCH 248/396] DOC: add note about pysat 3.1.0 --- pysatNASA/instruments/ses14_gold.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 6369495a..c9a86397 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -17,7 +17,8 @@ Warnings -------- -The cleaning parameters for the instrument are still under development. +- The cleaning parameters for the instrument are still under development. +- Loading multiple days of data requires a bugfix in pysat 3.1.0 or higher. Note ---- From dd3cbd59aff9dfd2070dbf3e243e626b4f8f8322 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 10 Apr 2023 16:29:40 -0400 Subject: [PATCH 249/396] STY: use pysat logger to warn --- pysatNASA/instruments/methods/cdaweb.py | 3 +-- pysatNASA/instruments/methods/general.py | 6 ++---- pysatNASA/instruments/methods/omni.py | 19 ++++++++++--------- pysatNASA/instruments/omni_hro.py | 7 +++---- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index acfbc92a..ac89f6bd 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -13,7 +13,6 @@ import os import pandas as pds import requests -import warnings import xarray as xr from bs4 import BeautifulSoup @@ -139,7 +138,7 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), else: if not use_cdflib: estr = 'The `use_cdflib` option is not currently enabled for xarray' - warnings.warn(estr) + pysat.logger.warn(estr) data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id, epoch_name=epoch_name, diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index 013e8c7c..d2d16008 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -1,7 +1,5 @@ """General methods for NASA instruments.""" -import warnings - import pysat @@ -50,7 +48,7 @@ def clean_warn(self): 'none' No cleaning applied, routine not called in this case. """ - warnings.warn(' '.join(('No cleaning routines available for', - self.platform, self.name))) + pysat.loggern.warn(' '.join(('No cleaning routines available for', + self.platform, self.name))) return diff --git a/pysatNASA/instruments/methods/omni.py b/pysatNASA/instruments/methods/omni.py index 8640340a..2844b0cb 100644 --- a/pysatNASA/instruments/methods/omni.py +++ b/pysatNASA/instruments/methods/omni.py @@ -5,9 +5,8 @@ import numpy as np import pandas as pds from scipy import stats -import warnings -from pysat import logger +import pysat def time_shift_to_magnetic_poles(inst): @@ -46,8 +45,8 @@ def time_shift_to_magnetic_poles(inst): time_x = inst['BSN_x'] * 6371.2 / -inst['Vx'] idx, = np.where(np.isnan(time_x)) if len(idx) > 0: - logger.info(time_x[idx]) - logger.info(time_x) + pysat.logger.info(time_x[idx]) + pysat.logger.info(time_x) time_x_offset = [pds.DateOffset(seconds=time) for time in time_x.astype(int)] new_index = [] @@ -115,8 +114,10 @@ def calculate_imf_steadiness(inst, steady_window=15, min_window_frac=0.75, max_wnum = np.floor(steady_window / sample_rate) if max_wnum != steady_window / sample_rate: steady_window = max_wnum * sample_rate - logger.warning("sample rate is not a factor of the statistical window") - logger.warning("new statistical window is {:.1f}".format(steady_window)) + pysat.logger.warning(" ".join(("sample rate is not a factor of the", + "statistical window"))) + pysat.logger.warning(" ".join(("new statistical window is", + "{:.1f}".format(steady_window)))) min_wnum = int(np.ceil(max_wnum * min_window_frac)) @@ -137,9 +138,9 @@ def calculate_imf_steadiness(inst, steady_window=15, min_window_frac=0.75, kwargs=circ_kwargs, raw=True) except TypeError: - warnings.warn(' '.join(['To automatically remove NaNs from the', - 'calculation, please upgrade to scipy 1.4 or', - 'newer'])) + pysat.logger.warn(' '.join(['To automatically remove NaNs from the', + 'calculation, please upgrade to scipy 1.4', + 'or newer.'])) circ_kwargs.pop('nan_policy') ca_std = \ inst['clock_angle'].rolling(min_periods=min_wnum, diff --git a/pysatNASA/instruments/omni_hro.py b/pysatNASA/instruments/omni_hro.py index aad7d05d..676165bb 100644 --- a/pysatNASA/instruments/omni_hro.py +++ b/pysatNASA/instruments/omni_hro.py @@ -42,10 +42,9 @@ import functools import numpy as np import pandas as pds -import warnings +import pysat from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import omni as mm_omni @@ -85,7 +84,7 @@ def init(self): 'magnetic field data, J. Geophys. Res.,', 'Vol. 110, No. A2, A02209,', '10.1029/2004JA010649.')) - logger.info(ackn_str) + pysat.logger.info(ackn_str) return @@ -208,7 +207,7 @@ def func_wrapper(*args, **kwargs): 'Please update your path to suppress this warning.', 'This redirect will be removed in v0.1.0.']) # Triggered if OMMBV is not installed - warnings.warn(warn_message, DeprecationWarning, stacklevel=2) + pysat.logger.warn(warn_message, DeprecationWarning, stacklevel=2) return func(*args, **kwargs) From dd9efa2647af0b08f4db8beff2f92abe65c88ebd Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 10 Apr 2023 16:30:18 -0400 Subject: [PATCH 250/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60b95485..6e8eb4f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed version cap for xarray * Added manual workflow to check that latest RC is installable through test pip * Update meta label type for instruments + * Use pysat logger to raise warnings ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 13416e34bbc4aee66b39ced728162683488196f8 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 10 Apr 2023 16:48:15 -0400 Subject: [PATCH 251/396] BUG: spelling --- pysatNASA/instruments/methods/general.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index d2d16008..ef29409f 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -48,7 +48,7 @@ def clean_warn(self): 'none' No cleaning applied, routine not called in this case. """ - pysat.loggern.warn(' '.join(('No cleaning routines available for', - self.platform, self.name))) + pysat.logger.warn(' '.join(('No cleaning routines available for', + self.platform, self.name))) return From 637252600a3513b4c23e3692ebfb859cc54d7f33 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 10 Apr 2023 17:12:29 -0400 Subject: [PATCH 252/396] BUG: deprecations --- pysatNASA/instruments/omni_hro.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/omni_hro.py b/pysatNASA/instruments/omni_hro.py index 676165bb..d7a205cf 100644 --- a/pysatNASA/instruments/omni_hro.py +++ b/pysatNASA/instruments/omni_hro.py @@ -42,6 +42,7 @@ import functools import numpy as np import pandas as pds +import warnings import pysat from pysat.instruments.methods import general as mm_gen @@ -207,7 +208,7 @@ def func_wrapper(*args, **kwargs): 'Please update your path to suppress this warning.', 'This redirect will be removed in v0.1.0.']) # Triggered if OMMBV is not installed - pysat.logger.warn(warn_message, DeprecationWarning, stacklevel=2) + warnings.warn(warn_message, DeprecationWarning, stacklevel=2) return func(*args, **kwargs) From b593330014254834dc69ec374ae085fe49c53bbc Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:07:40 -0600 Subject: [PATCH 253/396] Update pysatNASA/instruments/ses14_gold.py Co-authored-by: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> --- pysatNASA/instruments/ses14_gold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 9276d99f..23b1a057 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -59,7 +59,7 @@ 'tdisk': 'Level 2 Tdisk data for the GOLD instrument', 'o2den': 'Level 2 O2den data for the GOLD instrument', } -inst_ids = {'': ['nmax', 'tlimb', 'tdisk','o2den']} +inst_ids = {'': ['nmax', 'tlimb', 'tdisk', 'o2den']} pandas_format = False From 0712ffea2d703153ee05e5485f1ece4affc89797 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:07:48 -0600 Subject: [PATCH 254/396] Update pysatNASA/instruments/ses14_gold.py Co-authored-by: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> --- pysatNASA/instruments/ses14_gold.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 23b1a057..290e4312 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -40,7 +40,6 @@ import datetime as dt import functools import numpy as np -import xarray as xr from pysat.instruments.methods import general as ps_gen from pysat.utils.io import load_netcdf From 6f7cad75726de27553419fee01ee3905bc63f3d9 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:08:02 -0600 Subject: [PATCH 255/396] Update pysatNASA/instruments/ses14_gold.py Co-authored-by: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> --- pysatNASA/instruments/ses14_gold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 290e4312..b594816c 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -207,7 +207,7 @@ def load(fnames, tag='', inst_id=''): data['zdat'] = data['zdat'].isel(time=0) # Add time coordinate from utc_time - data['time'] = [dt.datetime.strptime(str(val), + data['time'] = [dt.datetime.strptime(str(val), "b'%Y-%m-%dT%H:%M:%S.%fZ'") for val in data['time_utc'].values] From 7cb4241438e4585dfbe830fc99181d02759a1585 Mon Sep 17 00:00:00 2001 From: land <69330209+landsito@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:08:09 -0600 Subject: [PATCH 256/396] Update pysatNASA/instruments/ses14_gold.py Co-authored-by: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> --- pysatNASA/instruments/ses14_gold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index b594816c..39995285 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -223,6 +223,6 @@ def load(fnames, tag='', inst_id=''): meta['zret'] = {meta.labels.notes: ''.join(('Index for retrieval', ' altitude values'))} meta['zdat'] = {meta.labels.notes: ''.join(('Index for data tangent', - ' altitude values'))} + ' altitude values'))} return data, meta From cc1a996194433a1add13e40c1aa8c3b914c431a1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:38:00 -0400 Subject: [PATCH 257/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysatNASA/instruments/ses14_gold.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 39995285..5ba3daa5 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -53,22 +53,18 @@ platform = 'ses14' name = 'gold' -tags = {'nmax': 'Level 2 Nmax data for the GOLD instrument', - 'tlimb': 'Level 2 Tlimb data for the GOLD instrument', - 'tdisk': 'Level 2 Tdisk data for the GOLD instrument', - 'o2den': 'Level 2 O2den data for the GOLD instrument', - } -inst_ids = {'': ['nmax', 'tlimb', 'tdisk', 'o2den']} +tags = {'nmax': 'Level 2 max dens data for the GOLD instrument', + 'tlimb': 'Level 2 limb temp data for the GOLD instrument', + 'tdisk': 'Level 2 disk temp data for the GOLD instrument', + 'o2den': 'Level 2 O2 dens data for the GOLD instrument'} +inst_ids = {'': [tag for tags in tags.keys()]} pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'nmax': dt.datetime(2020, 1, 1), - 'tlimb': dt.datetime(2020, 1, 1), - 'tdisk': dt.datetime(2020, 1, 1), - 'o2den': dt.datetime(2020, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(2020, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods From 7d91e23045e046c3b758bd06b0786bf8cf0a5674 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 14 Apr 2023 14:20:26 -0400 Subject: [PATCH 258/396] Update pysatNASA/instruments/ses14_gold.py --- pysatNASA/instruments/ses14_gold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 5ba3daa5..67fdfff4 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -57,7 +57,7 @@ 'tlimb': 'Level 2 limb temp data for the GOLD instrument', 'tdisk': 'Level 2 disk temp data for the GOLD instrument', 'o2den': 'Level 2 O2 dens data for the GOLD instrument'} -inst_ids = {'': [tag for tags in tags.keys()]} +inst_ids = {'': [tag for tag in tags.keys()]} pandas_format = False From 4ef6c25628cb870f1fb170ce538c9cd6cf548cc9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 14 Apr 2023 14:58:04 -0400 Subject: [PATCH 259/396] Update pysatNASA/instruments/ses14_gold.py --- pysatNASA/instruments/ses14_gold.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index e065d54b..c9a86397 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -41,7 +41,6 @@ import datetime as dt import functools import numpy as np -import xarray as xr import pysat from pysat.instruments.methods import general as ps_gen From 1e36f303e26b8f2a84f5db25889581b978ff18d4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:13:44 -0400 Subject: [PATCH 260/396] Update pysatNASA/instruments/ses14_gold.py --- pysatNASA/instruments/ses14_gold.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 1210f173..f64b445c 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -249,6 +249,7 @@ def load(fnames, tag='', inst_id=''): elif tag == 'o2den': epoch_name = 'nevents' + # TODO(#165): remove try/except notation once pysat 3.1.0 is released try: data, meta = load_netcdf(fnames, pandas_format=pandas_format, epoch_name=epoch_name, labels=labels, From 73d2d56acac7390ed36180fcaa1fa659c2189fc9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:15:06 -0400 Subject: [PATCH 261/396] Update pysatNASA/instruments/ses14_gold.py --- pysatNASA/instruments/ses14_gold.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index f64b445c..4fa32bd4 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -151,11 +151,6 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, Ending time for the file list. A None value will stop with the last file found. (default=None) - supported_tags : dict - dict of dicts. Keys are supported tag names for download. Value is - a dict with 'remote_dir', 'fname'. Inteded to be - pre-set with functools.partial then assigned to new instrument code. - (default=None) series_out : bool boolean to determine output type. True for pandas series of file names, and False for a list of the full web address. From 80e3d9598bc90d638a874a2237011a26767c9c68 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 10:59:43 -0400 Subject: [PATCH 262/396] BUG: install pysat rc --- .github/workflows/pysat_rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index c73bcf07..054a0dc7 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -29,7 +29,7 @@ jobs: run: brew reinstall gcc - name: Install pysat RC - run: pip install --no-deps -i https://test.pypi.org/simple/ pysat + run: pip install -i https://test.pypi.org/simple/ --no-deps --pre pysat - name: Install standard dependencies run: | From e4c38edd5804a1b2d8a357f366b3dd45658fece4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 11:26:44 -0400 Subject: [PATCH 263/396] BUG: pysatNASA RC --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 45d835e5..95f6b555 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -27,4 +27,4 @@ jobs: run: pip install -r requirements.txt - name: Install pysatNASA RC - run: pip install --no-deps -i https://test.pypi.org/simple/ pysatNASA + run: pip install --no-deps --pre -i https://test.pypi.org/simple/ pysatNASA From acb4382dbcc89c3f5e653a7c3d6e87edb332f39d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 12:37:32 -0400 Subject: [PATCH 264/396] STY: E275 fix --- pysatNASA/tests/test_instruments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 25776e41..b3334a17 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -85,7 +85,7 @@ def test_load_cdflib(self, inst_dict): assert UserWarning in categories else: # If error message does not match, raise error anyway - raise(verr) + raise (verr) # Make sure fake data is cleared assert target not in test_inst.data From 78b45177b92e505fd900819d72e146b497ced589 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 12:38:12 -0400 Subject: [PATCH 265/396] BUG: install setuptools --- .github/workflows/pysat_rc.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 054a0dc7..1ef34fd0 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -29,7 +29,9 @@ jobs: run: brew reinstall gcc - name: Install pysat RC - run: pip install -i https://test.pypi.org/simple/ --no-deps --pre pysat + run: | + pip install setuptools + pip install -i https://test.pypi.org/simple/ --no-deps --pre pysat - name: Install standard dependencies run: | From 83a2851fac0b4c7fb50ad65279767c4c50c70f6c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 12:42:05 -0400 Subject: [PATCH 266/396] BUG: upgrade pip --- .github/workflows/pysat_rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 1ef34fd0..baba9b14 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -30,7 +30,7 @@ jobs: - name: Install pysat RC run: | - pip install setuptools + pip install --upgrade pip pip install -i https://test.pypi.org/simple/ --no-deps --pre pysat - name: Install standard dependencies From ce3040ce3b276e927268b5702cb810f9002c7581 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 12:52:26 -0400 Subject: [PATCH 267/396] BUG: add extra url --- .github/workflows/pysat_rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index baba9b14..cd4a43d4 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -31,7 +31,7 @@ jobs: - name: Install pysat RC run: | pip install --upgrade pip - pip install -i https://test.pypi.org/simple/ --no-deps --pre pysat + pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ --no-deps --pre pysat - name: Install standard dependencies run: | From fc4b78c1a67a40f70d030fffba97db18cf4f996a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 12:54:33 -0400 Subject: [PATCH 268/396] BUG: no pysatCDF --- .github/workflows/pysat_rc.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index cd4a43d4..443ea312 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -36,7 +36,6 @@ jobs: - name: Install standard dependencies run: | pip install -r requirements.txt - pip install pysatCDF --no-binary=pysatCDF pip install -r test_requirements.txt - name: Set up pysat From 9d121b244b5f45950949f71000099d25b01b9a21 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 12:55:34 -0400 Subject: [PATCH 269/396] STY: consistency --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 95f6b555..0ce3935c 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -27,4 +27,4 @@ jobs: run: pip install -r requirements.txt - name: Install pysatNASA RC - run: pip install --no-deps --pre -i https://test.pypi.org/simple/ pysatNASA + run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysatNASA From ae88459baf8c96bd5561e835ed083baa5d4f1e5c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 12:56:24 -0400 Subject: [PATCH 270/396] STY: consistency --- .github/workflows/pysat_rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 443ea312..0f74e279 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -31,7 +31,7 @@ jobs: - name: Install pysat RC run: | pip install --upgrade pip - pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ --no-deps --pre pysat + pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat - name: Install standard dependencies run: | From 952ac1abae4015a7a5634d70e6f812ec22abdd40 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 13:20:22 -0400 Subject: [PATCH 271/396] MAINT: pip options --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7291779f..7a244034 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - pip install pysatCDF --no-binary=pysatCDF + pip install pysatCDF --no-cache-dir - name: Install standard dependencies run: | From 3f3d00bcbe1eb8c57519820efe3939ee8ac016c9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 13:23:15 -0400 Subject: [PATCH 272/396] MAINT: pip options --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7a244034..c3c88b75 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: - name: Install NEP29 dependencies if: ${{ matrix.numpy_ver != 'latest'}} run: | - pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} + pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} pip install pysatCDF --no-cache-dir - name: Install standard dependencies From 43e90ccb56358fb8448857e32279ddd753504237 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 13:27:47 -0400 Subject: [PATCH 273/396] MAINT: remove pysatCDF --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3c88b75..1b3f4fbe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,6 @@ jobs: if: ${{ matrix.numpy_ver != 'latest'}} run: | pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} - pip install pysatCDF --no-cache-dir - name: Install standard dependencies run: | From 28f9a1ddadf6cfcb07b2b12e0b7ec3b04fde5b81 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 13:28:30 -0400 Subject: [PATCH 274/396] STY: default pip --- .github/workflows/pysat_rc.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 0f74e279..b6292e18 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -29,9 +29,7 @@ jobs: run: brew reinstall gcc - name: Install pysat RC - run: | - pip install --upgrade pip - pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat + run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat - name: Install standard dependencies run: | From 9fb84653dbc700f46a35ac795faec5ee4c2eeb26 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 26 Apr 2023 13:53:01 -0400 Subject: [PATCH 275/396] DOC: update changelog --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60b95485..5bc0be47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,16 +30,17 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated platform methods to follow a consistent style and work with the general `init` function * Added unit tests for the different platform method attributes - * xarray support for TIMED SEE + * xarray support for TIMED SABER and SEE * Maintenance * Removed duplicate tests if pysatCDF not isntalled - * Only test pysatCDF on GitHub Actions for older numpy versions + * Removed pysatCDF tests on Github Actions workflows (see #167) * Updated actions and templates based on pysatEcosystem docs * Remove pandas cap on NEP29 tests * Updated dosctring style for consistency * Removed version cap for xarray * Added manual workflow to check that latest RC is installable through test pip * Update meta label type for instruments + * Updated Github Actions workflows for improved compliance with pip>=23.0 ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From edd96f55c3b9b546fa2dc06c902dd4d7b6ad2964 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 1 May 2023 15:43:12 -0400 Subject: [PATCH 276/396] TST: test import after rc install --- .github/workflows/pip_rc_install.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 0ce3935c..50999f1d 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -4,7 +4,7 @@ name: Test install of latest RC from pip -on: [workflow_dispatch] +on: [push] jobs: build: @@ -28,3 +28,11 @@ jobs: - name: Install pysatNASA RC run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysatNASA + + - name: Set up pysat + run: | + mkdir pysatData + python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" + + - name: Check that install imports correctly + run: python -c "import pysatNASA; print(pysatNASA.__version__)" From 83553b311db6de1c4ac912c433aa9f05b09363aa Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 4 May 2023 14:22:13 -0400 Subject: [PATCH 277/396] Update .github/workflows/pip_rc_install.yml --- .github/workflows/pip_rc_install.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 50999f1d..fb05d68b 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -35,4 +35,6 @@ jobs: python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" - name: Check that install imports correctly - run: python -c "import pysatNASA; print(pysatNASA.__version__)" + run: | + cd .. + python -c "import pysatNASA; print(pysatNASA.__version__)" From 0f810256bc441a11f9e5cc761c909bad971bce35 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 4 May 2023 14:24:39 -0400 Subject: [PATCH 278/396] Update .github/workflows/pip_rc_install.yml --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index fb05d68b..5191c994 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -4,7 +4,7 @@ name: Test install of latest RC from pip -on: [push] +on: [workflow_dispatch] jobs: build: From 92846cb1b31376dfe09ab9df32c6eabc2a357445 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 5 May 2023 14:30:48 -0400 Subject: [PATCH 279/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- CHANGELOG.md | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9952469a..23d73a47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed version cap for xarray * Added manual workflow to check that latest RC is installable through test pip * Update meta label type for instruments - * Use pyproject.toml to manage setup + * Use pyproject.toml to manage installation and metadata ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class diff --git a/pyproject.toml b/pyproject.toml index 5c14a2a2..a5628f4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,10 +7,10 @@ name = "pysatNASA" version = "0.0.5" description = "pysat support for NASA Instruments" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.6" license = {file = "LICENSE"} authors = [ - {name = "Jeff Klenzing, et al", email = "pysat.developers@gmail.com"}, + {name = "Jeff Klenzing, et al.", email = "pysat.developers@gmail.com"}, ] classifiers = [ "Development Status :: 3 - Alpha", From 08000ab7323517a3498e122365bbee94253e3092 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 5 May 2023 16:30:04 -0400 Subject: [PATCH 280/396] DOC: update docs --- README.md | 4 ++-- docs/installation.rst | 4 ++-- pyproject.toml | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9bdce9c5..c0235964 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,14 @@ some examples on how to use the routines pysatNASA uses common Python modules, as well as modules developed by and for the Space Physics community. This module officially supports -Python 3.8+. +Python 3.6+. | Common modules | Community modules | Optional Modules | | ---------------- | ----------------- |------------------| | beautifulsoup4 | cdflib | pysatCDF | | lxml | pysat>=3.0.4 | | | netCDF4 | | | -| numpy<1.24 | | | +| numpy | | | | pandas | | | | requests | | | | xarray | | | diff --git a/docs/installation.rst b/docs/installation.rst index ae8eedda..f74c7e4b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -53,10 +53,10 @@ Installation Options pip install --user . - C. Install with the intent to develop locally:: + C. Install with the intent to change the code:: - pip install -e . + pip install --user -e . .. extras-require:: pysatcdf :pyproject: diff --git a/pyproject.toml b/pyproject.toml index a5628f4d..7d65f625 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "License :: OSI Approved :: BSD License", "Natural Language :: English", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -30,7 +31,10 @@ classifiers = [ ] keywords = [ "pysat", - "ionosphere" + "ionosphere", + "magnetosphere", + "solar wind", + "thermosphere" ] dependencies = [ "beautifulsoup4", From 23da8f8c968ee1ce9d82ab71a06748ff9f151265 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 5 May 2023 16:31:41 -0400 Subject: [PATCH 281/396] TST: update tested versions --- .github/workflows/main.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 80643508..899b2005 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,12 +11,15 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.9", "3.10"] + python-version: ["3.10", "3.11"] numpy_ver: ["latest"] include: - - python-version: "3.8" + - python-version: "3.9" numpy_ver: "1.21" - os: "ubuntu-latest" + os: ubuntu-latest + - python-version: "3.6.8" + numpy_ver: "1.19.5" + os: "ubuntu-20.04" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} From a8567676688d3f2f585443102c8313ad1dc3a862 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 5 May 2023 16:32:08 -0400 Subject: [PATCH 282/396] DOC: update meta --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7d65f625..0f5cc26f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,9 @@ classifiers = [ "Natural Language :: English", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows" From b8ec8336a961b0a869568ac0785b5b526002184a Mon Sep 17 00:00:00 2001 From: jklenzing Date: Fri, 5 May 2023 17:04:29 -0400 Subject: [PATCH 283/396] TST: add readthedocs --- .readthedocs.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..869d6a59 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,29 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# If using Sphinx, optionally build your docs in additional formats such as PDF +# formats: +# - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt From 0af986bda2558345062bab587fc406d78d03df2f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 5 May 2023 17:10:19 -0400 Subject: [PATCH 284/396] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f927840..e9f725c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added manual workflow to check that latest RC is installable through test pip * Update meta label type for instruments * Updated Github Actions workflows for improved compliance with pip>=23.0 + * Added .readthedocs.yml to configure settings there. ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 77d693c3cb45b6c8e9488150f52f74f069746792 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 16 May 2023 13:39:56 +0900 Subject: [PATCH 285/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- .readthedocs.yml | 7 ------- CHANGELOG.md | 4 ++-- pysatNASA/tests/test_instruments.py | 2 +- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 869d6a59..d2c27c90 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,18 +10,11 @@ build: os: ubuntu-22.04 tools: python: "3.10" - # You can also specify other tool versions: - # nodejs: "19" - # rust: "1.64" - # golang: "1.19" # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py -# If using Sphinx, optionally build your docs in additional formats such as PDF -# formats: -# - pdf # Optionally declare the Python requirements required to build your docs python: diff --git a/CHANGELOG.md b/CHANGELOG.md index e9f725c8..866e864b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,14 +40,14 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps * Maintenance * Removed duplicate tests if pysatCDF not isntalled - * Removed pysatCDF tests on Github Actions workflows (see #167) + * Removed pysatCDF tests on GitHub Actions workflows (see #167) * Updated actions and templates based on pysatEcosystem docs * Remove pandas cap on NEP29 tests * Updated dosctring style for consistency * Removed version cap for xarray * Added manual workflow to check that latest RC is installable through test pip * Update meta label type for instruments - * Updated Github Actions workflows for improved compliance with pip>=23.0 + * Updated GitHub Actions workflows for improved compliance with pip>=23.0 * Added .readthedocs.yml to configure settings there. ## [0.0.4] - 2022-11-07 diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 138440e3..3084724e 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -87,7 +87,7 @@ def test_load_cdflib(self, inst_dict): assert UserWarning in categories else: # If error message does not match, raise error anyway - raise (verr) + raise ValueError(verr) # Make sure fake data is cleared assert target not in test_inst.data From ac2caad628da1290b8ed2bddff0cebda8615e6a4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 16 May 2023 13:40:38 +0900 Subject: [PATCH 286/396] Update main.yml --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1b3f4fbe..7e892137 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,8 +33,7 @@ jobs: - name: Install NEP29 dependencies if: ${{ matrix.numpy_ver != 'latest'}} - run: | - pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} + run: pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} - name: Install standard dependencies run: | From 1d48f497a8ff6c0d0a2962d60a58dbb70605e77e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 16 May 2023 13:51:31 +0900 Subject: [PATCH 287/396] BUG: operational tests --- .github/workflows/main.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45a03045..2b614b9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,8 +34,15 @@ jobs: if: ${{ matrix.os == 'macos-latest' }} run: brew reinstall gcc + - name: Install Operational dependencies + if: ${{ matrix.numpy_ver == "1.19.5"}} + run: | + pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} + pip install -r requirements.txt + pip install -r test_requirements.txt + - name: Install NEP29 dependencies - if: ${{ matrix.numpy_ver != 'latest'}} + if: ${{ matrix.numpy_ver == "1.21"}} run: | pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} pip install --upgrade-strategy only-if-needed .[test] From 859635bedd790341e1d0fe47ea4b60da019a74fb Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 16 May 2023 13:55:20 +0900 Subject: [PATCH 288/396] Update main.yml --- .github/workflows/main.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b614b9f..2fec7b86 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,13 +13,16 @@ jobs: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.10", "3.11"] numpy_ver: ["latest"] + test_config: ["latest"] include: - python-version: "3.9" numpy_ver: "1.21" os: ubuntu-latest + test_config: "NEP29" - python-version: "3.6.8" numpy_ver: "1.19.5" os: "ubuntu-20.04" + test_config: "Ops" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} @@ -35,20 +38,20 @@ jobs: run: brew reinstall gcc - name: Install Operational dependencies - if: ${{ matrix.numpy_ver == "1.19.5"}} + if: ${{ matrix.test_config == "Ops"}} run: | pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} pip install -r requirements.txt pip install -r test_requirements.txt - name: Install NEP29 dependencies - if: ${{ matrix.numpy_ver == "1.21"}} + if: ${{ matrix.test_config == "NEP29"}} run: | pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} pip install --upgrade-strategy only-if-needed .[test] - name: Install standard dependencies - if: ${{ matrix.numpy_ver == 'latest'}} + if: ${{ matrix.test_config == 'latest'}} run: | pip install .[test] From 0149729268778969209262c836455065f175c51c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 16 May 2023 13:58:47 +0900 Subject: [PATCH 289/396] BUG: quotes --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2fec7b86..357befc7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,14 +38,14 @@ jobs: run: brew reinstall gcc - name: Install Operational dependencies - if: ${{ matrix.test_config == "Ops"}} + if: ${{ matrix.test_config == 'Ops'}} run: | pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} pip install -r requirements.txt pip install -r test_requirements.txt - name: Install NEP29 dependencies - if: ${{ matrix.test_config == "NEP29"}} + if: ${{ matrix.test_config == 'NEP29'}} run: | pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} pip install --upgrade-strategy only-if-needed .[test] From e0ffbad65f36789f2ffd706d6da9dd7e69cd99c5 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 16 May 2023 14:18:28 +0900 Subject: [PATCH 290/396] Update main.yml --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 357befc7..2bb51850 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,6 +43,7 @@ jobs: pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} pip install -r requirements.txt pip install -r test_requirements.txt + pip install . - name: Install NEP29 dependencies if: ${{ matrix.test_config == 'NEP29'}} From f554299b8ab7e8032ee646034697f800d689b6a8 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 08:58:05 +0900 Subject: [PATCH 291/396] Update __init__.py --- pysatNASA/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/__init__.py b/pysatNASA/__init__.py index 2f8f3370..31d763a5 100644 --- a/pysatNASA/__init__.py +++ b/pysatNASA/__init__.py @@ -6,10 +6,10 @@ """ -import pkg_resources +import importlib from pysatNASA import constellations # noqa F401 from pysatNASA import instruments # noqa F401 # set version -__version__ = pkg_resources.get_distribution('pysatNASA').version +__version__ = importlib.metadata.version('pysatNASA') From edd0c41b08340f733944a5774ec441752c501ea5 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 09:01:43 +0900 Subject: [PATCH 292/396] Update __init__.py --- pysatNASA/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pysatNASA/__init__.py b/pysatNASA/__init__.py index 31d763a5..ea9f8ff2 100644 --- a/pysatNASA/__init__.py +++ b/pysatNASA/__init__.py @@ -7,9 +7,14 @@ """ import importlib +import importlib_metadata from pysatNASA import constellations # noqa F401 from pysatNASA import instruments # noqa F401 # set version -__version__ = importlib.metadata.version('pysatNASA') +try: + __version__ = importlib.metadata.version('pysatNASA') +except AttributeError: + # Python 3.6 requires a different version + __version__ = importlib_metadata.version('pysatNASA') From 2b006a16dadd0aa6817ec548156e28663d13ce93 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 09:12:24 +0900 Subject: [PATCH 293/396] Update .github/workflows/main.yml --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7e892137..7291779f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,9 @@ jobs: - name: Install NEP29 dependencies if: ${{ matrix.numpy_ver != 'latest'}} - run: pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} + run: | + pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} + pip install pysatCDF --no-binary=pysatCDF - name: Install standard dependencies run: | From ba59f29e7f5ce5d89f09b1339d1198677361b331 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 09:55:18 +0900 Subject: [PATCH 294/396] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0f5cc26f..55fc7da7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools", "pip >= 10"] +requires = ["setuptools >= 38.6", "pip >= 10"] build-backend = "setuptools.build_meta" [project] From a4c65919772d7ad677e3b0a76f5c0153f4aa8813 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 09:58:15 +0900 Subject: [PATCH 295/396] Update main.yml --- .github/workflows/main.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7291779f..a8865480 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,9 +33,7 @@ jobs: - name: Install NEP29 dependencies if: ${{ matrix.numpy_ver != 'latest'}} - run: | - pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - pip install pysatCDF --no-binary=pysatCDF + run: pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} - name: Install standard dependencies run: | From 432a449f828c534fc6ad68c8d11e7ccbc4a0bed2 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 09:59:48 +0900 Subject: [PATCH 296/396] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a8865480..3bbe6f57 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,7 @@ jobs: - name: Install NEP29 dependencies if: ${{ matrix.numpy_ver != 'latest'}} - run: pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} + run: pip install numpy==${{ matrix.numpy_ver }} - name: Install standard dependencies run: | From 771939dad1059957e4b76e9e526aab2b0f91aa78 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 10:02:46 +0900 Subject: [PATCH 297/396] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 55fc7da7..420becd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = ["setuptools >= 38.6", "pip >= 10"] -build-backend = "setuptools.build_meta" +build-backend = "setuptools.build_meta:__legacy__" [project] name = "pysatNASA" From 879f3a503cd1ec61ff40510d19ac8865a904d5a9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 10:42:47 +0900 Subject: [PATCH 298/396] Update setup.cfg --- setup.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.cfg b/setup.cfg index ccced987..17163008 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,7 @@ +[metadata] +name = "pysatNASA" +version = "0.0.5" + [flake8] max-line-length = 80 ignore = From 5e0cd34a855ba5d09bbf1fa83193b5566ebe39c6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 10:57:12 +0900 Subject: [PATCH 299/396] Update setup.cfg --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 17163008..b507df90 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] -name = "pysatNASA" -version = "0.0.5" +name = pysatNASA +version = 0.0.5 [flake8] max-line-length = 80 From f5baa1357a86fb61b9dd88b4575d15bb6ea443b6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 15:58:51 +0900 Subject: [PATCH 300/396] Update .readthedocs.yml --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index d2c27c90..23fa5ade 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,4 +1,4 @@ -# .readthedocs.yaml +# .readthedocs.yml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details From 3726577748a8640bb55ddee1f65fb3e4ef8d0fb8 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 19 May 2023 15:03:13 -0400 Subject: [PATCH 301/396] BUG: SSUSI dimensions may change There are dimensions that are not always present in the SSUSI data, and so this kwarg must be False. --- pysatNASA/instruments/methods/jhuapl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index e11971ea..e7c8753e 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -209,7 +209,7 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): # After loading all the data, determine which dimensions need to be # expanded. Pad the data so that all dimensions are the same shape - single_data = expand_coords(single_data, mdata, dims_equal=True) + single_data = expand_coords(single_data, mdata, dims_equal=False) # Combine all the data, indexing along time data = xr.combine_by_coords(single_data) From 2c8128477a5511d3d129d646383e0e1a7a3d3792 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 19 May 2023 15:22:02 -0400 Subject: [PATCH 302/396] DEP: added deprecation warning and filters Added a deprecation warning to `expand_coords` and filtered in in the local calls so that users are only warned when they call the function directly. --- pysatNASA/instruments/methods/jhuapl.py | 30 ++++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index e7c8753e..385e12e2 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pds import xarray as xr +import warnings from pysat.utils.io import load_netcdf @@ -62,6 +63,11 @@ def build_dtimes(data, var, epoch=None, epoch_var='time'): def expand_coords(data_list, mdata, dims_equal=False): """Ensure that dimensions do not vary from file to file. + .. deprecated:: 0.0.5 + This function is now included in `pysat.utils.coords` as + `expand_xarray_dims`, and so this function will be removed in + 0.1.0+ to reduce redundancy + Parameters ---------- data_list : list-like @@ -80,6 +86,12 @@ def expand_coords(data_list, mdata, dims_equal=False): when needed. """ + warnings.warn("".join(["This function is now included in ", + "`pysat.utils.coords` as `expand_xarray_dims`, and", + " so this function will be removed in 0.1.0+ to ", + "reduce redundancy"]), + DeprecationWarning, stacklevel=2) + # Get a list of all the dimensions if dims_equal: dims = list(data_list[0].dims.keys()) if len(data_list) > 0 else [] @@ -208,8 +220,11 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): mdata[var] = {mdata.labels.fill_val: mdata.header.NO_DATA_IN_BIN_VALUE} # After loading all the data, determine which dimensions need to be - # expanded. Pad the data so that all dimensions are the same shape - single_data = expand_coords(single_data, mdata, dims_equal=False) + # expanded. Pad the data so that all dimensions are the same shape. + # TODO(#169): Remove warning catches after expand_coords is removed + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + single_data = expand_coords(single_data, mdata, dims_equal=False) # Combine all the data, indexing along time data = xr.combine_by_coords(single_data) @@ -366,10 +381,13 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, # Combine all time dimensions if combine_times: - data_list = expand_coords([inners[dim] if dim == 'time' else - inners[dim].rename_dims({dim: 'time'}) - for dim in time_dims], mdata, - dims_equal=False) + # TODO(#169): Remove warning catches after expand_coords is removed + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + data_list = expand_coords([inners[dim] if dim == 'time' else + inners[dim].rename_dims({dim: 'time'}) + for dim in time_dims], mdata, + dims_equal=False) else: data_list = [inners[dim] for dim in time_dims] From 2993ec4572075b3d82ce663cb1863746c706dee3 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 19 May 2023 15:22:27 -0400 Subject: [PATCH 303/396] TST: added test for the new deprecation warning Added a unit test for the new deprecation warning. --- pysatNASA/tests/test_methods_jhuapl.py | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 pysatNASA/tests/test_methods_jhuapl.py diff --git a/pysatNASA/tests/test_methods_jhuapl.py b/pysatNASA/tests/test_methods_jhuapl.py new file mode 100644 index 00000000..62301c6d --- /dev/null +++ b/pysatNASA/tests/test_methods_jhuapl.py @@ -0,0 +1,39 @@ +"""Unit tests for the JHU-APL method functions.""" + +import warnings + +import pysat + +from pysatNASA.instruments.methods import jhuapl + + +class TestDeprecation(object): + """Unit tests for deprecated functions.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + warnings.simplefilter("always", DeprecationWarning) + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + return + + def test_expand_coords(self): + """Test that convert_timestamp_to_datetime is deprecated.""" + + warn_msgs = ["".join(["This function is now included in ", + "`pysat.utils.coords` as `expand_xarray_dims`, ", + "and so this function will be removed in 0.1.0+", + " to reduce redundancy"])] + + with warnings.catch_warnings(record=True) as war: + jhuapl.expand_coords([], None) + + # Ensure the minimum number of warnings were raised + assert len(war) >= len(warn_msgs) + + # Test the warning messages, ensuring each attribute is present + pysat.utils.testing.eval_warnings(war, warn_msgs) + return From 9490799448ef38cd4f6b38c29d54b375dd47f89b Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 19 May 2023 15:25:27 -0400 Subject: [PATCH 304/396] DOC: updated Changelog Updated the changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0634791c..bebdefb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). be removed before attempting to merge. * Deprecations * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps + * Deprecated expand_coords function, as it was introduced to pysat * Maintenance * Removed duplicate tests if pysatCDF not isntalled * Only test pysatCDF on GitHub Actions for older numpy versions From ee68ecdac599fed5aff3025af02f2ffee5c1c6ac Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 19 May 2023 15:29:25 -0400 Subject: [PATCH 305/396] STY: fixed flake8 issues Fixed issues revealed by CI flake8. --- pysatNASA/instruments/methods/jhuapl.py | 2 +- pysatNASA/tests/test_instruments.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 385e12e2..07af2428 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -4,8 +4,8 @@ import datetime as dt import numpy as np import pandas as pds -import xarray as xr import warnings +import xarray as xr from pysat.utils.io import load_netcdf diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index b19380bd..3084724e 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -87,7 +87,7 @@ def test_load_cdflib(self, inst_dict): assert UserWarning in categories else: # If error message does not match, raise error anyway - raise(verr) + raise ValueError(verr) # Make sure fake data is cleared assert target not in test_inst.data From 1cbcc2d136f000adcb871cc4cf2357943ff5ae3e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 19 May 2023 17:02:35 -0400 Subject: [PATCH 306/396] BUG: added catch for missing metadata Added a catch for missing metadata when determining the fill value. --- pysatNASA/instruments/methods/jhuapl.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 07af2428..8a5c2c47 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -133,9 +133,18 @@ def expand_coords(data_list, mdata, dims_equal=False): idim = list(sdata[dvar].dims).index(dim) new_shape[idim] = combo_dims[dim] + # Get the fill value + if dvar in mdata: + # If available, take it from the metadata + fill_val = mdata[dvar, mdata.labels.fill_val] + else: + # Otherwise, use the data type + ftype = type(sdata[dvar].values.flatten()[0]) + fill_val = mdata.labels.default_values_from_type( + mdata.labels.label_type['fill_val'], ftype) + # Set the new data for output - new_dat = np.full(shape=new_shape, fill_value=mdata[ - dvar, mdata.labels.fill_val]) + new_dat = np.full(shape=new_shape, fill_value=) new_dat[tuple(old_slice)] = sdata[dvar].values new_data[dvar] = (sdata[dvar].dims, new_dat) else: From 57b8dbe81039e33d8879a8ebd787b58f14f21b33 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 19 May 2023 17:06:19 -0400 Subject: [PATCH 307/396] BUG: added missing variable Added a missing variable to the full call. --- pysatNASA/instruments/methods/jhuapl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 8a5c2c47..a29765be 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -144,7 +144,7 @@ def expand_coords(data_list, mdata, dims_equal=False): mdata.labels.label_type['fill_val'], ftype) # Set the new data for output - new_dat = np.full(shape=new_shape, fill_value=) + new_dat = np.full(shape=new_shape, fill_value=fill_val) new_dat[tuple(old_slice)] = sdata[dvar].values new_data[dvar] = (sdata[dvar].dims, new_dat) else: From f86fd9d3fea6072350f2387b047c48b2d449d520 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Tue, 30 May 2023 13:36:08 -0400 Subject: [PATCH 308/396] STY: update meta --- pyproject.toml | 5 ++--- setup.cfg | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 420becd5..fe9c33c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,9 @@ [build-system] -requires = ["setuptools >= 38.6", "pip >= 10"] -build-backend = "setuptools.build_meta:__legacy__" +requires = ["setuptools", "pip >= 10"] +build-backend = "setuptools.build_meta" [project] name = "pysatNASA" -version = "0.0.5" description = "pysat support for NASA Instruments" readme = "README.md" requires-python = ">=3.6" diff --git a/setup.cfg b/setup.cfg index b507df90..400e122a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,4 @@ [metadata] -name = pysatNASA version = 0.0.5 [flake8] From 2b41e01dbbd5e662a1cd3d945a0e67128fadb64d Mon Sep 17 00:00:00 2001 From: jklenzing Date: Tue, 30 May 2023 13:40:07 -0400 Subject: [PATCH 309/396] BUG: fix metadata --- pyproject.toml | 1 + setup.cfg | 3 +++ 2 files changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fe9c33c5..0f5cc26f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pysatNASA" +version = "0.0.5" description = "pysat support for NASA Instruments" readme = "README.md" requires-python = ">=3.6" diff --git a/setup.cfg b/setup.cfg index 400e122a..08661e10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,7 @@ +# name and version must be maintained here as well for python 3.6 compatibility + [metadata] +name = pysatNASA version = 0.0.5 [flake8] From 8d3202d3419cf5bd5932e31c56ca6f639c6de909 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 30 May 2023 13:51:55 -0400 Subject: [PATCH 310/396] DEP: removed expand_coords Removed the local `expand_coords` function and replaced it with the new pysat `expand_xarray_dims` function. --- pysatNASA/instruments/methods/jhuapl.py | 114 ++---------------------- 1 file changed, 6 insertions(+), 108 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index a29765be..15575383 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -4,9 +4,9 @@ import datetime as dt import numpy as np import pandas as pds -import warnings import xarray as xr +from pysat.utils.coords import expand_xarray_dims from pysat.utils.io import load_netcdf @@ -60,102 +60,6 @@ def build_dtimes(data, var, epoch=None, epoch_var='time'): return dtimes -def expand_coords(data_list, mdata, dims_equal=False): - """Ensure that dimensions do not vary from file to file. - - .. deprecated:: 0.0.5 - This function is now included in `pysat.utils.coords` as - `expand_xarray_dims`, and so this function will be removed in - 0.1.0+ to reduce redundancy - - Parameters - ---------- - data_list : list-like - List of xr.Dataset objects with the same dimensions and variables - mdata : pysat.Meta - Metadata for the data in `data_list` - dims_equal : bool - Assert that all xr.Dataset objects have the same dimensions if True - (default=False) - - Returns - ------- - out_list : list-like - List of xr.Dataset objects with the same dimensions and variables, - now with dimensions that all have the same values and data padded - when needed. - - """ - warnings.warn("".join(["This function is now included in ", - "`pysat.utils.coords` as `expand_xarray_dims`, and", - " so this function will be removed in 0.1.0+ to ", - "reduce redundancy"]), - DeprecationWarning, stacklevel=2) - - # Get a list of all the dimensions - if dims_equal: - dims = list(data_list[0].dims.keys()) if len(data_list) > 0 else [] - else: - dims = list() - for sdata in data_list: - if len(dims) == 0: - dims = list(sdata.dims.keys()) - else: - for dim in list(sdata.dims.keys()): - if dim not in dims: - dims.append(dim) - - # After loading all the data, determine which dimensions may need to be - # expanded, as they could differ in dimensions from file to file - combo_dims = {dim: max([sdata.dims[dim] for sdata in data_list - if dim in sdata.dims]) for dim in dims} - - # Expand the data so that all dimensions are the same shape - out_list = list() - for i, sdata in enumerate(data_list): - # Determine which dimensions need to be updated - fix_dims = [dim for dim in sdata.dims.keys() - if sdata.dims[dim] < combo_dims[dim]] - - new_data = {} - update_new = False - for dvar in sdata.data_vars.keys(): - # See if any dimensions need to be updated - update_dims = list(set(sdata[dvar].dims) & set(fix_dims)) - - # Save the old data as is, or pad it to have the right dims - if len(update_dims) > 0: - update_new = True - new_shape = list(sdata[dvar].values.shape) - old_slice = [slice(0, ns) for ns in new_shape] - - for dim in update_dims: - idim = list(sdata[dvar].dims).index(dim) - new_shape[idim] = combo_dims[dim] - - # Get the fill value - if dvar in mdata: - # If available, take it from the metadata - fill_val = mdata[dvar, mdata.labels.fill_val] - else: - # Otherwise, use the data type - ftype = type(sdata[dvar].values.flatten()[0]) - fill_val = mdata.labels.default_values_from_type( - mdata.labels.label_type['fill_val'], ftype) - - # Set the new data for output - new_dat = np.full(shape=new_shape, fill_value=fill_val) - new_dat[tuple(old_slice)] = sdata[dvar].values - new_data[dvar] = (sdata[dvar].dims, new_dat) - else: - new_data[dvar] = sdata[dvar] - - # Get the updated dataset - out_list.append(xr.Dataset(new_data) if update_new else sdata) - - return out_list - - def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): """Load JHU APL EDR Aurora data and meta data. @@ -230,10 +134,7 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): # After loading all the data, determine which dimensions need to be # expanded. Pad the data so that all dimensions are the same shape. - # TODO(#169): Remove warning catches after expand_coords is removed - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - single_data = expand_coords(single_data, mdata, dims_equal=False) + single_data = expand_xarray_dims(single_data, mdata, dims_equal=False) # Combine all the data, indexing along time data = xr.combine_by_coords(single_data) @@ -390,13 +291,10 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, # Combine all time dimensions if combine_times: - # TODO(#169): Remove warning catches after expand_coords is removed - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - data_list = expand_coords([inners[dim] if dim == 'time' else - inners[dim].rename_dims({dim: 'time'}) - for dim in time_dims], mdata, - dims_equal=False) + data_list = expand_xarray_dims([inners[dim] if dim == 'time' else + inners[dim].rename_dims({dim: 'time'}) + for dim in time_dims], mdata, + dims_equal=False) else: data_list = [inners[dim] for dim in time_dims] From 175e571474da537574b3134707ef80243b584dd2 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 30 May 2023 13:52:16 -0400 Subject: [PATCH 311/396] TST: removed deprecation tests Removed the deprecation tests, as they are no longer needed. --- pysatNASA/tests/test_methods_jhuapl.py | 39 -------------------------- 1 file changed, 39 deletions(-) delete mode 100644 pysatNASA/tests/test_methods_jhuapl.py diff --git a/pysatNASA/tests/test_methods_jhuapl.py b/pysatNASA/tests/test_methods_jhuapl.py deleted file mode 100644 index 62301c6d..00000000 --- a/pysatNASA/tests/test_methods_jhuapl.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Unit tests for the JHU-APL method functions.""" - -import warnings - -import pysat - -from pysatNASA.instruments.methods import jhuapl - - -class TestDeprecation(object): - """Unit tests for deprecated functions.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - warnings.simplefilter("always", DeprecationWarning) - return - - def teardown_method(self): - """Clean up the unit test environment after each method.""" - - return - - def test_expand_coords(self): - """Test that convert_timestamp_to_datetime is deprecated.""" - - warn_msgs = ["".join(["This function is now included in ", - "`pysat.utils.coords` as `expand_xarray_dims`, ", - "and so this function will be removed in 0.1.0+", - " to reduce redundancy"])] - - with warnings.catch_warnings(record=True) as war: - jhuapl.expand_coords([], None) - - # Ensure the minimum number of warnings were raised - assert len(war) >= len(warn_msgs) - - # Test the warning messages, ensuring each attribute is present - pysat.utils.testing.eval_warnings(war, warn_msgs) - return From 2193d4842d24e0dfcc360970218a57b4d7d4e5f7 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 30 May 2023 13:52:34 -0400 Subject: [PATCH 312/396] DOC: updated changelog Removed deprecation warning from the changelog. --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bebdefb2..0634791c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,6 @@ This project adheres to [Semantic Versioning](https://semver.org/). be removed before attempting to merge. * Deprecations * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps - * Deprecated expand_coords function, as it was introduced to pysat * Maintenance * Removed duplicate tests if pysatCDF not isntalled * Only test pysatCDF on GitHub Actions for older numpy versions From da2229b9aafb48254ac4202f5e2dbd3caa597fc6 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 31 May 2023 17:37:54 -0400 Subject: [PATCH 313/396] BUG: add sleep to download --- pysatNASA/instruments/methods/cdaweb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 02407a85..4d7a48a8 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -13,6 +13,7 @@ import os import pandas as pds import requests +import time import warnings import xarray as xr @@ -549,6 +550,7 @@ def download(date_array, tag='', inst_id='', supported_tags=None, except requests.exceptions.RequestException as exception: logger.info(' '.join((str(exception), '- File not available for', date.strftime('%d %B %Y')))) + time.sleep(0.1) return @@ -644,6 +646,7 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, except requests.exceptions.RequestException as exception: logger.info(' '.join((str(exception), '- File: "', file, '" Is not available'))) + time.sleep(0.1) return From f95ddadbf0a49fd4b822a39cecec4bc298081450 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 31 May 2023 17:38:05 -0400 Subject: [PATCH 314/396] DOC: update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c86b9bd..aa3bcdea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -## [0.X.X] - 2023-XX-XX +## [0.0.5] - 2023-XX-XX * New Instruments * ACE EPAM * ACE MAG @@ -25,6 +25,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated GOLD nmax to sort scans by time. * Added 1 usec to GOLD nmax channel B times to ensure uniqueness * Fixed multi-file loads for cdf xarray datasets. + * Adds a 0.1 sec delay between file downloads to avoid excessive calls + to servers. * Documentation * Added TIMED-GUVI platform * Added missing sub-module imports From 74050cea0630ca4162439e808b126174aaa7b08e Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 31 May 2023 18:24:04 -0400 Subject: [PATCH 315/396] BUG: time v time --- pysatNASA/instruments/methods/cdaweb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 4d7a48a8..0a8838f0 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -13,7 +13,7 @@ import os import pandas as pds import requests -import time +from time import sleep import warnings import xarray as xr @@ -550,7 +550,7 @@ def download(date_array, tag='', inst_id='', supported_tags=None, except requests.exceptions.RequestException as exception: logger.info(' '.join((str(exception), '- File not available for', date.strftime('%d %B %Y')))) - time.sleep(0.1) + sleep(0.1) return @@ -646,7 +646,7 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, except requests.exceptions.RequestException as exception: logger.info(' '.join((str(exception), '- File: "', file, '" Is not available'))) - time.sleep(0.1) + sleep(0.1) return From 2dd9535bbf9f9ef1d95f6fd00fddcb6b7d58e780 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 31 May 2023 18:25:21 -0400 Subject: [PATCH 316/396] DOC: add comments --- pysatNASA/instruments/methods/cdaweb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 0a8838f0..a6e52b19 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -550,6 +550,7 @@ def download(date_array, tag='', inst_id='', supported_tags=None, except requests.exceptions.RequestException as exception: logger.info(' '.join((str(exception), '- File not available for', date.strftime('%d %B %Y')))) + # Pause to avoid excessive pings to server sleep(0.1) return @@ -646,6 +647,7 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, except requests.exceptions.RequestException as exception: logger.info(' '.join((str(exception), '- File: "', file, '" Is not available'))) + # Pause to avoid excessive pings to server sleep(0.1) return From ba1291ce4a2b5f98a73b4a3e5da046c6facd1b5d Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 31 May 2023 23:24:33 -0400 Subject: [PATCH 317/396] BUG: fix saber concatenation --- pysatNASA/instruments/timed_saber.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index d1e6aa0c..555d30a5 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -99,8 +99,11 @@ supported_tags=supported_tags) # Set the load routine +# Note that duplicate string and variables to Epoch are dropped to aid in +# concatonation of files. load = functools.partial(cdw.load, pandas_format=pandas_format, - drop_dims='record0') + drop_dims='record0', drop_vars='time', + use_cdflib=True) # Set the download routine download_tags = {'': {'': 'TIMED_L2A_SABER'}} From ef206f1e950d7937b3e32853233d17c72a0cdf02 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 31 May 2023 23:25:20 -0400 Subject: [PATCH 318/396] ENH: add drop_vars to load from xarray --- pysatNASA/instruments/methods/cdaweb.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 02407a85..cc797fd9 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -64,8 +64,8 @@ def try_inst_dict(inst_id, tag, supported_tags): def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), flatten_twod=True, pandas_format=True, epoch_name='Epoch', - drop_dims=None, meta_processor=None, meta_translation=None, - drop_meta_labels=None, use_cdflib=None): + drop_dims=None, drop_vars=None, meta_processor=None, + meta_translation=None, drop_meta_labels=None, use_cdflib=None): """Load NASA CDAWeb CDF files. Parameters @@ -97,6 +97,9 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), List of variable dimensions that should be dropped. Applied to data as loaded from the file. Used only from xarray Dataset. (default=None) + drop_vars : list or NoneType + List of variables that should be dropped. Applied to data as loaded + from the file. Used only from xarray Dataset. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -148,6 +151,7 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id, epoch_name=epoch_name, drop_dims=drop_dims, + drop_vars=drop_vars, file_cadence=file_cadence, meta_processor=meta_processor, meta_translation=meta_translation, @@ -269,8 +273,9 @@ def load_xarray(fnames, tag='', inst_id='', 'min_val': ('ValidMin', float), 'max_val': ('ValidMax', float), 'fill_val': ('FillVal', float)}, - epoch_name='Epoch', drop_dims=None, meta_processor=None, - meta_translation=None, drop_meta_labels=None): + epoch_name='Epoch', drop_dims=None, drop_vars=None, + meta_processor=None, meta_translation=None, + drop_meta_labels=None): """Load NASA CDAWeb CDF files into an xarray Dataset. Parameters @@ -303,6 +308,9 @@ def load_xarray(fnames, tag='', inst_id='', drop_dims : list or NoneType List of variable dimensions that should be dropped. Applied to data as loaded from the file. (default=None) + drop_vars : list or NoneType + List of variables that should be dropped. Applied to data as loaded + from the file. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -367,6 +375,8 @@ def load_xarray(fnames, tag='', inst_id='', temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True) if drop_dims: temp_data = temp_data.drop_dims(drop_dims) + if drop_vars: + temp_data = temp_data.drop_vars(drop_vars) ldata.append(temp_data) # Combine individual files together, concat along epoch From 2d739490a629a44be8b716388a7d2c273ce68a30 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 31 May 2023 23:25:44 -0400 Subject: [PATCH 319/396] STY: ues_cdflib True for xarray --- pysatNASA/instruments/ace_epam_l2.py | 2 +- pysatNASA/instruments/ace_mag_l2.py | 2 +- pysatNASA/instruments/ace_sis_l2.py | 2 +- pysatNASA/instruments/ace_swepam_l2.py | 2 +- pysatNASA/instruments/igs_gps.py | 3 ++- pysatNASA/instruments/jpl_gps.py | 3 ++- pysatNASA/instruments/timed_see.py | 4 ++-- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index 670ed3ad..4ef14b23 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -92,7 +92,7 @@ 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} load = functools.partial(cdw.load, pandas_format=pandas_format, - meta_translation=meta_translation) + meta_translation=meta_translation, use_cdflib=True) # Set the download routine download_tags = {'12sec': {'base': 'AC_H3_EPM'}, diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index 8b5c4ea6..bff8ad2a 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -94,7 +94,7 @@ 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} load = functools.partial(cdw.load, pandas_format=pandas_format, - meta_translation=meta_translation) + meta_translation=meta_translation, use_cdflib=True) # Set the download routine download_tags = {'1sec': {'base': 'AC_H3_MFI'}, diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index f69a4308..cf55bed9 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -87,7 +87,7 @@ 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} load = functools.partial(cdw.load, pandas_format=pandas_format, - meta_translation=meta_translation) + meta_translation=meta_translation, use_cdflib=True) # Set the download routine download_tags = {'256sec': {'base': 'AC_H1_SIS'}, diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index a4a5ae20..2d85e524 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -90,7 +90,7 @@ 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} load = functools.partial(cdw.load, pandas_format=pandas_format, - meta_translation=meta_translation) + meta_translation=meta_translation, use_cdflib=True) # Set the download routine download_tags = {'64sec': {'base': 'AC_H0_SWE'}, diff --git a/pysatNASA/instruments/igs_gps.py b/pysatNASA/instruments/igs_gps.py index e9e17bcc..5c64c03f 100644 --- a/pysatNASA/instruments/igs_gps.py +++ b/pysatNASA/instruments/igs_gps.py @@ -123,7 +123,8 @@ supported_tags=supported_tags) # Set the load routine -load = functools.partial(cdw.load, pandas_format=pandas_format) +load = functools.partial(cdw.load, pandas_format=pandas_format, + use_cdflib=True) # Set the download routine download = functools.partial(cdw.cdas_download, supported_tags=cdas_labels) diff --git a/pysatNASA/instruments/jpl_gps.py b/pysatNASA/instruments/jpl_gps.py index 43414b0b..255effa6 100644 --- a/pysatNASA/instruments/jpl_gps.py +++ b/pysatNASA/instruments/jpl_gps.py @@ -113,7 +113,8 @@ def init(self): supported_tags=supported_tags) # Set the load routine -load = functools.partial(cdw.load, pandas_format=pandas_format) +load = functools.partial(cdw.load, pandas_format=pandas_format, + use_cdflib=True) # Set the download routine download_tags = {'': {'roti': 'GPS_ROTI15MIN_JPL'}} diff --git a/pysatNASA/instruments/timed_see.py b/pysatNASA/instruments/timed_see.py index 1980443e..02cc5d3c 100644 --- a/pysatNASA/instruments/timed_see.py +++ b/pysatNASA/instruments/timed_see.py @@ -77,8 +77,8 @@ file_cadence=pds.DateOffset(months=1)) # Set the load routine -load = functools.partial(cdw.load, pandas_format=pandas_format, - file_cadence=pds.DateOffset(months=1)) +load = functools.partial(cdw.load, file_cadence=pds.DateOffset(months=1), + pandas_format=pandas_format, use_cdflib=True) # Set the download routine download_tags = {'': {'': 'TIMED_L3A_SEE'}} From babc30ce102e4d42ade96c029b3c03f77537170e Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 31 May 2023 23:26:16 -0400 Subject: [PATCH 320/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c86b9bd..53683f59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * xarray support for TIMED SABER and SEE * Added `drop_dims` kwarg to `load_xarray` interface so that orphan dims can be removed before attempting to merge. + * Improved usage of cdflib for users in xarray instruments * Deprecations * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps * Maintenance From 4a80aa07c018239ae85878132f6089382d821468 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 07:44:56 -0400 Subject: [PATCH 321/396] ENH: rename variables --- pysatNASA/instruments/methods/cdaweb.py | 26 ++++++++++++------------- pysatNASA/instruments/timed_saber.py | 7 ++++--- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index cc797fd9..30dbbae2 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -64,7 +64,7 @@ def try_inst_dict(inst_id, tag, supported_tags): def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), flatten_twod=True, pandas_format=True, epoch_name='Epoch', - drop_dims=None, drop_vars=None, meta_processor=None, + drop_dims=None, var_translation=None, meta_processor=None, meta_translation=None, drop_meta_labels=None, use_cdflib=None): """Load NASA CDAWeb CDF files. @@ -97,8 +97,8 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), List of variable dimensions that should be dropped. Applied to data as loaded from the file. Used only from xarray Dataset. (default=None) - drop_vars : list or NoneType - List of variables that should be dropped. Applied to data as loaded + var_translation : dict or NoneType + Variables that should be renamed. Applied to data as loaded from the file. Used only from xarray Dataset. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be @@ -151,7 +151,7 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id, epoch_name=epoch_name, drop_dims=drop_dims, - drop_vars=drop_vars, + var_translation=var_translation, file_cadence=file_cadence, meta_processor=meta_processor, meta_translation=meta_translation, @@ -270,10 +270,10 @@ def load_xarray(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), labels={'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', float), - 'max_val': ('ValidMax', float), - 'fill_val': ('FillVal', float)}, - epoch_name='Epoch', drop_dims=None, drop_vars=None, + 'min_val': ('ValidMin', (int, float)), + 'max_val': ('ValidMax', (int, float)), + 'fill_val': ('FillVal', (int, float))}, + epoch_name='Epoch', drop_dims=None, var_translation=None, meta_processor=None, meta_translation=None, drop_meta_labels=None): """Load NASA CDAWeb CDF files into an xarray Dataset. @@ -308,9 +308,9 @@ def load_xarray(fnames, tag='', inst_id='', drop_dims : list or NoneType List of variable dimensions that should be dropped. Applied to data as loaded from the file. (default=None) - drop_vars : list or NoneType - List of variables that should be dropped. Applied to data as loaded - from the file. (default=None) + var_translation : dict or NoneType + Variables that should be renamed. Applied to data as loaded + from the file. Used only from xarray Dataset. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -375,8 +375,8 @@ def load_xarray(fnames, tag='', inst_id='', temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True) if drop_dims: temp_data = temp_data.drop_dims(drop_dims) - if drop_vars: - temp_data = temp_data.drop_vars(drop_vars) + if var_translation: + temp_data = temp_data.rename(var_translation) ldata.append(temp_data) # Combine individual files together, concat along epoch diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index 555d30a5..1b238f75 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -99,10 +99,11 @@ supported_tags=supported_tags) # Set the load routine -# Note that duplicate string and variables to Epoch are dropped to aid in -# concatonation of files. +# Note that the time variable associated with tpaltitude is renamed to avoid +# conflict with renaming Epoch. load = functools.partial(cdw.load, pandas_format=pandas_format, - drop_dims='record0', drop_vars='time', + drop_dims='record0', + var_translation={'time': 'tp_time'}, use_cdflib=True) # Set the download routine From 5a13ace738f54388b21080931e4678a6ecf003b2 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 09:07:34 -0400 Subject: [PATCH 322/396] BUG: xarray meta --- pysatNASA/instruments/methods/cdaweb.py | 42 ++++++++++++------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 30dbbae2..c23b52f2 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -385,26 +385,6 @@ def load_xarray(fnames, tag='', inst_id='', all_vars = io.xarray_all_vars(data) - # Convert output epoch name to 'time' for pysat consistency - if epoch_name != 'time': - if 'time' not in all_vars: - if epoch_name in data.dims: - data = data.rename({epoch_name: 'time'}) - elif epoch_name in all_vars: - data = data.rename({epoch_name: 'time'}) - wstr = ''.join(['Epoch label: "', epoch_name, '"', - ' is not a dimension.']) - pysat.logger.warning(wstr) - else: - estr = ''.join(['Epoch label: "', epoch_name, '"', - ' not found in loaded data, ', - repr(all_vars)]) - raise ValueError(estr) - - epoch_name = 'time' - - all_vars = io.xarray_all_vars(data) - meta = pysat.Meta(labels=labels) full_mdict = {} @@ -421,8 +401,8 @@ def load_xarray(fnames, tag='', inst_id='', for key in all_vars: meta_dict = {} - for nc_key in data[key].attrs.keys(): - meta_dict[nc_key] = data[key].attrs[nc_key] + for nc_key in ldata[0][key].attrs.keys(): + meta_dict[nc_key] = ldata[0][key].attrs[nc_key] full_mdict[key] = meta_dict data[key].attrs = {} @@ -457,6 +437,24 @@ def load_xarray(fnames, tag='', inst_id='', for key in filt_mdict: meta[key] = filt_mdict[key] + # Convert output epoch name to 'time' for pysat consistency + if epoch_name != 'time': + if 'time' not in all_vars: + if epoch_name in data.dims: + data = data.rename({epoch_name: 'time'}) + elif epoch_name in all_vars: + data = data.rename({epoch_name: 'time'}) + wstr = ''.join(['Epoch label: "', epoch_name, '"', + ' is not a dimension.']) + pysat.logger.warning(wstr) + else: + estr = ''.join(['Epoch label: "', epoch_name, '"', + ' not found in loaded data, ', + repr(all_vars)]) + raise ValueError(estr) + + epoch_name = 'time' + # Remove attributes from the data object data.attrs = {} From b4526b185cbd88b6236c7ab1425047b5dcaa0243 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 09:17:33 -0400 Subject: [PATCH 323/396] ENH: use kwarg for combine_nested --- pysatNASA/instruments/methods/cdaweb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index c23b52f2..b6d24d55 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -381,7 +381,8 @@ def load_xarray(fnames, tag='', inst_id='', # Combine individual files together, concat along epoch if len(ldata) > 0: - data = xr.combine_nested(ldata, epoch_name) + data = xr.combine_nested(ldata, epoch_name, + combine_attrs='override') all_vars = io.xarray_all_vars(data) @@ -401,8 +402,8 @@ def load_xarray(fnames, tag='', inst_id='', for key in all_vars: meta_dict = {} - for nc_key in ldata[0][key].attrs.keys(): - meta_dict[nc_key] = ldata[0][key].attrs[nc_key] + for nc_key in data[key].attrs.keys(): + meta_dict[nc_key] = data[key].attrs[nc_key] full_mdict[key] = meta_dict data[key].attrs = {} @@ -438,6 +439,7 @@ def load_xarray(fnames, tag='', inst_id='', meta[key] = filt_mdict[key] # Convert output epoch name to 'time' for pysat consistency + # This needs to be done last so that meta is updated properly if epoch_name != 'time': if 'time' not in all_vars: if epoch_name in data.dims: From d01ae4f43de1e15684d372702585c23f3f7cc1e6 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 09:31:45 -0400 Subject: [PATCH 324/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53683f59..1866b63d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated GitHub Actions workflows for improved compliance with pip>=23.0 * Added .readthedocs.yml to configure settings there. * Use pyproject.toml to manage installation and metadata + * Set use_cdflib=True for supported xarray instruments ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From f516015186344925199fe7d93cd20bc4a5dca955 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 09:50:55 -0400 Subject: [PATCH 325/396] DOC: update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1866b63d..8e5e0849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * xarray support for TIMED SABER and SEE * Added `drop_dims` kwarg to `load_xarray` interface so that orphan dims can be removed before attempting to merge. + * Added `var_translation` kwarg to `load_xarray` interface so that variables can + be renamed before attempting to merge. * Improved usage of cdflib for users in xarray instruments * Deprecations * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps From 20a07c2685864110dd3d70a30a8780d747a3cb1e Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 11:19:52 -0400 Subject: [PATCH 326/396] MAINT: remove try except --- pysatNASA/instruments/ses14_gold.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 4fa32bd4..9523f33f 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -244,21 +244,11 @@ def load(fnames, tag='', inst_id=''): elif tag == 'o2den': epoch_name = 'nevents' - # TODO(#165): remove try/except notation once pysat 3.1.0 is released - try: - data, meta = load_netcdf(fnames, pandas_format=pandas_format, - epoch_name=epoch_name, labels=labels, - meta_translation=meta_translation, - combine_by_coords=False, - drop_meta_labels='FILLVAL') - except TypeError: - pysat.logger.warn(' '.join(("Loading multiple days of data may error.", - "Upgrade to pysat 3.1.0 or higher to", - "resolve this issue."))) - data, meta = load_netcdf(fnames, pandas_format=pandas_format, - epoch_name=epoch_name, labels=labels, - meta_translation=meta_translation, - drop_meta_labels='FILLVAL') + data, meta = load_netcdf(fnames, pandas_format=pandas_format, + epoch_name=epoch_name, labels=labels, + meta_translation=meta_translation, + combine_by_coords=False, + drop_meta_labels='FILLVAL') if tag in ['nmax', 'tdisk', 'tlimb']: # Add time coordinate from scan_start_time From 451ba0bf203e776eb96488d5f25d57a5d7b8cdab Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 11:20:59 -0400 Subject: [PATCH 327/396] MAINT: pysat 3.1.0 minimum --- README.md | 2 +- docs/installation.rst | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c0235964..0d5ab95a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Python 3.6+. | Common modules | Community modules | Optional Modules | | ---------------- | ----------------- |------------------| | beautifulsoup4 | cdflib | pysatCDF | -| lxml | pysat>=3.0.4 | | +| lxml | pysat>=3.1.0 | | | netCDF4 | | | | numpy | | | | pandas | | | diff --git a/docs/installation.rst b/docs/installation.rst index f74c7e4b..bb425a56 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -20,7 +20,7 @@ Python 3.8+ and pysat 3.0.4+. Common modules Community modules ================== ================= beautifulsoup4 cdflib>=0.4.4 - lxml pysat>=3.0.4 + lxml pysat>=3.1.0 netCDF4 numpy pandas diff --git a/pyproject.toml b/pyproject.toml index 0f5cc26f..646969a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "netCDF4", "numpy", "pandas", - "pysat >= 3.0.4", + "pysat >= 3.1", "requests", "xarray" ] From 1755f725e09023491a7f7ec867e460dcc48d0514 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 11:21:21 -0400 Subject: [PATCH 328/396] DOC: update docs --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index bb425a56..23ef4123 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -14,7 +14,7 @@ Prerequisites pysatNASA uses common Python modules, as well as modules developed by and for the Space Physics community. This module officially supports -Python 3.8+ and pysat 3.0.4+. +Python 3.8+ and pysat 3.1.0+. ================== ================= Common modules Community modules From 1f2199d62be95992cc589556286a384bb536e6e0 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 11:21:58 -0400 Subject: [PATCH 329/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e5e0849..7c968e2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added .readthedocs.yml to configure settings there. * Use pyproject.toml to manage installation and metadata * Set use_cdflib=True for supported xarray instruments + * Set pysat 3.1.0 minimum ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From c18a4fd7b285a1d1f431006472d1c961c770f380 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 11:37:01 -0400 Subject: [PATCH 330/396] MAINT: removed unused import --- pysatNASA/instruments/ses14_gold.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 9523f33f..9035a1b1 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -42,7 +42,6 @@ import functools import numpy as np -import pysat from pysat.instruments.methods import general as ps_gen from pysat.utils.io import load_netcdf From 4ec325846873739be7003c90c9110e51614825fb Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 12:31:54 -0400 Subject: [PATCH 331/396] STY: include strict_dim_check in jhuapl load --- pysatNASA/instruments/dmsp_ssusi.py | 3 ++- pysatNASA/instruments/methods/jhuapl.py | 10 ++++++++-- pysatNASA/instruments/timed_guvi.py | 6 ++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index 65f3fa4b..268985a8 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -106,7 +106,8 @@ supported_tags=supported_tags) # Set the load routine -load = functools.partial(jhuapl.load_edr_aurora, pandas_format=pandas_format) +load = functools.partial(jhuapl.load_edr_aurora, pandas_format=pandas_format, + strict_dim_check=False) # Set the download routine basic_tag = {'remote_dir': ''.join(('/pub/data/dmsp/dmsp{inst_id:s}/ssusi/', diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index e11971ea..99f2c277 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -135,7 +135,8 @@ def expand_coords(data_list, mdata, dims_equal=False): return out_list -def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): +def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, + strict_dim_check=True): """Load JHU APL EDR Aurora data and meta data. Parameters @@ -149,6 +150,10 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): (default='') pandas_format : bool False for xarray format, True for pandas (default=False) + strict_dim_check : bool + Used for xarray data (`pandas_format` is False). If True, warn the user + that the desired epoch is not present in `xarray.dims`. If False, + no warning is raised. (default=True) Returns ------- @@ -180,7 +185,8 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): # than a dimension or coordinate. Additionally, no coordinates # are assigned. sdata, mdata = load_netcdf(fname, epoch_name='TIME', epoch_unit='s', - labels=labels, pandas_format=pandas_format) + labels=labels, pandas_format=pandas_format, + strict_dim_check=strict_dim_check) # Calculate the time for this data file. The pysat `load_netcdf` routine # converts the 'TIME' parameter (seconds of day) into datetime using diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 36b8111b..85a6b8d7 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -188,10 +188,12 @@ def load(fnames, tag='', inst_id='', combine_times=False): """ if tag == 'edr-aur': data, meta = jhuapl.load_edr_aurora(fnames, tag, inst_id, - pandas_format=pandas_format) + pandas_format=pandas_format, + strict_dim_check=False) else: data, meta = jhuapl.load_sdr_aurora(fnames, tag, inst_id, pandas_format=pandas_format, - combine_times=combine_times) + combine_times=combine_times, + strict_dim_check=False) return data, meta From 89d1298a4a8386b861accf9e0332272fb36811bd Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Jun 2023 12:47:10 -0400 Subject: [PATCH 332/396] Update pysatNASA/instruments/timed_saber.py Co-authored-by: Angeline Burrell --- pysatNASA/instruments/timed_saber.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/timed_saber.py b/pysatNASA/instruments/timed_saber.py index 1b238f75..e1e5086a 100644 --- a/pysatNASA/instruments/timed_saber.py +++ b/pysatNASA/instruments/timed_saber.py @@ -98,9 +98,8 @@ list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) -# Set the load routine -# Note that the time variable associated with tpaltitude is renamed to avoid -# conflict with renaming Epoch. +# Set the load routine. Note that the time variable associated with +# tpaltitude is renamed to avoid conflict with renaming Epoch. load = functools.partial(cdw.load, pandas_format=pandas_format, drop_dims='record0', var_translation={'time': 'tp_time'}, From 2a0ea9f00b02c816340eaa4b0b2e3e5983f3cd77 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 1 Jun 2023 13:13:57 -0400 Subject: [PATCH 333/396] BUG: strict_dim_check in sdr_aurora --- pysatNASA/instruments/methods/jhuapl.py | 6 +++++- pysatNASA/instruments/timed_guvi.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 99f2c277..68e77a8e 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -224,7 +224,7 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, - combine_times=False): + strict_dim_check=True, combine_times=False): """Load JHU APL SDR data and meta data. Parameters @@ -238,6 +238,10 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, (default='') pandas_format : bool False for xarray format, True for pandas (default=False) + strict_dim_check : bool + Used for xarray data (`pandas_format` is False). If True, warn the user + that the desired epoch is not present in `xarray.dims`. If False, + no warning is raised. (default=True) combine_times : bool For SDR data, optionally combine the different datetime coordinates into a single time coordinate (default=False) diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 85a6b8d7..2a0f7da8 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -193,7 +193,7 @@ def load(fnames, tag='', inst_id='', combine_times=False): else: data, meta = jhuapl.load_sdr_aurora(fnames, tag, inst_id, pandas_format=pandas_format, - combine_times=combine_times, - strict_dim_check=False) + strict_dim_check=False, + combine_times=combine_times) return data, meta From e4070c55c1d0bd1a078393adfe47d532c85b5cfe Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Jun 2023 19:45:58 -0400 Subject: [PATCH 334/396] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6aedff4e..3684998a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ lxml netCDF4 numpy pandas -pysat>=3.0.4 +pysat>=3.1.0 requests xarray From 009bf658872e97d89488f9cc3aad3f44c5cc143a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Jun 2023 21:09:54 -0400 Subject: [PATCH 335/396] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 646969a3..5f7cd9f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ keywords = [ dependencies = [ "beautifulsoup4", "cdasws", - "cdflib >= 0.4.4", + "cdflib >= 0.4.4, <1.0", "lxml", "netCDF4", "numpy", From c26f1e87f3c1cc73033cf261e5077a6b018e441d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Jun 2023 21:23:38 -0400 Subject: [PATCH 336/396] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3684998a..898145bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4 cdasws -cdflib>=0.4.4 +cdflib>=0.4.4,<1.0 lxml netCDF4 numpy From 5d0fc54348bab0d8216d40ace5fbc2a3a544437d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 2 Jun 2023 12:42:23 -0400 Subject: [PATCH 337/396] STY: set sleep to 0.2 --- pysatNASA/instruments/methods/cdaweb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 900766e1..af0ad076 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -561,7 +561,7 @@ def download(date_array, tag='', inst_id='', supported_tags=None, logger.info(' '.join((str(exception), '- File not available for', date.strftime('%d %B %Y')))) # Pause to avoid excessive pings to server - sleep(0.1) + sleep(0.2) return @@ -658,7 +658,7 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, logger.info(' '.join((str(exception), '- File: "', file, '" Is not available'))) # Pause to avoid excessive pings to server - sleep(0.1) + sleep(0.2) return From f9392314a0f971be3a90fd20aca1db4b787fbffd Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 2 Jun 2023 12:54:55 -0400 Subject: [PATCH 338/396] Update pysatNASA/instruments/methods/jhuapl.py Co-authored-by: Angeline Burrell --- pysatNASA/instruments/methods/jhuapl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 68e77a8e..07eb8452 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -152,7 +152,7 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, False for xarray format, True for pandas (default=False) strict_dim_check : bool Used for xarray data (`pandas_format` is False). If True, warn the user - that the desired epoch is not present in `xarray.dims`. If False, + that the desired epoch, NAME, is not present as a dimension in the NetCDF file. If False, no warning is raised. (default=True) Returns From 7144c1dc4281d0b9fc53be5370c238cab60b157d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 2 Jun 2023 12:57:04 -0400 Subject: [PATCH 339/396] Update pysatNASA/instruments/methods/jhuapl.py --- pysatNASA/instruments/methods/jhuapl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 07eb8452..7c24aead 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -152,8 +152,8 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, False for xarray format, True for pandas (default=False) strict_dim_check : bool Used for xarray data (`pandas_format` is False). If True, warn the user - that the desired epoch, NAME, is not present as a dimension in the NetCDF file. If False, - no warning is raised. (default=True) + that the desired epoch, NAME, is not present as a dimension in the + NetCDF file. If False, no warning is raised. (default=True) Returns ------- From c66ee097aa7482b1fc48b07a51ae484674f04d17 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 2 Jun 2023 13:44:10 -0400 Subject: [PATCH 340/396] STY: update cdflib versions --- README.md | 2 +- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0d5ab95a..de310e27 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Python 3.6+. | Common modules | Community modules | Optional Modules | | ---------------- | ----------------- |------------------| -| beautifulsoup4 | cdflib | pysatCDF | +| beautifulsoup4 | cdflib>=0.4.4 | pysatCDF | | lxml | pysat>=3.1.0 | | | netCDF4 | | | | numpy | | | diff --git a/pyproject.toml b/pyproject.toml index 5f7cd9f4..646969a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ keywords = [ dependencies = [ "beautifulsoup4", "cdasws", - "cdflib >= 0.4.4, <1.0", + "cdflib >= 0.4.4", "lxml", "netCDF4", "numpy", diff --git a/requirements.txt b/requirements.txt index 898145bf..3684998a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4 cdasws -cdflib>=0.4.4,<1.0 +cdflib>=0.4.4 lxml netCDF4 numpy From 75e362bc45a2ed267b3d61a7dabd5f93eb37b467 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 2 Jun 2023 13:44:22 -0400 Subject: [PATCH 341/396] TST: cap on operational env tests --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 74b02c5f..61125636 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,6 +41,7 @@ jobs: if: ${{ matrix.test_config == 'Ops'}} run: | pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} + pip install "cdflib<1.0" pip install -r requirements.txt pip install -r test_requirements.txt pip install . From 027df456c143a4334ffed6fe36ef4c18435c54e9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 2 Jun 2023 15:33:23 -0400 Subject: [PATCH 342/396] STY: xarray support across cdflib versions --- pysatNASA/instruments/methods/cdaweb.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index b6d24d55..3e57736a 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -7,7 +7,6 @@ """ -import cdflib import datetime as dt import numpy as np import os @@ -26,6 +25,13 @@ from pysat.utils import io from pysatNASA.instruments.methods import CDF as libCDF +try: + # cdflib 1.0 syntax + from cdflib.xarray import cdf_to_xarray +except ModuleNotFoundError: + # cdflib 0.4 syntax required for backwards compatibility + from cdflib import cdf_to_xarray + try: import pysatCDF auto_CDF = pysatCDF.CDF @@ -372,7 +378,7 @@ def load_xarray(fnames, tag='', inst_id='', lfnames = fnames for lfname in lfnames: - temp_data = cdflib.cdf_to_xarray(lfname, to_datetime=True) + temp_data = cdf_to_xarray(lfname, to_datetime=True) if drop_dims: temp_data = temp_data.drop_dims(drop_dims) if var_translation: From fe13b258e72abf84fc55bc2ca2d388afcd7bfb24 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 2 Jun 2023 15:34:55 -0400 Subject: [PATCH 343/396] MAINT: cdflib 1.0+ compat --- pysatNASA/instruments/methods/_cdf.py | 40 +++++++++++++++++++-------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/pysatNASA/instruments/methods/_cdf.py b/pysatNASA/instruments/methods/_cdf.py index ca2a6155..0f152685 100644 --- a/pysatNASA/instruments/methods/_cdf.py +++ b/pysatNASA/instruments/methods/_cdf.py @@ -91,8 +91,13 @@ def __init__(self, filename, self.meta = {} self._dependencies = {} - self._variable_names = (self._cdf_info['rVariables'] - + self._cdf_info['zVariables']) + try: + self._variable_names = (self._cdf_info.rVariables + + self._cdf_info.zVariables) + except AttributeError: + # cdflib < 1.0 stores info as a dict + self._variable_names = (self._cdf_info['rVariables'] + + self._cdf_info['zVariables']) self.load_variables() @@ -156,8 +161,13 @@ def set_epoch(self, x_axis_var): """ - data_type_description = self._cdf_file.varinq( - x_axis_var)['Data_Type_Description'] + try: + data_type_description = self._cdf_file.varinq( + x_axis_var).Data_Type_Description + except AttributeError: + # cdflib < 1.0 stores this as a dict + data_type_description = self._cdf_file.varinq( + x_axis_var)['Data_Type_Description'] center_measurement = self._center_measurement cdf_file = self._cdf_file @@ -298,7 +308,12 @@ def load_variables(self): if not re.match(var_regex, variable_name): # Skip this variable continue - var_atts = self._cdf_file.varattsget(variable_name, to_np=True) + try: + var_atts = self._cdf_file.varattsget(variable_name, to_np=True) + except TypeError: + # cdflib 1.0+ drops to_np kwarg, assumes True + var_atts = self._cdf_file.varattsget(variable_name) + for k in var_atts: var_atts[k] = var_atts[k] # [0] @@ -319,13 +334,14 @@ def load_variables(self): continue if "FILLVAL" in var_atts: - if (var_properties['Data_Type_Description'] == 'CDF_FLOAT' - or var_properties['Data_Type_Description'] - == 'CDF_REAL4' - or var_properties['Data_Type_Description'] - == 'CDF_DOUBLE' - or var_properties['Data_Type_Description'] - == 'CDF_REAL8'): + try: + data_type_desc = var_properties.Data_Type_Description + except AttributeError: + # cdflib < 1.0 stores this as a dict + data_type_desc = var_properties['Data_Type_Description'] + + if data_type_desc in ['CDF_FLOAT', 'CDF_REAL4', 'CDF_DOUBLE', + 'CDF_REAL8']: if ydata[ydata == var_atts["FILLVAL"]].size != 0: ydata[ydata == var_atts["FILLVAL"]] = np.nan From 7723d4556a7577c0e9797d4635a203e8bcda4b7d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 2 Jun 2023 16:43:44 -0400 Subject: [PATCH 344/396] STY: whitespace --- pysatNASA/instruments/methods/jhuapl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 7c24aead..34b5982e 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -152,7 +152,7 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, False for xarray format, True for pandas (default=False) strict_dim_check : bool Used for xarray data (`pandas_format` is False). If True, warn the user - that the desired epoch, NAME, is not present as a dimension in the + that the desired epoch, NAME, is not present as a dimension in the NetCDF file. If False, no warning is raised. (default=True) Returns From 88c16e212d6d0d36a5f5cdf7cef6d7feac68fc08 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 2 Jun 2023 16:52:06 -0400 Subject: [PATCH 345/396] Update CHANGELOG.md Co-authored-by: Angeline Burrell --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 377dbf06..6a386c02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Use pyproject.toml to manage installation and metadata * Set use_cdflib=True for supported xarray instruments * Set pysat 3.1.0 minimum - * Use pysat logger to raise warnings + * Use pysat logger to raise non-deprecation warnings ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 027b6b826e0627902b4cc9c1e6bdf31e6361a1a1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 5 Jun 2023 11:05:44 -0400 Subject: [PATCH 346/396] Update pysatNASA/instruments/methods/jhuapl.py --- pysatNASA/instruments/methods/jhuapl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 34b5982e..6578b245 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -152,7 +152,7 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, False for xarray format, True for pandas (default=False) strict_dim_check : bool Used for xarray data (`pandas_format` is False). If True, warn the user - that the desired epoch, NAME, is not present as a dimension in the + that the desired epoch,'TIME', is not present as a dimension in the NetCDF file. If False, no warning is raised. (default=True) Returns From 8afa88c62d3d3139d4aea8db60376708d96ae1ef Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 5 Jun 2023 12:08:53 -0400 Subject: [PATCH 347/396] Apply suggestions from code review --- pysatNASA/instruments/methods/jhuapl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 6578b245..26700c03 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -152,7 +152,7 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, False for xarray format, True for pandas (default=False) strict_dim_check : bool Used for xarray data (`pandas_format` is False). If True, warn the user - that the desired epoch,'TIME', is not present as a dimension in the + that the desired epoch, 'TIME', is not present as a dimension in the NetCDF file. If False, no warning is raised. (default=True) Returns @@ -240,8 +240,8 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, False for xarray format, True for pandas (default=False) strict_dim_check : bool Used for xarray data (`pandas_format` is False). If True, warn the user - that the desired epoch is not present in `xarray.dims`. If False, - no warning is raised. (default=True) + that the desired epoch, 'TIME_DAY', is not present as a dimension in the + NetCDF file. If False, no warning is raised. (default=True)``` combine_times : bool For SDR data, optionally combine the different datetime coordinates into a single time coordinate (default=False) From 5a75090f6bc58ace421978bbc88cb9a2ff926462 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Mon, 5 Jun 2023 12:26:24 -0400 Subject: [PATCH 348/396] MAINT: use meta_kwargs in load_netdf --- pysatNASA/instruments/icon_euv.py | 2 +- pysatNASA/instruments/icon_fuv.py | 2 +- pysatNASA/instruments/icon_ivm.py | 2 +- pysatNASA/instruments/icon_mighti.py | 2 +- pysatNASA/instruments/methods/jhuapl.py | 6 ++++-- pysatNASA/instruments/ses14_gold.py | 3 ++- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pysatNASA/instruments/icon_euv.py b/pysatNASA/instruments/icon_euv.py index 258db2ab..76b2809b 100644 --- a/pysatNASA/instruments/icon_euv.py +++ b/pysatNASA/instruments/icon_euv.py @@ -220,7 +220,7 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): data, meta = pysat.utils.io.load_netcdf(fnames, epoch_name='Epoch', pandas_format=pandas_format, - labels=labels, + meta_kwargs={'labels': labels}, meta_processor=filter_metadata, meta_translation=meta_translation, drop_meta_labels=['Valid_Max', diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py index 49bfc652..c13935dd 100644 --- a/pysatNASA/instruments/icon_fuv.py +++ b/pysatNASA/instruments/icon_fuv.py @@ -221,7 +221,7 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): data, meta = pysat.utils.io.load_netcdf(fnames, epoch_name='Epoch', pandas_format=pandas_format, - labels=labels, + meta_kwargs={'labels': labels}, meta_processor=filter_metadata, meta_translation=meta_translation, drop_meta_labels=drop_labels) diff --git a/pysatNASA/instruments/icon_ivm.py b/pysatNASA/instruments/icon_ivm.py index 034b63e0..ff6012d0 100644 --- a/pysatNASA/instruments/icon_ivm.py +++ b/pysatNASA/instruments/icon_ivm.py @@ -281,7 +281,7 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): '_FillValue': 'FillVal'} data, meta = pysat.utils.io.load_netcdf(fnames, epoch_name='Epoch', - labels=labels, + meta_kwargs={'labels': labels}, meta_processor=filter_metadata, meta_translation=meta_translation, drop_meta_labels=['Valid_Max', diff --git a/pysatNASA/instruments/icon_mighti.py b/pysatNASA/instruments/icon_mighti.py index b195fcfd..8dbce265 100644 --- a/pysatNASA/instruments/icon_mighti.py +++ b/pysatNASA/instruments/icon_mighti.py @@ -329,7 +329,7 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): data, meta = pysat.utils.io.load_netcdf(fnames, epoch_name='Epoch', pandas_format=pandas_format, - labels=labels, + meta_kwargs={'labels': labels}, meta_processor=filter_metadata, meta_translation=meta_translation, drop_meta_labels=['Valid_Max', diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index e11971ea..afccedcd 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -180,7 +180,8 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False): # than a dimension or coordinate. Additionally, no coordinates # are assigned. sdata, mdata = load_netcdf(fname, epoch_name='TIME', epoch_unit='s', - labels=labels, pandas_format=pandas_format) + meta_kwargs={'labels': labels}, + pandas_format=pandas_format) # Calculate the time for this data file. The pysat `load_netcdf` routine # converts the 'TIME' parameter (seconds of day) into datetime using @@ -300,7 +301,8 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, # than a dimension or coordinate. Additionally, no coordinates # are assigned. sdata, mdata = load_netcdf(fname, epoch_name=load_time, epoch_unit='s', - labels=labels, pandas_format=pandas_format) + meta_kwargs={'labels': labels}, + pandas_format=pandas_format) # Calculate the time for this data file. The pysat `load_netcdf` routine # converts the 'TIME' parameter (seconds of day) into datetime using diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 9035a1b1..13be59b1 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -244,7 +244,8 @@ def load(fnames, tag='', inst_id=''): epoch_name = 'nevents' data, meta = load_netcdf(fnames, pandas_format=pandas_format, - epoch_name=epoch_name, labels=labels, + epoch_name=epoch_name, + meta_kwargs={'labels': labels}, meta_translation=meta_translation, combine_by_coords=False, drop_meta_labels='FILLVAL') From 8e9a4fb388e30ae86435c93343153c31d5a956a7 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 5 Jun 2023 12:29:34 -0400 Subject: [PATCH 349/396] Update jhuapl.py --- pysatNASA/instruments/methods/jhuapl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 26700c03..6dae78c1 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -310,7 +310,8 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, # than a dimension or coordinate. Additionally, no coordinates # are assigned. sdata, mdata = load_netcdf(fname, epoch_name=load_time, epoch_unit='s', - labels=labels, pandas_format=pandas_format) + labels=labels, pandas_format=pandas_format, + strict_dim_check=strict_dim_check) # Calculate the time for this data file. The pysat `load_netcdf` routine # converts the 'TIME' parameter (seconds of day) into datetime using From 2374a4bdfe820a69e2dbdf24cdeacd75047c6c83 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Tue, 6 Jun 2023 07:42:54 -0400 Subject: [PATCH 350/396] STY: float as primary --- pysatNASA/instruments/methods/cdaweb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 1f050123..c8bcb1b5 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -270,9 +270,9 @@ def load_xarray(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), labels={'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', (int, float)), - 'max_val': ('ValidMax', (int, float)), - 'fill_val': ('FillVal', (int, float))}, + 'min_val': ('ValidMin', (float, int)), + 'max_val': ('ValidMax', (float, int)), + 'fill_val': ('FillVal', (float, int))}, epoch_name='Epoch', drop_dims=None, var_translation=None, meta_processor=None, meta_translation=None, drop_meta_labels=None): From 9253a1d7c4a307b8d61aa4e16f31e5f378769161 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Tue, 6 Jun 2023 07:43:05 -0400 Subject: [PATCH 351/396] STY: warning, not warn --- pysatNASA/instruments/methods/general.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index 2a48ff1a..742a031b 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -54,7 +54,7 @@ def clean_warn(self): 'none' No cleaning applied, routine not called in this case. """ - pysat.logger.warn(' '.join(('No cleaning routines available for', - self.platform, self.name))) + pysat.logger.warning(' '.join(('No cleaning routines available for', + self.platform, self.name))) return From 7f140b18b924783587272327286d4819db6e803d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 6 Jun 2023 09:15:34 -0400 Subject: [PATCH 352/396] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb135982..5e711814 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Set use_cdflib=True for supported xarray instruments * Set pysat 3.1.0 minimum * Use pysat logger to raise non-deprecation warnings + * Update syntax based on latest pysat deprecations to make the code compatible with pysat 3.2.0. ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 78ad4340d8e3737c363a8de72b728b2c61094d38 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 6 Jun 2023 09:29:16 -0400 Subject: [PATCH 353/396] BUG: fix bad merge --- pysatNASA/instruments/methods/jhuapl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 28d73287..30d1d3eb 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -112,7 +112,6 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, sdata, mdata = load_netcdf(fname, epoch_name='TIME', epoch_unit='s', meta_kwargs={'labels': labels}, pandas_format=pandas_format, - labels=labels, pandas_format=pandas_format, strict_dim_check=strict_dim_check) # Calculate the time for this data file. The pysat `load_netcdf` routine @@ -239,7 +238,6 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, sdata, mdata = load_netcdf(fname, epoch_name=load_time, epoch_unit='s', meta_kwargs={'labels': labels}, pandas_format=pandas_format, - labels=labels, pandas_format=pandas_format, strict_dim_check=strict_dim_check) # Calculate the time for this data file. The pysat `load_netcdf` routine From 90c653ac59ba4de1816eebcca6c0c8334bc309e3 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 6 Jun 2023 12:56:16 -0400 Subject: [PATCH 354/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb135982..eb6bcb01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Set use_cdflib=True for supported xarray instruments * Set pysat 3.1.0 minimum * Use pysat logger to raise non-deprecation warnings + * Updated syntax compliance with cdflib 1.0+ ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 6572322a1dc93b830d428313ec7d995b1f3de1c2 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 6 Jun 2023 12:59:27 -0400 Subject: [PATCH 355/396] DOC: update docs --- docs/installation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 23ef4123..7b5858b7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -14,7 +14,7 @@ Prerequisites pysatNASA uses common Python modules, as well as modules developed by and for the Space Physics community. This module officially supports -Python 3.8+ and pysat 3.1.0+. +Python 3.6+ and pysat 3.1.0+. ================== ================= Common modules Community modules @@ -25,7 +25,7 @@ Python 3.8+ and pysat 3.1.0+. numpy pandas requests - xarray<2022.11 + xarray ================== ================= From 5cbd14fa7526a3b43d575a544e2357916e43f67f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 6 Jun 2023 15:44:11 -0400 Subject: [PATCH 356/396] Update pysatNASA/instruments/methods/_cdf.py Co-authored-by: Angeline Burrell --- pysatNASA/instruments/methods/_cdf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/_cdf.py b/pysatNASA/instruments/methods/_cdf.py index 0f152685..32a29861 100644 --- a/pysatNASA/instruments/methods/_cdf.py +++ b/pysatNASA/instruments/methods/_cdf.py @@ -91,10 +91,10 @@ def __init__(self, filename, self.meta = {} self._dependencies = {} - try: + if hasattr(self._cdf_info, 'rVariables') and hasattr(self._cdf_info, 'zVariables'): self._variable_names = (self._cdf_info.rVariables + self._cdf_info.zVariables) - except AttributeError: + else: # cdflib < 1.0 stores info as a dict self._variable_names = (self._cdf_info['rVariables'] + self._cdf_info['zVariables']) From 647cdb441d39041e930a71fc8010668ed72747ff Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 6 Jun 2023 15:54:09 -0400 Subject: [PATCH 357/396] STY: there is no try, only if --- pysatNASA/instruments/methods/_cdf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pysatNASA/instruments/methods/_cdf.py b/pysatNASA/instruments/methods/_cdf.py index 32a29861..98713df1 100644 --- a/pysatNASA/instruments/methods/_cdf.py +++ b/pysatNASA/instruments/methods/_cdf.py @@ -91,7 +91,7 @@ def __init__(self, filename, self.meta = {} self._dependencies = {} - if hasattr(self._cdf_info, 'rVariables') and hasattr(self._cdf_info, 'zVariables'): + if hasattr(self._cdf_info, 'rVariables'): self._variable_names = (self._cdf_info.rVariables + self._cdf_info.zVariables) else: @@ -161,10 +161,10 @@ def set_epoch(self, x_axis_var): """ - try: + if hasattr(self._cdf_file.varinq(x_axis_var), 'Data_Type_Description'): data_type_description = self._cdf_file.varinq( x_axis_var).Data_Type_Description - except AttributeError: + else: # cdflib < 1.0 stores this as a dict data_type_description = self._cdf_file.varinq( x_axis_var)['Data_Type_Description'] @@ -334,9 +334,9 @@ def load_variables(self): continue if "FILLVAL" in var_atts: - try: + if hasattr(var_properties, 'Data_Type_Description'): data_type_desc = var_properties.Data_Type_Description - except AttributeError: + else: # cdflib < 1.0 stores this as a dict data_type_desc = var_properties['Data_Type_Description'] From 73564823f87c4595e69c807a2d4fcf34361155dd Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:18:01 -0400 Subject: [PATCH 358/396] updated download functions to take data_path as an arg --- pysatNASA/instruments/methods/cdaweb.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 6bcda5e5..e0120cee 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -470,8 +470,8 @@ def load_xarray(fnames, tag='', inst_id='', # TODO(#103): Include support to unzip / untar files after download. -def download(date_array, tag='', inst_id='', supported_tags=None, - remote_url='https://cdaweb.gsfc.nasa.gov', data_path=None): +def download(date_array, data_path, tag='', inst_id='', supported_tags=None, + remote_url='https://cdaweb.gsfc.nasa.gov'): """Download NASA CDAWeb data. This routine is intended to be used by pysat instrument modules supporting @@ -481,6 +481,8 @@ def download(date_array, tag='', inst_id='', supported_tags=None, ---------- date_array : array-like Array of datetimes to download data for. Provided by pysat. + data_path : str + Path to data directory. tag : str Data product tag (default='') inst_id : str @@ -493,9 +495,6 @@ def download(date_array, tag='', inst_id='', supported_tags=None, remote_url : str Remote site to download data from (default='https://cdaweb.gsfc.nasa.gov') - data_path : str or NoneType - Path to data directory. If None is specified, the value previously - set in Instrument.files.data_path is used. (default=None) Examples -------- @@ -570,8 +569,8 @@ def download(date_array, tag='', inst_id='', supported_tags=None, return -def cdas_download(date_array, tag='', inst_id='', supported_tags=None, - data_path=None): +def cdas_download(date_array, data_path, tag='', inst_id='', + supported_tags=None): """Download NASA CDAWeb CDF data using cdasws. This routine is intended to be used by pysat instrument modules supporting @@ -581,6 +580,8 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, ---------- date_array : array-like Array of datetimes to download data for. Provided by pysat. + data_path : str + Path to data directory. tag : str Data product tag (default='') inst_id : str @@ -590,9 +591,6 @@ def cdas_download(date_array, tag='', inst_id='', supported_tags=None, a dict with 'remote_dir', 'fname'. Inteded to be pre-set with functools.partial then assigned to new instrument code. (default=None) - data_path : str or NoneType - Path to data directory. If None is specified, the value previously - set in Instrument.files.data_path is used. (default=None) Note ---- From 5de67df96aa2b483ccd47e72944e92d874f3d5ec Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:18:31 -0400 Subject: [PATCH 359/396] updated tests for data_path arg --- pysatNASA/tests/test_methods_cdaweb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index f601517d..dd7870e6 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -19,6 +19,7 @@ def setup_method(self): self.download_tags = pysatNASA.instruments.timed_guvi.download_tags self.kwargs = {'tag': None, 'inst_id': None} + self.saved_path = pysat.params['data_dirs'] return def teardown_method(self): @@ -61,6 +62,7 @@ def test_bad_kwarg_download(self, bad_key, bad_val, err_msg): with pytest.raises(ValueError) as excinfo: cdw.download(supported_tags=self.download_tags, date_array=date_array, + data_path=self.saved_path, tag=self.kwargs['tag'], inst_id=self.kwargs['inst_id']) assert str(excinfo.value).find(err_msg) >= 0 From 16b82890fff3f57ea4160f7447860da101e70fd4 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:18:42 -0400 Subject: [PATCH 360/396] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb6bcb01..bc7873fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Deprecations * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps * Maintenance + * Updated download functions to take data_path as an arg, not a kwarg * Removed duplicate tests if pysatCDF not isntalled * Removed pysatCDF tests on GitHub Actions workflows (see #167) * Updated actions and templates based on pysatEcosystem docs From d53368f27b709e17011aa1a6c85447cfd9e1aca1 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:40:32 -0400 Subject: [PATCH 361/396] updated to_pysat method to attach header level meta --- pysatNASA/instruments/methods/_cdf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/_cdf.py b/pysatNASA/instruments/methods/_cdf.py index 98713df1..f6c93cb4 100644 --- a/pysatNASA/instruments/methods/_cdf.py +++ b/pysatNASA/instruments/methods/_cdf.py @@ -87,6 +87,7 @@ def __init__(self, filename, self._cdf_file = cdflib.CDF(self._filename) self._cdf_info = self._cdf_file.cdf_info() + self.global_attrs = self._cdf_file.globalattsget() self.data = {} self.meta = {} self._dependencies = {} @@ -423,7 +424,7 @@ def to_pysat(self, flatten_twod=True, # and utilizing the attribute labels provided by the user meta = pysat.Meta(pds.DataFrame.from_dict(self.meta, orient='index'), labels=labels) - + meta.header = pysat.MetaHeader(header_data=self.global_attrs) cdata = self.data.copy() lower_names = [name.lower() for name in meta.keys()] for name, true_name in zip(lower_names, meta.keys()): From 74c413669ea6b3fa630435641f70773fd64853d5 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:41:03 -0400 Subject: [PATCH 362/396] added test to ensure attachment of header level meta --- pysatNASA/tests/test_instruments.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 3084724e..9a41a720 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -95,6 +95,22 @@ def test_load_cdflib(self, inst_dict): pytest.skip("Download data not available.") return + @pytest.mark.second + @pytest.mark.parametrize("inst_dict", instruments['cdf']) + @pytest.mark.skipif(cdflib_only, + reason=" ".join(("Additional load tests not required", + "when pysatCDF not installed"))) + def test_meta_header(self, inst_dict): + """Test that instruments have header level metadata attached + + Parameters + ---------- + inst_dict : dict + Dictionary containing info to instnatiate a specific instrument. + """ + test_inst, date = clslib.initialize_test_inst_and_date(inst_dict) + test_inst.load(date=date, use_header=True, use_cdflib=True) + assert test_inst.meta.to_dict() != {} class TestDeprecation(object): From d2219aaafa58f1424eddfd094896e38e028f0969 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:41:21 -0400 Subject: [PATCH 363/396] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc7873fc..85ef06b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). low, high * Added CDAWeb methods that can use cdasws to get the remote file list * Bug Fixes + * Pandas datasets made with cdflib now have header level meta * Updated CDAWeb routines to allow for data stored by year/day-of-year * Updated GOLD nmax to sort scans by time. * Added 1 usec to GOLD nmax channel B times to ensure uniqueness From c6a2daf3bd6fa71d00932fb07702c1096358cdd7 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:54:31 -0400 Subject: [PATCH 364/396] style fixes --- pysatNASA/tests/test_instruments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 9a41a720..bce5dcca 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -95,13 +95,14 @@ def test_load_cdflib(self, inst_dict): pytest.skip("Download data not available.") return + @pytest.mark.second @pytest.mark.parametrize("inst_dict", instruments['cdf']) @pytest.mark.skipif(cdflib_only, reason=" ".join(("Additional load tests not required", "when pysatCDF not installed"))) def test_meta_header(self, inst_dict): - """Test that instruments have header level metadata attached + """Test that instruments have header level metadata attached. Parameters ---------- From bfb6492260a2e697a3db93a8bbbe5bdc2936302b Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 12 Jun 2023 11:25:53 -0400 Subject: [PATCH 365/396] BUG: use custom load for ACE --- pysatNASA/instruments/ace_epam_l2.py | 7 +--- pysatNASA/instruments/ace_mag_l2.py | 6 +-- pysatNASA/instruments/ace_sis_l2.py | 6 +-- pysatNASA/instruments/ace_swepam_l2.py | 8 +--- pysatNASA/instruments/methods/ace.py | 51 ++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 22 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index 4ef14b23..c3e73b01 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -51,7 +51,6 @@ inst_ids = {'12sec': ['base'], '5min': ['key', 'base'], '1hr': ['key', 'base']} -pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -88,11 +87,7 @@ supported_tags=supported_tags) # Set the load routine -meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', - 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', - 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} -load = functools.partial(cdw.load, pandas_format=pandas_format, - meta_translation=meta_translation, use_cdflib=True) +load = functools.partial(mm_ace.load, to_pandas=True) # Set the download routine download_tags = {'12sec': {'base': 'AC_H3_EPM'}, diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index bff8ad2a..844b5a6a 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -90,11 +90,7 @@ supported_tags=supported_tags) # Set the load routine -meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', - 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', - 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} -load = functools.partial(cdw.load, pandas_format=pandas_format, - meta_translation=meta_translation, use_cdflib=True) +load = functools.partial(mm_ace.load, to_pandas=False) # Set the download routine download_tags = {'1sec': {'base': 'AC_H3_MFI'}, diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index cf55bed9..a6a46a9c 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -83,11 +83,7 @@ supported_tags=supported_tags) # Set the load routine -meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', - 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', - 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} -load = functools.partial(cdw.load, pandas_format=pandas_format, - meta_translation=meta_translation, use_cdflib=True) +load = functools.partial(mm_ace.load, to_pandas=False) # Set the download routine download_tags = {'256sec': {'base': 'AC_H1_SIS'}, diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index 2d85e524..e78dc9fe 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -51,7 +51,6 @@ inst_ids = {'64sec': ['base'], '5min': ['key'], '1hr': ['key', 'base']} -pandas_format = False # ---------------------------------------------------------------------------- # Instrument test attributes @@ -85,12 +84,9 @@ list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) + # Set the load routine -meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', - 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', - 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} -load = functools.partial(cdw.load, pandas_format=pandas_format, - meta_translation=meta_translation, use_cdflib=True) +load = functools.partial(mm_ace.load, to_pandas=True) # Set the download routine download_tags = {'64sec': {'base': 'AC_H0_SWE'}, diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index 69e70e7e..ff6a9584 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -3,6 +3,8 @@ import numpy as np +from pysatNASA.instruments.methods import cdaweb as cdw + ackn_str = ' '.join(("Please acknowledge the NASA National Space Science Data", "Center, the Space Physics Data Facility, and the ACE", "Principal Investigator, Edward C. Stone of the", @@ -34,6 +36,55 @@ } +def load(fnames, tag='', inst_id='', to_pandas=False): + """Load data via xarray and convert to pandas if needed. + + This routine is called as needed by pysat. It is not intended + for direct user interaction. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + tag : str + Tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + to_pandas : bool + If True, convert to pandas. If False, leave as xarray. (default=False) + + Returns + ------- + data : pds.DataFrame or xr.Dataset + A pandas DataFrame or xarray Dataset with data prepared for the + `pysat.Instrument`. + meta : pysat.Meta + Metadata formatted for a pysat.Instrument object. + + Note + ---- + Several variables relating to time stored in different formats are dropped. + These are redundant and complicate the load procedure. + + """ + + meta_translation = {'CATDESC': 'desc', 'FILLVAL': 'fill', + 'LABLAXIS': 'plot_label', 'VALIDMAX': 'value_max', + 'VALIDMIN': 'value_min', 'VAR_NOTES': 'notes'} + data, meta = cdw.load(fnames, tag=tag, inst_id=inst_id, pandas_format=False, + meta_translation=meta_translation, + drop_dims=['dim_empty', 'dim0', 'unit_time'], + use_cdflib=True) + + if to_pandas: + data = data.to_pandas() + + return data, meta + + def clean(self): """Clean ACE data to the specified level. From d221e965b5836790f6e5280295baed53320caad4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 12 Jun 2023 11:29:59 -0400 Subject: [PATCH 366/396] DOC: update default styles --- pysatNASA/instruments/methods/cdaweb.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 9f4e9032..76f87611 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -300,11 +300,11 @@ def load_xarray(fnames, tag='', inst_id='', labels : dict Dict where keys are the label attribute names and the values are tuples that have the label values and value types in that order. - (default={'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', np.float64), - 'max_val': ('value_max', np.float64), - 'fill_val': ('fill', np.float64)}) + (default={'units': ('Units', str), 'name': ('Long_Name', str), + 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), + 'min_val': ('ValidMin', (float, int)), + 'max_val': ('ValidMax', (float, int)), + 'fill_val': ('FillVal', (float, int))}) epoch_name : str Data key for epoch variable. The epoch variable is expected to be an array of integer or float values denoting time elapsed from an origin From 3e8cef81f365ea344a7e9a21469382284da1a27f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 12 Jun 2023 13:26:36 -0400 Subject: [PATCH 367/396] ENH: add str as meta type --- pysatNASA/instruments/methods/_cdf.py | 12 ++++++------ pysatNASA/instruments/methods/cdaweb.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pysatNASA/instruments/methods/_cdf.py b/pysatNASA/instruments/methods/_cdf.py index 98713df1..4cf78192 100644 --- a/pysatNASA/instruments/methods/_cdf.py +++ b/pysatNASA/instruments/methods/_cdf.py @@ -376,9 +376,9 @@ def load_variables(self): def to_pysat(self, flatten_twod=True, labels={'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', (int, float)), - 'max_val': ('ValidMax', (int, float)), - 'fill_val': ('FillVal', (int, float))}): + 'min_val': ('ValidMin', (float, int, str)), + 'max_val': ('ValidMax', (float, int, str)), + 'fill_val': ('FillVal', (float, int, str))}): """Export loaded CDF data into data, meta for pysat module. Parameters @@ -398,9 +398,9 @@ def to_pysat(self, flatten_twod=True, that order. (default={'units': ('units', str), 'name': ('long_name', str), 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', (int, float)), - 'max_val': ('value_max', (int, float)) - 'fill_val': ('fill', (int, float))}) + 'min_val': ('value_min', (float, int, str)), + 'max_val': ('value_max', (float, int, str)) + 'fill_val': ('fill', (float, int, str))}) Returns ------- diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 76f87611..235090eb 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -276,9 +276,9 @@ def load_xarray(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), labels={'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', (float, int)), - 'max_val': ('ValidMax', (float, int)), - 'fill_val': ('FillVal', (float, int))}, + 'min_val': ('ValidMin', (float, int, str)), + 'max_val': ('ValidMax', (float, int, str)), + 'fill_val': ('FillVal', (float, int, str))}, epoch_name='Epoch', drop_dims=None, var_translation=None, meta_processor=None, meta_translation=None, drop_meta_labels=None): @@ -302,9 +302,9 @@ def load_xarray(fnames, tag='', inst_id='', that have the label values and value types in that order. (default={'units': ('Units', str), 'name': ('Long_Name', str), 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), - 'min_val': ('ValidMin', (float, int)), - 'max_val': ('ValidMax', (float, int)), - 'fill_val': ('FillVal', (float, int))}) + 'min_val': ('ValidMin', (float, int, str)), + 'max_val': ('ValidMax', (float, int, str)), + 'fill_val': ('FillVal', (float, int, str))}) epoch_name : str Data key for epoch variable. The epoch variable is expected to be an array of integer or float values denoting time elapsed from an origin From 4eef68099729f5a6cd219f4b9eae5b62bdabae30 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 12 Jun 2023 13:54:53 -0400 Subject: [PATCH 368/396] BUG: don't clean the coords --- pysatNASA/instruments/methods/ace.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index ff6a9584..cbc97afd 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -94,8 +94,15 @@ def clean(self): """ + # Get a list of coords for the data + if self.pandas_format: + coords = [self.data.index.name] + else: + coords = [key for key in self.data.coords] + for key in self.variables: - if key != 'time': + # Skip over the coordinates when cleaning + if key not in coords: fill = self.meta[key, self.meta.labels.fill_val] # Replace fill with nan From 4f23ea0bd55a2d6800a6d7c4dd87022c2280446a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 12 Jun 2023 14:40:04 -0400 Subject: [PATCH 369/396] ENH: move ace clean to general --- pysatNASA/instruments/ace_epam_l2.py | 4 ++-- pysatNASA/instruments/ace_mag_l2.py | 4 ++-- pysatNASA/instruments/ace_sis_l2.py | 4 ++-- pysatNASA/instruments/ace_swepam_l2.py | 4 ++-- pysatNASA/instruments/methods/ace.py | 29 ------------------------ pysatNASA/instruments/methods/general.py | 29 ++++++++++++++++++++++++ 6 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pysatNASA/instruments/ace_epam_l2.py b/pysatNASA/instruments/ace_epam_l2.py index c3e73b01..61dddf8e 100644 --- a/pysatNASA/instruments/ace_epam_l2.py +++ b/pysatNASA/instruments/ace_epam_l2.py @@ -65,8 +65,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) -# Use default ace clean -clean = mm_ace.clean +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index 844b5a6a..75c2b61e 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -67,8 +67,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) -# Use default ace clean -clean = mm_ace.clean +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/ace_sis_l2.py b/pysatNASA/instruments/ace_sis_l2.py index a6a46a9c..188f5973 100644 --- a/pysatNASA/instruments/ace_sis_l2.py +++ b/pysatNASA/instruments/ace_sis_l2.py @@ -63,8 +63,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) -# Use default ace clean -clean = mm_ace.clean +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/ace_swepam_l2.py b/pysatNASA/instruments/ace_swepam_l2.py index e78dc9fe..b16b37ed 100644 --- a/pysatNASA/instruments/ace_swepam_l2.py +++ b/pysatNASA/instruments/ace_swepam_l2.py @@ -64,8 +64,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) -# Use default ace clean -clean = mm_ace.clean +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index cbc97afd..fa99e515 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- """Provides non-instrument specific routines for ACE data.""" -import numpy as np - from pysatNASA.instruments.methods import cdaweb as cdw ackn_str = ' '.join(("Please acknowledge the NASA National Space Science Data", @@ -83,30 +81,3 @@ def load(fnames, tag='', inst_id='', to_pandas=False): data = data.to_pandas() return data, meta - - -def clean(self): - """Clean ACE data to the specified level. - - Note - ---- - Basic cleaning to replace fill values with NaN - - """ - - # Get a list of coords for the data - if self.pandas_format: - coords = [self.data.index.name] - else: - coords = [key for key in self.data.coords] - - for key in self.variables: - # Skip over the coordinates when cleaning - if key not in coords: - fill = self.meta[key, self.meta.labels.fill_val] - - # Replace fill with nan - fill_mask = self[key] == fill - self[key] = self.data[key].where(~fill_mask) - self.meta[key] = {self.meta.labels.fill_val: np.nan} - return diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index 742a031b..2585b250 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -1,5 +1,7 @@ """General methods for NASA instruments.""" +import numpy as np + import pysat @@ -43,6 +45,33 @@ def init(self, module, name): return +def clean(self): + """Clean data to the specified level. + + Note + ---- + Basic cleaning to replace fill values with NaN + + """ + + # Get a list of coords for the data + if self.pandas_format: + coords = [self.data.index.name] + else: + coords = [key for key in self.data.coords] + + for key in self.variables: + # Skip over the coordinates when cleaning + if key not in coords: + fill = self.meta[key, self.meta.labels.fill_val] + + # Replace fill with nan + fill_mask = self[key] == fill + self[key] = self.data[key].where(~fill_mask) + self.meta[key] = {self.meta.labels.fill_val: np.nan} + return + + def clean_warn(self): """Warn user that cleaning not yet available for this data set. From f6d2b06aae34e1f17c966d8f946f77a4fb23727b Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 12 Jun 2023 14:42:27 -0400 Subject: [PATCH 370/396] STY: use clean for historical data --- pysatNASA/instruments/cnofs_plp.py | 18 ++---------------- pysatNASA/instruments/de2_fpi.py | 4 ++-- pysatNASA/instruments/de2_lang.py | 4 ++-- pysatNASA/instruments/de2_nacs.py | 4 ++-- pysatNASA/instruments/de2_rpa.py | 4 ++-- pysatNASA/instruments/de2_vefi.py | 4 ++-- pysatNASA/instruments/de2_wats.py | 4 ++-- pysatNASA/instruments/formosat1_ivm.py | 4 ++-- pysatNASA/instruments/igs_gps.py | 4 ++-- pysatNASA/instruments/iss_fpmu.py | 23 +++-------------------- pysatNASA/instruments/jpl_gps.py | 4 ++-- 11 files changed, 23 insertions(+), 54 deletions(-) diff --git a/pysatNASA/instruments/cnofs_plp.py b/pysatNASA/instruments/cnofs_plp.py index 71eb1064..1b5b546e 100644 --- a/pysatNASA/instruments/cnofs_plp.py +++ b/pysatNASA/instruments/cnofs_plp.py @@ -60,7 +60,6 @@ import datetime as dt import functools -import numpy as np from pysat.instruments.methods import general as mm_gen @@ -88,21 +87,8 @@ init = functools.partial(mm_nasa.init, module=mm_cnofs, name=name) -def clean(self): - """Clean C/NOFS PLP data to the specified level. - - Note - ---- - Basic cleaning to find valid Epoch values - - """ - - for key in self.data.columns: - if key != 'Epoch': - fill = self.meta[key, self.meta.labels.fill_val] - idx, = np.where(self[key] == fill) - self[idx, key] = np.nan - return +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- diff --git a/pysatNASA/instruments/de2_fpi.py b/pysatNASA/instruments/de2_fpi.py index ad5e064d..4f0d210e 100644 --- a/pysatNASA/instruments/de2_fpi.py +++ b/pysatNASA/instruments/de2_fpi.py @@ -81,8 +81,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/de2_lang.py b/pysatNASA/instruments/de2_lang.py index 5afeb803..01a77c56 100644 --- a/pysatNASA/instruments/de2_lang.py +++ b/pysatNASA/instruments/de2_lang.py @@ -79,8 +79,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/de2_nacs.py b/pysatNASA/instruments/de2_nacs.py index e6b9fa3a..dd6b7d16 100644 --- a/pysatNASA/instruments/de2_nacs.py +++ b/pysatNASA/instruments/de2_nacs.py @@ -104,8 +104,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index c9161118..e06b67b3 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -86,8 +86,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index 45bec0ad..9e4140e6 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -70,8 +70,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/de2_wats.py b/pysatNASA/instruments/de2_wats.py index cdcd7ef2..e3b52bb7 100644 --- a/pysatNASA/instruments/de2_wats.py +++ b/pysatNASA/instruments/de2_wats.py @@ -101,8 +101,8 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_de2, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/formosat1_ivm.py b/pysatNASA/instruments/formosat1_ivm.py index 6eb49a92..3ac177d9 100644 --- a/pysatNASA/instruments/formosat1_ivm.py +++ b/pysatNASA/instruments/formosat1_ivm.py @@ -74,8 +74,8 @@ def init(self): return -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/igs_gps.py b/pysatNASA/instruments/igs_gps.py index 5c64c03f..99cd4690 100644 --- a/pysatNASA/instruments/igs_gps.py +++ b/pysatNASA/instruments/igs_gps.py @@ -99,8 +99,8 @@ init = functools.partial(mm_nasa.init, module=mm_igs, name=name) -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions diff --git a/pysatNASA/instruments/iss_fpmu.py b/pysatNASA/instruments/iss_fpmu.py index 3fc44673..c98f907c 100644 --- a/pysatNASA/instruments/iss_fpmu.py +++ b/pysatNASA/instruments/iss_fpmu.py @@ -26,12 +26,12 @@ import datetime as dt import functools -import numpy as np from pysat.instruments.methods import general as mm_gen from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- # Instrument attributes @@ -79,25 +79,8 @@ def init(self): return -def clean(self): - """Clean ISS FPMU data to the specified level. - - Note - ---- - 'clean' - Replace Te and Ni fill values with NaN - 'dusty' - Same as clean - 'dirty' - Same as clean - 'none' - Not applied, default fill values are preserved - - """ - - # Replace Te data fill - self.data.replace(-999., np.nan, inplace=True) - - # Replace Ne data fill - self.data.replace(-9.9999998e+30, np.nan, inplace=True) - - return +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- diff --git a/pysatNASA/instruments/jpl_gps.py b/pysatNASA/instruments/jpl_gps.py index 255effa6..040ceac0 100644 --- a/pysatNASA/instruments/jpl_gps.py +++ b/pysatNASA/instruments/jpl_gps.py @@ -98,8 +98,8 @@ def init(self): return -# No cleaning, use standard warning function instead -clean = mm_nasa.clean_warn +# Use default clean +clean = mm_nasa.clean # ---------------------------------------------------------------------------- # Instrument functions From 7cee83aadcd09e764be728c284fec25de8062843 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 12 Jun 2023 14:43:20 -0400 Subject: [PATCH 371/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index beeebb41..7ce23506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added `var_translation` kwarg to `load_xarray` interface so that variables can be renamed before attempting to merge. * Improved usage of cdflib for users in xarray instruments + * Added a generalized `clean` routine to replace fill vals with NaNs * Deprecations * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps * Maintenance From e388da7af89ca799c459ed45be46f1b1d85c1dca Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 12 Jun 2023 14:46:54 -0400 Subject: [PATCH 372/396] BUG: xarray syntax --- pysatNASA/instruments/methods/ace.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index cbc97afd..d5702e0c 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -80,7 +80,11 @@ def load(fnames, tag='', inst_id='', to_pandas=False): use_cdflib=True) if to_pandas: - data = data.to_pandas() + try: + data = data.to_pandas() + except AttributeError: + # xarray 0.16 support required for operational server + data = data.to_dataframe() return data, meta From 231bf711208740bcfe597b324c26f5af8f1870b8 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 13 Jun 2023 10:12:48 -0400 Subject: [PATCH 373/396] MAINT: decode times --- pysatNASA/instruments/icon_euv.py | 3 ++- pysatNASA/instruments/icon_fuv.py | 3 ++- pysatNASA/instruments/icon_mighti.py | 3 ++- pysatNASA/instruments/methods/jhuapl.py | 2 ++ pysatNASA/instruments/ses14_gold.py | 3 ++- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/icon_euv.py b/pysatNASA/instruments/icon_euv.py index 76b2809b..f3abcf58 100644 --- a/pysatNASA/instruments/icon_euv.py +++ b/pysatNASA/instruments/icon_euv.py @@ -225,7 +225,8 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): meta_translation=meta_translation, drop_meta_labels=['Valid_Max', 'Valid_Min', - '_FillValue']) + '_FillValue'], + decode_times=False) # xarray can't merge if variable and dim names are the same if 'Altitude' in data.dims: diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py index c13935dd..7749f006 100644 --- a/pysatNASA/instruments/icon_fuv.py +++ b/pysatNASA/instruments/icon_fuv.py @@ -224,5 +224,6 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): meta_kwargs={'labels': labels}, meta_processor=filter_metadata, meta_translation=meta_translation, - drop_meta_labels=drop_labels) + drop_meta_labels=drop_labels, + decode_times=False) return data, meta diff --git a/pysatNASA/instruments/icon_mighti.py b/pysatNASA/instruments/icon_mighti.py index 8dbce265..a108e8d2 100644 --- a/pysatNASA/instruments/icon_mighti.py +++ b/pysatNASA/instruments/icon_mighti.py @@ -336,7 +336,8 @@ def load(fnames, tag='', inst_id='', keep_original_names=False): 'Valid_Min', 'Valid_Range', '_Fillvalue', - 'ScaleTyp']) + 'ScaleTyp'], + decode_times=False) # xarray can't merge if variable and dim names are the same if 'Altitude' in data.dims: diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 30d1d3eb..1f92c802 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -112,6 +112,7 @@ def load_edr_aurora(fnames, tag='', inst_id='', pandas_format=False, sdata, mdata = load_netcdf(fname, epoch_name='TIME', epoch_unit='s', meta_kwargs={'labels': labels}, pandas_format=pandas_format, + decode_times=False, strict_dim_check=strict_dim_check) # Calculate the time for this data file. The pysat `load_netcdf` routine @@ -238,6 +239,7 @@ def load_sdr_aurora(fnames, tag='', inst_id='', pandas_format=False, sdata, mdata = load_netcdf(fname, epoch_name=load_time, epoch_unit='s', meta_kwargs={'labels': labels}, pandas_format=pandas_format, + decode_times=False, strict_dim_check=strict_dim_check) # Calculate the time for this data file. The pysat `load_netcdf` routine diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 13be59b1..980143a2 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -248,7 +248,8 @@ def load(fnames, tag='', inst_id=''): meta_kwargs={'labels': labels}, meta_translation=meta_translation, combine_by_coords=False, - drop_meta_labels='FILLVAL') + drop_meta_labels='FILLVAL', + decode_times=False) if tag in ['nmax', 'tdisk', 'tlimb']: # Add time coordinate from scan_start_time From a9a73030a9e4f825443e5df5d64b927d93cc328e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 13 Jun 2023 10:13:49 -0400 Subject: [PATCH 374/396] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index beeebb41..7e278a66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Use pysat logger to raise non-deprecation warnings * Update syntax based on latest pysat deprecations to make the code compatible with pysat 3.2.0. * Updated syntax compliance with cdflib 1.0+ + * Updated usage of `decode_times` kwarg to maintain usage in xarray ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 20e3adb5e2964d7fa4834e787e28a040fd716be6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 13 Jun 2023 11:30:12 -0400 Subject: [PATCH 375/396] ENH: map multidim meta --- pysatNASA/instruments/ace_mag_l2.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pysatNASA/instruments/ace_mag_l2.py b/pysatNASA/instruments/ace_mag_l2.py index 844b5a6a..d1bdaa59 100644 --- a/pysatNASA/instruments/ace_mag_l2.py +++ b/pysatNASA/instruments/ace_mag_l2.py @@ -67,6 +67,27 @@ # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_ace, name=name) + +def preprocess(self): + """Adjust dimensionality of metadata.""" + + # TODO(https://github.com/pysat/pysat/issues/1078): Update the metadata by + # removing value_min0, value_min1, etc, once possible + self.meta['BGSEc'] = {'value_min': min([self.meta['BGSEc']['value_min0'], + self.meta['BGSEc']['value_min1'], + self.meta['BGSEc']['value_min2']]), + 'value_max': max([self.meta['BGSEc']['value_max0'], + self.meta['BGSEc']['value_max1'], + self.meta['BGSEc']['value_max2']]), + 'SCALEMIN': min([self.meta['BGSEc']['SCALEMIN0'], + self.meta['BGSEc']['SCALEMIN1'], + self.meta['BGSEc']['SCALEMIN2']]), + 'SCALEMAX': max([self.meta['BGSEc']['SCALEMAX0'], + self.meta['BGSEc']['SCALEMAX1'], + self.meta['BGSEc']['SCALEMAX2']])} + return + + # Use default ace clean clean = mm_ace.clean From 39373f61959e91ee70359d1e1d48e55fe3669dc3 Mon Sep 17 00:00:00 2001 From: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> Date: Tue, 13 Jun 2023 12:38:07 -0400 Subject: [PATCH 376/396] Update pysatNASA/tests/test_instruments.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/tests/test_instruments.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index bce5dcca..8d9aec3a 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -98,9 +98,6 @@ def test_load_cdflib(self, inst_dict): @pytest.mark.second @pytest.mark.parametrize("inst_dict", instruments['cdf']) - @pytest.mark.skipif(cdflib_only, - reason=" ".join(("Additional load tests not required", - "when pysatCDF not installed"))) def test_meta_header(self, inst_dict): """Test that instruments have header level metadata attached. From 9fbecf7dc6fdd7c169adb595c8ca60ca95a80c4f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 13 Jun 2023 12:59:54 -0400 Subject: [PATCH 377/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysatNASA/instruments/methods/ace.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index d5702e0c..b151c8b6 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -37,7 +37,7 @@ def load(fnames, tag='', inst_id='', to_pandas=False): - """Load data via xarray and convert to pandas if needed. + """Load ACE data via xarray and convert to pandas if needed. This routine is called as needed by pysat. It is not intended for direct user interaction. @@ -80,9 +80,9 @@ def load(fnames, tag='', inst_id='', to_pandas=False): use_cdflib=True) if to_pandas: - try: + if hasattr(data, 'to_pandas'): data = data.to_pandas() - except AttributeError: + else: # xarray 0.16 support required for operational server data = data.to_dataframe() @@ -102,7 +102,7 @@ def clean(self): if self.pandas_format: coords = [self.data.index.name] else: - coords = [key for key in self.data.coords] + coords = [key for key in self.data.coords.keys()] for key in self.variables: # Skip over the coordinates when cleaning From bc5c0b33524aab49dc9a92c0c2133e590bedccb4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 13 Jun 2023 15:19:00 -0400 Subject: [PATCH 378/396] Update pysatNASA/instruments/methods/general.py --- pysatNASA/instruments/methods/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/general.py b/pysatNASA/instruments/methods/general.py index 2585b250..cc6a24e9 100644 --- a/pysatNASA/instruments/methods/general.py +++ b/pysatNASA/instruments/methods/general.py @@ -58,7 +58,7 @@ def clean(self): if self.pandas_format: coords = [self.data.index.name] else: - coords = [key for key in self.data.coords] + coords = [key for key in self.data.coords.keys()] for key in self.variables: # Skip over the coordinates when cleaning From ab93499b539ca23e7a48121259af91e888c46e4a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:18:08 -0400 Subject: [PATCH 379/396] Update CHANGELOG.md Co-authored-by: Angeline Burrell --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e278a66..f0ea4867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Use pysat logger to raise non-deprecation warnings * Update syntax based on latest pysat deprecations to make the code compatible with pysat 3.2.0. * Updated syntax compliance with cdflib 1.0+ - * Updated usage of `decode_times` kwarg to maintain usage in xarray + * Updated use of `decode_times` kwarg when loading xarray data to maintain current behaviour ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class From 9189e5f99aef778f67de746ac825f7ab69ae8ca6 Mon Sep 17 00:00:00 2001 From: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 14 Jun 2023 11:28:49 -0400 Subject: [PATCH 380/396] Update pysatNASA/tests/test_instruments.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/tests/test_instruments.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 8d9aec3a..cd2528ce 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -107,7 +107,20 @@ def test_meta_header(self, inst_dict): Dictionary containing info to instnatiate a specific instrument. """ test_inst, date = clslib.initialize_test_inst_and_date(inst_dict) - test_inst.load(date=date, use_header=True, use_cdflib=True) + try: + test_inst.load(date=date, use_header=True, use_cdflib=True) + except ValueError as verr: + # Check if instrument is failing due to strict time flag + if str(verr).find('Loaded data') > 0: + test_inst.strict_time_flag = False + with warnings.catch_warnings(record=True) as war: + test_inst.load(date=date, use_header=True, use_cdflib=True) + assert len(war) >= 1 + categories = [war[j].category for j in range(0, len(war))] + assert UserWarning in categories + else: + # If error message does not match, raise error anyway + raise(verr) assert test_inst.meta.to_dict() != {} From 9cda86463f1f0f5a23325467ef1945c97c8c2704 Mon Sep 17 00:00:00 2001 From: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 14 Jun 2023 11:45:37 -0400 Subject: [PATCH 381/396] Update test_instruments.py whitespace --- pysatNASA/tests/test_instruments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index cd2528ce..cac08df8 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -120,7 +120,7 @@ def test_meta_header(self, inst_dict): assert UserWarning in categories else: # If error message does not match, raise error anyway - raise(verr) + raise (verr) assert test_inst.meta.to_dict() != {} From 4830072d69e85a5a1cdb714f41aa5fea906a4bc2 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 14 Jun 2023 14:55:55 -0400 Subject: [PATCH 382/396] added TODO linked to pysat version 3.3.0 --- pysatNASA/tests/test_instruments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index cac08df8..1098a451 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -95,7 +95,8 @@ def test_load_cdflib(self, inst_dict): pytest.skip("Download data not available.") return - + # TODO(https://github.com/pysat/pysat/issues/1020): This test should be + # removed when header level data is tested in version 3.3.0+ of pysat @pytest.mark.second @pytest.mark.parametrize("inst_dict", instruments['cdf']) def test_meta_header(self, inst_dict): From 9ce996cd12ee580fee8316f7edd7c3171916d089 Mon Sep 17 00:00:00 2001 From: JonathonMSmith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 14 Jun 2023 14:58:03 -0400 Subject: [PATCH 383/396] added **kwargs to ace load with TODO linked to pysat 3.3.0. And PEP8 changes --- pysatNASA/instruments/methods/ace.py | 4 +++- pysatNASA/tests/test_instruments.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index df25ddfa..f9c25a7b 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -34,7 +34,9 @@ } -def load(fnames, tag='', inst_id='', to_pandas=False): +# TODO(https://github.com/pysat/pysat/issues/1020): **kwargs can be removed +# when header level meta is tested in version 3.3.0+ the core pysat +def load(fnames, tag='', inst_id='', to_pandas=False, **kwargs): """Load ACE data via xarray and convert to pandas if needed. This routine is called as needed by pysat. It is not intended diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 1098a451..62bd837c 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -95,6 +95,7 @@ def test_load_cdflib(self, inst_dict): pytest.skip("Download data not available.") return + # TODO(https://github.com/pysat/pysat/issues/1020): This test should be # removed when header level data is tested in version 3.3.0+ of pysat @pytest.mark.second From f0ed7c506f5f2dbedf0d80d88f1ab04361fcf0ff Mon Sep 17 00:00:00 2001 From: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 14 Jun 2023 15:04:17 -0400 Subject: [PATCH 384/396] removed TODO related to kwargs --- pysatNASA/instruments/methods/ace.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pysatNASA/instruments/methods/ace.py b/pysatNASA/instruments/methods/ace.py index f9c25a7b..f9e78412 100644 --- a/pysatNASA/instruments/methods/ace.py +++ b/pysatNASA/instruments/methods/ace.py @@ -34,8 +34,6 @@ } -# TODO(https://github.com/pysat/pysat/issues/1020): **kwargs can be removed -# when header level meta is tested in version 3.3.0+ the core pysat def load(fnames, tag='', inst_id='', to_pandas=False, **kwargs): """Load ACE data via xarray and convert to pandas if needed. From eb8327d577172e5eb2eba504759c6b47c678df59 Mon Sep 17 00:00:00 2001 From: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> Date: Wed, 14 Jun 2023 15:44:00 -0400 Subject: [PATCH 385/396] Update pysatNASA/tests/test_instruments.py Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatNASA/tests/test_instruments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 62bd837c..8a589938 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -97,7 +97,7 @@ def test_load_cdflib(self, inst_dict): return # TODO(https://github.com/pysat/pysat/issues/1020): This test should be - # removed when header level data is tested in version 3.3.0+ of pysat + # removed when header level data is tested in version 3.2.0+ of pysat @pytest.mark.second @pytest.mark.parametrize("inst_dict", instruments['cdf']) def test_meta_header(self, inst_dict): From 45185f2ec689d5d29c46ba4f4edb21df070301dd Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 15 Jun 2023 09:18:03 -0400 Subject: [PATCH 386/396] DOC: update changelog --- CHANGELOG.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d86081..e6905490 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -## [0.0.5] - 2023-XX-XX +## [0.0.5] - 2023-06-23 * New Instruments * ACE EPAM * ACE MAG @@ -14,11 +14,6 @@ This project adheres to [Semantic Versioning](https://semver.org/). * IGS GPS (TEC and ROTI) * SES-14 GOLD -- tdisk, tlimb and o2den data products added * TIMED GUVI -* Add TIMED GUVI platform to support L1C intensity datasets. - * Type of sensor source handled by inst_id with options of - spectrograph, imaging - * Resolution of dataset handled by tag with - low, high * Added CDAWeb methods that can use cdasws to get the remote file list * Bug Fixes * Pandas datasets made with cdflib now have header level meta From abd6af6be1fce47df3ce619397e704cfb5f70712 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 15 Jun 2023 09:20:27 -0400 Subject: [PATCH 387/396] STY: update test versions --- .github/workflows/docs.yml | 2 +- .github/workflows/pip_rc_install.yml | 2 +- .github/workflows/pysat_rc.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fe6d6ca4..2b6d7e88 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] # Keep this version at the highest supported Python version + python-version: ["3.11"] # Keep this version at the highest supported Python version name: Documentation tests steps: diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 5191c994..cc60a7dc 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10"] # Keep this version at the highest supported Python version + python-version: ["3.11"] # Keep this version at the highest supported Python version name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index b6292e18..e0ab0742 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10"] # Keep this version at the highest supported Python version + python-version: ["3.11"] # Keep this version at the highest supported Python version name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} From cdf7a4f1c76d46bf432cff4d32c9c390fcfed5a1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 15 Jun 2023 09:26:20 -0400 Subject: [PATCH 388/396] MAINT: minimum setuptools --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 646969a3..b35d7492 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools", "pip >= 10"] +requires = ["setuptools >= 38.6", "pip >= 10"] build-backend = "setuptools.build_meta" [project] From 74d506eb32624631391b90e24572eec056cb3173 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 15 Jun 2023 09:42:38 -0400 Subject: [PATCH 389/396] DOC: update authors --- .zenodo.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index 1052a817..1b5b3db1 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -20,6 +20,11 @@ "name": "Smith, Jonathon", "orcid": "0000-0002-8191-4765" }, + { + "affilitation":"University of Colorado at Boulder", + "name": "Navarro, Luis", + "orcid": "0000-0002-6362-6575" + }, { "affiliation": "Predictive Science", "name": "Pembroke, Asher" @@ -27,11 +32,6 @@ { "name": "Spence, Carey", "orcid": "0000-0001-8340-5625" - }, - { - "affilitation":"University of Colorado at Boulder", - "name": "Navarro, Luis", - "orcid": "0000-0002-6362-6575" } ] } From db74b96b834cf98aa48d1d0d3a98b68347454bdb Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 15 Jun 2023 09:54:53 -0400 Subject: [PATCH 390/396] DOC: clean up changelog --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6905490..fc80dc0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,20 +14,19 @@ This project adheres to [Semantic Versioning](https://semver.org/). * IGS GPS (TEC and ROTI) * SES-14 GOLD -- tdisk, tlimb and o2den data products added * TIMED GUVI -* Added CDAWeb methods that can use cdasws to get the remote file list * Bug Fixes * Pandas datasets made with cdflib now have header level meta * Updated CDAWeb routines to allow for data stored by year/day-of-year * Updated GOLD nmax to sort scans by time. - * Added 1 usec to GOLD nmax channel B times to ensure uniqueness + * Added 1 usec to GOLD nmax channel B times to ensure timestamp uniqueness * Fixed multi-file loads for cdf xarray datasets. * Adds a 0.1 sec delay between file downloads to avoid excessive calls to servers. * Documentation - * Added TIMED-GUVI platform * Added missing sub-module imports * Added discussion of ICON constellation to docstrings, including caveats * Enhancements + * Added CDAWeb methods that can use `cdasws` to get the remote file list * Updated platform methods to follow a consistent style and work with the general `init` function * Added unit tests for the different platform method attributes @@ -39,14 +38,14 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Improved usage of cdflib for users in xarray instruments * Added a generalized `clean` routine to replace fill vals with NaNs * Deprecations - * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps + * Deprecated jpl_gps instrument module, moved roti instrument to igs_gps * Maintenance * Updated download functions to take data_path as an arg, not a kwarg - * Removed duplicate tests if pysatCDF not isntalled + * Removed duplicate tests if pysatCDF not installed * Removed pysatCDF tests on GitHub Actions workflows (see #167) * Updated actions and templates based on pysatEcosystem docs * Remove pandas cap on NEP29 tests - * Updated dosctring style for consistency + * Updated docstring style for consistency * Removed version cap for xarray * Added manual workflow to check that latest RC is installable through test pip * Update meta label type for instruments @@ -60,6 +59,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated syntax compliance with cdflib 1.0+ * Updated use of `decode_times` kwarg when loading xarray data to maintain current behaviour + ## [0.0.4] - 2022-11-07 * Update instrument tests with new test class * Support xarray datasets through cdflib From 408881abe787abf28eea8bc8032b2cc906c52348 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:55:47 -0400 Subject: [PATCH 391/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- README.md | 2 +- pysatNASA/instruments/de2_vefi.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index de310e27..9c96b121 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ pip install . Note: pre-1.0.0 version ----------------------- -pysatNASA is currently in an initial development phase and requires pysat 3.0.4. +pysatNASA is currently in an initial development phase and requires pysat 3.1.0. Feedback and contributions are appreciated. # Using with pysat diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index 9e4140e6..14ecdb53 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -27,10 +27,10 @@ 'de2' name 'vefi' -inst_id - None Supported tag 'dca' or 'ac' +inst_id + None Supported Warnings From e436a65f96dd4fbfaa1e53437ab973fc32457c51 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 15 Jun 2023 13:59:51 -0400 Subject: [PATCH 392/396] DOC: consistent properties in docstrings --- pysatNASA/instruments/de2_fpi.py | 4 ++-- pysatNASA/instruments/de2_lang.py | 4 ++-- pysatNASA/instruments/de2_nacs.py | 4 ++-- pysatNASA/instruments/de2_rpa.py | 4 ++-- pysatNASA/instruments/de2_wats.py | 4 ++-- pysatNASA/instruments/icon_euv.py | 2 ++ pysatNASA/instruments/icon_fuv.py | 2 ++ pysatNASA/instruments/ses14_gold.py | 2 ++ 8 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pysatNASA/instruments/de2_fpi.py b/pysatNASA/instruments/de2_fpi.py index 4f0d210e..519191d2 100644 --- a/pysatNASA/instruments/de2_fpi.py +++ b/pysatNASA/instruments/de2_fpi.py @@ -34,10 +34,10 @@ 'de2' name 'fpi' -inst_id - None Supported tag None Supported +inst_id + None Supported Warnings diff --git a/pysatNASA/instruments/de2_lang.py b/pysatNASA/instruments/de2_lang.py index 01a77c56..f48d4797 100644 --- a/pysatNASA/instruments/de2_lang.py +++ b/pysatNASA/instruments/de2_lang.py @@ -31,10 +31,10 @@ 'de2' name 'lang' -inst_id - None Supported tag None Supported +inst_id + None Supported Warnings diff --git a/pysatNASA/instruments/de2_nacs.py b/pysatNASA/instruments/de2_nacs.py index dd6b7d16..6da31c5a 100644 --- a/pysatNASA/instruments/de2_nacs.py +++ b/pysatNASA/instruments/de2_nacs.py @@ -56,10 +56,10 @@ 'de2' name 'nacs' -inst_id - None Supported tag None Supported +inst_id + None Supported Warnings diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index e06b67b3..7f8e4f16 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -38,10 +38,10 @@ 'de2' name 'rpa' -inst_id - None Supported tag None Supported +inst_id + None Supported Warnings diff --git a/pysatNASA/instruments/de2_wats.py b/pysatNASA/instruments/de2_wats.py index e3b52bb7..90590761 100644 --- a/pysatNASA/instruments/de2_wats.py +++ b/pysatNASA/instruments/de2_wats.py @@ -53,10 +53,10 @@ 'de2' name 'wats' -inst_id - None Supported tag None Supported +inst_id + None Supported Warnings diff --git a/pysatNASA/instruments/icon_euv.py b/pysatNASA/instruments/icon_euv.py index f3abcf58..a3658154 100644 --- a/pysatNASA/instruments/icon_euv.py +++ b/pysatNASA/instruments/icon_euv.py @@ -13,6 +13,8 @@ 'euv' tag None supported +inst_id + None Supported Warnings -------- diff --git a/pysatNASA/instruments/icon_fuv.py b/pysatNASA/instruments/icon_fuv.py index 7749f006..38515810 100644 --- a/pysatNASA/instruments/icon_fuv.py +++ b/pysatNASA/instruments/icon_fuv.py @@ -13,6 +13,8 @@ 'fuv' tag None supported +inst_id + None Supported Warnings -------- diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 980143a2..52fd3ae1 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -14,6 +14,8 @@ 'tlimb' 'tdisk' 'o2den' +inst_id + None Supported Warnings -------- From 1dbdb5539232281ef6ac2b5377b12197d499650f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 15 Jun 2023 14:00:25 -0400 Subject: [PATCH 393/396] STY: import order --- pysatNASA/tests/test_instruments.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 8a589938..fac14fe7 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -8,16 +8,17 @@ import warnings -import pysat import pytest -# Make sure to import your instrument library here -import pysatNASA - # Import the test classes from pysat +import pysat from pysat.tests.classes import cls_instrument_library as clslib from pysat.utils import testing +# Make sure to import your instrument library here +import pysatNASA + + try: import pysatCDF # noqa: F401 # If this successfully imports, tests need to be run with both pysatCDF From 91db4235311f5612efd42e491e53b43211831b48 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:03:51 -0400 Subject: [PATCH 394/396] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysatNASA/instruments/de2_rpa.py | 6 +++--- pysatNASA/instruments/de2_vefi.py | 6 +++--- pysatNASA/instruments/ses14_gold.py | 2 +- pysatNASA/tests/test_instruments.py | 4 +++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pysatNASA/instruments/de2_rpa.py b/pysatNASA/instruments/de2_rpa.py index 7f8e4f16..aa97b66d 100644 --- a/pysatNASA/instruments/de2_rpa.py +++ b/pysatNASA/instruments/de2_rpa.py @@ -72,12 +72,12 @@ platform = 'de2' name = 'rpa' tags = {'': '2 sec cadence RPA data'} -inst_ids = {'': [tag for tag in tags]} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods @@ -99,7 +99,7 @@ dataproduct = {'': 'ion2s'} fname = 'de2_{dp:s}_rpa_{datestr:s}.cdf' supported_tags = {'': {tag: fname.format(dp=dataproduct[tag], datestr=datestr) - for tag in tags}} + for tag in tags.keys()}} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index 14ecdb53..aba68448 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -56,12 +56,12 @@ tags = {'': '62 ms combination of Electric Field and Magnetometer', 'dca': '500 ms cadence DC Averaged Electric Field data', 'ac': '500 ms cadence AC Electric Field data'} -inst_ids = {'': [tag for tag in tags]} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- @@ -85,7 +85,7 @@ 'dca': 'dca500ms_vefi'} fname = 'de2_{fid:s}_{datestr:s}.cdf' supported_tags = {'': {tag: fname.format(fid=fid[tag], datestr=datestr) - for tag in tags}} + for tag in tags.keys()}} list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 52fd3ae1..d9b1e9fb 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -133,7 +133,7 @@ def download(date_array, tag='', inst_id='', data_path=None): # Set the list_remote_files routine def list_remote_files(tag='', inst_id='', start=None, stop=None, series_out=True): - """Return a list of every file for chosen remote data. + """List every file for remote GOLD data. This routine is intended to be used by pysat instrument modules supporting a particular NASA CDAWeb dataset. diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index fac14fe7..8b8ffb13 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -107,7 +107,8 @@ def test_meta_header(self, inst_dict): Parameters ---------- inst_dict : dict - Dictionary containing info to instnatiate a specific instrument. + Dictionary containing info to instantiate a specific instrument. + """ test_inst, date = clslib.initialize_test_inst_and_date(inst_dict) try: @@ -118,6 +119,7 @@ def test_meta_header(self, inst_dict): test_inst.strict_time_flag = False with warnings.catch_warnings(record=True) as war: test_inst.load(date=date, use_header=True, use_cdflib=True) + assert len(war) >= 1 categories = [war[j].category for j in range(0, len(war))] assert UserWarning in categories From 15191593b7a49502d45464381b5b1531da65c0c1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 20 Jun 2023 16:06:18 -0400 Subject: [PATCH 395/396] STY: inst_ids / tags --- pysatNASA/instruments/de2_fpi.py | 4 ++-- pysatNASA/instruments/de2_lang.py | 4 ++-- pysatNASA/instruments/de2_nacs.py | 4 ++-- pysatNASA/instruments/de2_wats.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pysatNASA/instruments/de2_fpi.py b/pysatNASA/instruments/de2_fpi.py index 519191d2..d27693ed 100644 --- a/pysatNASA/instruments/de2_fpi.py +++ b/pysatNASA/instruments/de2_fpi.py @@ -67,12 +67,12 @@ platform = 'de2' name = 'fpi' tags = {'': '8 s cadence Fabry-Perot Interferometer data'} -inst_ids = {'': ['']} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatNASA/instruments/de2_lang.py b/pysatNASA/instruments/de2_lang.py index f48d4797..28d4dbc9 100644 --- a/pysatNASA/instruments/de2_lang.py +++ b/pysatNASA/instruments/de2_lang.py @@ -65,12 +65,12 @@ platform = 'de2' name = 'lang' tags = {'': '500 ms cadence Langmuir Probe data'} -inst_ids = {'': ['']} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatNASA/instruments/de2_nacs.py b/pysatNASA/instruments/de2_nacs.py index 6da31c5a..ba949e95 100644 --- a/pysatNASA/instruments/de2_nacs.py +++ b/pysatNASA/instruments/de2_nacs.py @@ -90,12 +90,12 @@ platform = 'de2' name = 'nacs' tags = {'': '1 s cadence Neutral Atmosphere Composition Spectrometer data'} -inst_ids = {'': ['']} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatNASA/instruments/de2_wats.py b/pysatNASA/instruments/de2_wats.py index 90590761..5a784d24 100644 --- a/pysatNASA/instruments/de2_wats.py +++ b/pysatNASA/instruments/de2_wats.py @@ -87,12 +87,12 @@ platform = 'de2' name = 'wats' tags = {'': '2 s cadence Wind and Temperature Spectrometer data'} -inst_ids = {'': ['']} +inst_ids = {'': [tag for tag in tags.keys()]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(1983, 1, 1)}} +_test_dates = {'': {tag: dt.datetime(1983, 1, 1) for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods From 14957b69f59f7d20f5bec1f9d97331b9e8b83fc0 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 27 Jun 2023 13:49:13 -0700 Subject: [PATCH 396/396] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc80dc0f..0bdc06a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -## [0.0.5] - 2023-06-23 +## [0.0.5] - 2023-06-27 * New Instruments * ACE EPAM * ACE MAG