From 1fb947f222524a536d0841a4aaf5649247e8aa53 Mon Sep 17 00:00:00 2001 From: Tracy Esman <21086818+t-esman@users.noreply.github.com> Date: Fri, 2 Jun 2023 14:06:43 -0400 Subject: [PATCH 001/365] Update installation.rst --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 423522513..afc3e029b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,7 +6,7 @@ Installation Python and associated packages for science are freely available. Convenient science python package setups are available from ``_, -`Anaconda `_, and other locations +`Anaconda `_, and other locations (some platform specific). Anaconda also includes a developer environment that works well with pysat. Core science packages such as numpy, scipy, matplotlib, pandas and many others may also be installed directly via the From d80a8eb6797c4fc6b7edeaec1356628e2163c829 Mon Sep 17 00:00:00 2001 From: Tracy Esman <21086818+t-esman@users.noreply.github.com> Date: Fri, 2 Jun 2023 14:19:37 -0400 Subject: [PATCH 002/365] Update tutorial_load.rst --- docs/tutorial/tutorial_load.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial/tutorial_load.rst b/docs/tutorial/tutorial_load.rst index dfd13072f..76478a28d 100644 --- a/docs/tutorial/tutorial_load.rst +++ b/docs/tutorial/tutorial_load.rst @@ -67,7 +67,7 @@ pysat supports the use of two different data structures. You can either use a pandas `DataFrame `_, a highly capable class with labeled rows and columns, or an xarray -`DataSet `_ +`DataSet `_ for data sets with more dimensions. The type of data class is flagged using the attribute :py:attr:`pysat.Instrument.pandas_format`. This is set to ``True`` if a :py:class:`pandas.DataFrame` is returned by the corresponding From 2a919f08c7b4db423b46342ae3026970aa9e1dbd Mon Sep 17 00:00:00 2001 From: Tracy Esman <21086818+t-esman@users.noreply.github.com> Date: Fri, 2 Jun 2023 14:21:03 -0400 Subject: [PATCH 003/365] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d34d155c..90588ccdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +[3.1.X] - 2023-xx-xx +-------------------- +* Maintenance + * Update link redirects in docs. + [3.1.0] - 2023-05-31 -------------------- * New Features From 73b86d41c77926d1d0463945b900b47957969c35 Mon Sep 17 00:00:00 2001 From: Tracy Esman <21086818+t-esman@users.noreply.github.com> Date: Fri, 2 Jun 2023 14:22:58 -0400 Subject: [PATCH 004/365] Update .zenodo.json --- .zenodo.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index bdc218349..8a69eee50 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -66,6 +66,11 @@ "affiliation": "@olist", "name": "Leite, Silvio", "orcid": "0000-0003-1707-7963" + }, + { + "affiliation": "NASA NPP", + "name": "Esman, Teresa", + "orcid": "0000-0003-0382-6281" } ] } From 066a0e9b94f27fa2caa48e6f9ee7ab84b5b085ef Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 5 Jul 2023 16:23:22 -0400 Subject: [PATCH 005/365] TST: added a test for warnings in `clean` Added a test for logging messages, warnings, and errors that may occur in Instrument `clean` methods. Also tests to see if the clean level has been reset correctly. --- pysat/tests/classes/cls_instrument_library.py | 136 +++++++++++++++--- 1 file changed, 120 insertions(+), 16 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index bb047209e..0e32fbda0 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -32,6 +32,7 @@ class TestInstruments(InstLibTests): import datetime as dt from importlib import import_module +import logging import sys import tempfile import warnings @@ -40,8 +41,7 @@ class TestInstruments(InstLibTests): import pysat from pysat.utils import generate_instrument_list -from pysat.utils.testing import assert_hasattr -from pysat.utils.testing import assert_isinstance +from pysat.utils import testing def initialize_test_inst_and_date(inst_dict): @@ -66,9 +66,8 @@ def initialize_test_inst_and_date(inst_dict): test_inst = pysat.Instrument(inst_module=inst_dict['inst_module'], tag=inst_dict['tag'], inst_id=inst_dict['inst_id'], - temporary_file_list=True, - update_files=True, use_header=True, - **kwargs) + temporary_file_list=True, update_files=True, + use_header=True, **kwargs) test_dates = inst_dict['inst_module']._test_dates date = test_dates[inst_dict['inst_id']][inst_dict['tag']] return test_inst, date @@ -218,10 +217,10 @@ def test_modules_standard(self, inst_name): # Check for presence of basic instrument module attributes for mattr in self.module_attrs: - assert_hasattr(module, mattr) + testing.assert_hasattr(module, mattr) if mattr in self.attr_types.keys(): - assert_isinstance(getattr(module, mattr), - self.attr_types[mattr]) + testing.assert_isinstance(getattr(module, mattr), + self.attr_types[mattr]) # Check for presence of required instrument attributes for inst_id in module.inst_ids.keys(): @@ -230,7 +229,7 @@ def test_modules_standard(self, inst_name): inst_id=inst_id, use_header=True) # Test to see that the class parameters were passed in - assert_isinstance(inst, pysat.Instrument) + testing.assert_isinstance(inst, pysat.Instrument) assert inst.platform == module.platform assert inst.name == module.name assert inst.inst_id == inst_id @@ -239,10 +238,10 @@ def test_modules_standard(self, inst_name): # Test the required class attributes for iattr in self.inst_attrs: - assert_hasattr(inst, iattr) + testing.assert_hasattr(inst, iattr) if iattr in self.attr_types: - assert_isinstance(getattr(inst, iattr), - self.attr_types[iattr]) + testing.assert_isinstance(getattr(inst, iattr), + self.attr_types[iattr]) return @pytest.mark.all_inst @@ -287,7 +286,7 @@ def test_instrument_test_dates(self, inst_name): info = module._test_dates for inst_id in info.keys(): for tag in info[inst_id].keys(): - assert_isinstance(info[inst_id][tag], dt.datetime) + testing.assert_isinstance(info[inst_id][tag], dt.datetime) return @pytest.mark.first @@ -307,8 +306,10 @@ def test_download(self, inst_dict): test_inst, date = initialize_test_inst_and_date(inst_dict) # Check for username. - dl_dict = inst_dict['user_info'] if 'user_info' in \ - inst_dict.keys() else {} + if 'user_info' in inst_dict.keys(): + dl_dict = inst_dict['user_info'] + else: + dl_dict = {} test_inst.download(date, date, **dl_dict) assert len(test_inst.files.files) > 0 return @@ -352,7 +353,7 @@ def test_load(self, clean_level, inst_dict): assert UserWarning in categories else: # If error message does not match, raise error anyway - raise(verr) + raise ValueError(verr) # Make sure fake data is cleared assert target not in test_inst.data @@ -366,6 +367,109 @@ def test_load(self, clean_level, inst_dict): return + @pytest.mark.second + # Need to maintain download mark for backwards compatibility. + # Can remove once pysat 3.1.0 is released and libraries are updated. + @pytest.mark.load_options + @pytest.mark.download + @pytest.mark.parametrize("clean_level", ['none', 'dirty', 'dusty', 'clean']) + def test_clean_warn(self, clean_level, inst_dict, caplog): + """Test that appropriate warnings and errors are raised when cleaning. + + Parameters + ---------- + clean_level : str + Cleanliness level for loaded instrument data. + inst_dict : dict + Dictionary containing info to instantiate a specific instrument. + Set automatically from instruments['download'] when + `initialize_test_package` is run. + **kwags : dict + + """ + if hasattr(inst_dict['inst_module'], '_clean_warn'): + clean_warn = inst_dict['inst_module']._clean_warn + + if clean_level in clean_warn.keys(): + # Only need to test if there are clean warnings for this level + test_inst, date = initialize_test_inst_and_date(inst_dict) + (clean_method, clean_method_level, clean_method_msg, + final_level) = clean_warn[clean_level] + + if len(test_inst.files.files) > 0: + # Set the clean level + test_inst.clean_level = clean_level + target = 'Fake Data to be cleared' + test_inst.data = [target] + + try: + if clean_method == 'logger': + # A logging message is expected + with caplog.at_level( + getattr(logging, clean_method_level), + logger='pysat'): + test_inst.load(date=date, use_header=True) + + # Test the returned message + out_msg = caplog.text + assert out_msg.find(clean_method_msg) >= 0 + elif clean_method == 'warning': + # A warning message is expected + with warnings.catch_warnings(record=True) as war: + test_inst.load(date=date, use_header=True) + + # Test the warning output + testing.eval_warnings(war, [clean_method_msg], + clean_method_level) + elif clean_method == 'error': + # An error message is expected, evaluate error and + # error message + try: + testing.eval_bad_input( + test_inst.load, clean_method_level, + clean_method_msg, + input_kwargs={'date': date, + 'use_header': True}) + except AssertionError as aerr: + if str(aerr).find( + 'Loaded data is not unique') >= 0: + pytest.skip('Cannot test multiple errors') + else: + raise AssertionError(aerr) + else: + raise AttributeError( + 'unknown type of warning: {:}'.format( + clean_method)) + except ValueError as verr: + # Check if instrument is failing due to strict time flag + if str(verr).find('Loaded data') > 0: + test_inst.strict_time_flag = False + with warnings.catch_warnings(record=True) as war: + test_inst.load(date=date, use_header=True) + assert len(war) >= 1 + categories = [war[j].category + for j in range(0, len(war))] + assert UserWarning in categories + else: + # If error message does not match, raise error + raise ValueError(verr) + + # Test to see if the clean flag has the expected value + # afterwards + assert test_inst.clean_level == final_level, \ + "Clean level should now be {:s}, not {:s}".format( + final_level, test_inst.clean_level) + + # Make sure fake data is cleared + assert target not in test_inst.data + + # If cleaning not used, something should be in the file. Not + # used for clean levels since cleaning may remove all data + if clean_level == "none": + assert not test_inst.empty + + return + @pytest.mark.download def test_remote_file_list(self, inst_dict): """Test if optional list_remote_files routine exists and is callable. From 71f6d9d3f0f09a3bd9d9369f17a5ee0d902d6321 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 5 Jul 2023 16:24:47 -0400 Subject: [PATCH 006/365] TST: expanded test Instrument cleaning Added level resetting, logging messages, warnings, and errors as potential outputs to the clean method for test instruments. Also added a unit test for each of these elements. --- pysat/instruments/methods/testing.py | 21 ++++++++++- pysat/tests/test_instruments.py | 54 ++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index bf9cbeef1..95f143425 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -56,12 +56,31 @@ def clean(self, test_clean_kwarg=None): Parameters ---------- test_clean_kwarg : any - Testing keyword (default=None) + Testing keyword. If these keywords contain 'logger', 'warning', or + 'error', the message entered as the value to that key will be returned + as a logging.WARNING, UserWarning, or ValueError, respectively. If the + 'change' kwarg is set, the clean level will be changed to the specified + value. (default=None) """ self.test_clean_kwarg = test_clean_kwarg + if test_clean_kwarg is None: + test_clean_kwarg = {} + + if 'change' in test_clean_kwarg.keys(): + self.clean_level = test_clean_kwarg['change'] + + if 'logger' in test_clean_kwarg.keys(): + pysat.logger.warning(test_clean_kwarg['logger']) + + if 'warning' in test_clean_kwarg.keys(): + warnings.warn(test_clean_kwarg['warning'], UserWarning) + + if 'error' in test_clean_kwarg.keys(): + raise ValueError(test_clean_kwarg['error']) + return diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index 605270288..be8f27edf 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -146,6 +146,60 @@ def test_inst_max_latitude(self, inst_dict): return + @pytest.mark.second + @pytest.mark.parametrize("clean_level", ['clean', 'dusty', 'dirty']) + @pytest.mark.parametrize("change", [True, False]) + @pytest.mark.parametrize('warn_type', ['logger', 'warning', 'error']) + @pytest.mark.parametrize("inst_dict", instruments['download']) + def test_clean_with_warnings(self, clean_level, change, warn_type, + inst_dict, caplog): + """Run `test_clean_warn` with different warning behaviours. + + Parameters + ---------- + clean_level : str + Cleanliness level for loaded instrument data; must run the clean + routine (not include 'none'). + change : bool + Specify whether or not clean level should change. + warn_type : str + Desired type of warning or error to be raised. + inst_dict : dict + One of the dictionaries returned from + `InstLibTests.initialize_test_package` with instruments to test. + + """ + # Set the default values + warn_level = {'logger': 'WARN', 'warning': UserWarning, + 'error': ValueError} + warn_msg = 'Default warning message' + if change: + final_level = 'none' if clean_level == 'clean' else 'clean' + else: + final_level = clean_level + + # Construct the expected warnings + inst_dict['inst_module']._clean_warn = { + clean_level: (warn_type, warn_level[warn_type], warn_msg, + final_level)} + + # Set the additional Instrument kwargs + if 'kwargs' in inst_dict.keys(): + if 'test_clean_kwarg' in inst_dict['kwargs']: + inst_dict['kwargs']['test_clean_kwarg']['change'] = final_level + inst_dict['kwargs']['test_clean_kwarg'][warn_type] = warn_msg + else: + inst_dict['kwargs']['test_clean_kwarg'] = { + 'change': final_level, warn_type: warn_msg} + else: + inst_dict['kwargs'] = {'test_clean_kwarg': {'change': final_level, + warn_type: warn_msg}} + + # Run the test + self.test_clean_warn(clean_level, inst_dict, caplog) + + return + class TestDeprecation(object): """Unit test for deprecation warnings.""" From 6227fc92a0b57ce8da16b010498d0ecf5bf699dd Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 5 Jul 2023 16:25:22 -0400 Subject: [PATCH 007/365] DOC: updated new instrument instructions Updated the new instrument instructions for the clean method and `_clean_warn` test variable. --- docs/new_instrument.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 06f39603a..68b15a909 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -619,6 +619,13 @@ Cleans instrument for levels supplied in inst.clean_level. ``self`` is a :py:class:`pysat.Instrument` object. :py:func:`clean` should modify ``self`` in-place as needed; equivalent to a custom routine. +:py:func:`clean` is allowed to raise logger messages, warnings, and errors. If +the routine does this, be sure to test them by assigning the necessary +information to the :py:attr:`_clean_warn` attribute, described in Section +rst_new_inst-test_. :py:func:`clean` may also re-assign the cleaning level if +appropriate. If you do this, be sure to raise a logging warning, so that users +are aware that this change is happening and why the clean level they requested +is not appropriate. list_remote_files ^^^^^^^^^^^^^^^^^ @@ -731,7 +738,25 @@ combinations), and runs the tests using pytestmark. By default, routine, and will run an end-to-end test. If this is not the case, see the next section. +Another important test is for warnings and the re-setting of clean levels that +may come up when cleaning data. These may be specified using the +:py:attr:`_clean_warn` attribute, which should point to a dictionary that has a +tuple of four elements as the value. The first element should be 'logger', +'warning', or 'error', specifying the method through which the warning is being +reported. The second element specifies either the logging level (as a string) +or the warning/error type (e.g., ``ValueError``). The third element provides the +warning message as a string and the final element provides the expected clean +level after running the clean routine. +.. code:: python + + # ------------------------------------------ + # Instrument test attributes + + _clean_warn = {'dusty': ('logger', 'WARN', "I am a warning!", 'clean')} + + + .. _rst_test-special: Special Test Configurations From 665ab427c4fe6aeac055f6473e986f9359fd714d Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 5 Jul 2023 16:25:45 -0400 Subject: [PATCH 008/365] DOC: updated changelog Updated the changelog with a description of this pull request. --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90588ccdb..8274e2f15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,11 @@ This project adheres to [Semantic Versioning](https://semver.org/). [3.1.X] - 2023-xx-xx -------------------- +* New Features + * Added tests for warnings, logging messages, and errors in the Instrument + clean method. * Maintenance - * Update link redirects in docs. + * Update link redirects in docs. [3.1.0] - 2023-05-31 -------------------- From b84e0052612c3f9661f0ce4b5dd8a5284cd8e33f Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 5 Jul 2023 16:37:13 -0400 Subject: [PATCH 009/365] STY: updated reference style Updated the reference style in the docs. --- docs/new_instrument.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 68b15a909..b95bc0306 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -621,11 +621,11 @@ Cleans instrument for levels supplied in inst.clean_level. modify ``self`` in-place as needed; equivalent to a custom routine. :py:func:`clean` is allowed to raise logger messages, warnings, and errors. If the routine does this, be sure to test them by assigning the necessary -information to the :py:attr:`_clean_warn` attribute, described in Section -rst_new_inst-test_. :py:func:`clean` may also re-assign the cleaning level if -appropriate. If you do this, be sure to raise a logging warning, so that users -are aware that this change is happening and why the clean level they requested -is not appropriate. +information to the :py:attr:`_clean_warn` attribute, described in +:ref:`Testing Support `_. :py:func:`clean` may also +re-assign the cleaning level if appropriate. If you do this, be sure to raise a +logging warning, so that users are aware that this change is happening and why +the clean level they requested is not appropriate. list_remote_files ^^^^^^^^^^^^^^^^^ From 67c1d8c15cbae57b845d5f3009e79971f412e264 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 5 Jul 2023 16:40:22 -0400 Subject: [PATCH 010/365] BUG: fixing reference Fixing bug in reference format. --- docs/new_instrument.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index b95bc0306..655a0ad06 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -622,7 +622,7 @@ modify ``self`` in-place as needed; equivalent to a custom routine. :py:func:`clean` is allowed to raise logger messages, warnings, and errors. If the routine does this, be sure to test them by assigning the necessary information to the :py:attr:`_clean_warn` attribute, described in -:ref:`Testing Support `_. :py:func:`clean` may also +:ref:`Testing Support `. :py:func:`clean` may also re-assign the cleaning level if appropriate. If you do this, be sure to raise a logging warning, so that users are aware that this change is happening and why the clean level they requested is not appropriate. From ac2eb50f45ee26af70c1aab81ecfe8bc75d14426 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 6 Jul 2023 10:16:44 -0400 Subject: [PATCH 011/365] BUG: fixed dict evalutation Fixed logic in clean method, allowing key search only if input is a dict. --- pysat/instruments/methods/testing.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 95f143425..fe66275bf 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -66,20 +66,18 @@ def clean(self, test_clean_kwarg=None): self.test_clean_kwarg = test_clean_kwarg - if test_clean_kwarg is None: - test_clean_kwarg = {} + if isinstance(test_clean_kwarg, dict): + if 'change' in test_clean_kwarg.keys(): + self.clean_level = test_clean_kwarg['change'] - if 'change' in test_clean_kwarg.keys(): - self.clean_level = test_clean_kwarg['change'] + if 'logger' in test_clean_kwarg.keys(): + pysat.logger.warning(test_clean_kwarg['logger']) - if 'logger' in test_clean_kwarg.keys(): - pysat.logger.warning(test_clean_kwarg['logger']) + if 'warning' in test_clean_kwarg.keys(): + warnings.warn(test_clean_kwarg['warning'], UserWarning) - if 'warning' in test_clean_kwarg.keys(): - warnings.warn(test_clean_kwarg['warning'], UserWarning) - - if 'error' in test_clean_kwarg.keys(): - raise ValueError(test_clean_kwarg['error']) + if 'error' in test_clean_kwarg.keys(): + raise ValueError(test_clean_kwarg['error']) return From 2b5ae2ef91b359cb2271004d4300c0cfced32b1a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 6 Jul 2023 10:34:40 -0400 Subject: [PATCH 012/365] DOC: Update docs/new_instrument.rst Updated example for `_clean_warn` in docs to include inst_id and tag. --- docs/new_instrument.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 655a0ad06..71234c454 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -753,7 +753,7 @@ level after running the clean routine. # ------------------------------------------ # Instrument test attributes - _clean_warn = {'dusty': ('logger', 'WARN', "I am a warning!", 'clean')} + _clean_warn = {inst_id: {tag: {'dusty': ('logger', 'WARN', "I am a warning!", 'clean')} for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} From 56dadc7707b80e19e3a32e55788df3982940bc7f Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 6 Jul 2023 11:47:05 -0400 Subject: [PATCH 013/365] ENH: removed unreachable lines Removed lines that cannot be reached, as the clean method doesn't run if the clean level is 'none'. --- pysat/tests/classes/cls_instrument_library.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 0e32fbda0..e9947d383 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -463,11 +463,6 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Make sure fake data is cleared assert target not in test_inst.data - # If cleaning not used, something should be in the file. Not - # used for clean levels since cleaning may remove all data - if clean_level == "none": - assert not test_inst.empty - return @pytest.mark.download From cd75aeed01cb39fa831dbd83341989ae2653dfff Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 12:40:39 -0400 Subject: [PATCH 014/365] MAINT: updated `_clean_warn` format Updated the `_clean_warn` format to accept a list of tuples. --- docs/new_instrument.rst | 24 ++- pysat/tests/classes/cls_instrument_library.py | 143 +++++++++--------- pysat/tests/test_instruments.py | 32 ++-- 3 files changed, 113 insertions(+), 86 deletions(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 71234c454..d88e0e386 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -732,7 +732,7 @@ will not be present in Input/Output operations. The standardized :py:mod:`pysat` tests are available in :py:mod:`pysat.tests.instrument_test_class`. The test collection in test_instruments.py imports this class, collects a list of all available -instruments (including potential :py:data:`tag`/:py:data:`inst_id` +instruments (including potential :py:attr:`tag`/:py:attr:`inst_id` combinations), and runs the tests using pytestmark. By default, :py:mod:`pysat` assumes that your instrument has a fully functional download routine, and will run an end-to-end test. If this is not the case, see the next @@ -741,19 +741,27 @@ section. Another important test is for warnings and the re-setting of clean levels that may come up when cleaning data. These may be specified using the :py:attr:`_clean_warn` attribute, which should point to a dictionary that has a -tuple of four elements as the value. The first element should be 'logger', -'warning', or 'error', specifying the method through which the warning is being -reported. The second element specifies either the logging level (as a string) -or the warning/error type (e.g., ``ValueError``). The third element provides the -warning message as a string and the final element provides the expected clean -level after running the clean routine. +list with tuples of four elements as the value. The first tuple element should +be 'logger', 'warning', or 'error', specifying the method through which the +warning is being reported. The second tuple element specifies either the logging +level (as a string) or the warning/error type (e.g., ``ValueError``). The third +tuple element provides the warning message as a string and the final element +provides the expected clean level after running the clean routine. The list +allows multiple types of warning messages to be tested for a given +:py:attr:`inst_id`, :py:attr:`tag`, and :py:attr:`clean_level` combination. .. code:: python # ------------------------------------------ # Instrument test attributes - _clean_warn = {inst_id: {tag: {'dusty': ('logger', 'WARN', "I am a warning!", 'clean')} for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} + _clean_warn = {inst_id: {tag: {'dusty': [ + ('logger', 'WARN', "I am a warning!", 'clean'), + ('warning', UserWarning, + 'I am a serios warning!', 'dusty'), + ('error, ValueError, 'I am an error', 'dusty')]} + for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index e9947d383..5bdf6faf7 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -393,75 +393,80 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): if clean_level in clean_warn.keys(): # Only need to test if there are clean warnings for this level test_inst, date = initialize_test_inst_and_date(inst_dict) - (clean_method, clean_method_level, clean_method_msg, - final_level) = clean_warn[clean_level] - - if len(test_inst.files.files) > 0: - # Set the clean level - test_inst.clean_level = clean_level - target = 'Fake Data to be cleared' - test_inst.data = [target] - - try: - if clean_method == 'logger': - # A logging message is expected - with caplog.at_level( - getattr(logging, clean_method_level), - logger='pysat'): - test_inst.load(date=date, use_header=True) - - # Test the returned message - out_msg = caplog.text - assert out_msg.find(clean_method_msg) >= 0 - elif clean_method == 'warning': - # A warning message is expected - with warnings.catch_warnings(record=True) as war: - test_inst.load(date=date, use_header=True) - - # Test the warning output - testing.eval_warnings(war, [clean_method_msg], - clean_method_level) - elif clean_method == 'error': - # An error message is expected, evaluate error and - # error message - try: - testing.eval_bad_input( - test_inst.load, clean_method_level, - clean_method_msg, - input_kwargs={'date': date, - 'use_header': True}) - except AssertionError as aerr: - if str(aerr).find( - 'Loaded data is not unique') >= 0: - pytest.skip('Cannot test multiple errors') - else: - raise AssertionError(aerr) - else: - raise AttributeError( - 'unknown type of warning: {:}'.format( - clean_method)) - except ValueError as verr: - # Check if instrument is failing due to strict time flag - if str(verr).find('Loaded data') > 0: - test_inst.strict_time_flag = False - with warnings.catch_warnings(record=True) as war: - test_inst.load(date=date, use_header=True) - assert len(war) >= 1 - categories = [war[j].category - for j in range(0, len(war))] - assert UserWarning in categories - else: - # If error message does not match, raise error - raise ValueError(verr) - - # Test to see if the clean flag has the expected value - # afterwards - assert test_inst.clean_level == final_level, \ - "Clean level should now be {:s}, not {:s}".format( - final_level, test_inst.clean_level) - - # Make sure fake data is cleared - assert target not in test_inst.data + clean_warnings = clean_warn[clean_level] + + for (clean_method, clean_method_level, clean_method_msg, + final_level) in clean_warnings: + if len(test_inst.files.files) > 0: + # Set the clean level + test_inst.clean_level = clean_level + target = 'Fake Data to be cleared' + test_inst.data = [target] + + try: + if clean_method == 'logger': + # A logging message is expected + with caplog.at_level( + getattr(logging, clean_method_level), + logger='pysat'): + test_inst.load(date=date, use_header=True) + + # Test the returned message + out_msg = caplog.text + assert out_msg.find(clean_method_msg) >= 0 + elif clean_method == 'warning': + # A warning message is expected + with warnings.catch_warnings( + record=True) as war: + test_inst.load(date=date, use_header=True) + + # Test the warning output + testing.eval_warnings(war, [clean_method_msg], + clean_method_level) + elif clean_method == 'error': + # An error message is expected, evaluate error + # and the error message + try: + testing.eval_bad_input( + test_inst.load, clean_method_level, + clean_method_msg, + input_kwargs={'date': date, + 'use_header': True}) + except AssertionError as aerr: + if str(aerr).find( + 'Loaded data is not unique') >= 0: + pytest.skip( + 'Cannot test multiple errors') + else: + raise AssertionError(aerr) + else: + raise AttributeError( + 'unknown type of warning: {:}'.format( + clean_method)) + except ValueError as verr: + # Check if instrument is failing due to strict time + # flag + if str(verr).find('Loaded data') > 0: + test_inst.strict_time_flag = False + with warnings.catch_warnings( + record=True) as war: + test_inst.load(date=date, use_header=True) + assert len(war) >= 1 + categories = [war[j].category + for j in range(0, len(war))] + assert UserWarning in categories + else: + # If error message does not match, raise error + raise ValueError(verr) + + # Test to see if the clean flag has the expected value + # afterwards + assert test_inst.clean_level == final_level, \ + "Clean level should now be {:s}, not {:s}".format( + final_level, test_inst.clean_level) + + # Make sure fake data is cleared + assert target not in test_inst.data return diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index be8f27edf..37d6ed8bf 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -149,7 +149,8 @@ def test_inst_max_latitude(self, inst_dict): @pytest.mark.second @pytest.mark.parametrize("clean_level", ['clean', 'dusty', 'dirty']) @pytest.mark.parametrize("change", [True, False]) - @pytest.mark.parametrize('warn_type', ['logger', 'warning', 'error']) + @pytest.mark.parametrize('warn_type', ['logger', 'warning', 'error', + 'mult']) @pytest.mark.parametrize("inst_dict", instruments['download']) def test_clean_with_warnings(self, clean_level, change, warn_type, inst_dict, caplog): @@ -179,21 +180,34 @@ def test_clean_with_warnings(self, clean_level, change, warn_type, final_level = clean_level # Construct the expected warnings - inst_dict['inst_module']._clean_warn = { - clean_level: (warn_type, warn_level[warn_type], warn_msg, - final_level)} + if warn_type == 'mult': + inst_dict['inst_module']._clean_warn = { + inst_dict['inst_id']: {inst_dict['tag']: {clean_level: [ + ('logger', warn_level['logger'], warn_msg, final_level), + ('warning', warn_level['warning'], warn_msg, final_level), + ('error', warn_level['error'], warn_msg, final_level)]}}} + else: + inst_dict['inst_module']._clean_warn = { + inst_dict['inst_id']: {inst_dict['tag']: {clean_level: [ + (warn_type, warn_level[warn_type], warn_msg, + final_level)]}}} # Set the additional Instrument kwargs if 'kwargs' in inst_dict.keys(): if 'test_clean_kwarg' in inst_dict['kwargs']: inst_dict['kwargs']['test_clean_kwarg']['change'] = final_level - inst_dict['kwargs']['test_clean_kwarg'][warn_type] = warn_msg else: - inst_dict['kwargs']['test_clean_kwarg'] = { - 'change': final_level, warn_type: warn_msg} + inst_dict['kwargs']['test_clean_kwarg'] = {'change': + final_level} + else: + inst_dict['kwargs'] = {'test_clean_kwarg': {'change': final_level}} + + if warn_type == 'mult': + inst_dict['kwargs']['test_clean_kwarg']['logger'] = warn_msg + inst_dict['kwargs']['test_clean_kwarg']['warning'] = warn_msg + inst_dict['kwargs']['test_clean_kwarg']['error'] = warn_msg else: - inst_dict['kwargs'] = {'test_clean_kwarg': {'change': final_level, - warn_type: warn_msg}} + inst_dict['kwargs']['test_clean_kwarg'][warn_type] = warn_msg # Run the test self.test_clean_warn(clean_level, inst_dict, caplog) From cf273b527902fc311e9b8df3babedb9230dfbb4f Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 13:22:40 -0400 Subject: [PATCH 015/365] DOC: improved documentation flow Moved the discussion of the new clean_warn test attribute to its own section and updated the reference. --- docs/new_instrument.rst | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index d88e0e386..1712ad9a3 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -622,7 +622,7 @@ modify ``self`` in-place as needed; equivalent to a custom routine. :py:func:`clean` is allowed to raise logger messages, warnings, and errors. If the routine does this, be sure to test them by assigning the necessary information to the :py:attr:`_clean_warn` attribute, described in -:ref:`Testing Support `. :py:func:`clean` may also +:ref:`rst_test-clean`. :py:func:`clean` may also re-assign the cleaning level if appropriate. If you do this, be sure to raise a logging warning, so that users are aware that this change is happening and why the clean level they requested is not appropriate. @@ -738,6 +738,22 @@ combinations), and runs the tests using pytestmark. By default, routine, and will run an end-to-end test. If this is not the case, see the next section. + +.. _rst_test-special: + +Special Test Configurations +--------------------------- + +The following test attributes may or may not be necessary for your new +:py:class:`~pysat._instrument.Instrument`. The descriptions should provide +insight into when and how they should be used. + + +.. _rst_test-clean: + +Warnings in the Clean method +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Another important test is for warnings and the re-setting of clean levels that may come up when cleaning data. These may be specified using the :py:attr:`_clean_warn` attribute, which should point to a dictionary that has a @@ -764,11 +780,7 @@ allows multiple types of warning messages to be tested for a given for inst_id in inst_ids.keys()} - -.. _rst_test-special: - -Special Test Configurations ---------------------------- +.. _rst_test-nodownload: No Download Available ^^^^^^^^^^^^^^^^^^^^^ From 728153bb763ea178c8a77463168fb3a4849d1ddf Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 6 Jul 2023 13:27:12 -0400 Subject: [PATCH 016/365] MAINT: added missing quotation Added a missing quotation mark. --- docs/new_instrument.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 1712ad9a3..f6d5a2cc1 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -775,7 +775,7 @@ allows multiple types of warning messages to be tested for a given ('logger', 'WARN', "I am a warning!", 'clean'), ('warning', UserWarning, 'I am a serios warning!', 'dusty'), - ('error, ValueError, 'I am an error', 'dusty')]} + ('error', ValueError, 'I am an error', 'dusty')]} for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} From 150e691e1c492ffa85b7a24e9faa40dfaff6b936 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 16:33:38 -0400 Subject: [PATCH 017/365] BUG: fixed dict access Fixed the dict access for the expected clean warnings. --- pysat/tests/classes/cls_instrument_library.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 5bdf6faf7..08e4238c5 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -388,7 +388,8 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): """ if hasattr(inst_dict['inst_module'], '_clean_warn'): - clean_warn = inst_dict['inst_module']._clean_warn + clean_warn = inst_dict['inst_module']._clean_warn[ + inst_dict['inst_id']][inst_dict['tag']] if clean_level in clean_warn.keys(): # Only need to test if there are clean warnings for this level From 78e85aa9b5fe3d801f1d48a81c2f39f0fa6ba415 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 18:09:13 -0400 Subject: [PATCH 018/365] BUG: handle strict time flag before messages Ensure the strict time flag is not raising a ValueError when testing the clean warning messages. --- pysat/tests/classes/cls_instrument_library.py | 108 +++++++++--------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 08e4238c5..4f73cf428 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -396,6 +396,30 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): test_inst, date = initialize_test_inst_and_date(inst_dict) clean_warnings = clean_warn[clean_level] + try: + test_inst.load(date=date, use_header=True) + except Exception as err: + # Catch all potential input errors, and only ensure that + # the one caused by the strict time flag is prevented from + # occurring on future load calls. + if str(err).find('Loaded data') > 0: + # Change the flags that may have caused + # the error to be raised, to see if it the + # strict time flag + test_inst.strict_time_flag = False + test_inst.clean_level = 'none' + + # Evaluate the warning + with warnings.catch_warnings(record=True) as war: + test_inst.load(date=date, use_header=True) + + assert len(war) >= 1 + categories = [war[j].category for j in range(len(war))] + assert UserWarning in categories + + # Reset the clean level + test_inst.clean_level = clean_level + for (clean_method, clean_method_level, clean_method_msg, final_level) in clean_warnings: if len(test_inst.files.files) > 0: @@ -404,61 +428,35 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): target = 'Fake Data to be cleared' test_inst.data = [target] - try: - if clean_method == 'logger': - # A logging message is expected - with caplog.at_level( - getattr(logging, clean_method_level), - logger='pysat'): - test_inst.load(date=date, use_header=True) - - # Test the returned message - out_msg = caplog.text - assert out_msg.find(clean_method_msg) >= 0 - elif clean_method == 'warning': - # A warning message is expected - with warnings.catch_warnings( - record=True) as war: - test_inst.load(date=date, use_header=True) - - # Test the warning output - testing.eval_warnings(war, [clean_method_msg], - clean_method_level) - elif clean_method == 'error': - # An error message is expected, evaluate error - # and the error message - try: - testing.eval_bad_input( - test_inst.load, clean_method_level, - clean_method_msg, - input_kwargs={'date': date, - 'use_header': True}) - except AssertionError as aerr: - if str(aerr).find( - 'Loaded data is not unique') >= 0: - pytest.skip( - 'Cannot test multiple errors') - else: - raise AssertionError(aerr) - else: - raise AttributeError( - 'unknown type of warning: {:}'.format( - clean_method)) - except ValueError as verr: - # Check if instrument is failing due to strict time - # flag - if str(verr).find('Loaded data') > 0: - test_inst.strict_time_flag = False - with warnings.catch_warnings( - record=True) as war: - test_inst.load(date=date, use_header=True) - assert len(war) >= 1 - categories = [war[j].category - for j in range(0, len(war))] - assert UserWarning in categories - else: - # If error message does not match, raise error - raise ValueError(verr) + if clean_method == 'logger': + # A logging message is expected + with caplog.at_level( + getattr(logging, clean_method_level), + logger='pysat'): + test_inst.load(date=date, use_header=True) + + # Test the returned message + out_msg = caplog.text + assert out_msg.find(clean_method_msg) >= 0 + elif clean_method == 'warning': + # A warning message is expected + with warnings.catch_warnings(record=True) as war: + test_inst.load(date=date, use_header=True) + + # Test the warning output + testing.eval_warnings(war, [clean_method_msg], + clean_method_level) + elif clean_method == 'error': + # An error message is expected, evaluate error + # and the error message + testing.eval_bad_input( + test_inst.load, clean_method_level, + clean_method_msg, + input_kwargs={'date': date, 'use_header': True}) + else: + raise AttributeError( + 'unknown type of warning: {:}'.format( + clean_method)) # Test to see if the clean flag has the expected value # afterwards From 0d2c069f2dd5a8cfa4d0acd92da6de02781d2e05 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 18:11:00 -0400 Subject: [PATCH 019/365] BUG: remove error from mult Cannot assess other warnings when an error is raised, only test multiple non-fatal warnings. --- pysat/tests/test_instruments.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index 37d6ed8bf..d5cede4e2 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -181,11 +181,11 @@ def test_clean_with_warnings(self, clean_level, change, warn_type, # Construct the expected warnings if warn_type == 'mult': + # Note that we cannot test errors along with other warnings inst_dict['inst_module']._clean_warn = { inst_dict['inst_id']: {inst_dict['tag']: {clean_level: [ - ('logger', warn_level['logger'], warn_msg, final_level), ('warning', warn_level['warning'], warn_msg, final_level), - ('error', warn_level['error'], warn_msg, final_level)]}}} + ('logger', warn_level['logger'], warn_msg, final_level)]}}} else: inst_dict['inst_module']._clean_warn = { inst_dict['inst_id']: {inst_dict['tag']: {clean_level: [ @@ -194,18 +194,14 @@ def test_clean_with_warnings(self, clean_level, change, warn_type, # Set the additional Instrument kwargs if 'kwargs' in inst_dict.keys(): - if 'test_clean_kwarg' in inst_dict['kwargs']: - inst_dict['kwargs']['test_clean_kwarg']['change'] = final_level - else: - inst_dict['kwargs']['test_clean_kwarg'] = {'change': - final_level} + # Ensure the test instrument cleaning kwarg is reset + inst_dict['kwargs']['test_clean_kwarg'] = {'change': final_level} else: inst_dict['kwargs'] = {'test_clean_kwarg': {'change': final_level}} if warn_type == 'mult': inst_dict['kwargs']['test_clean_kwarg']['logger'] = warn_msg inst_dict['kwargs']['test_clean_kwarg']['warning'] = warn_msg - inst_dict['kwargs']['test_clean_kwarg']['error'] = warn_msg else: inst_dict['kwargs']['test_clean_kwarg'][warn_type] = warn_msg From 0c3798ace24656e82097e4922ece36e92f901b59 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 18:59:07 -0400 Subject: [PATCH 020/365] TST: improved error message Improved the logging error message in the clean warning test to be more useful. --- pysat/tests/classes/cls_instrument_library.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 4f73cf428..ff14bdcf9 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -437,7 +437,9 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Test the returned message out_msg = caplog.text - assert out_msg.find(clean_method_msg) >= 0 + assert out_msg.find(clean_method_msg) >= 0, \ + "{:s} not in output: {:s}".format( + clean_method_msg, out_msg) elif clean_method == 'warning': # A warning message is expected with warnings.catch_warnings(record=True) as war: From c769ba87e298a1e76206f30d84ab94d3e0740734 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 10 Jul 2023 15:36:14 -0400 Subject: [PATCH 021/365] MAINT: fixed spelling Fixed spelling and line length in the docs. --- docs/new_instrument.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index f6d5a2cc1..1db654491 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -774,8 +774,8 @@ allows multiple types of warning messages to be tested for a given _clean_warn = {inst_id: {tag: {'dusty': [ ('logger', 'WARN', "I am a warning!", 'clean'), ('warning', UserWarning, - 'I am a serios warning!', 'dusty'), - ('error', ValueError, 'I am an error', 'dusty')]} + 'I am a serious warning!', 'dusty'), + ('error', ValueError, "I'm an error", 'dusty')]} for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} From 1eadc88158225a58a92ae674a9520cf2d06f52ba Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 10 Jul 2023 15:49:00 -0400 Subject: [PATCH 022/365] ENH: improved clean warning test Improved the clean warning test by: - Creating a function for testing/setting the strict time flag, - Removing 'none' from the clean level parametrization, - Fixing the docstrings, - Adding more comments, and - Adding pytest skip statements. --- pysat/tests/classes/cls_instrument_library.py | 110 +++++++++++------- 1 file changed, 71 insertions(+), 39 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index ff14bdcf9..481cddc74 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -73,6 +73,50 @@ def initialize_test_inst_and_date(inst_dict): return test_inst, date +def set_strict_time_flag(test_inst, date, raise_error=False): + """Ensure the strict time flag does not interfere with other tests. + + Parameters + ---------- + test_inst : pysat.Instrument + Test instrument + date : dt.datetime + Date for loading data + raise_error : bool + Raise the load error if it is not the strict time flag error + (default=False) + + """ + + try: + test_inst.load(date=date, use_header=True) + except Exception as err: + # Catch all potential input errors, and only ensure that the one caused + # by the strict time flag is prevented from occurring on future load + # calls. + if str(err).find('Loaded data') > 0: + # Change the flags that may have caused the error to be raised, to + # see if it the strict time flag + test_inst.strict_time_flag = False + orig_clean_level = str(test_inst.clean_level) + test_inst.clean_level = 'none' + + # Evaluate the warning + with warnings.catch_warnings(record=True) as war: + test_inst.load(date=date, use_header=True) + + assert len(war) >= 1 + categories = [war[j].category for j in range(len(war))] + assert UserWarning in categories + + # Reset the clean level + test_inst.clean_level = orig_clean_level + elif raise_error: + raise err + + return + + class InstLibTests(object): """Provide standardized tests for pysat instrument libraries. @@ -340,20 +384,10 @@ def test_load(self, clean_level, inst_dict): test_inst.clean_level = clean_level target = 'Fake Data to be cleared' test_inst.data = [target] - try: - test_inst.load(date=date, use_header=True) - except ValueError as verr: - # Check if instrument is failing due to strict time flag - if str(verr).find('Loaded data') > 0: - test_inst.strict_time_flag = False - with warnings.catch_warnings(record=True) as war: - test_inst.load(date=date, use_header=True) - assert len(war) >= 1 - categories = [war[j].category for j in range(0, len(war))] - assert UserWarning in categories - else: - # If error message does not match, raise error anyway - raise ValueError(verr) + + # Make sure the strict time flag doesn't interfere with + # the cleaning tests + set_strict_time_flag(test_inst, date, raise_error=True) # Make sure fake data is cleared assert target not in test_inst.data @@ -372,7 +406,7 @@ def test_load(self, clean_level, inst_dict): # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options @pytest.mark.download - @pytest.mark.parametrize("clean_level", ['none', 'dirty', 'dusty', 'clean']) + @pytest.mark.parametrize("clean_level", ['dirty', 'dusty', 'clean']) def test_clean_warn(self, clean_level, inst_dict, caplog): """Test that appropriate warnings and errors are raised when cleaning. @@ -384,42 +418,27 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): Dictionary containing info to instantiate a specific instrument. Set automatically from instruments['download'] when `initialize_test_package` is run. - **kwags : dict """ + # Not all Instruments have warning messages to test, only run tests + # when the desired test attribute is defined if hasattr(inst_dict['inst_module'], '_clean_warn'): clean_warn = inst_dict['inst_module']._clean_warn[ inst_dict['inst_id']][inst_dict['tag']] + # Cleaning warnings may vary by clean level, test the warning + # messages at the current clean level, specified by `clean_level` if clean_level in clean_warn.keys(): # Only need to test if there are clean warnings for this level test_inst, date = initialize_test_inst_and_date(inst_dict) clean_warnings = clean_warn[clean_level] - try: - test_inst.load(date=date, use_header=True) - except Exception as err: - # Catch all potential input errors, and only ensure that - # the one caused by the strict time flag is prevented from - # occurring on future load calls. - if str(err).find('Loaded data') > 0: - # Change the flags that may have caused - # the error to be raised, to see if it the - # strict time flag - test_inst.strict_time_flag = False - test_inst.clean_level = 'none' - - # Evaluate the warning - with warnings.catch_warnings(record=True) as war: - test_inst.load(date=date, use_header=True) - - assert len(war) >= 1 - categories = [war[j].category for j in range(len(war))] - assert UserWarning in categories - - # Reset the clean level - test_inst.clean_level = clean_level + # Make sure the strict time flag doesn't interfere with + # the cleaning tests + set_strict_time_flag(test_inst, date) + # Cycle through each of the potential cleaning messages + # for this Instrument module, inst ID, tag, and clean level for (clean_method, clean_method_level, clean_method_msg, final_level) in clean_warnings: if len(test_inst.files.files) > 0: @@ -468,6 +487,19 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Make sure fake data is cleared assert target not in test_inst.data + else: + pytest.skip("".join(["Can't test clean warnings for ", + "Instrument ", + repr(inst_dict['inst_module']), + " level ", clean_level, + " (no downloaded files)"])) + else: + pytest.skip("".join(["No clean warnings for Instrument ", + repr(inst_dict['inst_module']), " level ", + clean_level])) + else: + pytest.skip("No clean warnings for Instrument {:s}".format( + repr(inst_dict['inst_module']))) return From 35fdc51cbf04d236d57780dd279efbce7357fca6 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 11 Jul 2023 11:46:00 -0400 Subject: [PATCH 023/365] DOC: Update pysat/tests/classes/cls_instrument_library.py Fixed comments to be accurate. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysat/tests/classes/cls_instrument_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 481cddc74..d3e9dde11 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -386,7 +386,7 @@ def test_load(self, clean_level, inst_dict): test_inst.data = [target] # Make sure the strict time flag doesn't interfere with - # the cleaning tests + # the load tests set_strict_time_flag(test_inst, date, raise_error=True) # Make sure fake data is cleared From 31313412fa67ea201881679ab0b9f64909aa7a48 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 11 Jul 2023 14:04:49 -0400 Subject: [PATCH 024/365] TST: added clean_level input Allow the strict flag function to run with or without the clean method. --- pysat/tests/classes/cls_instrument_library.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index d3e9dde11..35cad0ffb 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -73,7 +73,7 @@ def initialize_test_inst_and_date(inst_dict): return test_inst, date -def set_strict_time_flag(test_inst, date, raise_error=False): +def set_strict_time_flag(test_inst, date, raise_error=False, clean_off=True): """Ensure the strict time flag does not interfere with other tests. Parameters @@ -85,6 +85,9 @@ def set_strict_time_flag(test_inst, date, raise_error=False): raise_error : bool Raise the load error if it is not the strict time flag error (default=False) + clean_off : bool + Turn off the clean method when re-loading data and testing the + strict time flag (default=True) """ @@ -98,8 +101,11 @@ def set_strict_time_flag(test_inst, date, raise_error=False): # Change the flags that may have caused the error to be raised, to # see if it the strict time flag test_inst.strict_time_flag = False - orig_clean_level = str(test_inst.clean_level) - test_inst.clean_level = 'none' + + if clean_off: + # Turn the clean method off + orig_clean_level = str(test_inst.clean_level) + test_inst.clean_level = 'none' # Evaluate the warning with warnings.catch_warnings(record=True) as war: @@ -109,8 +115,9 @@ def set_strict_time_flag(test_inst, date, raise_error=False): categories = [war[j].category for j in range(len(war))] assert UserWarning in categories - # Reset the clean level - test_inst.clean_level = orig_clean_level + if clean_off: + # Reset the clean level + test_inst.clean_level = orig_clean_level elif raise_error: raise err @@ -380,14 +387,15 @@ def test_load(self, clean_level, inst_dict): test_inst, date = initialize_test_inst_and_date(inst_dict) if len(test_inst.files.files) > 0: - # Set Clean Level + # Set the clean level test_inst.clean_level = clean_level target = 'Fake Data to be cleared' test_inst.data = [target] # Make sure the strict time flag doesn't interfere with - # the load tests - set_strict_time_flag(test_inst, date, raise_error=True) + # the load tests, and re-run with desired clean level + set_strict_time_flag(test_inst, date, raise_error=True, + clean_off=False) # Make sure fake data is cleared assert target not in test_inst.data From a356931c7d0c68793c3da1bc42e01c716e37c14b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 12 Jul 2023 13:26:43 -0400 Subject: [PATCH 025/365] STY: rename function Rename the test function as per discussion. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysat/tests/classes/cls_instrument_library.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 35cad0ffb..b72ee011d 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -73,8 +73,8 @@ def initialize_test_inst_and_date(inst_dict): return test_inst, date -def set_strict_time_flag(test_inst, date, raise_error=False, clean_off=True): - """Ensure the strict time flag does not interfere with other tests. +def load_and_set_strict_time_flag(test_inst, date, raise_error=False, clean_off=True): + """Load data and set the strict time flag if needed for other tests. Parameters ---------- @@ -394,8 +394,8 @@ def test_load(self, clean_level, inst_dict): # Make sure the strict time flag doesn't interfere with # the load tests, and re-run with desired clean level - set_strict_time_flag(test_inst, date, raise_error=True, - clean_off=False) + load_and_set_strict_time_flag(test_inst, date, raise_error=True, + clean_off=False) # Make sure fake data is cleared assert target not in test_inst.data @@ -443,7 +443,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Make sure the strict time flag doesn't interfere with # the cleaning tests - set_strict_time_flag(test_inst, date) + load_and_set_strict_time_flag(test_inst, date) # Cycle through each of the potential cleaning messages # for this Instrument module, inst ID, tag, and clean level From 3bacdef240707f6aaf16f67376fe7d17aa5b4247 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 12 Jul 2023 13:29:17 -0400 Subject: [PATCH 026/365] STY: pep8 --- pysat/tests/classes/cls_instrument_library.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index b72ee011d..7429c7ff6 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -73,7 +73,8 @@ def initialize_test_inst_and_date(inst_dict): return test_inst, date -def load_and_set_strict_time_flag(test_inst, date, raise_error=False, clean_off=True): +def load_and_set_strict_time_flag(test_inst, date, raise_error=False, + clean_off=True): """Load data and set the strict time flag if needed for other tests. Parameters From b04eaac52afe8544bd017148b75d828630f51024 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 19 Jul 2023 13:12:02 -0400 Subject: [PATCH 027/365] ENH: updated ValueError message Updated a ValueError message to be more informative, as valid tags vary with inst_id. --- pysat/_instrument.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 744b6c302..d9964ed39 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1396,10 +1396,12 @@ def _assign_attrs(self, by_name=False): raise ValueError(estr) if self.tag not in self.inst_module.inst_ids[self.inst_id]: - tag_str = ', '.join([tkey.__repr__() for tkey - in self.inst_module.inst_ids[self.inst_id]]) + tag_id_str = repr(self.inst_module.inst_ids[self.inst_id]).replace( + "{", "'inst ID': ['tag'] combinations are: ") + tag_id_str = tag_id_str.replace("}", "") estr = ''.join(("'", self.tag, "' is not one of the supported ", - 'tags. Supported tags are: ', tag_str, '.')) + "tags for inst ID ['", self.inst_id, "']. ", + 'Supported ', tag_id_str)) raise ValueError(estr) # Assign the Instrument methods From 40e16dccb1a259e81948017f1b598e92406321fb Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 19 Jul 2023 13:12:30 -0400 Subject: [PATCH 028/365] TST: updated ValueError test Updated the ValueError test message for a bad tag to reflect the changes in the message. --- pysat/tests/classes/cls_instrument_property.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_property.py b/pysat/tests/classes/cls_instrument_property.py index 554cd7fef..dcb0b262c 100644 --- a/pysat/tests/classes/cls_instrument_property.py +++ b/pysat/tests/classes/cls_instrument_property.py @@ -693,7 +693,7 @@ def test_optional_unknown_data_dir(self, caplog): [({'inst_id': 'invalid_inst_id'}, "'invalid_inst_id' is not one of the supported inst_ids."), ({'inst_id': '', 'tag': 'bad_tag'}, - "'bad_tag' is not one of the supported tags.")]) + "'bad_tag' is not one of the supported tags")]) def test_error_bad_instrument_object(self, kwargs, estr): """Ensure instantiation with invalid inst_id or tag errors. From a1be583e49b3272ae5ae4070e0e1ecdc06a4f298 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 19 Jul 2023 13:13:15 -0400 Subject: [PATCH 029/365] DOC: updated changelog Updated the changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8274e2f15..f920be4b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). clean method. * Maintenance * Update link redirects in docs. + * Improved Instrument ValueError messages. [3.1.0] - 2023-05-31 -------------------- From 37b231b22ee65fc6b94a397d067103f026f9a64a Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 16:16:18 -0400 Subject: [PATCH 030/365] ENH: allow custom `concat_data` methods Allow instruments to specify their own `concat_data` methods, making allowances for multiple or non-existent time indices. --- pysat/_instrument.py | 78 +++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index d9964ed39..fc3056b0e 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -322,7 +322,7 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', # Expected function keywords exp_keys = ['list_files', 'load', 'preprocess', 'download', - 'list_remote_files', 'clean', 'init'] + 'list_remote_files', 'clean', 'init', 'concat_data'] for fkey in exp_keys: func_name = _kwargs_keys_to_func_name(fkey) func = getattr(self, func_name) @@ -568,7 +568,8 @@ def __eq__(self, other): # required their own path for equality, string comparisons! partial_funcs = ['_init_rtn', '_clean_rtn', '_preprocess_rtn', '_list_files_rtn', '_download_rtn', - '_list_remote_files_rtn', '_load_rtn'] + '_list_remote_files_rtn', '_load_rtn', + '_concat_data_rtn'] # If the type is the same then check everything that is attached to # the Instrument object. Includes attributes, methods, variables, etc. @@ -1310,7 +1311,7 @@ def _assign_attrs(self, by_name=False): methods init, preprocess, and clean functions - load, list_files, download, and list_remote_files + load, list_files, download, and list_remote_files, concat_data attributes directory_format, file_format, multi_file_day, orbit_info, and pandas_format @@ -1320,7 +1321,7 @@ def _assign_attrs(self, by_name=False): """ # Declare the standard Instrument methods and attributes inst_methods = {'required': ['init', 'clean'], - 'optional': ['preprocess']} + 'optional': ['preprocess', 'concat_data']} inst_funcs = {'required': ['load', 'list_files', 'download'], 'optional': ['list_remote_files']} inst_attrs = {'directory_format': None, 'file_format': None, @@ -2375,6 +2376,11 @@ def concat_data(self, new_data, prepend=False, **kwargs): except if the user includes a value for dim as a keyword argument. """ + # Add any concat_data kwargs + for ckey in self.kwargs['concat_data'].keys(): + if ckey not in kwargs.keys(): + kwargs[ckey] = self.kwargs['concat_data'][ckey] + # Order the data to be concatenated in a list if not isinstance(new_data, list): new_data = [new_data] @@ -2383,36 +2389,41 @@ def concat_data(self, new_data, prepend=False, **kwargs): new_data.append(self.data) else: new_data.insert(0, self.data) - - # Retrieve the appropriate concatenation function - if self.pandas_format: - # Specifically do not sort unless otherwise specified - if 'sort' not in kwargs: - kwargs['sort'] = False - concat_func = pds.concat + + if self._concat_data_rtn.__name__.find('_pass_method') == 0: + # There is no custom concat function, use the pysat standard method. + # Start by retrieving the appropriate concatenation function + if self.pandas_format: + # Specifically do not sort unless otherwise specified + if 'sort' not in kwargs: + kwargs['sort'] = False + concat_func = pds.concat + else: + # Ensure the dimensions are equal + equal_dims = True + idat = 0 + while idat < len(new_data) - 1 and equal_dims: + if new_data[idat].dims != new_data[idat + 1].dims: + equal_dims = False + idat += 1 + + if not equal_dims: + # Update the dimensions, padding data where necessary + new_data = pysat.utils.coords.expand_xarray_dims( + new_data, self.meta, exclude_dims=[self.index.name]) + + # Specify the dimension, if not otherwise specified + if 'dim' not in kwargs: + kwargs['dim'] = self.index.name + + # Set the concat function + concat_func = xr.concat + + # Assign the concatenated data to the instrument + self.data = concat_func(new_data, **kwargs) else: - # Ensure the dimensions are equal - equal_dims = True - idat = 0 - while idat < len(new_data) - 1 and equal_dims: - if new_data[idat].dims != new_data[idat + 1].dims: - equal_dims = False - idat += 1 - - if not equal_dims: - # Update the dimensions, padding data where necessary - new_data = pysat.utils.coords.expand_xarray_dims( - new_data, self.meta, exclude_dims=['time']) - - # Specify the dimension, if not otherwise specified - if 'dim' not in kwargs: - kwargs['dim'] = self.index.name - - # Set the concat function - concat_func = xr.concat - - # Assign the concatenated data to the instrument - self.data = concat_func(new_data, **kwargs) + self._concat_data_rtn(new_data, **kwargs) + return def custom_attach(self, function, at_pos='end', args=None, kwargs=None): @@ -3328,7 +3339,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, if not self.empty: if (self.index[-1] == last_pad) & (not want_last_pad): self.data = self[:-1] - else: # If self.pad is False, load single day self.data, meta = self._load_data(date=self.date, fid=self._fid, From 1bd63537bc655eb7348af57982ee803b56f181f3 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 16:28:52 -0400 Subject: [PATCH 031/365] DOC: update changelog Add a description of the changes to the log. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f920be4b3..c8b5df1c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * New Features * Added tests for warnings, logging messages, and errors in the Instrument clean method. + * Allow Instruments to define custom `concat_data` methods. * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. From 4ea3cbe13c77bb9788ac74612b28e628604672b8 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 16:29:19 -0400 Subject: [PATCH 032/365] DOC: update new instrument guidelines Update the new instrument guidelines to include the `concat_data` method. --- docs/new_instrument.rst | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 1db654491..9a0255ef6 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -546,10 +546,11 @@ If provided, :py:mod:`pysat` supports the definition and use of keywords for an instrument module so that users may define their preferred default values. A custom keyword for an instrument module must be defined in each function that will receive that keyword argument if provided by the user. All instrument -functions, :py:func:`init`, :py:func:`preprocess`, :py:func:`load`, -:py:func:`clean`, :py:func:`list_files`, :py:func:`list_remote_files`, and -:py:func:`download` support custom keywords. The same keyword may be used in -more than one function but the same value will be passed to each. +functions, :py:func:`init`, :py:func:`preprocess`, :py:func:`concat_data`, +:py:func:`load`, :py:func:`clean`, :py:func:`list_files`, +:py:func:`list_remote_files`, and :py:func:`download` support custom keywords. +The same keyword may be used in more than one function but the same value will +be passed to each. An example :py:func:`load` function definition with two custom keyword arguments. @@ -650,6 +651,25 @@ The user can search for subsets of files through optional keywords, such as: inst.remote_file_list(year=2019) inst.remote_file_list(year=2019, month=1, day=1) +concat_data +^^^^^^^^^^^ + +Combines data from multiple Instruments of the same type, used internally to +combine data from different load periods. The default method concatonates data +using the :py:attr:`inst.index` name. However, some data sets have multiple +different time indices along which data should be concatonated. In such cases +(e.g., TIMED-GUVI SDR-Imaging data from :py:mod:`pysatNASA`), a custom +:py:meth:`concat_data` method must be supplied. If available, this method +will be used instead of the default +:py:meth:`~pysat._instrument.Instrument.concat_data`, after the default +method handles the prepending of the data that needs to be combined. + +.. code:: python + + def concat_data(self, new_data, **kwargs): + # Perform custom concatonation here, updating self.data + return + Logging ------- From bd82e10ad4cda7b1ca03738438b24ba69832f708 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 20 Jul 2023 16:36:13 -0400 Subject: [PATCH 033/365] MAINT: added TODO for loading Currently, secondary time data is not down-selected in the load function. Added TODO statements in the places where data is being selected by time as reminders to address this issue. --- pysat/_instrument.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index fc3056b0e..5fec219d1 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -3309,6 +3309,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # __getitem__ used below to get data from instrument object. # Details for handling pandas and xarray are different and # handled by __getitem__. + # TODO: fix data selection for other time indices self.data = self[first_pad:temp_time] if not self.empty: if self.index[-1] == temp_time: @@ -3324,6 +3325,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # Pad data using access mechanisms that work for both pandas # and xarray self.data = self._next_data.copy() + # TODO: fix data selection for other time indices self.data = self[temp_time:last_pad] if len(self.index) > 0: if (self.index[0] == temp_time): @@ -3333,6 +3335,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, self.data = stored_data if len(self.index) > 0: + # TODO: fix data selection for other time indices self.data = self[first_pad:last_pad] # Want exclusive end slicing behavior from above @@ -3439,6 +3442,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # Remove the excess data padding, if any applied if (self.pad is not None) & (not self.empty) & (not verifyPad): + # TODO: fix data selection for other time indices self.data = self[first_time: last_time] if not self.empty: if (self.index[-1] == last_time) & (not want_last_pad): From fdad8f4fe6e50434d66f2a2b09c4e8c12ab1bbc3 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 25 Jul 2023 11:19:12 -0400 Subject: [PATCH 034/365] BUG: fixed xarray time selection Fixed xarray time selection for multiple time indices. --- pysat/_instrument.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 5fec219d1..290c4e75e 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -868,12 +868,19 @@ def __getitem_xarray__(self, key): """ if 'Epoch' in self.data.indexes: - epoch_name = 'Epoch' + epoch_names = ['Epoch'] elif 'time' in self.data.indexes: - epoch_name = 'time' + epoch_names = ['time'] else: return xr.Dataset(None) + # Find secondary time indexes that may need to be sliced + if len(self.data.indexes) > 1: + for ind in self.data.indexes.keys(): + if(ind != epoch_names[0] and self.data.indexes[ind].dtype + == self.data.indexes[epoch_names[0]].dtype): + epoch_names.append(ind) + if isinstance(key, tuple): if len(key) == 2: # Support slicing time, variable name @@ -885,7 +892,8 @@ def __getitem_xarray__(self, key): data_subset = self.data[key[1]] # If the input is a tuple, `key[0]` must be linked to the epoch. - key_dict = {'indexers': {epoch_name: key[0]}} + key_dict = {'indexers': {epoch_name: key[0] + for epoch_name in epoch_names}} try: # Assume key[0] is an integer return data_subset.isel(**key_dict) @@ -934,7 +942,8 @@ def __getitem_xarray__(self, key): except (TypeError, KeyError, ValueError): # If that didn't work, likely need to use `isel` or `sel` # Link key to the epoch. - key_dict = {'indexers': {epoch_name: key}} + key_dict = {'indexers': {epoch_name: key + for epoch_name in epoch_names}} try: # Try to get all data variables, but for a subset of time # using integer indexing @@ -3309,7 +3318,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # __getitem__ used below to get data from instrument object. # Details for handling pandas and xarray are different and # handled by __getitem__. - # TODO: fix data selection for other time indices self.data = self[first_pad:temp_time] if not self.empty: if self.index[-1] == temp_time: @@ -3325,7 +3333,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # Pad data using access mechanisms that work for both pandas # and xarray self.data = self._next_data.copy() - # TODO: fix data selection for other time indices self.data = self[temp_time:last_pad] if len(self.index) > 0: if (self.index[0] == temp_time): @@ -3335,7 +3342,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, self.data = stored_data if len(self.index) > 0: - # TODO: fix data selection for other time indices self.data = self[first_pad:last_pad] # Want exclusive end slicing behavior from above @@ -3442,7 +3448,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # Remove the excess data padding, if any applied if (self.pad is not None) & (not self.empty) & (not verifyPad): - # TODO: fix data selection for other time indices self.data = self[first_time: last_time] if not self.empty: if (self.index[-1] == last_time) & (not want_last_pad): From d2b242041cd6b6b137c130d7c58dcd1052cd2016 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 25 Jul 2023 11:22:21 -0400 Subject: [PATCH 035/365] STY: remove whitespace Remove extra whitespace. --- pysat/_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 290c4e75e..ae04bb8e6 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -2398,7 +2398,7 @@ def concat_data(self, new_data, prepend=False, **kwargs): new_data.append(self.data) else: new_data.insert(0, self.data) - + if self._concat_data_rtn.__name__.find('_pass_method') == 0: # There is no custom concat function, use the pysat standard method. # Start by retrieving the appropriate concatenation function From 35e2341d187d6b9d054982cfc9fab2e646e66e99 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 25 Jul 2023 12:08:36 -0400 Subject: [PATCH 036/365] ENH: added extra time dimension to test inst Added an optional extra time dimension to the xarray test instrument. --- pysat/instruments/methods/testing.py | 71 ++++++++++++++++++++++++++-- pysat/instruments/pysat_ndtesting.py | 38 +++++++++++---- 2 files changed, 97 insertions(+), 12 deletions(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index fe66275bf..e60be7171 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -7,6 +7,7 @@ import pandas as pds import time import warnings +import xarray as xr import pysat from pysat.utils import NetworkLock @@ -82,7 +83,67 @@ def clean(self, test_clean_kwarg=None): return -# Optional method +# Optional methods +def concat_data(self, new_data, extra_time_dims=None, **kwargs): + """Concatonate data to self.data for extra time dimensions. + + Parameters + ---------- + new_data : xarray.Dataset or list of such objects + New data objects to be concatonated + extra_time_dims : list-like or NoneType + List of extra time dimensions that require concatonation (default=None) + **kwargs : dict + Optional keyword arguments passed to xr.concat + + """ + # Establish the time dimensions + time_dims = [self.index.name] + + if extra_time_dims is not None: + time_dims.extend(list(extra_time_dims)) + + # Concatonate using the appropriate method for the number of time + # dimensions + if len(time_dims) == 1: + # There is only one time dimensions, but other dimensions may + # need to be adjusted + new_data = pysat.utils.coords.expand_xarray_dims( + new_data, self.meta, exclude_dims=time_dims) + + # Combine the data + self.data = xr.combine_by_coords(new_data, **kwargs) + else: + inners = None + for ndata in new_data: + # Separate into inner datasets + inner_keys = {dim: [key for key in ndata.keys() + if dim in ndata[key].dims] for dim in time_dims} + inner_dat = {dim: ndata.get(inner_keys[dim]) for dim in time_dims} + + # Add 'single_var's into 'time' dataset to keep track + sv_keys = [val.name for val in ndata.values() + if 'single_var' in val.dims] + singlevar_set = ndata.get(sv_keys) + inner_dat[self.index.name] = xr.merge([inner_dat[self.index.name], + singlevar_set]) + + # Concatenate along desired dimension with previous data + if inners is None: + # No previous data, assign the data separated by dimension + inners = dict(inner_dat) + else: + # Concatenate with existing data + inners = {dim: xr.concat([inners[dim], inner_dat[dim]], + dim=dim) for dim in time_dims} + + # Combine all time dimensions + if inners is not None: + data_list = [inners[dim] for dim in time_dims] + self.data = xr.merge(data_list) + return + + def preprocess(self, test_preprocess_kwarg=None): """Perform standard preprocessing. @@ -96,12 +157,12 @@ def preprocess(self, test_preprocess_kwarg=None): Testing keyword (default=None) """ - self.test_preprocess_kwarg = test_preprocess_kwarg return +# Utility functions def initialize_test_meta(epoch_name, data_keys): """Initialize meta data for test instruments. @@ -213,8 +274,12 @@ def initialize_test_meta(epoch_name, data_keys): 'units': 'km', 'meta': alt_profile_meta} + # Optional and standard metadata for xarray + for var in data_keys: + if var.find('variable_profiles') == 0: + meta[var] = {'desc': 'Profiles with variable altitude.'} + # Standard metadata required for xarray. - meta['variable_profiles'] = {'desc': 'Profiles with variable altitude.'} meta['profile_height'] = {'value_min': 0, 'value_max': 14, 'fill': -1, 'desc': 'Altitude of profile data.'} meta['variable_profile_height'] = {'long_name': 'Variable Profile Height'} diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 581f44446..6f8e87842 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -17,6 +17,8 @@ tags = {'': 'Regular testing data set'} inst_ids = {'': ['']} _test_dates = {'': {'': dt.datetime(2009, 1, 1)}} +_test_load_opt = {'': {'': [{'num_extra_time_coords': 0}, + {'num_extra_time_coords': 1}]}} epoch_name = u'time' @@ -27,12 +29,14 @@ clean = mm_test.clean # Optional method, preprocess +concat_data = mm_test.concat_data preprocess = mm_test.preprocess def load(fnames, tag='', inst_id='', non_monotonic_index=False, non_unique_index=False, malformed_index=False, start_time=None, - num_samples=864, test_load_kwarg=None, max_latitude=90.): + num_samples=864, test_load_kwarg=None, max_latitude=90.0, + num_extra_time_coords=0): """Load the test files. Parameters @@ -65,7 +69,9 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, correctly. (default=None) max_latitude : float Latitude simulated as `max_latitude` * cos(theta(t))`, where - theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.). + theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.0) + num_extra_time_coords : int + Number of extra time coordinates to include. (default=0) Returns ------- @@ -169,6 +175,14 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, data.coords['y'] = (('y'), np.arange(17)) data.coords['z'] = (('z'), np.arange(15)) + # Add extra time coords + for i in range(num_extra_time_coords): + ckey = 'time{:d}'.format(i) + tindex = data.indexes[epoch_name][:-1 * (i + 1)] + data.coords[ckey] = ( + (ckey), [itime + dt.timedelta(microseconds=1 + i) + for i, itime in enumerate(tindex)]) + # Create altitude 'profile' at each location to simulate remote data num = len(data['uts']) data['profiles'] = ( @@ -188,14 +202,20 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, data['images'] = ((epoch_name, 'x', 'y'), data['dummy3'].values[ :, np.newaxis, np.newaxis] * np.ones((num, 17, 17))) - data.coords['image_lat'] = \ - ((epoch_name, 'x', 'y'), - np.arange(17)[np.newaxis, - np.newaxis, - :] * np.ones((num, 17, 17))) + data.coords['image_lat'] = ((epoch_name, 'x', 'y'), + np.arange(17)[np.newaxis, np.newaxis, :] + * np.ones((num, 17, 17))) data.coords['image_lon'] = ((epoch_name, 'x', 'y'), - np.arange(17)[np.newaxis, np.newaxis, - :] * np.ones((num, 17, 17))) + np.arange(17)[np.newaxis, np.newaxis, :] + * np.ones((num, 17, 17))) + + # There may be data that depends on alternate time indices + for i in range(num_extra_time_coords): + alt_epoch = 'time{:d}'.format(i) + data['variable_profiles{:d}'.format(i)] = ( + (alt_epoch, 'z'), np.full(shape=(data.coords[alt_epoch].shape[0], + data.coords['z'].shape[0]), + fill_value=100.0 + i)) meta = mm_test.initialize_test_meta(epoch_name, data.keys()) return data, meta From fcf146236df0d9931c7a9e34e77ef2f0d3a87ef3 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 25 Jul 2023 12:59:33 -0400 Subject: [PATCH 037/365] BUG: fixed custom concat data function Fixed a bug in the custom concat data function, where the wrong xarray function was called. --- pysat/instruments/methods/testing.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index e60be7171..2ab416b71 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -111,8 +111,11 @@ def concat_data(self, new_data, extra_time_dims=None, **kwargs): new_data = pysat.utils.coords.expand_xarray_dims( new_data, self.meta, exclude_dims=time_dims) - # Combine the data - self.data = xr.combine_by_coords(new_data, **kwargs) + # Specify the dimension, if not otherwise specified + if 'dim' not in kwargs: + kwargs['dim'] = self.index.name + + self.data = xr.concat(new_data, **kwargs) else: inners = None for ndata in new_data: From 78e52074a6136db166dc699d1a367dc84ba50c63 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 25 Jul 2023 13:27:56 -0400 Subject: [PATCH 038/365] BUG: fixed extra time selection with limited range bug Fixed a bug encountered using pysatNASA TIMED GUVI SDR-spectrograph low-res data, in which the extra time dimensions span a fraction of the day, and so the pandas DatetimeIndex index finding can't work. --- pysat/_instrument.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index ae04bb8e6..e5ee7cd17 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -950,7 +950,21 @@ def __getitem_xarray__(self, key): return self.data.isel(**key_dict) except (KeyError, TypeError): # Try to get a subset of time, using label based indexing - return self.data.sel(**key_dict) + try: + return self.data.sel(**key_dict) + except KeyError as kerr: + if str(kerr).find('Timestamp') >= 0 and len( + epoch_names) > 0: + # The problem is probably coming from a limited + # time range in the ancillery epochs, remove them + # from selection + pysat.logger.warning( + ''.join(['Removing ', repr(epoch_names[1:]), + ' dimensions from data selection'])) + key_dict = {'indexers': {epoch_names[0]: key}} + return self.data.sel(**key_dict) + else: + raise kerr def __setitem__(self, key, new_data): """Set data in `pysat.Instrument` object. From 206da7f5ab50960980b658c9522b116f3257adb4 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 25 Jul 2023 13:29:25 -0400 Subject: [PATCH 039/365] STY: removed extra whitespace Removed extra whitespace. --- pysat/instruments/methods/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 2ab416b71..8ce6761ea 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -114,7 +114,7 @@ def concat_data(self, new_data, extra_time_dims=None, **kwargs): # Specify the dimension, if not otherwise specified if 'dim' not in kwargs: kwargs['dim'] = self.index.name - + self.data = xr.concat(new_data, **kwargs) else: inners = None From 251fc5a39fcb187b5059e1fa943f1e8e9b2f3c90 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 1 Aug 2023 13:55:42 -0400 Subject: [PATCH 040/365] TST: add multi-day load --- pysat/tests/classes/cls_instrument_library.py | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 7429c7ff6..66d19aa94 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -74,7 +74,7 @@ def initialize_test_inst_and_date(inst_dict): def load_and_set_strict_time_flag(test_inst, date, raise_error=False, - clean_off=True): + clean_off=True, concat=False): """Load data and set the strict time flag if needed for other tests. Parameters @@ -89,11 +89,19 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False, clean_off : bool Turn off the clean method when re-loading data and testing the strict time flag (default=True) + concat : bool + If True, load multiple days to concat. If False, load single day. + (default=False) """ + kwargs = {'use_header': True} + + if concat: + kwargs['end_date'] = date + dt.timedelta(days=1) + try: - test_inst.load(date=date, use_header=True) + test_inst.load(date=date, **kwargs) except Exception as err: # Catch all potential input errors, and only ensure that the one caused # by the strict time flag is prevented from occurring on future load @@ -110,7 +118,7 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False, # Evaluate the warning with warnings.catch_warnings(record=True) as war: - test_inst.load(date=date, use_header=True) + test_inst.load(date=date, **kwargs) assert len(war) >= 1 categories = [war[j].category for j in range(len(war))] @@ -362,7 +370,8 @@ def test_download(self, inst_dict): dl_dict = inst_dict['user_info'] else: dl_dict = {} - test_inst.download(date, date, **dl_dict) + # Note this will download two consecutive days + test_inst.download(date, **dl_dict) assert len(test_inst.files.files) > 0 return @@ -415,6 +424,42 @@ def test_load(self, clean_level, inst_dict): # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options @pytest.mark.download + def test_load_multiple_days(self, inst_dict): + """Test that instruments load at each cleaning level. + + Parameters + ---------- + clean_level : str + Cleanliness level for loaded instrument data. + inst_dict : dict + Dictionary containing info to instantiate a specific instrument. + Set automatically from instruments['download'] when + `initialize_test_package` is run. + + """ + + test_inst, date = initialize_test_inst_and_date(inst_dict) + if len(test_inst.files.files) > 0: + # Set the clean level + target = 'Fake Data to be cleared' + test_inst.data = [target] + + # Make sure the strict time flag doesn't interfere with + # the load tests, and re-run with desired clean level + load_and_set_strict_time_flag(test_inst, date, raise_error=True, + clean_off=False, concat=True) + + # Make sure fake data is cleared + assert target not in test_inst.data + + assert not test_inst.empty + else: + pytest.skip("Download data not available") + + return + + @pytest.mark.second + @pytest.mark.load_options @pytest.mark.parametrize("clean_level", ['dirty', 'dusty', 'clean']) def test_clean_warn(self, clean_level, inst_dict, caplog): """Test that appropriate warnings and errors are raised when cleaning. From b1fa4244068aada26b3a5b850737ac8eb058c7f9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 1 Aug 2023 13:56:01 -0400 Subject: [PATCH 041/365] STY: remove old marks --- pysat/tests/classes/cls_instrument_library.py | 6 ------ pysat/tests/test_instruments.py | 9 --------- 2 files changed, 15 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 66d19aa94..5436312eb 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -376,10 +376,7 @@ def test_download(self, inst_dict): return @pytest.mark.second - # Need to maintain download mark for backwards compatibility. - # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options - @pytest.mark.download @pytest.mark.parametrize("clean_level", ['none', 'dirty', 'dusty', 'clean']) def test_load(self, clean_level, inst_dict): """Test that instruments load at each cleaning level. @@ -420,10 +417,7 @@ def test_load(self, clean_level, inst_dict): return @pytest.mark.second - # Need to maintain download mark for backwards compatibility. - # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options - @pytest.mark.download def test_load_multiple_days(self, inst_dict): """Test that instruments load at each cleaning level. diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index d5cede4e2..a7eaabcc0 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -266,15 +266,6 @@ def test_old_initialize_inst_and_date(self): testing.eval_warnings(war, self.warn_msgs) return - def test_old_pytest_mark_presence(self): - """Test that pytest mark is backwards compatible.""" - - n_args = len(InstLibTests.test_load.pytestmark) - mark_names = [InstLibTests.test_load.pytestmark[j].name - for j in range(0, n_args)] - - assert "download" in mark_names - @pytest.mark.parametrize("inst_module", ['pysat_testing2d', 'pysat_testing_xarray', 'pysat_testing2d_xarray']) From 17b4ec4a1ac2e81ed57f478068e9f3d151e53208 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 1 Aug 2023 13:56:59 -0400 Subject: [PATCH 042/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b5df1c0..26c8eb1b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added tests for warnings, logging messages, and errors in the Instrument clean method. * Allow Instruments to define custom `concat_data` methods. + * Added test for loading multiple files and concatenating. * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. From ab2bacf8c856aa456ead56d801792b54a4fa228a Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 2 Aug 2023 10:05:02 -0400 Subject: [PATCH 043/365] BUG: end date usage --- pysat/tests/classes/cls_instrument_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 5436312eb..4bbba44ec 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -98,7 +98,7 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False, kwargs = {'use_header': True} if concat: - kwargs['end_date'] = date + dt.timedelta(days=1) + kwargs['end_date'] = date + dt.timedelta(days=2) try: test_inst.load(date=date, **kwargs) From 9554dff264b05448e8396088e6d74731ea186a50 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 2 Aug 2023 15:37:22 -0400 Subject: [PATCH 044/365] BUG: changed time trigger method Changed additional time index to trigger on different tag instead of a load option. --- pysat/instruments/pysat_ndtesting.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 6f8e87842..373815c68 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -14,11 +14,9 @@ name = 'ndtesting' pandas_format = False -tags = {'': 'Regular testing data set'} -inst_ids = {'': ['']} -_test_dates = {'': {'': dt.datetime(2009, 1, 1)}} -_test_load_opt = {'': {'': [{'num_extra_time_coords': 0}, - {'num_extra_time_coords': 1}]}} +tags = {'': 'Regular testing data set', 'two_times': 'Two time indices'} +inst_ids = {'': [tag for tag in tags.keys()]} +_test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}} epoch_name = u'time' @@ -35,8 +33,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, non_unique_index=False, malformed_index=False, start_time=None, - num_samples=864, test_load_kwarg=None, max_latitude=90.0, - num_extra_time_coords=0): + num_samples=864, test_load_kwarg=None, max_latitude=90.0): """Load the test files. Parameters @@ -70,8 +67,6 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, max_latitude : float Latitude simulated as `max_latitude` * cos(theta(t))`, where theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.0) - num_extra_time_coords : int - Number of extra time coordinates to include. (default=0) Returns ------- @@ -176,6 +171,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, data.coords['z'] = (('z'), np.arange(15)) # Add extra time coords + num_extra_time_coords = 1 if tag == 'two_times' else 0 for i in range(num_extra_time_coords): ckey = 'time{:d}'.format(i) tindex = data.indexes[epoch_name][:-1 * (i + 1)] From 2dcd549ccb8b667a90480048b6aff33cb96d92e1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 2 Aug 2023 18:26:45 -0400 Subject: [PATCH 045/365] BUG: fixed concat call The concat call was incorrect, fixed it. --- pysat/instruments/pysat_ndtesting.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 373815c68..efd83aa53 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -26,11 +26,26 @@ # Clean method clean = mm_test.clean -# Optional method, preprocess -concat_data = mm_test.concat_data +# Optional methods preprocess = mm_test.preprocess +def concat_data(self, new_data, **kwargs): + """Concatonate data using the appropriate method by tag. + + Parameters + ---------- + new_data : list-like + List of xarray Datasets + + """ + extra_time_dims = ['time1'] if self.tag == 'two_times' else None + mm_test.concat_data(self, new_data, extra_time_dims=extra_time_dims, + **kwargs) + + return + + def load(fnames, tag='', inst_id='', non_monotonic_index=False, non_unique_index=False, malformed_index=False, start_time=None, num_samples=864, test_load_kwarg=None, max_latitude=90.0): From 13fa25880e9532853844aabe6279c0c5f2df1f91 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Wed, 2 Aug 2023 19:28:42 -0400 Subject: [PATCH 046/365] STY: assert multiple days --- pysat/tests/classes/cls_instrument_library.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 4bbba44ec..8a57e451e 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -33,6 +33,7 @@ class TestInstruments(InstLibTests): import datetime as dt from importlib import import_module import logging +import numpy as np import sys import tempfile import warnings @@ -445,8 +446,8 @@ def test_load_multiple_days(self, inst_dict): # Make sure fake data is cleared assert target not in test_inst.data - - assert not test_inst.empty + # Make sure more than one day has been loaded + assert len(np.unique(test_inst.index.day)) > 1 else: pytest.skip("Download data not available") From d2837f1e6b168ac28888a8fcb36f0ef7d7e8ad0a Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 3 Aug 2023 11:16:25 -0400 Subject: [PATCH 047/365] TST: added new load test Added a load test with padded data to the general instruments library of tests, to ensure that each Instrument data set runs the `concat_data` method. --- CHANGELOG.md | 1 + pysat/tests/classes/cls_instrument_library.py | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b5df1c0..f887c593c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * New Features * Added tests for warnings, logging messages, and errors in the Instrument clean method. + * Added loading test with padding for Instruments. * Allow Instruments to define custom `concat_data` methods. * Maintenance * Update link redirects in docs. diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 7429c7ff6..a566bf5cd 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -37,6 +37,7 @@ class TestInstruments(InstLibTests): import tempfile import warnings +import pandas as pds import pytest import pysat @@ -512,6 +513,63 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): return + @pytest.mark.second + # Need to maintain download mark for backwards compatibility. + # Can remove once pysat 3.1.0 is released and libraries are updated. + @pytest.mark.load_options + @pytest.mark.download + @pytest.mark.parametrize('pad', [pds.DateOffset(days=1), {'days': 1}, None, + dt.timedelta(days=1)]) + def test_load_w_pad(self, pad, inst_dict): + """Test that instruments load at each cleaning level. + + Parameters + ---------- + pad : pds.DateOffset, dict, or NoneType + Valid pad value for initializing an instrument + inst_dict : dict + Dictionary containing info to instantiate a specific instrument. + Set automatically from instruments['download'] when + `initialize_test_package` is run. + + """ + # Update the Instrument dict with the desired pad + if 'kwargs' in inst_dict.keys(): + inst_dict['kwargs']['pad'] = pad + else: + inst_dict['kwargs'] = {'pad': pad} + + # Assign the expected representation + if type(pad) in [dict]: + pad_repr = repr(pds.DateOffset(days=1)) + elif type(pad) in [dt.timedelta]: + pad_repr = "1 day, 0:00:00" + else: + pad_repr = repr(pad) + + test_inst, date = initialize_test_inst_and_date(inst_dict) + if len(test_inst.files.files) > 0: + # Make sure the strict time flag doesn't interfere with + # the load tests + load_and_set_strict_time_flag(test_inst, date, raise_error=True, + clean_off=False) + + assert not test_inst.empty + + # Evaluate the data index length + assert (test_inst.index[-1] + - test_inst.index[0]).total_seconds() < 86400.0 + + # Evaluate the recorded pad + inst_str = test_inst.__str__() + assert inst_str.find( + 'Data Padding: {:s}'.format(pad_repr)) > 0, "".join([ + "bad pad value: ", pad_repr, " not in ", inst_str]) + else: + pytest.skip("Download data not available") + + return + @pytest.mark.download def test_remote_file_list(self, inst_dict): """Test if optional list_remote_files routine exists and is callable. From cd51df70176cbd51a903102e474d78fd5bb5ecca Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 4 Aug 2023 10:08:33 -0400 Subject: [PATCH 048/365] TST: reduce number of tests Reduce the number of tests by removing two of the padding options. --- pysat/tests/classes/cls_instrument_library.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index a566bf5cd..a241659b6 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -518,8 +518,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options @pytest.mark.download - @pytest.mark.parametrize('pad', [pds.DateOffset(days=1), {'days': 1}, None, - dt.timedelta(days=1)]) + @pytest.mark.parametrize('pad', [{'day': 1}, dt.timedelta(days=1)]) def test_load_w_pad(self, pad, inst_dict): """Test that instruments load at each cleaning level. From 9f600fad649546f4ab173c7bb880ec60e2c39b3a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 4 Aug 2023 14:52:39 -0400 Subject: [PATCH 049/365] BUG: fixed dict input Fixed the dict input key name. --- pysat/tests/classes/cls_instrument_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index a241659b6..3f62d442a 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -518,7 +518,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options @pytest.mark.download - @pytest.mark.parametrize('pad', [{'day': 1}, dt.timedelta(days=1)]) + @pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)]) def test_load_w_pad(self, pad, inst_dict): """Test that instruments load at each cleaning level. From 44ecbe9640cdc0cf54dce058ee2c036e89818cfb Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 4 Aug 2023 16:17:46 -0400 Subject: [PATCH 050/365] REV: change trigger method Change trigger method for extra time dimensions back to load options to reduce number of unit tests that need to run. --- pysat/instruments/pysat_ndtesting.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index efd83aa53..71038f0f3 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -14,9 +14,11 @@ name = 'ndtesting' pandas_format = False -tags = {'': 'Regular testing data set', 'two_times': 'Two time indices'} +tags = {'': 'Regular testing data set'} inst_ids = {'': [tag for tag in tags.keys()]} _test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}} +_test_load_opt = {'': {'': [{'num_extra_time_coords': 0}, + {'num_extra_time_coords': 1}]}} epoch_name = u'time' @@ -39,7 +41,7 @@ def concat_data(self, new_data, **kwargs): List of xarray Datasets """ - extra_time_dims = ['time1'] if self.tag == 'two_times' else None + extra_time_dims = ['time1'] if 'time1' in self.variables else None mm_test.concat_data(self, new_data, extra_time_dims=extra_time_dims, **kwargs) @@ -48,7 +50,8 @@ def concat_data(self, new_data, **kwargs): def load(fnames, tag='', inst_id='', non_monotonic_index=False, non_unique_index=False, malformed_index=False, start_time=None, - num_samples=864, test_load_kwarg=None, max_latitude=90.0): + num_samples=864, test_load_kwarg=None, max_latitude=90.0, + num_extra_time_coords=0): """Load the test files. Parameters @@ -82,6 +85,8 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, max_latitude : float Latitude simulated as `max_latitude` * cos(theta(t))`, where theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.0) + num_extra_time_coords : int + Number of extra time coordinates to include. (default=0) Returns ------- @@ -186,7 +191,6 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, data.coords['z'] = (('z'), np.arange(15)) # Add extra time coords - num_extra_time_coords = 1 if tag == 'two_times' else 0 for i in range(num_extra_time_coords): ckey = 'time{:d}'.format(i) tindex = data.indexes[epoch_name][:-1 * (i + 1)] From b62e1ab86c6d8fbcb05719079c35aa7b26edd40d Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 4 Aug 2023 17:50:04 -0400 Subject: [PATCH 051/365] BUG: fixed bug in concat method Changed the method of selecting the extra time variable names in the concat_data method. --- pysat/instruments/pysat_ndtesting.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 71038f0f3..63668a197 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -41,9 +41,13 @@ def concat_data(self, new_data, **kwargs): List of xarray Datasets """ - extra_time_dims = ['time1'] if 'time1' in self.variables else None - mm_test.concat_data(self, new_data, extra_time_dims=extra_time_dims, - **kwargs) + # Select the extra time variable names + time_vars = [var for var in self.variables + if var.find('time') == 0 and var != 'time'] + if len(time_vars) == 0: + time_vars = None + + mm_test.concat_data(self, new_data, extra_time_dims=time_vars, **kwargs) return From bdd320970b2b5219516677201a73f9e501b93ae5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 7 Aug 2023 13:38:22 -0400 Subject: [PATCH 052/365] STY: improved concat definition Improved the way the custom concat method is called, simplifying it so that a wrapper is not needed. Also made sure the same kwarg is needed for this function and load, which expands the number of lines covered in the Instrument class. --- pysat/instruments/methods/testing.py | 19 ++++++---- pysat/instruments/pysat_ndtesting.py | 54 ++++++++++++---------------- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 8ce6761ea..cd3354642 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -84,24 +84,31 @@ def clean(self, test_clean_kwarg=None): # Optional methods -def concat_data(self, new_data, extra_time_dims=None, **kwargs): +def concat_data(self, new_data, num_extra_time_coords=0, **kwargs): """Concatonate data to self.data for extra time dimensions. Parameters ---------- new_data : xarray.Dataset or list of such objects New data objects to be concatonated - extra_time_dims : list-like or NoneType - List of extra time dimensions that require concatonation (default=None) + num_extra_time_coords : int + Number of extra time dimensions that require concatonation (default=0) **kwargs : dict Optional keyword arguments passed to xr.concat + Note + ---- + Expects the extra time dimensions to have a variable name that starts + with 'time', and no other dimensions to have a name that fits this format. + """ # Establish the time dimensions - time_dims = [self.index.name] + time_dims = [var for var in self.variables if var.find('time') == 0] - if extra_time_dims is not None: - time_dims.extend(list(extra_time_dims)) + if len(time_dims) != num_extra_time_coords + 1: + raise ValueError( + 'unexpected number of time dimensions: len({:}) != {:d}'.format( + time_dims, num_extra_time_coords + 1)) # Concatonate using the appropriate method for the number of time # dimensions diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 63668a197..7a63d54ba 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -29,29 +29,10 @@ clean = mm_test.clean # Optional methods +concat_data = mm_test.concat_data preprocess = mm_test.preprocess -def concat_data(self, new_data, **kwargs): - """Concatonate data using the appropriate method by tag. - - Parameters - ---------- - new_data : list-like - List of xarray Datasets - - """ - # Select the extra time variable names - time_vars = [var for var in self.variables - if var.find('time') == 0 and var != 'time'] - if len(time_vars) == 0: - time_vars = None - - mm_test.concat_data(self, new_data, extra_time_dims=time_vars, **kwargs) - - return - - def load(fnames, tag='', inst_id='', non_monotonic_index=False, non_unique_index=False, malformed_index=False, start_time=None, num_samples=864, test_load_kwarg=None, max_latitude=90.0, @@ -190,9 +171,9 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, dtype=np.int64)) # Add dummy coords - data.coords['x'] = (('x'), np.arange(17)) - data.coords['y'] = (('y'), np.arange(17)) - data.coords['z'] = (('z'), np.arange(15)) + data.coords['x'] = (('x'), np.arange(7)) + data.coords['y'] = (('y'), np.arange(7)) + data.coords['z'] = (('z'), np.arange(5)) # Add extra time coords for i in range(num_extra_time_coords): @@ -206,27 +187,36 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, num = len(data['uts']) data['profiles'] = ( (epoch_name, 'profile_height'), - data['dummy3'].values[:, np.newaxis] * np.ones((num, 15))) - data.coords['profile_height'] = ('profile_height', np.arange(15)) + data['dummy3'].values[:, np.newaxis] * np.ones( + (num, data.coords['z'].shape[0]))) + data.coords['profile_height'] = ('profile_height', + np.arange(len(data.coords['z']))) # Profiles that could have different altitude values data['variable_profiles'] = ( (epoch_name, 'z'), data['dummy3'].values[:, np.newaxis] - * np.ones((num, 15))) + * np.ones((num, data.coords['z'].shape[0]))) data.coords['variable_profile_height'] = ( - (epoch_name, 'z'), np.arange(15)[np.newaxis, :] * np.ones((num, 15))) + (epoch_name, 'z'), np.arange(data.coords['z'].shape[0])[np.newaxis, :] + * np.ones((num, data.coords['z'].shape[0]))) # Create fake image type data, projected to lat / lon at some location # from satellite. data['images'] = ((epoch_name, 'x', 'y'), data['dummy3'].values[ - :, np.newaxis, np.newaxis] * np.ones((num, 17, 17))) + :, np.newaxis, np.newaxis] + * np.ones((num, data.coords['x'].shape[0], + data.coords['y'].shape[0]))) data.coords['image_lat'] = ((epoch_name, 'x', 'y'), - np.arange(17)[np.newaxis, np.newaxis, :] - * np.ones((num, 17, 17))) + np.arange(data.coords['x'].shape[0])[ + np.newaxis, np.newaxis, :] + * np.ones((num, data.coords['x'].shape[0], + data.coords['y'].shape[0]))) data.coords['image_lon'] = ((epoch_name, 'x', 'y'), - np.arange(17)[np.newaxis, np.newaxis, :] - * np.ones((num, 17, 17))) + np.arange(data.coords['x'].shape[0])[ + np.newaxis, np.newaxis, :] + * np.ones((num, data.coords['x'].shape[0], + data.coords['y'].shape[0]))) # There may be data that depends on alternate time indices for i in range(num_extra_time_coords): From 965985411e2d79d87abf5ada9f78285db38fbd68 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 7 Aug 2023 13:44:47 -0400 Subject: [PATCH 053/365] TST: added unit test for ValueError Added a unit test for the proper-use-check in the new concat_data method. --- pysat/tests/test_methods_testing.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py index 0885032a7..ead66c57b 100644 --- a/pysat/tests/test_methods_testing.py +++ b/pysat/tests/test_methods_testing.py @@ -83,3 +83,13 @@ def test_generate_times_kwargs(self, num, kwargs, output): delta_time = [dt.timedelta(seconds=sec) for sec in uts] assert (index.to_pydatetime() - delta_time == dates).all return + + def test_concat_bad_kwarg(self): + """Test `concat_data` errors with bad kwarg input.""" + + testing.eval_bad_input(mm_test.concat_data, ValueError, + 'unexpected number of time dimensions', + input_args=(self.test_inst, self.test_inst), + input_kwargs={'num_extra_time_coords': 10}) + return + From e2b1eb9eaeded55efe09330845699e6d06db9039 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 7 Aug 2023 13:47:34 -0400 Subject: [PATCH 054/365] STY: removed whitespace Removed extra whitespace. --- pysat/tests/test_methods_testing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py index ead66c57b..8ff6c2140 100644 --- a/pysat/tests/test_methods_testing.py +++ b/pysat/tests/test_methods_testing.py @@ -92,4 +92,3 @@ def test_concat_bad_kwarg(self): input_args=(self.test_inst, self.test_inst), input_kwargs={'num_extra_time_coords': 10}) return - From 39fa2424094bd5ec030dcb6fa28298619b570cab Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 7 Aug 2023 14:52:05 -0400 Subject: [PATCH 055/365] BUG: fixed index name handling Fixed potential issues with the index name handling in the custom concat_data method. --- pysat/instruments/methods/testing.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index cd3354642..e86e0550d 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -102,8 +102,11 @@ def concat_data(self, new_data, num_extra_time_coords=0, **kwargs): with 'time', and no other dimensions to have a name that fits this format. """ - # Establish the time dimensions - time_dims = [var for var in self.variables if var.find('time') == 0] + # Establish the time dimensions, ensuring the standard variable is included + # whether or not it is treated as a variable + time_dims = [self.index.name] + time_dims.extend([var for var in self.variables if var.find('time') == 0 + and var != self.index.name]) if len(time_dims) != num_extra_time_coords + 1: raise ValueError( From 99b3f7bd3330b061b19e43571ab5ab524dcca5e9 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Mon, 7 Aug 2023 15:16:47 -0400 Subject: [PATCH 056/365] MAINT: move to pyproject --- pyproject.toml | 92 +++++++++++++++++++++++++++++++++++++++++++++++ pysat/__init__.py | 17 +++++---- pysat/version.txt | 1 - setup.cfg | 65 +-------------------------------- 4 files changed, 103 insertions(+), 72 deletions(-) create mode 100644 pyproject.toml delete mode 100644 pysat/version.txt diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..daffc9cbe --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,92 @@ +[build-system] +requires = ["setuptools", "pip >= 10"] +build-backend = "setuptools.build_meta" + +[project] +name = "pysat" +version = "3.1.0" +description = "Supports science analysis across disparate data platforms" +readme = "README.md" +requires-python = ">=3.6" +license = {file = "LICENSE"} +authors = [ + {name = "Russell Stoneback, et al.", email = "pysat.developers@gmail.com"}, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering :: Astronomy", + "Topic :: Scientific/Engineering :: Physics", + "Topic :: Scientific/Engineering :: Atmospheric Science", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Operating System :: Microsoft :: Windows" +] +keywords = [ + "pysat", + "ionosphere", + "atmosphere", + "thermosphere", + "magnetosphere", + "heliosphere", + "observations", + "models", + "space", + "satellites", + "analysis" +] +dependencies = [ + "dask", + "netCDF4", + "numpy >= 1.12", + "pandas", + "portalocker", + "pytest", + "scipy", + "toolz", + "xarray" +] + +[project.optional-dependencies] +test = [ + "coveralls < 3.3", + "flake8", + "flake8-docstrings", + "hacking >= 1.0, 6.0", + "pysatSpaceWeather", + "pytest", + "pytest-cov", + "pytest-ordering" +] +doc = [ + "extras_require", + "ipython", + "m2r2", + "numpydoc", + "sphinx<7.0", + "sphinx_rtd_theme" +] + +[project.urls] +Documentation = "https://pysat.readthedocs.io/en/latest/" +Source = "https://github.com/pysat/pysat" + +[tool.coverage.report] +omit = ["*/instruments/templates/"] + +[tool.pytest.ini_options] +addopts = "-vs --cov=pysatNASA" +markers = [ + "all_inst", + "download", + "no_download", + "load_options", + "first", + "second" +] diff --git a/pysat/__init__.py b/pysat/__init__.py index 122b541bc..f8a110a90 100644 --- a/pysat/__init__.py +++ b/pysat/__init__.py @@ -34,9 +34,10 @@ """ +import importlib + import logging import os -from portalocker import Lock logger = logging.getLogger(__name__) handler = logging.StreamHandler() @@ -48,9 +49,13 @@ # Import and set user and pysat parameters object from pysat import _params -# set version -here = os.path.abspath(os.path.dirname(__file__)) -version_filename = os.path.join(here, 'version.txt') +# Set version +try: + __version__ = importlib.metadata.version('pysat') +except AttributeError: + # Python 3.6 requires a different version + import importlib_metadata + __version__ = importlib_metadata.version('pysat') # Get home directory home_dir = os.path.expanduser('~') @@ -59,6 +64,7 @@ pysat_dir = os.path.join(home_dir, '.pysat') # Set directory for test data +here = os.path.abspath(os.path.dirname(__file__)) test_data_path = os.path.join(here, 'tests', 'test_data') # Create a .pysat directory or parameters file if one doesn't exist. @@ -96,9 +102,6 @@ # Load up existing parameters file params = _params.Parameters() -# Load up version information -with Lock(version_filename, 'r', params['file_timeout']) as version_file: - __version__ = version_file.read().strip() from pysat._files import Files from pysat._instrument import Instrument diff --git a/pysat/version.txt b/pysat/version.txt deleted file mode 100644 index fd2a01863..000000000 --- a/pysat/version.txt +++ /dev/null @@ -1 +0,0 @@ -3.1.0 diff --git a/setup.cfg b/setup.cfg index 579ecfbc8..84e7c49c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,60 +1,6 @@ [metadata] name = pysat -version = file: pysat/version.txt -url = https://github.com/pysat/pysat -author = Russell Stoneback, et al. -author_email = pysat.developers@gmail.com -description = 'Supports science analysis across disparate data platforms' -keywords = - pysat - ionosphere - atmosphere - thermosphere - magnetosphere - heliosphere - observations - models - space - satellites - analysis -classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Science/Research - Topic :: Scientific/Engineering :: Astronomy - Topic :: Scientific/Engineering :: Physics - Topic :: Scientific/Engineering :: Atmospheric Science - License :: OSI Approved :: BSD License - Natural Language :: English - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Operating System :: MacOS :: MacOS X - Operating System :: POSIX :: Linux - Operating System :: Microsoft :: Windows -license_file = LICENSE -long_description = file: README.md -long_description_content_type = text/markdown - -[options] -python_requires = >= 3.6 -setup_requires = setuptools >= 38.6; pip >= 10 -include_package_data = True -zip_safe = False -packages = find: -install_requires = dask - netCDF4 - numpy - pandas - portalocker - pytest - scipy - toolz - xarray - -[coverage:report] -omit = - */instruments/templates/* +version = 3.1.0 [flake8] max-line-length = 80 @@ -65,12 +11,3 @@ ignore = pysat/__init__.py E402 F401 pysat/instruments/methods/__init__.py F401 pysat/utils/__init__.py F401 - -[tool:pytest] -markers = - all_inst: tests all instruments - download: tests for downloadable instruments - no_download: tests for instruments without download support - load_options: tests for instruments including optional load kwargs - first: first tests to run - second: second tests to run From 404ae8bd572b222e43457713a70b17fb64cfda93 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Mon, 7 Aug 2023 15:17:20 -0400 Subject: [PATCH 057/365] MAINT: update test install syntax --- .github/workflows/docs.yml | 7 +++---- .github/workflows/main.yml | 33 ++++++++++++++++++++------------- .github/workflows/stats.yml | 7 ++----- .readthedocs.yml | 28 +++++++++++++--------------- 4 files changed, 38 insertions(+), 37 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 39a113b76..153cfc945 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,11 +8,11 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false matrix: - python-version: [3.9] + python-version: ["3.10"] name: Documentation tests steps: @@ -25,8 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r test_requirements.txt - pip install -r requirements.txt + pip install .[doc] - name: Set up pysat run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f577605e..51489005d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,15 +11,18 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python-version: ["3.9", "3.10"] + python-version: ["3.10", "3.11"] numpy_ver: ["latest"] + test_config: ["latest"] include: - - python-version: "3.8" - numpy_ver: "1.20" + - python-version: "3.9" + numpy_ver: "1.21" os: ubuntu-latest + test_config: "NEP29" - python-version: "3.6.8" numpy_ver: "1.19.5" os: "ubuntu-20.04" + test_config: "Ops" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} @@ -30,19 +33,23 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install NEP29/Operational dependencies - if: ${{ matrix.numpy_ver != 'latest'}} + - name: Install Operational dependencies + if: ${{ matrix.test_config == 'Ops'}} run: | - pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }} + pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} + pip install -r requirements.txt + pip install -r test_requirements.txt + pip install . - - name: Install standard dependencies and pysat + - name: Install NEP29 dependencies + if: ${{ matrix.test_config == 'NEP29'}} run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - python setup.py install + pip install numpy==${{ matrix.numpy_ver }} + pip install --upgrade-strategy only-if-needed .[test] - - name: Install requirements for testing setup - run: pip install -r test_requirements.txt + - name: Install standard dependencies + if: ${{ matrix.test_config == 'latest'}} + run: pip install .[test] - name: Set up pysat run: | @@ -61,4 +68,4 @@ jobs: - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls --rcfile=setup.cfg --service=github + run: coveralls --rcfile=pyproject.toml --service=github diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index 6cde36c2f..f22ad9ab3 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9"] + python-version: ["3.11"] name: Summary of instrument libraries steps: @@ -25,10 +25,7 @@ jobs: - name: Install dependencies run: | - pip install -r requirements.txt - # Need to install pysat before ecosystem - python setup.py install - pip install -r test_requirements.txt + pip install .test pip install `awk '{print $1}' ecosystem.txt` - name: Set up pysat diff --git a/.readthedocs.yml b/.readthedocs.yml index defa293da..125f0b3de 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,22 +5,20 @@ # Required version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.10" + # Build documentation in the docs/ directory with Sphinx sphinx: - configuration: docs/conf.py - -# Build documentation with MkDocs -#mkdocs: -# configuration: mkdocs.yml + configuration: docs/conf.py -# Optionally build your docs in additional formats such as PDF -formats: - - pdf - -# Optionally set the version of Python and requirements -# required to build your docs +# Declare the Python requirements required to build your docs +# This method includes a local build of the package python: - version: 3.7 - install: - - requirements: docs/requirements.txt - + install: + - method: pip + path: . + extra_requirements: + - doc From 3b86de009eb74f95879b042ea5668e4bf55042b5 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Mon, 7 Aug 2023 15:17:30 -0400 Subject: [PATCH 058/365] TST: add rc install test --- .github/workflows/pip_rc_install.yml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/pip_rc_install.yml diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml new file mode 100644 index 000000000..a6dd2c0f2 --- /dev/null +++ b/.github/workflows/pip_rc_install.yml @@ -0,0 +1,40 @@ +# This workflow will install Python dependencies and the latest RC of pysatCDAAC from test pypi. +# This test should be manually run before a pysatCDAAC RC is officially approved and versioned. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Test install of latest RC from pip + +on: [push] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.11"] # Keep this version at the highest supported Python version + + name: Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install standard dependencies + run: pip install -r requirements.txt + + - name: Install pysat RC + run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat + + - name: Set up pysat + run: | + mkdir pysatData + python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" + + - name: Check that install imports correctly + run: | + cd .. + python -c "import pysat; print(pysat.__version__)" From 259c22e62614f68580ad9d8a56342246f0e7dd2b Mon Sep 17 00:00:00 2001 From: jklenzing Date: Mon, 7 Aug 2023 15:17:53 -0400 Subject: [PATCH 059/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f920be4b3..6d2c53a8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. + * Implement pyproject to manage metadata [3.1.0] - 2023-05-31 -------------------- From 853930b56456c738262f72fbeb2f4a453167ad47 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Mon, 7 Aug 2023 15:19:52 -0400 Subject: [PATCH 060/365] BUG: sphinx versions --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index daffc9cbe..19a3e9b5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,8 @@ doc = [ "ipython", "m2r2", "numpydoc", - "sphinx<7.0", - "sphinx_rtd_theme" + "sphinx < 7.0", + "sphinx_rtd_theme >= 1.2.2" ] [project.urls] From de5ed552ee4ce91ab39b7ffcbec9a72724fa65d7 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Mon, 7 Aug 2023 15:21:07 -0400 Subject: [PATCH 061/365] MAINT: set to workflow dispatch --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index a6dd2c0f2..c782985e0 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -4,7 +4,7 @@ name: Test install of latest RC from pip -on: [push] +on: [workflow_dispatch] jobs: build: From f96541b920689430a834908351fe35d370dfa5a6 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Mon, 7 Aug 2023 15:22:33 -0400 Subject: [PATCH 062/365] BUG: dependency format --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 19a3e9b5e..ddb3e6e75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ test = [ "coveralls < 3.3", "flake8", "flake8-docstrings", - "hacking >= 1.0, 6.0", + "hacking >= 1.0, <6.0", "pysatSpaceWeather", "pytest", "pytest-cov", From 10f8da22721a35ab873eb6f9ebcc6c16bbb4a90e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 8 Aug 2023 11:48:19 -0400 Subject: [PATCH 063/365] TST: reduced number of test samples Changed the test frequency to 15 min instead of 100 sec. --- pysat/instruments/pysat_ndtesting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 7a63d54ba..c529e7af1 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -35,7 +35,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, non_unique_index=False, malformed_index=False, start_time=None, - num_samples=864, test_load_kwarg=None, max_latitude=90.0, + num_samples=96, test_load_kwarg=None, max_latitude=90.0, num_extra_time_coords=0): """Load the test files. @@ -62,7 +62,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, will begin at midnight. (default=None) num_samples : int Maximum number of times to generate. Data points will not go beyond the - current day. (default=864) + current day. (default=96) test_load_kwarg : any Keyword used for pysat unit testing to ensure that functionality for custom keywords defined in instrument support functions is working @@ -89,8 +89,8 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, iperiod = mm_test.define_period() drange = mm_test.define_range() - # Using 100s frequency for compatibility with seasonal analysis unit tests - uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='100S', + # Using 900s (15 min) frequency + uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='900S', start_time=start_time) # TODO(#1094): Remove in pysat 3.2.0 if malformed_index: From d9692df06b7b0438b25f843fc9a4a07a5e6bad92 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 9 Aug 2023 14:17:19 -0400 Subject: [PATCH 064/365] MAINT: remove setup.py --- setup.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index dfba3800e..000000000 --- a/setup.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2020, Authors -# Full license can be found in License.md -# ----------------------------------------------------------------------------- -"""Setup routine for pysat. - -Note ----- -package metadata stored in setup.cfg - -""" - -from setuptools import setup - -setup() From 6be826103648d2dc795009876356c2be53334251 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 9 Aug 2023 14:17:34 -0400 Subject: [PATCH 065/365] DOC: update docs --- README.md | 6 +++--- docs/dependency.rst | 12 ++++++------ docs/installation.rst | 14 +++++++------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 30470e6cb..8db411201 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ JGR-Space Physics [Publication](https://doi.org/10.1029/2018JA025297) [Citation Info](https://pysat.readthedocs.io/en/latest/citing.html) -Come join us on Slack! An invitation to the pysat workspace is available +Come join us on Slack! An invitation to the pysat workspace is available in the 'About' section of the [pysat GitHub Repository.](https://github.com/pysat/pysat) Development meetings are generally held fortnightly. @@ -66,7 +66,7 @@ pip install pysat ``` git clone https://github.com/pysat/pysat.git cd pysat -python setup.py install +pip install . ``` An advantage to installing through github is access to the development branches. The latest bugfixes can be found in the `develop` branch. However, this branch @@ -76,7 +76,7 @@ virtual environment or using `python setup.py develop`. git clone https://github.com/pysat/pysat.git cd pysat git checkout develop -python setup.py develop +pip install -e . ``` * Note that pysat requires a number of packages for the install. * dask diff --git a/docs/dependency.rst b/docs/dependency.rst index 7f0495328..f88438fba 100644 --- a/docs/dependency.rst +++ b/docs/dependency.rst @@ -31,8 +31,8 @@ could look something like: | | |-- __init__.py | | `-- test_instruments.py | `-- __init__.py - |-- README.md - `-- setup.py + |-- pyproject.toml + `-- README.md The instruments folder includes a file for each instrument object. The @@ -53,7 +53,7 @@ suite of instrument tests to run on instruments. These are imported from the ``test_instruments.py`` file can be copied directly into the library, updating the instrument library name as indicated. -The ``setup.py`` file should include pysat as a dependency, as well as any +The ``pyproject.toml`` file should include pysat as a dependency, as well as any other packages required by the instruments. A more complicated structure could include analysis routines, @@ -84,8 +84,8 @@ The structure then could look like: | | |-- compare.py | | `-- contrast.py | `-- __init__.py - |-- README.md - `-- setup.py + |-- pyproject.toml + `-- README.md .. _pysat-dep-testinst: @@ -330,7 +330,7 @@ leap days. Tips and Tricks --------------- -Remember to include pysat as a dependency in your setup.py or setup.cfg file. +Remember to include pysat as a dependency in your pyproject.toml file. The CI environment will also need to be configured to install pysat and its dependencies. You may need to install pysat from github rather than pip if diff --git a/docs/installation.rst b/docs/installation.rst index afc3e029b..24104dfae 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -26,7 +26,7 @@ To use Anaconda's tools for creating a suitable virtual environment, .. _inst-standard: - + Standard installation --------------------- @@ -57,7 +57,7 @@ pysat may also be installed directly from the source repository on github:: git clone https://github.com/pysat/pysat.git cd pysat - python setup.py install --user + pip install --user . An advantage to installing through github is access to the development branches. The latest bugfixes can be found in the ``develop`` branch. However, this @@ -67,9 +67,9 @@ virtual environment and using:: git clone https://github.com/pysat/pysat.git cd pysat git checkout develop - python setup.py develop + pip install -e . -The use of `develop` rather than `install` in the setup command installs the -code 'in-place', so any changes to the software do not have to be reinstalled -to take effect. It is not related to changing the pysat working branch from -``main`` to ``develop`` in the preceeding line. +The use of `-e` in the setup command installs the code 'in-place', so any +changes to the software do not have to be reinstalled to take effect. It is not +related to changing the pysat working branch from ``main`` to ``develop`` in the +preceeding line. From 1e01596da53b513787f51a8715db6890b4cf8e31 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 10 Aug 2023 10:39:34 -0400 Subject: [PATCH 066/365] BUG: update omit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ddb3e6e75..cb11257ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ Documentation = "https://pysat.readthedocs.io/en/latest/" Source = "https://github.com/pysat/pysat" [tool.coverage.report] -omit = ["*/instruments/templates/"] +omit = ["*/instruments/templates/*"] [tool.pytest.ini_options] addopts = "-vs --cov=pysatNASA" From 8d6818e3d8b2eb3d3a6f221435777a5e6da51f74 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Thu, 10 Aug 2023 11:07:07 -0400 Subject: [PATCH 067/365] BUG: wrong package --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cb11257ed..745fcdc2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ Source = "https://github.com/pysat/pysat" omit = ["*/instruments/templates/*"] [tool.pytest.ini_options] -addopts = "-vs --cov=pysatNASA" +addopts = "--cov=pysat" markers = [ "all_inst", "download", From 5871b077ad4b17b55d6b2e51b97816bc95bba55f Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 7 Sep 2023 15:42:36 -0400 Subject: [PATCH 068/365] BUG: fixed load `_prev` and `_next` inputs Ensure that kwargs passed directly into the load function are also passed into the supporting load functions: `_load_prev` and `_load_next`. --- pysat/_instrument.py | 54 ++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index e5ee7cd17..002c57469 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1598,7 +1598,7 @@ def _load_data(self, date=None, fid=None, inc=None, load_kwargs=None): inc : dt.timedelta, int, or NoneType Increment of files or dates to load, starting from the root date or fid (default=None) - load_kwargs : dict + load_kwargs : dict or NoneType Dictionary of keywords that may be options for specific instruments. If None, uses `self.kwargs['load']`. (default=None) @@ -1709,9 +1709,15 @@ def _load_data(self, date=None, fid=None, inc=None, load_kwargs=None): return data, mdata - def _load_next(self): + def _load_next(self, load_kwargs=None): """Load the next days data (or file) without incrementing the date. + Parameters + ---------- + load_kwargs : dict or NoneType + Dictionary of keywords that may be options for specific instruments. + If None, uses `self.kwargs['load']`. (default=None) + Returns ------- data : pds.DataFrame or xr.Dataset @@ -1728,16 +1734,24 @@ def _load_next(self): or the file. Looks for `self._load_by_date` flag. """ + load_data_kwargs = {'inc': self.load_step, 'load_kwargs': load_kwargs} + if self._load_by_date: - next_date = self.date + self.load_step - return self._load_data(date=next_date, inc=self.load_step) + load_data_kwargs['date'] = self.date + self.load_step else: - next_id = self._fid + self.load_step + 1 - return self._load_data(fid=next_id, inc=self.load_step) + load_data_kwargs['fid'] = self._fid + self.load_step + 1 + + return self._load_data(**load_data_kwargs) - def _load_prev(self): + def _load_prev(self, load_kwargs=None): """Load the previous days data (or file) without decrementing the date. + Parameters + ---------- + load_kwargs : dict or NoneType + Dictionary of keywords that may be options for specific instruments. + If None, uses `self.kwargs['load']`. (default=None) + Returns ------- data : pds.DataFrame or xr.Dataset @@ -1754,14 +1768,14 @@ def _load_prev(self): or the file. Looks for `self._load_by_date` flag. """ - load_kwargs = {'inc': self.load_step} + load_data_kwargs = {'inc': self.load_step, 'load_kwargs': load_kwargs} if self._load_by_date: - load_kwargs['date'] = self.date - self.load_step + load_data_kwargs['date'] = self.date - self.load_step else: - load_kwargs['fid'] = self._fid - self.load_step - 1 + load_data_kwargs['fid'] = self._fid - self.load_step - 1 - return self._load_data(**load_kwargs) + return self._load_data(**load_data_kwargs) def _set_load_parameters(self, date=None, fid=None): """Set the necesssary load attributes. @@ -3219,11 +3233,13 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, pysat.logger.debug('Initializing data cache.') # Using current date or fid - self._prev_data, self._prev_meta = self._load_prev() + self._prev_data, self._prev_meta = self._load_prev( + load_kwargs=kwargs) self._curr_data, self._curr_meta = self._load_data( date=self.date, fid=self._fid, inc=self.load_step, load_kwargs=kwargs) - self._next_data, self._next_meta = self._load_next() + self._next_data, self._next_meta = self._load_next( + load_kwargs=kwargs) else: if self._next_data_track == curr: pysat.logger.debug('Using data cache. Loading next.') @@ -3233,7 +3249,8 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, self._prev_meta = self._curr_meta self._curr_data = self._next_data self._curr_meta = self._next_meta - self._next_data, self._next_meta = self._load_next() + self._next_data, self._next_meta = self._load_next( + load_kwargs=kwargs) elif self._prev_data_track == curr: pysat.logger.debug('Using data cache. Loading previous.') # Moving backward in time @@ -3242,7 +3259,8 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, self._next_meta = self._curr_meta self._curr_data = self._prev_data self._curr_meta = self._prev_meta - self._prev_data, self._prev_meta = self._load_prev() + self._prev_data, self._prev_meta = self._load_prev( + load_kwargs=kwargs) else: # Jumped in time/or switched from filebased to date based # access @@ -3250,11 +3268,13 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, del self._prev_data del self._curr_data del self._next_data - self._prev_data, self._prev_meta = self._load_prev() + self._prev_data, self._prev_meta = self._load_prev( + load_kwargs=kwargs) self._curr_data, self._curr_meta = self._load_data( date=self.date, fid=self._fid, inc=self.load_step, load_kwargs=kwargs) - self._next_data, self._next_meta = self._load_next() + self._next_data, self._next_meta = self._load_next( + load_kwargs=kwargs) # Make sure datetime indices for all data is monotonic if self.pandas_format: From 2671e7e8f29f256bd7a1e2f7416d3d7e6eccb926 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 7 Sep 2023 15:43:37 -0400 Subject: [PATCH 069/365] STY: removed concat kwarg Removed the concat kwarg that was used to verify correct input and the test that supported it. --- pysat/instruments/methods/testing.py | 13 +++++-------- pysat/tests/test_methods_testing.py | 9 --------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index e86e0550d..2a4b6af15 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -84,15 +84,13 @@ def clean(self, test_clean_kwarg=None): # Optional methods -def concat_data(self, new_data, num_extra_time_coords=0, **kwargs): +def concat_data(self, new_data, **kwargs): """Concatonate data to self.data for extra time dimensions. Parameters ---------- new_data : xarray.Dataset or list of such objects New data objects to be concatonated - num_extra_time_coords : int - Number of extra time dimensions that require concatonation (default=0) **kwargs : dict Optional keyword arguments passed to xr.concat @@ -108,11 +106,6 @@ def concat_data(self, new_data, num_extra_time_coords=0, **kwargs): time_dims.extend([var for var in self.variables if var.find('time') == 0 and var != self.index.name]) - if len(time_dims) != num_extra_time_coords + 1: - raise ValueError( - 'unexpected number of time dimensions: len({:}) != {:d}'.format( - time_dims, num_extra_time_coords + 1)) - # Concatonate using the appropriate method for the number of time # dimensions if len(time_dims) == 1: @@ -292,6 +285,10 @@ def initialize_test_meta(epoch_name, data_keys): if var.find('variable_profiles') == 0: meta[var] = {'desc': 'Profiles with variable altitude.'} + if len(var) > 17: + tvar = 'time{:s}'.format(var[17:]) + meta[tvar] = {'desc': 'Additional time variable.'} + # Standard metadata required for xarray. meta['profile_height'] = {'value_min': 0, 'value_max': 14, 'fill': -1, 'desc': 'Altitude of profile data.'} diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py index 8ff6c2140..0885032a7 100644 --- a/pysat/tests/test_methods_testing.py +++ b/pysat/tests/test_methods_testing.py @@ -83,12 +83,3 @@ def test_generate_times_kwargs(self, num, kwargs, output): delta_time = [dt.timedelta(seconds=sec) for sec in uts] assert (index.to_pydatetime() - delta_time == dates).all return - - def test_concat_bad_kwarg(self): - """Test `concat_data` errors with bad kwarg input.""" - - testing.eval_bad_input(mm_test.concat_data, ValueError, - 'unexpected number of time dimensions', - input_args=(self.test_inst, self.test_inst), - input_kwargs={'num_extra_time_coords': 10}) - return From 894f1c6cd41129e80c5defd1b40d6ae96b0bb1cf Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 7 Sep 2023 15:49:12 -0400 Subject: [PATCH 070/365] REV: reverted test instrument time step Reverted the test instrument time step, since it didn't solve the problem and was set this way originally for compatibility with other tests. --- pysat/instruments/pysat_ndtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index c529e7af1..03fb0290f 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -89,8 +89,8 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, iperiod = mm_test.define_period() drange = mm_test.define_range() - # Using 900s (15 min) frequency - uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='900S', + # Using 100s frequency for compatibility with seasonal analysis unit tests + uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='100S', start_time=start_time) # TODO(#1094): Remove in pysat 3.2.0 if malformed_index: From 1ac853c39c3b94203f5e3dc18ba82db44cff9a35 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 7 Sep 2023 15:49:40 -0400 Subject: [PATCH 071/365] BUG: updated coordinate metadata Updated the coordinate metadata to reflect the new, smaller size. --- pysat/instruments/methods/testing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 2a4b6af15..8c0b2838f 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -299,13 +299,13 @@ def initialize_test_meta(epoch_name, data_keys): 'notes': 'function of image_lat and image_lon'} meta['x'] = {'desc': 'x-value of image pixel', 'notes': 'Dummy Variable', - 'value_min': 0, 'value_max': 17, 'fill': -1} + 'value_min': 0, 'value_max': 7, 'fill': -1} meta['y'] = {'desc': 'y-value of image pixel', 'notes': 'Dummy Variable', - 'value_min': 0, 'value_max': 17, 'fill': -1} + 'value_min': 0, 'value_max': 7, 'fill': -1} meta['z'] = {'desc': 'z-value of profile height', 'notes': 'Dummy Variable', - 'value_min': 0, 'value_max': 15, 'fill': -1} + 'value_min': 0, 'value_max': 5, 'fill': -1} meta['image_lat'] = {'desc': 'Latitude of image pixel', 'notes': 'Dummy Variable', 'value_min': -90., 'value_max': 90.} From 5adcad2a36c21b32caf566fd59c783c3d8e8fc77 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 8 Sep 2023 13:19:36 -0400 Subject: [PATCH 072/365] BUG: adding verbose flag Temporarily adding verbose flag, to see where things are failing remotely. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f577605e..a76a98a00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: pytest --cov=pysat/ + run: pytest -v --cov=pysat/ - name: Publish results to coveralls env: From 1ff5978084466c43545d1aa05ff4216be9633bec Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 10:13:14 -0400 Subject: [PATCH 073/365] MAINT: updated `to_inst` behaviour Updated `to_inst` behaviour, only combining dimensions instead of coordinates that are not also dimensions. --- CHANGELOG.md | 2 ++ pysat/_constellation.py | 34 ++++++++++++++++--------------- pysat/tests/test_constellation.py | 3 --- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f887c593c..ebe685761 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. + * Updated `Constellation.to_inst` method definition of coords, using dims + to combine common dimensions instead. [3.1.0] - 2023-05-31 -------------------- diff --git a/pysat/_constellation.py b/pysat/_constellation.py index 0252c7275..34e5866c9 100644 --- a/pysat/_constellation.py +++ b/pysat/_constellation.py @@ -670,22 +670,24 @@ def to_inst(self, common_coord=True, fill_method=None): # Get the common coordinates needed for all data for cinst in self.instruments: if not cinst.pandas_format: - for new_coord in cinst.data.coords.keys(): - if new_coord not in coords.keys(): - coords[new_coord] = cinst.data.coords[new_coord] - elif new_coord != 'time': - # Two instruments have the same coordinate, if they - # are not identical, we need to establish a common - # range and resolution. Note that this will only - # happen if the coordinates share the same names. - if(len(coords[new_coord]) - != len(cinst.data.coords[new_coord]) - or coords[new_coord].values - != cinst.data.coords[new_coord].values): - coords[new_coord] = establish_common_coord( - [coords[new_coord].values, - cinst.data.coords[new_coord].values], - common=common_coord) + for new_coord in cinst.data.dims.keys(): + if new_coord in cinst.data.coords.keys(): + if new_coord not in coords.keys(): + coords[new_coord] = cinst.data.coords[new_coord] + elif new_coord != 'time': + # Two instruments have the same coordinate, if + # they are not identical, we need to establish + # a common range and resolution. Note that + # this will only happen if the coordinates + # share the same names. + if(len(coords[new_coord]) + != len(cinst.data.coords[new_coord]) + or coords[new_coord].values + != cinst.data.coords[new_coord].values): + coords[new_coord] = establish_common_coord( + [coords[new_coord].values, + cinst.data.coords[new_coord].values], + common=common_coord) data = xr.Dataset(coords=coords) diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py index 5dda00d34..5986b71e8 100644 --- a/pysat/tests/test_constellation.py +++ b/pysat/tests/test_constellation.py @@ -478,9 +478,6 @@ def test_to_inst_xarray(self, common_coord, fill_method): testing.assert_lists_equal(self.dims, list(out_inst.data.dims.keys())) testing.assert_list_contains(self.dims, list(out_inst.data.coords.keys())) - testing.assert_list_contains(['variable_profile_height', 'image_lon', - 'image_lat'], - list(out_inst.data.coords.keys())) for cinst in self.const.instruments: for var in cinst.variables: From 37ce5d32050d3b73e77128300b2d6abd725884d0 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 10:15:08 -0400 Subject: [PATCH 074/365] REV: removed verbose flag Removed the temporary setting of the verbosity flag. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a76a98a00..7f577605e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: pytest -v --cov=pysat/ + run: pytest --cov=pysat/ - name: Publish results to coveralls env: From d1633c15d0765b465ba2278a542b9c66e64765d5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 10:17:45 -0400 Subject: [PATCH 075/365] TST: added temporary skip for new test Added a skip for the new test for Python 3.6. --- pysat/tests/classes/cls_instrument_library.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 3f62d442a..c0c183cab 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -518,6 +518,8 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options @pytest.mark.download + @pytest.skipif(sys.version_info < (3, 7), + reason='See if everything else passes without this test') @pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)]) def test_load_w_pad(self, pad, inst_dict): """Test that instruments load at each cleaning level. From 9f854ca75ae58108fe6186fc675ceeb22e9d8787 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 10:22:48 -0400 Subject: [PATCH 076/365] REV: revert default times Restore the old number of times, for support of other ecosystem packages. --- pysat/instruments/pysat_ndtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 03fb0290f..7a63d54ba 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -35,7 +35,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, non_unique_index=False, malformed_index=False, start_time=None, - num_samples=96, test_load_kwarg=None, max_latitude=90.0, + num_samples=864, test_load_kwarg=None, max_latitude=90.0, num_extra_time_coords=0): """Load the test files. @@ -62,7 +62,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, will begin at midnight. (default=None) num_samples : int Maximum number of times to generate. Data points will not go beyond the - current day. (default=96) + current day. (default=864) test_load_kwarg : any Keyword used for pysat unit testing to ensure that functionality for custom keywords defined in instrument support functions is working From 1c8f3fa41545edddf06aacf7eb5be2c9376f9294 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 10:25:03 -0400 Subject: [PATCH 077/365] BUG: fixed skipif evaluation Fixed the skipif usage. --- pysat/tests/classes/cls_instrument_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index c0c183cab..8addb4730 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -518,7 +518,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options @pytest.mark.download - @pytest.skipif(sys.version_info < (3, 7), + @pytest.skipif(sys.version_info.minor < 7, reason='See if everything else passes without this test') @pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)]) def test_load_w_pad(self, pad, inst_dict): From 181d480e6bbb0b423ac3ebf3d2d338626eea443c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 10:26:44 -0400 Subject: [PATCH 078/365] BUG: added missing `mark` Added `mark` to the pytest line. --- pysat/tests/classes/cls_instrument_library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 8addb4730..68de0f7bd 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -518,8 +518,8 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options @pytest.mark.download - @pytest.skipif(sys.version_info.minor < 7, - reason='See if everything else passes without this test') + @pytest.mark.skipif(sys.version_info.minor < 7, + reason='See if everything else passes') @pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)]) def test_load_w_pad(self, pad, inst_dict): """Test that instruments load at each cleaning level. From 87c3c2a3efedc336b66c665f8b0acf6e3b96c946 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 15:38:28 -0400 Subject: [PATCH 079/365] STY: added kwarg to `__getitem__` Added a data kwarg to `__getitem__` to streamline the data loading process. --- pysat/_instrument.py | 125 +++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 002c57469..581ffe943 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -743,7 +743,7 @@ def __str__(self): return output_str - def __getitem__(self, key): + def __getitem__(self, key, data=None): """Access data in `pysat.Instrument` object. Parameters @@ -751,6 +751,8 @@ def __getitem__(self, key): key : str, tuple, or dict Data variable name, tuple with a slice, or dict used to locate desired data. + data : pds.DataFrame, xr.Dataset, or NoneType + Desired data object to select from or None to use `data` attribute Raises ------ @@ -789,20 +791,22 @@ def __getitem__(self, key): inst[datetime1:datetime2, 'name1':'name2'] """ + if data is None: + data = self.data if self.pandas_format: if isinstance(key, str): - return self.data[key] + return data[key] elif isinstance(key, tuple): try: # Pass keys directly through - return self.data.loc[key[0], key[1]] + return data.loc[key[0], key[1]] except (KeyError, TypeError) as err1: # TypeError for single integer. KeyError for list, array, # slice of integers. Assume key[0] is integer # (including list or slice). try: - return self.data.loc[self.data.index[key[0]], key[1]] + return data.loc[data.index[key[0]], key[1]] except IndexError as err2: err_message = '\n'.join(("original messages:", str(err1), str(err2))) @@ -812,15 +816,15 @@ def __getitem__(self, key): else: try: # Integer based indexing - return self.data.iloc[key] + return data.iloc[key] except (TypeError, ValueError): # If it's not an integer, TypeError is thrown. If it's a # list, ValueError is thrown. - return self.data[key] + return data[key] else: - return self.__getitem_xarray__(key) + return self.__getitem_xarray__(key, data=data) - def __getitem_xarray__(self, key): + def __getitem_xarray__(self, key, data=None): """Access data in `pysat.Instrument` object with `xarray.Dataset`. Parameters @@ -828,6 +832,8 @@ def __getitem_xarray__(self, key): key : str, tuple, or dict Data variable name, tuple with a slice, or dict used to locate desired data + data : xr.Dataset or NoneType + Desired data object to select from or None to use `data` attribute Returns ------- @@ -866,19 +872,21 @@ def __getitem_xarray__(self, key): inst[datetime1:datetime2, 'name'] """ + if data is None: + data = self.data - if 'Epoch' in self.data.indexes: + if 'Epoch' in data.indexes: epoch_names = ['Epoch'] - elif 'time' in self.data.indexes: + elif 'time' in data.indexes: epoch_names = ['time'] else: return xr.Dataset(None) # Find secondary time indexes that may need to be sliced - if len(self.data.indexes) > 1: - for ind in self.data.indexes.keys(): - if(ind != epoch_names[0] and self.data.indexes[ind].dtype - == self.data.indexes[epoch_names[0]].dtype): + if len(data.indexes) > 1: + for ind in data.indexes.keys(): + if(ind != epoch_names[0] and data.indexes[ind].dtype + == data.indexes[epoch_names[0]].dtype): epoch_names.append(ind) if isinstance(key, tuple): @@ -886,10 +894,10 @@ def __getitem_xarray__(self, key): # Support slicing time, variable name if isinstance(key[1], slice): # Extract subset of variables before epoch selection. - data_subset = self.data[self.variables[key[1]]] + data_subset = data[self.variables[key[1]]] else: # Extract single variable before epoch selection. - data_subset = self.data[key[1]] + data_subset = data[key[1]] # If the input is a tuple, `key[0]` must be linked to the epoch. key_dict = {'indexers': {epoch_name: key[0] @@ -918,7 +926,7 @@ def __getitem_xarray__(self, key): for i, dim in enumerate(self[var_name].dims): indict[dim] = key[0][i] - return self.data[var_name][indict] + return data[var_name][indict] else: # Multidimensional indexing where the multiple dimensions are # not contained within another object @@ -934,11 +942,11 @@ def __getitem_xarray__(self, key): for i, dim in enumerate(self[var_name].dims): indict[dim] = key[i] - return self.data[var_name][indict] + return data[var_name][indict] else: try: # Grab a particular variable by name - return self.data[key] + return data[key] except (TypeError, KeyError, ValueError): # If that didn't work, likely need to use `isel` or `sel` # Link key to the epoch. @@ -947,11 +955,11 @@ def __getitem_xarray__(self, key): try: # Try to get all data variables, but for a subset of time # using integer indexing - return self.data.isel(**key_dict) + return data.isel(**key_dict) except (KeyError, TypeError): # Try to get a subset of time, using label based indexing try: - return self.data.sel(**key_dict) + return data.sel(**key_dict) except KeyError as kerr: if str(kerr).find('Timestamp') >= 0 and len( epoch_names) > 0: @@ -962,7 +970,7 @@ def __getitem_xarray__(self, key): ''.join(['Removing ', repr(epoch_names[1:]), ' dimensions from data selection'])) key_dict = {'indexers': {epoch_names[0]: key}} - return self.data.sel(**key_dict) + return data.sel(**key_dict) else: raise kerr @@ -3210,8 +3218,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # Check for consistency between loading range and data padding, if any if self.pad is not None: if self._load_by_date: - tdate = dt.datetime(2009, 1, 1) - if tdate + self.load_step < tdate + loop_pad: + if date + self.load_step < date + loop_pad: estr = ''.join(('Data padding window must be shorter than ', 'data loading window. Load a greater ', 'range of data or shorten the padding.')) @@ -3262,7 +3269,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, self._prev_data, self._prev_meta = self._load_prev( load_kwargs=kwargs) else: - # Jumped in time/or switched from filebased to date based + # Jumped in time/or switched from file based to date based # access pysat.logger.debug('Resetting data cache.') del self._prev_data @@ -3294,14 +3301,16 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, self._next_data = getattr(self._next_data, sort_method)(*sort_args) - # Make tracking indexes consistent with new loads + # Make tracking indexes consistent with new loads, as date loading + # and file loading have to be treated differently due to change in + # inclusive/exclusive range end treatment. Loading by file is + # inclusive. if self._load_by_date: + # Arithmatic uses datetime or DateOffset objects self._next_data_track = curr + self.load_step self._prev_data_track = curr - self.load_step else: - # File and date loads have to be treated differently - # due to change in inclusive/exclusive range end - # treatment. Loading by file is inclusive. + # Aritmatic uses integers self._next_data_track = curr + self.load_step + 1 self._prev_data_track = curr - self.load_step - 1 @@ -3341,46 +3350,36 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, "by file."))) # Pad data based upon passed parameter - if (not self._empty(self._prev_data)) & (not self.empty): - stored_data = self.data # .copy() - temp_time = copy.deepcopy(self.index[0]) - - # Pad data using access mechanisms that works for both pandas - # and xarray - self.data = self._prev_data.copy() - - # __getitem__ used below to get data from instrument object. - # Details for handling pandas and xarray are different and - # handled by __getitem__. - self.data = self[first_pad:temp_time] - if not self.empty: - if self.index[-1] == temp_time: - self.data = self[:-1] - self.concat_data(stored_data, prepend=False) - else: - self.data = stored_data - - if (not self._empty(self._next_data)) & (not self.empty): - stored_data = self.data # .copy() - temp_time = copy.deepcopy(self.index[-1]) - - # Pad data using access mechanisms that work for both pandas - # and xarray - self.data = self._next_data.copy() - self.data = self[temp_time:last_pad] - if len(self.index) > 0: - if (self.index[0] == temp_time): - self.data = self[1:] - self.concat_data(stored_data, prepend=True) - else: - self.data = stored_data + if not self._empty(self._prev_data) and not self.empty: + # __getitem__ used to handle any pandas/xarray differences in + # data slicing + pdata = self.__getitem__(slice(first_pad, self.index[0]), + data=self._prev_data) + if not self._empty(pdata): + if pdata[self.index.name][-1] == self.index[0]: + pdata = self.__getitem__(slice(-1), data=pdata) + self.concat_data(pdata, prepend=True) + del pdata + + if not self._empty(self._next_data) and not self.empty: + # __getitem__ used to handle any pandas/xarray differences in + # data slicing + + ndata = self.__getitem__(slice(self.index[-1], last_pad), + data=self._next_data) + if not self._empty(ndata): + if ndata[self.index.name][0] == self.index[-1]: + ndata = self.__getitem__( + slice(1, len(ndata[self.index.name])), data=ndata) + self.concat_data(ndata, prepend=False) + del ndata if len(self.index) > 0: self.data = self[first_pad:last_pad] # Want exclusive end slicing behavior from above if not self.empty: - if (self.index[-1] == last_pad) & (not want_last_pad): + if (self.index[-1] == last_pad) and (not want_last_pad): self.data = self[:-1] else: # If self.pad is False, load single day From 0fa26b5b0959333eecbf9a945182ce16a200f659 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 15:38:50 -0400 Subject: [PATCH 080/365] TST: removed skipif Removed the skipif, to see if the memory issue has been improved. --- pysat/tests/classes/cls_instrument_library.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 68de0f7bd..3f62d442a 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -518,8 +518,6 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options @pytest.mark.download - @pytest.mark.skipif(sys.version_info.minor < 7, - reason='See if everything else passes') @pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)]) def test_load_w_pad(self, pad, inst_dict): """Test that instruments load at each cleaning level. From a92ba9422afa2a1f23f7441248881a30e48e27c6 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 15:40:44 -0400 Subject: [PATCH 081/365] STY: removed extra line Removed an extra line that only had whitespace. --- pysat/_instrument.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 581ffe943..3d58350d1 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -3364,7 +3364,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, if not self._empty(self._next_data) and not self.empty: # __getitem__ used to handle any pandas/xarray differences in # data slicing - ndata = self.__getitem__(slice(self.index[-1], last_pad), data=self._next_data) if not self._empty(ndata): From dc5c622265380da8cc1c13c6c294235d0bc746d1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 15:52:22 -0400 Subject: [PATCH 082/365] BUG: fixed index handling Fixed index handling to be pandas/xarray agnostic within the load method. --- pysat/_instrument.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 3d58350d1..9d5781e1c 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -3356,7 +3356,9 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, pdata = self.__getitem__(slice(first_pad, self.index[0]), data=self._prev_data) if not self._empty(pdata): - if pdata[self.index.name][-1] == self.index[0]: + # Test the data index, slicing if necessary + pindex = self._index(data=pdata) + if pindex[-1] == self.index[0]: pdata = self.__getitem__(slice(-1), data=pdata) self.concat_data(pdata, prepend=True) del pdata @@ -3367,7 +3369,9 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, ndata = self.__getitem__(slice(self.index[-1], last_pad), data=self._next_data) if not self._empty(ndata): - if ndata[self.index.name][0] == self.index[-1]: + # Test the data index, slicing if necessary + nindex = self._index(data=ndata) + if nindex[0] == self.index[-1]: ndata = self.__getitem__( slice(1, len(ndata[self.index.name])), data=ndata) self.concat_data(ndata, prepend=False) From 98c4761ed23297f578875476ac28bbe574f55670 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 15:58:21 -0400 Subject: [PATCH 083/365] DOC: updated the changelog Added a description of the new Instrument changes to the changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9a0fdbd7..ca78e0db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). clean method. * Added loading test with padding for Instruments. * Allow Instruments to define custom `concat_data` methods. + * Added data kwarg to the Instrument class `__getitem__` method and reduced + memory usage in the `load` method. * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. From 3cb0f5aecd8e05293d6d7dc2836a40915a0ddf03 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 16:44:27 -0400 Subject: [PATCH 084/365] TST: added testing skip Try to run new test only on one of the test instruments for Python 3.6. --- pysat/tests/classes/cls_instrument_library.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 3f62d442a..9b6865dc5 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -532,6 +532,13 @@ def test_load_w_pad(self, pad, inst_dict): `initialize_test_package` is run. """ + # Skip all but one instrument for Python 3.6 + if sys.version_info.minor < 7 and inst_dict[ + 'inst_module'].__name__.find('pysat_testing') == len( + inst_dict['inst_module'].__name__) - len('pysat_testing'): + pytest.skip("skipping 'test_load_w_pad' for {:}".format(inst_dict)) + return + # Update the Instrument dict with the desired pad if 'kwargs' in inst_dict.keys(): inst_dict['kwargs']['pad'] = pad From 77fcb2370dacc5b35e07bea9932cf756a3a80bd0 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 16:46:15 -0400 Subject: [PATCH 085/365] STY: removed whitespace Removed extra whitespace. --- pysat/tests/classes/cls_instrument_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 9b6865dc5..fe95ebb50 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -538,7 +538,7 @@ def test_load_w_pad(self, pad, inst_dict): inst_dict['inst_module'].__name__) - len('pysat_testing'): pytest.skip("skipping 'test_load_w_pad' for {:}".format(inst_dict)) return - + # Update the Instrument dict with the desired pad if 'kwargs' in inst_dict.keys(): inst_dict['kwargs']['pad'] = pad From a143898ced6bfa902ad8ff065d90617bf5537935 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Sep 2023 17:01:42 -0400 Subject: [PATCH 086/365] TST: add more skips Add more skips, with information about what is being skipped. --- pysat/tests/classes/cls_instrument_library.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index fe95ebb50..c4a380af8 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -533,10 +533,11 @@ def test_load_w_pad(self, pad, inst_dict): """ # Skip all but one instrument for Python 3.6 - if sys.version_info.minor < 7 and inst_dict[ - 'inst_module'].__name__.find('pysat_testing') == len( - inst_dict['inst_module'].__name__) - len('pysat_testing'): - pytest.skip("skipping 'test_load_w_pad' for {:}".format(inst_dict)) + if sys.version_info.minor < 7: + pytest.skip("skipping 3.6 for {:} ({:} =? {:})".format( + inst_dict, inst_dict['inst_module'].__name__.find( + 'pysat_testing'), len(inst_dict['inst_module'].__name__) + - len('pysat_testing'))) return # Update the Instrument dict with the desired pad From 8e34d084652b23959b09c1c529eb785ad73a400a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 12 Sep 2023 09:54:30 -0400 Subject: [PATCH 087/365] STY: reduced code duplication Reduced code duplication in Instrument by adding a hidden method that finds the standard epoch name(s) in the xarray Dataset. --- CHANGELOG.md | 2 ++ pysat/_instrument.py | 60 +++++++++++++++++++++++++++++++++----------- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca78e0db3..234631ee8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Allow Instruments to define custom `concat_data` methods. * Added data kwarg to the Instrument class `__getitem__` method and reduced memory usage in the `load` method. + * Added a hidden method the Instrument class `_get_epoch_name_from_data` to + reduce code duplication. * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 9d5781e1c..8f69a41d6 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -875,11 +875,9 @@ def __getitem_xarray__(self, key, data=None): if data is None: data = self.data - if 'Epoch' in data.indexes: - epoch_names = ['Epoch'] - elif 'time' in data.indexes: - epoch_names = ['time'] - else: + # Find the standard epoch index name(s) + epoch_names = self._get_epoch_name_from_data(data=data) + if len(epoch_names) == 0: return xr.Dataset(None) # Find secondary time indexes that may need to be sliced @@ -1089,13 +1087,16 @@ def __setitem__(self, key, new_data): new = {'data': new} in_data = new.pop('data') - if 'Epoch' in self.data.indexes: - epoch_name = 'Epoch' - elif 'time' in self.data.indexes: - epoch_name = 'time' - else: + epoch_names = self._get_epoch_name_from_data() + if len(epoch_names) == 0: raise ValueError(' '.join(('Unsupported time index name,', '"Epoch" or "time".'))) + else: + if len(epoch_names) > 1: + pysat.logger.error("".join(["Multiple standard time index ", + "names found, defaulting to ", + epoch_names[0]])) + epoch_name = epoch_names[0] if isinstance(key, tuple): # User provided more than one thing in assignment location @@ -1252,6 +1253,34 @@ def __iter__(self): # ----------------------------------------------------------------------- # Define all hidden methods + def _get_epoch_name_from_data(self, data=None): + """Get the standard epoch name used in this data object. + + Parameters + ---------- + data : pds.DataFrame, xr.Dataset, or NoneType + Desired data object to select from or None to use `data` attribute + + Returns + ------- + epoch_names : list + List of standard epoch names included in the data indexes + + """ + # Initalize output + epoch_names = [] + + # If no data is provided, use the Instrument attribute + if data is None: + data = self.data + + if hasattr(data, 'indexes'): + for ename in ['Epoch', 'time']: + if ename in data.indexes: + epoch_names.append(ename) + + return epoch_names + def _empty(self, data=None): """Determine whether or not data has been loaded. @@ -1308,12 +1337,13 @@ def _index(self, data=None): if self.pandas_format: return data.index else: - if 'time' in data.indexes: - return data.indexes['time'] - elif 'Epoch' in data.indexes: - return data.indexes['Epoch'] - else: + epoch_names = self.get_epoch_name_from_data(data=data) + + if len(epoch_names) == 0: return pds.Index([]) + else: + # Xarray preferred epoch name order is opposite + return data.indexes[epoch_names[-1]] def _pass_method(*args, **kwargs): """Empty default method for updatable Instrument methods.""" From b562952a2b34d6566947ce1e6a16268c8a67a720 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 12 Sep 2023 09:55:09 -0400 Subject: [PATCH 088/365] TST: added TODO for skip Added a TODO for the skip, since the solution to the current failure is out of scope for the current pull request. --- pysat/tests/classes/cls_instrument_library.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index c4a380af8..da0dc1195 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -532,7 +532,9 @@ def test_load_w_pad(self, pad, inst_dict): `initialize_test_package` is run. """ - # Skip all but one instrument for Python 3.6 + # Skip for Python 3.6, keeping information that will allow adding + # or skipping particular instruments. + # TODO(#1136): Remove skip once memory management is improved if sys.version_info.minor < 7: pytest.skip("skipping 3.6 for {:} ({:} =? {:})".format( inst_dict, inst_dict['inst_module'].__name__.find( From 08f27646ba685f58e4f8ff8036c7e46dcc5e6163 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 12 Sep 2023 10:31:39 -0400 Subject: [PATCH 089/365] BUG: fixed typo Fixed a typo in a method name call. --- pysat/_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 8f69a41d6..cf6826f72 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1337,7 +1337,7 @@ def _index(self, data=None): if self.pandas_format: return data.index else: - epoch_names = self.get_epoch_name_from_data(data=data) + epoch_names = self._get_epoch_name_from_data(data=data) if len(epoch_names) == 0: return pds.Index([]) From 16b491da068e63db26d67e4a33bf06948c384bc7 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 15 Sep 2023 12:39:46 -0400 Subject: [PATCH 090/365] MAINT: fixed spelling Fixed spelling, because words are hard. Co-authored-by: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> --- pysat/_instrument.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index cf6826f72..b5091d70f 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -3336,11 +3336,11 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # inclusive/exclusive range end treatment. Loading by file is # inclusive. if self._load_by_date: - # Arithmatic uses datetime or DateOffset objects + # Arithmetic uses datetime or DateOffset objects self._next_data_track = curr + self.load_step self._prev_data_track = curr - self.load_step else: - # Aritmatic uses integers + # Arithmetic uses integers self._next_data_track = curr + self.load_step + 1 self._prev_data_track = curr - self.load_step - 1 From 2d72fb902c930236995be1d89587eb45cec54054 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 15 Sep 2023 13:09:17 -0400 Subject: [PATCH 091/365] BUG: pad requires time Test for index length as well as empty data when padding. --- pysat/_instrument.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index b5091d70f..aa02e0734 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -3388,9 +3388,10 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, if not self._empty(pdata): # Test the data index, slicing if necessary pindex = self._index(data=pdata) - if pindex[-1] == self.index[0]: - pdata = self.__getitem__(slice(-1), data=pdata) - self.concat_data(pdata, prepend=True) + if len(pindex) > 0: + if pindex[-1] == self.index[0]: + pdata = self.__getitem__(slice(-1), data=pdata) + self.concat_data(pdata, prepend=True) del pdata if not self._empty(self._next_data) and not self.empty: @@ -3401,10 +3402,12 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, if not self._empty(ndata): # Test the data index, slicing if necessary nindex = self._index(data=ndata) - if nindex[0] == self.index[-1]: - ndata = self.__getitem__( - slice(1, len(ndata[self.index.name])), data=ndata) - self.concat_data(ndata, prepend=False) + if len(nindex) > 0: + if nindex[0] == self.index[-1]: + ndata = self.__getitem__( + slice(1, len(ndata[self.index.name])), + data=ndata) + self.concat_data(ndata, prepend=False) del ndata if len(self.index) > 0: From 065e77a3eb5040e70d80165effce3156b1b44591 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 15 Sep 2023 16:51:21 -0400 Subject: [PATCH 092/365] ENH: added `include` kwarg to `concat_data` Added a kwarg to `concat_data` that allows the currently loaded data to be inserted in any location within the new data list. Applied this new kwarg to reduce `concat_data` calls in the `load` method. --- pysat/_instrument.py | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index aa02e0734..20f812984 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -2427,7 +2427,7 @@ def copy(self): return inst_copy - def concat_data(self, new_data, prepend=False, **kwargs): + def concat_data(self, new_data, prepend=False, include=None, **kwargs): """Concatonate data to self.data for xarray or pandas as needed. Parameters @@ -2437,6 +2437,9 @@ def concat_data(self, new_data, prepend=False, **kwargs): prepend : bool If True, assign new data before existing data; if False append new data (default=False) + include : int or NoneType + Index at which `self.data` should be included in `new_data` or None + to use `prepend` (default=None) **kwargs : dict Optional keyword arguments passed to pds.concat or xr.concat @@ -2450,6 +2453,13 @@ def concat_data(self, new_data, prepend=False, **kwargs): For xarray, `dim=Instrument.index.name` is passed along to xarray.concat except if the user includes a value for dim as a keyword argument. + Examples + -------- + :: + + # Concatonate data before and after the existing Instrument data + inst.concat_data([prev_data, next_data], include=1) + """ # Add any concat_data kwargs for ckey in self.kwargs['concat_data'].keys(): @@ -2459,11 +2469,14 @@ def concat_data(self, new_data, prepend=False, **kwargs): # Order the data to be concatenated in a list if not isinstance(new_data, list): new_data = [new_data] - - if prepend: - new_data.append(self.data) + + if include is None: + if prepend: + new_data.append(self.data) + else: + new_data.insert(0, self.data) else: - new_data.insert(0, self.data) + new_data.insert(include, self.data) if self._concat_data_rtn.__name__.find('_pass_method') == 0: # There is no custom concat function, use the pysat standard method. @@ -3380,8 +3393,10 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, "by file."))) # Pad data based upon passed parameter + cdata = list() + include = None if not self._empty(self._prev_data) and not self.empty: - # __getitem__ used to handle any pandas/xarray differences in + # __getitem__ is used to handle any pandas/xarray differences in # data slicing pdata = self.__getitem__(slice(first_pad, self.index[0]), data=self._prev_data) @@ -3391,11 +3406,11 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, if len(pindex) > 0: if pindex[-1] == self.index[0]: pdata = self.__getitem__(slice(-1), data=pdata) - self.concat_data(pdata, prepend=True) - del pdata + cdata.append(pdata) + include = 1 if not self._empty(self._next_data) and not self.empty: - # __getitem__ used to handle any pandas/xarray differences in + # __getitem__ is used to handle any pandas/xarray differences in # data slicing ndata = self.__getitem__(slice(self.index[-1], last_pad), data=self._next_data) @@ -3407,8 +3422,13 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, ndata = self.__getitem__( slice(1, len(ndata[self.index.name])), data=ndata) - self.concat_data(ndata, prepend=False) - del ndata + cdata.append(ndata) + if include is None: + include = 0 + + # Concatonate the current, previous, and next data + if len(cdata) > 0: + self.concat_data(cdata, include=include) if len(self.index) > 0: self.data = self[first_pad:last_pad] From ed72675218ec67ab3a7adf6585a2855664b72b85 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 18 Sep 2023 10:10:51 -0400 Subject: [PATCH 093/365] TST: added test for `include` kwarg Added a test for the `include` kwarg in `concat_data`. --- pysat/tests/classes/cls_instrument_access.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py index edc8b759a..b66b11e4a 100644 --- a/pysat/tests/classes/cls_instrument_access.py +++ b/pysat/tests/classes/cls_instrument_access.py @@ -574,7 +574,8 @@ def test_inequality_reduced_object(self): @pytest.mark.parametrize("prepend, sort_dim_toggle", [(True, True), (True, False), (False, False)]) - def test_concat_data(self, prepend, sort_dim_toggle): + @pytest.mark.parametrize("include", [True, False]) + def test_concat_data(self, prepend, sort_dim_toggle, include): """Test `pysat.Instrument.data` concatenation. Parameters @@ -586,6 +587,8 @@ def test_concat_data(self, prepend, sort_dim_toggle): If True, sort variable names in pandas before concatenation. If False, do not sort for pandas objects. For xarray objects, rename the epoch if True. + include : bool + Use `include` kwarg instead of `prepend` for the same behaviour. """ @@ -611,6 +614,9 @@ def test_concat_data(self, prepend, sort_dim_toggle): data2 = data2.rename({self.xarray_epoch_name: 'Epoch2'}) self.testInst.data = self.testInst.data.rename( {self.xarray_epoch_name: 'Epoch2'}) + if include: + # To prepend new data, put existing data at end and vice versa + kwargs['include'] = 1 if prepend else 0 # Concat together self.testInst.concat_data(data2, **kwargs) From 2b53191a5c4355b71df57d01804858e378661178 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 18 Sep 2023 10:13:02 -0400 Subject: [PATCH 094/365] DOC: updated changelog Added a description of the pull to the changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 234631ee8..31c5ec17e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). clean method. * Added loading test with padding for Instruments. * Allow Instruments to define custom `concat_data` methods. + * Added `include` kwarg to `Instrument.concat_data` to expand allowed inputs. * Added data kwarg to the Instrument class `__getitem__` method and reduced memory usage in the `load` method. * Added a hidden method the Instrument class `_get_epoch_name_from_data` to From 9ef05b7218e5f1360a676a6652f561b5dd0661de Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 18 Sep 2023 10:28:46 -0400 Subject: [PATCH 095/365] STY: remove excess whitespace Removed extra whitespace. --- pysat/_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 20f812984..5405e1ee6 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -2469,7 +2469,7 @@ def concat_data(self, new_data, prepend=False, include=None, **kwargs): # Order the data to be concatenated in a list if not isinstance(new_data, list): new_data = [new_data] - + if include is None: if prepend: new_data.append(self.data) From 1a94949be501a340759cc89c894d5451f3e346eb Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 20 Sep 2023 17:39:39 -0400 Subject: [PATCH 096/365] ENH: improved meta delete Added `drop` methods to MetaLabels and MetaHeader classes, as well as `__delitem__` to Meta. For consistency and ease of access, also updated `Meta.__getitem__` to access MetaHeader attributes. --- pysat/_meta.py | 102 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/pysat/_meta.py b/pysat/_meta.py index 85d5c8d54..21fa40798 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -13,6 +13,7 @@ import pysat import pysat.utils._core as core_utils +from pysat.utils import listify from pysat.utils import testing @@ -674,6 +675,8 @@ def match_name(func, var_name, index_or_column): meta_row.at['children'] = '' meta_row.at['children'] = None return meta_row + elif key in self.header.global_attrs: + return getattr(self.header, key) else: raise KeyError("Key '{:}' not found in MetaData".format(key)) else: @@ -681,6 +684,55 @@ def match_name(func, var_name, index_or_column): "{}; ".format(key.__repr__()), "expected tuple, list, or str"])) + def __delitem__(self, key): + """Remove metadata. + + Maps to pandas DataFrame methods. + + Parameters + ---------- + key : str, tuple, or list + A single variable name, a tuple, or a list + + Raises + ------ + KeyError + If a properly formatted key is not present + + Examples + -------- + :: + + import pysat + inst = pysat.Instrument('pysat', 'testing2d') + inst.load(date=inst.inst_module._test_dates['']['']) + meta = inst.meta + + # Any data or label of Meta, including the top-level header + # attributes may be removed using `del` + del meta['uts'] + + """ + + try: + # This is a data variable + self.drop(key) + except KeyError: + keys = listify(key) + + for key in keys: + if key in self.data.columns: + # This is a metadata label + self.data = self.data.drop(key, axis=1) + + # Also drop this from Labels + self.labels.drop(key) + elif key in self.header.global_attrs: + self.header.drop(key) + else: + raise KeyError("{:} not found in Meta".format(repr(key))) + return + def __contains__(self, data_var): """Check variable name, not distinguishing by case. @@ -2029,6 +2081,37 @@ def default_values_from_attr(self, attr_name, data_type=None): return default_val + def drop(self, names): + """Removes data from MetaLabels. + + Parameters + ---------- + names : str or list-like + Attribute or MetaData name(s) + + Raises + ------ + AttributeError or KeyError + If any part of `names` is missing and cannot be dropped + + """ + # Ennsure the input is list-likee + names = listify(names) + + # Cycle through each name to drop + for name in names: + if name in self.label_attrs.keys(): + lname = self.label_attrs[name] + delattr(self, lname) + del self.label_type[lname] + del self.label_attrs[name] + else: + lname = getattr(self, name) + delattr(self, name) + del self.label_type[name] + del self.label_attrs[lname] + return + def update(self, lattr, lname, ltype): """Update MetaLabels with a new label. @@ -2207,6 +2290,25 @@ def __eq__(self, other): return True + def drop(self, names): + """Drop variables (names) from MetaHeader. + + Parameters + ---------- + names : list-like + List of strings specifying the variable names to drop + + """ + names = listify(names) + + for name in names: + # Delete the attribute + delattr(self, name) + + # Remove the attribute from the attribute list + self.global_attrs.pop(self.global_attrs.index(name)) + return + def to_dict(self): """Convert the header data to a dictionary. From c22785c4a6f0204f65ac7371e844c2d11920aa51 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 20 Sep 2023 17:40:24 -0400 Subject: [PATCH 097/365] TST: added drop and del tests Added unit and integration tests for the new `drop` and `__delitem__` methods. --- pysat/tests/test_meta.py | 19 +++++++++++++ pysat/tests/test_meta_header.py | 47 ++++++++++++++++++++++++++++++++- pysat/tests/test_meta_labels.py | 30 +++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index a4ec9aec8..fcf984b4a 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -510,6 +510,25 @@ def test_str(self, long_str, inst_kwargs): assert out.find('ND Metadata variables:') < 0 return + @pytest.mark.parametrize("del_key", ["uts", "units"]) + def test_del(self, del_key): + """Test deletion of Meta data. + + Parameters + ---------- + del_key : str + Key for Meta data, MetaHeater attribute, or MetaLabels label + + """ + # Set the meta data + self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'}) + + del self.meta[del_key] + + assert del_key not in self.meta.data.index + assert del_key not in self.meta.data.columns + return + def test_self_equality(self): """Test Meta equality for the same object.""" diff --git a/pysat/tests/test_meta_header.py b/pysat/tests/test_meta_header.py index 42008523a..401a34c51 100644 --- a/pysat/tests/test_meta_header.py +++ b/pysat/tests/test_meta_header.py @@ -121,12 +121,29 @@ def test_to_dict(self, header_data): assert header_data == out_dict return + def test_drop(self): + """Test the MetaHeader.drop method.""" + # Get the current output dict + out_dict = self.meta_header.to_dict() + + # Add an attribute + self.meta_header.PI = "pysat development team" + assert "PI" in self.meta_header.global_attrs + assert out_dict != self.meta_header.to_dict() + + # Remove the attribute + self.meta_header.drop('PI') + + # Ensure the current output matches the original output + assert out_dict == self.meta_header.to_dict() + return + # ---------------------------------------- # Test the integration with the Meta class @pytest.mark.parametrize("header_data", [{}, {"test": "value"}]) def test_init_metaheader_in_meta(self, header_data): - """Test changing case of meta labels after initialization. + """Test changing case of MetaHeader after Meta initialization. Parameters ---------- @@ -142,3 +159,31 @@ def test_init_metaheader_in_meta(self, header_data): # Ensure both initialization methods work the same assert meta.header == self.meta_header return + + def test_get_metaheader_in_meta(self): + """Test MetaHeader attribute retrieval from Meta.""" + + # Initalize the header data through the meta object + test_attr = "PI" + test_val = "pysat development team" + meta = pysat.Meta(header_data={test_attr: test_val}) + + # Test value retrieval from Meta and MetaHeader + assert getattr(meta.header, test_attr) == test_val + assert meta[test_attr] == test_val + return + + def test_delete_metaheader_in_meta(self): + """Test MetaHeader attribute deletion from Meta.""" + + # Initalize the header data through the meta object + test_attr = "PI" + test_val = "pysat development team" + meta = pysat.Meta(header_data={test_attr: test_val}) + + # Delete MetaHeader data + del meta[test_attr] + + # Test for empty MetaHeader + assert meta.header.to_dict() == {} + return diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py index ee4b6658e..3678dacd9 100644 --- a/pysat/tests/test_meta_labels.py +++ b/pysat/tests/test_meta_labels.py @@ -11,6 +11,7 @@ import pytest import pysat +from pysat.utils import listify from pysat.utils import testing @@ -185,6 +186,24 @@ def test_default_value_from_type_int_inputs(self, in_val, comp_val): return + @pytest.mark.parametrize("drop_labels", [["units", "fill_val"], "units"]) + def test_drop(self, drop_labels): + """Test successfull drop from MetaLabels. + + Parameters + ---------- + drop_labels : str or list-like + Label or labels to drop + + """ + # Drop the desired label(s) + self.meta_labels.drop(drop_labels) + + # Ensure the labels are missing + for dlabel in listify(drop_labels): + assert not hasattr(self.meta_labels, dlabel) + return + def test_update(self): """Test successful update of MetaLabels.""" self.meta_labels.update('new_label', 'new_name', int) @@ -197,6 +216,17 @@ def test_update(self): # ---------------------------------------- # Test the integration with the Meta class + def test_del_from_meta(self): + """Test successfull deletion of MetaLabels attribute from Meta.""" + # Delete the desired label + del_label = list(self.meta_labels.label_type.keys())[0] + del self.meta[del_label] + + # Ensure the label is missing + assert not hasattr(self.meta.labels, del_label) + assert del_label not in self.meta.data.columns + return + def test_change_case_of_meta_labels(self): """Test changing case of meta labels after initialization.""" From 16201f9aac09374cf18bc8d8715cd5384b03c657 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 20 Sep 2023 17:42:44 -0400 Subject: [PATCH 098/365] DOC: updated changelog Added a summary of the changes to the changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31c5ec17e..acab459a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). memory usage in the `load` method. * Added a hidden method the Instrument class `_get_epoch_name_from_data` to reduce code duplication. + * Added `__delitem__` to Meta and `drop` to MetaHeader and MetaLabels classes. + * Modified Meta to allow MetaHeader attribute access directly from Meta. * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. From 6c25355cd789728f3a6160f12c2b413b5441b37c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 20 Sep 2023 17:55:59 -0400 Subject: [PATCH 099/365] DOC: fixed docstring case Changed the docstring case to be imperative. --- pysat/_meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index 21fa40798..8b764f103 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -2082,7 +2082,7 @@ def default_values_from_attr(self, attr_name, data_type=None): return default_val def drop(self, names): - """Removes data from MetaLabels. + """Remove data from MetaLabels. Parameters ---------- From 001e3cbe4122e218d4daf4475eac3911ccd79753 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 20 Sep 2023 17:56:20 -0400 Subject: [PATCH 100/365] STY: removed extra whitespace Removed extra whitespace. --- pysat/tests/test_meta_header.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pysat/tests/test_meta_header.py b/pysat/tests/test_meta_header.py index 401a34c51..64af8b967 100644 --- a/pysat/tests/test_meta_header.py +++ b/pysat/tests/test_meta_header.py @@ -162,7 +162,6 @@ def test_init_metaheader_in_meta(self, header_data): def test_get_metaheader_in_meta(self): """Test MetaHeader attribute retrieval from Meta.""" - # Initalize the header data through the meta object test_attr = "PI" test_val = "pysat development team" @@ -175,7 +174,6 @@ def test_get_metaheader_in_meta(self): def test_delete_metaheader_in_meta(self): """Test MetaHeader attribute deletion from Meta.""" - # Initalize the header data through the meta object test_attr = "PI" test_val = "pysat development team" From 63ba91810a6da555eeb669cd2df216f973f2e715 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 20 Sep 2023 17:56:40 -0400 Subject: [PATCH 101/365] MAINT: fixed spelling Fixed spelling of 'successful'. --- pysat/tests/test_meta_labels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py index 3678dacd9..17195f174 100644 --- a/pysat/tests/test_meta_labels.py +++ b/pysat/tests/test_meta_labels.py @@ -188,7 +188,7 @@ def test_default_value_from_type_int_inputs(self, in_val, comp_val): @pytest.mark.parametrize("drop_labels", [["units", "fill_val"], "units"]) def test_drop(self, drop_labels): - """Test successfull drop from MetaLabels. + """Test successful drop from MetaLabels. Parameters ---------- @@ -217,7 +217,7 @@ def test_update(self): # Test the integration with the Meta class def test_del_from_meta(self): - """Test successfull deletion of MetaLabels attribute from Meta.""" + """Test successful deletion of MetaLabels attribute from Meta.""" # Delete the desired label del_label = list(self.meta_labels.label_type.keys())[0] del self.meta[del_label] From 16df694521b3eff60fb4ff308495e419b86c845f Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 28 Sep 2023 09:06:48 -0400 Subject: [PATCH 102/365] TST: added skip Added a skip to test memory issues. --- pysat/tests/test_meta.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index fcf984b4a..8be80db03 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -510,6 +510,7 @@ def test_str(self, long_str, inst_kwargs): assert out.find('ND Metadata variables:') < 0 return + @pytest.mark.skip("Does this test use too much memory?") @pytest.mark.parametrize("del_key", ["uts", "units"]) def test_del(self, del_key): """Test deletion of Meta data. From 93c7e41d05c1e9707437bdc9fbe2f6d32e5e34fc Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 28 Sep 2023 11:26:05 -0400 Subject: [PATCH 103/365] STY: updated drop Changed drop to act like `__delitem__` and changed `__delitem__` to redirect user to `drop`. --- pysat/_meta.py | 77 +++++++++++++++++++------------------------------- 1 file changed, 29 insertions(+), 48 deletions(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index 8b764f103..aa1287311 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -687,51 +687,13 @@ def match_name(func, var_name, index_or_column): def __delitem__(self, key): """Remove metadata. - Maps to pandas DataFrame methods. - - Parameters - ---------- - key : str, tuple, or list - A single variable name, a tuple, or a list - Raises ------ - KeyError - If a properly formatted key is not present - - Examples - -------- - :: - - import pysat - inst = pysat.Instrument('pysat', 'testing2d') - inst.load(date=inst.inst_module._test_dates['']['']) - meta = inst.meta - - # Any data or label of Meta, including the top-level header - # attributes may be removed using `del` - del meta['uts'] + NotImplementedError + Redirects the user to use `drop` """ - - try: - # This is a data variable - self.drop(key) - except KeyError: - keys = listify(key) - - for key in keys: - if key in self.data.columns: - # This is a metadata label - self.data = self.data.drop(key, axis=1) - - # Also drop this from Labels - self.labels.drop(key) - elif key in self.header.global_attrs: - self.header.drop(key) - else: - raise KeyError("{:} not found in Meta".format(repr(key))) - return + raise NotImplementedError('`del` not supported, use `Meta.drop`') def __contains__(self, data_var): """Check variable name, not distinguishing by case. @@ -1047,18 +1009,37 @@ def drop(self, names): Parameters ---------- - names : list-like - List of strings specifying the variable names to drop - - """ + names : str or list-like + String or list of strings specifying the variable names to drop - # Drop the lower dimension data - self.data = self._data.drop(names, axis=0) + Raises + ------ + KeyError + If any of the keys provided in `names` is not found in the + lower dimensional data, higher dimensional data, labels, or + header data - # Drop the higher dimension data + """ + # Ensure the input is list-like + names = listify(names) for name in names: if name in self._ho_data: + # Drop the higher dimension data self._ho_data.pop(name) + + if name in self.keys(): + # Drop the lower dimension data + self.data = self._data.drop(name, axis=0) + elif name in self.data.columns: + # This is a metadata label + self.data = self._data.drop(name, axis=1) + + # Also drop this from Labels + self.labels.drop(name) + elif key in self.header.global_attrs: + self.header.drop(name) + else: + raise KeyError("{:} not found in Meta".format(repr(name))) return def keep(self, keep_names): From 0c69fee3105063ac09a84aeda63bc2704a5e0e8e Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 28 Sep 2023 11:27:25 -0400 Subject: [PATCH 104/365] TST: added drop tests Added drop tests for labels, string input, higher-order input, and a combination of those with normal data variables. --- pysat/tests/test_meta.py | 69 ++++++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 8be80db03..408e32bc7 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -510,10 +510,8 @@ def test_str(self, long_str, inst_kwargs): assert out.find('ND Metadata variables:') < 0 return - @pytest.mark.skip("Does this test use too much memory?") - @pytest.mark.parametrize("del_key", ["uts", "units"]) - def test_del(self, del_key): - """Test deletion of Meta data. + def test_delitem(self): + """Test deletion of Meta data informs user of correct path. Parameters ---------- @@ -524,10 +522,9 @@ def test_del(self, del_key): # Set the meta data self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'}) - del self.meta[del_key] - - assert del_key not in self.meta.data.index - assert del_key not in self.meta.data.columns + with pytest.raises(NotImplementedError, + match='`del` not supported, use `Meta.drop`'): + del self.meta['uts'] return def test_self_equality(self): @@ -1208,8 +1205,33 @@ def test_meta_merge(self): meta_dict[label].__repr__()) return + @pytest.mark.parametrize("names", ['uts', ['uts', 'mlt'], 'units', + ['units', 'uts']]) + def test_meta_drop(self, names): + """Test successful deletion of meta data for different types of data. + + Parameters + ---------- + names : int + Number of variables to drop in a single go. + + """ + # Set meta data + self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'}) + + # Drop the values + self.meta.drop(names) + + # Test the successful deletion + for name in pysat.utils.listify(names): + if name in self.testInst.variables: + assert name not in self.meta.keys(), "didn't drop variable" + else: + assert name not in self.meta.data.columns, "didn't drop label" + return + @pytest.mark.parametrize("num_drop", [0, 1, 3]) - def test_meta_drop(self, num_drop): + def test_meta_num_drop(self, num_drop): """Test successful deletion of meta data for specific values. Parameters @@ -1547,6 +1569,35 @@ def test_update_epoch(self): # ------------------------------- # Tests for higher order metadata + # TODO(#789): remove tests for higher order meta + @pytest.mark.parametrize("names", ['series_profiles', + ['uts', 'series_profiles'], + ['units', 'series_profiles']]) + def test_meta_drop_higher_order(self, names): + """Test successful deletion of meta data for different types of data. + + Parameters + ---------- + names : str or list + Variables or labels to drop from metadata. + + """ + # Set meta data + self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'}) + + # Drop the values + self.meta.drop(names) + + # Test the successful deletion + for name in pysat.utils.listify(names): + if name in self.testInst.variables: + assert name not in self.meta.keys(), "didn't drop LO variable" + assert name not in self.meta.ho_data.keys(), \ + "didn't drop HO variable" + else: + assert name not in self.meta.data.columns, "didn't drop label" + return + # TODO(#789): remove tests for higher order meta @pytest.mark.parametrize('meta_dict', [ None, {'units': 'V', 'long_name': 'test name'}, From e5b2e8ea2a0be24659e6eb4cbbe1841065bb2024 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 28 Sep 2023 11:29:40 -0400 Subject: [PATCH 105/365] BUG: fixed variable name Fixed the variable name to reflect the drop standards. --- pysat/_meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index aa1287311..cb1bd0338 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -1036,7 +1036,7 @@ def drop(self, names): # Also drop this from Labels self.labels.drop(name) - elif key in self.header.global_attrs: + elif name in self.header.global_attrs: self.header.drop(name) else: raise KeyError("{:} not found in Meta".format(repr(name))) From 3e410f3aa537ee14db1d1eb9b7d4981c9a4f2b9e Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 28 Sep 2023 13:35:24 -0400 Subject: [PATCH 106/365] STY: remove delitem Remove delitem completely. --- pysat/_meta.py | 11 ----------- pysat/tests/test_meta.py | 17 ----------------- 2 files changed, 28 deletions(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index cb1bd0338..70191be83 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -684,17 +684,6 @@ def match_name(func, var_name, index_or_column): "{}; ".format(key.__repr__()), "expected tuple, list, or str"])) - def __delitem__(self, key): - """Remove metadata. - - Raises - ------ - NotImplementedError - Redirects the user to use `drop` - - """ - raise NotImplementedError('`del` not supported, use `Meta.drop`') - def __contains__(self, data_var): """Check variable name, not distinguishing by case. diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 408e32bc7..aa993792c 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -510,23 +510,6 @@ def test_str(self, long_str, inst_kwargs): assert out.find('ND Metadata variables:') < 0 return - def test_delitem(self): - """Test deletion of Meta data informs user of correct path. - - Parameters - ---------- - del_key : str - Key for Meta data, MetaHeater attribute, or MetaLabels label - - """ - # Set the meta data - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'}) - - with pytest.raises(NotImplementedError, - match='`del` not supported, use `Meta.drop`'): - del self.meta['uts'] - return - def test_self_equality(self): """Test Meta equality for the same object.""" From c9b104b511557b9bc3771b1da6970c30d2db1665 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 28 Sep 2023 14:33:15 -0400 Subject: [PATCH 107/365] BUG: replace del with drop Replace del with drop in the integration tests. --- pysat/tests/test_meta_header.py | 4 ++-- pysat/tests/test_meta_labels.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pysat/tests/test_meta_header.py b/pysat/tests/test_meta_header.py index 64af8b967..376777af9 100644 --- a/pysat/tests/test_meta_header.py +++ b/pysat/tests/test_meta_header.py @@ -172,7 +172,7 @@ def test_get_metaheader_in_meta(self): assert meta[test_attr] == test_val return - def test_delete_metaheader_in_meta(self): + def test_drop_metaheader_in_meta(self): """Test MetaHeader attribute deletion from Meta.""" # Initalize the header data through the meta object test_attr = "PI" @@ -180,7 +180,7 @@ def test_delete_metaheader_in_meta(self): meta = pysat.Meta(header_data={test_attr: test_val}) # Delete MetaHeader data - del meta[test_attr] + meta.drop(test_attr) # Test for empty MetaHeader assert meta.header.to_dict() == {} diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py index 17195f174..2693a176c 100644 --- a/pysat/tests/test_meta_labels.py +++ b/pysat/tests/test_meta_labels.py @@ -216,11 +216,11 @@ def test_update(self): # ---------------------------------------- # Test the integration with the Meta class - def test_del_from_meta(self): + def test_drop_from_meta(self): """Test successful deletion of MetaLabels attribute from Meta.""" # Delete the desired label del_label = list(self.meta_labels.label_type.keys())[0] - del self.meta[del_label] + self.meta.drop(del_label) # Ensure the label is missing assert not hasattr(self.meta.labels, del_label) From 069884ad9f47f85954c5561f1b23da683c61de81 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 28 Sep 2023 14:33:35 -0400 Subject: [PATCH 108/365] TST: added skip Added a skip to both new meta tests. --- pysat/tests/test_meta.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index aa993792c..502ee7bc8 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -1188,6 +1188,7 @@ def test_meta_merge(self): meta_dict[label].__repr__()) return + @pytest.mark.skip('See if memory is still an issue') @pytest.mark.parametrize("names", ['uts', ['uts', 'mlt'], 'units', ['units', 'uts']]) def test_meta_drop(self, names): @@ -1553,6 +1554,7 @@ def test_update_epoch(self): # Tests for higher order metadata # TODO(#789): remove tests for higher order meta + @pytest.mark.skip('See if memory is still an issue') @pytest.mark.parametrize("names", ['series_profiles', ['uts', 'series_profiles'], ['units', 'series_profiles']]) From 742902701405f3448b70de498f4201fb45909a8e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 15:06:22 -0400 Subject: [PATCH 109/365] ENH: pass overwrite through --- pysat/utils/registry.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py index 97193ccbc..ac42838e1 100644 --- a/pysat/utils/registry.py +++ b/pysat/utils/registry.py @@ -208,7 +208,7 @@ class Foo(object): return -def register_by_module(module): +def register_by_module(module, overwrite=False): """Register all sub-modules attached to input module. Enables instantiation of a third-party Instrument module using @@ -221,6 +221,9 @@ def register_by_module(module): module : Python module Module with one or more pysat.Instrument support modules attached as sub-modules to the input `module` + overwrite : bool + If True, an existing registration will be updated + with the new module information. Raises ------ @@ -249,7 +252,7 @@ def register_by_module(module): module_names = [module.__name__ + '.' + mod for mod in module_names] # Register all of the sub-modules - register(module_names) + register(module_names, overwrite=overwrite) return From 55956dbe8e16a3e3552c46653a34b0038290bf1f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 15:07:26 -0400 Subject: [PATCH 110/365] DOC: update links --- pysat/_files.py | 4 ++-- pysat/_instrument.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pysat/_files.py b/pysat/_files.py index 21203b136..337a0243f 100644 --- a/pysat/_files.py +++ b/pysat/_files.py @@ -46,8 +46,8 @@ class Files(object): `month`, `day`, etc. will be filled in as needed using python string formatting. The default file format structure is supplied in the instrument `list_files` routine. See - `pysat.files.parse_delimited_filenames` and - `pysat.files.parse_fixed_width_filenames` for more information. + `pysat.utils.files.parse_delimited_filenames` and + `pysat.utils.files.parse_fixed_width_filenames` for more information. (default=None) write_to_disk : bool If true, the list of Instrument files will be written to disk. diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 5405e1ee6..5e53cce32 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -75,8 +75,8 @@ class Instrument(object): `month`, `day`, etc. will be filled in as needed using python string formatting. The default file format structure is supplied in the instrument `list_files` routine. See - `pysat.files.parse_delimited_filenames` and - `pysat.files.parse_fixed_width_filenames` for more information. + `pysat.utils.files.parse_delimited_filenames` and + `pysat.utils.files.parse_fixed_width_filenames` for more information. The value will be None if not specified by the user at instantiation. (default=None) temporary_file_list : bool From 41b65a0b93fa7ae96ecd5c058212bfd8c6dbb290 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 15:11:22 -0400 Subject: [PATCH 111/365] DOC: update docstrings --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31c5ec17e..441e38162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,12 +15,14 @@ This project adheres to [Semantic Versioning](https://semver.org/). memory usage in the `load` method. * Added a hidden method the Instrument class `_get_epoch_name_from_data` to reduce code duplication. + * Added the overwrite kwarg to `utils.registry.register_by_module` * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. * Updated `Constellation.to_inst` method definition of coords, using dims to combine common dimensions instead. * Implement pyproject to manage metadata + * Updated docstring references to `pysat.utils.files` in other modules. [3.1.0] - 2023-05-31 -------------------- From 06aa436ee77470c966b5872d19638c3f657ffcad Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 15:17:27 -0400 Subject: [PATCH 112/365] MAINT: remove sphinx cap --- pyproject.toml | 2 +- test_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 745fcdc2f..73277f7ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ doc = [ "ipython", "m2r2", "numpydoc", - "sphinx < 7.0", + "sphinx", "sphinx_rtd_theme >= 1.2.2" ] diff --git a/test_requirements.txt b/test_requirements.txt index 823ea6406..99a4e74fb 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -8,5 +8,5 @@ numpydoc pysatSpaceWeather pytest-cov pytest-ordering -sphinx<7.0 +sphinx sphinx_rtd_theme From a5e4030c397aa3ff51fb2f4bc6ee66ae5c66ac8c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 15:17:48 -0400 Subject: [PATCH 113/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31c5ec17e..c09845187 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated `Constellation.to_inst` method definition of coords, using dims to combine common dimensions instead. * Implement pyproject to manage metadata + * Remove Sphinx cap [3.1.0] - 2023-05-31 -------------------- From 6e64411ac4d16eeac2dfdd0526e8b887101b2b5f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 16:02:37 -0400 Subject: [PATCH 114/365] STY: any / all --- pysat/_instrument.py | 23 ++++++++++++----------- pysat/_meta.py | 18 +++++++++--------- pysat/instruments/methods/general.py | 4 ++-- pysat/tests/test_utils_io.py | 5 +++-- pysat/utils/time.py | 4 ++-- 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 5405e1ee6..21a64eb34 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -409,9 +409,9 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', # Check to make sure value is reasonable if self.file_format is not None: # Check if it is an iterable string - if(not isinstance(self.file_format, str) - or (self.file_format.find("{") < 0) - or (self.file_format.find("}") < 0)): + if any([(not isinstance(self.file_format, str)), + (self.file_format.find("{") < 0), + (self.file_format.find("}") < 0)]): raise ValueError(''.join(['file format set to default, ', 'supplied string must be iterable ', '[{:}]'.format(self.file_format)])) @@ -883,8 +883,9 @@ def __getitem_xarray__(self, key, data=None): # Find secondary time indexes that may need to be sliced if len(data.indexes) > 1: for ind in data.indexes.keys(): - if(ind != epoch_names[0] and data.indexes[ind].dtype - == data.indexes[epoch_names[0]].dtype): + if all([ind != epoch_names[0], + data.indexes[ind].dtype + == data.indexes[epoch_names[0]].dtype]): epoch_names.append(ind) if isinstance(key, tuple): @@ -2270,8 +2271,8 @@ def bounds(self, value=None): if self.files.stop_date is not None: # Ensure the start and stop times intersect with # the file list - if(start <= self.files.stop_date - and stops[i] >= self.files.start_date): + if all([start <= self.files.stop_date, + stops[i] >= self.files.start_date]): good_bounds.append(i) if len(good_bounds) > 0: @@ -3843,8 +3844,8 @@ def download(self, start=None, stop=None, date_array=None, # Get current bounds curr_bound = self.bounds if self._iter_type == 'date': - if(curr_bound[0][0] == first_date - and curr_bound[1][0] == last_date): + if all([curr_bound[0][0] == first_date, + curr_bound[1][0] == last_date]): pysat.logger.info(' '.join(('Updating instrument', 'object bounds by date'))) self.bounds = (self.files.start_date, @@ -3859,8 +3860,8 @@ def download(self, start=None, stop=None, date_array=None, dsel2 = slice(last_date, last_date + dt.timedelta(hours=23, minutes=59, seconds=59)) - if(curr_bound[0][0] == self.files[dsel1][0] - and curr_bound[1][0] == self.files[dsel2][-1]): + if all([curr_bound[0][0] == self.files[dsel1][0], + curr_bound[1][0] == self.files[dsel2][-1]]): pysat.logger.info(' '.join(('Updating instrument', 'object bounds by file'))) dsel1 = slice(self.files.start_date, diff --git a/pysat/_meta.py b/pysat/_meta.py index 85d5c8d54..b04e4d7bc 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -182,8 +182,8 @@ def __init__(self, metadata=None, header_data=None, # Set the NaN export list self._export_nan = [] if export_nan is None else export_nan for lvals in labels.values(): - if(lvals[0] not in self._export_nan - and float in pysat.utils.listify(lvals[1])): + if all([lvals[0] not in self._export_nan, + float in pysat.utils.listify(lvals[1])]): self._export_nan.append(lvals[0]) # Set the labels @@ -427,9 +427,9 @@ def __setitem__(self, data_vars, input_dat): to_be_set, self.labels.label_type[iattr]): # If this is a disagreement between byte data # and an expected str, resolve it here - if(isinstance(to_be_set, bytes) - and str in pysat.utils.listify( - self.labels.label_type[iattr])): + if all([isinstance(to_be_set, bytes), + str in pysat.utils.listify( + self.labels.label_type[iattr])]): to_be_set = core_utils.stringify(to_be_set) else: # This type is incorrect, try casting it @@ -612,8 +612,8 @@ def match_name(func, var_name, index_or_column): return self.data.loc[new_index, new_name] except KeyError as kerr: # This may instead be a child variable, check for children - if(hasattr(self[new_index], 'children') - and self[new_index].children is None): + if all([hasattr(self[new_index], 'children'), + self[new_index].children is None]): raise kerr try: @@ -821,8 +821,8 @@ def _insert_default_values(self, data_var, data_type=None): for i, lattr in enumerate(self.labels.label_type.keys()): labels.append(getattr(self.labels, lattr)) lattrs.append(lattr) - if(isinstance(self.labels.label_type[lattr], tuple) - and data_type is not None): + if all([isinstance(self.labels.label_type[lattr], tuple), + data_type is not None]): need_data_type[lattr] = True else: need_data_type[lattr] = False diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py index 732163702..1d6f17343 100644 --- a/pysat/instruments/methods/general.py +++ b/pysat/instruments/methods/general.py @@ -131,8 +131,8 @@ def list_files(tag='', inst_id='', data_path='', format_str=None, new_out = out.asfreq('D') for i, out_month in enumerate(out.index): - if(out_month.month == emonth.month - and out_month.year == emonth.year): + if all([out_month.month == emonth.month, + out_month.year == emonth.year]): out_month = emonth crange = pds.date_range(start=out_month, periods=2, diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 7ac9c1e11..25c769e5b 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -991,8 +991,9 @@ def test_filter_netcdf4_metadata(self, remove, check_type, export_nan, assert mkey not in export_nan, \ "{:} should have been exported".format(repr(mkey)) else: - if(mkey in export_nan and not np.issubdtype(data_type, str) - and np.isnan(mdict[mkey])): + if all([mkey in export_nan, + not np.issubdtype(data_type, str), + np.isnan(mdict[mkey])]): assert np.isnan(fdict[mkey]) else: if mkey in check_type and fdict[mkey] != mdict[mkey]: diff --git a/pysat/utils/time.py b/pysat/utils/time.py index 4523aaf85..6ffa84d12 100644 --- a/pysat/utils/time.py +++ b/pysat/utils/time.py @@ -364,8 +364,8 @@ def filter_datetime_input(date): if hasattr(date, '__iter__'): out_date = [] for in_date in date: - if(in_date.tzinfo is not None - and in_date.utcoffset() is not None): + if all([in_date.tzinfo is not None, + in_date.utcoffset() is not None]): in_date = in_date.astimezone(tz=dt.timezone.utc) out_date.append(dt.datetime(in_date.year, in_date.month, From 911ac243881e2c7600b48ef15c9bb2ec33515093 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 16:02:44 -0400 Subject: [PATCH 115/365] STY: whitespace --- pysat/tests/test_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py index be2b38daa..c50c19dd1 100644 --- a/pysat/tests/test_files.py +++ b/pysat/tests/test_files.py @@ -586,7 +586,8 @@ def test_instrument_has_no_files(self): inst = pysat.Instrument(platform='pysat', name='testing', update_files=True) reload(pysat.instruments.pysat_testing) - assert(inst.files.files.empty) + assert inst.files.files.empty + return def test_instrument_has_files(self): From 8bfac919ae424c4ce127a626fa79e08d7a75dc64 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 16:03:10 -0400 Subject: [PATCH 116/365] MAINT: consistent version limits --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 73277f7ba..d65b86913 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ doc = [ "m2r2", "numpydoc", "sphinx", - "sphinx_rtd_theme >= 1.2.2" + "sphinx_rtd_theme" ] [project.urls] From 4a9d13a4b0ce552cdad5eb617ca19c9911e25c43 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 16:09:31 -0400 Subject: [PATCH 117/365] STY: any --- pysat/_constellation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysat/_constellation.py b/pysat/_constellation.py index 34e5866c9..c51a43fd3 100644 --- a/pysat/_constellation.py +++ b/pysat/_constellation.py @@ -680,10 +680,10 @@ def to_inst(self, common_coord=True, fill_method=None): # a common range and resolution. Note that # this will only happen if the coordinates # share the same names. - if(len(coords[new_coord]) - != len(cinst.data.coords[new_coord]) - or coords[new_coord].values - != cinst.data.coords[new_coord].values): + if any([len(cinst.data.coords[new_coord]) + != len(coords[new_coord]), + cinst.data.coords[new_coord].values + != coords[new_coord].values]): coords[new_coord] = establish_common_coord( [coords[new_coord].values, cinst.data.coords[new_coord].values], From 2f977402b7569491d688fe6dbf6eeeaed17a3387 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 2 Oct 2023 16:19:39 -0400 Subject: [PATCH 118/365] STY: use a slice --- pysat/_orbits.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysat/_orbits.py b/pysat/_orbits.py index 93effb84d..7f68f77bd 100644 --- a/pysat/_orbits.py +++ b/pysat/_orbits.py @@ -465,7 +465,8 @@ def _equa_breaks(self, orbit_index_period=24.0): # values new_ind = [] for idx in ind: - tidx, = np.where(lt_diff[(idx - 5):(idx + 6)] + sub_idx = slice((idx - 5), (idx + 6)) + tidx, = np.where(lt_diff[sub_idx] > 10 * typical_lt_diff) if len(tidx) != 0: @@ -473,9 +474,8 @@ def _equa_breaks(self, orbit_index_period=24.0): # Iterate over samples and check. for sub_tidx in tidx: # Look at time change vs local time change - if(ut_diff[idx - 5:idx + 6].iloc[sub_tidx] - < lt_diff[idx - 5:idx + 6].iloc[sub_tidx] - / orbit_index_period * self.orbit_period): + if(ut_diff[sub_idx].iloc[sub_tidx] * orbit_index_period + < lt_diff[sub_idx].iloc[sub_tidx] * self.orbit_period): # The change in UT is small compared to the change # in the orbit index this is flagged as a false From ee79d50002be2b4f6f25c442dec904b38c8039c3 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 10:11:45 -0400 Subject: [PATCH 119/365] MAINT: sphinx_rtd_theme minimum --- pyproject.toml | 2 +- test_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d65b86913..73277f7ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ doc = [ "m2r2", "numpydoc", "sphinx", - "sphinx_rtd_theme" + "sphinx_rtd_theme >= 1.2.2" ] [project.urls] diff --git a/test_requirements.txt b/test_requirements.txt index 99a4e74fb..dc679dcce 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -9,4 +9,4 @@ pysatSpaceWeather pytest-cov pytest-ordering sphinx -sphinx_rtd_theme +sphinx_rtd_theme>=1.2.2 From bc4cf0ee27dcb36fda61dacae30fa19a72915c1d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 10:11:54 -0400 Subject: [PATCH 120/365] MAINT: pandas cap --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 73277f7ba..9ec0b533b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "dask", "netCDF4", "numpy >= 1.12", - "pandas", + "pandas < 2.1.1", "portalocker", "pytest", "scipy", diff --git a/requirements.txt b/requirements.txt index 6671a1449..059c99271 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ dask netCDF4 numpy>=1.12 -pandas +pandas<2.1.1 portalocker pytest scipy From 7e5977e8d57596153cf6d7dd761ea392e7b72232 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 10:34:33 -0400 Subject: [PATCH 121/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c09845187..b547a3889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). to combine common dimensions instead. * Implement pyproject to manage metadata * Remove Sphinx cap + * Add pandas cap [3.1.0] - 2023-05-31 -------------------- From 7d00795819c57023b70cee4347e99234e56aa86d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 3 Oct 2023 10:57:55 -0400 Subject: [PATCH 122/365] Apply suggestions from code review --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ec0b533b..311c0382d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "dask", "netCDF4", "numpy >= 1.12", - "pandas < 2.1.1", + "pandas < 2.1", "portalocker", "pytest", "scipy", diff --git a/requirements.txt b/requirements.txt index 059c99271..c5577679a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ dask netCDF4 numpy>=1.12 -pandas<2.1.1 +pandas<2.1 portalocker pytest scipy From c5caa2a7ec559de583f0eca84763055abf19e53a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 11:09:40 -0400 Subject: [PATCH 123/365] MAINT: remove pysat_testing2d --- pysat/instruments/__init__.py | 2 +- pysat/instruments/methods/testing.py | 33 ---- pysat/instruments/pysat_testing2d.py | 231 --------------------------- 3 files changed, 1 insertion(+), 265 deletions(-) delete mode 100644 pysat/instruments/pysat_testing2d.py diff --git a/pysat/instruments/__init__.py b/pysat/instruments/__init__.py index d1fb406db..248e80f36 100644 --- a/pysat/instruments/__init__.py +++ b/pysat/instruments/__init__.py @@ -6,7 +6,7 @@ __all__ = ['pysat_ndtesting', 'pysat_netcdf', 'pysat_testing', 'pysat_testmodel', 'pysat_testing_xarray', - 'pysat_testing2d', 'pysat_testing2d_xarray'] + 'pysat_testing2d_xarray'] for inst in __all__: exec("from pysat.instruments import {x}".format(x=inst)) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 8c0b2838f..cb25480af 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -247,39 +247,6 @@ def initialize_test_meta(epoch_name, data_keys): 'Note the value_max is largest netCDF4 supports, ', 'but is lower than actual 64-bit int limit.'])} - # Children metadata required for 2D pandas. - # TODO(#789): Delete after removal of Meta children. - series_profile_meta = pysat.Meta() - series_profile_meta['series_profiles'] = {'desc': 'Testing series data.', - 'value_min': 0, - 'value_max': np.inf, - 'units': 'm/s'} - meta['series_profiles'] = {'meta': series_profile_meta, - 'value_min': 0., 'value_max': 25., 'units': 'km', - 'fill': np.nan, - 'desc': ''.join(['Testing series profiles ', - 'indexed by float.'])} - - # Children metadata required for 2D pandas. - # TODO(#789): Delete after removal of Meta children. - data_types = {'density': float, 'fraction': float, 'alt_profiles': float, - 'variable_profiles': float, 'profile_height': int, - 'variable_profile_height': int, 'images': int, 'x': int, - 'y': int, 'z': int, 'image_lat': float, 'image_lon': float} - alt_profile_meta = pysat.Meta() - alt_profile_meta['density'] = {'desc': 'Simulated density values.', - 'units': 'Log N/cc', - 'value_min': 0, 'value_max': np.inf} - alt_profile_meta['fraction'] = {'value_min': 0., 'value_max': 1., - 'desc': ''.join(['Simulated fractional O+ ', - 'composition.'])} - meta['alt_profiles'] = {'value_min': 0., 'value_max': 25., 'fill': np.nan, - 'desc': ''.join([ - 'Testing profile multi-dimensional data ', - 'indexed by float.']), - 'units': 'km', - 'meta': alt_profile_meta} - # Optional and standard metadata for xarray for var in data_keys: if var.find('variable_profiles') == 0: diff --git a/pysat/instruments/pysat_testing2d.py b/pysat/instruments/pysat_testing2d.py deleted file mode 100644 index 76819a6a4..000000000 --- a/pysat/instruments/pysat_testing2d.py +++ /dev/null @@ -1,231 +0,0 @@ -# -*- coding: utf-8 -*- -"""Produces fake instrument data for testing. - -.. deprecated:: 3.0.2 - Support for 2D pandas objects will be removed in 3.2.0+. This instrument - module simulates an object that will no longer be supported. - -""" - -import datetime as dt -import functools -import numpy as np -import warnings - -import pandas as pds - -import pysat -from pysat.instruments.methods import testing as mm_test - -platform = 'pysat' -name = 'testing2d' -tags = {'': 'Regular testing data set'} -inst_ids = {'': ['']} -_test_dates = {'': {'': dt.datetime(2009, 1, 1)}} - - -# Init method -def init(self, test_init_kwarg=None): - """Initialize the test instrument. - - Parameters - ---------- - self : pysat.Instrument - This object - test_init_kwarg : any - Testing keyword (default=None) - - """ - - warnings.warn(" ".join(["The instrument module `pysat_testing2d` has been", - "deprecated and will be removed in 3.2.0+. This", - "module simulates an object that will no longer be", - "supported."]), - DeprecationWarning, stacklevel=2) - - mm_test.init(self, test_init_kwarg=test_init_kwarg) - return - - -# Clean method -clean = mm_test.clean - -# Optional method, preprocess -preprocess = mm_test.preprocess - - -def load(fnames, tag='', inst_id='', malformed_index=False, - start_time=None, num_samples=864, test_load_kwarg=None, - max_latitude=90.): - """Load the test files. - - Parameters - ---------- - fnames : list - List of filenames - tag : str - Tag name used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. (default='') - inst_id : str - Instrument ID used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. (default='') - malformed_index : bool - If True, the time index will be non-unique and non-monotonic. - (default=False) - start_time : dt.timedelta or NoneType - Offset time of start time since midnight UT. If None, instrument data - will begin at midnight. - (default=None) - num_samples : int - Maximum number of times to generate. Data points will not go beyond the - current day. (default=864) - test_load_kwarg : any - Keyword used for pysat unit testing to ensure that functionality for - custom keywords defined in instrument support functions is working - correctly. (default=None) - max_latitude : float - Latitude simulated as `max_latitude` * cos(theta(t))`, where - theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.). - - Returns - ------- - data : pds.DataFrame - Testing data - meta : pysat.Meta - Testing metadata - - """ - - # Support keyword testing - pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg)))) - - # Create an artificial satellite data set - iperiod = mm_test.define_period() - drange = mm_test.define_range() - - # Using 100s frequency for compatibility with seasonal analysis unit tests - uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='100S', - start_time=start_time) - # Seed the DataFrame with a UT array - data = pds.DataFrame(np.mod(uts, 86400.), columns=['uts']) - - # Need to create simple orbits here. Have start of first orbit - # at 2009,1, 0 UT. 14.84 orbits per day. Figure out how far in time from - # the root start a measurement is and use that info to create a signal - # that is continuous from that start. Going to presume there are 5820 - # seconds per orbit (97 minute period). - time_delta = dates[0] - dt.datetime(2009, 1, 1) - - # MLT runs 0-24 each orbit - data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(), uts, - period=iperiod['lt'], - data_range=drange['lt']) - - # SLT, 20 second offset from `mlt`. - data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20, - uts, period=iperiod['lt'], - data_range=drange['lt']) - - # Create a fake longitude, resets every 6240 seconds. Sat moves at - # 360/5820 deg/s, Earth rotates at 360/86400, takes extra time to go - # around full longitude. - data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(), - uts, period=iperiod['lon'], - data_range=drange['lon']) - - # Create latitude signal for testing polar orbits - angle = mm_test.generate_fake_data(time_delta.total_seconds(), - uts, period=iperiod['angle'], - data_range=drange['angle']) - data['latitude'] = max_latitude * np.cos(angle) - - # Create constant altitude at 400 km - alt0 = 400.0 - data['altitude'] = alt0 * np.ones(data['latitude'].shape) - - # Dummy variable data for different types - data['string_dummy'] = ['test'] * len(data) - data['unicode_dummy'] = [u'test'] * len(data) - data['int8_dummy'] = np.ones(len(data), dtype=np.int8) - data['int16_dummy'] = np.ones(len(data), dtype=np.int16) - data['int32_dummy'] = np.ones(len(data), dtype=np.int32) - data['int64_dummy'] = np.ones(len(data), dtype=np.int64) - - if malformed_index: - mm_test._warn_malformed_kwarg() - index = mm_test.non_monotonic_index(index) - index = mm_test.non_unique_index(index) - - data.index = index - data.index.name = 'Epoch' - - # Higher rate time signal (for scalar >= 2). This time signal is used - # for 2D profiles associated with each time in main DataFrame. - num_profiles = 50 if num_samples >= 50 else num_samples - end_date = dates[0] + dt.timedelta(seconds=2 * num_profiles - 1) - high_rate_template = pds.date_range(dates[0], end_date, freq='2S') - - # Create a few simulated profiles. This results in a pds.DataFrame at - # each time with mixed variables. - profiles = [] - - # DataFrame at each time with numeric variables only - alt_profiles = [] - - # Series at each time, numeric data only - series_profiles = [] - - # Frame indexed by date times - frame = pds.DataFrame({'density': - data.iloc[0:num_profiles]['mlt'].values.copy(), - 'dummy_str': ['test'] * num_profiles, - 'dummy_ustr': [u'test'] * num_profiles}, - index=data.index[0:num_profiles], - columns=['density', 'dummy_str', 'dummy_ustr']) - - # Frame indexed by float - dd = np.arange(num_profiles) * 1.2 - ff = np.arange(num_profiles) / num_profiles - ii = np.arange(num_profiles) * 0.5 - frame_alt = pds.DataFrame({'density': dd, 'fraction': ff}, - index=ii, - columns=['density', 'fraction']) - - # Series version of storage - series_alt = pds.Series(dd, index=ii, name='series_profiles') - - for time in data.index: - frame.index = high_rate_template + (time - data.index[0]) - profiles.append(frame) - alt_profiles.append(frame_alt) - series_profiles.append(series_alt) - - # Store multiple data types into main frame - data['profiles'] = pds.Series(profiles, index=data.index) - data['alt_profiles'] = pds.Series(alt_profiles, index=data.index) - data['series_profiles'] = pds.Series(series_profiles, index=data.index) - - # Set the meta data - meta = mm_test.initialize_test_meta('epoch', data.keys()) - - # Reset profiles as children meta - profile_meta = pysat.Meta() - profile_meta['density'] = {'long_name': 'density', 'units': 'N/cc', - 'desc': 'Fake "density" signal for testing.', - 'value_min': 0., 'value_max': 25., - 'fill': np.nan} - profile_meta['dummy_str'] = {'long_name': 'dummy_str', - 'desc': 'String data for testing.'} - profile_meta['dummy_ustr'] = {'long_name': 'dummy_ustr', - 'desc': 'Unicode string data for testing.'} - - # Update profiles metadata with sub-variable information - meta['profiles'] = {'meta': profile_meta} - - return data, meta - - -list_files = functools.partial(mm_test.list_files, test_dates=_test_dates) -list_remote_files = functools.partial(mm_test.list_remote_files, - test_dates=_test_dates) -download = functools.partial(mm_test.download) From 8ea5b51dab386f6b7f3c7ae823a726e2b27821b8 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 11:11:58 -0400 Subject: [PATCH 124/365] MAINT: remove meta children --- pysat/_meta.py | 69 +------------------------------------------------- 1 file changed, 1 insertion(+), 68 deletions(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index 85d5c8d54..4b4fe55f5 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -500,47 +500,9 @@ def __setitem__(self, data_vars, input_dat): in_dict = input_data.to_dict() if 'children' in in_dict: child = in_dict.pop('children') - if child is not None: - # If there is data in the child object, assign it here - self._warn_meta_children() - self.ho_data[data_vars] = child # Remaining items are simply assigned via recursive call self[data_vars] = in_dict - elif isinstance(input_data, Meta): - # Dealing with a higher order data set - self._warn_meta_children() - # `data_vars` is only a single name here (by choice for support) - if (data_vars in self._ho_data) and input_data.empty: - # No actual metadata provided and there is already some - # higher order metadata in self - return - - # Get Meta approved variable data names - new_item_name = self.var_case_name(data_vars) - - # Ensure that Meta labels of object to be assigned are - # consistent with self. input_data accepts self's labels. - input_data.accept_default_labels(self) - - # Go through and ensure Meta object to be added has variable and - # attribute names consistent with other variables and attributes - # this covers custom attributes not handled by default routine - # above - attr_names = [item for item in input_data.attrs()] - input_data.data.columns = self.attr_case_name(attr_names) - - # Same thing for variables - input_data.data.index = self.var_case_name(input_data.data.index) - - # Assign Meta object now that things are consistent with Meta - # object settings, but first make sure there are lower dimension - # metadata parameters, passing in an empty dict fills in defaults - # if there is no existing metadata info. - self[new_item_name] = {} - - # Now add to higher order data - self._ho_data[new_item_name] = input_data return def __getitem__(self, key): @@ -565,7 +527,7 @@ def __getitem__(self, key): :: import pysat - inst = pysat.Instrument('pysat', 'testing2d') + inst = pysat.Instrument('pysat', 'testing') inst.load(date=inst.inst_module._test_dates['']['']) meta = inst.meta @@ -1568,26 +1530,6 @@ def to_dict(self, preserve_case=False): for orig_key in meta_dict: export_dict[case_key][orig_key] = meta_dict[orig_key] - # Higher Order Data - # TODO(#789): remove in pysat 3.2.0 - for key in self.ho_data: - if preserve_case: - case_key = self.var_case_name(key) - else: - case_key = key.lower() - - if case_key not in export_dict: - export_dict[case_key] = {} - for ho_key in self.ho_data[key].data.index: - if preserve_case: - case_ho_key = self.var_case_name(ho_key) - else: - case_ho_key = ho_key.lower() - - new_key = '_'.join((case_key, case_ho_key)) - export_dict[new_key] = \ - self.ho_data[key].data.loc[ho_key].to_dict() - return export_dict @classmethod @@ -1650,15 +1592,6 @@ def from_csv(cls, filename=None, col_names=None, sep=None, **kwargs): raise ValueError(''.join(['Unable to retrieve information from ', filename])) - # TODO(#789): remove in pysat 3.2.0 - def _warn_meta_children(self): - """Warn the user that higher order metadata is deprecated.""" - - warnings.warn(" ".join(["Support for higher order metadata has been", - "deprecated and will be removed in 3.2.0+."]), - DeprecationWarning, stacklevel=2) - return - class MetaLabels(object): """Store metadata labels for Instrument instance. From 6dd81bf5e5db856ce4d03d5907c08dc670082fbe Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 11:12:30 -0400 Subject: [PATCH 125/365] MAINT: update export --- pysat/utils/io.py | 145 +--------------------------------------------- 1 file changed, 2 insertions(+), 143 deletions(-) diff --git a/pysat/utils/io.py b/pysat/utils/io.py index 571b6a31d..0483d1653 100644 --- a/pysat/utils/io.py +++ b/pysat/utils/io.py @@ -986,152 +986,11 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4', nc_key) full_mdict[key] = meta_dict - # TODO(#913): Remove 2D support - if len(data.variables[key].dimensions) == 2: - # Part of a DataFrame to store within the main DataFrame - two_d_keys.append(key) - two_d_dims.append(data.variables[key].dimensions) - - if len(data.variables[key].dimensions) >= 3: - raise ValueError(' '.join(('pysat only supports 1D and 2D', + if len(data.variables[key].dimensions) >= 2: + raise ValueError(' '.join(('pysat only supports 1D', 'data in pandas. Please use', 'xarray for this file.'))) - # TODO(#913): Remove 2D support - # We now have a list of keys that need to go into a dataframe, - # could be more than one, collect unique dimensions for 2D keys. - for dim in set(two_d_dims): - # First or second dimension could be epoch. Use other - # dimension name as variable name. - if dim[0] == epoch_name: - obj_key = dim[1] - elif dim[1] == epoch_name: - obj_key = dim[0] - else: - estr = ''.join(['Epoch label: "', epoch_name, '"', - ' was not found in loaded dimensions [', - ', '.join(dim), ']']) - raise KeyError(estr) - - # Collect variable names associated with dimension - idx_bool = [dim == i for i in two_d_dims] - idx, = np.where(np.array(idx_bool)) - obj_var_keys = [] - clean_var_keys = [] - for i in idx: - obj_var_keys.append(two_d_keys[i]) - clean_var_keys.append( - two_d_keys[i].split(obj_key + '_')[-1]) - - # Figure out how to index this data, it could provide its - # own index - or we may have to create simple integer based - # DataFrame access. If the dimension is stored as its own - # variable then use that info for index. - if obj_key in obj_var_keys: - # String used to indentify dimension also in - # data.variables will be used as an index. - index_key_name = obj_key - - # If the object index uses UNIX time, process into - # datetime index. - if data.variables[obj_key].getncattr( - name_label) == epoch_name: - # Found the name to be used in DataFrame index - index_name = epoch_name - time_index_flag = True - else: - time_index_flag = False - - # Label to be used in DataFrame index - index_name = data.variables[obj_key].getncattr( - name_label) - else: - # Dimension is not itself a variable - index_key_name = None - - # Iterate over the variables and grab metadata - dim_meta_data = {} - - # Store attributes in metadata, except for the dimension name. - for key, clean_key in zip(obj_var_keys, clean_var_keys): - meta_dict = {} - for nc_key in data.variables[key].ncattrs(): - meta_dict[nc_key] = data.variables[key].getncattr( - nc_key) - - dim_meta_data[clean_key] = meta_dict - - dim_meta_dict = {'meta': dim_meta_data} - - # Add top level meta - if index_key_name is not None: - for nc_key in data.variables[obj_key].ncattrs(): - dim_meta_dict[nc_key] = data.variables[ - obj_key].getncattr(nc_key) - full_mdict[obj_key] = dim_meta_dict - - # Iterate over all variables with this dimension - # data storage, whole shebang. - loop_dict = {} - - # List holds a series of slices, parsed from dict above. - loop_list = [] - for key, clean_key in zip(obj_var_keys, clean_var_keys): - loop_dict[clean_key] = data.variables[ - key][:, :].flatten(order='C') - - # Find the number of time values - loop_lim = data.variables[obj_var_keys[0]].shape[0] - - # Find the number of values per time - step = len(data.variables[obj_var_keys[0]][0, :]) - - # Check if there is an index we should use - if not (index_key_name is None): - time_var = loop_dict.pop(index_key_name) - if time_index_flag: - # Create datetime index from data - time_var = pds.to_datetime(time_var, unit=epoch_unit, - origin=epoch_origin) - new_index = time_var - new_index_name = index_name - else: - # Using integer indexing if no index identified - new_index = np.arange((loop_lim * step), - dtype=np.int64) % step - new_index_name = 'index' - - # Load all data into frame - if len(loop_dict.keys()) > 1: - loop_frame = pds.DataFrame(loop_dict, - columns=clean_var_keys) - if obj_key in loop_frame: - del loop_frame[obj_key] - - # Break massive frame into bunch of smaller frames - for i in np.arange(loop_lim, dtype=np.int64): - loop_list.append(loop_frame.iloc[(step * i): - (step * (i + 1)), :]) - loop_list[-1].index = new_index[(step * i): - (step * (i + 1))] - loop_list[-1].index.name = new_index_name - else: - loop_frame = pds.Series(loop_dict[clean_var_keys[0]], - name=obj_var_keys[0]) - - # Break massive series into bunch of smaller series - for i in np.arange(loop_lim, dtype=np.int64): - loop_list.append(loop_frame.iloc[(step * i): - (step * (i + 1))]) - loop_list[-1].index = new_index[(step * i): - (step * (i + 1))] - loop_list[-1].index.name = new_index_name - - # Add 2D object data, all based on a unique dimension within - # netCDF, to loaded data dictionary. - loaded_vars[obj_key] = loop_list - del loop_list - # Prepare dataframe index for this netcdf file if epoch_name not in loaded_vars.keys(): estr = ''.join(['Epoch label: "', epoch_name, '"', From f39fa5ec8c0ff8a7b8b68878e665886fee4cf914 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 11:13:16 -0400 Subject: [PATCH 126/365] MAINT: remove 2d pandas tests --- pysat/tests/classes/cls_instrument_access.py | 136 ---- pysat/tests/test_instrument.py | 47 -- pysat/tests/test_instrument_index.py | 3 +- pysat/tests/test_instrument_padding.py | 1 - pysat/tests/test_instruments.py | 3 +- pysat/tests/test_meta.py | 642 +------------------ pysat/tests/test_meta_labels.py | 26 - pysat/tests/test_utils_coords.py | 12 +- pysat/tests/test_utils_io.py | 94 +-- 9 files changed, 16 insertions(+), 948 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py index b66b11e4a..5e89a7d88 100644 --- a/pysat/tests/classes/cls_instrument_access.py +++ b/pysat/tests/classes/cls_instrument_access.py @@ -958,142 +958,6 @@ def test_basic_variable_renaming(self, lowercase, mapper): assert key not in self.testInst.meta.keys() return - @pytest.mark.parametrize("mapper", [ - {'profiles': {'density': 'ionization'}}, - {'profiles': {'density': 'mass'}, - 'alt_profiles': {'density': 'volume'}}, - str.upper]) - def test_ho_pandas_variable_renaming(self, mapper): - """Test rename of higher order pandas variable. - - Parameters - ---------- - mapper : dict or function - A function or dict that maps how the variables will be renamed. - - """ - # TODO(#789): Remove when meta children support is dropped. - - # Initialize the testing dict - if isinstance(mapper, dict): - values = mapper - else: - values = {var: mapper(var) for var in self.testInst.variables} - - # Check for pysat_testing2d instrument - if self.testInst.platform == 'pysat': - if self.testInst.name == 'testing2d': - self.testInst.load(self.ref_time.year, self.ref_doy, - use_header=True) - self.testInst.rename(mapper) - for key in values: - for ikey in values[key]: - # Check column name unchanged - assert key in self.testInst.data - assert key in self.testInst.meta - - # Check for new name in HO data - check_var = self.testInst.meta[key]['children'] - - if isinstance(values[key], dict): - map_val = values[key][ikey] - else: - map_val = mapper(ikey) - - assert map_val in self.testInst[0, key] - assert map_val in check_var - - # Ensure old name not present - assert ikey not in self.testInst[0, key] - if map_val.lower() != ikey: - assert ikey not in check_var - return - - @pytest.mark.parametrize("values", [{'profiles': - {'help': 'I need somebody'}}, - {'fake_profi': - {'help': 'Not just anybody'}}, - {'wrong_profile': - {'help': 'You know I need someone'}, - 'fake_profiles': - {'Beatles': 'help!'}, - 'profiles': - {'density': 'valid_change'}}, - {'fake_profile': - {'density': 'valid HO change'}}, - {'Nope_profiles': - {'density': 'valid_HO_change'}}]) - def test_ho_pandas_unknown_variable_error_renaming(self, values): - """Test higher order pandas variable rename raises error if unknown. - - Parameters - ---------- - values : dict - Variables to be renamed. A dict where each key is the current - variable and its value is the new variable name. - - """ - # TODO(#789): Remove when meta children support is dropped. - - # Check for pysat_testing2d instrument - if self.testInst.platform == 'pysat': - if self.testInst.name == 'testing2d': - self.testInst.load(self.ref_time.year, self.ref_doy, - use_header=True) - - # Check for error for unknown column or HO variable name - testing.eval_bad_input(self.testInst.rename, ValueError, - "cannot rename", [values]) - else: - pytest.skip("Not implemented for this instrument") - return - - @pytest.mark.parametrize("values", [{'profiles': {'density': 'Ionization'}}, - {'profiles': {'density': 'MASa'}, - 'alt_profiles': - {'density': 'VoLuMe'}}]) - def test_ho_pandas_variable_renaming_lowercase(self, values): - """Test rename higher order pandas variable uses lowercase. - - Parameters - ---------- - values : dict - Variables to be renamed. A dict where each key is the current - variable and its value is the new variable name. - - """ - # TODO(#789): Remove when meta children support is dropped. - - # Check for pysat_testing2d instrument - if self.testInst.platform == 'pysat': - if self.testInst.name == 'testing2d': - self.testInst.load(self.ref_time.year, self.ref_doy, - use_header=True) - self.testInst.rename(values) - for key in values: - for ikey in values[key]: - # Check column name unchanged - assert key in self.testInst.data - assert key in self.testInst.meta - - # Check for new name in HO data - test_val = values[key][ikey] - assert test_val in self.testInst[0, key] - check_var = self.testInst.meta[key]['children'] - - # Case insensitive check - assert values[key][ikey] in check_var - - # Ensure new case in there - check_var = check_var[values[key][ikey]].name - assert values[key][ikey] == check_var - - # Ensure old name not present - assert ikey not in self.testInst[0, key] - check_var = self.testInst.meta[key]['children'] - assert ikey not in check_var - return - def test_generic_meta_translator(self): """Test `generic_meta_translator`.""" diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 705d3eaa5..18861ca59 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -12,7 +12,6 @@ import pysat import pysat.instruments.pysat_ndtesting import pysat.instruments.pysat_testing -import pysat.instruments.pysat_testing2d import pysat.instruments.pysat_testing_xarray from pysat.tests.classes.cls_instrument_access import InstAccessTests @@ -228,32 +227,6 @@ def teardown_method(self): return -# TODO(#908): remove below class when pysat_testing2d is removed. -class TestBasics2D(TestBasics): - """Basic tests for 2D pandas `pysat.Instrument`.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - - reload(pysat.instruments.pysat_testing2d) - self.testInst = pysat.Instrument(platform='pysat', name='testing2d', - num_samples=50, - clean_level='clean', - update_files=True, - use_header=True, - **self.testing_kwargs) - self.ref_time = pysat.instruments.pysat_testing2d._test_dates[''][''] - self.ref_doy = int(self.ref_time.strftime('%j')) - self.out = None - return - - def teardown_method(self): - """Clean up the unit test environment after each method.""" - - del self.testInst, self.out, self.ref_time, self.ref_doy - return - - class TestBasicsNDXarray(TestBasics): """Basic tests for ND xarray `pysat.Instrument`. @@ -815,23 +788,3 @@ def test_load_use_header(self): # Evaluate the warning output self.eval_warnings() return - - def test_set_2d_pandas_data(self): - """Check that setting 2D data for pandas raises a DeprecationWarning.""" - - test_inst = pysat.Instrument('pysat', 'testing2d', use_header=True) - test_date = pysat.instruments.pysat_testing2d._test_dates[''][''] - test_inst.load(date=test_date) - with warnings.catch_warnings(record=True) as war: - test_inst['new_profiles'] = 2 * test_inst['profiles'] - - warn_msgs = [" ".join(["Support for 2D pandas instrument", - "data has been deprecated and will", - "be removed in 3.2.0+."])] - - # Ensure the minimum number of warnings were raised. - assert len(war) >= len(warn_msgs) - - # Test the warning messages, ensuring each attribute is present. - testing.eval_warnings(war, warn_msgs) - return diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py index ea41140a5..ac87a74ce 100644 --- a/pysat/tests/test_instrument_index.py +++ b/pysat/tests/test_instrument_index.py @@ -107,8 +107,7 @@ def eval_warnings(self): return # TODO(#1094): Remove in pysat 3.2.0, potentially with class - @pytest.mark.parametrize('name', ['testing', 'ndtesting', 'testing_xarray', - 'testing2d']) + @pytest.mark.parametrize('name', ['testing', 'ndtesting', 'testing_xarray']) def test_kwarg_malformed_index(self, name): """Test deprecation of `malformed_index` kwarg. diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py index 89610277b..59e02e687 100644 --- a/pysat/tests/test_instrument_padding.py +++ b/pysat/tests/test_instrument_padding.py @@ -10,7 +10,6 @@ import pysat import pysat.instruments.pysat_ndtesting import pysat.instruments.pysat_testing -import pysat.instruments.pysat_testing2d import pysat.instruments.pysat_testing_xarray from pysat.utils import testing from pysat.utils.time import filter_datetime_input diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index d5cede4e2..b1f03a42d 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -275,8 +275,7 @@ def test_old_pytest_mark_presence(self): assert "download" in mark_names - @pytest.mark.parametrize("inst_module", ['pysat_testing2d', - 'pysat_testing_xarray', + @pytest.mark.parametrize("inst_module", ['pysat_testing_xarray', 'pysat_testing2d_xarray']) def test_deprecated_instruments(self, inst_module): """Check that instantiating old instruments raises a DeprecationWarning. diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index a4ec9aec8..9269046ea 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -201,31 +201,6 @@ def test_getitem_w_index(self): assert str(ierr).find('expected tuple, list, or str') >= 0 return - # TODO(#913): remove tests for 2D metadata - @pytest.mark.parametrize("parent_child", [ - (['alt_profiles', 'profiles'], 'density'), - (['alt_profiles', 'profiles'], ['density', 'dummy_str']), - (['alt_profiles', 'profiles'], 'density', 'units'), - (['alt_profiles', 'profiles'], 'density', ['units', 'long_name'])]) - def test_getitem_w_ho_child_slicing(self, parent_child): - """Test raises NotImplementedError with parent/child slicing. - - Parameters - ---------- - parent_child : list - List of inputs with unsupported parent/child slicing options - - """ - - # Set the meta object - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'}) - - with pytest.raises(NotImplementedError) as ierr: - self.meta[parent_child] - - assert str(ierr).find("retrieve child meta data from multiple") >= 0 - return - def test_concat_strict_w_collision(self): """Test raises KeyError when new meta names overlap.""" @@ -244,25 +219,6 @@ def test_concat_strict_w_collision(self): return - # TODO(#913): remove tests for 2d metadata - def test_concat_strict_w_ho_collision(self): - """Test raises KeyError when higher-order variable nams overlap.""" - - # Set the meta object - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'}) - - # Create a second object with the same higher-order data variables - concat_meta = pysat.Meta() - for dvar in self.meta.keys_nD(): - concat_meta[dvar] = self.meta[dvar] - - # Test the error message - testing.eval_bad_input( - self.meta.concat, KeyError, - 'Duplicated keys (variable names) in Meta.keys()', [concat_meta], - {'strict': True}) - return - def test_multiple_meta_assignment_error(self): """Test that assignment of multiple metadata raises a ValueError.""" @@ -318,20 +274,6 @@ def test_meta_csv_load_w_errors(self, bad_key, bad_val, err_msg): input_kwargs=kwargs) return - # TODO(#913): remove tests for 2D metadata - def test_meta_rename_bad_ho_input(self): - """Test raises ValueError when treating normal data like HO data.""" - - # Initialize the meta data - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'}) - - # Set a bad mapping dictionary - mapper = {'mlt': {'mlt_profile': 'mlt_density_is_not_real'}} - - testing.eval_bad_input(self.meta.rename, ValueError, - "unknown mapped value at 'mlt'", [mapper]) - return - # ------------------------- # Test the Warning messages @@ -459,11 +401,9 @@ def test_repr(self): assert out.find('Meta(') >= 0 return - # TODO(#913): remove tests for 2d metadata @pytest.mark.parametrize('long_str', [True, False]) @pytest.mark.parametrize('inst_kwargs', - [None, {'platform': 'pysat', 'name': 'testing'}, - {'platform': 'pysat', 'name': 'testing2d'}]) + [None, {'platform': 'pysat', 'name': 'testing'}]) def test_str(self, long_str, inst_kwargs): """Test long string output with custom meta data. @@ -501,10 +441,7 @@ def test_str(self, long_str, inst_kwargs): else: assert out.find('Standard Metadata variables:') < 0 - if inst_kwargs is not None and inst_kwargs['name'] == 'testing2d': - assert out.find('ND Metadata variables:') > 0 - else: - assert out.find('ND Metadata variables:') < 0 + assert out.find('ND Metadata variables:') < 0 else: assert out.find('Standard Metadata variables:') < 0 assert out.find('ND Metadata variables:') < 0 @@ -536,7 +473,7 @@ def test_equality(self): return # TODO(#908): remove tests for deprecated instruments - @pytest.mark.parametrize("inst_name", ["testing", "testing2d", + @pytest.mark.parametrize("inst_name", ["testing", "ndtesting", "testing_xarray", "testmodel"]) def test_equality_w_copy(self, inst_name): @@ -576,41 +513,6 @@ def test_inequality(self, emeta): assert emeta != self.meta, "meta equality not detectinng differences" return - # TODO(#789): remove tests for higher order meta - @pytest.mark.parametrize('val_dict', [ - {'units': 'U', 'long_name': 'HO Val', 'radn': 'raiden'}, - {'units': 'MetaU', 'long_name': 'HO Val'}]) - def test_inequality_with_higher_order_meta(self, val_dict): - """Test inequality with higher order metadata. - - Parameters - ---------- - val_dict : dict - Information to be added to higher order metadata variable - - """ - - meta_dict = {'units': {'ho_val': 'U', 'ho_prof': 'e-'}, - 'long_name': {'ho_val': 'HO Val', 'ho_prof': 'HO Profile'}} - - # Set the default meta object - self.meta['ho_data'] = pysat.Meta(pds.DataFrame(meta_dict)) - - # Set the altered meta object - cmeta = pysat.Meta() - - for vkey in val_dict.keys(): - if vkey in meta_dict.keys(): - meta_dict[vkey]['ho_val'] = val_dict[vkey] - else: - meta_dict[vkey] = {'ho_val': val_dict[vkey]} - - cmeta['ho_data'] = pysat.Meta(pds.DataFrame(meta_dict)) - - # Evaluate the inequality - assert cmeta != self.meta - return - @pytest.mark.parametrize("label_key", ["units", "name", "notes", "desc", "min_val", "max_val", "fill_val"]) def test_value_inequality(self, label_key): @@ -648,7 +550,7 @@ def test_value_inequality(self, label_key): return # TODO(#908): remove tests for deprecated instruments - @pytest.mark.parametrize("inst_name", ["testing", "testing2d", + @pytest.mark.parametrize("inst_name", ["testing", "ndtesting", "testing_xarray", "testmodel"]) def test_pop(self, inst_name): @@ -793,8 +695,7 @@ def test_multiple_meta_assignment(self, custom_attr, assign_type): self.eval_meta_settings() return - # TODO(#913): remove tests for 2D metadata - @pytest.mark.parametrize('inst_name', ['testing', 'testing2d']) + @pytest.mark.parametrize('inst_name', ['testing']) @pytest.mark.parametrize('num_mvals', [0, 1, 3]) @pytest.mark.parametrize('num_dvals', [0, 1, 3]) def test_selected_meta_retrieval(self, inst_name, num_mvals, num_dvals): @@ -962,8 +863,7 @@ def test_meta_immutable_at_instrument_instantiation(self): return - # TODO(#913): remove tests for 2D metadata - @pytest.mark.parametrize('inst_name', ['testing', 'testing2d']) + @pytest.mark.parametrize('inst_name', ['testing']) def test_assign_nonstandard_metalabels(self, inst_name): """Test labels do not conform to the standard values if set that way. @@ -1524,449 +1424,6 @@ def test_update_epoch(self): return - # ------------------------------- - # Tests for higher order metadata - - # TODO(#789): remove tests for higher order meta - @pytest.mark.parametrize('meta_dict', [ - None, {'units': 'V', 'long_name': 'test name'}, - {'units': 'V', 'long_name': 'test name', - 'meta': pysat.Meta(metadata=pds.DataFrame( - {'units': {'dummy_frame1': 'A', 'dummy_frame2': ''}, - 'desc': {'dummy_frame1': '', - 'dummy_frame2': 'A filler description'}, - 'long_name': {'dummy_frame1': 'Dummy 1', - 'dummy_frame2': 'Dummy 2'}}))}, - {'units': 'V', 'long_name': 'test name', 'bananas': 0, - 'meta': pysat.Meta(metadata=pds.DataFrame( - {'units': {'dummy_frame1': 'A', 'dummy_frame2': ''}, - 'desc': {'dummy_frame1': '', - 'dummy_frame2': 'A filler description'}, - 'long_name': {'dummy_frame1': 'Dummy 1', - 'dummy_frame2': 'Dummy 2'}, - 'bananas': {'dummy_frame1': 1, 'dummy_frame2': 2}}))}]) - def test_inst_ho_data_assignment(self, meta_dict): - """Test the assignment of the higher order metadata. - - Parameters - ---------- - meta_dict : dict or NoneType - Dictionary used to create test metadata - - """ - - # Initialize the Meta data - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'}) - - # Alter the Meta data - frame = pds.DataFrame({fkey: np.arange(10) for fkey in self.frame_list}, - columns=self.frame_list) - inst_data = [frame for i in range(self.testInst.index.shape[0])] - self.dval = 'test_val' - - if meta_dict is None: - self.testInst[self.dval] = inst_data - meta_dict = {'units': '', 'long_name': self.dval, 'desc': ''} - else: - meta_dict.update({'data': inst_data}) - self.testInst[self.dval] = meta_dict - - # Remove data key for evaluation - del meta_dict['data'] - - self.meta = self.testInst.meta - - # Test the ND metadata results - self.eval_ho_meta_settings(meta_dict) - return - - # TODO(#789): remove tests for higher order meta - @pytest.mark.parametrize("num_ho, num_lo", [(1, 1), (2, 2)]) - def test_assign_mult_higher_order_meta_from_dict(self, num_ho, num_lo): - """Test assign higher order metadata from dict with multiple types. - - Parameters - ---------- - num_ho : int - Number of higher order data values to initialize - num_lo : int - Number of lower order data valuess to initialize - - """ - - # Initialize the lower order evaluation data - self.default_val['units'] = 'U' - - # Initialize the higher-order meta data - ho_meta = pysat.Meta() - for flist in self.frame_list: - ho_meta[flist] = {'units': 'U', 'long_name': flist} - - # Initialize the meta dict for setting the data values - dvals = ['higher_{:d}'.format(i) for i in range(num_ho)] - dvals.extend(['lower_{:d}'.format(i) for i in range(num_lo)]) - - meta_dict = {'units': ['U' for i in range(len(dvals))], - 'long_name': [val for val in dvals], - 'meta': [ho_meta for i in range(num_ho)]} - meta_dict['meta'].extend([None for i in range(num_lo)]) - - # Assign and test the meta data - self.meta[dvals] = meta_dict - - for i, self.dval in enumerate(dvals): - if i < num_ho: - eval_dict = {label: meta_dict[label][i] - for label in meta_dict.keys()} - self.eval_ho_meta_settings(eval_dict) - else: - self.eval_meta_settings() - return - - # TODO(#789): remove tests for higher order meta - def test_inst_ho_data_assign_meta_then_data(self): - """Test assignment of higher order metadata before assigning data.""" - - # Initialize the Meta data - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'}) - - # Alter the Meta data - frame = pds.DataFrame({fkey: np.arange(10) for fkey in self.frame_list}, - columns=self.frame_list) - inst_data = [frame for i in range(self.testInst.index.shape[0])] - meta_dict = {'data': inst_data, 'units': 'V', 'long_name': 'The Doors', - 'meta': pysat.Meta(metadata=pds.DataFrame( - {'units': {dvar: "{:d}".format(i) - for i, dvar in enumerate(self.frame_list)}, - 'desc': {dvar: "{:s} desc".format(dvar) - for dvar in self.frame_list}, - 'long_name': {dvar: dvar - for dvar in self.frame_list}}))} - self.dval = 'test_data' - - # Assign the metadata - self.testInst[self.dval] = meta_dict - - # Alter the data - self.testInst[self.dval] = inst_data - - # Test the ND metadata results - self.meta = self.testInst.meta - del meta_dict['data'] - self.eval_ho_meta_settings(meta_dict) - return - - # TODO(#913): remove tests for 2D metadata - def test_inst_ho_data_assign_meta_different_labels(self): - """Test the higher order assignment of custom metadata labels.""" - - # Initialize the Meta data - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'}) - - # Alter the higher order metadata - ho_meta = pysat.Meta(labels={'units': ('barrels', str), - 'desc': ('Monkeys', str), - 'meta': ('meta', object)}) - self.frame_list = list( - self.testInst.meta['profiles']['children'].keys()) - for dvar in self.frame_list: - if dvar == 'density': - ho_meta[dvar] = {'barrels': 'A'} - else: - ho_meta[dvar] = {'Monkeys': 'are fun', 'bananas': 2} - - # The 'units', 'desc' and other labels used on self.testInst are - # applied to the input metadata to ensure everything remains - # consistent across the object. - self.testInst['profiles'] = {'data': self.testInst.data['profiles'], - 'units': 'V', 'long_name': 'The Doors', - 'meta': ho_meta} - self.meta = self.testInst.meta - - # Test the nD metadata - assert self.testInst.meta['profiles', 'long_name'] == 'The Doors' - testing.assert_list_contains(self.frame_list, - self.meta.ho_data['profiles']) - testing.assert_list_contains(self.frame_list, - self.meta['profiles']['children']) - - for label in ['units', 'desc']: - assert self.meta['profiles']['children'].hasattr_case_neutral(label) - - assert self.meta['profiles']['children']['density', 'units'] == 'A' - assert self.meta['profiles']['children']['density', 'desc'] == '' - - for dvar in ['dummy_str', 'dummy_ustr']: - assert self.meta['profiles']['children'][dvar, 'desc'] == 'are fun' - assert self.meta['profiles']['children'][dvar, 'bananas'] == 2 - return - - # TODO(#789): remove tests for higher order meta - def test_concat_w_ho(self): - """Test `meta.concat` adds new meta objects with higher order data.""" - - # Create meta data to concatenate - meta2 = pysat.Meta() - meta2['new3'] = {'units': 'hey3', 'long_name': 'crew_brew'} - meta2['new4'] = pysat.Meta(pds.DataFrame({ - 'units': {'new41': 'hey4'}, 'long_name': {'new41': 'crew_brew'}, - 'bob_level': {'new41': 'max'}})) - - # Perform and test for successful concatenation - self.meta = self.meta.concat(meta2) - assert self.meta['new3'].units == 'hey3' - assert self.meta['new4'].children['new41'].units == 'hey4' - return - - # TODO(#913): remove tests for 2d metadata - def test_concat_not_strict_w_ho_collision(self): - """Test non-strict concat with overlapping higher-order data.""" - - # Set the meta object - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'}) - - # Create a second object with the same higher-order data variables - concat_meta = pysat.Meta() - for dvar in self.meta.keys_nD(): - concat_meta[dvar] = self.meta[dvar] - - # Change the units of the HO data variables - for cvar in concat_meta[dvar].children.keys(): - # HERE FIX, DOESN'T WORK - concat_meta[dvar].children[cvar] = { - concat_meta.labels.units: "UpdatedUnits"} - - # Concatenate the data - self.meta = self.meta.concat(concat_meta, strict=False) - - # Test that the Meta data kept the original values - testing.assert_list_contains(list(concat_meta.keys_nD()), - list(self.meta.keys_nD())) - - for dvar in concat_meta.keys_nD(): - testing.assert_lists_equal(list(concat_meta[dvar].children.keys()), - list(self.meta[dvar].children.keys())) - - for cvar in concat_meta[dvar].children.keys(): - assert self.meta[dvar].children[ - cvar, self.meta.labels.units].find('Updated') >= 0 - return - - # TODO(#789): remove tests for higher order meta - @pytest.mark.parametrize("label", ['meta_label', 'META_LABEL', 'Meta_Label', - 'MeTa_lAbEl']) - def test_ho_attribute_name_case(self, label): - """Test that `meta.attribute_case_name` preserves the HO stored case. - - Parameters - ---------- - label : str - Label name - - """ - - # Only set `label` in the child data variable - self.dval = 'test_val' - self.meta[self.dval] = self.default_val - cmeta = pysat.Meta() - cval = "_".join([self.dval, "child"]) - cmeta[cval] = {label: 'Test meta data for child meta label'} - self.meta[self.dval] = cmeta - - # Test the meta method using different input variations - assert self.meta.attr_case_name(label.upper()) == label - assert self.meta.attr_case_name(label.lower()) == label - assert self.meta.attr_case_name(label.capitalize()) == label - assert self.meta.attr_case_name(label) == label - return - - # TODO(#789): remove tests for higher order meta - @pytest.mark.parametrize("label", ['meta_label', 'META_LABEL', 'Meta_Label', - 'MeTa_lAbEl']) - def test_ho_hasattr_case_neutral(self, label): - """Test `meta.hasattr_case_neutral` identifies the HO label name. - - Parameters - ---------- - label : str - Label name - - """ - - # Only set `label` in the child data variable - self.dval = 'test_val' - self.meta[self.dval] = self.default_val - cmeta = pysat.Meta() - cval = "_".join([self.dval, "child"]) - cmeta[cval] = {label: 'Test meta data for child meta label'} - self.meta[self.dval] = cmeta - - # Test the meta method using different input variations - assert self.meta[self.dval].children.hasattr_case_neutral(label.upper()) - assert self.meta[self.dval].children.hasattr_case_neutral(label.lower()) - assert self.meta[self.dval].children.hasattr_case_neutral( - label.capitalize()) - assert self.meta[self.dval].children.hasattr_case_neutral(label) - return - - # TODO(#913): remove tests for 2D metadata - def test_ho_meta_rename_function(self): - """Test `meta.rename` method with ho data using a function.""" - - # Set the meta object - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'}) - - # Rename the meta variables to be all upper case, this will differ - # from the Instrument variables, as pysat defaults to lower case - self.meta.rename(str.upper) - - for dvar in self.testInst.vars_no_time: - mvar = dvar.upper() - - # Test the lower order variables - assert dvar not in self.meta.keys(), \ - "variable not renamed: {:}".format(repr(dvar)) - assert mvar in self.meta.keys(), \ - "renamed variable missing: {:}".format(repr(mvar)) - - if mvar in self.meta.keys_nD(): - # Get the variable names from the children - if hasattr(self.testInst[dvar][0], 'columns'): - columns = getattr(self.testInst[dvar][0], 'columns') - else: - columns = [dvar] - - # Test the higher order variables - for cvar in columns: - cmvar = cvar.upper() - assert cmvar in self.meta[mvar].children, \ - "renamed HO variable missing: {:} ({:})".format( - repr(cmvar), repr(mvar)) - - return - - # TODO(#913): remove tests for 2D metadata - def test_ho_meta_rename_dict(self): - """Test `meta.rename` method with ho data using a dict.""" - - # Set the meta object - self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'}) - - # Create a renaming dictionary, which only changes up to four of the - # variable names - rename_dict = {dvar: dvar.upper() - for i, dvar in enumerate(self.testInst.vars_no_time) - if i < 3 or dvar == 'profiles'} - rename_dict['profiles'] = {'density': 'DeNsItY'} - - # Rename the meta variables to be all upper case, this will differ - # from the Instrument variables, as pysat defaults to lower case - self.meta.rename(rename_dict) - - for dvar in self.testInst.vars_no_time: - # Test the lower order variables - if dvar in rename_dict.keys(): - mvar = rename_dict[dvar] - - if isinstance(mvar, dict): - assert dvar in self.meta.keys_nD() - - # Get the variable names from the children - if hasattr(self.testInst[dvar][0], 'columns'): - columns = getattr(self.testInst[dvar][0], 'columns') - else: - columns = [dvar] - - # Test the higher order variables. - for cvar in columns: - if cvar in mvar.keys(): - cmvar = mvar[cvar] - assert cmvar in self.meta[dvar].children.keys(), \ - "renamed HO variable missing: {:} ({:})".format( - repr(cmvar), repr(dvar)) - else: - assert cvar in self.meta[dvar].children.keys(), \ - "unmapped HO var altered: {:} ({:})".format( - repr(cvar), repr(dvar)) - else: - assert dvar not in self.meta.keys(), \ - "variable not renamed: {:}".format(repr(dvar)) - assert mvar in self.meta.keys(), \ - "renamed variable missing: {:}".format(repr(mvar)) - else: - mvar = dvar - assert dvar in self.meta.keys(), \ - "unmapped variable renamed: {:}".format(repr(dvar)) - - return - - # TODO(#789): remove tests for higher order meta - @pytest.mark.parametrize("label", ['meta_label', 'META_LABEL', 'Meta_Label', - 'MeTa_lAbEl']) - def test_get_attribute_name_case_preservation_w_higher_order_list_in(self, - label): - """Test that get attribute names preserves the case with ho metadata. - - Parameters - ---------- - label : str - String label used for testing metadata - - """ - - # Set a meta data variable - self.dval = 'test_val' - self.meta[self.dval] = self.default_val - - # Set an attribute with case in `label` - cval = ''.join([self.dval, '21']) - meta2 = pysat.Meta() - meta2[cval] = {label: 'Test meta data for meta label'} - - # Attach child metadata to root meta - dval2 = ''.join([self.dval, '2']) - self.meta[dval2] = meta2 - - # Attempt to assign to same label at root but potentially different - # case. - self.meta[self.dval] = {label.lower(): 'Test meta data for meta label'} - - # Create inputs and get the attribute case names - ins = [label.upper(), label.lower(), label.capitalize(), - label] - outputs = self.meta.attr_case_name(ins) - - targets = [label] * len(ins) - - # Confirm original input case retained. - assert np.all(outputs == targets) - - return - - # TODO(#789): remove tests for higher order meta - def test_ho_data_retrieval_case_insensitive(self): - """Test that higher order data variables are case insensitive.""" - - # Initalize the meta data - self.dval = "test_val" - self.meta[self.dval] = self.default_val - - cmeta = pysat.Meta() - cval = '_'.join([self.dval, 'child']) - cmeta[cval] = self.default_val - self.meta[self.dval] = cmeta - - # Test that the data value is present using real key and upper-case - # version of that key - assert self.dval in self.meta.keys() - - # Test the child variable, which should only be present through the - # children attribute. Cannot specify keys for case-insensitive look-up. - assert cval not in self.meta.keys() - assert cval in self.meta[self.dval].children.keys() - assert cval.upper() in self.meta[self.dval].children - return - class TestMetaImmutable(TestMeta): """Unit tests for immutable metadata.""" @@ -2000,8 +1457,7 @@ def teardown_method(self): return # TODO(#789): remove tests for higher order meta - @pytest.mark.parametrize("prop,set_val", [('data', pds.DataFrame()), - ('ho_data', {})]) + @pytest.mark.parametrize("prop,set_val", [('data', pds.DataFrame())]) def test_meta_mutable_properties(self, prop, set_val): """Test that @properties are always mutable. @@ -2197,73 +1653,6 @@ def test_transfer_attributes_to_instrument_leading_underscore(self): return -class TestDeprecation(object): - """Unit tests for DeprecationWarnings in the Meta class.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - - warnings.simplefilter("always", DeprecationWarning) - self.meta = pysat.Meta() - self.warn_msgs = [] - return - - def teardown_method(self): - """Clean up the unit test environment after each method.""" - - del self.meta, self.warn_msgs - return - - def test_higher_order_meta_deprecation(self): - """Test that setting higher order meta raises DeprecationWarning.""" - - # Initialize higher-order metadata to add to the main meta object - ho_meta = pysat.Meta() - ho_meta['series_profiles'] = {'long_name': 'series'} - - # Raise and catch warnings - with warnings.catch_warnings(record=True) as war: - self.meta['series_profiles'] = {'meta': ho_meta, - 'long_name': 'series'} - - # Evaluate warnings - self.warn_msgs = ["Support for higher order metadata has been"] - self.warn_msgs = np.array(self.warn_msgs) - - # Ensure the minimum number of warnings were raised - assert len(war) >= len(self.warn_msgs) - - # Test the warning messages, ensuring each attribute is present - testing.eval_warnings(war, self.warn_msgs) - - return - - def test_higher_order_meta_rename_deprecation(self): - """Test that renaming higher order meta raises DeprecationWarning.""" - - # Initialize higher-order metadata to add to the main meta object - ho_meta = pysat.Meta() - ho_meta['series_profiles'] = {'long_name': 'series'} - self.meta['series_profiles'] = {'meta': ho_meta, - 'long_name': 'series'} - - # Raise and catch warnings - with warnings.catch_warnings(record=True) as war: - self.meta.rename(str.upper) - - # Evaluate warnings - self.warn_msgs = ["Support for higher order metadata has been"] - self.warn_msgs = np.array(self.warn_msgs) - - # Ensure the minimum number of warnings were raised - assert len(war) >= len(self.warn_msgs) - - # Test the warning messages, ensuring each attribute is present - testing.eval_warnings(war, self.warn_msgs) - - return - - class TestToDict(object): """Test `.to_dict` method using pysat test Instruments.""" @@ -2376,23 +1765,6 @@ def setup_method(self): return -class TestToDictPandas2D(TestToDict): - """Test `.to_dict` methods using pysat test Instruments.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - - self.testInst = pysat.Instrument('pysat', 'testing2d', - num_samples=5, use_header=True) - self.stime = pysat.instruments.pysat_testing2d._test_dates[''][''] - self.testInst.load(date=self.stime) - - # For output - self.out = None - - return - - class TestToDictXarrayModel(TestToDict): """Test `.to_dict` methods using pysat test Instruments.""" diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py index ee4b6658e..d1ee69b12 100644 --- a/pysat/tests/test_meta_labels.py +++ b/pysat/tests/test_meta_labels.py @@ -211,29 +211,3 @@ def test_change_case_of_meta_labels(self): assert (self.meta['new2'].Units == 'hey2') assert (self.meta['new2'].Long_Name == 'boo2') return - - def test_case_change_of_meta_labels_w_ho(self): - """Test change case of meta labels after initialization with HO data.""" - - # Set the initial labels - self.meta_labels = {'units': ('units', str), 'name': ('long_Name', str)} - self.meta = pysat.Meta(labels=self.meta_labels) - meta2 = pysat.Meta(labels=self.meta_labels) - - # Set meta data values - meta2['new21'] = {'units': 'hey2', 'long_name': 'boo2'} - self.meta['new'] = {'units': 'hey', 'long_name': 'boo'} - self.meta['new2'] = meta2 - - # Change the label name - self.meta.labels.units = 'Units' - self.meta.labels.name = 'Long_Name' - - # Evaluate the results in the main data - assert (self.meta['new'].Units == 'hey') - assert (self.meta['new'].Long_Name == 'boo') - - # Evaluate the results in the higher order data - assert (self.meta['new2'].children['new21'].Units == 'hey2') - assert (self.meta['new2'].children['new21'].Long_Name == 'boo2') - return diff --git a/pysat/tests/test_utils_coords.py b/pysat/tests/test_utils_coords.py index bd08e9917..1ce7fea09 100644 --- a/pysat/tests/test_utils_coords.py +++ b/pysat/tests/test_utils_coords.py @@ -199,8 +199,7 @@ def test_bad_lon_name_calc_solar_local_time(self): return - @pytest.mark.parametrize("name", ["testmodel", "testing2d", - "ndtesting"]) + @pytest.mark.parametrize("name", ["testmodel", "ndtesting"]) def test_lon_broadcasting_calc_solar_local_time(self, name): """Test calc_solar_local_time with longitude coordinates.""" @@ -216,8 +215,7 @@ def test_lon_broadcasting_calc_solar_local_time(self, name): assert self.py_inst['slt'].min() >= 0.0 return - @pytest.mark.parametrize("name", ["testmodel", "testing2d", - "ndtesting"]) + @pytest.mark.parametrize("name", ["testmodel", "ndtesting"]) def test_lon_broadcasting_calc_solar_local_time_no_mod_multiday(self, name): """Test non modulated solar local time output for a 2 day range.""" @@ -235,8 +233,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod_multiday(self, name): assert self.py_inst['slt'].min() >= 0.0 return - @pytest.mark.parametrize("name", ["testmodel", "testing2d", - "ndtesting"]) + @pytest.mark.parametrize("name", ["testmodel", "ndtesting"]) def test_lon_broadcasting_calc_solar_local_time_no_mod_ref_date(self, name): """Test non modulated SLT output for a 2 day range with a ref date.""" @@ -256,8 +253,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod_ref_date(self, name): assert self.py_inst['slt'].min() >= 24.0 return - @pytest.mark.parametrize("name", ["testmodel", "testing2d", - "ndtesting"]) + @pytest.mark.parametrize("name", ["testmodel", "ndtesting"]) def test_lon_broadcasting_calc_solar_local_time_no_mod(self, name): """Test SLT calc with longitude coordinates and no modulus.""" diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 7ac9c1e11..27ca83f6d 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -772,8 +772,8 @@ def test_read_netcdf4_with_time_meta_labels(self, kwargs, target): "Variable {:} not loaded correctly".format(var) return - def test_load_netcdf_pandas_3d_error(self): - """Test load_netcdf error with a pandas 3D file.""" + def test_load_netcdf_pandas_2d_error(self): + """Test load_netcdf error with a pandas 2D file.""" # Create a bunch of files by year and doy outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') @@ -783,49 +783,12 @@ def test_load_netcdf_pandas_3d_error(self): # Evaluate the error raised and the expected message testing.eval_bad_input( io.load_netcdf, ValueError, - "only supports 1D and 2D data in pandas", input_args=[outfile], + "only supports 1D data in pandas", input_args=[outfile], input_kwargs={"epoch_name": 'time', "pandas_format": True}) return -class TestLoadNetCDF2DPandas(TestLoadNetCDF): - """Unit tests for `load_netcdf` using 2d pandas data.""" - - def setup_method(self): - """Set up the test environment.""" - - # Create temporary directory - self.tempdir = tempfile.TemporaryDirectory() - self.saved_path = pysat.params['data_dirs'] - pysat.params['data_dirs'] = self.tempdir.name - - self.testInst = pysat.Instrument(platform='pysat', name='testing2d', - update_files=True, num_samples=100, - use_header=True) - self.stime = pysat.instruments.pysat_testing2d._test_dates[''][''] - self.epoch_name = 'time' - - # Initialize the loaded data object - self.loaded_inst = None - return - - def teardown_method(self): - """Clean up the test environment.""" - - pysat.params['data_dirs'] = self.saved_path - - # Clear the attributes with data in them - del self.loaded_inst, self.testInst, self.stime, self.epoch_name - - # Remove the temporary directory - self.tempdir.cleanup() - - # Clear the directory attributes - del self.tempdir, self.saved_path - return - - class TestNetCDF4Integration(object): """Integration tests for the netCDF4 I/O utils.""" @@ -1049,17 +1012,6 @@ def test_add_netcdf4_standards_to_meta(self, missing): assert label not in init_meta[var] assert label in new_meta[var] - if self.testInst.name == 'testing2D': - assert 'Depend_1' not in init_meta[var] - - # Check for higher dimensional data properties - if self.testInst.name == 'testing2D': - for var in self.testInst.vars_no_time: - if self.testInst.meta[var].children is not None: - assert 'Depend_1' in new_meta[var] - else: - assert 'Depend_1' not in new_meta[var] - return @pytest.mark.parametrize('meta_trans', [{'units': ['testingFillVal', @@ -1335,23 +1287,6 @@ def setup_method(self): return -class TestNetCDF4IntegrationPandas2D(TestNetCDF4Integration): - """Integration tests for the netCDF4 I/O utils using pandas2d Instrument.""" - - def setup_method(self): - """Create a testing environment.""" - - # Create an instrument object that has a meta with some - # variables allowed to be nan within metadata when exporting. - self.testInst = pysat.Instrument('pysat', 'testing2d', num_samples=5, - use_header=True) - self.testInst.load(date=self.testInst.inst_module._test_dates[''][''], - use_header=True) - self.pformat = self.testInst.pandas_format - - return - - class TestNetCDF4Integration2DXarray(TestNetCDF4Integration): """Integration tests for the netCDF4 I/O utils using 2dxarray Instrument.""" @@ -1846,29 +1781,6 @@ def teardown_method(self): return -class TestMetaTranslation2DPandas(TestMetaTranslation): - """Test meta translation when writing/loading files testing2d Instrument.""" - - def setup_method(self): - """Create test environment.""" - - self.test_inst = pysat.Instrument('pysat', 'testing2d', - num_samples=5, use_header=True) - self.test_date = pysat.instruments.pysat_testing2d._test_dates[''][''] - self.test_inst.load(date=self.test_date) - self.meta_dict = self.test_inst.meta.to_dict() - self.out = None - - return - - def teardown_method(self): - """Cleanup test environment.""" - - del self.test_inst, self.test_date, self.out, self.meta_dict - - return - - class TestMetaTranslationModel(TestMetaTranslation): """Test meta translation when writing/loading files testmodel Instrument.""" From 11fe41aed6d4f15fce3c93004674fa0a27257667 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 11:13:44 -0400 Subject: [PATCH 127/365] DOC: remove testing2d from docs --- docs/api.rst | 9 --------- docs/dependency.rst | 7 ------- docs/instruments/testing_instruments.rst | 7 ------- 3 files changed, 23 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index b1768d984..d14c971a4 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -241,15 +241,6 @@ pysat_testing_xarray :members: -.. _api-pysat-testing2d: - -pysat_testing2d -^^^^^^^^^^^^^^^ - -.. automodule:: pysat.instruments.pysat_testing2d - :members: - - .. _api-pysat-testing2d_xarray: pysat_testing2d_xarray diff --git a/docs/dependency.rst b/docs/dependency.rst index f88438fba..fa31acf32 100644 --- a/docs/dependency.rst +++ b/docs/dependency.rst @@ -272,13 +272,6 @@ pysat_testing_xarray :ref:`api-pysat-testing_xarray` returns a satellite-like object with 1D data as a function of latitude, longitude, and altitude in a xarray format. -pysat_testing2d -^^^^^^^^^^^^^^^ -:ref:`api-pysat-testing2d` is another satellite-like object that also returns -profile data as a function of altitude at some distance from the satellite. It -is similar to a Radio Occultation or other instruments that have altitude -profiles. - pysat_ndtesting ^^^^^^^^^^^^^^^^^^^^^^ :ref:`api-pysat-ndtesting` is a satellite-like object that returns all diff --git a/docs/instruments/testing_instruments.rst b/docs/instruments/testing_instruments.rst index 0fa1299c3..2361a0765 100644 --- a/docs/instruments/testing_instruments.rst +++ b/docs/instruments/testing_instruments.rst @@ -21,13 +21,6 @@ and altitude in a xarray format. See :ref:`api-pysat-testing_xarray` for more details. -pysat_testing2d -^^^^^^^^^^^^^^^ -An instrument with satellite-like data like :py:mod:`pysat_testing`, but with a -2D data variable, 'profile', that is similar to Radio Occultation data. See -:ref:`api-pysat-testing2d` for more details. - - pysat_ndtesting ^^^^^^^^^^^^^^^ An instrument with satellite-like data like :py:mod:`pysat_testing` that From b810d16f05cdaa1c574d85cd7b8227c459baaffa Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 11:31:49 -0400 Subject: [PATCH 128/365] DOC: update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b547a3889..73f76d005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -[3.1.X] - 2023-xx-xx +[3.2.0] - 2023-xx-xx -------------------- * New Features * Added tests for warnings, logging messages, and errors in the Instrument @@ -23,6 +23,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Implement pyproject to manage metadata * Remove Sphinx cap * Add pandas cap + * Remove deprecated `pysat_testing2d` instrument + * Remove deprecated meta children info [3.1.0] - 2023-05-31 -------------------- From 19de5df3e44beddcbb9ac3c27aab1b4637dac0ad Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 11:39:21 -0400 Subject: [PATCH 129/365] pandas cap of 2.1.1 --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 311c0382d..9ec0b533b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "dask", "netCDF4", "numpy >= 1.12", - "pandas < 2.1", + "pandas < 2.1.1", "portalocker", "pytest", "scipy", diff --git a/requirements.txt b/requirements.txt index c5577679a..059c99271 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ dask netCDF4 numpy>=1.12 -pandas<2.1 +pandas<2.1.1 portalocker pytest scipy From e85dcbe33196df0a9732e1f90ef22a4c29197515 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 11:42:33 -0400 Subject: [PATCH 130/365] STY: pep8 --- pysat/_meta.py | 2 -- pysat/utils/io.py | 8 -------- 2 files changed, 10 deletions(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index 4b4fe55f5..274e1aefc 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -498,8 +498,6 @@ def __setitem__(self, data_vars, input_dat): # Outputs from Meta object are a Series. Thus, this takes in input # from a Meta object. Set data using standard assignment via a dict. in_dict = input_data.to_dict() - if 'children' in in_dict: - child = in_dict.pop('children') # Remaining items are simply assigned via recursive call self[data_vars] = in_dict diff --git a/pysat/utils/io.py b/pysat/utils/io.py index 0483d1653..a935a6e98 100644 --- a/pysat/utils/io.py +++ b/pysat/utils/io.py @@ -928,8 +928,6 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4', saved_meta = None running_idx = 0 running_store = [] - two_d_keys = [] - two_d_dims = [] if meta_kwargs is None: meta_kwargs = {} @@ -957,12 +955,6 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4', else: drop_meta_labels = pysat.utils.listify(drop_meta_labels) - # Need to use name label later to identify variables with long_name 'Epoch' - name_label = meta.labels.name - for key in meta_translation.keys(): - if meta.labels.name in meta_translation[key]: - name_label = key - # Load data for each file for fname in fnames: with netCDF4.Dataset(fname, mode='r', format=file_format) as data: From 848e573b170d41956c0dbde15598e4d9d4481695 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 12:42:39 -0400 Subject: [PATCH 131/365] STY: restore line with exception --- pysat/_meta.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysat/_meta.py b/pysat/_meta.py index 274e1aefc..138f6beda 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -498,6 +498,8 @@ def __setitem__(self, data_vars, input_dat): # Outputs from Meta object are a Series. Thus, this takes in input # from a Meta object. Set data using standard assignment via a dict. in_dict = input_data.to_dict() + if 'children' in in_dict: + child = in_dict.pop('children') # noqa: F841 # Remaining items are simply assigned via recursive call self[data_vars] = in_dict From 6252ed0d9045538487a89fe00007bbbbd0f13737 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 3 Oct 2023 16:03:39 -0400 Subject: [PATCH 132/365] BUG: clean up meta children --- pysat/_instrument.py | 100 +------------- pysat/_meta.py | 195 +++------------------------ pysat/instruments/methods/general.py | 9 -- pysat/instruments/methods/testing.py | 2 - pysat/tests/test_meta.py | 144 ++------------------ pysat/tests/test_utils_io.py | 3 +- 6 files changed, 36 insertions(+), 417 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 5405e1ee6..31a85c254 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1051,30 +1051,6 @@ def __setitem__(self, key, new_data): # the rest of the keys are presumed to be metadata in_data = new.pop('data') - # TODO(#908): remove code below with removal of 2D pandas support. - if hasattr(in_data, '__iter__'): - if not isinstance(in_data, pds.DataFrame) and isinstance( - next(iter(in_data), None), pds.DataFrame): - # Input is a list_like of frames, denoting higher order data - warnings.warn(" ".join(["Support for 2D pandas instrument", - "data has been deprecated and will", - "be removed in 3.2.0+. Please", - "either raise an issue with the", - "developers or modify the load", - "statement to use an", - "xarray.Dataset."]), - DeprecationWarning, stacklevel=2) - - if ('meta' not in new) and (key not in self.meta.keys_nD()): - # Create an empty Meta instance but with variable names. - # This will ensure the correct defaults for all - # subvariables. Meta can filter out empty metadata as - # needed, the check above reduces the need to create - # Meta instances. - ho_meta = pysat.Meta(**self.meta_kwargs) - ho_meta[in_data[0].columns] = {} - self.meta[key] = ho_meta - # Assign data and any extra metadata self.data[key] = in_data self._update_data_types(key) @@ -2915,77 +2891,13 @@ def rename(self, mapper, lowercase_data_labels=False): map_key = pysat.utils.get_mapped_value(vkey, mapper) if map_key is not None: - # Treat higher-order pandas and normal pandas separately - if vkey in self.meta.keys_nD(): - # Variable name is in higher order list - hdict = {} - if isinstance(map_key, dict): - # Changing a variable name within a higher order - # object using a dictionary. First ensure the - # variable exist. - for hkey in map_key.keys(): - if hkey not in self.meta[ - vkey]['children'].keys(): - estr = ' '.join( - ('cannot rename', repr(hkey), - 'because it is not a known ', - 'higher-order variable under', - repr(vkey), '.')) - raise ValueError(estr) - hdict = map_key - else: - # This is either a value or a mapping function - for hkey in self.meta[vkey]['children'].keys(): - hmap = pysat.utils.get_mapped_value(hkey, - mapper) - if hmap is not None: - hdict[hkey] = hmap - - pdict[vkey] = map_key - - # Check for lowercase flag - change = True - if lowercase_data_labels: - gdict = {hkey: hdict[hkey].lower() - for hkey in hdict.keys() - if hkey != hdict[hkey].lower()} - - if len(list(gdict.keys())) == 0: - change = False - else: - gdict = hdict - - # Change the higher-order variable names frame-by-frame - if change: - for i in np.arange(len(self.index)): - if isinstance(self[i, vkey], pds.Series): - if self[i, vkey].name in gdict: - new_name = gdict[self[i, vkey].name] - self[i, vkey].rename(new_name, - inplace=True) - else: - tkey = list(gdict.keys())[0] - if self[i, vkey].name != gdict[tkey]: - estr = ' '.join( - ('cannot rename', hkey, - 'because, it is not a known' - 'known higher-order ', - 'variable under', vkey, 'at', - 'index {:d}.'.format(i))) - raise ValueError(estr) - else: - self[i, vkey].rename(columns=gdict, - inplace=True) - + # Add to the pandas renaming dictionary after accounting + # for the `lowercase_data_labels` flag. + if lowercase_data_labels: + if vkey != map_key.lower(): + pdict[vkey] = map_key.lower() else: - # This is a normal variable. Add it to the pandas - # renaming dictionary after accounting for the - # `lowercase_data_labels` flag. - if lowercase_data_labels: - if vkey != map_key.lower(): - pdict[vkey] = map_key.lower() - else: - pdict[vkey] = map_key + pdict[vkey] = map_key # Change variable names for attached data object self.data.rename(columns=pdict, inplace=True) diff --git a/pysat/_meta.py b/pysat/_meta.py index 138f6beda..e550a69b7 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -192,9 +192,6 @@ def __init__(self, metadata=None, header_data=None, # Set the data types, if provided self._data_types = data_types - # Initialize higher order (nD) data structure container, a dict - self._ho_data = {} - # Use any user provided data to instantiate object with data. # Attributes unit and name labels are called within. if metadata is not None: @@ -249,23 +246,17 @@ def __str__(self, long_str=True): """ # Get the desired variables as lists labs = [var for var in self.attrs()] - vdim = [var for var in self.keys() if var not in self.keys_nD()] - nchild = {var: len([kk for kk in self[var]['children'].keys()]) - for var in self.keys_nD()} - ndim = ["{:} -> {:d} children".format(var, nchild[var]) - for var in self.keys_nD()] + vdim = [var for var in self.keys()] # Get the lengths of each list nlabels = len(labs) nvdim = len(vdim) - nndim = len(ndim) # Print the short output out_str = "pysat Meta object\n" out_str += "-----------------\n" out_str += "Tracking {:d} metadata values\n".format(nlabels) out_str += "Metadata for {:d} standard variables\n".format(nvdim) - out_str += "Metadata for {:d} ND variables\n".format(nndim) # Print the global meta data. `max_num` should be divisible by 2 and # `ncol`. @@ -283,10 +274,6 @@ def __str__(self, long_str=True): out_str += "\nStandard Metadata variables:\n" out_str += core_utils.fmt_output_in_cols(vdim, ncols=ncol, max_num=max_num) - if nndim > 0: - out_str += "\nND Metadata variables:\n" - out_str += core_utils.fmt_output_in_cols(ndim, ncols=ncol, - max_num=max_num) return out_str @@ -540,14 +527,6 @@ def __getitem__(self, key): meta[:, 'units'] meta[:, ['units', 'long_name']] - # For higher order data, slicing is not supported for multiple - # parents with any children - meta['profiles', 'density', 'units'] - meta['profiles', 'density', ['units', 'long_name']] - meta['profiles', ['density', 'dummy_str'], ['units', 'long_name']] - meta['profiles', ('units', 'long_name')] - meta[['series_profiles', 'profiles'], ('units', 'long_name')] - """ # Define a local convenience function def match_name(func, var_name, index_or_column): @@ -567,43 +546,10 @@ def match_name(func, var_name, index_or_column): # If tuple length is 2, index, column new_index = match_name(self.var_case_name, key[0], self.data.index) - try: - # Assume this is a label name - new_name = match_name(self.attr_case_name, key[1], - self.data.columns) - return self.data.loc[new_index, new_name] - except KeyError as kerr: - # This may instead be a child variable, check for children - if(hasattr(self[new_index], 'children') - and self[new_index].children is None): - raise kerr - - try: - new_child_index = match_name( - self.attr_case_name, key[1], - self[new_index].children.data.index) - return self.ho_data[new_index].data.loc[new_child_index] - except AttributeError: - raise NotImplementedError( - ''.join(['Cannot retrieve child meta data ', - 'from multiple parents'])) - - elif len(key) == 3: - # If tuple length is 3, index, child_index, column - new_index = match_name(self.var_case_name, key[0], - self.data.index) - try: - new_child_index = match_name( - self.attr_case_name, key[1], - self[new_index].children.data.index) - except AttributeError: - raise NotImplementedError( - 'Cannot retrieve child meta data from multiple parents') - - new_name = match_name(self.attr_case_name, key[2], + # Assume this is a label name + new_name = match_name(self.attr_case_name, key[1], self.data.columns) - return self.ho_data[new_index].data.loc[new_child_index, - new_name] + return self.data.loc[new_index, new_name] elif isinstance(key, list): # If key is a list, selection works as-is @@ -627,14 +573,6 @@ def match_name(func, var_name, index_or_column): # above and commented .copy code below have been kept. Remove # for any subsequent releases if things are still ok. meta_row = self.data.loc[new_key] # .copy() - if new_key in self.keys_nD(): - meta_row.at['children'] = self.ho_data[new_key] # .copy() - else: - # Not higher order meta. Assign value of None. First, we - # assign a string, and then None. Ensures column is not - # a numeric data type. - meta_row.at['children'] = '' - meta_row.at['children'] = None return meta_row else: raise KeyError("Key '{:}' not found in MetaData".format(key)) @@ -663,10 +601,6 @@ def __contains__(self, data_var): if data_var.lower() in [ikey.lower() for ikey in self.keys()]: does_contain = True - if not does_contain: - if data_var.lower() in [ikey.lower() for ikey in self.keys_nD()]: - does_contain = True - return does_contain def __eq__(self, other_meta): @@ -716,39 +650,6 @@ def __eq__(self, other_meta): other_meta[key, attr]): return False - # Check the higher order products. Recursive call into this function - # didn't work, so spell out the details. - keys1 = [key for key in self.keys_nD()] - keys2 = [key for key in other_meta.keys_nD()] - try: - testing.assert_lists_equal(keys1, keys2) - except AssertionError: - return False - - # Check the higher order variables within each nD key are the same. - # NaN is treated as equal, though mathematically NaN is not equal - # to anything. - for key in self.keys_nD(): - for iter1, iter2 in [(self[key].children.keys(), - other_meta[key].children.keys()), - (self[key].children.attrs(), - other_meta[key].children.attrs())]: - list1 = [value for value in iter1] - list2 = [value for value in iter2] - - try: - testing.assert_lists_equal(list1, list2) - except AssertionError: - return False - - # Check if all elements are individually equal - for ckey in self[key].children.keys(): - for cattr in self[key].children.attrs(): - if not testing.nan_equal( - self[key].children[ckey, cattr], - other_meta[key].children[ckey, cattr]): - return False - # If we made it this far, things are good return True @@ -875,12 +776,6 @@ def _label_setter(self, new_label, current_label, default_type, 'Meta instantiation.')) pysat.logger.info(mstr) - # Check higher order structures and recursively change labels - for key in self.keys_nD(): - # Update children - self.ho_data[key]._label_setter(new_label, current_label, - default_type, use_names_default) - return # ----------------------------------------------------------------------- @@ -902,22 +797,6 @@ def data(self, new_frame): self._data = new_frame return - @property - def ho_data(self): - """Retrieve higher order data. - - May be set using `ho_data.setter(new_dict)`, where `new_dict` is a - dict containing the higher order metadata. - - """ - return self._ho_data - - @ho_data.setter - def ho_data(self, new_dict): - # Set the ho_data property. See docstring for property above. - self._ho_data = new_dict - return - @property def empty(self): """Return boolean True if there is no metadata. @@ -962,13 +841,8 @@ def drop(self, names): """ - # Drop the lower dimension data self.data = self._data.drop(names, axis=0) - # Drop the higher dimension data - for name in names: - if name in self._ho_data: - self._ho_data.pop(name) return def keep(self, keep_names): @@ -991,6 +865,7 @@ def keep(self, keep_names): # Drop names not specified in keep_names list self.drop(drop_names) + return def apply_meta_labels(self, other_meta): @@ -1119,11 +994,6 @@ def keys(self): for ikey in self.data.index: yield ikey - def keys_nD(self): - """Yield keys for higher order metadata.""" - for ndkey in self.ho_data: - yield ndkey - def attrs(self): """Yield metadata products stored for each variable name.""" for dcol in self.data.columns: @@ -1191,8 +1061,8 @@ def attr_case_name(self, name): # Create a list of all attribute names and lower case attribute names self_keys = [key for key in self.attrs()] - for key in list(self.keys_nD()): - self_keys.extend(self.ho_data[key].data.columns) + # for key in list(self.keys_nD()): + # self_keys.extend(self.ho_data[key].data.columns) lower_self_keys = [key.lower() for key in self_keys] case_names = [] @@ -1240,37 +1110,14 @@ def rename(self, mapper): # Update the attribute name map_var = core_utils.get_mapped_value(var, mapper) if map_var is not None: - if isinstance(map_var, dict): - if var in self.keys_nD(): - child_meta = self[var].children.copy() - child_meta.rename(map_var) - self.ho_data[var] = child_meta - else: - raise ValueError('unknown mapped value at {:}'.format( - repr(var))) - else: - # Get and update the meta data - hold_meta = self[var].copy() - hold_meta.name = map_var - - # Remove the metadata under the previous variable name - self.drop(var) - if var in self.ho_data: - del self.ho_data[var] - - # Re-add the meta data with the updated variable name - self[map_var] = hold_meta - - # Determine if the attribute is present in higher order - # structures - if map_var in self.keys_nD(): - # The children attribute is a Meta class object. - # Recursively call the current routine. The only way to - # avoid Meta undoing the renaming process is to assign - # the meta data to `ho_data`. - child_meta = self[map_var].children.copy() - child_meta.rename(mapper) - self.ho_data[map_var] = child_meta + hold_meta = self[var].copy() + hold_meta.name = map_var + + # Remove the metadata under the previous variable name + self.drop(var) + + # Re-add the meta data with the updated variable name + self[map_var] = hold_meta return @@ -1314,14 +1161,10 @@ def concat(self, other_meta, strict=False): other_meta_updated = other_meta.copy() other_meta_updated.labels = self.labels - # Concat 1D metadata in data frames to copy of current metadata + # Concat metadata in data frames to copy of current metadata for key in other_meta_updated.keys(): mdata.data.loc[key] = other_meta.data.loc[key] - # Combine the higher order meta data - for key in other_meta_updated.keys_nD(): - mdata.ho_data[key] = other_meta.ho_data[key] - return mdata def copy(self): @@ -1351,8 +1194,6 @@ def pop(self, label_name): if new_name in self.keys(): output = self[new_name] self.data = self.data.drop(new_name, axis=0) - else: - output = self.ho_data.pop(new_name) else: raise KeyError('Key not present in metadata variables') @@ -1661,10 +1502,6 @@ class MetaLabels(object): but the original case is preserved. Case preservation is built in to support writing files with a desired case to meet standards. - Metadata for higher order data objects, those that have - multiple products under a single variable name in a `pysat.Instrument` - object, are stored by providing a Meta object under the single name. - Supports any custom metadata values in addition to the expected metadata attributes (units, name, notes, desc, value_min, value_max, and fill). These base attributes may be used to programatically access and set types diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py index 732163702..400480ddd 100644 --- a/pysat/instruments/methods/general.py +++ b/pysat/instruments/methods/general.py @@ -216,15 +216,6 @@ def remove_leading_text(inst, target=None): inst.meta.data = inst.meta.data.rename( index=lambda x: x.split(prepend_str)[-1]) - orig_keys = [kk for kk in inst.meta.keys_nD()] - - for keynd in orig_keys: - if keynd.find(prepend_str) >= 0: - new_key = keynd.split(prepend_str)[-1] - new_meta = inst.meta.pop(keynd) - new_meta.data = new_meta.data.rename( - index=lambda x: x.split(prepend_str)[-1]) - inst.meta[new_key] = new_meta return diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index cb25480af..08705398d 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -284,8 +284,6 @@ def initialize_test_meta(epoch_name, data_keys): for var in meta.keys(): if var not in data_keys: meta.drop(var) - if var in meta.keys_nD(): - meta.ho_data.pop(var) return meta diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 9269046ea..adfaf9c6c 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -106,62 +106,8 @@ def eval_meta_settings(self, isfloat=True): self.default_val[lkey].__repr__()) assert 'children' not in self.meta.data.columns - assert self.dval not in self.meta.keys_nD() return - def eval_ho_meta_settings(self, meta_dict): - """Test the Meta settings for higher order data. - - Parameters - ---------- - meta_dict : dict - Dict with meta data labels as keys and values to test against - - """ - - # Test the ND metadata results - testing.assert_list_contains(self.frame_list, - list(self.meta.ho_data[self.dval].keys())) - testing.assert_list_contains( - self.frame_list, list(self.meta[self.dval]['children'].keys())) - - # Test the meta settings at the base and nD level - for label in meta_dict.keys(): - if label == 'meta': - testing.assert_lists_equal( - list(self.meta[self.dval]['children'].attrs()), - list(meta_dict[label].attrs())) - testing.assert_lists_equal( - list(self.meta[self.dval]['children'].keys()), - list(meta_dict[label].keys())) - - for lvar in self.meta[self.dval]['children'].attrs(): - for dvar in self.meta[self.dval]['children'].keys(): - if lvar in self.default_nan: - assert np.isnan( - self.meta[self.dval]['children'][dvar, lvar]), \ - "{:s} child {:s} {:s} value {:} != NaN".format( - self.dval.__repr__(), dvar.__repr__(), - lvar.__repr__(), - self.meta[self.dval]['children'][ - dvar, lvar].__repr__()) - else: - assert (self.meta[self.dval]['children'][dvar, lvar] - == meta_dict[label][dvar, lvar]), \ - "{:s} child {:s} {:s} value {:} != {:}".format( - self.dval.__repr__(), dvar.__repr__(), - lvar.__repr__(), - self.meta[self.dval]['children'][ - dvar, lvar].__repr__(), - meta_dict[label][dvar, lvar].__repr__()) - else: - assert self.meta[self.dval]['children'].hasattr_case_neutral( - label) - assert self.meta[self.dval, label] == meta_dict[label], \ - "{:s} label value {:} != {:}".format( - label, self.meta[self.dval, label].__repr__(), - meta_dict[label].__repr__()) - return # ----------------------- @@ -425,7 +371,6 @@ def test_str(self, long_str, inst_kwargs): # Evaluate the common parts of the output string assert out.find('pysat Meta object') >= 0 assert out.find('standard variables') > 0 - assert out.find('ND variables') > 0 assert out.find('global attributes') > 0 # Evaluate the extra parts of the long output string @@ -441,10 +386,8 @@ def test_str(self, long_str, inst_kwargs): else: assert out.find('Standard Metadata variables:') < 0 - assert out.find('ND Metadata variables:') < 0 else: assert out.find('Standard Metadata variables:') < 0 - assert out.find('ND Metadata variables:') < 0 return def test_self_equality(self): @@ -573,7 +516,6 @@ def test_pop(self, inst_name): # Test the popped object labels pop_attrs = list(mpop.keys()) - pop_attrs.pop(pop_attrs.index('children')) testing.assert_lists_equal(pop_attrs, list(self.meta.attrs())) # Test the popped object values @@ -581,9 +523,6 @@ def test_pop(self, inst_name): comp_values = [mcomp[pattr] for pattr in pop_attrs] testing.assert_lists_equal(pop_values, comp_values) - if mpop['children'] is not None: - assert mpop['children'] == mcomp['children'] - # Test that the popped variable is no longer in the main object assert dvar not in self.meta.keys(), "pop did not remove metadata" @@ -720,20 +659,6 @@ def test_selected_meta_retrieval(self, inst_name, num_mvals, num_dvals): mvals = [getattr(self.meta.labels, mattr) for mattr in list(self.meta_labels.keys())[:num_mvals]] - # If dvals is greater than zero and there is higher order data, - # make sure at least one is included - nd_inds = list() - if len(dvals) > 0: - nd_vals = [key for key in self.meta.keys_nD()] - - if len(nd_vals) > 0: - nd_inds = [dvals.index(self.dval) for self.dval in nd_vals - if self.dval in dvals] - - if len(nd_inds) == 0: - dvals[0] = nd_vals[0] - nd_inds = [0] - # Retrieve meta data for desired values sel_meta = self.meta[dvals, mvals] @@ -742,21 +667,6 @@ def test_selected_meta_retrieval(self, inst_name, num_mvals, num_dvals): testing.assert_lists_equal(dvals, list(sel_meta.index)) testing.assert_lists_equal(mvals, list(sel_meta.columns)) - # If there is higher order data, test the retrieval - for pind in nd_inds: - # Get the desired number of child variables - self.dval = dvals[pind] - cvals = [kk for kk in self.meta[self.dval].children.keys()][ - :num_dvals] - - # Retrieve meta data for desired values - sel_meta = self.meta[self.dval, cvals, mvals] - - # Evaluate retrieved data - assert isinstance(sel_meta, pds.DataFrame) - testing.assert_lists_equal(cvals, list(sel_meta.index)) - testing.assert_lists_equal(mvals, list(sel_meta.columns)) - return def test_replace_meta(self): @@ -1456,32 +1366,18 @@ def teardown_method(self): del self.mutable return - # TODO(#789): remove tests for higher order meta - @pytest.mark.parametrize("prop,set_val", [('data', pds.DataFrame())]) - def test_meta_mutable_properties(self, prop, set_val): - """Test that @properties are always mutable. - - Parameters - ---------- - prop : str - Attribute on `self.meta` to be tested - set_val : any - Value to be assigned to `prop` on `self.meta` - - """ + def test_meta_mutable_properties(self): + """Test that @properties are always mutable.""" # Set anything that can be immutable to be immutable self.meta.mutable = self.mutable # Test that data and label values can be updated - for prop, set_val in [('data', pds.DataFrame()), ('ho_data', {})]: - try: - # Pandas does not support dataframe equality - setattr(self.meta, prop, set_val) - except AttributeError: - raise AssertionError( - "Couldn't update mutable property {:}".format( - prop.__repr__())) + try: + # Pandas does not support dataframe equality + setattr(self.meta, 'data', pds.DataFrame()) + except AttributeError: + raise AssertionError("Couldn't update mutable property 'data'") @pytest.mark.parametrize("label", ['units', 'name', 'desc', 'notes', 'min_val', 'max_val', 'fill_val']) @@ -1691,40 +1587,26 @@ def test_to_dict(self, preserve_case): # Confirm type assert isinstance(self.out, dict) - # Check for higher order products - ho_vars = [] - for var in self.testInst.meta.keys(): - if 'children' in self.testInst.meta[var]: - if self.testInst.meta[var]['children'] is not None: - for subvar in self.testInst.meta[var]['children'].keys(): - ho_vars.append('_'.join([var, subvar])) - # Confirm the contents of the output for variables for var in self.out.keys(): - if var not in ho_vars: - for label in self.out[var]: - assert label in self.testInst.meta.data.columns - assert testing.nan_equal(self.out[var][label], - self.testInst.meta[var][label]), \ - 'Differing values.' + for label in self.out[var]: + assert label in self.testInst.meta.data.columns + assert testing.nan_equal(self.out[var][label], + self.testInst.meta[var][label]), \ + 'Differing values.' # Confirm case if not preserve_case: # Outputs should all be lower case for key in self.out.keys(): assert key == key.lower(), 'Output not lower case.' - for key in ho_vars: - assert key == key.lower(), 'Output not lower case.' - assert key.lower() in self.out, 'Missing output variable.' else: # Case should be preserved for key in self.out.keys(): assert key == self.testInst.meta.var_case_name(key), \ 'Output case different.' - for key in ho_vars: - assert key in self.out, 'Output case different, or missing.' - num_target_vars = len(ho_vars) + len(list(self.testInst.meta.keys())) + num_target_vars = len(list(self.testInst.meta.keys())) assert num_target_vars == len(self.out), \ 'Different number of variables.' diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 27ca83f6d..bdee5cb43 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -284,8 +284,7 @@ def test_inst_write_and_read_netcdf(self, add_path): and var not in updated_attrs] tvars = [var for var in self.testInst.meta.keys() - if var not in self.testInst.meta.keys_nD() - and var.lower() not in ["epoch", "time"]] + if var.lower() not in ["epoch", "time"]] fvars = [var for var in netcdf_inst.meta.keys() if var.lower() not in ["epoch", "time"]] From 1918f73e73ee5a484e79403ec916b60169b21e7e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 4 Oct 2023 11:08:19 -0400 Subject: [PATCH 133/365] MAINT: scrub code for higher order meta --- pysat/_meta.py | 146 +++++++-------- pysat/tests/test_utils_io.py | 32 +--- pysat/utils/io.py | 347 +++-------------------------------- 3 files changed, 92 insertions(+), 433 deletions(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index e550a69b7..83fad9c47 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -401,86 +401,72 @@ def __setitem__(self, data_vars, input_dat): # Time to actually add the metadata for ikey in input_data: - if ikey not in ['children', 'meta']: - for i, var in enumerate(data_vars): - to_be_set = input_data[ikey][i] - good_set = True - - # See if this meta data key has already been defined - # in MetaLabels - if ikey in self.labels.label_attrs.keys(): - iattr = self.labels.label_attrs[ikey] - if not isinstance( - to_be_set, self.labels.label_type[iattr]): - # If this is a disagreement between byte data - # and an expected str, resolve it here - if(isinstance(to_be_set, bytes) - and str in pysat.utils.listify( - self.labels.label_type[iattr])): - to_be_set = core_utils.stringify(to_be_set) - else: - # This type is incorrect, try casting it - wmsg = ''.join(['Metadata with type ', - repr(type(to_be_set)), - ' does not match expected ', - 'type ', - repr(self.labels.label_type[ - iattr])]) - try: - if hasattr(to_be_set, '__iter__'): - if str in pysat.utils.listify( - self.labels.label_type[ - iattr]): - to_be_set = '\n\n'.join( - [str(tval) for tval in - to_be_set]) - else: - raise TypeError("can't recast") + for i, var in enumerate(data_vars): + to_be_set = input_data[ikey][i] + good_set = True + + # See if this meta data key has already been defined + # in MetaLabels + if ikey in self.labels.label_attrs.keys(): + iattr = self.labels.label_attrs[ikey] + if not isinstance( + to_be_set, self.labels.label_type[iattr]): + # If this is a disagreement between byte data + # and an expected str, resolve it here + if(isinstance(to_be_set, bytes) + and str in pysat.utils.listify( + self.labels.label_type[iattr])): + to_be_set = core_utils.stringify(to_be_set) + else: + # This type is incorrect, try casting it + wmsg = ''.join(['Metadata with type ', + repr(type(to_be_set)), + ' does not match expected ', + 'type ', + repr(self.labels.label_type[ + iattr])]) + try: + if hasattr(to_be_set, '__iter__'): + if str in pysat.utils.listify( + self.labels.label_type[iattr]): + to_be_set = '\n\n'.join( + [str(tval) for tval in + to_be_set]) else: - to_be_set = pysat.utils.listify( - self.labels.label_type[ - iattr])[0](to_be_set) - - # Inform user data was recast - pysat.logger.info(''.join(( - wmsg, '. Recasting input for ', - repr(var), ' with key ', - repr(ikey)))) - except (TypeError, ValueError): - # Warn user data was dropped - warnings.warn(''.join(( - wmsg, '. Dropping input for ', - repr(var), ' with key ', - repr(ikey)))) - good_set = False - else: - # Extend the meta labels. Ensure the attribute - # name has no spaces and that bytes are used instead - # of strings. - iattr = ikey.replace(" ", "_") - itype = type(to_be_set) - if itype == bytes: - itype = str - - # Update the MetaLabels object and the existing - # metadata to ensure all data have all labels - self.labels.update(iattr, ikey, itype) - self._label_setter(ikey, ikey, type(to_be_set)) - - # Set the data - if good_set: - self._data.loc[var, ikey] = to_be_set - else: - # Key is 'meta' or 'children', providing higher order - # metadata. Meta inputs could be part of a larger multiple - # parameter assignment, so not all names may actually have - # a 'meta' object to add. - for item, val in zip(data_vars, input_data['meta']): - if val is not None: - # Assign meta data, using a recursive call... - # Heads to if Meta instance call. - self[item] = val - + raise TypeError("can't recast") + else: + to_be_set = pysat.utils.listify( + self.labels.label_type[ + iattr])[0](to_be_set) + + # Inform user data was recast + pysat.logger.info(''.join(( + wmsg, '. Recasting input for ', + repr(var), ' with key ', repr(ikey)))) + except (TypeError, ValueError): + # Warn user data was dropped + warnings.warn(''.join(( + wmsg, '. Dropping input for ', + repr(var), ' with key ', + repr(ikey)))) + good_set = False + else: + # Extend the meta labels. Ensure the attribute + # name has no spaces and that bytes are used instead + # of strings. + iattr = ikey.replace(" ", "_") + itype = type(to_be_set) + if itype == bytes: + itype = str + + # Update the MetaLabels object and the existing + # metadata to ensure all data have all labels + self.labels.update(iattr, ikey, itype) + self._label_setter(ikey, ikey, type(to_be_set)) + + # Set the data + if good_set: + self._data.loc[var, ikey] = to_be_set elif isinstance(input_data, pds.Series): # Outputs from Meta object are a Series. Thus, this takes in input # from a Meta object. Set data using standard assignment via a dict. @@ -1061,8 +1047,6 @@ def attr_case_name(self, name): # Create a list of all attribute names and lower case attribute names self_keys = [key for key in self.attrs()] - # for key in list(self.keys_nD()): - # self_keys.extend(self.ho_data[key].data.columns) lower_self_keys = [key.lower() for key in self_keys] case_names = [] diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index bdee5cb43..47e468922 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -105,15 +105,7 @@ def eval_loaded_data(self, test_case=True): # Test the data values for each variable for dkey in keys: lkey = dkey.lower() - if lkey in ['profiles', 'alt_profiles', 'series_profiles']: - # Test the loaded higher-dimension data - for tframe, lframe in zip(self.testInst[dkey], - self.loaded_inst[dkey]): - assert np.all(tframe == lframe), "unequal {:s} data".format( - dkey) - else: - # Test the standard data structures - assert np.all(self.testInst[dkey] == self.loaded_inst[dkey]) + assert np.all(self.testInst[dkey] == self.loaded_inst[dkey]) # Check that names are lower case when written pysat.utils.testing.assert_lists_equal(keys, new_keys, test_case=False) @@ -1232,14 +1224,6 @@ def test_meta_processor_to_from_netcdf4(self, assign_flag): def test_missing_metadata(self): """Test writing file with no metadata.""" - # Collect a list of higher order meta - ho_vars = [] - for var in self.testInst.meta.keys(): - if 'children' in self.testInst.meta[var]: - if self.testInst.meta[var]['children'] is not None: - for subvar in self.testInst.meta[var]['children'].keys(): - ho_vars.append((subvar, var)) - # Drop all metadata self.testInst.meta.keep([]) @@ -1257,12 +1241,6 @@ def test_missing_metadata(self): # Test the warning testing.eval_warnings(war, exp_warns, warn_type=UserWarning) - # Test warning for higher order data as well (pandas) - for (svar, var) in ho_vars: - wstr = ''.join(['Unable to find MetaData for ', - svar, ' subvariable of ', var]) - exp_warns.append(wstr) - # Test the warning testing.eval_warnings(war, exp_warns, warn_type=UserWarning) @@ -1701,8 +1679,7 @@ def test_remove_netcdf4_standards(self, caplog): # Enforcing netcdf4 standards removes 'fill', min, and max information # for string variables. This is not re-added by the `remove_` function # call since, strictly speaking, we don't know what to add back in. - # Also exepmting a check on long_name for higher order data with a time - # index. When loading files, pysat specifically checks for 'Epoch' as + # When loading files, pysat specifically checks for 'Epoch' as # the long_name. So, ensuring long_name for such variables is written # could break loading for existent files. I could fake it, and assign # the standard name as long_name when loading, and while that would @@ -1712,11 +1689,6 @@ def test_remove_netcdf4_standards(self, caplog): assert var in filt_meta, 'Lost metadata variable {}'.format(var) for key in self.meta_dict[var].keys(): - # Creating exception for time-index of higher order data. The - # long_name comes out differently. - if var == 'profiles' and (key == 'long_name'): - continue - # Test remaining variables accounting for possible exceptions # for string variables. if key not in ['fill', 'value_min', 'value_max']: diff --git a/pysat/utils/io.py b/pysat/utils/io.py index a935a6e98..0fb4cd4d1 100644 --- a/pysat/utils/io.py +++ b/pysat/utils/io.py @@ -235,142 +235,28 @@ def add_netcdf4_standards_to_metadict(inst, in_meta_dict, epoch_name, meta_dict.update(time_meta) - if inst[var].dtype == np.dtype('O') and coltype != str: - # This is a Series or DataFrame, possibly with more dimensions. - # Series and DataFrame data must be treated differently. - try: - # Assume it is a DataFrame and get a list of subvariables - subvars = inst[0, var].columns - is_frame = True - except AttributeError: - # Data is Series of Series, which doesn't have columns. - subvars = [inst[0, var].name] - is_frame = False - - # Get the dimensions and their names - dims = np.shape(inst[0, var]) - obj_dim_names = [] - if len(dims) == 1: - # Pad the dimensions so that the rest of the code works - # for either a Series or a DataFrame. - dims = (dims[0], 0) - - for dim in dims[:-1]: - # Don't need to go over last dimension value, - # it covers number of columns (if a DataFrame). - obj_dim_names.append(var) - - # Set the base-level meta data. - meta_dict['Depend_1'] = obj_dim_names[-1] - - # Cycle through each of the sub-variable, updating metadata. - for svar in subvars: - # Find the subvariable data within the main variable, - # checking that this is not an empty DataFrame or - # Series. Determine the underlying data types. - good_data_loc = 0 - for idat in np.arange(len(inst.data)): - if len(inst[idat, var]) > 0: - good_data_loc = idat - break - - # Get the correct location of the sub-variable based on - # the object type. - if is_frame: - good_data = inst[good_data_loc, var][svar] - else: - good_data = inst[good_data_loc, var] - - # Get subvariable information - _, sctype, sdflag = inst._get_data_info(good_data) - - if not sdflag: - # Not a datetime index - smeta_dict = {'Depend_0': epoch_name, - 'Depend_1': obj_dim_names[-1], - 'Display_Type': 'Multidimensional', - 'Format': inst._get_var_type_code(sctype), - 'Var_Type': 'data'} - else: - # Attach datetime index metadata - smeta_dict = return_epoch_metadata(inst, epoch_name) - smeta_dict.pop('MonoTon') - - # Construct name, variable_subvariable, and store. - sname = '_'.join([lower_var, svar.lower()]) - if sname in out_meta_dict: - out_meta_dict[sname].update(smeta_dict) - else: - warnings.warn(''.join(['Unable to find MetaData for ', - svar, ' subvariable of ', var])) - out_meta_dict[sname] = smeta_dict - - # Filter metadata - remove = True if sctype == str else False - out_meta_dict[sname] = filter_netcdf4_metadata( - inst, out_meta_dict[sname], sctype, remove=remove, - check_type=check_type, export_nan=export_nan, varname=sname) - - # Get information on the subvar index. This information - # stored under primary variable name. - sub_index = inst[good_data_loc, var].index - _, coltype, datetime_flag = inst._get_data_info(sub_index) - meta_dict['Format'] = inst._get_var_type_code(coltype) - - # Deal with index information for holding variable - _, index_type, index_flag = inst._get_data_info( - inst[good_data_loc, var].index) - - # Update metadata when a datetime index found - if index_flag: - update_dict = return_epoch_metadata(inst, epoch_name) - update_dict.pop('MonoTon') - update_dict.update(meta_dict) - else: - if inst[good_data_loc, var].index.name is not None: - name = inst[good_data_loc, var].index.name - else: - name = var - update_dict = {inst.meta.labels.name: name} - update_dict.update(meta_dict) - - if lower_var in out_meta_dict: - out_meta_dict[lower_var].update(update_dict) - else: - warnings.warn(''.join(['Unable to find MetaData for ', - var])) - out_meta_dict[lower_var] = update_dict - - # Filter metdata for other netCDF4 requirements - remove = True if index_type == str else False - out_meta_dict[lower_var] = filter_netcdf4_metadata( - inst, out_meta_dict[lower_var], index_type, remove=remove, - check_type=check_type, export_nan=export_nan, varname=lower_var) - + meta_dict['Format'] = inst._get_var_type_code(coltype) + + if not inst.pandas_format: + for i, dim in enumerate(list(inst[var].dims)): + meta_dict['Depend_{:1d}'.format(i)] = dim + num_dims = len(inst[var].dims) + if num_dims >= 2: + meta_dict['Display_Type'] = 'Multidimensional' + + # Update the meta data + if lower_var in out_meta_dict: + out_meta_dict[lower_var].update(meta_dict) else: - # Dealing with 1D data or xarray format - meta_dict['Format'] = inst._get_var_type_code(coltype) - - if not inst.pandas_format: - for i, dim in enumerate(list(inst[var].dims)): - meta_dict['Depend_{:1d}'.format(i)] = dim - num_dims = len(inst[var].dims) - if num_dims >= 2: - meta_dict['Display_Type'] = 'Multidimensional' - - # Update the meta data - if lower_var in out_meta_dict: - out_meta_dict[lower_var].update(meta_dict) - else: - warnings.warn(''.join(['Unable to find MetaData for ', - var])) - out_meta_dict[lower_var] = meta_dict + warnings.warn(''.join(['Unable to find MetaData for ', + var])) + out_meta_dict[lower_var] = meta_dict - # Filter metdata for other netCDF4 requirements - remove = True if coltype == str else False - out_meta_dict[lower_var] = filter_netcdf4_metadata( - inst, out_meta_dict[lower_var], coltype, remove=remove, - check_type=check_type, export_nan=export_nan, varname=lower_var) + # Filter metdata for other netCDF4 requirements + remove = True if coltype == str else False + out_meta_dict[lower_var] = filter_netcdf4_metadata( + inst, out_meta_dict[lower_var], coltype, remove=remove, + check_type=check_type, export_nan=export_nan, varname=lower_var) return out_meta_dict @@ -418,11 +304,6 @@ def remove_netcdf4_standards_from_meta(mdict, epoch_name, labels): lower_sub_keys = [ckey.lower() for ckey in mdict[key].keys()] sub_keys = list(mdict[key].keys()) - if 'meta' in lower_sub_keys: - # Higher dimensional data, recursive treatment. - mdict[key]['meta'] = remove_netcdf4_standards_from_meta( - mdict[key]['meta'], '', labels) - # Check for presence of time information for lval in lower_sub_keys: if lval in lower_time_vals: @@ -631,13 +512,6 @@ def apply_table_translation_from_file(trans_table, meta_dict): wstr = 'Translation label "{}" not found for variable "{}".' pysat.logger.debug(wstr.format(trans_key, var_key)) - # Check for higher order metadata - if 'meta' in meta_dict[var_key].keys(): - # Recursive call to process metadata - ldict = meta_dict[var_key]['meta'] - filt_dict[var_key]['meta'] = \ - apply_table_translation_from_file(trans_table, ldict) - return filt_dict @@ -673,12 +547,6 @@ def meta_array_expander(meta_dict): for key in meta_dict.keys(): loop_dict = {} for meta_key in meta_dict[key].keys(): - # Check for higher order data from 2D pandas support - if meta_key == 'meta': - loop_dict[meta_key] = meta_array_expander( - meta_dict[key][meta_key]) - continue - tst_array = np.asarray(meta_dict[key][meta_key]) if tst_array.shape == (): loop_dict[meta_key] = meta_dict[key][meta_key] @@ -1037,23 +905,7 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4', # Assign filtered metadata to pysat.Meta instance for key in filt_mdict: - if 'meta' in filt_mdict[key].keys(): - # Higher order metadata - dim_meta = pysat.Meta(**meta_kwargs) - for skey in filt_mdict[key]['meta'].keys(): - dim_meta[skey] = filt_mdict[key]['meta'][skey] - - # Remove HO metdata that was just transfered elsewhere - filt_mdict[key].pop('meta') - - # Assign standard metdata - meta[key] = filt_mdict[key] - - # Assign HO metadata - meta[key] = {'meta': dim_meta} - else: - # Standard metadata - meta[key] = filt_mdict[key] + meta[key] = filt_mdict[key] return data, meta @@ -1489,8 +1341,6 @@ def inst_to_netcdf(inst, fname, base_instrument=None, epoch_name=None, Stores 1-D data along dimension 'Epoch' - the date time index. - Stores higher order data (e.g. dataframes within series) separately - - The name of the main variable column is used to prepend subvariable names within netCDF, var_subvar_sub - A netCDF4 dimension is created for each main variable column @@ -1640,7 +1490,6 @@ def inst_to_netcdf(inst, fname, base_instrument=None, epoch_name=None, meta_translation = copy.deepcopy(meta_translation) # Ensure `meta_translation` has default values for items not assigned. - # This is needed for the higher order pandas support and may be removed. def_meta_trans = default_to_netcdf_translation_table(inst) for key in def_meta_trans.keys(): if key not in meta_translation: @@ -1767,156 +1616,10 @@ def inst_to_netcdf(inst, fname, base_instrument=None, epoch_name=None, cdfkey[:] = data.values else: - # Still dealing with an object, not just a Series of - # strings. Maps to `if` check on coltypes, being - # string-based. Presuming a Series with a DataFrame or - # Series in each location. Start by collecting some - # basic info on dimensions sizes, names, then create - # corresponding netCDF4 dimensions total dimensions - # stored for object are epoch plus ones created below - dims = np.shape(inst[key].iloc[0]) - obj_dim_names = [] - - # Pad dimensions so that the rest of the code works - # for either a Series or a DataFrame. - if len(dims) == 1: - dims = (dims[0], 0) - - # Don't need to go over last dimension value, - # it covers number of columns (if a frame). - for i, dim in enumerate(dims[:-1]): - obj_dim_names.append(case_key) - out_data.createDimension(obj_dim_names[-1], dim) - - # Create simple tuple with information needed to create - # the right dimensions for variables that will - # be written to file. - var_dim = tuple([epoch_name] + obj_dim_names) - - # Determine whether data is in a DataFrame or Series - try: - # Start by assuming it is a DataFrame - iterable = inst[key].iloc[0].columns - is_frame = True - except AttributeError: - # Otherwise get sub-variables for a Series - iterable = [inst[key].iloc[0].name] - is_frame = False - - # Find the subvariable data within the main variable, - # checking that this is not an empty DataFrame or - # Series. Determine the underlying data types. - good_data_loc = 0 - for idat in np.arange(len(inst.data)): - if len(inst.data[key].iloc[0]) > 0: - good_data_loc = idat - break - - # Found a place with data, if there is one - # now iterate over the subvariables, get data info - # create netCDF4 variables and store the data - # stored name is variable_subvariable. - for col in iterable: - if is_frame: - # We are working with a DataFrame, so - # multiple subvariables stored under a single - # main variable heading. - idx = inst[key].iloc[good_data_loc][col] - data, coltype, _ = inst._get_data_info(idx) - - # netCDF4 doesn't support string compression - if coltype == str: - lzlib = False - else: - lzlib = zlib - - cdfkey = out_data.createVariable( - '_'.join((case_key, col)), coltype, - dimensions=var_dim, zlib=lzlib, - complevel=complevel, shuffle=shuffle) - - # Set metadata - lkey = '_'.join((lower_key, col.lower())) - cdfkey.setncatts(export_meta[lkey]) - - # Attach data. It may be slow to repeatedly - # call the store method as well astype method - # below collect data into a numpy array, then - # write the full array in one go. - temp_cdf_data = np.zeros( - (num, dims[0])).astype(coltype) - for i in range(num): - temp_cdf_data[i, :] = inst[ - key].iloc[i][col].values - - # Write data - cdfkey[:, :] = temp_cdf_data - else: - # We are dealing with a Series. Get - # information from within the series. - idx = inst[key].iloc[good_data_loc] - data, coltype, _ = inst._get_data_info(idx) - - # netCDF4 doesn't support string compression - if coltype == str: - lzlib = False - else: - lzlib = zlib - - cdfkey = out_data.createVariable( - case_key + '_data', coltype, - dimensions=var_dim, zlib=lzlib, - complevel=complevel, shuffle=shuffle) - - # Set metadata - tempk = '_'.join([lower_key, lower_key]) - cdfkey.setncatts(export_meta[tempk]) - - # Attach data - temp_cdf_data = np.zeros( - (num, dims[0]), dtype=coltype) - for i in range(num): - temp_cdf_data[i, :] = inst[i, key].values - - # Write data - cdfkey[:, :] = temp_cdf_data - - # We are done storing the actual data for the given - # higher order variable. Now we need to store the index - # for all of that fancy data. - - # Get index information - data, coltype, datetime_flag = inst._get_data_info( - inst[key].iloc[good_data_loc].index) - - # Create dimension variable to store index in netCDF4 - cdfkey = out_data.createVariable(case_key, coltype, - dimensions=var_dim, - zlib=zlib, - complevel=complevel, - shuffle=shuffle) - # Set meta data - cdfkey.setncatts(export_meta[lower_key]) - - # Treat time and non-time data differently - if datetime_flag: - # Set data - temp_cdf_data = np.zeros((num, dims[0]), - dtype=coltype) - for i in range(num): - temp_cdf_data[i, :] = inst[i, key].index.values - cdfkey[:, :] = (temp_cdf_data * 1.0E-6).astype( - coltype) - - else: - # Set data - temp_cdf_data = np.zeros((num, dims[0]), - dtype=coltype) - - for i in range(num): - temp_cdf_data[i, :] = inst[ - key].iloc[i].index.astype(coltype) - cdfkey[:, :] = temp_cdf_data + raise ValueError(' '.join(('Recursive data detected.', + 'Please ensure that the', + 'DataFrame does not contain', + 'other pandas objects.'))) else: # Attach the metadata to a separate xarray.Dataset object, ensuring # the Instrument data object is unchanged. From 4a2e3f2e4e42d67a45df4496510f1d363226708b Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 4 Oct 2023 11:08:32 -0400 Subject: [PATCH 134/365] DOC: update documentation refs --- docs/tutorial/tutorial_files.rst | 2 +- pysat/_instrument.py | 34 +++----------------------- pysat/_meta.py | 41 +++++--------------------------- 3 files changed, 10 insertions(+), 67 deletions(-) diff --git a/docs/tutorial/tutorial_files.rst b/docs/tutorial/tutorial_files.rst index 067518cb0..04599168e 100644 --- a/docs/tutorial/tutorial_files.rst +++ b/docs/tutorial/tutorial_files.rst @@ -150,7 +150,7 @@ as fill, _FillValue, and FillVal. When writing files pysat processes metadata for both xarray and pandas before writing the file. For xarray, pysat leverages xarray's built-in file writing capabilities. For pandas, pysat interfaces with netCDF4 directly to translate -both 1D and higher dimensional data into netCDF4. +data into netCDF4. .. _tutorial-files-meta: diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 31a85c254..9d5c5b3b7 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -2814,33 +2814,6 @@ def rename(self, mapper, lowercase_data_labels=False): inst.rename(str.upper) - If using a pandas-type Instrument with higher-order data and a - dictionary mapper, the upper-level data key must contain a dictionary - for renaming the dependent data variables. The upper-level data key - cannot be renamed. Note that this rename will be invoked individually - for all times in the dataset. - :: - - # Applies to higher-order datasets that are loaded into pandas - inst = pysat.Instrument('pysat', 'testing2D') - inst.load(2009, 1) - mapper = {'uts': 'pysat_uts', - 'profiles': {'density': 'pysat_density'}} - inst.rename(mapper) - print(inst[0, 'profiles'].columns) # 'density' will be updated - - # To rename higher-order data at both levels using a dictionary, - # you need two calls - mapper2 = {'profiles': 'pysat_profile'} - inst.rename(mapper2) - print(inst[0, 'pysat_profile'].columns) - - # A function will affect both standard and higher-order data. - # Remember this function also updates the Meta data. - inst.rename(str.capitalize) - print(inst.meta['Pysat_profile']['children']) - - pysat supports differing case for variable labels across the data and metadata objects attached to an Instrument. Since Meta is case-preserving (on assignment) but case-insensitive to access, the @@ -2850,10 +2823,9 @@ def rename(self, mapper, lowercase_data_labels=False): :: # Example with lowercase_data_labels - inst = pysat.Instrument('pysat', 'testing2D') + inst = pysat.Instrument('pysat', 'testing') inst.load(2009, 1) - mapper = {'uts': 'Pysat_UTS', - 'profiles': {'density': 'PYSAT_density'}} + mapper = {'uts': 'Pysat_UTS'} inst.rename(mapper, lowercase_data_labels=True) # Note that 'Pysat_UTS' was applied to data as 'pysat_uts' @@ -2886,7 +2858,7 @@ def rename(self, mapper, lowercase_data_labels=False): # Initialize dict for renaming normal pandas data pdict = {} - # Collect normal variables and rename higher order variables + # Collect and rename variables for vkey in self.variables: map_key = pysat.utils.get_mapped_value(vkey, mapper) diff --git a/pysat/_meta.py b/pysat/_meta.py index 83fad9c47..25a63fac7 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -67,10 +67,6 @@ class Meta(object): but the original case is preserved. Case preseveration is built in to support writing files with a desired case to meet standards. - Metadata for higher order data objects, those that have - multiple products under a single variable name in a `pysat.Instrument` - object, are stored by providing a Meta object under the single name. - Supports any custom metadata values in addition to the expected metadata attributes (units, name, notes, desc, value_min, value_max, and fill). These base attributes may be used to programatically access and set types @@ -126,15 +122,6 @@ class Meta(object): meta['var_name4'].children['name41'] meta['var_name4'].children['name42'] - # You may, of course, have a mixture of 1D and nD data - meta = pysat.Meta() - meta['dm'] = {'units': 'hey', 'long_name': 'boo'} - meta['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'} - meta2 = pysat.Meta() - meta2[['higher', 'lower']] = {'meta': [meta, None], - 'units': [None, 'boo'], - 'long_name': [None, 'boohoo']} - # Meta data may be assigned from another Meta object using dict-like # assignments key1 = 'var_name' @@ -794,8 +781,6 @@ def empty(self): """ - # Only need to check on lower data since lower data - # is set when higher metadata assigned. if self.data.empty: return True else: @@ -813,7 +798,6 @@ def merge(self, other): for key in other.keys(): if key not in self: - # Copies over both lower and higher dimensional data self[key] = other[key] return @@ -999,10 +983,6 @@ def hasattr_case_neutral(self, attr_name): True if the case-insensitive check for attribute name is successful, False if no attribute name is present. - Note - ---- - Does not check higher order meta objects - """ if attr_name.lower() in [dcol.lower() for dcol in self.data.columns]: @@ -1025,11 +1005,9 @@ def attr_case_name(self, name): Note ---- - Checks first within standard attributes. If not found there, checks - attributes for higher order data structures. If not found, returns - supplied name as it is available for use. Intended to be used to help - ensure that the same case is applied to all repetitions of a given - variable name. + Checks first within standard attributes. If not found, returns supplied + name as it is available for use. Intended to be used to help ensure that + the same case is applied to all repetitions of a given variable name. """ @@ -1074,18 +1052,11 @@ def rename(self, mapper): Dictionary with old names as keys and new names as variables or a function to apply to all names - Raises - ------ - ValueError - When normal data is treated like higher-order data in dict mapping. - Note ---- - Checks first within standard attributes. If not found there, checks - attributes for higher order data structures. If not found, returns - supplied name as it is available for use. Intended to be used to help - ensure that the same case is applied to all repetitions of a given - variable name. + Checks first within standard attributes. If not found, returns supplied + name as it is available for use. Intended to be used to help ensure that + the same case is applied to all repetitions of a given variable name. """ From bf6ed856dfaafe05c1a41f700659741653cc88e8 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 4 Oct 2023 11:31:52 -0400 Subject: [PATCH 135/365] BUG: remove unused variable --- pysat/tests/test_utils_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 47e468922..cb3dd7b87 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -104,7 +104,6 @@ def eval_loaded_data(self, test_case=True): # Test the data values for each variable for dkey in keys: - lkey = dkey.lower() assert np.all(self.testInst[dkey] == self.loaded_inst[dkey]) # Check that names are lower case when written From 294fe4c8defb68271336c1527d8cfec7af77fdc7 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 4 Oct 2023 13:34:37 -0400 Subject: [PATCH 136/365] STY: no children --- pysat/_meta.py | 13 ------------- pysat/tests/test_meta.py | 3 --- pysat/utils/io.py | 4 ---- 3 files changed, 20 deletions(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index 25a63fac7..bf14905e6 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -117,11 +117,6 @@ class Meta(object): meta2['var_name42'] = {'long_name': 'name2of4', 'units': 'Units2'} meta['var_name4'] = {'meta': meta2} - # An alternative method to achieve the same result is: - meta['var_name4'] = meta2 - meta['var_name4'].children['name41'] - meta['var_name4'].children['name42'] - # Meta data may be assigned from another Meta object using dict-like # assignments key1 = 'var_name' @@ -454,14 +449,6 @@ def __setitem__(self, data_vars, input_dat): # Set the data if good_set: self._data.loc[var, ikey] = to_be_set - elif isinstance(input_data, pds.Series): - # Outputs from Meta object are a Series. Thus, this takes in input - # from a Meta object. Set data using standard assignment via a dict. - in_dict = input_data.to_dict() - if 'children' in in_dict: - child = in_dict.pop('children') # noqa: F841 - # Remaining items are simply assigned via recursive call - self[data_vars] = in_dict return diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index adfaf9c6c..de3bf2c6e 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -105,9 +105,6 @@ def eval_meta_settings(self, isfloat=True): lkey.__repr__(), self.meta[self.dval, lkey].__repr__(), self.default_val[lkey].__repr__()) - assert 'children' not in self.meta.data.columns - return - return # ----------------------- diff --git a/pysat/utils/io.py b/pysat/utils/io.py index 0fb4cd4d1..2b972ca4a 100644 --- a/pysat/utils/io.py +++ b/pysat/utils/io.py @@ -882,10 +882,6 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4', for label in drop_meta_labels: if label in full_mdict[var]: full_mdict[var].pop(label) - if 'meta' in full_mdict[var]: - for var2 in full_mdict[var]['meta'].keys(): - if label in full_mdict[var]['meta'][var2]: - full_mdict[var]['meta'][var2].pop(label) # Second, remove some items pysat added for netcdf compatibility. filt_mdict = remove_netcdf4_standards_from_meta(full_mdict, epoch_name, From df91d43564eea3bdee2ac50ebec6fd61a89ba2c4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:10:53 -0400 Subject: [PATCH 137/365] Update pysat/utils/registry.py Co-authored-by: Angeline Burrell --- pysat/utils/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py index ac42838e1..fd0beec76 100644 --- a/pysat/utils/registry.py +++ b/pysat/utils/registry.py @@ -223,7 +223,7 @@ def register_by_module(module, overwrite=False): attached as sub-modules to the input `module` overwrite : bool If True, an existing registration will be updated - with the new module information. + with the new module information. (default=False) Raises ------ From 63afe6ec14ee039a6159fddde21d3f254b3b4716 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:11:46 -0400 Subject: [PATCH 138/365] Update registry.py --- pysat/utils/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py index fd0beec76..77c1f8338 100644 --- a/pysat/utils/registry.py +++ b/pysat/utils/registry.py @@ -92,7 +92,7 @@ def register(module_names, overwrite=False): specify package name and instrument modules overwrite : bool If True, an existing registration will be updated - with the new module information. + with the new module information. (default=False) Raises ------ From ad4509296907006f1013b5ef1ccfdad3bf0f0fb6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 5 Oct 2023 12:17:41 -0400 Subject: [PATCH 139/365] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysat/_meta.py | 1 + pysat/tests/test_meta.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/pysat/_meta.py b/pysat/_meta.py index bf14905e6..ca2fd982a 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -506,6 +506,7 @@ def match_name(func, var_name, index_or_column): # If tuple length is 2, index, column new_index = match_name(self.var_case_name, key[0], self.data.index) + # Assume this is a label name new_name = match_name(self.attr_case_name, key[1], self.data.columns) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index de3bf2c6e..77b19437a 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -1373,8 +1373,12 @@ def test_meta_mutable_properties(self): try: # Pandas does not support dataframe equality setattr(self.meta, 'data', pds.DataFrame()) + + # Test that data is empty + assert self.meta.data.empty, "`meta.data` not updated correctly" except AttributeError: raise AssertionError("Couldn't update mutable property 'data'") + return @pytest.mark.parametrize("label", ['units', 'name', 'desc', 'notes', 'min_val', 'max_val', 'fill_val']) From 25053cd65f4cbeb77501cc2df1e6729227bb2806 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 5 Oct 2023 12:18:40 -0400 Subject: [PATCH 140/365] BUG: allow input as series --- pysat/_meta.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysat/_meta.py b/pysat/_meta.py index bf14905e6..fb094baa3 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -449,6 +449,10 @@ def __setitem__(self, data_vars, input_dat): # Set the data if good_set: self._data.loc[var, ikey] = to_be_set + elif isinstance(input_data, pds.Series): + # Outputs from Meta object are a Series. Thus, this takes in input + # from a Meta object. Set data using standard assignment via a dict. + self[data_vars] = input_data.to_dict() return From 2d631d94d5cfade83f01ec40a0da5db9ef3f67d4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 5 Oct 2023 12:19:43 -0400 Subject: [PATCH 141/365] STY: whitespace --- pysat/tests/test_meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 77b19437a..5e3b1699a 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -1373,7 +1373,7 @@ def test_meta_mutable_properties(self): try: # Pandas does not support dataframe equality setattr(self.meta, 'data', pds.DataFrame()) - + # Test that data is empty assert self.meta.data.empty, "`meta.data` not updated correctly" except AttributeError: From 2f29f0ecbe1499a34f7b56e10ab62ab86bf054e6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 5 Oct 2023 12:30:18 -0400 Subject: [PATCH 142/365] STY: e275 --- pysat/_orbits.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pysat/_orbits.py b/pysat/_orbits.py index 7f68f77bd..dcbc604b2 100644 --- a/pysat/_orbits.py +++ b/pysat/_orbits.py @@ -474,8 +474,11 @@ def _equa_breaks(self, orbit_index_period=24.0): # Iterate over samples and check. for sub_tidx in tidx: # Look at time change vs local time change - if(ut_diff[sub_idx].iloc[sub_tidx] * orbit_index_period - < lt_diff[sub_idx].iloc[sub_tidx] * self.orbit_period): + false_alarm = ( + ut_diff[sub_idx].iloc[sub_tidx] * orbit_index_period + < lt_diff[sub_idx].iloc[sub_tidx] + * self.orbit_period) + if false_alarm: # The change in UT is small compared to the change # in the orbit index this is flagged as a false From 8a505c5e69eeb31c936829ffc6c2fd92d8ff801c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 5 Oct 2023 13:44:44 -0400 Subject: [PATCH 143/365] MAINT: remove deprecated instruments --- pysat/instruments/__init__.py | 3 +- pysat/instruments/pysat_testing2d_xarray.py | 66 ------ pysat/instruments/pysat_testing_xarray.py | 223 -------------------- 3 files changed, 1 insertion(+), 291 deletions(-) delete mode 100644 pysat/instruments/pysat_testing2d_xarray.py delete mode 100644 pysat/instruments/pysat_testing_xarray.py diff --git a/pysat/instruments/__init__.py b/pysat/instruments/__init__.py index 248e80f36..882080c8b 100644 --- a/pysat/instruments/__init__.py +++ b/pysat/instruments/__init__.py @@ -5,8 +5,7 @@ """ __all__ = ['pysat_ndtesting', 'pysat_netcdf', 'pysat_testing', - 'pysat_testmodel', 'pysat_testing_xarray', - 'pysat_testing2d_xarray'] + 'pysat_testmodel'] for inst in __all__: exec("from pysat.instruments import {x}".format(x=inst)) diff --git a/pysat/instruments/pysat_testing2d_xarray.py b/pysat/instruments/pysat_testing2d_xarray.py deleted file mode 100644 index 334821385..000000000 --- a/pysat/instruments/pysat_testing2d_xarray.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -"""Produces fake instrument data for testing. - -.. deprecated:: 3.1.0 - This module has been renamed pysat_ndtesting. A copy inheriting the - routines from the new location is maintained here for backwards- - compatibility. This instrument will be removed in 3.2.0+ to reduce - redundancy. - -""" - -import datetime as dt -import functools -import numpy as np -import warnings - -import xarray as xr - -import pysat -from pysat.instruments.methods import testing as mm_test -from pysat.instruments import pysat_ndtesting - -platform = 'pysat' -name = 'testing2d_xarray' - -tags = pysat_ndtesting.tags -inst_ids = pysat_ndtesting.inst_ids -pandas_format = pysat_ndtesting.pandas_format -_test_dates = pysat_ndtesting._test_dates - - -# Init method -def init(self, test_init_kwarg=None): - """Initialize the test instrument. - - Parameters - ---------- - self : pysat.Instrument - This object - test_init_kwarg : any - Testing keyword (default=None) - - """ - - warnings.warn(" ".join(["The instrument module `pysat_testing2d_xarray`", - "has been deprecated and will be removed in", - "3.2.0+. Please use `pysat_ndtesting` instead."]), - DeprecationWarning, stacklevel=2) - - mm_test.init(self, test_init_kwarg=test_init_kwarg) - return - - -# Clean method -clean = pysat_ndtesting.clean - -# Optional method, preprocess -preprocess = pysat_ndtesting.preprocess - -load = pysat_ndtesting.load - -list_files = functools.partial(pysat_ndtesting.list_files, - test_dates=_test_dates) -list_remote_files = functools.partial(pysat_ndtesting.list_remote_files, - test_dates=_test_dates) -download = functools.partial(pysat_ndtesting.download) diff --git a/pysat/instruments/pysat_testing_xarray.py b/pysat/instruments/pysat_testing_xarray.py deleted file mode 100644 index 0edd7365b..000000000 --- a/pysat/instruments/pysat_testing_xarray.py +++ /dev/null @@ -1,223 +0,0 @@ -# -*- coding: utf-8 -*- -"""Produces fake instrument data for testing. - -.. deprecated:: 3.0.2 - All data present in this instrument is duplicated in pysat_ndtesting. - This instrument will be removed in 3.2.0+ to reduce redundancy. - -""" - -import datetime as dt -import functools -import numpy as np -import warnings - -import xarray as xr - -import pysat -from pysat.instruments.methods import testing as mm_test - -# pysat required parameters -platform = 'pysat' -name = 'testing_xarray' - -# Dictionary of data 'tags' and corresponding description -tags = {'': 'Regular testing data set'} - -# Dictionary of satellite IDs, list of corresponding tags -inst_ids = {'': ['']} -_test_dates = {'': {'': dt.datetime(2009, 1, 1)}} -pandas_format = False - -epoch_name = u'time' - - -# Init method -def init(self, test_init_kwarg=None): - """Initialize the test instrument. - - Parameters - ---------- - self : pysat.Instrument - This object - test_init_kwarg : any - Testing keyword (default=None) - - """ - - warnings.warn(" ".join(["The instrument module `pysat_testing_xarray` has", - "been deprecated and will be removed in 3.2.0+."]), - DeprecationWarning, stacklevel=2) - - mm_test.init(self, test_init_kwarg=test_init_kwarg) - return - - -# Clean method -clean = mm_test.clean - -# Optional method, preprocess -preprocess = mm_test.preprocess - - -def load(fnames, tag='', inst_id='', sim_multi_file_right=False, - sim_multi_file_left=False, non_monotonic_index=False, - non_unique_index=False, malformed_index=False, start_time=None, - num_samples=86400, test_load_kwarg=None, max_latitude=90.): - """Load the test files. - - Parameters - ---------- - fnames : list - List of filenames. - tag : str - Tag name used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. (default='') - inst_id : str - Instrument ID used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. (default='') - sim_multi_file_right : bool - Adjusts date range to be 12 hours in the future or twelve hours beyond - `root_date`. (default=False) - sim_multi_file_left : bool - Adjusts date range to be 12 hours in the past or twelve hours before - `root_date`. (default=False) - non_monotonic_index : bool - If True, time index will be non-monotonic (default=False) - non_unique_index : bool - If True, time index will be non-unique (default=False) - malformed_index : bool - If True, the time index will be non-unique and non-monotonic. Deprecated - and scheduled for removal in pysat 3.2.0. (default=False) - start_time : dt.timedelta or NoneType - Offset time of start time since midnight UT. If None, instrument data - will begin at midnight. (default=None) - num_samples : int - Maximum number of times to generate. Data points will not go beyond the - current day. (default=86400) - test_load_kwarg : any - Keyword used for pysat unit testing to ensure that functionality for - custom keywords defined in instrument support functions is working - correctly. (default=None) - max_latitude : float - Latitude simulated as `max_latitude` * cos(theta(t))`, where - theta is a linear periodic signal bounded by [0, 2 * pi). (default=90.) - - Returns - ------- - data : xr.Dataset - Testing data - meta : pysat.Meta - Metadata - - """ - - # Support keyword testing - pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg)))) - - # Create an artificial satellite data set - iperiod = mm_test.define_period() - drange = mm_test.define_range() - - uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='1S', - start_time=start_time) - - if sim_multi_file_right: - root_date = dt.datetime(2009, 1, 1, 12) - elif sim_multi_file_left: - root_date = dt.datetime(2008, 12, 31, 12) - else: - root_date = dt.datetime(2009, 1, 1) - - # TODO(#1094): Remove in pysat 3.2.0 - if malformed_index: - # Warn that kwarg is deprecated and set new kwargs. - mm_test._warn_malformed_kwarg() - non_monotonic_index = True - non_unique_index = True - - if non_monotonic_index: - index = mm_test.non_monotonic_index(index) - if non_unique_index: - index = mm_test.non_unique_index(index) - - data = xr.Dataset({'uts': ((epoch_name), uts)}, - coords={epoch_name: index}) - - # Need to create simple orbits here. Have start of first orbit - # at 2009,1, 0 UT. 14.84 orbits per day. Figure out how far in time from - # the root start a measurement is and use that info to create a signal - # that is continuous from that start. Going to presume there are 5820 - # seconds per orbit (97 minute period). - time_delta = dates[0] - root_date - mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts, - period=iperiod['lt'], - data_range=drange['lt']) - data['mlt'] = ((epoch_name), mlt) - - # SLT, 20 second offset from `mlt`. - slt = mm_test.generate_fake_data(time_delta.total_seconds() + 20, uts, - period=iperiod['lt'], - data_range=drange['lt']) - data['slt'] = ((epoch_name), slt) - - # Create a fake longitude, resets every 6240 seconds. Sat moves at - # 360/5820 deg/s, Earth rotates at 360/86400, takes extra time to go - # around full longitude. - longitude = mm_test.generate_fake_data(time_delta.total_seconds(), uts, - period=iperiod['lon'], - data_range=drange['lon']) - data['longitude'] = ((epoch_name), longitude) - - # Create latitude area for testing polar orbits - angle = mm_test.generate_fake_data(time_delta.total_seconds(), uts, - period=iperiod['angle'], - data_range=drange['angle']) - latitude = max_latitude * np.cos(angle) - data['latitude'] = ((epoch_name), latitude) - - # Create constant altitude at 400 km - alt0 = 400.0 - altitude = alt0 * np.ones(data['latitude'].shape) - data['altitude'] = ((epoch_name), altitude) - - # Fake orbit number - fake_delta = dates[0] - dt.datetime(2008, 1, 1) - orbit_num = mm_test.generate_fake_data(fake_delta.total_seconds(), - uts, period=iperiod['lt'], - cyclic=False) - - data['orbit_num'] = ((epoch_name), orbit_num) - - # Create some fake data to support testing of averaging routines - mlt_int = data['mlt'].astype(int).data - long_int = (data['longitude'] / 15.).astype(int).data - data['dummy1'] = ((epoch_name), mlt_int) - data['dummy2'] = ((epoch_name), long_int) - data['dummy3'] = ((epoch_name), mlt_int + long_int * 1000.) - data['dummy4'] = ((epoch_name), uts) - data['string_dummy'] = ((epoch_name), - ['test'] * len(data.indexes[epoch_name])) - data['unicode_dummy'] = ((epoch_name), - [u'test'] * len(data.indexes[epoch_name])) - data['int8_dummy'] = ((epoch_name), - np.ones(len(data.indexes[epoch_name]), dtype=np.int8)) - data['int16_dummy'] = ((epoch_name), - np.ones(len(data.indexes[epoch_name]), - dtype=np.int16)) - data['int32_dummy'] = ((epoch_name), - np.ones(len(data.indexes[epoch_name]), - dtype=np.int32)) - data['int64_dummy'] = ((epoch_name), - np.ones(len(data.indexes[epoch_name]), - dtype=np.int64)) - - # Set the meta data - meta = mm_test.initialize_test_meta(epoch_name, data.keys()) - return data, meta - - -list_files = functools.partial(mm_test.list_files, test_dates=_test_dates) -list_remote_files = functools.partial(mm_test.list_remote_files, - test_dates=_test_dates) -download = functools.partial(mm_test.download) From 0054093077a7a802c332727c8db96e5ce39b726b Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 5 Oct 2023 13:52:55 -0400 Subject: [PATCH 144/365] ENH: add capability to ndtesting to mirror deprecated instrument --- pysat/instruments/pysat_ndtesting.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 7a63d54ba..f10be34a4 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -33,10 +33,11 @@ preprocess = mm_test.preprocess -def load(fnames, tag='', inst_id='', non_monotonic_index=False, +def load(fnames, tag='', inst_id='', sim_multi_file_right=False, + sim_multi_file_left=False, non_monotonic_index=False, non_unique_index=False, malformed_index=False, start_time=None, - num_samples=864, test_load_kwarg=None, max_latitude=90.0, - num_extra_time_coords=0): + num_samples=864, sample_rate='100S', test_load_kwarg=None, + max_latitude=90.0, num_extra_time_coords=0): """Load the test files. Parameters @@ -49,6 +50,12 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, inst_id : str Instrument ID used to identify particular data set to be loaded. This input is nominally provided by pysat itself. (default='') + sim_multi_file_right : bool + Adjusts date range to be 12 hours in the future or twelve hours beyond + `root_date`. (default=False) + sim_multi_file_left : bool + Adjusts date range to be 12 hours in the past or twelve hours before + `root_date`. (default=False) non_monotonic_index : bool If True, time index will be non-monotonic (default=False) non_unique_index : bool @@ -63,6 +70,8 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, num_samples : int Maximum number of times to generate. Data points will not go beyond the current day. (default=864) + sample_rate : str + Frequency of data points, using pandas conventions. (default='100s') test_load_kwarg : any Keyword used for pysat unit testing to ensure that functionality for custom keywords defined in instrument support functions is working @@ -90,8 +99,16 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, drange = mm_test.define_range() # Using 100s frequency for compatibility with seasonal analysis unit tests - uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='100S', + uts, index, dates = mm_test.generate_times(fnames, num_samples, + freq=sample_rate, start_time=start_time) + if sim_multi_file_right: + root_date = dt.datetime(2009, 1, 1, 12) + elif sim_multi_file_left: + root_date = dt.datetime(2008, 12, 31, 12) + else: + root_date = dt.datetime(2009, 1, 1) + # TODO(#1094): Remove in pysat 3.2.0 if malformed_index: # Warn that kwarg is deprecated and set new kwargs. @@ -112,7 +129,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False, # the root start a measurement is and use that info to create a signal # that is continuous from that start. Going to presume there are 5820 # seconds per orbit (97 minute period). - time_delta = dates[0] - dt.datetime(2009, 1, 1) + time_delta = dates[0] - root_date # MLT runs 0-24 each orbit mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts, From f68067fa57b6b973e39a66856f36afe428909de0 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 5 Oct 2023 14:09:24 -0400 Subject: [PATCH 145/365] MAINT: replace deprecated instruments in tests --- pysat/tests/test_instrument.py | 31 +------------- pysat/tests/test_instrument_custom.py | 4 +- pysat/tests/test_instrument_index.py | 2 +- pysat/tests/test_instrument_padding.py | 57 +++++++++++++++++--------- pysat/tests/test_instruments.py | 29 ------------- pysat/tests/test_meta.py | 29 ++----------- pysat/tests/test_orbits.py | 24 +++++------ pysat/tests/test_utils_coords.py | 9 ++-- pysat/tests/test_utils_io.py | 47 ++------------------- 9 files changed, 64 insertions(+), 168 deletions(-) diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 18861ca59..aeb1b8cdb 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -12,7 +12,6 @@ import pysat import pysat.instruments.pysat_ndtesting import pysat.instruments.pysat_testing -import pysat.instruments.pysat_testing_xarray from pysat.tests.classes.cls_instrument_access import InstAccessTests from pysat.tests.classes.cls_instrument_integration import InstIntegrationTests @@ -199,34 +198,6 @@ def teardown_method(self): return -# TODO(#908): remove below class when pysat_testing_xarray is removed. -class TestBasicsXarray(TestBasics): - """Basic tests for xarray `pysat.Instrument`.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - - reload(pysat.instruments.pysat_testing_xarray) - self.testInst = pysat.Instrument(platform='pysat', - name='testing_xarray', - num_samples=10, - clean_level='clean', - update_files=True, - use_header=True, - **self.testing_kwargs) - self.ref_time = pysat.instruments.pysat_testing_xarray._test_dates[ - ''][''] - self.ref_doy = int(self.ref_time.strftime('%j')) - self.out = None - return - - def teardown_method(self): - """Clean up the unit test environment after each method.""" - - del self.testInst, self.out, self.ref_time, self.ref_doy - return - - class TestBasicsNDXarray(TestBasics): """Basic tests for ND xarray `pysat.Instrument`. @@ -548,7 +519,7 @@ def test_eq_different_object(self): num_samples=10, clean_level='clean', update_files=True, use_header=True) - obj2 = pysat.Instrument(platform='pysat', name='testing_xarray', + obj2 = pysat.Instrument(platform='pysat', name='ndtesting', num_samples=10, clean_level='clean', update_files=True, use_header=True) assert not (obj1 == obj2) diff --git a/pysat/tests/test_instrument_custom.py b/pysat/tests/test_instrument_custom.py index 2c3f5ba30..b7888c7ac 100644 --- a/pysat/tests/test_instrument_custom.py +++ b/pysat/tests/test_instrument_custom.py @@ -235,10 +235,10 @@ class TestBasicsXarray(TestBasics): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', num_samples=10, clean_level='clean', use_header=True) - self.load_date = pysat.instruments.pysat_testing_xarray._test_dates + self.load_date = pysat.instruments.pysat_ndtesting._test_dates self.load_date = self.load_date[''][''] self.testInst.load(date=self.load_date) self.custom_args = [2] diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py index ac87a74ce..451598019 100644 --- a/pysat/tests/test_instrument_index.py +++ b/pysat/tests/test_instrument_index.py @@ -107,7 +107,7 @@ def eval_warnings(self): return # TODO(#1094): Remove in pysat 3.2.0, potentially with class - @pytest.mark.parametrize('name', ['testing', 'ndtesting', 'testing_xarray']) + @pytest.mark.parametrize('name', ['testing', 'ndtesting']) def test_kwarg_malformed_index(self, name): """Test deprecation of `malformed_index` kwarg. diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py index 59e02e687..ad30f55ac 100644 --- a/pysat/tests/test_instrument_padding.py +++ b/pysat/tests/test_instrument_padding.py @@ -10,7 +10,6 @@ import pysat import pysat.instruments.pysat_ndtesting import pysat.instruments.pysat_testing -import pysat.instruments.pysat_testing_xarray from pysat.utils import testing from pysat.utils.time import filter_datetime_input @@ -143,9 +142,11 @@ class TestDataPaddingbyFileXarray(TestDataPaddingbyFile): def setup_method(self): """Set up the unit test environment for each method.""" - reload(pysat.instruments.pysat_testing_xarray) + reload(pysat.instruments.pysat_ndtesting) self.testInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', + num_samples=86400, + sample_rate='1s', clean_level='clean', pad={'minutes': 5}, update_files=True, @@ -153,7 +154,9 @@ def setup_method(self): self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile') self.rawInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', + num_samples=86400, + sample_rate='1s', clean_level='clean', update_files=True, use_header=True) @@ -205,16 +208,20 @@ class TestOffsetRightFileDataPaddingBasicsXarray(TestDataPaddingbyFile): def setup_method(self): """Set up the unit test environment for each method.""" - reload(pysat.instruments.pysat_testing_xarray) + reload(pysat.instruments.pysat_ndtesting) self.testInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', + num_samples=86400, + sample_rate='1s', clean_level='clean', update_files=True, sim_multi_file_right=True, pad={'minutes': 5}, use_header=True) self.rawInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', + num_samples=86400, + sample_rate='1s', clean_level='clean', update_files=True, sim_multi_file_right=True, @@ -499,9 +506,11 @@ class TestDataPaddingXArray(TestDataPadding): def setup_method(self): """Set up the unit test environment for each method.""" - reload(pysat.instruments.pysat_testing_xarray) + reload(pysat.instruments.pysat_ndtesting) self.testInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', + num_samples=86400, + sample_rate='1s', clean_level='clean', pad={'minutes': 5}, update_files=True, @@ -524,9 +533,11 @@ class TestDataPaddingXArrayNonMonotonic(TestDataPadding): def setup_method(self): """Set up the unit test environment for each method.""" - reload(pysat.instruments.pysat_testing_xarray) + reload(pysat.instruments.pysat_ndtesting) self.testInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', + num_samples=86400, + sample_rate='1s', clean_level='clean', pad={'minutes': 5}, non_monotonic_index=True, @@ -603,9 +614,11 @@ class TestMultiFileRightDataPaddingBasicsXarray(TestDataPadding): def setup_method(self): """Set up the unit test environment for each method.""" - reload(pysat.instruments.pysat_testing_xarray) + reload(pysat.instruments.pysat_ndtesting) self.testInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', + num_samples=86400, + sample_rate='1s', clean_level='clean', update_files=True, sim_multi_file_right=True, @@ -630,9 +643,11 @@ class TestMultiFileRightDataPaddingBasicsXarrayNonMonotonic(TestDataPadding): def setup_method(self): """Set up the unit test environment for each method.""" - reload(pysat.instruments.pysat_testing_xarray) + reload(pysat.instruments.pysat_ndtesting) self.testInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', + num_samples=86400, + sample_rate='1s', clean_level='clean', update_files=True, sim_multi_file_right=True, @@ -713,9 +728,11 @@ class TestMultiFileLeftDataPaddingBasicsXarray(TestDataPadding): def setup_method(self): """Set up the unit test environment for each method.""" - reload(pysat.instruments.pysat_testing_xarray) + reload(pysat.instruments.pysat_ndtesting) self.testInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', + num_samples=86400, + sample_rate='1s', clean_level='clean', update_files=True, sim_multi_file_left=True, @@ -740,11 +757,13 @@ class TestMultiFileLeftDataPaddingBasicsXarrayNonMonotonic(TestDataPadding): def setup_method(self): """Set up the unit test environment for each method.""" - reload(pysat.instruments.pysat_testing_xarray) + reload(pysat.instruments.pysat_ndtesting) self.testInst = pysat.Instrument(platform='pysat', - name='testing_xarray', + name='ndtesting', clean_level='clean', update_files=True, + num_samples=86400, + sample_rate='1s', sim_multi_file_left=True, non_monotonic_index=True, pad={'minutes': 5}, diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index b1f03a42d..470a83278 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -274,32 +274,3 @@ def test_old_pytest_mark_presence(self): for j in range(0, n_args)] assert "download" in mark_names - - @pytest.mark.parametrize("inst_module", ['pysat_testing_xarray', - 'pysat_testing2d_xarray']) - def test_deprecated_instruments(self, inst_module): - """Check that instantiating old instruments raises a DeprecationWarning. - - Parameters - ---------- - inst_module : str - name of deprecated module. - - """ - - with warnings.catch_warnings(record=True) as war: - pysat.Instrument(inst_module=getattr(pysat.instruments, - inst_module), - use_header=True) - - warn_msgs = [" ".join(["The instrument module", - "`{:}`".format(inst_module), - "has been deprecated and will be removed", - "in 3.2.0+."])] - - # Ensure the minimum number of warnings were raised. - assert len(war) >= len(warn_msgs) - - # Test the warning messages, ensuring each attribute is present. - testing.eval_warnings(war, warn_msgs) - return diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 5e3b1699a..9de618392 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -412,10 +412,7 @@ def test_equality(self): assert cmeta == self.meta, "identical meta objects differ" return - # TODO(#908): remove tests for deprecated instruments - @pytest.mark.parametrize("inst_name", ["testing", - "ndtesting", "testing_xarray", - "testmodel"]) + @pytest.mark.parametrize("inst_name", ["testing", "ndtesting", "testmodel"]) def test_equality_w_copy(self, inst_name): """Test that meta remains the same when copied. @@ -489,10 +486,7 @@ def test_value_inequality(self, label_key): "differences not detected in label {:s}".format(label_key) return - # TODO(#908): remove tests for deprecated instruments - @pytest.mark.parametrize("inst_name", ["testing", - "ndtesting", "testing_xarray", - "testmodel"]) + @pytest.mark.parametrize("inst_name", ["testing", "ndtesting", "testmodel"]) def test_pop(self, inst_name): """Test meta attributes are retained when extracted using pop. @@ -1614,23 +1608,6 @@ def test_to_dict(self, preserve_case): return -class TestToDictXarray(TestToDict): - """Test `.to_dict` methods using pysat test Instruments.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - - self.testInst = pysat.Instrument('pysat', 'testing_xarray', - num_samples=5, use_header=True) - self.stime = pysat.instruments.pysat_testing_xarray._test_dates[''][''] - self.testInst.load(date=self.stime) - - # For output - self.out = None - - return - - class TestToDictXarrayND(TestToDict): """Test `.to_dict` methods using pysat test Instruments.""" @@ -1639,7 +1616,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'ndtesting', num_samples=5, use_header=True) - self.stime = pysat.instruments.pysat_testing_xarray._test_dates[''][''] + self.stime = pysat.instruments.pysat_ndtesting._test_dates[''][''] self.testInst.load(date=self.stime) # For output diff --git a/pysat/tests/test_orbits.py b/pysat/tests/test_orbits.py index 28d3173f6..c849d29a9 100644 --- a/pysat/tests/test_orbits.py +++ b/pysat/tests/test_orbits.py @@ -589,12 +589,12 @@ class TestGeneralOrbitsMLTxarray(TestGeneralOrbitsMLT): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'mlt'}, update_files=True, use_header=True) - self.stime = pysat.instruments.pysat_testing_xarray._test_dates[''][''] + self.stime = pysat.instruments.pysat_ndtesting._test_dates[''][''] return def teardown_method(self): @@ -701,7 +701,7 @@ class TestGeneralOrbitsLongXarray(TestGeneralOrbitsMLT): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'longitude', 'kind': 'longitude'}, @@ -745,13 +745,13 @@ class TestGeneralOrbitsOrbitNumberXarray(TestGeneralOrbitsMLT): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'orbit_num', 'kind': 'orbit'}, update_files=True, use_header=True) - self.stime = pysat.instruments.pysat_testing_xarray._test_dates[''][''] + self.stime = pysat.instruments.pysat_ndtesting._test_dates[''][''] return def teardown_method(self): @@ -789,13 +789,13 @@ class TestGeneralOrbitsLatitudeXarray(TestGeneralOrbitsMLT): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'latitude', 'kind': 'polar'}, update_files=True, use_header=True) - self.stime = pysat.instruments.pysat_testing_xarray._test_dates[''][''] + self.stime = pysat.instruments.pysat_ndtesting._test_dates[''][''] return def teardown_method(self): @@ -874,7 +874,7 @@ class TestOrbitsGappyDataXarray(TestOrbitsGappyData): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'mlt'}, update_files=True, @@ -946,7 +946,7 @@ class TestOrbitsGappyData2Xarray(TestOrbitsGappyData2): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'mlt'}, use_header=True) @@ -991,7 +991,7 @@ class TestOrbitsGappyLongDataXarray(TestOrbitsGappyData): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'longitude', 'kind': 'longitude'}, @@ -1037,7 +1037,7 @@ class TestOrbitsGappyOrbitNumDataXarray(TestOrbitsGappyData): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'orbit_num', 'kind': 'orbit'}, @@ -1084,7 +1084,7 @@ class TestOrbitsGappyOrbitLatDataXarray(TestOrbitsGappyData): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'latitude', 'kind': 'polar'}, diff --git a/pysat/tests/test_utils_coords.py b/pysat/tests/test_utils_coords.py index 1ce7fea09..8edd4b113 100644 --- a/pysat/tests/test_utils_coords.py +++ b/pysat/tests/test_utils_coords.py @@ -63,8 +63,7 @@ def teardown_method(self): del self.py_inst, self.inst_time return - @pytest.mark.parametrize("name", ["testing", "testing_xarray", - "ndtesting", "testmodel"]) + @pytest.mark.parametrize("name", ["testing", "ndtesting", "testmodel"]) def test_update_longitude(self, name): """Test `update_longitude` successful run.""" @@ -119,7 +118,7 @@ def teardown_method(self): del self.py_inst, self.inst_time return - @pytest.mark.parametrize("name", ["testing", "testing_xarray"]) + @pytest.mark.parametrize("name", ["testing", "ndtesting"]) def test_calc_solar_local_time(self, name): """Test SLT calculation with longitudes from 0-360 deg for 0 UTH.""" @@ -143,7 +142,7 @@ def test_calc_solar_local_time(self, name): assert np.min(np.abs(cos_diff)) > 1.0 - 1.0e-6 return - @pytest.mark.parametrize("name", ["testing", "testing_xarray"]) + @pytest.mark.parametrize("name", ["testing", "ndtesting"]) def test_calc_solar_local_time_inconsistent_keywords(self, name, caplog): """Test that ref_date only works when apply_modulus=False.""" @@ -274,7 +273,7 @@ def test_single_lon_calc_solar_local_time(self): """Test calc_solar_local_time with a single longitude value.""" # Instantiate instrument and load data - self.py_inst = pysat.Instrument(platform='pysat', name="testing_xarray", + self.py_inst = pysat.Instrument(platform='pysat', name="ndtesting", use_header=True) self.py_inst.load(date=self.inst_time) lon_name = 'lon2' diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index cb3dd7b87..f41cf2e79 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -1247,24 +1247,7 @@ def test_missing_metadata(self): class TestNetCDF4IntegrationXarray(TestNetCDF4Integration): - """Integration tests for the netCDF4 I/O utils using xarray data.""" - - def setup_method(self): - """Create a testing environment.""" - - # Create an instrument object that has a meta with some - # variables allowed to be nan within metadata when exporting. - self.testInst = pysat.Instrument('pysat', 'testing_xarray', - num_samples=5, use_header=True) - self.testInst.load(date=self.testInst.inst_module._test_dates[''][''], - use_header=True) - self.pformat = self.testInst.pandas_format - - return - - -class TestNetCDF4Integration2DXarray(TestNetCDF4Integration): - """Integration tests for the netCDF4 I/O utils using 2dxarray Instrument.""" + """Integration tests for the netCDF4 I/O utils using xarray Instrument.""" def setup_method(self): """Create a testing environment.""" @@ -1305,7 +1288,7 @@ def setup_method(self): # Create an instrument object that has a meta with some # variables allowed to be nan within metadata when exporting. - self.testInst = pysat.Instrument('pysat', 'testing_xarray', + self.testInst = pysat.Instrument('pysat', 'ndtesting', num_samples=5, use_header=True) self.testInst.load(date=self.testInst.inst_module._test_dates[''][''], use_header=True) @@ -1706,36 +1689,12 @@ def test_remove_netcdf4_standards(self, caplog): class TestMetaTranslationXarray(TestMetaTranslation): """Test meta translation when writing/loading files xarray Instrument.""" - def setup_method(self): - """Create test environment.""" - - self.test_inst = pysat.Instrument('pysat', 'testing_xarray', - num_samples=5, use_header=True) - self.test_date = pysat.instruments.pysat_testing_xarray._test_dates - self.test_date = self.test_date[''][''] - self.test_inst.load(date=self.test_date) - self.meta_dict = self.test_inst.meta.to_dict() - self.out = None - - return - - def teardown_method(self): - """Cleanup test environment.""" - - del self.test_inst, self.test_date, self.out, self.meta_dict - - return - - -class TestMetaTranslation2DXarray(TestMetaTranslation): - """Test meta translation when writing/loading files xarray2d Instrument.""" - def setup_method(self): """Create test environment.""" self.test_inst = pysat.Instrument('pysat', 'ndtesting', num_samples=5, use_header=True) - self.test_date = pysat.instruments.pysat_testing_xarray._test_dates + self.test_date = pysat.instruments.pysat_ndtesting._test_dates self.test_date = self.test_date[''][''] self.test_inst.load(date=self.test_date) self.meta_dict = self.test_inst.meta.to_dict() From b021068fc715bb39301994ef92b6c5495406e84e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 5 Oct 2023 14:09:36 -0400 Subject: [PATCH 146/365] ENH: add orbit number to ndtesting --- pysat/instruments/pysat_ndtesting.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index f10be34a4..b12a09f9e 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -5,6 +5,7 @@ import functools import numpy as np +import pandas as pds import xarray as xr import pysat @@ -164,6 +165,13 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, altitude = alt0 * np.ones(data['latitude'].shape) data['altitude'] = ((epoch_name), altitude) + # Fake orbit number + fake_delta = dates[0] - (_test_dates[''][''] - pds.DateOffset(years=1)) + data['orbit_num'] = ((epoch_name), + mm_test.generate_fake_data(fake_delta.total_seconds(), + uts, period=iperiod['lt'], + cyclic=False)) + # Create some fake data to support testing of averaging routines mlt_int = data['mlt'].astype(int).data long_int = (data['longitude'] / 15.).astype(int).data From 4b05449ab02eff8139687beb1e86ee4e24191304 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 5 Oct 2023 14:10:07 -0400 Subject: [PATCH 147/365] DOC: remove deprecated instruments from docs --- docs/api.rst | 18 ------------------ docs/dependency.rst | 5 ----- docs/instruments/testing_instruments.rst | 14 -------------- docs/quickstart.rst | 2 +- 4 files changed, 1 insertion(+), 38 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index d14c971a4..89c4a4ddb 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -232,24 +232,6 @@ pysat_testing :members: -.. _api-pysat-testing_xarray: - -pysat_testing_xarray -^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: pysat.instruments.pysat_testing_xarray - :members: - - -.. _api-pysat-testing2d_xarray: - -pysat_testing2d_xarray -^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: pysat.instruments.pysat_testing2d_xarray - :members: - - .. _api-pysat-testmodel: pysat_testmodel diff --git a/docs/dependency.rst b/docs/dependency.rst index fa31acf32..4b0251cb3 100644 --- a/docs/dependency.rst +++ b/docs/dependency.rst @@ -267,11 +267,6 @@ pysat_testing object with 1D data as a function of latitude, longitude, and altitude in a pandas format. Most similar to in situ data. -pysat_testing_xarray -^^^^^^^^^^^^^^^^^^^^ -:ref:`api-pysat-testing_xarray` returns a satellite-like object with 1D data as -a function of latitude, longitude, and altitude in a xarray format. - pysat_ndtesting ^^^^^^^^^^^^^^^^^^^^^^ :ref:`api-pysat-ndtesting` is a satellite-like object that returns all diff --git a/docs/instruments/testing_instruments.rst b/docs/instruments/testing_instruments.rst index 2361a0765..f92558e33 100644 --- a/docs/instruments/testing_instruments.rst +++ b/docs/instruments/testing_instruments.rst @@ -14,13 +14,6 @@ An instrument with satellite-like data as a function of latitude, longitude, and altitude in a pandas format. See :ref:`api-pysat-testing` for more details. -pysat_testing_xarray -^^^^^^^^^^^^^^^^^^^^ -An instrument with satellite-like data as a function of latitude, longitude, -and altitude in a xarray format. See :ref:`api-pysat-testing_xarray` for more -details. - - pysat_ndtesting ^^^^^^^^^^^^^^^ An instrument with satellite-like data like :py:mod:`pysat_testing` that @@ -28,13 +21,6 @@ also has an imager-like 3D data variable. See :ref:`api-pysat-ndtesting` for more details. -pysat_testing2d_xarray -^^^^^^^^^^^^^^^^^^^^^^ -An instrument with satellite-like data like :py:mod:`pysat_testing_xarray` that -also has an imager-like 3D data variable. See :ref:`api-pysat-testing2d_xarray` -for more details. - - pysat_testmodel ^^^^^^^^^^^^^^^ An instrument with model-like data that returns a 4D object as a function of diff --git a/docs/quickstart.rst b/docs/quickstart.rst index eecb860a5..f1796aaea 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -67,7 +67,7 @@ installations. print(inst.data) # Testing out the xarray installation - inst = pysat.Instrument('pysat', 'testing_xarray') + inst = pysat.Instrument('pysat', 'ndtesting') inst.load(2009, 1) print(inst.data) From 7b5396e48ca8dbb56d9086b41b04c646061bbf32 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 5 Oct 2023 14:12:30 -0400 Subject: [PATCH 148/365] DOC: update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73f76d005..db8d3fe9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ This project adheres to [Semantic Versioning](https://semver.org/). memory usage in the `load` method. * Added a hidden method the Instrument class `_get_epoch_name_from_data` to reduce code duplication. + * Added options to customize `pysat_ndtesting` instrument with sample rate, + shift in time. + * Added orbit number to `pysat_ndtesting`. * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. @@ -25,6 +28,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Add pandas cap * Remove deprecated `pysat_testing2d` instrument * Remove deprecated meta children info + * Remove deprecated `pysat_testing_xarray` instrument + * Remove deprecated `pysat_testing2d_xarray` instrument [3.1.0] - 2023-05-31 -------------------- From 74e7101c8fa135d3e4e319c72f16903dcf8bb6e4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 5 Oct 2023 16:10:51 -0400 Subject: [PATCH 149/365] Update pysat/tests/test_meta.py Co-authored-by: Angeline Burrell --- pysat/tests/test_meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 5e3b1699a..4f7bf9777 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -1375,7 +1375,7 @@ def test_meta_mutable_properties(self): setattr(self.meta, 'data', pds.DataFrame()) # Test that data is empty - assert self.meta.data.empty, "`meta.data` not updated correctly" + assert self.meta.empty, "`meta.data` not updated correctly" except AttributeError: raise AssertionError("Couldn't update mutable property 'data'") return From 65cc8daf7a4fbf3569e85d385e3f12b5e80e88e6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 12:25:22 -0400 Subject: [PATCH 150/365] STY: remove unaccessible error --- pysat/utils/io.py | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/pysat/utils/io.py b/pysat/utils/io.py index 2b972ca4a..63a4692fb 100644 --- a/pysat/utils/io.py +++ b/pysat/utils/io.py @@ -1588,34 +1588,24 @@ def inst_to_netcdf(inst, fname, base_instrument=None, epoch_name=None, # Not datetime data, just store as is. cdfkey[:] = data.values.astype(coltype) else: - # It is a Series of objects. First, figure out what the - # individual object types are. Then, act as needed. - - # Use info in coltype to get real datatype of object - if coltype == str: - if '_FillValue' in export_meta[lower_key].keys(): - str_fill = export_meta[lower_key]['_FillValue'] - del export_meta[lower_key]['_FillValue'] - else: - str_fill = '' + if '_FillValue' in export_meta[lower_key].keys(): + str_fill = export_meta[lower_key]['_FillValue'] + del export_meta[lower_key]['_FillValue'] + else: + str_fill = '' - cdfkey = out_data.createVariable(case_key, coltype, - dimensions=epoch_name, - complevel=complevel, - shuffle=shuffle, - fill_value=str_fill) + cdfkey = out_data.createVariable(case_key, coltype, + dimensions=epoch_name, + complevel=complevel, + shuffle=shuffle, + fill_value=str_fill) - # Set metadata - cdfkey.setncatts(export_meta[lower_key]) + # Set metadata + cdfkey.setncatts(export_meta[lower_key]) - # Time to actually write the data now - cdfkey[:] = data.values + # Time to actually write the data now + cdfkey[:] = data.values - else: - raise ValueError(' '.join(('Recursive data detected.', - 'Please ensure that the', - 'DataFrame does not contain', - 'other pandas objects.'))) else: # Attach the metadata to a separate xarray.Dataset object, ensuring # the Instrument data object is unchanged. From b25e836d21ae5b447b0388564bf552a7fd9bc564 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 12:45:39 -0400 Subject: [PATCH 151/365] BUG: nest if --- pysat/tests/test_utils_io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 25c769e5b..cf6d03695 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -992,9 +992,9 @@ def test_filter_netcdf4_metadata(self, remove, check_type, export_nan, "{:} should have been exported".format(repr(mkey)) else: if all([mkey in export_nan, - not np.issubdtype(data_type, str), - np.isnan(mdict[mkey])]): - assert np.isnan(fdict[mkey]) + not np.issubdtype(data_type, str)]): + if np.isnan(mdict[mkey]): + assert np.isnan(fdict[mkey]) else: if mkey in check_type and fdict[mkey] != mdict[mkey]: assert fdict[mkey] == data_type(mdict[mkey]), \ From 6395f80839e14af76317cc5377d2e7852c148c65 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 12:48:46 -0400 Subject: [PATCH 152/365] MAINT: remove hacking cap --- pyproject.toml | 2 +- test_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ec0b533b..a5a48123d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ test = [ "coveralls < 3.3", "flake8", "flake8-docstrings", - "hacking >= 1.0, <6.0", + "hacking >= 1.0, "pysatSpaceWeather", "pytest", "pytest-cov", diff --git a/test_requirements.txt b/test_requirements.txt index dc679dcce..8a33141b8 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ coveralls<3.3 flake8 flake8-docstrings -hacking>=1.0,<6.0 +hacking>=1.0 ipython m2r2 numpydoc From 2d6e94304b76404e729851cd758033969ab59952 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 12:54:20 -0400 Subject: [PATCH 153/365] BUG: " --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a5a48123d..eb0dbe732 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ test = [ "coveralls < 3.3", "flake8", "flake8-docstrings", - "hacking >= 1.0, + "hacking >= 1.0", "pysatSpaceWeather", "pytest", "pytest-cov", From a55d4f730c8c2947993147b66dcd5d1e7c96ba76 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 12:56:08 -0400 Subject: [PATCH 154/365] DOC: update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b547a3889..ed07e4687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Implement pyproject to manage metadata * Remove Sphinx cap * Add pandas cap + * Update usage of whitespace and if statements (E275) + * Remove hacking cap [3.1.0] - 2023-05-31 -------------------- From e4eb7ea782a9b2dde453d5c98fe717b760ca25db Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 13:33:38 -0400 Subject: [PATCH 155/365] MAINT: remove instrument test class --- pysat/tests/instrument_test_class.py | 78 ---------------------------- 1 file changed, 78 deletions(-) delete mode 100644 pysat/tests/instrument_test_class.py diff --git a/pysat/tests/instrument_test_class.py b/pysat/tests/instrument_test_class.py deleted file mode 100644 index c35e0ff3a..000000000 --- a/pysat/tests/instrument_test_class.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Standardized class and functions to test instruments for pysat libraries. - -Note ----- -Not directly called by pytest, but imported as part of test_instruments.py. -Can be imported directly for external instrument libraries of pysat instruments. - -""" - -import warnings - -import pysat.tests.classes.cls_instrument_library as cls_inst_lib - - -def initialize_test_inst_and_date(inst_dict): - """Initialize the instrument object to test and date. - - .. deprecated:: 3.0.2 - `initialize_test_inst_and_date` will be removed in pysat 3.2.0, it is - moved to `pysat.tests.classes.cls_instrument_library`. - - Parameters - ---------- - inst_dict : dict - Dictionary containing specific instrument info, generated by - generate_instrument_list - - Returns - ------- - test_inst : pysat.Instrument - instrument object to be tested - date : dt.datetime - test date from module - - """ - - warnings.warn(" ".join(["`initialize_test_inst_and_date` has been moved to", - "`pysat.tests.classes.cls_instrument_library`.", - "The link here will be removed in 3.2.0+."]), - DeprecationWarning, stacklevel=2) - return cls_inst_lib.initialize_test_inst_and_date(inst_dict) - - -class InstTestClass(cls_inst_lib.InstLibTests): - """Provide standardized tests for pysat instrument libraries. - - .. deprecated:: 3.0.2 - `InstTestClass` will be removed in pysat 3.2.0, it is replaced by - `pysat.tests.classes.cls_instrument_library.InstLibTests`. - - Note - ---- - Uses class level setup and teardown so that all tests use the same - temporary directory. We do not want to geneate a new tempdir for each test, - as the load tests need to be the same as the download tests. - - Not directly run by pytest, but inherited through test_instruments.py - - Users will need to run `apply_marks_to_tests` before setting up the test - class. - - """ - - def __init_subclass__(self): - """Throw a warning if used as a subclass.""" - - warnings.warn(" ".join( - ["`InstTestClass` has been deprecated and will be removed in", - "3.2.0+. Please update code to use the `InstLibTests` class", - "under `pysat.tests.classes.cls_instrument_library`."]), - DeprecationWarning, stacklevel=2) - warnings.warn(" ".join( - ["`test_load` now uses `@pytest.mark.load_options` in place", - "of `@pytest.mark.download`. The old behavior will be removed in", - "3.2.0+. Please update code or use the new" - "`InstLibTests.initialize_test_package` function", - "under `pysat.tests.classes.cls_instrument_library`."]), - DeprecationWarning, stacklevel=2) From 162fa2450b0e977813d8818a2d477dc49befda79 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 13:34:03 -0400 Subject: [PATCH 156/365] MAINT: remove dep tests --- pysat/tests/test_instruments.py | 66 --------------------------------- 1 file changed, 66 deletions(-) diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index 470a83278..e118d0a52 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -16,7 +16,6 @@ import pysat import pysat.tests.classes.cls_instrument_library as cls_inst_lib from pysat.tests.classes.cls_instrument_library import InstLibTests -import pysat.tests.instrument_test_class as itc from pysat.utils import testing # Optional code to pass through user and password info to test instruments @@ -209,68 +208,3 @@ def test_clean_with_warnings(self, clean_level, change, warn_type, self.test_clean_warn(clean_level, inst_dict, caplog) return - - -class TestDeprecation(object): - """Unit test for deprecation warnings.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - - warnings.simplefilter("always", DeprecationWarning) - return - - def teardown_method(self): - """Clean up the unit test environment after each method.""" - - return - - def test_subclass_inst_test_class(self): - """Check that subclass of old instrument library tests is deprecated.""" - - with warnings.catch_warnings(record=True) as war: - - class OldClass(itc.InstTestClass): - """Dummy subclass.""" - - pass - - self.warn_msgs = ["`InstTestClass` has been deprecated", - "`test_load` now uses `@pytest.mark.load_options`"] - self.warn_msgs = np.array(self.warn_msgs) - - # Ensure the minimum number of warnings were raised - assert len(war) >= len(self.warn_msgs) - - # Test the warning messages, ensuring each attribute is present - testing.eval_warnings(war, self.warn_msgs) - return - - def test_old_initialize_inst_and_date(self): - """Check that subclass of old instrument library tests is deprecated.""" - - with warnings.catch_warnings(record=True) as war: - try: - itc.initialize_test_inst_and_date({}) - except KeyError: - # empty dict produces KeyError - pass - - self.warn_msgs = ["`initialize_test_inst_and_date` has been moved to"] - self.warn_msgs = np.array(self.warn_msgs) - - # Ensure the minimum number of warnings were raised - assert len(war) >= len(self.warn_msgs) - - # Test the warning messages, ensuring each attribute is present - testing.eval_warnings(war, self.warn_msgs) - return - - def test_old_pytest_mark_presence(self): - """Test that pytest mark is backwards compatible.""" - - n_args = len(InstLibTests.test_load.pytestmark) - mark_names = [InstLibTests.test_load.pytestmark[j].name - for j in range(0, n_args)] - - assert "download" in mark_names From 3adadc7061a8a5debdd26980b6437766ce298b1e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 13:34:13 -0400 Subject: [PATCH 157/365] MAINT: update usage --- pysat/utils/registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py index 97193ccbc..b0e24001a 100644 --- a/pysat/utils/registry.py +++ b/pysat/utils/registry.py @@ -52,7 +52,7 @@ import logging import pysat -import pysat.tests.instrument_test_class as itc +import pysat.tests.classes.cls_instrument_library as itc def load_saved_modules(): @@ -147,7 +147,7 @@ def register(module_names, overwrite=False): raise # Second, check that module is itself pysat compatible - validate = itc.InstTestClass() + validate = itc.InstLibTests() # Work with test code, create dummy structure to make things work class Foo(object): From c272e4a88dbf5bc4d84911aaa128752fd26e29a2 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 13:34:23 -0400 Subject: [PATCH 158/365] DOC: update comments --- pysat/tests/test_instrument_listgen.py | 2 +- pysat/utils/_core.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pysat/tests/test_instrument_listgen.py b/pysat/tests/test_instrument_listgen.py index 8414f9fbb..7932651b0 100644 --- a/pysat/tests/test_instrument_listgen.py +++ b/pysat/tests/test_instrument_listgen.py @@ -45,7 +45,7 @@ def test_for_missing_test_date(self): # If an instrument does not have the _test_dates attribute, it should # still be added to the list for other checks to be run. - # This will be caught later by InstTestClass.test_instrument_test_dates. + # This will be caught later by InstLibTests.test_instrument_test_dates. assert not hasattr(self.test_library.pysat_testing, '_test_dates') inst_list = generate_instrument_list(self.test_library) assert 'pysat_testing' in inst_list['names'] diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index 1241d34ef..cd32629f2 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -443,7 +443,7 @@ def generate_instrument_list(inst_loc, user_info=None): # If this can't be imported, we can't pull out the info for the # download / no_download tests. Leaving in basic tests for all # instruments, but skipping the rest. The import error will be - # caught as part of the pytest.mark.all_inst tests in InstTestClass + # caught as part of the pytest.mark.all_inst tests in InstLibTests pass else: # try to grab basic information about the module so we @@ -453,7 +453,7 @@ def generate_instrument_list(inst_loc, user_info=None): except AttributeError: # If a module does not have a test date, add it anyway for # other tests. This will be caught later by - # InstTestClass.test_instrument_test_dates + # InstLibTests.test_instrument_test_dates info = {} info[''] = {'': dt.datetime(2009, 1, 1)} module._test_dates = info From ce56952eacffc4ea9fd3c50ee84a499969cd6972 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 13:34:50 -0400 Subject: [PATCH 159/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index db8d3fe9c..93f3160f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Remove deprecated meta children info * Remove deprecated `pysat_testing_xarray` instrument * Remove deprecated `pysat_testing2d_xarray` instrument + * Remove deprecated `instrument_test_class` [3.1.0] - 2023-05-31 -------------------- From e3a1fcc47bf02f38b9930e4348371cf1451f03fd Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 14:18:41 -0400 Subject: [PATCH 160/365] MAINT: remove #1094 --- pysat/instruments/methods/testing.py | 10 ----- pysat/instruments/pysat_ndtesting.py | 17 ++------ pysat/instruments/pysat_testing.py | 14 +------ pysat/tests/test_instrument_index.py | 62 ---------------------------- 4 files changed, 5 insertions(+), 98 deletions(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 08705398d..3f02ff71a 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -770,13 +770,3 @@ def non_unique_index(index): new_index = pds.to_datetime(new_index) return new_index - - -def _warn_malformed_kwarg(): - """Warn user that kwarg has been deprecated.""" - - dstr = ' '.join(['The kwarg malformed_index has been deprecated and', - 'will be removed in pysat 3.2.0+. Please use', - 'non_monotonic_index or non_unique_index to specify', - 'desired behaviour.']) - warnings.warn(dstr, DeprecationWarning, stacklevel=2) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index b12a09f9e..d455eebcb 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -36,9 +36,9 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, sim_multi_file_left=False, non_monotonic_index=False, - non_unique_index=False, malformed_index=False, start_time=None, - num_samples=864, sample_rate='100S', test_load_kwarg=None, - max_latitude=90.0, num_extra_time_coords=0): + non_unique_index=False, start_time=None, num_samples=864, + sample_rate='100S', test_load_kwarg=None, max_latitude=90.0, + num_extra_time_coords=0): """Load the test files. Parameters @@ -61,10 +61,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, If True, time index will be non-monotonic (default=False) non_unique_index : bool If True, time index will be non-unique (default=False) - malformed_index : bool - If True, the time index will be non-unique and non-monotonic. Deprecated - and scheduled for removal in pysat 3.2.0. - (default=False) start_time : dt.timedelta or NoneType Offset time of start time since midnight UT. If None, instrument data will begin at midnight. (default=None) @@ -110,13 +106,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, else: root_date = dt.datetime(2009, 1, 1) - # TODO(#1094): Remove in pysat 3.2.0 - if malformed_index: - # Warn that kwarg is deprecated and set new kwargs. - mm_test._warn_malformed_kwarg() - non_monotonic_index = True - non_unique_index = True - if non_monotonic_index: index = mm_test.non_monotonic_index(index) if non_unique_index: diff --git a/pysat/instruments/pysat_testing.py b/pysat/instruments/pysat_testing.py index c96dccdfe..90592c754 100644 --- a/pysat/instruments/pysat_testing.py +++ b/pysat/instruments/pysat_testing.py @@ -40,8 +40,8 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, sim_multi_file_left=False, root_date=None, non_monotonic_index=False, - non_unique_index=False, malformed_index=False, start_time=None, - num_samples=86400, test_load_kwarg=None, max_latitude=90.): + non_unique_index=False,start_time=None, num_samples=86400, + test_load_kwarg=None, max_latitude=90.): """Load the test files. Parameters @@ -67,9 +67,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, If True, time index will be non-monotonic (default=False) non_unique_index : bool If True, time index will be non-unique (default=False) - malformed_index : bool - If True, the time index will be non-unique and non-monotonic. Deprecated - and scheduled for removal in pysat 3.2.0. (default=False) start_time : dt.timedelta or NoneType Offset time of start time since midnight UT. If None, instrument data will begin at midnight. (default=None) @@ -164,13 +161,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, data['int32_dummy'] = np.ones(len(data), dtype=np.int32) data['int64_dummy'] = np.ones(len(data), dtype=np.int64) - # TODO(#1094): Remove in pysat 3.2.0 - if malformed_index: - # Warn that kwarg is deprecated and set new kwargs. - mm_test._warn_malformed_kwarg() - non_monotonic_index = True - non_unique_index = True - # Activate if non-monotonic index is needed. if np.any([non_monotonic_index, (tag == 'non_strict')]): index = mm_test.non_monotonic_index(index) diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py index 451598019..5c33c65b2 100644 --- a/pysat/tests/test_instrument_index.py +++ b/pysat/tests/test_instrument_index.py @@ -76,65 +76,3 @@ def teardown_method(self): del self.ref_time, self.name return - - -class TestDeprecation(object): - """Unit test for deprecation warnings for index.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - - warnings.simplefilter("always", DeprecationWarning) - self.ref_time = pysat.instruments.pysat_testing._test_dates[''][''] - self.warn_msgs = [] - self.war = "" - return - - def teardown_method(self): - """Clean up the unit test environment after each method.""" - - del self.ref_time, self.warn_msgs, self.war - return - - def eval_warnings(self): - """Evaluate the number and message of the raised warnings.""" - - # Ensure the minimum number of warnings were raised. - assert len(self.war) >= len(self.warn_msgs) - - # Test the warning messages, ensuring each attribute is present. - testing.eval_warnings(self.war, self.warn_msgs) - return - - # TODO(#1094): Remove in pysat 3.2.0, potentially with class - @pytest.mark.parametrize('name', ['testing', 'ndtesting']) - def test_kwarg_malformed_index(self, name): - """Test deprecation of `malformed_index` kwarg. - - Parameters - ---------- - name : str - name of instrument that uses the deprecated `malformed_index` kwarg. - - """ - - test_inst = pysat.Instrument(platform='pysat', - name=name, - strict_time_flag=False, - use_header=True, - malformed_index=True) - - # Catch the warnings - with warnings.catch_warnings(record=True) as self.war: - test_inst.load(date=self.ref_time) - - self.warn_msgs = np.array([" ".join(["The kwarg malformed_index has", - "been deprecated"])]) - - # Evaluate the warning output - self.eval_warnings() - - # Check that resulting index is both non-monotonic and non-unique - assert not test_inst.index.is_monotonic_increasing - assert not test_inst.index.is_unique - return From 799f613f8b6a66c8643c0cc8335a6339f0339ca5 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 14:19:49 -0400 Subject: [PATCH 161/365] MAINT: remove #947 --- pysat/instruments/methods/general.py | 37 ---------------------------- pysat/tests/test_methods_general.py | 35 -------------------------- 2 files changed, 72 deletions(-) diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py index 400480ddd..a9afcdf8e 100644 --- a/pysat/instruments/methods/general.py +++ b/pysat/instruments/methods/general.py @@ -148,43 +148,6 @@ def list_files(tag='', inst_id='', data_path='', format_str=None, return out -def convert_timestamp_to_datetime(inst, sec_mult=1.0, epoch_name='time'): - """Use datetime instead of timestamp for Epoch. - - .. deprecated:: 3.0.2 - This routine has been deprecated with the addition of the kwargs - `epoch_unit` and `epoch_origin` to `pysat.utils.io.load_netcdf4`. - This routing will be removed in 3.2.0. - - Parameters - ---------- - inst : pysat.Instrument - associated pysat.Instrument object - sec_mult : float - Multiplier needed to convert epoch time to seconds (default=1.0) - epoch_name : str - variable name for instrument index (default='Epoch') - - Note - ---- - If the variable represented by epoch_name is not a float64, data is passed - through unchanged. - - """ - - warnings.warn(" ".join(["New kwargs added to `pysat.utils.io.load_netCDF4`", - "for generalized handling, deprecated", - "function will be removed in pysat 3.2.0+"]), - DeprecationWarning, stacklevel=2) - - if inst.data[epoch_name].dtype == 'float64': - inst.data[epoch_name] = pds.to_datetime( - [dt.datetime.utcfromtimestamp(int(np.floor(epoch_time * sec_mult))) - for epoch_time in inst.data[epoch_name]]) - - return - - def remove_leading_text(inst, target=None): """Remove leading text on variable names. diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py index d08108f72..a9724553e 100644 --- a/pysat/tests/test_methods_general.py +++ b/pysat/tests/test_methods_general.py @@ -297,38 +297,3 @@ def test_load_file_with_kwargs(self): self.eval_data_cols() assert len(self.data.columns) == len(self.data_cols) return - - -class TestDeprecation(object): - """Unit tests for deprecated methods.""" - - def setup_method(self): - """Set up the unit test environment for each method.""" - - warnings.simplefilter("always", DeprecationWarning) - return - - def teardown_method(self): - """Clean up the unit test environment after each method.""" - - return - - def test_convert_timestamp_to_datetime(self): - """Test that convert_timestamp_to_datetime is deprecated.""" - - warn_msgs = [" ".join( - ["New kwargs added to `pysat.utils.io.load_netCDF4`", - "for generalized handling, deprecated", - "function will be removed in pysat 3.2.0+"])] - - test = pysat.Instrument('pysat', 'testing', use_header=True) - test.load(2009, 1) - with warnings.catch_warnings(record=True) as war: - gen.convert_timestamp_to_datetime(test, epoch_name='uts') - - # Ensure the minimum number of warnings were raised - assert len(war) >= len(warn_msgs) - - # Test the warning messages, ensuring each attribute is present - pysat.utils.testing.eval_warnings(war, warn_msgs) - return From 365829a1e818f7ab2cbff6a45090a678a90563f3 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 14:20:57 -0400 Subject: [PATCH 162/365] MAINT: remove #807 --- pysat/_instrument.py | 18 ------------------ pysat/tests/test_instrument.py | 23 ----------------------- 2 files changed, 41 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 9d5c5b3b7..1e90ab233 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1518,24 +1518,6 @@ def _assign_attrs(self, by_name=False): else: missing.append(iattr) - # Check and see if this instrument has deprecated _test_download_travis - # TODO(#807): Remove this check once _test_download_travis is removed. - if hasattr(self.inst_module, '_test_download_travis'): - local_attr = getattr(self.inst_module, '_test_download_travis') - - # Test to see that this attribute is set for the desired - # `inst_id` and `tag`. - if self.inst_id in local_attr.keys(): - if self.tag in local_attr[self.inst_id].keys(): - # Update the test attribute value - setattr(self, '_test_download_ci', - local_attr[self.inst_id][self.tag]) - warnings.warn(" ".join(["`_test_download_travis` has been", - "deprecated and will be replaced", - "by `_test_download_ci` in", - "3.2.0+"]), - DeprecationWarning, stacklevel=2) - if len(missing) > 0: pysat.logger.debug(' '.join(['These Instrument test attributes', 'kept their default values:', diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index aeb1b8cdb..2ef0adda8 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -649,29 +649,6 @@ def test_download_freq_kwarg(self): self.eval_warnings() return - def test_download_travis_attr(self): - """Test deprecation of instrument attribute `_test_download_travis`.""" - - inst_module = pysat.instruments.pysat_testing - # Add deprecated attribute. - inst_module._test_download_travis = {'': {'': False}} - - self.warn_msgs = np.array([" ".join(["`_test_download_travis` has been", - "deprecated and will be replaced", - "by `_test_download_ci` in", - "3.2.0+"])]) - - # Catch the warnings. - with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(inst_module=inst_module, use_header=True) - - # Ensure attributes set properly. - assert tinst._test_download_ci is False - - # Evaluate the warning output - self.eval_warnings() - return - def test_filter_netcdf4_metadata(self): """Test deprecation warning generated by `_filter_netcdf4_metadata`.""" From 54b886577859b2bba9d2f94010dca57083a75d11 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 14:55:13 -0400 Subject: [PATCH 163/365] MAINT: remove #788 --- pysat/_instrument.py | 18 +++--------------- pysat/tests/test_instrument.py | 17 ----------------- 2 files changed, 3 insertions(+), 32 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 1e90ab233..ae720567c 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -3597,10 +3597,6 @@ def download(self, start=None, stop=None, date_array=None, **kwargs): """Download data for given Instrument object from start to stop. - .. deprecated:: 3.2.0 - `freq`, which sets the step size for downloads, will be removed in - the 3.2.0+ release. - Parameters ---------- start : pandas.datetime or NoneType @@ -3635,17 +3631,9 @@ def download(self, start=None, stop=None, date_array=None, pandas.DatetimeIndex """ - # Test for deprecated kwargs - if 'freq' in kwargs.keys(): - warnings.warn("".join(["`pysat.Instrument.download` kwarg `freq` ", - "has been deprecated and will be removed ", - "in pysat 3.2.0+. Use `date_array` for ", - "non-daily frequencies instead."]), - DeprecationWarning, stacklevel=2) - freq = kwargs['freq'] - del kwargs['freq'] - else: - freq = 'D' + + # Set frequency to daily. + freq = 'D' # Make sure directories are there, otherwise create them try: diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 2ef0adda8..95fbf75b2 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -632,23 +632,6 @@ def test_generic_meta_translator(self): self.eval_warnings() return - def test_download_freq_kwarg(self): - """Test deprecation of download kwarg `freq`.""" - - # Catch the warnings - with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(use_header=True, **self.in_kwargs) - tinst.download(start=self.ref_time, freq='D') - - self.warn_msgs = np.array(["".join(["`pysat.Instrument.download` kwarg", - " `freq` has been deprecated and ", - "will be removed in pysat ", - "3.2.0+"])]) - - # Evaluate the warning output - self.eval_warnings() - return - def test_filter_netcdf4_metadata(self): """Test deprecation warning generated by `_filter_netcdf4_metadata`.""" From a633a6a9b5978f74cea3d2d8b34fd203be200a9f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 14:57:37 -0400 Subject: [PATCH 164/365] DOC: update changelog --- CHANGELOG.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93f3160f1..15743291d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,11 +26,15 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Implement pyproject to manage metadata * Remove Sphinx cap * Add pandas cap - * Remove deprecated `pysat_testing2d` instrument - * Remove deprecated meta children info - * Remove deprecated `pysat_testing_xarray` instrument - * Remove deprecated `pysat_testing2d_xarray` instrument - * Remove deprecated `instrument_test_class` + * Removed deprecated `pysat_testing2d` instrument + * Removed deprecated meta children info + * Removed deprecated `pysat_testing_xarray` instrument + * Removed deprecated `pysat_testing2d_xarray` instrument + * Removed deprecated `instrument_test_class` + * Removed deprecated `malformed_index` kwarg in test instrumennts + * Removed deprecated `convert_timestamp_to_datetime` function + * Removed deprecated `_test_download_travis` flag + * Removed deprecated `freq` kwarg from `download` [3.1.0] - 2023-05-31 -------------------- From fc27c60caf46f84098c7699901c0c84358839d64 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 6 Oct 2023 17:49:35 -0400 Subject: [PATCH 165/365] STY: fix whitespace --- pysat/_meta.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index 7034d44c4..e51b06af4 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -395,9 +395,9 @@ def __setitem__(self, data_vars, input_dat): to_be_set, self.labels.label_type[iattr]): # If this is a disagreement between byte data # and an expected str, resolve it here - if(isinstance(to_be_set, bytes) - and str in pysat.utils.listify( - self.labels.label_type[iattr])): + if all([isinstance(to_be_set, bytes), + str in pysat.utils.listify( + self.labels.label_type[iattr])]): to_be_set = core_utils.stringify(to_be_set) else: # This type is incorrect, try casting it From 1608f4a9103a556b366d3b6f4960680490fce12e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 10 Oct 2023 10:59:04 -0400 Subject: [PATCH 166/365] BUG: fix logic --- pysat/_instrument.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 461a4f3f5..306f19c33 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -409,11 +409,17 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', # Check to make sure value is reasonable if self.file_format is not None: # Check if it is an iterable string - if any([(not isinstance(self.file_format, str)), - (self.file_format.find("{") < 0), - (self.file_format.find("}") < 0)]): + if isinstance(self.file_format, str): + if any([self.file_format.find("{") < 0, + self.file_format.find("}") < 0]): + raise ValueError(''.join(['file format set to default, ', + 'supplied string must be ', + 'iterable [{', + self.file_format, '}]'])) + else: raise ValueError(''.join(['file format set to default, ', - 'supplied string must be iterable ', + 'supplied format must be iterable ', + 'string', '[{:}]'.format(self.file_format)])) # Set up empty data and metadata. From 6fe60e46c1b7da5e7ab2897534f57d2299087dc6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 10 Oct 2023 11:15:52 -0400 Subject: [PATCH 167/365] Update pysat/tests/test_utils_io.py Co-authored-by: Angeline Burrell --- pysat/tests/test_utils_io.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index f41cf2e79..14b53dfdb 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -1694,8 +1694,7 @@ def setup_method(self): self.test_inst = pysat.Instrument('pysat', 'ndtesting', num_samples=5, use_header=True) - self.test_date = pysat.instruments.pysat_ndtesting._test_dates - self.test_date = self.test_date[''][''] + self.test_date = pysat.instruments.pysat_ndtesting._test_dates[''][''] self.test_inst.load(date=self.test_date) self.meta_dict = self.test_inst.meta.to_dict() self.out = None From 9268edec8293581837c134c63ef7614dfddadd4c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 10 Oct 2023 11:16:23 -0400 Subject: [PATCH 168/365] STY: update root_date --- pysat/instruments/pysat_ndtesting.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index b12a09f9e..ab354f974 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -35,7 +35,7 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, - sim_multi_file_left=False, non_monotonic_index=False, + sim_multi_file_left=False, root_date=None, non_monotonic_index=False, non_unique_index=False, malformed_index=False, start_time=None, num_samples=864, sample_rate='100S', test_load_kwarg=None, max_latitude=90.0, num_extra_time_coords=0): @@ -57,6 +57,9 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, sim_multi_file_left : bool Adjusts date range to be 12 hours in the past or twelve hours before `root_date`. (default=False) + root_date : NoneType + Optional central date, uses _test_dates if not specified. + (default=None) non_monotonic_index : bool If True, time index will be non-monotonic (default=False) non_unique_index : bool @@ -103,12 +106,15 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, uts, index, dates = mm_test.generate_times(fnames, num_samples, freq=sample_rate, start_time=start_time) + + # Specify the date tag locally and determine the desired date range + pds_offset = dt.timedelta(hours=12) if sim_multi_file_right: - root_date = dt.datetime(2009, 1, 1, 12) + root_date = root_date or _test_dates[''][''] + pds_offset elif sim_multi_file_left: - root_date = dt.datetime(2008, 12, 31, 12) + root_date = root_date or _test_dates[''][''] - pds_offset else: - root_date = dt.datetime(2009, 1, 1) + root_date = root_date or _test_dates[''][''] # TODO(#1094): Remove in pysat 3.2.0 if malformed_index: From 7ea9a3d0bb891280caa87ed956c0ab26b5f54eb1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 10 Oct 2023 14:06:51 -0400 Subject: [PATCH 169/365] STY: improve error message --- pysat/_instrument.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 306f19c33..1edc9a299 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -414,7 +414,8 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', self.file_format.find("}") < 0]): raise ValueError(''.join(['file format set to default, ', 'supplied string must be ', - 'iterable [{', + 'iterable string with key ', + 'formatting [{', self.file_format, '}]'])) else: raise ValueError(''.join(['file format set to default, ', From 2ca3e2fd42cf72b92496ee51f7be57ce29f7bfb1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 10 Oct 2023 16:54:09 -0400 Subject: [PATCH 170/365] BUG: fixed bad merge --- pysat/instruments/pysat_ndtesting.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 7c4de51d9..63c4dfd07 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -39,10 +39,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, non_unique_index=False, start_time=None, num_samples=864, sample_rate='100S', test_load_kwarg=None, max_latitude=90.0, num_extra_time_coords=0): - sim_multi_file_left=False, root_date=None, non_monotonic_index=False, - non_unique_index=False, malformed_index=False, start_time=None, - num_samples=864, sample_rate='100S', test_load_kwarg=None, - max_latitude=90.0, num_extra_time_coords=0): """Load the test files. Parameters From 7fb5ced53dde270ec3648532b5e964b5107f428f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 10 Oct 2023 16:57:05 -0400 Subject: [PATCH 171/365] STY: whitespace --- pysat/instruments/pysat_testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/instruments/pysat_testing.py b/pysat/instruments/pysat_testing.py index 90592c754..6bbc08b02 100644 --- a/pysat/instruments/pysat_testing.py +++ b/pysat/instruments/pysat_testing.py @@ -40,7 +40,7 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, sim_multi_file_left=False, root_date=None, non_monotonic_index=False, - non_unique_index=False,start_time=None, num_samples=86400, + non_unique_index=False, start_time=None, num_samples=86400, test_load_kwarg=None, max_latitude=90.): """Load the test files. From e4079f5ae7b83d85c9021a4c46775b9fa0c44e7f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 11 Oct 2023 15:40:08 -0400 Subject: [PATCH 172/365] STY: clarify error message --- pysat/_instrument.py | 8 +++----- pysat/tests/test_files.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 1edc9a299..b9153ce49 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -412,15 +412,13 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', if isinstance(self.file_format, str): if any([self.file_format.find("{") < 0, self.file_format.find("}") < 0]): - raise ValueError(''.join(['file format set to default, ', - 'supplied string must be ', + raise ValueError(''.join(['Supplied format string must be ', 'iterable string with key ', 'formatting [{', self.file_format, '}]'])) else: - raise ValueError(''.join(['file format set to default, ', - 'supplied format must be iterable ', - 'string', + raise ValueError(''.join(['Supplied format string must be ', + 'iterable string', '[{:}]'.format(self.file_format)])) # Set up empty data and metadata. diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py index c50c19dd1..cdc5039a0 100644 --- a/pysat/tests/test_files.py +++ b/pysat/tests/test_files.py @@ -1087,7 +1087,7 @@ def test_files_non_standard_file_format_template_fail(self, file_format): 'temporary_file_list': self.temporary_file_list} testing.eval_bad_input(pysat.Instrument, ValueError, - 'file format set to default', + 'Supplied format string', input_kwargs=in_kwargs) return From 75be1817970f31ebf83d76183f25227c2c17504d Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 12 Oct 2023 11:46:21 -0400 Subject: [PATCH 173/365] MAINT: dropped HO support Dropped higher order support in `Meta.drop`. Also adjusted method to reduce number of drop calls and allow partially good data to be dropped without raising and error. --- pysat/_meta.py | 52 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index 79b6f9ca2..668530550 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -807,31 +807,51 @@ def drop(self, names): Raises ------ KeyError - If any of the keys provided in `names` is not found in the - lower dimensional data, higher dimensional data, labels, or - header data + If all of the keys provided in `names` is not found in the + standard metadata, labels, or header metadata. If a subset is + missing, a logger warning is issued instead. """ # Ensure the input is list-like names = listify(names) - for name in names: - if name in self._ho_data: - # Drop the higher dimension data - self._ho_data.pop(name) + # Divide the names by category + data_names = [] + label_names = [] + header_names = [] + bad_names = [] + for name in names: if name in self.keys(): - # Drop the lower dimension data - self.data = self._data.drop(name, axis=0) + data_names.append(name) elif name in self.data.columns: - # This is a metadata label - self.data = self._data.drop(name, axis=1) - - # Also drop this from Labels - self.labels.drop(name) + label_names.append(name) elif name in self.header.global_attrs: - self.header.drop(name) + header_names.append(name) + else: + bad_names.append(name) + + # Drop the data + if len(data_names) > 0: + # Drop the lower dimension data + self.data = self._data.drop(data_names, axis=0) + + if len(label_names) > 0: + # This is a metadata label + self.data = self._data.drop(label_names, axis=1) + + # Also drop this from Labels + self.labels.drop(label_names) + + if len(header_names) > 0: + # There is header metadata to drop + self.header.drop(header_names) + + if len(bad_names) > 0: + estr = "{:} not found in Meta".format(repr(bad_names)) + if len(data_names) + len(label_names) + len(header_names) == 0: + raise KeyError(estr) else: - raise KeyError("{:} not found in Meta".format(repr(name))) + pysat.logger.warning(estr) return def keep(self, keep_names): From aa34e9634a10e4dbfae655549ad28de0716897d5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 12 Oct 2023 11:46:50 -0400 Subject: [PATCH 174/365] TST: added drop error tests Added error and logging tests for the Meta.drop method. --- pysat/tests/test_meta.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 3b1088fa1..38be65eec 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -126,6 +126,13 @@ def test_pop_w_bad_key(self): input_args=['not_a_key']) return + def test_drop_w_bad_name(self): + """Test that a bad name will raise a KeyError for `meta.drop`.""" + + testing.eval_bad_input(self.meta.drop, KeyError, 'not found in Meta', + input_args=['not_a_name']) + return + def test_getitem_w_bad_key(self): """Test that a bad key will raise a KeyError in meta access.""" @@ -280,6 +287,24 @@ def test_set_meta_with_wrong_type_drop(self, bad_val): # ------------------------- # Test the logging messages + def test_drop_with_some_bad_names(self, caplog): + """Test a logger warning is raised if not all names can be dropped.""" + + with caplog.at_level(logging.WARN, logger='pysat'): + self.meta.drop(['uts', 'units', 'fake_var']) + + # Test the warning + captured = caplog.text + estr = "missing expected message in: {:}".format(captured) + assert captured.find('not found in Meta') >= 0, estr + + # Check that correct meta data and labels were dropped + assert 'uts' not in self.meta.keys(), 'Did not drop metadata' + assert not hasattr(self.meta.labels, 'units'), 'Did not drop MetaLabel' + assert 'units' not in self.meta.data.columns, 'Did not drop meta label' + + return + @pytest.mark.parametrize('bad_val', [[1, 2], 1, 2.0, True, None]) def test_set_meta_with_wrong_type_cast(self, bad_val, caplog): """Test that setting meta as recastable type raises appropriate warning. From f1db3af3ee93b9eeef1f59e0f515af690f8a26d7 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 12 Oct 2023 15:50:05 -0400 Subject: [PATCH 175/365] ENH: added `__delitem__` Re-added `__delitem__`, which now just calls `drop`. --- pysat/_meta.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pysat/_meta.py b/pysat/_meta.py index 668530550..51751f982 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -571,6 +571,24 @@ def __contains__(self, data_var): return does_contain + def __delitem__(self, key): + """Delete a key by calling `drop` method. + + Parameters + ---------- + key : str or list-like + A meta data variable, label, or MetaHeader attribute; which are + considered in that order. + + Raises + ------ + KeyError + If all key values are unavailable + + """ + self.drop(key) + return + def __eq__(self, other_meta): """Check equality between Meta instances. From 3a008eb08b7403bea19284d4257d8ded6e6fa573 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 12 Oct 2023 15:50:37 -0400 Subject: [PATCH 176/365] TST: removed skip Removed the skip statement as tests are passing and added a parameter to test drop and delete together. --- pysat/tests/test_meta.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 38be65eec..e5cb68c64 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -1014,23 +1014,28 @@ def test_meta_merge(self): meta_dict[label].__repr__()) return - @pytest.mark.skip('See if memory is still an issue') @pytest.mark.parametrize("names", ['uts', ['uts', 'mlt'], 'units', ['units', 'uts']]) - def test_meta_drop(self, names): + @pytest.mark.parametrize("is_drop", [True, False]) + def test_meta_drop(self, names, is_drop): """Test successful deletion of meta data for different types of data. Parameters ---------- names : int Number of variables to drop in a single go. + is_drop : bool + Use `drop` if True, use `del` if False. """ # Set meta data self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'}) # Drop the values - self.meta.drop(names) + if is_drop: + self.meta.drop(names) + else: + del self.meta[names] # Test the successful deletion for name in pysat.utils.listify(names): @@ -1041,13 +1046,16 @@ def test_meta_drop(self, names): return @pytest.mark.parametrize("num_drop", [0, 1, 3]) - def test_meta_num_drop(self, num_drop): + @pytest.mark.parametrize("is_drop", [True, False]) + def test_meta_num_drop(self, num_drop, is_drop): """Test successful deletion of meta data for specific values. Parameters ---------- num_drop : int Number of variables to drop in a single go. + is_drop : bool + Use `drop` if True, use `del` if False. """ @@ -1060,7 +1068,10 @@ def test_meta_num_drop(self, num_drop): [val for val in self.meta.keys()]) # Drop the values - self.meta.drop(self.dval) + if is_drop: + self.meta.drop(self.dval) + else: + del self.meta[self.dval] # Test the successful deletion meta_vals = [val for val in self.meta.keys()] From ecd2933d69192ac5c207c107905f0509fc6f3706 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 12 Oct 2023 15:50:51 -0400 Subject: [PATCH 177/365] DOC: updated changelog Improved the changelog description. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ada664d21..dc43ecbcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). reduce code duplication. * Added `__delitem__` to Meta and `drop` to MetaHeader and MetaLabels classes. * Modified Meta to allow MetaHeader attribute access directly from Meta. + * Expanded `Meta.drop` to traverse attached MetaLabel and MetaHeader data. * Added options to customize `pysat_ndtesting` instrument with sample rate, shift in time. * Added orbit number to `pysat_ndtesting`. From ce0f75206e61899dff6944b813156d5561f156a9 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 17 Oct 2023 10:21:33 -0400 Subject: [PATCH 178/365] MAINT: removed `use_header` Removed `use_header` as a standard kwarg for `load`. Adjusted deprecation warning to notify user of the end of support for the optional kwarg. --- pysat/_instrument.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index d66ae92b8..4ffa28a1d 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -331,7 +331,7 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', default_kwargs = _get_supported_keywords(func) # Expand the dict to include method keywords for load. - # TODO(#1020): Remove this if statement for the 3.2.0+ release + # TODO(#1020): Remove this if statement when `use_header` is removed if fkey == 'load': meth = getattr(self, fkey) default_kwargs.update(_get_supported_keywords(meth)) @@ -2920,7 +2920,7 @@ def generic_meta_translator(self, input_meta): def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, end_date=None, fname=None, stop_fname=None, verifyPad=False, - use_header=False, **kwargs): + **kwargs): """Load the instrument data and metadata. Parameters @@ -2957,9 +2957,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, verifyPad : bool If True, padding data not removed for debugging. Padding parameters are provided at Instrument instantiation. (default=False) - use_header : bool - If True, moves custom Meta attributes to MetaHeader instead of - Instrument (default=False) **kwargs : dict Dictionary of keywords that may be options for specific instruments. @@ -3017,6 +3014,22 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, inst.load(fname=inst.files[0], stop_fname=inst.files[1]) """ + # If the `use_header` kwarg is included, set it here. Otherwise set + # it to True. + # TODO(#1020): removed this logic after kwarg not supported. + if 'use_header' in kwargs.keys(): + use_header = kwargs['use_header'] + warnings.warn(''.join(['Meta now contains a class for global ', + 'metadata (MetaHeader). Allowing attachment', + ' of global attributes to Instrument ', + 'through `use_header=False` will be ', + 'Deprecated in pysat 3.3.0+. Remove ', + '`use_header` kwarg (now same as ', + '`use_header=True`) to stop this warning.']), + DeprecationWarning, stacklevel=2) + else: + use_header = True + # Add the load kwargs from initialization those provided on input for lkey in self.kwargs['load'].keys(): # Only use the initialized kwargs if a request hasn't been @@ -3368,19 +3381,11 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # Transfer any extra attributes in meta to the Instrument object. # Metadata types need to be initialized before preprocess is run. - # TODO(#1020): Change the way this kwarg is handled + # TODO(#1020): Remove warning and logic when kwarg is removed if use_header or ('use_header' in self.kwargs['load'] and self.kwargs['load']['use_header']): self.meta.transfer_attributes_to_header() else: - warnings.warn(''.join(['Meta now contains a class for global ', - 'metadata (MetaHeader). Default attachment ', - 'of global attributes to Instrument will ', - 'be Deprecated in pysat 3.2.0+. Set ', - '`use_header=True` in this load call or ', - 'on Instrument instantiation to remove this', - ' warning.']), DeprecationWarning, - stacklevel=2) self.meta.transfer_attributes_to_instrument(self) # Transfer loaded data types to meta. From 588a735033570fc203fec7d1ec4de4f95d686161 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 17 Oct 2023 10:22:18 -0400 Subject: [PATCH 179/365] DOC: updated basic tutorial Updated basic tutorial by changing the description of the old `use_header` kwarg. --- docs/tutorial/tutorial_basics.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/tutorial/tutorial_basics.rst b/docs/tutorial/tutorial_basics.rst index 63fcb1bdb..59ab6d4a0 100644 --- a/docs/tutorial/tutorial_basics.rst +++ b/docs/tutorial/tutorial_basics.rst @@ -518,15 +518,16 @@ the instrument PI, etc. Previous versions of pysat stored this data as custom attributes attached to the :py:class:`pysat.Instrument`, instead of keeping all metadata in the :py:class:`pysat.Meta` object. -To avoid breaking existing workflows, global metadata is only loaded into -:py:class:`pysat.MetaHeader` through completely internal pysat processes or -after setting the :py:data:`use_header` keyword argument. +Global metadata is loaded into the :py:class:`pysat.MetaHeader` by default, but +to avoid breaking existing workflows, loading this metadata directly into the +:py:class:`~pysat._instrument.Instrument` by setting the :py:data:`use_header` +keyword argument. .. code:: python - # This will show: Metadata for 0 global attributes - dmsp.load(date=start, use_header=True) - print(dmsp.meta.header) + # This will raise a warning that future releases will require use of + # the MetaHeader class + dmsp.load(date=start, use_header=False) You can manually add global metadata the same way you would assign an attribute. From 28cfb88c3204b33611204719dec707e83719abc8 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 17 Oct 2023 10:23:06 -0400 Subject: [PATCH 180/365] TST: removed `use_header` kwarg Removed the `use_header` kwarg from the test suite and updated the deprecation test. --- pysat/constellations/testing.py | 6 +- pysat/constellations/testing_partial.py | 4 +- pysat/tests/classes/cls_instrument_access.py | 105 ++++++++-------- pysat/tests/classes/cls_instrument_library.py | 15 ++- .../tests/classes/cls_instrument_property.py | 22 ++-- pysat/tests/test_constellation.py | 44 ++++--- pysat/tests/test_files.py | 5 +- pysat/tests/test_instrument.py | 85 ++++++------- pysat/tests/test_instrument_custom.py | 17 +-- pysat/tests/test_instrument_index.py | 10 +- pysat/tests/test_instrument_padding.py | 115 ++++++------------ pysat/tests/test_instruments.py | 12 +- pysat/tests/test_meta.py | 14 +-- pysat/tests/test_methods_general.py | 8 +- pysat/tests/test_methods_testing.py | 2 +- pysat/tests/test_orbits.py | 64 +++------- pysat/tests/test_utils.py | 4 +- pysat/tests/test_utils_coords.py | 37 +++--- pysat/tests/test_utils_files.py | 10 +- pysat/tests/test_utils_io.py | 74 +++++------ 20 files changed, 258 insertions(+), 395 deletions(-) diff --git a/pysat/constellations/testing.py b/pysat/constellations/testing.py index 1b7917e80..1d03e6833 100644 --- a/pysat/constellations/testing.py +++ b/pysat/constellations/testing.py @@ -13,8 +13,8 @@ import pysat instruments = [pysat.Instrument('pysat', 'testing', clean_level='clean', - num_samples=10, use_header=True), + num_samples=10), pysat.Instrument('pysat', 'ndtesting', clean_level='clean', - num_samples=16, use_header=True), + num_samples=16), pysat.Instrument('pysat', 'testmodel', clean_level='clean', - num_samples=18, use_header=True)] + num_samples=18)] diff --git a/pysat/constellations/testing_partial.py b/pysat/constellations/testing_partial.py index 800e9850c..d8b8bcb1b 100644 --- a/pysat/constellations/testing_partial.py +++ b/pysat/constellations/testing_partial.py @@ -9,6 +9,6 @@ import pysat instruments = [pysat.Instrument('pysat', 'testing', clean_level='clean', - num_samples=10, use_header=True), + num_samples=10), pysat.Instrument('pysat', 'testing', tag='no_download', - clean_level='clean', use_header=True)] + clean_level='clean')] diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py index 5e89a7d88..e32d244f1 100644 --- a/pysat/tests/classes/cls_instrument_access.py +++ b/pysat/tests/classes/cls_instrument_access.py @@ -47,8 +47,7 @@ def test_instrument_complete_by_init(self): """Test Instrument object fully complete by self._init_rtn().""" # Create a base instrument to compare against - inst_copy = pysat.Instrument(inst_module=self.testInst.inst_module, - use_header=True) + inst_copy = pysat.Instrument(inst_module=self.testInst.inst_module) # Get instrument module and init funtcion inst_mod = self.testInst.inst_module @@ -71,7 +70,7 @@ def temp_init(inst, test_init_kwarg=None): # Instantiate instrument with test module which invokes needed test # code in the background - pysat.Instrument(inst_module=inst_mod, use_header=True) + pysat.Instrument(inst_module=inst_mod) # Restore nominal init function inst_mod.init = inst_mod_init @@ -125,8 +124,7 @@ def test_basic_instrument_load(self, kwargs): """ # Load data by year and day of year - self.testInst.load(self.ref_time.year, self.ref_doy, **kwargs, - use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy, **kwargs) # Test that the loaded date range is correct self.eval_successful_load() @@ -152,7 +150,7 @@ def test_basic_instrument_load_no_data(self, caplog, pad): # Test doesn't check against loading by filename since that produces # an error if there is no file. Loading by yr, doy no different # than date in this case. - self.testInst.load(date=no_data_d, pad=pad, use_header=True) + self.testInst.load(date=no_data_d, pad=pad) # Confirm by checking against caplog that metadata was # not assigned. @@ -182,7 +180,7 @@ def test_basic_instrument_load_two_days(self): end_date = self.ref_time + dt.timedelta(days=2) end_doy = int(end_date.strftime("%j")) self.testInst.load(self.ref_time.year, self.ref_doy, end_date.year, - end_doy, use_header=True) + end_doy) # Test that the loaded date range is correct self.eval_successful_load(end_date=end_date) @@ -195,8 +193,7 @@ def test_basic_instrument_bad_keyword_at_load(self): testing.eval_bad_input(self.testInst.load, TypeError, "load() got an unexpected keyword", input_kwargs={'date': self.ref_time, - 'unsupported_keyword': True, - 'use_header': True}) + 'unsupported_keyword': True}) return def test_basic_instrument_load_yr_no_doy(self): @@ -205,7 +202,7 @@ def test_basic_instrument_load_yr_no_doy(self): # Check that the correct error is raised estr = 'Unknown or incomplete input combination.' testing.eval_bad_input(self.testInst.load, TypeError, estr, - [self.ref_time.year], {'use_header': True}) + [self.ref_time.year]) return @pytest.mark.parametrize('doy', [0, 367, 1000, -1, -10000]) @@ -221,7 +218,7 @@ def test_basic_instrument_load_yr_bad_doy(self, doy): estr = 'Day of year (doy) is only valid between and ' testing.eval_bad_input(self.testInst.load, ValueError, estr, - [self.ref_time.year, doy], {'use_header': True}) + [self.ref_time.year, doy]) return @pytest.mark.parametrize('end_doy', [0, 367, 1000, -1, -10000]) @@ -239,7 +236,7 @@ def test_basic_instrument_load_yr_bad_end_doy(self, end_doy): testing.eval_bad_input(self.testInst.load, ValueError, estr, [self.ref_time.year, 1], {'end_yr': self.ref_time.year, - 'end_doy': end_doy, 'use_header': True}) + 'end_doy': end_doy}) return def test_basic_instrument_load_yr_no_end_doy(self): @@ -248,7 +245,7 @@ def test_basic_instrument_load_yr_no_end_doy(self): estr = 'Both end_yr and end_doy must be set' testing.eval_bad_input(self.testInst.load, ValueError, estr, [self.ref_time.year, self.ref_doy, - self.ref_time.year], {'use_header': True}) + self.ref_time.year]) return @pytest.mark.parametrize("kwargs", [{'yr': 2009, 'doy': 1, @@ -277,7 +274,6 @@ def test_basic_instrument_load_mixed_inputs(self, kwargs): """ - kwargs['use_header'] = True estr = 'An inconsistent set of inputs have been' testing.eval_bad_input(self.testInst.load, ValueError, estr, input_kwargs=kwargs) @@ -286,7 +282,7 @@ def test_basic_instrument_load_mixed_inputs(self, kwargs): def test_basic_instrument_load_no_input(self): """Test that `.load()` loads all data.""" - self.testInst.load(use_header=True) + self.testInst.load() assert (self.testInst.index[0] == self.testInst.files.start_date) assert (self.testInst.index[-1] >= self.testInst.files.stop_date) assert (self.testInst.index[-1] <= self.testInst.files.stop_date @@ -316,7 +312,6 @@ def test_instrument_load_errors_with_multifile(self, load_in, verr): else: load_kwargs = dict() - load_kwargs['use_header'] = True testing.eval_bad_input(self.testInst.load, ValueError, verr, input_kwargs=load_kwargs) return @@ -324,7 +319,7 @@ def test_instrument_load_errors_with_multifile(self, load_in, verr): def test_basic_instrument_load_by_date(self): """Test loading by date.""" - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) self.eval_successful_load() return @@ -332,8 +327,7 @@ def test_basic_instrument_load_by_dates(self): """Test date range loading, `date` and `end_date`.""" end_date = self.ref_time + dt.timedelta(days=2) - self.testInst.load(date=self.ref_time, end_date=end_date, - use_header=True) + self.testInst.load(date=self.ref_time, end_date=end_date) self.eval_successful_load(end_date=end_date) return @@ -341,15 +335,14 @@ def test_basic_instrument_load_by_date_with_extra_time(self): """Ensure `.load(date=date)` only uses date portion of datetime.""" # Put in a date that has more than year, month, day - self.testInst.load(date=(self.ref_time + dt.timedelta(minutes=71)), - use_header=True) + self.testInst.load(date=(self.ref_time + dt.timedelta(minutes=71))) self.eval_successful_load() return def test_basic_instrument_load_data(self): """Test that correct day loads (checking down to the sec).""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.eval_successful_load() return @@ -361,7 +354,7 @@ def test_basic_instrument_load_leap_year(self): self.ref_time = dt.datetime(2008, 12, 31) self.ref_doy = 366 - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.eval_successful_load() return @@ -398,7 +391,7 @@ def test_file_load_bad_start_file(self, operator): """ - self.testInst.load(fname=self.testInst.files[1], use_header=True) + self.testInst.load(fname=self.testInst.files[1]) # Set new bounds that do not include this date. self.testInst.bounds = (self.testInst.files[0], self.testInst.files[2], @@ -418,7 +411,7 @@ def test_file_load_bad_start_date(self, operator): """ - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) # Set new bounds that do not include this date. self.testInst.bounds = (self.ref_time + dt.timedelta(days=1), @@ -435,7 +428,7 @@ def test_basic_fname_instrument_load(self): # If mangle_file_date is true, index will not match exactly. # Find the closest point instead. ind = np.argmin(abs(self.testInst.files.files.index - self.ref_time)) - self.testInst.load(fname=self.testInst.files[ind], use_header=True) + self.testInst.load(fname=self.testInst.files[ind]) self.eval_successful_load() return @@ -455,7 +448,7 @@ def test_fname_load_default(self, operator, direction): # If mangle_file_date is true, index will not match exactly. # Find the closest point. ind = np.argmin(abs(self.testInst.files.files.index - self.ref_time)) - self.testInst.load(fname=self.testInst.files[ind], use_header=True) + self.testInst.load(fname=self.testInst.files[ind]) getattr(self.testInst, operator)() # Modify ref time since iterator changes load date. @@ -468,8 +461,7 @@ def test_fname_load_default(self, operator, direction): def test_filename_load(self): """Test if file is loadable by filename with no path.""" - self.testInst.load(fname=self.ref_time.strftime('%Y-%m-%d.nofile'), - use_header=True) + self.testInst.load(fname=self.ref_time.strftime('%Y-%m-%d.nofile')) self.eval_successful_load() return @@ -481,7 +473,7 @@ def test_filenames_load(self): stop_fname = self.ref_time + foff stop_fname = stop_fname.strftime('%Y-%m-%d.nofile') self.testInst.load(fname=self.ref_time.strftime('%Y-%m-%d.nofile'), - stop_fname=stop_fname, use_header=True) + stop_fname=stop_fname) assert self.testInst.index[0] == self.ref_time assert self.testInst.index[-1] >= self.ref_time + foff assert self.testInst.index[-1] <= self.ref_time + (2 * foff) @@ -499,8 +491,7 @@ def test_filenames_load_out_of_order(self): testing.eval_bad_input(self.testInst.load, ValueError, estr, input_kwargs={'fname': stop_fname, - 'stop_fname': check_fname, - 'use_header': True}) + 'stop_fname': check_fname}) return def test_eq_no_data(self): @@ -513,7 +504,7 @@ def test_eq_no_data(self): def test_eq_both_with_data(self): """Test equality when the same object with loaded data.""" - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) inst_copy = self.testInst.copy() assert inst_copy == self.testInst return @@ -521,7 +512,7 @@ def test_eq_both_with_data(self): def test_eq_one_with_data(self): """Test equality when the same objects but only one with loaded data.""" - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) inst_copy = self.testInst.copy() inst_copy.data = self.testInst._null_data assert inst_copy != self.testInst @@ -530,7 +521,7 @@ def test_eq_one_with_data(self): def test_eq_different_data_type(self): """Test equality different data type.""" - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) inst_copy = self.testInst.copy() # Can only change data types if Instrument empty @@ -596,12 +587,12 @@ def test_concat_data(self, prepend, sort_dim_toggle, include): ref_time2 = self.ref_time + pds.tseries.frequencies.to_offset( self.testInst.files.files.index.freqstr) doy2 = int(ref_time2.strftime('%j')) - self.testInst.load(ref_time2.year, doy2, use_header=True) + self.testInst.load(ref_time2.year, doy2) data2 = self.testInst.data len2 = len(self.testInst.index) # Load a different data set into the instrument - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) len1 = len(self.testInst.index) # Set the keyword arguments @@ -655,7 +646,7 @@ def test_empty_flag_data_empty(self): def test_empty_flag_data_not_empty(self): """Test the status of the empty flag for loaded data.""" - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) assert not self.testInst.empty return @@ -666,7 +657,7 @@ def test_index_attribute(self): assert isinstance(self.testInst.index, pds.Index) # Test an index is present with data loaded in an Instrument - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) assert isinstance(self.testInst.index, pds.Index) return @@ -674,7 +665,7 @@ def test_index_return(self): """Test that the index is returned in the proper format.""" # Load data - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) # Ensure we get the index back if self.testInst.pandas_format: @@ -697,7 +688,7 @@ def test_basic_data_access_by_name(self, labels): """ - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) assert np.all((self.testInst[labels] == self.testInst.data[labels]).values) return @@ -716,7 +707,7 @@ def test_data_access_by_indices_and_name(self, index): """ - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) assert np.all(self.testInst[index, 'mlt'] == self.testInst.data['mlt'][index]) return @@ -724,7 +715,7 @@ def test_data_access_by_indices_and_name(self, index): def test_data_access_by_row_slicing(self): """Check that each variable is downsampled.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) result = self.testInst[0:10] for variable, array in result.items(): assert len(array) == len(self.testInst.data[variable].values[0:10]) @@ -737,7 +728,7 @@ def test_data_access_by_row_slicing_and_name_slicing(self): if not self.testInst.pandas_format: pytest.skip("name slicing not implemented for xarray") - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) result = self.testInst[0:10, 'uts':'mlt'] for variable, array in result.items(): assert len(array) == len(self.testInst.data[variable].values[0:10]) @@ -747,7 +738,7 @@ def test_data_access_by_row_slicing_and_name_slicing(self): def test_data_access_by_datetime_and_name(self): """Check that datetime can be used to access data.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.out = dt.datetime(2009, 1, 1, 0, 0, 0) assert np.all(self.testInst[self.out, 'uts'] == self.testInst.data['uts'].values[0]) @@ -756,7 +747,7 @@ def test_data_access_by_datetime_and_name(self): def test_data_access_by_datetime_slicing_and_name(self): """Check that a slice of datetimes can be used to access data.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) time_step = (self.testInst.index[1] - self.testInst.index[0]).value / 1.E9 offset = dt.timedelta(seconds=(10 * time_step)) @@ -769,7 +760,7 @@ def test_data_access_by_datetime_slicing_and_name(self): def test_setting_data_by_name(self): """Test setting data by name.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst['doubleMLT'] = 2. * self.testInst['mlt'] assert np.all(self.testInst['doubleMLT'] == 2. * self.testInst['mlt']) return @@ -777,7 +768,7 @@ def test_setting_data_by_name(self): def test_setting_series_data_by_name(self): """Test setting series data by name.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst['doubleMLT'] = 2. * pds.Series( self.testInst['mlt'].values, index=self.testInst.index) assert np.all(self.testInst['doubleMLT'] == 2. * self.testInst['mlt']) @@ -789,7 +780,7 @@ def test_setting_series_data_by_name(self): def test_setting_pandas_dataframe_by_names(self): """Test setting pandas dataframe by name.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst[['doubleMLT', 'tripleMLT']] = pds.DataFrame( {'doubleMLT': 2. * self.testInst['mlt'].values, 'tripleMLT': 3. * self.testInst['mlt'].values}, @@ -801,7 +792,7 @@ def test_setting_pandas_dataframe_by_names(self): def test_setting_data_by_name_single_element(self): """Test setting data by name for a single element.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst['doubleMLT'] = 2. assert np.all(self.testInst['doubleMLT'] == 2.) assert len(self.testInst['doubleMLT']) == len(self.testInst.index) @@ -813,7 +804,7 @@ def test_setting_data_by_name_single_element(self): def test_setting_data_by_name_with_meta(self): """Test setting data by name with meta.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst['doubleMLT'] = {'data': 2. * self.testInst['mlt'], 'units': 'hours', 'long_name': 'double trouble'} @@ -825,7 +816,7 @@ def test_setting_data_by_name_with_meta(self): def test_setting_partial_data(self): """Test setting partial data by index.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.out = self.testInst if self.testInst.pandas_format: self.testInst[0:3] = 0 @@ -860,7 +851,7 @@ def test_setting_partial_data_by_inputs(self, changed, fixed): """ - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst['doubleMLT'] = 2. * self.testInst['mlt'] self.testInst[changed, 'doubleMLT'] = 0 assert (self.testInst[fixed, 'doubleMLT'] @@ -871,7 +862,7 @@ def test_setting_partial_data_by_inputs(self, changed, fixed): def test_modifying_data_inplace(self): """Test modification of data inplace.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst['doubleMLT'] = 2. * self.testInst['mlt'] self.testInst['doubleMLT'] += 100 assert (self.testInst['doubleMLT'] @@ -890,7 +881,7 @@ def test_getting_all_data_by_index(self, index): """ - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) inst_subset = self.testInst[index] if self.testInst.pandas_format: assert len(inst_subset) == len(index) @@ -913,7 +904,7 @@ def test_unknown_variable_error_renaming(self, values): """ # Check for error for unknown variable name - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) # Capture the ValueError and message testing.eval_bad_input(self.testInst.rename, ValueError, @@ -944,7 +935,7 @@ def test_basic_variable_renaming(self, lowercase, mapper): values = {var: mapper(var) for var in self.testInst.variables} # Test single variable - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst.rename(mapper, lowercase_data_labels=lowercase) for key in values: diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index da0dc1195..030500179 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -68,7 +68,7 @@ def initialize_test_inst_and_date(inst_dict): tag=inst_dict['tag'], inst_id=inst_dict['inst_id'], temporary_file_list=True, update_files=True, - use_header=True, **kwargs) + **kwargs) test_dates = inst_dict['inst_module']._test_dates date = test_dates[inst_dict['inst_id']][inst_dict['tag']] return test_inst, date @@ -94,7 +94,7 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False, """ try: - test_inst.load(date=date, use_header=True) + test_inst.load(date=date) except Exception as err: # Catch all potential input errors, and only ensure that the one caused # by the strict time flag is prevented from occurring on future load @@ -111,7 +111,7 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False, # Evaluate the warning with warnings.catch_warnings(record=True) as war: - test_inst.load(date=date, use_header=True) + test_inst.load(date=date) assert len(war) >= 1 categories = [war[j].category for j in range(len(war))] @@ -279,7 +279,7 @@ def test_modules_standard(self, inst_name): for inst_id in module.inst_ids.keys(): for tag in module.inst_ids[inst_id]: inst = pysat.Instrument(inst_module=module, tag=tag, - inst_id=inst_id, use_header=True) + inst_id=inst_id) # Test to see that the class parameters were passed in testing.assert_isinstance(inst, pysat.Instrument) @@ -462,7 +462,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): with caplog.at_level( getattr(logging, clean_method_level), logger='pysat'): - test_inst.load(date=date, use_header=True) + test_inst.load(date=date) # Test the returned message out_msg = caplog.text @@ -472,7 +472,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): elif clean_method == 'warning': # A warning message is expected with warnings.catch_warnings(record=True) as war: - test_inst.load(date=date, use_header=True) + test_inst.load(date=date) # Test the warning output testing.eval_warnings(war, [clean_method_msg], @@ -482,8 +482,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # and the error message testing.eval_bad_input( test_inst.load, clean_method_level, - clean_method_msg, - input_kwargs={'date': date, 'use_header': True}) + clean_method_msg, input_kwargs={'date': date}) else: raise AttributeError( 'unknown type of warning: {:}'.format( diff --git a/pysat/tests/classes/cls_instrument_property.py b/pysat/tests/classes/cls_instrument_property.py index dcb0b262c..5908fb720 100644 --- a/pysat/tests/classes/cls_instrument_property.py +++ b/pysat/tests/classes/cls_instrument_property.py @@ -378,7 +378,7 @@ def test_inst_attributes_not_overwritten(self): greeting = '... listen!' self.testInst.hei = greeting - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) assert self.testInst.hei == greeting return @@ -416,15 +416,11 @@ def test_str_w_orbit(self): """Test string output with Orbit data.""" reload(pysat.instruments.pysat_testing) - orbit_info = {'index': 'mlt', - 'kind': 'local time', + orbit_info = {'index': 'mlt', 'kind': 'local time', 'period': np.timedelta64(97, 'm')} testInst = pysat.Instrument(platform='pysat', name='testing', - num_samples=10, - clean_level='clean', - update_files=True, - orbit_info=orbit_info, - use_header=True) + num_samples=10, clean_level='clean', + update_files=True, orbit_info=orbit_info) self.out = testInst.__str__() @@ -434,7 +430,7 @@ def test_str_w_orbit(self): assert self.out.find('Loaded Orbit Number: 0') > 0 # Activate orbits, check that message has changed - testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + testInst.load(self.ref_time.year, self.ref_doy) testInst.orbits.next() self.out = testInst.__str__() assert self.out.find('Loaded Orbit Number: 1') > 0 @@ -462,7 +458,7 @@ def passfunc(self): def test_str_w_load_lots_data(self): """Test string output with loaded data with many variables.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.out = self.testInst.__str__() assert self.out.find('Number of variables:') > 0 assert self.out.find('...') > 0 @@ -472,7 +468,7 @@ def test_str_w_load_less_data(self): """Test string output with loaded data with few (4) variables.""" # Load the test data - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) # Ensure the desired data variable is present and delete all others # 4-6 variables are needed to test all lines; choose the lesser limit @@ -548,7 +544,7 @@ def test_instrument_function_keywords(self, caplog, func, kwarg, val): with caplog.at_level(logging.INFO, logger='pysat'): # Trigger load functions - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) # Refresh files to trigger other functions self.testInst.files.refresh() @@ -607,7 +603,7 @@ def test_instrument_function_keyword_liveness(self, caplog, func, kwarg): with caplog.at_level(logging.INFO, logger='pysat'): # Trigger load functions - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) # Refresh files to trigger other functions self.testInst.files.refresh() diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py index 5986b71e8..608251db2 100644 --- a/pysat/tests/test_constellation.py +++ b/pysat/tests/test_constellation.py @@ -43,8 +43,7 @@ def test_construct_constellation(self, ikeys, ivals, ilen): # Initialize the Constellation using the desired kwargs const = pysat.Constellation( - **{ikey: ivals[i] for i, ikey in enumerate(ikeys)}, - use_header=True) + **{ikey: ivals[i] for i, ikey in enumerate(ikeys)}) # Test that the appropriate number of Instruments were loaded. Each # fake Instrument has 5 tags and 1 inst_id. @@ -72,7 +71,7 @@ def test_some_bad_construct_constellation(self, caplog): # Load the Constellation and capture log output with caplog.at_level(logging.WARNING, logger='pysat'): const = pysat.Constellation(platforms=['Executor', 'platname1'], - tags=[''], use_header=True) + tags=['']) # Test the partial Constellation initialization assert len(const.instruments) == 2 @@ -164,7 +163,7 @@ def test_getitem(self): """Test Constellation iteration through instruments attribute.""" self.in_kwargs['const_module'] = None - self.const = pysat.Constellation(**self.in_kwargs, use_header=True) + self.const = pysat.Constellation(**self.in_kwargs) tst_get_inst = self.const[:] pysat.utils.testing.assert_lists_equal(self.instruments, tst_get_inst) return @@ -173,7 +172,7 @@ def test_repr_w_inst(self): """Test Constellation string output with instruments loaded.""" self.in_kwargs['const_module'] = None - self.const = pysat.Constellation(**self.in_kwargs, use_header=True) + self.const = pysat.Constellation(**self.in_kwargs) out_str = self.const.__repr__() assert out_str.find("Constellation(instruments") >= 0 @@ -183,7 +182,7 @@ def test_str_w_inst(self): """Test Constellation string output with instruments loaded.""" self.in_kwargs['const_module'] = None - self.const = pysat.Constellation(**self.in_kwargs, use_header=True) + self.const = pysat.Constellation(**self.in_kwargs) out_str = self.const.__str__() assert out_str.find("pysat Constellation ") >= 0 @@ -216,7 +215,7 @@ def test_str_with_data(self, common_index, cstr): self.in_kwargs["common_index"] = common_index self.const = pysat.Constellation(**self.in_kwargs) - self.const.load(date=self.ref_time, use_header=True) + self.const.load(date=self.ref_time) out_str = self.const.__str__() assert out_str.find("pysat Constellation ") >= 0 @@ -239,7 +238,7 @@ def double_mlt(inst): # Add the custom function self.const.custom_attach(double_mlt, at_pos='end') - self.const.load(date=self.ref_time, use_header=True) + self.const.load(date=self.ref_time) # Test the added value for inst in self.const: @@ -255,7 +254,7 @@ def setup_method(self): """Set up the unit test environment for each method.""" self.inst = list(constellations.testing.instruments) - self.const = pysat.Constellation(instruments=self.inst, use_header=True) + self.const = pysat.Constellation(instruments=self.inst) self.ref_time = pysat.instruments.pysat_testing._test_dates[''][''] self.attrs = ["platforms", "names", "tags", "inst_ids", "instruments", "bounds", "empty", "empty_partial", "index_res", @@ -342,7 +341,7 @@ def test_empty_flag_data_empty_partial_load(self): """Test the status of the empty flag for partially loaded data.""" self.const = pysat.Constellation( - const_module=constellations.testing_partial, use_header=True) + const_module=constellations.testing_partial) self.const.load(date=self.ref_time) assert self.const.empty_partial assert not self.const.empty @@ -352,7 +351,7 @@ def test_empty_flag_data_not_empty_partial_load(self): """Test the alt status of the empty flag for partially loaded data.""" self.const = pysat.Constellation( - const_module=constellations.testing_partial, use_header=True) + const_module=constellations.testing_partial) self.const.load(date=self.ref_time) assert not self.const._empty(all_inst=False) return @@ -361,7 +360,7 @@ def test_empty_flag_data_not_empty(self): """Test the status of the empty flag for loaded data.""" # Load data and test the status flag - self.const.load(date=self.ref_time, use_header=True) + self.const.load(date=self.ref_time) assert not self.const.empty return @@ -372,7 +371,7 @@ def test_full_data_index(self, ikwarg): # Test the attribute with loaded data self.const = pysat.Constellation(instruments=self.inst, **ikwarg) - self.const.load(date=self.ref_time, use_header=True) + self.const.load(date=self.ref_time) assert isinstance(self.const.index, pds.Index) assert self.const.index[0] == self.ref_time @@ -394,7 +393,7 @@ def test_full_data_date(self): """Test the date property when no data is loaded.""" # Test the attribute with loaded data - self.const.load(date=self.ref_time, use_header=True) + self.const.load(date=self.ref_time) assert self.const.date == self.ref_time return @@ -403,7 +402,7 @@ def test_full_variables(self): """Test the variables property when no data is loaded.""" # Test the attribute with loaded data - self.const.load(date=self.ref_time, use_header=True) + self.const.load(date=self.ref_time) assert len(self.const.variables) > 0 assert 'uts_pysat_testing' in self.const.variables @@ -501,10 +500,9 @@ def test_to_inst_pandas_w_pad(self): """ # Redefine the Instrument and constellation self.inst = pysat.Instrument( - inst_module=pysat.instruments.pysat_testing, use_header=True, - pad=pds.DateOffset(hours=1), num_samples=10) - self.const = pysat.Constellation(instruments=[self.inst], - use_header=True) + inst_module=pysat.instruments.pysat_testing, num_samples=10, + pad=pds.DateOffset(hours=1)) + self.const = pysat.Constellation(instruments=[self.inst]) # Load the data self.inst.load(date=self.ref_time) @@ -539,11 +537,11 @@ def test_to_inst_mult_pad_clean(self): pad = pds.DateOffset(hours=1) self.inst = [ pysat.Instrument(inst_module=pysat.instruments.pysat_testing, - use_header=True, pad=pad, num_samples=10), + pad=pad, num_samples=10), pysat.Instrument(inst_module=pysat.instruments.pysat_testing, - use_header=True, pad=2 * pad, - clean_level=clean_level, num_samples=10)] - self.const = pysat.Constellation(instruments=self.inst, use_header=True) + pad=2 * pad, clean_level=clean_level, + num_samples=10)] + self.const = pysat.Constellation(instruments=self.inst) # Load the Instrument and Constellation data self.inst[-1].load(date=self.ref_time) diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py index cdc5039a0..0933d2ed7 100644 --- a/pysat/tests/test_files.py +++ b/pysat/tests/test_files.py @@ -148,8 +148,7 @@ def setup_method(self): self.testInst = pysat.Instrument( inst_module=pysat.instruments.pysat_testing, clean_level='clean', - temporary_file_list=self.temporary_file_list, update_files=True, - use_header=True) + temporary_file_list=self.temporary_file_list, update_files=True) # Create instrument directories in tempdir create_dir(self.testInst) @@ -215,7 +214,7 @@ def test_equality_with_copy(self): def test_equality_with_copy_with_data(self): """Test that copy is the same as original, loaded `inst.data`.""" # Load data - self.testInst.load(date=self.start, use_header=True) + self.testInst.load(date=self.start) # Make copy self.out = self.testInst.files.copy() diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 95fbf75b2..6c132fdca 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -49,10 +49,8 @@ def setup_method(self): reload(pysat.instruments.pysat_testing) self.testInst = pysat.Instrument(platform='pysat', name='testing', - num_samples=10, - clean_level='clean', + num_samples=10, clean_level='clean', update_files=True, - use_header=True, **self.testing_kwargs) self.ref_time = pysat.instruments.pysat_testing._test_dates[''][''] self.ref_doy = int(self.ref_time.strftime('%j')) @@ -90,10 +88,8 @@ def setup_method(self): self.ref_time + pds.DateOffset(years=2) - pds.DateOffset(days=1), freq=self.freq) self.testInst = pysat.Instrument(platform='pysat', name='testing', - num_samples=10, - clean_level='clean', + num_samples=10, clean_level='clean', update_files=True, - use_header=True, file_date_range=date_range, **self.testing_kwargs) self.ref_doy = int(self.ref_time.strftime('%j')) @@ -122,10 +118,8 @@ def setup_method(self): + pds.DateOffset(years=2, days=-1), freq=self.freq) self.testInst = pysat.Instrument(platform='pysat', name='testing', - num_samples=10, - clean_level='clean', + num_samples=10, clean_level='clean', update_files=True, - use_header=True, file_date_range=date_range, **self.testing_kwargs) self.ref_doy = int(self.ref_time.strftime('%j')) @@ -155,10 +149,8 @@ def setup_method(self): + pds.DateOffset(years=5, days=-1), freq=self.freq) self.testInst = pysat.Instrument(platform='pysat', name='testing', - num_samples=10, - clean_level='clean', + num_samples=10, clean_level='clean', update_files=True, - use_header=True, file_date_range=date_range, **self.testing_kwargs) self.ref_doy = int(self.ref_time.strftime('%j')) @@ -180,11 +172,8 @@ def setup_method(self): reload(pysat.instruments.pysat_testing) imod = pysat.instruments.pysat_testing - self.testInst = pysat.Instrument(inst_module=imod, - num_samples=10, - clean_level='clean', - update_files=True, - use_header=True, + self.testInst = pysat.Instrument(inst_module=imod, num_samples=10, + clean_level='clean', update_files=True, **self.testing_kwargs) self.ref_time = imod._test_dates[''][''] self.ref_doy = int(self.ref_time.strftime('%j')) @@ -211,12 +200,9 @@ def setup_method(self): """Set up the unit test environment for each method.""" reload(pysat.instruments.pysat_ndtesting) - self.testInst = pysat.Instrument(platform='pysat', - name='ndtesting', - num_samples=10, - clean_level='clean', + self.testInst = pysat.Instrument(platform='pysat', name='ndtesting', + num_samples=10, clean_level='clean', update_files=True, - use_header=True, **self.testing_kwargs) self.ref_time = pysat.instruments.pysat_ndtesting._test_dates[''][''] self.ref_doy = int(self.ref_time.strftime('%j')) @@ -232,7 +218,7 @@ def teardown_method(self): def test_setting_data_as_tuple(self): """Test setting data as a tuple.""" - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst['doubleMLT'] = ('time', 2. * self.testInst['mlt'].values) assert np.all(self.testInst['doubleMLT'] == 2. * self.testInst['mlt']) return @@ -267,7 +253,7 @@ def test_data_access_by_2d_indices_and_name(self, index): """ - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) assert np.all(self.testInst[index, index, 'profiles'] == self.testInst.data['profiles'][index, index]) return @@ -275,7 +261,7 @@ def test_data_access_by_2d_indices_and_name(self, index): def test_data_access_by_2d_tuple_indices_and_name(self): """Check that variables and be accessed by multi-dim tuple index.""" - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) index = ([0, 1, 2, 3], [0, 1, 2, 3]) assert np.all(self.testInst[index, 'profiles'] == self.testInst.data['profiles'][index[0], index[1]]) @@ -284,7 +270,7 @@ def test_data_access_by_2d_tuple_indices_and_name(self): def test_data_access_bad_dimension_tuple(self): """Test raises ValueError for mismatched tuple index and data dims.""" - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) index = ([0, 1, 2, 3], [0, 1, 2, 3], [0, 1, 2, 3]) with pytest.raises(ValueError) as verr: @@ -297,7 +283,7 @@ def test_data_access_bad_dimension_tuple(self): def test_data_access_bad_dimension_for_multidim(self): """Test raises ValueError for mismatched index and data dimensions.""" - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) index = [0, 1, 2, 3] with pytest.raises(ValueError) as verr: @@ -324,7 +310,7 @@ def test_setting_partial_data_by_2d_indices_and_name(self, changed, fixed): """ - self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True) + self.testInst.load(self.ref_time.year, self.ref_doy) self.testInst['doubleProfile'] = 2. * self.testInst['profiles'] self.testInst[changed, changed, 'doubleProfile'] = 0 assert np.all(np.all(self.testInst[fixed, fixed, 'doubleProfile'] @@ -373,7 +359,7 @@ def test_set_xarray_single_value_warnings(self, val, warn_msg): warnings.simplefilter("always") - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) with warnings.catch_warnings(record=True) as self.war: self.testInst["new_val"] = val @@ -391,7 +377,7 @@ def test_set_xarray_single_value_errors(self): """ - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) self.testInst.data = self.testInst.data.assign_coords( {'preset_val': np.array([1.0, 2.0])}) @@ -413,7 +399,7 @@ def test_set_xarray_single_value_broadcast(self, new_val): """ - self.testInst.load(date=self.ref_time, use_header=True) + self.testInst.load(date=self.ref_time) self.testInst.data = self.testInst.data.assign_coords( {'preset_val': 1.0}) @@ -433,12 +419,10 @@ def setup_method(self): reload(pysat.instruments.pysat_testing) self.testInst = pysat.Instrument(platform='pysat', name='testing', - num_samples=10, - clean_level='clean', + num_samples=10, clean_level='clean', update_files=True, mangle_file_dates=True, strict_time_flag=True, - use_header=True, **self.testing_kwargs) self.ref_time = pysat.instruments.pysat_testing._test_dates[''][''] self.ref_doy = int(self.ref_time.strftime('%j')) @@ -517,11 +501,11 @@ def test_eq_different_object(self): obj1 = pysat.Instrument(platform='pysat', name='testing', num_samples=10, clean_level='clean', - update_files=True, use_header=True) + update_files=True) obj2 = pysat.Instrument(platform='pysat', name='ndtesting', num_samples=10, clean_level='clean', - update_files=True, use_header=True) + update_files=True) assert not (obj1 == obj2) return @@ -567,7 +551,7 @@ def test_instrument_labels(self): # Catch the warnings with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(use_header=True, **self.in_kwargs) + tinst = pysat.Instrument(**self.in_kwargs) self.warn_msgs = np.array(["`labels` is deprecated, use `meta_kwargs`"]) @@ -597,7 +581,7 @@ def test_instrument_meta_labels(self, use_kwargs): # Catch the warnings with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(use_header=True, **self.in_kwargs) + tinst = pysat.Instrument(**self.in_kwargs) labels = tinst.meta_labels self.warn_msgs = np.array(["Deprecated attribute, returns `meta_kwarg"]) @@ -622,7 +606,7 @@ def test_generic_meta_translator(self): # Catch the warnings with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(use_header=True, **self.in_kwargs) + tinst = pysat.Instrument(**self.in_kwargs) tinst.generic_meta_translator(tinst.meta) self.warn_msgs = np.array(["".join(["This function has been deprecated", @@ -637,8 +621,8 @@ def test_filter_netcdf4_metadata(self): # Catch the warnings with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(use_header=True, **self.in_kwargs) - tinst.load(date=self.ref_time, use_header=True) + tinst = pysat.Instrument(**self.in_kwargs) + tinst.load(date=self.ref_time) mdata_dict = tinst.meta._data.to_dict() tinst._filter_netcdf4_metadata(mdata_dict, coltype='str') @@ -659,7 +643,7 @@ def test_to_netcdf4(self): # Catch the warnings with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(use_header=True, **self.in_kwargs) + tinst = pysat.Instrument(**self.in_kwargs) try: tinst.to_netcdf4() except ValueError: @@ -686,7 +670,7 @@ def test_inst_init_with_none(self, kwargs): """ with warnings.catch_warnings(record=True) as self.war: - pysat.Instrument('pysat', 'testing', use_header=True, **kwargs) + pysat.Instrument('pysat', 'testing', **kwargs) self.warn_msgs = np.array(["".join(["The usage of None in `tag` and ", "`inst_id` has been deprecated ", @@ -697,19 +681,20 @@ def test_inst_init_with_none(self, kwargs): self.eval_warnings() return + # TODO(#1020): Remove test when keyword `use_header` is removed. def test_load_use_header(self): """Test that user is informed of MetaHeader on load.""" # Determine the expected warnings self.warn_msgs = np.array(["".join(['Meta now contains a class for ', 'global metadata (MetaHeader). ', - 'Default attachment of global ', - 'attributes to Instrument will be', - ' Deprecated in pysat 3.2.0+. Set ', - '`use_header=True` in this load ', - 'call or on Instrument ', - 'instantiation to remove this', - ' warning.'])]) + 'Allowing attachment of global ', + 'attributes to Instrument through', + ' `use_header=False` will be ', + 'Deprecated in pysat 3.3.0+. ', + 'Remove `use_header` kwarg (now ', + 'same as `use_header=True`) to ', + 'stop this warning.'])]) # Capture the warnings with warnings.catch_warnings(record=True) as self.war: diff --git a/pysat/tests/test_instrument_custom.py b/pysat/tests/test_instrument_custom.py index b7888c7ac..ab8f10caa 100644 --- a/pysat/tests/test_instrument_custom.py +++ b/pysat/tests/test_instrument_custom.py @@ -43,7 +43,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'testing', num_samples=10, clean_level='clean', - update_files=False, use_header=True) + update_files=False) self.out = '' return @@ -73,9 +73,7 @@ def setup_method(self): """Set up the unit test environment for each method.""" self.testInst = pysat.Instrument('pysat', 'testing', num_samples=10, - clean_level='clean', - update_files=True, - use_header=True) + clean_level='clean', update_files=True) self.load_date = pysat.instruments.pysat_testing._test_dates[''][''] self.testInst.load(date=self.load_date) self.custom_args = [2] @@ -236,8 +234,7 @@ def setup_method(self): """Set up the unit test environment for each method.""" self.testInst = pysat.Instrument('pysat', 'ndtesting', - num_samples=10, clean_level='clean', - use_header=True) + num_samples=10, clean_level='clean') self.load_date = pysat.instruments.pysat_ndtesting._test_dates self.load_date = self.load_date[''][''] self.testInst.load(date=self.load_date) @@ -259,8 +256,7 @@ def setup_method(self): self.testConst = pysat.Constellation(instruments=[ pysat.Instrument('pysat', 'testing', num_samples=10, - clean_level='clean', update_files=True, - use_header=True) + clean_level='clean', update_files=True) for i in range(5)]) self.load_date = pysat.instruments.pysat_testing._test_dates[''][''] self.testConst.load(date=self.load_date) @@ -371,8 +367,7 @@ def test_custom_inst_keyword_instantiation(self): {'function': mult_data, 'args': self.custom_args}] testConst2 = pysat.Constellation(instruments=[ pysat.Instrument('pysat', 'testing', num_samples=10, - clean_level='clean', custom=custom, - use_header=True) + clean_level='clean', custom=custom) for i in range(5)]) # Ensure all instruments within both constellations have the same @@ -400,7 +395,7 @@ def test_custom_const_keyword_instantiation(self): 'apply_inst': False}] testConst2 = pysat.Constellation( instruments=[pysat.Instrument('pysat', 'testing', num_samples=10, - clean_level='clean', use_header=True) + clean_level='clean') for i in range(5)], custom=custom) # Ensure both constellations have the same custom_* attributes diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py index 5c33c65b2..a36d8a68b 100644 --- a/pysat/tests/test_instrument_index.py +++ b/pysat/tests/test_instrument_index.py @@ -47,13 +47,9 @@ def test_index_error_messages(self, kwargs, msg): """ - test_inst = pysat.Instrument(platform='pysat', - name=self.name, - num_samples=10, - clean_level='clean', - update_files=True, - strict_time_flag=True, - use_header=True, + test_inst = pysat.Instrument(platform='pysat', name=self.name, + num_samples=10, clean_level='clean', + update_files=True, strict_time_flag=True, **kwargs) year, doy = pysat.utils.time.getyrdoy(self.ref_time) testing.eval_bad_input(test_inst.load, ValueError, msg, diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py index ad30f55ac..a359916b6 100644 --- a/pysat/tests/test_instrument_padding.py +++ b/pysat/tests/test_instrument_padding.py @@ -23,15 +23,11 @@ def setup_method(self): reload(pysat.instruments.pysat_testing) self.testInst = pysat.Instrument(platform='pysat', name='testing', clean_level='clean', - pad={'minutes': 5}, - update_files=True, - use_header=True) + pad={'minutes': 5}, update_files=True) self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile') self.rawInst = pysat.Instrument(platform='pysat', name='testing', - clean_level='clean', - update_files=True, - use_header=True) + clean_level='clean', update_files=True) self.rawInst.bounds = self.testInst.bounds self.delta = dt.timedelta(seconds=0) return @@ -143,23 +139,15 @@ def setup_method(self): """Set up the unit test environment for each method.""" reload(pysat.instruments.pysat_ndtesting) - self.testInst = pysat.Instrument(platform='pysat', - name='ndtesting', - num_samples=86400, - sample_rate='1s', + self.testInst = pysat.Instrument(platform='pysat', name='ndtesting', + num_samples=86400, sample_rate='1s', clean_level='clean', - pad={'minutes': 5}, - update_files=True, - use_header=True) + pad={'minutes': 5}, update_files=True) self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile') - self.rawInst = pysat.Instrument(platform='pysat', - name='ndtesting', - num_samples=86400, - sample_rate='1s', - clean_level='clean', - update_files=True, - use_header=True) + self.rawInst = pysat.Instrument(platform='pysat', name='ndtesting', + num_samples=86400, sample_rate='1s', + clean_level='clean', update_files=True) self.rawInst.bounds = self.testInst.bounds self.delta = dt.timedelta(seconds=0) return @@ -182,14 +170,11 @@ def setup_method(self): clean_level='clean', update_files=True, sim_multi_file_right=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.rawInst = pysat.Instrument(platform='pysat', name='testing', - tag='', - clean_level='clean', + tag='', clean_level='clean', update_files=True, - sim_multi_file_right=True, - use_header=True) + sim_multi_file_right=True) self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile') self.rawInst.bounds = self.testInst.bounds self.delta = dt.timedelta(seconds=0) @@ -216,16 +201,11 @@ def setup_method(self): clean_level='clean', update_files=True, sim_multi_file_right=True, - pad={'minutes': 5}, - use_header=True) - self.rawInst = pysat.Instrument(platform='pysat', - name='ndtesting', - num_samples=86400, - sample_rate='1s', - clean_level='clean', - update_files=True, - sim_multi_file_right=True, - use_header=True) + pad={'minutes': 5}) + self.rawInst = pysat.Instrument(platform='pysat', name='ndtesting', + num_samples=86400, sample_rate='1s', + clean_level='clean', update_files=True, + sim_multi_file_right=True) self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile') self.rawInst.bounds = self.testInst.bounds self.delta = dt.timedelta(seconds=0) @@ -246,16 +226,12 @@ def setup_method(self): reload(pysat.instruments.pysat_testing) self.testInst = pysat.Instrument(platform='pysat', name='testing', - clean_level='clean', - update_files=True, + clean_level='clean', update_files=True, sim_multi_file_left=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.rawInst = pysat.Instrument(platform='pysat', name='testing', - clean_level='clean', - update_files=True, - sim_multi_file_left=True, - use_header=True) + clean_level='clean', update_files=True, + sim_multi_file_left=True) self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile') self.rawInst.bounds = self.testInst.bounds self.delta = dt.timedelta(seconds=0) @@ -278,8 +254,7 @@ def setup_method(self): self.testInst = pysat.Instrument(platform='pysat', name='testing', clean_level='clean', pad={'minutes': 5}, - update_files=True, - use_header=True) + update_files=True) self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 self.delta = dt.timedelta(minutes=5) @@ -319,8 +294,7 @@ def test_data_padding_offset_instantiation(self, pad): self.testInst = pysat.Instrument(platform='pysat', name='testing', clean_level='clean', pad=pad, - update_files=True, - use_header=True) + update_files=True) self.testInst.load(self.ref_time.year, self.ref_doy, verifyPad=True) self.eval_index_start_end() return @@ -354,8 +328,7 @@ def test_padding_exceeds_load_window(self): self.testInst = pysat.Instrument(platform='pysat', name='testing', clean_level='clean', pad={'days': 2}, - update_files=True, - use_header=True) + update_files=True) testing.eval_bad_input(self.testInst.load, ValueError, 'Data padding window must be shorter than ', @@ -486,8 +459,7 @@ def setup_method(self): clean_level='clean', pad={'minutes': 5}, non_monotonic_index=True, - update_files=True, - use_header=True) + update_files=True) self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 self.delta = dt.timedelta(minutes=5) @@ -513,8 +485,7 @@ def setup_method(self): sample_rate='1s', clean_level='clean', pad={'minutes': 5}, - update_files=True, - use_header=True) + update_files=True) self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 self.delta = dt.timedelta(minutes=5) @@ -541,8 +512,7 @@ def setup_method(self): clean_level='clean', pad={'minutes': 5}, non_monotonic_index=True, - update_files=True, - use_header=True) + update_files=True) self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 self.delta = dt.timedelta(minutes=5) @@ -566,8 +536,7 @@ def setup_method(self): clean_level='clean', update_files=True, sim_multi_file_right=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.testInst.multi_file_day = True self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 @@ -593,8 +562,7 @@ def setup_method(self): update_files=True, sim_multi_file_right=True, non_monotonic_index=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.testInst.multi_file_day = True self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 @@ -622,8 +590,7 @@ def setup_method(self): clean_level='clean', update_files=True, sim_multi_file_right=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.testInst.multi_file_day = True self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 @@ -652,8 +619,7 @@ def setup_method(self): update_files=True, sim_multi_file_right=True, non_monotonic_index=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.testInst.multi_file_day = True self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 @@ -679,8 +645,7 @@ def setup_method(self): clean_level='clean', update_files=True, sim_multi_file_left=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.testInst.multi_file_day = True self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 @@ -707,8 +672,7 @@ def setup_method(self): update_files=True, sim_multi_file_left=True, non_monotonic_index=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.testInst.multi_file_day = True self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 @@ -736,8 +700,7 @@ def setup_method(self): clean_level='clean', update_files=True, sim_multi_file_left=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.testInst.multi_file_day = True self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 @@ -758,16 +721,12 @@ def setup_method(self): """Set up the unit test environment for each method.""" reload(pysat.instruments.pysat_ndtesting) - self.testInst = pysat.Instrument(platform='pysat', - name='ndtesting', - clean_level='clean', - update_files=True, - num_samples=86400, - sample_rate='1s', + self.testInst = pysat.Instrument(platform='pysat', name='ndtesting', + clean_level='clean', update_files=True, + num_samples=86400, sample_rate='1s', sim_multi_file_left=True, non_monotonic_index=True, - pad={'minutes': 5}, - use_header=True) + pad={'minutes': 5}) self.testInst.multi_file_day = True self.ref_time = dt.datetime(2009, 1, 2) self.ref_doy = 2 diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index e118d0a52..4f7f87df8 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -64,11 +64,10 @@ def test_inst_start_time(self, inst_dict, kwarg, output): _, date = cls_inst_lib.initialize_test_inst_and_date(inst_dict) if kwarg: self.test_inst = pysat.Instrument( - inst_module=inst_dict['inst_module'], start_time=kwarg, - use_header=True) + inst_module=inst_dict['inst_module'], start_time=kwarg) else: self.test_inst = pysat.Instrument( - inst_module=inst_dict['inst_module'], use_header=True) + inst_module=inst_dict['inst_module']) self.test_inst.load(date=date) @@ -92,7 +91,7 @@ def test_inst_num_samples(self, inst_dict): num = 10 _, date = cls_inst_lib.initialize_test_inst_and_date(inst_dict) self.test_inst = pysat.Instrument(inst_module=inst_dict['inst_module'], - num_samples=num, use_header=True) + num_samples=num) self.test_inst.load(date=date) assert len(self.test_inst['uts']) == num @@ -115,7 +114,7 @@ def test_inst_file_date_range(self, inst_dict): _, date = cls_inst_lib.initialize_test_inst_and_date(inst_dict) self.test_inst = pysat.Instrument(inst_module=inst_dict['inst_module'], file_date_range=file_date_range, - update_files=True, use_header=True) + update_files=True) file_list = self.test_inst.files.files assert all(file_date_range == file_list.index) @@ -134,8 +133,7 @@ def test_inst_max_latitude(self, inst_dict): """ _, date = cls_inst_lib.initialize_test_inst_and_date(inst_dict) - self.test_inst = pysat.Instrument(inst_module=inst_dict['inst_module'], - use_header=True) + self.test_inst = pysat.Instrument(inst_module=inst_dict['inst_module']) if self.test_inst.name != 'testmodel': self.test_inst.load(date=date, max_latitude=10.) assert np.all(np.abs(self.test_inst['latitude']) <= 10.) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index e5cb68c64..0bee40e27 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -67,7 +67,7 @@ class `mutable` attribute. (default=False) """ if inst_kwargs is not None: # Load the test Instrument - self.testInst = pysat.Instrument(**inst_kwargs, use_header=True) + self.testInst = pysat.Instrument(**inst_kwargs) stime = self.testInst.inst_module._test_dates[''][''] self.testInst.load(date=stime) @@ -1476,8 +1476,7 @@ class TestMetaMutable(object): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument(platform='pysat', name='testing', - use_header=True) + self.testInst = pysat.Instrument(platform='pysat', name='testing') self.testInst.load(date=self.testInst.inst_module._test_dates['']['']) self.meta = self.testInst.meta self.meta.mutable = True @@ -1612,8 +1611,7 @@ class TestToDict(object): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testing', num_samples=5, - use_header=True) + self.testInst = pysat.Instrument('pysat', 'testing', num_samples=5) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.testInst.load(date=self.stime) @@ -1676,8 +1674,7 @@ class TestToDictXarrayND(TestToDict): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'ndtesting', - num_samples=5, use_header=True) + self.testInst = pysat.Instrument('pysat', 'ndtesting', num_samples=5) self.stime = pysat.instruments.pysat_ndtesting._test_dates[''][''] self.testInst.load(date=self.stime) @@ -1693,8 +1690,7 @@ class TestToDictXarrayModel(TestToDict): def setup_method(self): """Set up the unit test environment for each method.""" - self.testInst = pysat.Instrument('pysat', 'testmodel', - num_samples=5, use_header=True) + self.testInst = pysat.Instrument('pysat', 'testmodel', num_samples=5) self.stime = pysat.instruments.pysat_testmodel._test_dates[''][''] self.testInst.load(date=self.stime) diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py index a9724553e..38a4bd0b4 100644 --- a/pysat/tests/test_methods_general.py +++ b/pysat/tests/test_methods_general.py @@ -114,7 +114,7 @@ def setup_method(self): # Load a test instrument self.testInst = pysat.Instrument('pysat', 'testing', num_samples=12, - clean_level='clean', use_header=True) + clean_level='clean') self.testInst.load(2009, 1) self.npts = len(self.testInst['uts']) return @@ -199,10 +199,8 @@ def setup_method(self): """Set up the unit test environment for each method.""" # Load a test instrument - self.testInst = pysat.Instrument('pysat', 'ndtesting', - num_samples=12, - clean_level='clean', - use_header=True) + self.testInst = pysat.Instrument('pysat', 'ndtesting', num_samples=12, + clean_level='clean') self.testInst.load(2009, 1) self.npts = len(self.testInst['uts']) return diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py index 0885032a7..5c2818037 100644 --- a/pysat/tests/test_methods_testing.py +++ b/pysat/tests/test_methods_testing.py @@ -16,7 +16,7 @@ class TestMethodsTesting(object): def setup_method(self): """Set up the unit test environment for each method.""" - self.test_inst = pysat.Instrument('pysat', 'testing', use_header=True) + self.test_inst = pysat.Instrument('pysat', 'testing') # Get list of filenames. self.fnames = [self.test_inst.files.files.values[0]] diff --git a/pysat/tests/test_orbits.py b/pysat/tests/test_orbits.py index c849d29a9..85525ffcb 100644 --- a/pysat/tests/test_orbits.py +++ b/pysat/tests/test_orbits.py @@ -140,7 +140,6 @@ def test_orbit_w_bad_orbit_info(self, info): """ self.in_kwargs['orbit_info'] = info - self.in_kwargs['use_header'] = True self.testInst = pysat.Instrument(*self.in_args, **self.in_kwargs) self.testInst.load(date=self.stime) @@ -168,7 +167,6 @@ def test_orbit_polar_w_missing_orbit_index(self, info): """ self.in_kwargs['orbit_info'] = info - self.in_kwargs['use_header'] = True self.testInst = pysat.Instrument(*self.in_args, **self.in_kwargs) # Force index to None beforee loading and iterating @@ -182,7 +180,6 @@ def test_orbit_repr(self): """Test the Orbit representation.""" self.in_kwargs['orbit_info'] = {'index': 'mlt'} - self.in_kwargs['use_header'] = True self.testInst = pysat.Instrument(*self.in_args, **self.in_kwargs) out_str = self.testInst.orbits.__repr__() @@ -193,7 +190,6 @@ def test_orbit_str(self): """Test the Orbit string representation with data.""" self.in_kwargs['orbit_info'] = {'index': 'mlt'} - self.in_kwargs['use_header'] = True self.testInst = pysat.Instrument(*self.in_args, **self.in_kwargs) self.testInst.load(date=self.stime) out_str = self.testInst.orbits.__str__() @@ -212,8 +208,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', orbit_info={'index': 'mlt'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.inc_min = 97 self.etime = None @@ -391,8 +386,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', orbit_info={'index': 'mlt'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] return @@ -592,8 +586,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'mlt'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_ndtesting._test_dates[''][''] return @@ -620,8 +613,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', orbit_info={'index': 'mlt'}, - update_files=True, - use_header=True) + update_files=True) self.testInst.bounds = (self.testInst.files.files.index[0], self.testInst.files.files.index[11], '2D', dt.timedelta(days=3)) @@ -683,8 +675,7 @@ def setup_method(self): clean_level='clean', orbit_info={'index': 'longitude', 'kind': 'longitude'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] return @@ -705,8 +696,7 @@ def setup_method(self): clean_level='clean', orbit_info={'index': 'longitude', 'kind': 'longitude'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] return @@ -727,8 +717,7 @@ def setup_method(self): clean_level='clean', orbit_info={'index': 'orbit_num', 'kind': 'orbit'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] return @@ -749,8 +738,7 @@ def setup_method(self): clean_level='clean', orbit_info={'index': 'orbit_num', 'kind': 'orbit'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_ndtesting._test_dates[''][''] return @@ -771,8 +759,7 @@ def setup_method(self): clean_level='clean', orbit_info={'index': 'latitude', 'kind': 'polar'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] return @@ -793,8 +780,7 @@ def setup_method(self): clean_level='clean', orbit_info={'index': 'latitude', 'kind': 'polar'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_ndtesting._test_dates[''][''] return @@ -835,8 +821,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', orbit_info={'index': 'mlt'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.gaps = self.stime + self.deltime self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps}) @@ -877,8 +862,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'mlt'}, - update_files=True, - use_header=True) + update_files=True) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.gaps = self.stime + self.deltime self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps}) @@ -919,8 +903,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', - orbit_info={'index': 'mlt'}, - use_header=True) + orbit_info={'index': 'mlt'}) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.testInst.custom_attach(filter_data, kwargs={'times': self.times}) return @@ -948,8 +931,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', - orbit_info={'index': 'mlt'}, - use_header=True) + orbit_info={'index': 'mlt'}) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.testInst.custom_attach(filter_data, kwargs={'times': self.times}) return @@ -970,8 +952,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', orbit_info={'index': 'longitude', - 'kind': 'longitude'}, - use_header=True) + 'kind': 'longitude'}) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.gaps = self.stime + self.deltime @@ -994,8 +975,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'longitude', - 'kind': 'longitude'}, - use_header=True) + 'kind': 'longitude'}) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.gaps = self.stime + self.deltime self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps}) @@ -1017,8 +997,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', orbit_info={'index': 'orbit_num', - 'kind': 'orbit'}, - use_header=True) + 'kind': 'orbit'}) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.gaps = self.stime + self.deltime self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps}) @@ -1040,8 +1019,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'orbit_num', - 'kind': 'orbit'}, - use_header=True) + 'kind': 'orbit'}) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.gaps = self.stime + self.deltime self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps}) @@ -1063,8 +1041,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', orbit_info={'index': 'latitude', - 'kind': 'polar'}, - use_header=True) + 'kind': 'polar'}) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.gaps = self.stime + self.deltime @@ -1087,8 +1064,7 @@ def setup_method(self): self.testInst = pysat.Instrument('pysat', 'ndtesting', clean_level='clean', orbit_info={'index': 'latitude', - 'kind': 'polar'}, - use_header=True) + 'kind': 'polar'}) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.gaps = self.stime + self.deltime diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index a6869bf4c..b88c961f6 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -50,7 +50,7 @@ def test_update_fill_values_numbers(self, name, variables): """ # Initalize the instrument - inst = pysat.Instrument('pysat', name, use_header=True) + inst = pysat.Instrument('pysat', name) inst.load(date=self.ref_time) # Ensure there are fill values to check @@ -82,7 +82,7 @@ def test_update_fill_values_by_type(self, name): """ # Initalize the instrument - inst = pysat.Instrument('pysat', name, use_header=True) + inst = pysat.Instrument('pysat', name) inst.load(date=self.ref_time) # Ensure there are fill values to check diff --git a/pysat/tests/test_utils_coords.py b/pysat/tests/test_utils_coords.py index 8edd4b113..d63077cc2 100644 --- a/pysat/tests/test_utils_coords.py +++ b/pysat/tests/test_utils_coords.py @@ -67,8 +67,7 @@ def teardown_method(self): def test_update_longitude(self, name): """Test `update_longitude` successful run.""" - self.py_inst = pysat.Instrument(platform='pysat', name=name, - use_header=True) + self.py_inst = pysat.Instrument(platform='pysat', name=name) self.py_inst.load(date=self.inst_time) # Test instruments initially define longitude between 0-360 deg @@ -85,8 +84,7 @@ def test_update_longitude(self, name): def test_bad_lon_name_update_longitude(self): """Test update_longitude with a bad longitude name.""" - self.py_inst = pysat.Instrument(platform='pysat', name="testing", - use_header=True) + self.py_inst = pysat.Instrument(platform='pysat', name="testing") self.py_inst.load(date=self.inst_time) testing.eval_bad_input(coords.update_longitude, ValueError, @@ -124,7 +122,7 @@ def test_calc_solar_local_time(self, name): # Instantiate instrument and load data self.py_inst = pysat.Instrument(platform='pysat', name=name, - num_samples=1, use_header=True) + num_samples=1) self.py_inst.load(date=self.inst_time) coords.calc_solar_local_time(self.py_inst, lon_name="longitude", @@ -148,7 +146,7 @@ def test_calc_solar_local_time_inconsistent_keywords(self, name, caplog): # Instantiate instrument and load data self.py_inst = pysat.Instrument(platform='pysat', name=name, - num_samples=1, use_header=True) + num_samples=1) self.py_inst.load(date=self.inst_time) # Apply solar local time method and capture logging output @@ -167,8 +165,7 @@ def test_calc_solar_local_time_w_neg_longitude(self): """Test calc_solar_local_time with longitudes from -180 to 180 deg.""" # Instantiate instrument and load data - self.py_inst = pysat.Instrument(platform='pysat', name="testing", - use_header=True) + self.py_inst = pysat.Instrument(platform='pysat', name="testing") self.py_inst.load(date=self.inst_time) coords.calc_solar_local_time(self.py_inst, lon_name="longitude", @@ -186,8 +183,7 @@ def test_bad_lon_name_calc_solar_local_time(self): """Test raises ValueError with a bad longitude name.""" # Instantiate instrument and load data - self.py_inst = pysat.Instrument(platform='pysat', name="testing", - use_header=True) + self.py_inst = pysat.Instrument(platform='pysat', name="testing") self.py_inst.load(date=self.inst_time) # Test that the correct Exception and error message are raised @@ -203,8 +199,7 @@ def test_lon_broadcasting_calc_solar_local_time(self, name): """Test calc_solar_local_time with longitude coordinates.""" # Instantiate instrument and load data - self.py_inst = pysat.Instrument(platform='pysat', name=name, - use_header=True) + self.py_inst = pysat.Instrument(platform='pysat', name=name) self.py_inst.load(date=self.inst_time) coords.calc_solar_local_time(self.py_inst, lon_name="longitude", slt_name='slt') @@ -219,8 +214,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod_multiday(self, name): """Test non modulated solar local time output for a 2 day range.""" # Instantiate instrument and load data - self.py_inst = pysat.Instrument(platform='pysat', name=name, - use_header=True) + self.py_inst = pysat.Instrument(platform='pysat', name=name) self.py_inst.load(date=self.inst_time, end_date=self.inst_time + dt.timedelta(days=2)) coords.calc_solar_local_time(self.py_inst, lon_name="longitude", @@ -237,8 +231,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod_ref_date(self, name): """Test non modulated SLT output for a 2 day range with a ref date.""" # Instantiate instrument and load data - self.py_inst = pysat.Instrument(platform='pysat', name=name, - use_header=True) + self.py_inst = pysat.Instrument(platform='pysat', name=name) self.py_inst.load(date=self.inst_time, end_date=self.inst_time + dt.timedelta(days=2)) coords.calc_solar_local_time(self.py_inst, lon_name="longitude", @@ -257,8 +250,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod(self, name): """Test SLT calc with longitude coordinates and no modulus.""" # Instantiate instrument and load data - self.py_inst = pysat.Instrument(platform='pysat', name=name, - use_header=True) + self.py_inst = pysat.Instrument(platform='pysat', name=name) self.py_inst.load(date=self.inst_time) coords.calc_solar_local_time(self.py_inst, lon_name="longitude", slt_name='slt', apply_modulus=False) @@ -273,8 +265,7 @@ def test_single_lon_calc_solar_local_time(self): """Test calc_solar_local_time with a single longitude value.""" # Instantiate instrument and load data - self.py_inst = pysat.Instrument(platform='pysat', name="ndtesting", - use_header=True) + self.py_inst = pysat.Instrument(platform='pysat', name="ndtesting") self.py_inst.load(date=self.inst_time) lon_name = 'lon2' @@ -362,7 +353,7 @@ class TestExpandXarrayDims(object): def setup_method(self): """Set up the unit test environment.""" self.test_inst = pysat.Instrument( - inst_module=pysat.instruments.pysat_ndtesting, use_header=True) + inst_module=pysat.instruments.pysat_ndtesting) self.start_time = pysat.instruments.pysat_ndtesting._test_dates[''][''] self.data_list = [] self.out = None @@ -397,12 +388,12 @@ def set_data_meta(self, dims_equal): # Load a second data set with half the time samples self.test_inst = pysat.Instrument( inst_module=self.test_inst.inst_module, - num_samples=num_samples, use_header=True) + num_samples=num_samples) else: # Load a second data set with different dimensions apart from time self.test_inst = pysat.Instrument( inst_module=pysat.instruments.pysat_testmodel, - num_samples=num_samples, use_header=True) + num_samples=num_samples) self.test_inst.load(date=self.start_time + dt.timedelta(days=1)) self.data_list.append(self.test_inst.data) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 87ba21683..b8dc67667 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -152,15 +152,13 @@ def setup_method(self): self.insts_kwargs = [] # Data by day, ACE SIS data - self.insts.append(pysat.Instrument('ace', 'sis', tag='historic', - use_header=True)) + self.insts.append(pysat.Instrument('ace', 'sis', tag='historic')) test_dates = pysatSpaceWeather.instruments.ace_sis._test_dates self.insts_dates.append([test_dates['']['historic']] * 2) self.insts_kwargs.append({}) # Data with date mangling, regular F10.7 data, stored monthly - self.insts.append(pysat.Instrument('sw', 'f107', tag='historic', - use_header=True)) + self.insts.append(pysat.Instrument('sw', 'f107', tag='historic')) test_dates = pysatSpaceWeather.instruments.sw_f107._test_dates self.insts_dates.append([test_dates['']['historic'], test_dates['']['historic'] @@ -265,7 +263,7 @@ def test_updating_directories(self, capsys): # Refresh inst with the old directory template set to get now 'old' # path information. inst2 = pysat.Instrument(inst.platform, inst.name, tag=inst.tag, - inst_id=inst.inst_id, use_header=True) + inst_id=inst.inst_id) # Check that directories with simpler platform org were NOT removed. assert os.path.isdir(inst2.files.data_path) @@ -326,7 +324,7 @@ def setup_method(self): self.testInst = pysat.Instrument( inst_module=pysat.instruments.pysat_testing, clean_level='clean', - update_files=True, use_header=True) + update_files=True) # Create instrument directories in tempdir pysat.utils.files.check_and_make_path(self.testInst.files.data_path) diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 14b9a5c94..210be40ea 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -61,8 +61,7 @@ def setup_method(self): pysat.params['data_dirs'] = self.tempdir.name self.testInst = pysat.Instrument(platform='pysat', name='testing', - num_samples=100, update_files=True, - use_header=True) + num_samples=100, update_files=True) self.stime = pysat.instruments.pysat_testing._test_dates[''][''] self.epoch_name = 'time' @@ -114,7 +113,7 @@ def test_basic_write_and_read_netcdf_mixed_case_data_format(self): """Test basic netCDF4 read/write with mixed case data variables.""" # Create a bunch of files by year and doy outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) # Modify data names in data if self.testInst.pandas_format: @@ -157,7 +156,7 @@ def test_basic_write_and_read_netcdf_mixed_case_meta_format(self): """Test basic netCDF4 read/write with mixed case metadata variables.""" # Create a bunch of files by year and doy outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) # Modify data and metadata names in data self.testInst.meta.rename(str.upper) @@ -200,7 +199,7 @@ def test_basic_write_and_read_netcdf_export_pysat_info(self, kwargs, """ # Create a bunch of files by year and doy outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) io.inst_to_netcdf(self.testInst, fname=outfile, preserve_meta_case=True, epoch_name=default_epoch_name, **kwargs) @@ -235,7 +234,7 @@ def test_inst_write_and_read_netcdf(self, add_path): 'pysat_test_ncdf_%Y%j.nc')) # Load and write the test instrument data - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) self.testInst.to_netcdf4(fname=outfile, epoch_name=default_epoch_name) # Load the written file directly into an Instrument @@ -244,7 +243,7 @@ def test_inst_write_and_read_netcdf(self, add_path): netcdf_inst = pysat.Instrument( 'pysat', 'netcdf', data_dir=file_path, update_files=True, file_format=file_root, pandas_format=self.testInst.pandas_format, - use_header=True, epoch_name=default_epoch_name, **tkwargs) + epoch_name=default_epoch_name, **tkwargs) # Confirm data path is correct assert os.path.normpath(netcdf_inst.files.data_path) \ @@ -253,7 +252,7 @@ def test_inst_write_and_read_netcdf(self, add_path): # Deleting the test file here via os.remove(...) does work # Load data - netcdf_inst.load(date=self.stime, use_header=True) + netcdf_inst.load(date=self.stime) # Test the loaded Instrument data self.loaded_inst = netcdf_inst.data @@ -316,7 +315,7 @@ def test_write_netcdf4_duplicate_variable_names(self): # Create a bunch of files by year and doy outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) self.testInst['MLT'] = 1 # Evaluate the expected error and message @@ -348,7 +347,7 @@ def test_read_netcdf4_bad_epoch_name(self, write_epoch, err_msg, err_type): # Load data outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) # Write file io.inst_to_netcdf(self.testInst, fname=outfile, epoch_name=write_epoch) @@ -392,7 +391,7 @@ def test_read_netcdf4_epoch_not_xarray_dimension(self, caplog, write_epoch, # Load data outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) # Write file io.inst_to_netcdf(self.testInst, outfile, epoch_name=write_epoch) @@ -430,7 +429,7 @@ def test_write_and_read_netcdf4_w_kwargs(self, wkwargs, lkwargs): # Create a new file based on loaded test data outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) if 'epoch_name' not in wkwargs.keys(): wkwargs['epoch_name'] = default_epoch_name io.inst_to_netcdf(self.testInst, fname=outfile, **wkwargs) @@ -471,7 +470,7 @@ def test_read_netcdf4_w_epoch_kwargs(self, kwargs, target): # Create a bunch of files by year and doy outfile = os.path.join(self.tempdir.name, 'pysat_{:}_ncdf.nc'.format(self.testInst.name)) - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) io.inst_to_netcdf(self.testInst, fname=outfile, epoch_name=default_epoch_name) @@ -523,7 +522,7 @@ def test_read_netcdf4_w_epoch_kwargs(self, kwargs, target): def test_netcdf_prevent_attribute_override(self): """Test that attributes will not be overridden by default.""" - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) # Test that `bespoke` attribute is initially missing assert not hasattr(self.testInst, 'bespoke') @@ -543,7 +542,7 @@ def test_netcdf_prevent_attribute_override(self): def test_netcdf_attribute_override(self): """Test that attributes in the netCDF file may be overridden.""" - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) self.testInst.meta.mutable = True self.testInst.meta.bespoke = True @@ -583,7 +582,7 @@ def test_decode_times(self, decode_times): # Create a file outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) io.inst_to_netcdf(self.testInst, fname=outfile, epoch_name=default_epoch_name) @@ -633,7 +632,7 @@ def test_drop_labels(self, drop_labels): # Create a file with additional metadata outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) # Add additional metadata self.testInst.meta['mlt'] = {drop_label: 1.} @@ -687,8 +686,7 @@ def setup_method(self): self.testInst = pysat.Instrument(platform='pysat', name='ndtesting', - update_files=True, num_samples=100, - use_header=True) + update_files=True, num_samples=100) self.stime = pysat.instruments.pysat_ndtesting._test_dates[ ''][''] self.epoch_name = 'time' @@ -737,7 +735,7 @@ def test_read_netcdf4_with_time_meta_labels(self, kwargs, target): # Prepare output test data outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) # Modify the variable attributes directly before writing to file self.testInst.meta['uts'] = {'units': 'seconds'} @@ -767,7 +765,7 @@ def test_load_netcdf_pandas_2d_error(self): # Create a bunch of files by year and doy outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.testInst.load(date=self.stime, use_header=True) + self.testInst.load(date=self.stime) io.inst_to_netcdf(self.testInst, fname=outfile) # Evaluate the error raised and the expected message @@ -801,10 +799,8 @@ def setup_method(self): # Create an instrument object that has a meta with some # variables allowed to be nan within metadata when exporting. - self.testInst = pysat.Instrument('pysat', 'testing', num_samples=5, - use_header=True) - self.testInst.load(date=self.testInst.inst_module._test_dates[''][''], - use_header=True) + self.testInst = pysat.Instrument('pysat', 'testing', num_samples=5) + self.testInst.load(date=self.testInst.inst_module._test_dates['']['']) self.pformat = self.testInst.pandas_format return @@ -1256,9 +1252,8 @@ def setup_method(self): # Create an instrument object that has a meta with some # variables allowed to be nan within metadata when exporting. self.testInst = pysat.Instrument('pysat', 'ndtesting', - num_samples=5, use_header=True) - self.testInst.load(date=self.testInst.inst_module._test_dates[''][''], - use_header=True) + num_samples=5) + self.testInst.load(date=self.testInst.inst_module._test_dates['']['']) self.pformat = self.testInst.pandas_format return @@ -1272,10 +1267,8 @@ def setup_method(self): # Create an instrument object that has a meta with some # variables allowed to be nan within metadata when exporting. - self.testInst = pysat.Instrument('pysat', 'testmodel', num_samples=5, - use_header=True) - self.testInst.load(date=self.testInst.inst_module._test_dates[''][''], - use_header=True) + self.testInst = pysat.Instrument('pysat', 'testmodel', num_samples=5) + self.testInst.load(date=self.testInst.inst_module._test_dates['']['']) self.pformat = self.testInst.pandas_format return @@ -1289,10 +1282,8 @@ def setup_method(self): # Create an instrument object that has a meta with some # variables allowed to be nan within metadata when exporting. - self.testInst = pysat.Instrument('pysat', 'ndtesting', - num_samples=5, use_header=True) - self.testInst.load(date=self.testInst.inst_module._test_dates[''][''], - use_header=True) + self.testInst = pysat.Instrument('pysat', 'ndtesting', num_samples=5) + self.testInst.load(date=self.testInst.inst_module._test_dates['']['']) self.epoch_name = 'time' return @@ -1411,8 +1402,7 @@ class TestMetaTranslation(object): def setup_method(self): """Create test environment.""" - self.test_inst = pysat.Instrument('pysat', 'testing', num_samples=5, - use_header=True) + self.test_inst = pysat.Instrument('pysat', 'testing', num_samples=5) self.test_date = pysat.instruments.pysat_testing._test_dates[''][''] self.test_inst.load(date=self.test_date) self.meta_dict = self.test_inst.meta.to_dict() @@ -1694,7 +1684,7 @@ def setup_method(self): """Create test environment.""" self.test_inst = pysat.Instrument('pysat', 'ndtesting', - num_samples=5, use_header=True) + num_samples=5) self.test_date = pysat.instruments.pysat_ndtesting._test_dates[''][''] self.test_inst.load(date=self.test_date) self.meta_dict = self.test_inst.meta.to_dict() @@ -1716,8 +1706,7 @@ class TestMetaTranslationModel(TestMetaTranslation): def setup_method(self): """Create test environment.""" - self.test_inst = pysat.Instrument('pysat', 'testmodel', - num_samples=5, use_header=True) + self.test_inst = pysat.Instrument('pysat', 'testmodel', num_samples=5) self.test_date = pysat.instruments.pysat_testmodel._test_dates[''][''] self.test_inst.load(date=self.test_date) self.meta_dict = self.test_inst.meta.to_dict() @@ -1782,8 +1771,7 @@ def test_load_netcdf_labels(self, inst_name, load_func): # Create a test file testInst = pysat.Instrument(platform='pysat', name=inst_name, - num_samples=100, update_files=True, - use_header=True) + num_samples=100, update_files=True) testInst.load(date=testInst.inst_module._test_dates['']['']) io.inst_to_netcdf(testInst, fname=self.outfile) From 2f4f07e21a8536faaaf05254796c2c4eca89bc54 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 17 Oct 2023 10:23:24 -0400 Subject: [PATCH 181/365] DOC: updated changelog Added a description of the changes to the changelog. --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7afe9f3b..30fe2f368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated `Constellation.to_inst` method definition of coords, using dims to combine common dimensions instead. * Implement pyproject to manage metadata -* Updated docstring references to `pysat.utils.files` in other modules. + * Updated docstring references to `pysat.utils.files` in other modules. * Remove Sphinx cap * Add pandas cap * Update usage of whitespace and if statements (E275) @@ -42,6 +42,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed deprecated `convert_timestamp_to_datetime` function * Removed deprecated `_test_download_travis` flag * Removed deprecated `freq` kwarg from `download` + * Removed deprecated `use_header` kwarg from `load` and changed default + behaviour to `use_header=True` [3.1.0] - 2023-05-31 -------------------- From 9ca8e683621b4bbfb69663703775ab08a6f1efa8 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 17 Oct 2023 16:47:02 -0400 Subject: [PATCH 182/365] BUG: `use_header` can be used at init Make sure the `use_header` kwarg is allowed for `load` at initialization. --- pysat/_instrument.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 4ffa28a1d..111322b25 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -333,8 +333,7 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', # Expand the dict to include method keywords for load. # TODO(#1020): Remove this if statement when `use_header` is removed if fkey == 'load': - meth = getattr(self, fkey) - default_kwargs.update(_get_supported_keywords(meth)) + default_kwargs['use_header'] = True # Confirm there are no reserved keywords present for kwarg in kwargs.keys(): From 6c1d784af4ab547304aff2527a53a48310da8c66 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 18 Oct 2023 10:21:46 -0400 Subject: [PATCH 183/365] STY: removed duplication in parsing Reduced duplication in file parsing code, ensuring both routines have the same type of output in the same situations. --- pysat/utils/files.py | 189 +++++++++++++++++++++++++++++-------------- 1 file changed, 127 insertions(+), 62 deletions(-) diff --git a/pysat/utils/files.py b/pysat/utils/files.py index ce0c18727..ed473b5fd 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -126,6 +126,122 @@ def process_parsed_filenames(stored, two_digit_year_break=None): return pds.Series(None, dtype='object') +def init_parse_filenames(files, format_str): + """Initalize the output for the file parsing functions. + + Parameters + ---------- + files : list + List of files, typically provided by + `pysat.utils.files.search_local_system_formatted_filename`. + format_str : str + Provides the naming pattern of the instrument files and the + locations of date information so an ordered list may be produced. + Supports all provided string formatting codes though only 'year', + 'month', 'day', 'hour', 'minute', 'second', 'version', 'revision', + and 'cycle' will be used for time and sorting information. For example, + `instrument-{year:4d}_{month:02d}-{day:02d}_v{version:02d}.cdf`, or + `*-{year:4d}_{month:02d}hithere{day:02d}_v{version:02d}.cdf` + + Returns + ------- + stored : collections.OrderedDict + Information parsed from filenames that includes: 'year', 'month', 'day', + 'hour', 'minute', 'second', 'version', 'revision', and 'cycle', as + well as any other user provided template variables. Also + includes `files`, an input list of files, and `format_str`. + search_dict : dict + An output dict with the following keys: + - 'search_string' (format_str with data to be parsed replaced with ?) + - 'keys' (keys for data to be parsed) + - 'lengths' (string length for data to be parsed) + - 'string_blocks' (the filenames are broken into fixed width segments). + + See Also + -------- + pysat.utils.files.parse_fixed_width_filenames + pysat.utils.files.parse_delimited_filenames + pysat.utils.files.construct_searchstring_from_format + + """ + # Create storage for data to be parsed from filenames + ordered_keys = ['year', 'month', 'day', 'hour', 'minute', 'second', + 'version', 'revision', 'cycle'] + stored = collections.OrderedDict({kk: None for kk in ordered_keys}) + + # Only define search dictionary if there are files to search + if len(files) == 0: + # Include keys that should only be added at the end, if there are no + # files to process + stored['format_str'] = format_str + stored['files'] = [] + search_dict = dict() + else: + # Parse format string to get information needed to parse filenames + search_dict = construct_searchstring_from_format(format_str, + wildcard=False) + + # Add non-standard keys + for key in search_dict['keys']: + if key not in stored: + stored[key] = None + + return stored, search_dict + + +def finish_parse_filenames(stored, files, format_str): + """Reshape and finalize the output for the file parsing functions. + + Parameters + ---------- + stored : collections.OrderedDict + Information parsed from filenames that includes: 'year', 'month', 'day', + 'hour', 'minute', 'second', 'version', 'revision', and 'cycle', as + well as any other user provided template variables. + files : list + List of files, typically provided by + `pysat.utils.files.search_local_system_formatted_filename`. + format_str : str + Provides the naming pattern of the instrument files and the + locations of date information so an ordered list may be produced. + Supports all provided string formatting codes though only 'year', + 'month', 'day', 'hour', 'minute', 'second', 'version', 'revision', + and 'cycle' will be used for time and sorting information. For example, + `instrument-{year:4d}_{month:02d}-{day:02d}_v{version:02d}.cdf`, or + `*-{year:4d}_{month:02d}hithere{day:02d}_v{version:02d}.cdf` + + Returns + ------- + stored : collections.OrderedDict + Information parsed from filenames that includes: 'year', 'month', 'day', + 'hour', 'minute', 'second', 'version', 'revision', and 'cycle', as + well as any other user provided template variables. Also + includes `files`, an input list of files, and `format_str`. + + See Also + -------- + pysat.utils.files.parse_fixed_width_filenames + pysat.utils.files.parse_delimited_filenames + + """ + + # Convert to numpy arrays + for key in stored.keys(): + if stored[key] is not None: + try: + # Assume key value is numeric integer + stored[key] = np.array(stored[key]).astype(np.int64) + except ValueError: + # Store key value as string + stored[key] = np.array(stored[key]) + + # Include files and file format in output + stored['files'] = files + stored['format_str'] = format_str + + return stored + + def parse_fixed_width_filenames(files, format_str): """Extract specified info from a list of files with a fixed name width. @@ -165,24 +281,11 @@ def parse_fixed_width_filenames(files, format_str): """ # Create storage for data to be parsed from filenames - ordered_keys = ['year', 'month', 'day', 'hour', 'minute', 'second', - 'version', 'revision', 'cycle'] - stored = collections.OrderedDict({kk: list() for kk in ordered_keys}) + stored, search_dict = init_parse_filenames(files, format_str) if len(files) == 0: - stored['files'] = [] - # Include format string as convenience for later functions - stored['format_str'] = format_str return stored - # Parse format string to get information needed to parse filenames - search_dict = construct_searchstring_from_format(format_str) - - # Add non-standard keys - for key in search_dict['keys']: - if key not in stored: - stored[key] = [] - # Determine the locations the date/version information in a filename is # stored and use these indices to slice out date from filenames. idx = 0 @@ -208,25 +311,15 @@ def parse_fixed_width_filenames(files, format_str): val = temp[key_str_idx[0][j]:] else: val = temp[key_str_idx[0][j]:key_str_idx[1][j]] - stored[key].append(val) - - # Convert to numpy arrays - for key in stored.keys(): - if len(stored[key]) == 0: - stored[key] = None - else: - try: - # Assume key value is numeric integer - stored[key] = np.array(stored[key]).astype(np.int64) - except ValueError: - # Store key value as string - stored[key] = np.array(stored[key]) - # Include files in output - stored['files'] = files + # Save the parsed variable for this key and file + if stored[key] is None: + stored[key] = [val] + else: + stored[key].append(val) - # Include format string as convenience for later functions - stored['format_str'] = format_str + # Convert to numpy arrays and add additional information to output + stored = finish_parse_filenames(stored, files, format_str) return stored @@ -281,26 +374,11 @@ def parse_delimited_filenames(files, format_str, delimiter): """ # Create storage for data to be parsed from filenames - ordered_keys = ['year', 'month', 'day', 'hour', 'minute', 'second', - 'version', 'revision', 'cycle'] - stored = collections.OrderedDict({kk: None for kk in ordered_keys}) + stored, search_dict = init_parse_filenames(files, format_str) - # Exit early if there are no files if len(files) == 0: - stored['files'] = [] - - # Include format string as convenience for later functions - stored['format_str'] = format_str return stored - # Parse format string to get information needed to parse filenames - search_dict = construct_searchstring_from_format(format_str, wildcard=False) - - # Add non-standard keys - for key in search_dict['keys']: - if key not in stored: - stored[key] = None - # Going to parse the string on the delimiter. It is possible that other # regions have the delimiter but aren't going to be parsed out. # Reconstruct string from `snips` and use `{}` in place of `keys` and @@ -361,21 +439,8 @@ def parse_delimited_filenames(files, format_str, delimiter): loop_split_idx = loop_split_idx[j + 1:] break - # Convert to numpy arrays - for key in stored.keys(): - if stored[key] is not None: - try: - # Assume key value is numeric integer - stored[key] = np.array(stored[key]).astype(np.int64) - except ValueError: - # Store key value as string - stored[key] = np.array(stored[key]) - - # Include files in output - stored['files'] = files - - # Include format string as convenience for later functions - stored['format_str'] = format_str + # Convert to numpy arrays and add additional information to output + stored = finish_parse_filenames(stored, files, format_str) return stored From 87abad2932870058f387cae8dd54c3585b1fdb76 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 18 Oct 2023 10:22:42 -0400 Subject: [PATCH 184/365] TST: added file parsing unit tests Added unit tests for `parse_fixed_width_filenames` and the new supporting functions created to reduce code duplication. --- pysat/tests/test_utils_files.py | 137 ++++++++++++++++++++++++++++++-- 1 file changed, 132 insertions(+), 5 deletions(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 87ba21683..f63a2d647 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -21,8 +21,8 @@ from pysat.utils import testing -class TestParseDelimitedFilenames(object): - """Unit tests for the `parse_delimited_filename` function.""" +class TestParseFilenames(object): + """Unit tests for the file parsing functions.""" def setup_method(self): """Set up the unit test environment for each method.""" @@ -46,7 +46,7 @@ def teardown_method(self): del self.fkwargs, self.file_dict, self.kw_format del self.temporary_file_list - def eval_parse_delimited_filename(self): + def eval_parsed_filenames(self): """Evaluate the output of a `parse_delimited_filename` unit test. Returns @@ -106,7 +106,7 @@ def test_parse_delimited_filename(self, sep_char, flead, good_kwargs): sep_char) # Test each of the return values - assert self.eval_parse_delimited_filename() + assert self.eval_parsed_filenames() return def test_parse_delimited_filename_empty(self): @@ -121,7 +121,134 @@ def test_parse_delimited_filename_empty(self): self.file_dict = futils.parse_delimited_filenames([], fname, sep_char) # Test each of the return values - assert self.eval_parse_delimited_filename() + assert self.eval_parsed_filenames() + return + + @pytest.mark.parametrize("sep_char,flead,good_kwargs", [ + ("_", "*_", ['year', 'month', 'day', 'hour', 'minute', 'version']), + ('?', "test", ['year', 'day', 'hour', 'minute', 'second', 'cycle', + 'revision']), ('fun', '*', [])]) + def test_parse_fixed_filename(self, sep_char, flead, good_kwargs): + """Check ability to parse list of fixed width files. + + Parameters + ---------- + sep_char : str + Separation character to use in joining the filename + flead : str + File prefix + good_kwargs : list + List of kwargs to include in the file format + + """ + # Format the test input + fname = '{:s}{:s}.cdf'.format(flead, sep_char.join( + [self.kw_format[fkey] for fkey in good_kwargs])) + + # Adjust the test input/comparison data for this run + bad_kwargs = [fkey for fkey in self.fkwargs[0] + if fkey not in good_kwargs] + + for kwargs in self.fkwargs: + for fkey in bad_kwargs: + del kwargs[fkey] + + # Create the input file list + file_list = [fname.format(**kwargs) for kwargs in self.fkwargs] + + # Get the test results + self.file_dict = futils.parse_fixed_width_filenames(file_list, fname) + + # Test each of the return values + assert self.eval_parsed_filenames() + return + + def test_parse_fixed_width_filename_empty(self): + """Check ability to parse list of fixed-width files with no files.""" + # Format the test input + fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}', + '{minute:02d}', '{second:02d}', '{cycle:2s}.txt')) + self.fkwargs = [] + + # Get the test results + self.file_dict = futils.parse_fixed_width_filenames([], fname) + + # Test each of the return values + assert self.eval_parsed_filenames() + return + + def test_init_parse_filename_empty(self): + """Check the `init_parse_filenames` output with no files.""" + # Format the test input + fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}', + '{minute:02d}', '{second:02d}', '{cycle:2s}.txt')) + self.fkwargs = [] + + # Get the test results + self.file_dict, sdict = futils.init_parse_filenames([], fname) + + # Test each of the return values + assert self.eval_parsed_filenames() + assert len(sdict.keys()) == 0, "Search dict was defined unnecessarily" + return + + def test_init_parse_filename_with_files(self): + """Check the `init_parse_filenames` output with files.""" + # Format the test input + fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}', + '{minute:02d}', '{second:02d}', '{cycle:2s}.txt')) + + # Create the input file list + file_list = [fname.format(**kwargs) for kwargs in self.fkwargs] + + # Get the test results + self.file_dict, sdict = futils.init_parse_filenames(file_list, fname) + + # Test the initalized dictionaries + testing.assert_lists_equal(['search_string', 'keys', 'lengths', + 'string_blocks'], list(sdict.keys())) + + for skey in sdict['keys']: + assert skey in self.file_dict.keys(), "Missing key {:}".format(skey) + + for fkey in self.file_dict.keys(): + assert self.file_dict[fkey] is None, "File dict not initalized" + + assert "files" not in self.file_dict.keys(), "'files' key set early" + assert "format_str" not in self.file_dict.keys(), \ + "'format_str' key set early" + return + + def test_finish_parsed_filenames(self): + """Test output restucturing for `finish_parsed_filenames`.""" + # Format the test input + fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}', + '{minute:02d}', '{second:02d}', '{cycle:2s}.txt')) + + # Create the input file list and dict + file_list = [fname.format(**kwargs) for kwargs in self.fkwargs] + self.file_dict = {'int': [1], 'none': None, 'float': [1.0], + 'str': ['hi']} + + # Get the test results + self.file_dict = futils.finish_parse_filenames(self.file_dict, + file_list, fname) + + # Test the output + for fkey in self.file_dict: + if fkey == 'none': + assert self.file_dict[fkey] is None + elif fkey == 'files': + testing.assert_lists_equal(file_list, self.file_dict[fkey]) + elif fkey == 'format_str': + assert fname == self.file_dict[fkey] + else: + testing.assert_isinstance(self.file_dict[fkey], np.ndarray) + + if fkey == 'str': + testing.assert_isinstance(self.file_dict[fkey][0], str) + else: + testing.assert_isinstance(self.file_dict[fkey][0], np.int64) return From 2ad73cba048c797ce1d41166b083fa4f01324ad2 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 18 Oct 2023 10:30:18 -0400 Subject: [PATCH 185/365] STY: renamed support functions Renamed the support functions to have hidden names, in order to not clutter the file utility namespace. --- pysat/tests/test_utils_files.py | 14 +- pysat/utils/files.py | 226 ++++++++++++++++---------------- 2 files changed, 122 insertions(+), 118 deletions(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index f63a2d647..2c1cfb552 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -178,14 +178,14 @@ def test_parse_fixed_width_filename_empty(self): return def test_init_parse_filename_empty(self): - """Check the `init_parse_filenames` output with no files.""" + """Check the `_init_parse_filenames` output with no files.""" # Format the test input fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}', '{minute:02d}', '{second:02d}', '{cycle:2s}.txt')) self.fkwargs = [] # Get the test results - self.file_dict, sdict = futils.init_parse_filenames([], fname) + self.file_dict, sdict = futils._init_parse_filenames([], fname) # Test each of the return values assert self.eval_parsed_filenames() @@ -193,7 +193,7 @@ def test_init_parse_filename_empty(self): return def test_init_parse_filename_with_files(self): - """Check the `init_parse_filenames` output with files.""" + """Check the `_init_parse_filenames` output with files.""" # Format the test input fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}', '{minute:02d}', '{second:02d}', '{cycle:2s}.txt')) @@ -202,7 +202,7 @@ def test_init_parse_filename_with_files(self): file_list = [fname.format(**kwargs) for kwargs in self.fkwargs] # Get the test results - self.file_dict, sdict = futils.init_parse_filenames(file_list, fname) + self.file_dict, sdict = futils._init_parse_filenames(file_list, fname) # Test the initalized dictionaries testing.assert_lists_equal(['search_string', 'keys', 'lengths', @@ -220,7 +220,7 @@ def test_init_parse_filename_with_files(self): return def test_finish_parsed_filenames(self): - """Test output restucturing for `finish_parsed_filenames`.""" + """Test output restucturing for `_finish_parsed_filenames`.""" # Format the test input fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}', '{minute:02d}', '{second:02d}', '{cycle:2s}.txt')) @@ -231,8 +231,8 @@ def test_finish_parsed_filenames(self): 'str': ['hi']} # Get the test results - self.file_dict = futils.finish_parse_filenames(self.file_dict, - file_list, fname) + self.file_dict = futils._finish_parse_filenames(self.file_dict, + file_list, fname) # Test the output for fkey in self.file_dict: diff --git a/pysat/utils/files.py b/pysat/utils/files.py index ed473b5fd..3e5ebd49b 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -19,114 +19,10 @@ from pysat.utils._core import listify from pysat.utils.time import create_datetime_index +# Define hidden support functions -def process_parsed_filenames(stored, two_digit_year_break=None): - """Create a Files pandas Series of filenames from a formatted dict. - - Parameters - ---------- - stored : collections.orderedDict - Ordered dictionary produced by `parse_fixed_width_filenames` or - `parse_delimited_filenames`, containing date, time, version, and - other information extracted from the filenames. - two_digit_year_break : int or NoneType - If filenames only store two digits for the year, then - '1900' will be added for years >= two_digit_year_break - and '2000' will be added for years < two_digit_year_break. - If None, then four-digit years are assumed. (default=None) - - Returns - ------- - pds.Series - Series, indexed by datetime, with file strings - - Note - ---- - If two files have the same date and time information in the filename then - the file with the higher version/revision/cycle is used. Series returned - only has one file per datetime. Version is required for this filtering, - revision and cycle are optional. - - See Also - -------- - pysat.utils.files.parse_fixed_width_filenames, - pysat.utils.files.parse_delimited_filenames - - """ - - search_dict = construct_searchstring_from_format(stored['format_str']) - keys = search_dict['keys'] - - if len(stored['files']) > 0: - # Deal with the possibility of two digit years. Years above - # or equal to break are considered to be 1900+, while - # years below break are considered to be 2000+ - if two_digit_year_break is not None: - idx, = np.where(np.array(stored['year']) - >= two_digit_year_break) - stored['year'][idx] = stored['year'][idx] + 1900 - idx, = np.where(np.array(stored['year']) < two_digit_year_break) - stored['year'][idx] = stored['year'][idx] + 2000 - - # Need to sort the information for things to work - rec_arr = [stored[key] for key in keys] - rec_arr.append(stored['files']) - - # Sort all arrays by creating a sortable records array - # withs keys corresponding to the files - val_keys = keys + ['files'] - rec_arr = np.rec.fromarrays(rec_arr, names=val_keys) - rec_arr.sort(order=val_keys, axis=0) - - # Pull out sorted info - for key in keys: - stored[key] = rec_arr[key] - files = rec_arr['files'] - # Add hour and minute information to 'second' - if stored['second'] is None: - stored['second'] = np.zeros(len(files)) - if stored['hour'] is not None: - stored['second'] += 3600 * stored['hour'] - if stored['minute'] is not None: - stored['second'] += 60 * stored['minute'] - - # The version shouldn't be set to zero, it is required to remove - # duplicate datetimes - if stored['revision'] is None: - stored['revision'] = np.zeros(len(files)) - if stored['cycle'] is None: - stored['cycle'] = np.zeros(len(files)) - - index = create_datetime_index(year=stored['year'], - month=stored['month'], - day=stored['day'], - uts=stored['second']) - - # If version, revision, and cycle are supplied, use these parameters - # to weed out files that have been replaced with updated versions. - # First, check for duplicate index times - dups = index[index.duplicated()].unique() - if (len(dups) > 0) and (stored['version'] is not None): - # We have duplicates, keep the highest version/revision combo - version = pds.Series(stored['version'], index=index) - revision = pds.Series(stored['revision'], index=index) - cycle = pds.Series(stored['cycle'], index=index) - revive = version * 100000. + revision + cycle * 1e-5 - frame = pds.DataFrame({'files': files, 'revive': revive, - 'time': index}, index=index) - frame = frame.sort_values(by=['time', 'revive'], - ascending=[True, False]) - frame = frame.drop_duplicates(subset='time', keep='first') - - return frame['files'] - else: - return pds.Series(files, index=index).sort_index() - else: - return pds.Series(None, dtype='object') - - -def init_parse_filenames(files, format_str): +def _init_parse_filenames(files, format_str): """Initalize the output for the file parsing functions. Parameters @@ -189,7 +85,7 @@ def init_parse_filenames(files, format_str): return stored, search_dict -def finish_parse_filenames(stored, files, format_str): +def _finish_parse_filenames(stored, files, format_str): """Reshape and finalize the output for the file parsing functions. Parameters @@ -242,6 +138,114 @@ def finish_parse_filenames(stored, files, format_str): return stored +# Define file utility functions + +def process_parsed_filenames(stored, two_digit_year_break=None): + """Create a Files pandas Series of filenames from a formatted dict. + + Parameters + ---------- + stored : collections.orderedDict + Ordered dictionary produced by `parse_fixed_width_filenames` or + `parse_delimited_filenames`, containing date, time, version, and + other information extracted from the filenames. + two_digit_year_break : int or NoneType + If filenames only store two digits for the year, then + '1900' will be added for years >= two_digit_year_break + and '2000' will be added for years < two_digit_year_break. + If None, then four-digit years are assumed. (default=None) + + Returns + ------- + pds.Series + Series, indexed by datetime, with file strings + + Note + ---- + If two files have the same date and time information in the filename then + the file with the higher version/revision/cycle is used. Series returned + only has one file per datetime. Version is required for this filtering, + revision and cycle are optional. + + See Also + -------- + pysat.utils.files.parse_fixed_width_filenames, + pysat.utils.files.parse_delimited_filenames + + """ + + search_dict = construct_searchstring_from_format(stored['format_str']) + keys = search_dict['keys'] + + if len(stored['files']) > 0: + # Deal with the possibility of two digit years. Years above + # or equal to break are considered to be 1900+, while + # years below break are considered to be 2000+ + if two_digit_year_break is not None: + idx, = np.where(np.array(stored['year']) + >= two_digit_year_break) + stored['year'][idx] = stored['year'][idx] + 1900 + idx, = np.where(np.array(stored['year']) < two_digit_year_break) + stored['year'][idx] = stored['year'][idx] + 2000 + + # Need to sort the information for things to work + rec_arr = [stored[key] for key in keys] + rec_arr.append(stored['files']) + + # Sort all arrays by creating a sortable records array + # withs keys corresponding to the files + val_keys = keys + ['files'] + rec_arr = np.rec.fromarrays(rec_arr, names=val_keys) + rec_arr.sort(order=val_keys, axis=0) + + # Pull out sorted info + for key in keys: + stored[key] = rec_arr[key] + files = rec_arr['files'] + + # Add hour and minute information to 'second' + if stored['second'] is None: + stored['second'] = np.zeros(len(files)) + if stored['hour'] is not None: + stored['second'] += 3600 * stored['hour'] + if stored['minute'] is not None: + stored['second'] += 60 * stored['minute'] + + # The version shouldn't be set to zero, it is required to remove + # duplicate datetimes + if stored['revision'] is None: + stored['revision'] = np.zeros(len(files)) + if stored['cycle'] is None: + stored['cycle'] = np.zeros(len(files)) + + index = create_datetime_index(year=stored['year'], + month=stored['month'], + day=stored['day'], + uts=stored['second']) + + # If version, revision, and cycle are supplied, use these parameters + # to weed out files that have been replaced with updated versions. + # First, check for duplicate index times + dups = index[index.duplicated()].unique() + if (len(dups) > 0) and (stored['version'] is not None): + # We have duplicates, keep the highest version/revision combo + version = pds.Series(stored['version'], index=index) + revision = pds.Series(stored['revision'], index=index) + cycle = pds.Series(stored['cycle'], index=index) + revive = version * 100000. + revision + cycle * 1e-5 + frame = pds.DataFrame({'files': files, 'revive': revive, + 'time': index}, index=index) + frame = frame.sort_values(by=['time', 'revive'], + ascending=[True, False]) + frame = frame.drop_duplicates(subset='time', keep='first') + + return frame['files'] + else: + return pds.Series(files, index=index).sort_index() + else: + return pds.Series(None, dtype='object') + + def parse_fixed_width_filenames(files, format_str): """Extract specified info from a list of files with a fixed name width. @@ -281,7 +285,7 @@ def parse_fixed_width_filenames(files, format_str): """ # Create storage for data to be parsed from filenames - stored, search_dict = init_parse_filenames(files, format_str) + stored, search_dict = _init_parse_filenames(files, format_str) if len(files) == 0: return stored @@ -319,7 +323,7 @@ def parse_fixed_width_filenames(files, format_str): stored[key].append(val) # Convert to numpy arrays and add additional information to output - stored = finish_parse_filenames(stored, files, format_str) + stored = _finish_parse_filenames(stored, files, format_str) return stored @@ -374,7 +378,7 @@ def parse_delimited_filenames(files, format_str, delimiter): """ # Create storage for data to be parsed from filenames - stored, search_dict = init_parse_filenames(files, format_str) + stored, search_dict = _init_parse_filenames(files, format_str) if len(files) == 0: return stored @@ -440,7 +444,7 @@ def parse_delimited_filenames(files, format_str, delimiter): break # Convert to numpy arrays and add additional information to output - stored = finish_parse_filenames(stored, files, format_str) + stored = _finish_parse_filenames(stored, files, format_str) return stored From 99755802963589c710cba1c2abf292318200ddae Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 18 Oct 2023 10:32:23 -0400 Subject: [PATCH 186/365] DOC: updated changelog Added a summary of the bug fixes, tests, and file restructuring to the changelog. --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7afe9f3b..b3e982414 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,14 +21,19 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added options to customize `pysat_ndtesting` instrument with sample rate, shift in time. * Added orbit number to `pysat_ndtesting`. - * Added the overwrite kwarg to `utils.registry.register_by_module` + * Added the overwrite kwarg to `utils.registry.register_by_module`. + * Added unit tests for all file parsing functions in `utils.files`. + * Reduced code duplication in the `utils.files.parse_fixed_width_filenames` + and `utils.files.parse_delimited_filenames` functions +* Bug Fix + * Fixed `utils.files.parse_fixed_width_filenames` output for empty file list * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. * Updated `Constellation.to_inst` method definition of coords, using dims to combine common dimensions instead. * Implement pyproject to manage metadata -* Updated docstring references to `pysat.utils.files` in other modules. + * Updated docstring references to `pysat.utils.files` in other modules. * Remove Sphinx cap * Add pandas cap * Update usage of whitespace and if statements (E275) From 11c949c3e8f969830d2ea0bec8a27527678cf2c3 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 18 Oct 2023 10:32:43 -0400 Subject: [PATCH 187/365] DOC: udpated docstrings Updated an incomplete test docstring. --- pysat/tests/test_utils_files.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 2c1cfb552..38b4deecb 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -85,7 +85,18 @@ def eval_parsed_filenames(self): ('-', "test", ['year', 'day', 'hour', 'minute', 'second', 'cycle', 'revision']), ('fun', 'test', [])]) def test_parse_delimited_filename(self, sep_char, flead, good_kwargs): - """Check ability to parse list of delimited files.""" + """Check ability to parse list of delimited files. + + Parameters + ---------- + sep_char : str + Separation character to use in joining the filename + flead : str + File prefix + good_kwargs : list + List of kwargs to include in the file format + + """ # Format the test input fname = '{:s}{:s}.cdf'.format(flead, sep_char.join( [self.kw_format[fkey] for fkey in good_kwargs])) From 449db5ba568344df4f58f5f561ea467653a8231b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 18 Oct 2023 10:38:31 -0400 Subject: [PATCH 188/365] DOC: update wording in docstring Parser didn't recognize imperative mood. --- pysat/utils/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/utils/files.py b/pysat/utils/files.py index 3e5ebd49b..01177aff7 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -23,7 +23,7 @@ def _init_parse_filenames(files, format_str): - """Initalize the output for the file parsing functions. + """Set the initial output for the file parsing functions. Parameters ---------- From 5dc4572369bf4c60ae7e0a22a91108525517dd0b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 18 Oct 2023 16:40:11 -0400 Subject: [PATCH 189/365] TST: added `process_parsed_filenames` unit tests Added unit tests for the `process_parsed_filenames` function. Also improved parsing evaluation method to assert a minimum number of keys and improve coverage results. --- pysat/tests/test_utils_files.py | 142 ++++++++++++++++++++++++++++---- 1 file changed, 126 insertions(+), 16 deletions(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 0b62a497d..5a2b2535b 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -6,6 +6,7 @@ """Tests the `pysat.utils.files` functions.""" import datetime as dt +from collections import OrderedDict from importlib import reload import numpy as np import os @@ -47,17 +48,9 @@ def teardown_method(self): del self.temporary_file_list def eval_parsed_filenames(self): - """Evaluate the output of a `parse_delimited_filename` unit test. - - Returns - ------- - bool - True if there is data to evalute, False if data dict is empty - - """ + """Evaluate the output of a `parse_delimited_filename` unit test.""" # Evaluate the returned data dict - if len(self.file_dict.keys()) < 2: - return False + assert len(self.file_dict.keys()) >= 2, "insufficient keys in file dict" # Extract the test lists if len(self.fkwargs) > 0: @@ -78,7 +71,7 @@ def eval_parsed_filenames(self): assert self.file_dict[fkey] is None, \ "unused format key has a value" - return True + return @pytest.mark.parametrize("sep_char,flead,good_kwargs", [ ("_", "test_", ['year', 'month', 'day', 'hour', 'minute', 'version']), @@ -117,7 +110,7 @@ def test_parse_delimited_filename(self, sep_char, flead, good_kwargs): sep_char) # Test each of the return values - assert self.eval_parsed_filenames() + self.eval_parsed_filenames() return def test_parse_delimited_filename_empty(self): @@ -132,7 +125,7 @@ def test_parse_delimited_filename_empty(self): self.file_dict = futils.parse_delimited_filenames([], fname, sep_char) # Test each of the return values - assert self.eval_parsed_filenames() + self.eval_parsed_filenames() return @pytest.mark.parametrize("sep_char,flead,good_kwargs", [ @@ -171,7 +164,7 @@ def test_parse_fixed_filename(self, sep_char, flead, good_kwargs): self.file_dict = futils.parse_fixed_width_filenames(file_list, fname) # Test each of the return values - assert self.eval_parsed_filenames() + self.eval_parsed_filenames() return def test_parse_fixed_width_filename_empty(self): @@ -185,7 +178,7 @@ def test_parse_fixed_width_filename_empty(self): self.file_dict = futils.parse_fixed_width_filenames([], fname) # Test each of the return values - assert self.eval_parsed_filenames() + self.eval_parsed_filenames() return def test_init_parse_filename_empty(self): @@ -199,7 +192,7 @@ def test_init_parse_filename_empty(self): self.file_dict, sdict = futils._init_parse_filenames([], fname) # Test each of the return values - assert self.eval_parsed_filenames() + self.eval_parsed_filenames() assert len(sdict.keys()) == 0, "Search dict was defined unnecessarily" return @@ -263,6 +256,123 @@ def test_finish_parsed_filenames(self): return +class TestProcessParsedFilenames(object): + """Unit tests for `process_parsed_filenames` function.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.stored = OrderedDict({'year': np.full(shape=3, fill_value=2001), + 'month': np.full(shape=3, fill_value=2), + 'day': np.ones(shape=3, dtype=np.int64), + 'hour': np.zeros(shape=3, dtype=np.int64), + 'minute': np.zeros(shape=3, dtype=np.int64), + 'second': np.arange(0, 3, 1), + 'version': np.arange(0, 3, 1), + 'revision': np.arange(3, 0, -1), + 'cycle': np.array([1, 3, 2])}) + self.format_str = "_".join(["test", "{year:04d}", "{month:02d}", + "{day:02d}", "{hour:02d}", "{minute:02d}", + "{second:02d}", "v{version:02d}", + "r{revision:02d}", "c{cycle:02d}.cdf"]) + + return + + def teardown_method(self): + """Clean up the unit test environment for each method.""" + + del self.stored, self.format_str + return + + def complete_stored(self): + """Add the 'files' and 'format_str' kwargs to the `stored` dict.""" + + file_list = [] + for ind in range(len(self.stored['year'])): + ind_dict = {skey: self.stored[skey][ind] + for skey in self.stored.keys()} + file_list.append(self.format_str.format(**ind_dict)) + + self.stored['files'] = file_list + self.stored['format_str'] = self.format_str + return + + @pytest.mark.parametrize("year_break", [0, 50]) + def test_two_digit_years(self, year_break): + """Test the results of using different year breaks for YY formats.""" + # Complete the ordered dict of file information + self.stored['year'] -= 2000 + self.format_str = self.format_str.replace('year:04', 'year:02') + self.complete_stored() + + # Get the file series + series = futils.process_parsed_filenames( + self.stored, two_digit_year_break=year_break) + + # Test the series year + test_year = series.index.max().year + century = 1900 if year_break == 0 else 2000 + assert test_year - century < 100, "year break caused wrong century" + + # Test that the series length is correct and all filenames are unique + assert series.shape == self.stored['year'].shape + assert np.unique(series.values).shape == self.stored['year'].shape + return + + def test_version_selection(self): + """Test version selection dominates when time is the same.""" + # Complete the ordered dict of file information + self.stored['second'] = np.zeros(shape=self.stored['year'].shape, + dtype=np.int64) + self.complete_stored() + + # Get the file series + series = futils.process_parsed_filenames(self.stored) + + # Ensure there is only one file and that it has the highest version + ver_num = "v{:02d}".format(self.stored['version'].max()) + assert series.shape == (1, ) + assert series.values[0].find(ver_num) > 0 + return + + def test_revision_selection(self): + """Test revision selection dominates after time and version.""" + # Complete the ordered dict of file information + self.stored['second'] = np.zeros(shape=self.stored['year'].shape, + dtype=np.int64) + self.stored['version'] = np.zeros(shape=self.stored['year'].shape, + dtype=np.int64) + self.complete_stored() + + # Get the file series + series = futils.process_parsed_filenames(self.stored) + + # Ensure there is only one file and that it has the highest version + rev_num = "r{:02d}".format(self.stored['revision'].max()) + assert series.shape == (1, ) + assert series.values[0].find(rev_num) > 0 + return + + def test_cycle_selection(self): + """Test cycle selection dominates after time, version, and revision.""" + # Complete the ordered dict of file information + self.stored['second'] = np.zeros(shape=self.stored['year'].shape, + dtype=np.int64) + self.stored['version'] = np.zeros(shape=self.stored['year'].shape, + dtype=np.int64) + self.stored['revision'] = np.zeros(shape=self.stored['year'].shape, + dtype=np.int64) + self.complete_stored() + + # Get the file series + series = futils.process_parsed_filenames(self.stored) + + # Ensure there is only one file and that it has the highest version + cyc_num = "c{:02d}".format(self.stored['cycle'].max()) + assert series.shape == (1, ) + assert series.values[0].find(cyc_num) > 0 + return + + class TestFileDirectoryTranslations(CICleanSetup): """Unit tests for file directory setup.""" From 95d4646fbcf9c2b86209f8af4f0959dbe02be67d Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 18 Oct 2023 17:20:16 -0400 Subject: [PATCH 190/365] DOC: add missing default values Add the missing default values to the docstring. Also improve the error message to be more informative. --- pysat/utils/files.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pysat/utils/files.py b/pysat/utils/files.py index 01177aff7..93f3b376e 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -19,8 +19,8 @@ from pysat.utils._core import listify from pysat.utils.time import create_datetime_index -# Define hidden support functions +# Define hidden support functions def _init_parse_filenames(files, format_str): """Set the initial output for the file parsing functions. @@ -464,7 +464,7 @@ def construct_searchstring_from_format(format_str, wildcard=False): `instrument_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf` wildcard : bool If True, replaces each '?' sequence that would normally - be returned with a single '*'. + be returned with a single '*'. (default=False) Returns ------- @@ -531,10 +531,10 @@ def construct_searchstring_from_format(format_str, wildcard=False): out_dict['search_string'] += '*' break else: - estr = ''.join(["Couldn't determine formatting width. ", - "This may be due to the use of unsupported ", - "wildcard characters."]) - raise ValueError(estr) + raise ValueError( + ''.join(["Couldn't determine formatting width, check ", + "formatting length specification (e.g., ", + "{day:03d} for day of year)."])) return out_dict From 411d63fc38d6702c34b2073a3e0bdc58736a77ea Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 18 Oct 2023 17:20:54 -0400 Subject: [PATCH 191/365] TST: add searchstring unit tests Added unit tests for the `construct_searchstring_from_format` file utility function. --- pysat/tests/test_utils_files.py | 124 ++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 5a2b2535b..8fae3eaf0 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -22,6 +22,130 @@ from pysat.utils import testing +class TestConstructSearchstring(object): + """Unit tests for the `construct_searchstring_from_format` function.""" + + def setup_method(self): + """Set up the unit test environment for each method.""" + self.out_dict = {} + self.num_fmt = None + self.str_len = None + self.fill_len = None + return + + def teardown_method(self): + """Clean up the unit test environment after each method.""" + + del self.out_dict, self.num_fmt, self.str_len, self.fill_len + return + + def eval_output(self): + """Evaluate the output dictionary.""" + + testing.assert_lists_equal(['search_string', 'keys', 'lengths', + 'string_blocks'], + list(self.out_dict.keys())) + + assert len(self.out_dict['keys']) == self.num_fmt + assert len(''.join(self.out_dict['string_blocks'])) == self.str_len + assert sum(self.out_dict['lengths']) == self.fill_len + + if self.out_dict['search_string'].find('*') < 0: + assert len( + self.out_dict['search_string']) == self.fill_len + self.str_len + else: + assert len( + self.out_dict['search_string']) <= self.fill_len + self.str_len + return + + @pytest.mark.parametrize("format_str,nfmt,slen,flen", [ + ("", 0, 0, 0), ("test", 0, 4, 0), ("{year:02d}{month:02d}", 2, 0, 4), + ("test_{year:04d}.ext", 1, 9, 4)]) + def test_searchstring_success(self, format_str, nfmt, slen, flen): + """Test successful construction of a searchable string. + + Parameters + ---------- + format_str : str + The naming pattern of the instrument files and the locations of + date/version/revision/cycle information needed to create an ordered + list. + nfmt : int + Number of formatting options included in the format string + slen : int + Length of non-formatted string segments + flen : int + Length of formatted segments + + """ + # Set the evaluation criteria + self.num_fmt = nfmt + self.str_len = slen + self.fill_len = flen + + # Get the streachstring dictionary + self.out_dict = futils.construct_searchstring_from_format(format_str) + + # Evaluate the output + self.eval_output() + return + + @pytest.mark.parametrize("format_str,nfmt,slen,flen, nwc", [ + ("", 0, 0, 0, 0), ("test", 0, 4, 0, 0), + ("{year:02d}{month:02d}", 2, 0, 4, 2), + ("test_{year:04d}_{month:02d}.ext", 2, 10, 6, 2)]) + def test_searchstring_w_wildcard(self, format_str, nfmt, slen, flen, nwc): + """Test successful construction of a searchable string with wildcards. + + Parameters + ---------- + format_str : str + The naming pattern of the instrument files and the locations of + date/version/revision/cycle information needed to create an ordered + list. + nfmt : int + Number of formatting options included in the format string + slen : int + Length of non-formatted string segments + flen : int + Length of formatted segments + nwc : int + Number of wildcard (*) symbols + + """ + # Set the evaluation criteria + self.num_fmt = nfmt + self.str_len = slen + self.fill_len = flen + + # Get the streachstring dictionary + self.out_dict = futils.construct_searchstring_from_format(format_str, + True) + + # Evaluate the output + self.eval_output() + assert len(self.out_dict['search_string'].split('*')) == nwc + 1 + return + + def test_searchstring_noformat(self): + """Test failure if the input argument is NoneType.""" + + testing.eval_bad_input(futils.construct_searchstring_from_format, + ValueError, + 'Must supply a filename template (format_str).', + input_args=[None]) + return + + def test_searchstring_bad_wildcard(self): + """Test failure if unsupported wildcard use is encountered.""" + + testing.eval_bad_input(futils.construct_searchstring_from_format, + ValueError, + "Couldn't determine formatting width, check", + input_args=["test{year:02d}{month:d}.txt"]) + return + + class TestParseFilenames(object): """Unit tests for the file parsing functions.""" From c19fa8518990667703e08a161db6827528be0ffd Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 19 Oct 2023 14:16:06 -0400 Subject: [PATCH 192/365] BUG: fixed bug in loading with pad Fixed a bug in loading with a pad, where an empty slice could be attempted. --- pysat/_instrument.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 111322b25..7e3318fa5 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1307,7 +1307,7 @@ def _index(self, data=None): Returns ------- - pds.Series + index : pds.Series Series containing the time indices for the Instrument data """ @@ -1316,15 +1316,17 @@ def _index(self, data=None): data = self.data if self.pandas_format: - return data.index + index = data.index else: epoch_names = self._get_epoch_name_from_data(data=data) if len(epoch_names) == 0: - return pds.Index([]) + index = pds.Index([]) else: # Xarray preferred epoch name order is opposite - return data.indexes[epoch_names[-1]] + index = data.indexes[epoch_names[-1]] + + return index def _pass_method(*args, **kwargs): """Empty default method for updatable Instrument methods.""" @@ -3301,11 +3303,10 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, if not self._empty(ndata): # Test the data index, slicing if necessary nindex = self._index(data=ndata) - if len(nindex) > 0: + if len(nindex) > 1: if nindex[0] == self.index[-1]: ndata = self.__getitem__( - slice(1, len(ndata[self.index.name])), - data=ndata) + slice(1, len(nindex)), data=ndata) cdata.append(ndata) if include is None: include = 0 From dc285e64fcd427f5403f0d4f77e494b9b4d3a660 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 19 Oct 2023 14:17:01 -0400 Subject: [PATCH 193/365] ENH: add index name to CSV loading If there is no time index name when loading CSV data into a pandas DataFrame, include the default name 'Epoch'. --- pysat/instruments/methods/general.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py index 5dee2195b..ef0e0d9a4 100644 --- a/pysat/instruments/methods/general.py +++ b/pysat/instruments/methods/general.py @@ -256,5 +256,12 @@ def load_csv_data(fnames, read_csv_kwargs=None): for fname in fnames: fdata.append(pds.read_csv(fname, **read_csv_kwargs)) - data = pds.DataFrame() if len(fdata) == 0 else pds.concat(fdata, axis=0) + if len(fdata) == 0: + data = pds.DataFrame() + else: + data = pds.concat(fdata, axis=0) + + if data.index.name is None: + data.index.name = "Epoch" + return data From 2dcd4345e6f82ae898535517db0cf7c6d03996b7 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 19 Oct 2023 14:18:41 -0400 Subject: [PATCH 194/365] TST: fixed bug in padding test Fixed a bug in the padding test, so that datasets that only load data before their load date (e.g., some forecast/nowcast files) are allowed to produce empty datasets after the time selection that happens with padding data. --- pysat/tests/classes/cls_instrument_library.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 030500179..316f2fdde 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -562,17 +562,25 @@ def test_load_w_pad(self, pad, inst_dict): load_and_set_strict_time_flag(test_inst, date, raise_error=True, clean_off=False) - assert not test_inst.empty - - # Evaluate the data index length - assert (test_inst.index[-1] - - test_inst.index[0]).total_seconds() < 86400.0 - - # Evaluate the recorded pad - inst_str = test_inst.__str__() - assert inst_str.find( - 'Data Padding: {:s}'.format(pad_repr)) > 0, "".join([ - "bad pad value: ", pad_repr, " not in ", inst_str]) + if test_inst.empty: + # This will be empty if this is a forecast file that doesn't + # include the load date + test_inst.pad = None + load_and_set_strict_time_flag(test_inst, date, raise_error=True, + clean_off=False) + assert not test_inst.empty, "No data on {:}".format(date) + assert test_inst.index.max() < date, \ + "Padding should have left data and didn't" + else: + # Padding was successful, evaluate the data index length + assert (test_inst.index[-1] + - test_inst.index[0]).total_seconds() < 86400.0 + + # Evaluate the recorded pad + inst_str = test_inst.__str__() + assert inst_str.find( + 'Data Padding: {:s}'.format(pad_repr)) > 0, "".join([ + "bad pad value: ", pad_repr, " not in ", inst_str]) else: pytest.skip("Download data not available") From 9e0a910750ae1adef2bf390d89d2b18c354caccb Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 19 Oct 2023 18:10:51 -0400 Subject: [PATCH 195/365] TST: added tests for general methods Added tests for uncovered lines in the general methods submodule. --- pysat/tests/test_methods_general.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py index 38a4bd0b4..3c3202c72 100644 --- a/pysat/tests/test_methods_general.py +++ b/pysat/tests/test_methods_general.py @@ -11,6 +11,15 @@ from pysat.utils import testing +def test_filename_creator(): + """Test the `filename_creator` placeholder.""" + + testing.eval_bad_input(gen.filename_creator, NotImplementedError, + 'This feature has not been implemented yet', + input_args=[0.0]) + return + + class TestListFiles(object): """Unit tests for `pysat.instrument.methods.general.list_files`.""" @@ -268,7 +277,7 @@ def eval_data_cols(self): return def test_load_single_file(self): - """Test the CVS data load with a single file.""" + """Test the CSV data load with a single file.""" self.data = gen.load_csv_data(self.csv_file) assert isinstance(self.data.index, pds.RangeIndex) @@ -277,7 +286,7 @@ def test_load_single_file(self): return def test_load_file_list(self): - """Test the CVS data load with multiple files.""" + """Test the CSV data load with multiple files.""" self.data = gen.load_csv_data([self.csv_file, self.csv_file]) assert self.data.index.dtype == 'int64' @@ -286,7 +295,7 @@ def test_load_file_list(self): return def test_load_file_with_kwargs(self): - """Test the CVS data load with kwargs.""" + """Test the CSV data load with kwargs.""" self.data = gen.load_csv_data([self.csv_file], read_csv_kwargs={"parse_dates": True, @@ -295,3 +304,12 @@ def test_load_file_with_kwargs(self): self.eval_data_cols() assert len(self.data.columns) == len(self.data_cols) return + + def test_load_empty_filelist(self): + """Test the CSV data loading with an empty file list.""" + + self.data = gen.load_csv_data([]) + + # Evaluate the empty output + assert self.data.empty + return From 868cee259a6bf89a86905e58c6b3d73c34bd4c67 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 19 Oct 2023 18:13:48 -0400 Subject: [PATCH 196/365] STY: fixed indent Fixed the indentation to be correct. --- pysat/tests/test_methods_general.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py index 3c3202c72..3272b5d63 100644 --- a/pysat/tests/test_methods_general.py +++ b/pysat/tests/test_methods_general.py @@ -15,8 +15,8 @@ def test_filename_creator(): """Test the `filename_creator` placeholder.""" testing.eval_bad_input(gen.filename_creator, NotImplementedError, - 'This feature has not been implemented yet', - input_args=[0.0]) + 'This feature has not been implemented yet', + input_args=[0.0]) return From 58f50f133ad464a8bd05803a81279bbe0df881c2 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 23 Oct 2023 18:06:35 -0400 Subject: [PATCH 197/365] TST: added file utils unit tests Added file utility unit tests for the `search_local_system_formatted_filename` and `update_data_directory_structure` functions. --- pysat/tests/test_utils_files.py | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 8fae3eaf0..ff3269218 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -709,6 +709,40 @@ def teardown_method(self): del self.testInst, self.out, self.tempdir, self.start, self.stop return + def test_updating_directories_no_registration(self, capsys): + """Test directory structure update method without registered insts.""" + # New Template + templ = '{platform}' + + # Convert directories to simpler platform structure, to get output + futils.update_data_directory_structure(new_template=templ, + full_breakdown=True) + + # Capture printouts and test the results + captured = capsys.readouterr() + assert captured.find("No registered instruments detected.") >= 0, \ + "Expected output not captured in: {:}".format(captured) + return + + def test_search_local_system_formatted_filename(self): + """Test `search_local_system_formatted_filename` success.""" + # Create a temporary file with a unique, searchable name + prefix = "test_me" + suffix = "tstfile" + searchstr = "*".join([prefix, suffix]) + with tempfile.NamedTemporaryFile( + dir=self.testInst.files.data_path, prefix=prefix, + suffix=suffix) as temp: + files = futils.search_local_system_formatted_filename( + self.testInst.files.data_path, searchstr) + + assert len(files) == 1, "unexpected number of files in search results" + assert files[0].find( + prefix) >= 0, "unexpected file prefix in search results" + assert files[0].find( + suffix) > 0, "unexpected file extension in search results" + return + def test_get_file_information(self): """Test `utils.files.get_file_information` success with existing files. From 262accfcbb7a2c65dd0cb83385de823f69be92c3 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 23 Oct 2023 18:06:59 -0400 Subject: [PATCH 198/365] DOC: improved changelog description Updated the changelog description to be more correct. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9de1d907..942d2eca5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). shift in time. * Added orbit number to `pysat_ndtesting`. * Added the overwrite kwarg to `utils.registry.register_by_module`. - * Added unit tests for all file parsing functions in `utils.files`. + * Added unit tests for all functions in `utils.files`. * Reduced code duplication in the `utils.files.parse_fixed_width_filenames` and `utils.files.parse_delimited_filenames` functions * Bug Fix From 30e0314b366825eaf7945e62f97668e7dbe930a8 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 23 Oct 2023 18:12:09 -0400 Subject: [PATCH 199/365] STY: removed unused variable Removed an unused variable definition. --- pysat/tests/test_utils_files.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index ff3269218..de5069304 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -730,9 +730,8 @@ def test_search_local_system_formatted_filename(self): prefix = "test_me" suffix = "tstfile" searchstr = "*".join([prefix, suffix]) - with tempfile.NamedTemporaryFile( - dir=self.testInst.files.data_path, prefix=prefix, - suffix=suffix) as temp: + with tempfile.NamedTemporaryFile(dir=self.testInst.files.data_path, + prefix=prefix, suffix=suffix): files = futils.search_local_system_formatted_filename( self.testInst.files.data_path, searchstr) From a64e4e4ba27f7b29e68e68430a635a194f13906c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 23 Oct 2023 18:14:37 -0400 Subject: [PATCH 200/365] STY: changed import order Changed import order to make flake8 happy. --- pysat/tests/test_utils_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index de5069304..54c59c5d3 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -6,9 +6,9 @@ """Tests the `pysat.utils.files` functions.""" import datetime as dt -from collections import OrderedDict from importlib import reload import numpy as np +from collections import OrderedDict import os import pandas as pds import platform From 3664b6112a0a72fbca758cd4e61d281ef30beedb Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 23 Oct 2023 18:20:46 -0400 Subject: [PATCH 201/365] STY: import order Attempt two at fixing the import order. --- pysat/tests/test_utils_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 54c59c5d3..240f601cf 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -5,10 +5,10 @@ # ---------------------------------------------------------------------------- """Tests the `pysat.utils.files` functions.""" +from collections import OrderedDict import datetime as dt from importlib import reload import numpy as np -from collections import OrderedDict import os import pandas as pds import platform From ad1d1b6824503e02c15cdf018dad0343ee353bd1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 24 Oct 2023 10:50:22 -0400 Subject: [PATCH 202/365] TST: fixed bug in capsys Fixed bug in capsys call to get STDOUT. Also added skip for local systems, as test is not expected to pass there. --- pysat/tests/test_utils_files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 240f601cf..48772ec38 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -709,19 +709,19 @@ def teardown_method(self): del self.testInst, self.out, self.tempdir, self.start, self.stop return + @pytest.mark.skipif(os.environ.get('CI') != 'true', reason="CI test only") def test_updating_directories_no_registration(self, capsys): """Test directory structure update method without registered insts.""" - # New Template - templ = '{platform}' - # Convert directories to simpler platform structure, to get output + templ = '{platform}' futils.update_data_directory_structure(new_template=templ, full_breakdown=True) # Capture printouts and test the results captured = capsys.readouterr() - assert captured.find("No registered instruments detected.") >= 0, \ - "Expected output not captured in: {:}".format(captured) + captxt = captured.out + assert captxt.find("No registered instruments detected.") >= 0, \ + "Expected output not captured in STDOUT: {:}".format(captxt) return def test_search_local_system_formatted_filename(self): From 8398f284d37d8cf977505ef6902c009e11a020aa Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 26 Oct 2023 15:05:02 -0400 Subject: [PATCH 203/365] ENH: allow direct assignment to meta Allow direct assignment of data to Meta (e.g., meta['var', 'label'] = value). --- pysat/_meta.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index 4501300f9..5850dbb4d 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -318,13 +318,17 @@ def __setitem__(self, data_vars, input_dat): Parameters ---------- - data_vars : str, list + data_vars : str, list, tuple Data variable names for the input metadata input_dat : dict, pds.Series, or Meta Input metadata to be assigned - """ + Raises + ------ + ValueError + For unexpected input type that does not allow metadata to be set. + """ input_data = deepcopy(input_dat) if isinstance(input_data, dict): @@ -454,6 +458,14 @@ def __setitem__(self, data_vars, input_dat): # Outputs from Meta object are a Series. Thus, this takes in input # from a Meta object. Set data using standard assignment via a dict. self[data_vars] = input_data.to_dict() + else: + # The input data is a value, this only works if `data_vars` is + # a tuple that contains the data variable and the metadata label + if isinstance(data_vars, tuple) and len(data_vars) == 2: + self[data_vars[0]] = {data_vars[1]: input_data} + else: + raise ValueError( + "unexpected input combination, can't set metadata") return From ff28aa8c6832971e275afabb2736b6f24545ffc3 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 26 Oct 2023 15:05:20 -0400 Subject: [PATCH 204/365] DOC: updated docs Added a direct assignment example to the docs. --- docs/tutorial/tutorial_basics.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorial/tutorial_basics.rst b/docs/tutorial/tutorial_basics.rst index 59ab6d4a0..1e1311801 100644 --- a/docs/tutorial/tutorial_basics.rst +++ b/docs/tutorial/tutorial_basics.rst @@ -439,13 +439,13 @@ Explorer `(ICON) `_. # Retrieve units using general labels dmsp.meta['ti', dmsp.meta.labels.units] - # Update units for ion temperature - dmsp.meta['ti'] = {dmsp.meta.labels.units: 'Kelvin'} + # Update units for ion temperature using direct assignment + dmsp.meta['ti', dmsp.meta.labels.units] = 'Kelvin' - # Update display name for ion temperature, using LaTeX notation + # Update display name for ion temp, using dict assignment and LaTeX notation dmsp.meta['ti'] = {dmsp.meta.labels.name: 'T$_i$'} - # Add new meta data + # Add new meta data for multiple labels using dict assignment dmsp.meta['new'] = {dmsp.meta.labels.units: 'unitless', dmsp.meta.labels.name: 'New display name'} From b2fe06128303da7e03972700c96547f5ceee9cf8 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 26 Oct 2023 15:05:34 -0400 Subject: [PATCH 205/365] DOC: updated changelog Added a summary of the PR to the changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9de1d907..683cf85ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added unit tests for all file parsing functions in `utils.files`. * Reduced code duplication in the `utils.files.parse_fixed_width_filenames` and `utils.files.parse_delimited_filenames` functions + * Added ability to set Meta data using `meta['data_var', 'label'] = value` + structure * Bug Fix * Fixed `utils.files.parse_fixed_width_filenames` output for empty file list * Maintenance From df1eec4ed234d3ea9c5bb3844233c289081d902f Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 26 Oct 2023 15:05:49 -0400 Subject: [PATCH 206/365] TST: added unit tests Added unit tests for the new meta setting behaviour. --- pysat/tests/test_meta.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 0bee40e27..804278828 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -142,6 +142,16 @@ def test_getitem_w_bad_key(self): assert str(kerr).find('not found in MetaData') >= 0 return + def test_setitem_w_bad_input_combo(self): + """Test that bad input calls will raise ValueError when setting data.""" + + with pytest.raises(ValueError) as verr: + self.meta[['uts', 'units']] = 'seconds' + + assert str(verr).find( + "unexpected input combination, can't set metadata") >= 0 + return + def test_getitem_w_index(self): """Test raises NotImplementedError with an integer index.""" @@ -578,6 +588,22 @@ def test_accept_default_labels(self): return + def test_meta_assign_single_val(self): + """Test basic assignment of a single metadata value.""" + # Ensure the data has not been set already + data_name = 'special_data' + label_name = self.meta.labels.notes + meta_val = "test me" + assert data_name not in self.meta.keys(), "bad testing set up" + + # Assign notes metadata + self.meta[data_name, label_name] = meta_val + + # Test the assigned metadata + assert data_name in self.meta.keys() + assert self.meta[data_name, label_name] == meta_val + return + @pytest.mark.parametrize("custom_attr", [None, 'custom_meta']) @pytest.mark.parametrize("assign_type", [dict, pds.Series]) def test_meta_assignment(self, custom_attr, assign_type): From b4f9680bb04358961b7638c30a19ea363c3b017a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 26 Oct 2023 15:11:21 -0400 Subject: [PATCH 207/365] DOC: updated docstring Updated the allowed `input_dat` type. --- pysat/_meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/_meta.py b/pysat/_meta.py index 5850dbb4d..e44c2172c 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -320,7 +320,7 @@ def __setitem__(self, data_vars, input_dat): ---------- data_vars : str, list, tuple Data variable names for the input metadata - input_dat : dict, pds.Series, or Meta + input_dat : dict, pds.Series, Meta, int, float, str, bool, or NoneType Input metadata to be assigned Raises From b64619a361662207de338130bdbb888bc8549973 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 27 Oct 2023 11:55:10 -0400 Subject: [PATCH 208/365] ENH: added `drop` and `delitem` Added the `drop` and `__delitem__` methods to the Instrument class, which delete data variables from the `data` and `Meta` objects. --- pysat/_instrument.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 7e3318fa5..1caf0d199 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1164,6 +1164,24 @@ def __setitem__(self, key, new_data): return + def __delitem__(self, key): + """Delete a key by calling `drop` method. + + Parameters + ---------- + key : str or list-like + A meta data variable, label, or MetaHeader attribute; which are + considered in that order. + + Raises + ------ + KeyError + If all key values are unavailable + + """ + self.drop(key) + return + def __iter__(self): """Load data for subsequent days or files. @@ -2580,6 +2598,48 @@ def custom_clear(self): self.custom_kwargs = [] return + def drop(self, names): + """Drop variables from Instrument. + + Parameters + ---------- + names : str or list-like + String or list of strings specifying the variables names to drop + + Raises + ------ + KeyError + If all of the variable names provided in `names` are not found + in the variable list. If a subset is missing, a logger warning is + issued instead. + + """ + # Ensure the input is list-like + names = pysat.utils.listify(names) + + # Ensure the names are present in the list of variables + good_names = [name for name in names if name in self.variables] + + if len(good_names) > 0: + # Drop the Instrument data using the appropriate methods + if self.pandas_format: + self.data = self.data.drop(columns=good_names) + else: + self.data = self.data.drop_vars(good_names) + + # Drop the meta data associated with this variable + self.meta.drop(good_names) + + if len(good_names) < len(names): + if len(good_names) == 0: + raise KeyError("{:} not found in Instrument variables".format( + names)) + else: + pysat.logger.warning( + "{:} not found in Instrument variables".format( + [name for name in names if name not in good_names])) + return + def today(self): """Get today's date (UTC), with no hour, minute, second, etc. From 19c39d3850a5e071cc730c2917fbef18fdc41774 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 27 Oct 2023 11:56:22 -0400 Subject: [PATCH 209/365] ENH: added `drop` and `delitem` to const Added the `drop` and `__delitem__` methods to the Constellation class, which delete data variables from the `data` and `Meta` objects in all the appropriate Instruments. May use the names from the individual Instruments or from the Constellation `variables` attribute. --- pysat/_constellation.py | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/pysat/_constellation.py b/pysat/_constellation.py index c51a43fd3..6dc87a8e4 100644 --- a/pysat/_constellation.py +++ b/pysat/_constellation.py @@ -378,6 +378,23 @@ def is_plural(num): return output_str + def __delitem__(self, key): + """Delete a key by calling `drop` method. + + Parameters + ---------- + key : str or list-like + A Constellation variable or list of variables. + + Raises + ------ + KeyError + If all key values are unavailable + + """ + self.drop(key) + return + # ----------------------------------------------------------------------- # Define all hidden methods @@ -592,6 +609,61 @@ def date(self): else: return None + def drop(self, names): + """Drop variables (names) from metadata. + + Parameters + ---------- + names : str or list-like + String or list of strings specifying the variable names to drop + + Raises + ------ + KeyError + If all of the keys provided in `names` is not found in the + standard metadata, labels, or header metadata. If a subset is + missing, a logger warning is issued instead. + + """ + # Ensure the input is list-like + names = pysat.utils.listify(names) + + # Divide the names by instrument + good_inst_names = [list() for inst in self.instruments] + bad_names = list() + inst_strs = ['_'.join([attr for attr in [inst.platform, inst.name, + inst.tag, inst.inst_id] + if len(attr) > 0]) for inst in self.instruments] + for name in names: + got_name = False + for i, inst in enumerate(self.instruments): + if name in inst.variables: + good_inst_names[i].append(name) + got_name = True + elif name in self.variables and name.find(inst_strs[i]) > 0: + good_inst_names[i].append(name.split("_{:s}".format( + inst_strs[i]))[0]) + got_name = True + + if not got_name: + bad_names.append(name) + + # If there are no good names, raise a KeyError + if len(bad_names) == len(names): + raise KeyError('{:} not found in Constellation'.format(names)) + + # Drop names by instrument + for i, inst in enumerate(self.instruments): + if len(good_inst_names[i]) > 0: + inst.drop(good_inst_names[i]) + + # If there are some bad names, raise a logging warning + if len(bad_names) > 0: + pysat.logger.warning('{:} not found in Constellation'.format( + bad_names)) + + return + @property def empty(self): """Boolean flag reflecting lack of data. From b91adaaf540aba11b666cb806769c92323bb8f7f Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 27 Oct 2023 11:56:47 -0400 Subject: [PATCH 210/365] TST: added Instrument del tests Added unit tests for the Instrument del and drop methods. --- pysat/tests/classes/cls_instrument_access.py | 60 ++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py index e32d244f1..3570eb66d 100644 --- a/pysat/tests/classes/cls_instrument_access.py +++ b/pysat/tests/classes/cls_instrument_access.py @@ -130,6 +130,66 @@ def test_basic_instrument_load(self, kwargs): self.eval_successful_load() return + @pytest.mark.parametrize("method", ["del", "drop"]) + @pytest.mark.parametrize("del_all", [True, False]) + def test_basic_instrument_del(self, method, del_all): + """Test that data can be deleted from an Instrument. + + Parameters + ---------- + method : str + String specifying the deletion method + del_all : bool + Delete a single variable if False, delete all if True + + """ + + # Load data by year and day of year + self.testInst.load(self.ref_time.year, self.ref_doy) + + # Get a variable name(s) to delete + var = self.testInst.variables if del_all else self.testInst.variables[0] + + # Delete the variable + if method == 'del': + del self.testInst[var] + else: + self.testInst.drop(var) + + # Test that the absence of the desired variable(s) + if del_all: + assert self.testInst.empty + assert len(self.testInst.variables) == 0 + else: + assert var not in self.testInst.variables + return + + def test_basic_instrument_bad_var_drop(self): + """Check for error when deleting absent data variable.""" + # Load data by year and day of year + self.testInst.load(self.ref_time.year, self.ref_doy) + + # Test that the correct error is raised + testing.eval_bad_input(self.testInst.drop, KeyError, + "not found in Instrument variables", + input_args=["not_a_data_variable"]) + return + + def test_basic_instrument_partial_bad_var_drop(self, caplog): + """Check for log warning when deleting present and absent variables.""" + # Load data by year and day of year + self.testInst.load(self.ref_time.year, self.ref_doy) + + dvars = [self.testInst.variables[0], "not_a_data_var"] + + # Test that the correct warning is raised + with caplog.at_level(logging.INFO, logger='pysat'): + self.testInst.drop(dvars) + + captured = caplog.text + assert captured.find("not found in Instrument variables") > 0 + return + @pytest.mark.parametrize('pad', [None, dt.timedelta(days=1)]) def test_basic_instrument_load_no_data(self, caplog, pad): """Test Instrument load with no data for appropriate log messages. From f44f1c5948354aec01d76ee5421eff7c477ff573 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 27 Oct 2023 11:57:07 -0400 Subject: [PATCH 211/365] TST: added Constellation del tests Added unit tests for the Constellation delete and drop methods. --- pysat/tests/test_constellation.py | 88 +++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py index 608251db2..0712de18f 100644 --- a/pysat/tests/test_constellation.py +++ b/pysat/tests/test_constellation.py @@ -572,3 +572,91 @@ def test_to_inst_mult_pad_clean(self): list(self.inst[1].index)) return + + @pytest.mark.parametrize("method", ["del", "drop"]) + def test_delitem_all_inst(self, method): + """Test Constellation deletion of data variables from all instruments. + + Parameters + ---------- + method : str + String specifying the deletion method + + """ + # Load the Constellation data + self.const.load(date=self.ref_time) + + # Delete the UTS data from all instruments + dvar = "uts" + if method == "del": + del self.const[dvar] + else: + self.const.drop(dvar) + + # Test that this variable is gone from all Instruments + for inst in self.const.instruments: + assert dvar not in inst.variables + + # Test that the constellation variable list has been updated + for var in self.const.variables: + assert var.find(dvar) != 0 + + return + + @pytest.mark.parametrize("method", ["del", "drop"]) + def test_delitem_one_inst(self, method): + """Test Constellation deletion of data variables from one instrument. + + Parameters + ---------- + method : str + String specifying the deletion method + + """ + # Load the Constellation data + self.const.load(date=self.ref_time) + + # Delete the UTS data from all instruments + dvar = "uts_pysat_testing" + if method == "del": + del self.const[dvar] + else: + self.const.drop(dvar) + + # Test that this variable is gone from only the desired Instrument + for inst in self.const.instruments: + if inst.platform == "pysat" and inst.name == "testing": + assert "uts" not in inst.variables + else: + assert "uts" in inst.variables + + # Test that the constellation variable list has been updated + assert dvar not in self.const.variables + + return + + def test_bad_var_drop(self): + """Check for error when deleting absent data variable.""" + # Load the Constellation data + self.const.load(date=self.ref_time) + + # Test that the correct error is raised + testing.eval_bad_input(self.const.drop, KeyError, + "not found in Constellation", + input_args=["not_a_data_variable"]) + return + + def test_partial_bad_var_drop(self, caplog): + """Check for log warning when deleting present and absent variables.""" + # Load the Constellation data + self.const.load(date=self.ref_time) + + dvars = [self.const.variables[0], "not_a_data_var"] + + # Test that the correct warning is raised + with caplog.at_level(logging.INFO, logger='pysat'): + self.const.drop(dvars) + + captured = caplog.text + assert captured.find("not found in Constellation") > 0 + return From 2aea6a363a436af9a1ccbc179417bca10cef3cc5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 27 Oct 2023 11:57:23 -0400 Subject: [PATCH 212/365] DOC: updated changelog Added a summary of changes to the changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9de1d907..3c0455f16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added `__delitem__` to Meta and `drop` to MetaHeader and MetaLabels classes. * Modified Meta to allow MetaHeader attribute access directly from Meta. * Expanded `Meta.drop` to traverse attached MetaLabel and MetaHeader data. + * Added `__delitem__` and `drop` to Instrument and Constellation classes. * Added options to customize `pysat_ndtesting` instrument with sample rate, shift in time. * Added orbit number to `pysat_ndtesting`. From d203ec7e32a9e9b258f42b30ed90b9420eaeb168 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 27 Oct 2023 12:05:22 -0400 Subject: [PATCH 213/365] STY: fixed indent Fixed a visual indent. --- pysat/_constellation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/_constellation.py b/pysat/_constellation.py index 6dc87a8e4..3baf1e87b 100644 --- a/pysat/_constellation.py +++ b/pysat/_constellation.py @@ -632,7 +632,7 @@ def drop(self, names): good_inst_names = [list() for inst in self.instruments] bad_names = list() inst_strs = ['_'.join([attr for attr in [inst.platform, inst.name, - inst.tag, inst.inst_id] + inst.tag, inst.inst_id] if len(attr) > 0]) for inst in self.instruments] for name in names: got_name = False From 74188e32b2f12d0f67c68b1a906af4e15e08ce15 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 31 Oct 2023 15:00:49 -0400 Subject: [PATCH 214/365] STY: reduce load calc in `pysat_testing.py` Reduce the amount of unnecessary calculations performed in the pysat_testing.py load method. --- pysat/instruments/pysat_testing.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pysat/instruments/pysat_testing.py b/pysat/instruments/pysat_testing.py index 6bbc08b02..2779fd8ed 100644 --- a/pysat/instruments/pysat_testing.py +++ b/pysat/instruments/pysat_testing.py @@ -93,6 +93,10 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, # Support keyword testing pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg)))) + # If no download should be simulated, return empty `data` and `meta` objects + if tag == 'no_download': + return pds.DataFrame(), pysat.Meta() + # Create an artificial satellite data set iperiod = mm_test.define_period() drange = mm_test.define_range() @@ -172,16 +176,14 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, data.index = index data.index.name = 'Epoch' + # If we only want data and not metadata stop now + if tag == 'default_meta': + return data, pysat.Meta() + # Set the meta data meta = mm_test.initialize_test_meta('Epoch', data.keys()) - # TODO(#1120): Move logic up so that empty data is returned first. - if tag == 'default_meta': - return data, pysat.Meta() - elif tag == 'no_download': - return pds.DataFrame(), pysat.Meta() - else: - return data, meta + return data, meta list_files = functools.partial(mm_test.list_files, test_dates=_test_dates) From c71884c59f21f3c4d939a1010163219c664c17c5 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 31 Oct 2023 15:01:30 -0400 Subject: [PATCH 215/365] ENH: added 'no_download' support for xarray Added the 'no_download' test for xarray. --- pysat/instruments/pysat_ndtesting.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 63c4dfd07..d4c24c6a6 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -15,9 +15,11 @@ name = 'ndtesting' pandas_format = False -tags = {'': 'Regular testing data set'} +tags = {'': 'Regular testing data set', + 'no_download': 'simulate an instrument without download support'} inst_ids = {'': [tag for tag in tags.keys()]} _test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}} +_test_download = {'': {'no_download': False}} _test_load_opt = {'': {'': [{'num_extra_time_coords': 0}, {'num_extra_time_coords': 1}]}} @@ -94,6 +96,10 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, # Support keyword testing pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg)))) + # If no download should be simulated, return empty `data` and `meta` objects + if tag == 'no_download': + return xr.Dataset(), pysat.Meta() + # Create an artificial satellite data set iperiod = mm_test.define_period() drange = mm_test.define_range() From e6830a15da7f3b8622e4226733dcaaedbff8ba16 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 31 Oct 2023 15:02:03 -0400 Subject: [PATCH 216/365] DOC: updated new instrument instructions Added the expected output for loading no data to the new instrument instructions. --- docs/new_instrument.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 9a0255ef6..2351f6107 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -414,6 +414,9 @@ The load module method signature should appear as: commmonly specify the data set to be loaded - The :py:func:`load` routine should return a tuple with :py:attr:`data` as the first element and a :py:class:`pysat.Meta` object as the second element. + If there is no data to load, :py:attr:`data` should return an empty + :py:class:`pandas.DataFrame` or :py:class:`xarray.Dataset` and :py:attr:`meta` + should return an empty :py:class:`pysat.Meta` object. - For simple time-series data sets, :py:attr:`data` is a :py:class:`pandas.DataFrame`, column names are the data labels, rows are indexed by :py:class:`datetime.datetime` objects. From 260eb227e6955a83a45d915e7f825dff38ee8e3c Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 31 Oct 2023 15:02:19 -0400 Subject: [PATCH 217/365] TST: added empty load test Added a test for loading times without data. --- pysat/tests/classes/cls_instrument_library.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 316f2fdde..a40f64f77 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -39,6 +39,7 @@ class TestInstruments(InstLibTests): import pandas as pds import pytest +import xarray as xr import pysat from pysat.utils import generate_instrument_list @@ -411,6 +412,39 @@ def test_load(self, clean_level, inst_dict): return + @pytest.mark.second + @pytest.mark.load_options + def test_load_empty(self, inst_dict): + """Test that instruments load empty objects if no data is available. + + Parameters + ---------- + inst_dict : dict + Dictionary containing info to instantiate a specific instrument. + Set automatically from instruments['download'] when + `initialize_test_package` is run. + + """ + + # Get the instrument information and update the date to be in the future + test_inst, date = initialize_test_inst_and_date(inst_dict) + date = dt.datetime(dt.datetime.utcnow().year + 100, 1, 1) + + # Make sure the strict time flag doesn't interfere with the load test + load_and_set_strict_time_flag(test_inst, date, raise_error=True) + + # Check the empty status + assert test_inst.empty, "Data was loaded for a far-future time" + assert test_inst.meta == pysat.Meta(), "Meta data is not empty" + if test_inst.pandas_format: + assert all(test_inst.data == pds.DataFrame()), "Data not empty" + else: + assert test_inst.data.dims == xr.Dataset().dims, "Dims not empty" + assert test_inst.data.data_vars == xr.Dataset().data_vars, \ + "Data variables not empty" + + return + @pytest.mark.second # Need to maintain download mark for backwards compatibility. # Can remove once pysat 3.1.0 is released and libraries are updated. From fb0ff24ed39bca7dbef1cd8226527c52fd73bf7e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 31 Oct 2023 15:02:33 -0400 Subject: [PATCH 218/365] DOC: updated changelog Added a summary of the changes to the changelog. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67364ca24..388222f20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * New Features * Added tests for warnings, logging messages, and errors in the Instrument clean method. - * Added loading test with padding for Instruments. + * Added Instrument loading test with padding and for times without data. * Allow Instruments to define custom `concat_data` methods. * Added `include` kwarg to `Instrument.concat_data` to expand allowed inputs. * Added data kwarg to the Instrument class `__getitem__` method and reduced From d4fa58f209cbd26590ff604b7e112c01f9b71574 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 31 Oct 2023 15:08:34 -0400 Subject: [PATCH 219/365] STY: removed trailing whitespace Removed trailing whitespace. --- pysat/instruments/pysat_ndtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index d4c24c6a6..3d2815170 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -98,7 +98,7 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, # If no download should be simulated, return empty `data` and `meta` objects if tag == 'no_download': - return xr.Dataset(), pysat.Meta() + return xr.Dataset(), pysat.Meta() # Create an artificial satellite data set iperiod = mm_test.define_period() From 8d20e8633e405a375a27a2045c0aff938b48f434 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 31 Oct 2023 15:49:11 -0400 Subject: [PATCH 220/365] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysat/tests/classes/cls_instrument_library.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index e737baf87..c51ed79e8 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -425,8 +425,6 @@ def test_load_multiple_days(self, inst_dict): Parameters ---------- - clean_level : str - Cleanliness level for loaded instrument data. inst_dict : dict Dictionary containing info to instantiate a specific instrument. Set automatically from instruments['download'] when @@ -436,17 +434,12 @@ def test_load_multiple_days(self, inst_dict): test_inst, date = initialize_test_inst_and_date(inst_dict) if len(test_inst.files.files) > 0: - # Set the clean level - target = 'Fake Data to be cleared' - test_inst.data = [target] # Make sure the strict time flag doesn't interfere with # the load tests, and re-run with desired clean level load_and_set_strict_time_flag(test_inst, date, raise_error=True, clean_off=False, concat=True) - # Make sure fake data is cleared - assert target not in test_inst.data # Make sure more than one day has been loaded assert len(np.unique(test_inst.index.day)) > 1 else: From acc29f985942b8c9367b1653f814f65eb8bd445e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 31 Oct 2023 15:49:45 -0400 Subject: [PATCH 221/365] MAINT: remove old marks --- pysat/tests/classes/cls_instrument_library.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index e737baf87..03389a360 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -553,10 +553,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): return @pytest.mark.second - # Need to maintain download mark for backwards compatibility. - # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options - @pytest.mark.download @pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)]) def test_load_w_pad(self, pad, inst_dict): """Test that instruments load at each cleaning level. From 2fa067021376adf6af8d76126b4099db30c69214 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 31 Oct 2023 15:52:19 -0400 Subject: [PATCH 222/365] Apply suggestions from review --- pysat/tests/classes/cls_instrument_library.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index af1ae1f78..6d21afe56 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -76,7 +76,7 @@ def initialize_test_inst_and_date(inst_dict): def load_and_set_strict_time_flag(test_inst, date, raise_error=False, - clean_off=True, concat=False): + clean_off=True, set_end_date=False): """Load data and set the strict time flag if needed for other tests. Parameters @@ -91,15 +91,15 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False, clean_off : bool Turn off the clean method when re-loading data and testing the strict time flag (default=True) - concat : bool - If True, load multiple days to concat. If False, load single day. + set_end_date : bool + If True, load with setting the end date. If False, load single day. (default=False) """ - kwargs = {'use_header': True} + kwargs = {} - if concat: + if set_end_date: kwargs['end_date'] = date + dt.timedelta(days=2) try: From 6cecdedba75524840683490a0f05bcb565ba8b61 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 31 Oct 2023 18:59:00 -0400 Subject: [PATCH 223/365] BUG: kwarg --- pysat/tests/classes/cls_instrument_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 6d21afe56..6b661dc3b 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -438,7 +438,7 @@ def test_load_multiple_days(self, inst_dict): # Make sure the strict time flag doesn't interfere with # the load tests, and re-run with desired clean level load_and_set_strict_time_flag(test_inst, date, raise_error=True, - clean_off=False, concat=True) + clean_off=False, set_end_date=True) # Make sure more than one day has been loaded assert len(np.unique(test_inst.index.day)) > 1 From 1afbd203bdb02990a31099cb4504a815b7eefa6e Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 1 Nov 2023 16:17:05 -0400 Subject: [PATCH 224/365] REV: remove 'no_download' from ndxarray Remove the 'no_download' option from the ndxarray test Instrument. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysat/instruments/pysat_ndtesting.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 3d2815170..63c4dfd07 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -15,11 +15,9 @@ name = 'ndtesting' pandas_format = False -tags = {'': 'Regular testing data set', - 'no_download': 'simulate an instrument without download support'} +tags = {'': 'Regular testing data set'} inst_ids = {'': [tag for tag in tags.keys()]} _test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}} -_test_download = {'': {'no_download': False}} _test_load_opt = {'': {'': [{'num_extra_time_coords': 0}, {'num_extra_time_coords': 1}]}} @@ -96,10 +94,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False, # Support keyword testing pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg)))) - # If no download should be simulated, return empty `data` and `meta` objects - if tag == 'no_download': - return xr.Dataset(), pysat.Meta() - # Create an artificial satellite data set iperiod = mm_test.define_period() drange = mm_test.define_range() From 478ba17403e23609ece8b993f2c133705aeed7bc Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 2 Nov 2023 12:47:55 -0400 Subject: [PATCH 225/365] DOC: remove old comments Remove comments that were referring to code that was removed previously. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysat/tests/classes/cls_instrument_library.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 92a6ae3d5..d7bdef0a4 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -453,8 +453,6 @@ def test_load_empty(self, inst_dict): return @pytest.mark.second - # Need to maintain download mark for backwards compatibility. - # Can remove once pysat 3.1.0 is released and libraries are updated. @pytest.mark.load_options def test_load_multiple_days(self, inst_dict): """Test that instruments load at each cleaning level. From 5c6b52aa98a191525d7d579a40af3a8a9be11306 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 22 Nov 2023 20:51:40 -0500 Subject: [PATCH 226/365] DOC: remove todo --- pysat/tests/test_files.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py index 0933d2ed7..c475b7cf9 100644 --- a/pysat/tests/test_files.py +++ b/pysat/tests/test_files.py @@ -1295,8 +1295,6 @@ def teardown_method(self): temporary_file_list=self.temporary_file_list) pysat.params['data_dirs'] = self.data_paths - # TODO(#871): This needs to be replaced or expanded based on the tests that - # portalocker uses def test_race_condition(self): """Test that multiple instances of pysat instrument creation run.""" processes = 5 From 7a8b5b409419c2d1efbfbccb78652c6f4bdd3310 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 22 Nov 2023 20:55:08 -0500 Subject: [PATCH 227/365] STY: use temporary directory --- pysat/tests/test_utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index b88c961f6..4e819b2eb 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -13,6 +13,7 @@ import portalocker import pytest import shutil +import sys import tempfile import warnings @@ -624,19 +625,22 @@ class TestNetworkLock(object): def setup_method(self): """Set up the unit test environment.""" + # Use a temporary directory so that the user's setup is not altered. + self.temp_dir = tempfile.TemporaryDirectory() + # Create and write a temporary file - self.fname = 'temp_lock_file.txt' + self.fname = os.path.join(self.temp_dir.name, 'temp_lock_file.txt') with open(self.fname, 'w') as fh: fh.write('spam and eggs') return def teardown_method(self): """Clean up the unit test environment.""" - # Remove the temporary file - os.remove(self.fname) + # Remove the temporary directory. + self.temp_dir.cleanup() # Delete the test class attributes - del self.fname + del self.fname, self.temp_dir return def test_with_timeout(self): From 33199c795d9c7b3bdbee77bee3bc815ea132170c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 22 Nov 2023 20:55:15 -0500 Subject: [PATCH 228/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04680ecc9..2acb2c1ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed deprecated `freq` kwarg from `download` * Removed deprecated `use_header` kwarg from `load` and changed default behaviour to `use_header=True` + * Use temporary directories for files created during test_utils.py [3.1.0] - 2023-05-31 -------------------- From a84bd0f5c0762edf412bdc2ac800bc642b26f20f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 23 Nov 2023 09:38:17 -0500 Subject: [PATCH 229/365] STY: pep8 --- pysat/tests/test_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index 4e819b2eb..3aeb87d81 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -13,7 +13,6 @@ import portalocker import pytest import shutil -import sys import tempfile import warnings From e9a119dc98c2648b36fb79c5c2242e82e72db07f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 24 Nov 2023 12:14:20 -0500 Subject: [PATCH 230/365] TST: continue-on-error for coveralls --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51489005d..845985ebd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,3 +69,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: coveralls --rcfile=pyproject.toml --service=github + continue-on-error: true From 6ce10e17f2500aa4b8e28d089d2fdb35ef50460a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Fri, 24 Nov 2023 14:04:52 -0500 Subject: [PATCH 231/365] TST: use coveralls app --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 845985ebd..2f47b9ab2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,5 +68,4 @@ jobs: - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls --rcfile=pyproject.toml --service=github - continue-on-error: true + uses: coverallsapp/github-action@v2 From 685648112fe2d7c67cd6baeeba9ba82f1abe40d4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Mon, 27 Nov 2023 13:40:51 -0500 Subject: [PATCH 232/365] MAINT: update coveralls --- .github/workflows/main.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2f47b9ab2..8b814363a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,4 +68,17 @@ jobs: - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: coverallsapp/github-action@v2 + COVERALLS_PARALLEL: true + run: coveralls --rcfile=pyproject.toml --service=github + + finish: + name: Finish Coverage Analysis + needs: build + runs-on: ubuntu-latest + steps: + - name: Coveralls Finished + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip install --upgrade coveralls + coveralls --service=github --finish From 669186909864175191ceaf0c7b3efc609d0eec8a Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:45:29 -0500 Subject: [PATCH 233/365] DOC: updated issue forms Updated the issue forms to be up to current standards. --- .github/ISSUE_TEMPLATE/bug_report.md | 26 +++++++++++------------ .github/ISSUE_TEMPLATE/feature_request.md | 24 +++++++++++++++------ .github/ISSUE_TEMPLATE/question.md | 19 +++++++++++++++++ 3 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/question.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 2764ce0c5..c29622e00 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,29 +1,29 @@ --- name: Bug report about: Create a report to help us improve +labels: bug +title: "BUG: " --- -**Describe the bug** -A clear and concise description of what the bug is. +# Description +A clear and concise description of what the bug is, including a description +of what you expected the outcome to be. -**To Reproduce** +# To Reproduce this bug: Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -**Expected behavior** -A clear and concise description of what you expected to happen. +Consider including images or test files to help others reproduce the bug and +solve the problem. -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Version [e.g. 22] +## Test configuration + - OS: [e.g., Hal] + - Version [e.g., Python 3.47] - Other details about your setup that could be relevant -**Additional context** -Add any other context about the problem here. +# Additional context +Add any other context about the problem here, including expected behaviour. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 066b2d920..d02da2ef4 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,17 +1,27 @@ --- name: Feature request about: Suggest an idea for this project +title: "ENH: " +labels: enhancement --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +# Description +A clear and concise description of the new feature or behaviour you would like. -**Describe the solution you'd like** +## Potential impact + +- Is the feature related to an existing problem? +- How critical is this feature to your workflow? +- How wide of an impact to you anticipate this enhancement having? +- Would this break any existing functionality? + +## Potential solution(s) A clear and concise description of what you want to happen. -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +# Alternatives +A clear description of any alternative solutions or features you've considered. -**Additional context** -Add any other context or screenshots about the feature request here. +# Additional context +Add any other context or screenshots about the feature request here, potentially +including your operational configuration. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..463725bae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,19 @@ +--- +name: Question +about: A question about this project +title: "QUEST: " +labels: question + +--- + +# Description +A clear and concise summary of your query + +## Example code (optional) +If relevant, include sample code, images, or files so that others can understand +the full context of your question. + +## Configuration + - OS: [e.g., Hal] + - Version: [e.g., Python 3.47] + - Other details about your setup that could be relevant From 9edbe901aa2996ec71d2bbb83b4cc391bfb90e5e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:46:03 -0500 Subject: [PATCH 234/365] DOC: updated pull request template Updated the pull request template to include all current check items. --- .github/pull_request_template.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 692a10c51..75ee10d47 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,6 @@ # Description -Addresses # (issue) +Addresses #(issue) Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required @@ -23,7 +23,10 @@ instructions so we can reproduce. Please also list any relevant details for your test configuration - Test A -- Test B + +``` +Test B +``` **Test Configuration**: * Operating system: Hal @@ -35,6 +38,7 @@ your test configuration - [ ] Make sure you are merging into the ``develop`` (not ``main``) branch - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code +- [ ] I have linted the files updated in this pull request - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings @@ -42,6 +46,8 @@ your test configuration - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules - [ ] Add a note to ``CHANGELOG.md``, summarizing the changes +- [ ] Update zenodo.json file for new code contributors -If this is a release PR, replace the first item of the above checklist with the release -checklist on the wiki: https://github.com/pysat/pysat/wiki/Checklist-for-Release +If this is a release PR, replace the first item of the above checklist with the +release checklist on the wiki: +https://github.com/pysat/pysat/wiki/Checklist-for-Release From 79c82cbcce370700477d06b76cdc8af10f100ada Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:46:50 -0500 Subject: [PATCH 235/365] TST: updated CI yamls Updated the CI yamls to be better formatted and contain the new coveralls logic. --- .github/workflows/docs.yml | 11 +++++------ .github/workflows/main.yml | 26 +++++++++++++++++++++----- .github/workflows/pip_rc_install.yml | 7 ++++--- .github/workflows/stats.yml | 7 ++++--- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 153cfc945..328ad1813 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,5 +1,6 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow will install Python dependencies, run tests and lint with a +# variety of Python versions. For more information see: +# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Documentation Check @@ -8,7 +9,7 @@ on: [push, pull_request] jobs: build: - runs-on: "ubuntu-latest" + runs-on: ["ubuntu-latest"] strategy: fail-fast: false matrix: @@ -23,9 +24,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install .[doc] + run: pip install .[doc] - name: Set up pysat run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51489005d..e07d0779c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,6 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow will install Python dependencies, run tests and lint with a +# variety of Python versions. For more information see: +# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Pytest with Flake8 @@ -10,15 +11,17 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] python-version: ["3.10", "3.11"] numpy_ver: ["latest"] test_config: ["latest"] include: + # NEP29 compliance settings - python-version: "3.9" numpy_ver: "1.21" os: ubuntu-latest test_config: "NEP29" + # Operational compliance settings - python-version: "3.6.8" numpy_ver: "1.19.5" os: "ubuntu-20.04" @@ -36,7 +39,7 @@ jobs: - name: Install Operational dependencies if: ${{ matrix.test_config == 'Ops'}} run: | - pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} + pip install numpy==${{ matrix.numpy_ver }} pip install -r requirements.txt pip install -r test_requirements.txt pip install . @@ -63,9 +66,22 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: pytest --cov=pysat/ + run: pytest - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_PARALLEL: true run: coveralls --rcfile=pyproject.toml --service=github + + finish: + name: Finish Coverage Analysis + needs: build + runs-on: ubuntu-latest + steps: + - name: Coveralls Finished + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip install --upgrade coveralls + coveralls --service=github --finish diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index c782985e0..0611a98b7 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -1,6 +1,7 @@ -# This workflow will install Python dependencies and the latest RC of pysatCDAAC from test pypi. -# This test should be manually run before a pysatCDAAC RC is officially approved and versioned. -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow will install Python dependencies and the latest RC of pysat from +# test pypi. This test should be manually run before a pysat RC is officially +# approved and versioned. For more information see: +# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Test install of latest RC from pip diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index f22ad9ab3..eed4565a7 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -1,5 +1,6 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow will install Python dependencies, run tests and lint with a +# variety of Python versions. For more information see: +# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Statistics of supported instruments @@ -9,7 +10,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ["ubuntu-latest"] strategy: fail-fast: false matrix: From 1c229a36c79f5524ab79c8fabb9030f1387b5786 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:47:17 -0500 Subject: [PATCH 236/365] DOC: updated code of conduct Updated code of conduct to comply with style standards. --- CODE_OF_CONDUCT.md | 62 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index bb9195cc8..b6d84d819 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,11 +2,17 @@ ## Our Pledge -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, +body size, disability, ethnicity, gender identity and expression, level of +experience, nationality, personal appearance, race, religion, or sexual +identity and orientation. ## Our Standards -Examples of behavior that contributes to creating a positive environment include: +Examples of behavior that contributes to creating a positive environment +include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences @@ -16,31 +22,61 @@ Examples of behavior that contributes to creating a positive environment include Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery and unwelcome sexual attention or advances +* The use of sexualized language or imagery and unwelcome sexual attention or + advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting ## Our Responsibilities -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. ## Scope -This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an +appointed representative at an online or offline event. Representation of a +project may be further defined and clarified by project maintainers. ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at pysat.developers@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at pysat.developers@gmail.com. The +pysat project team will review and investigate all complaints, and will respond +in a way that it deems appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an +incident. Further details of specific enforcement policies may be posted +separately. -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.4, available at +[https://contributor-covenant.org/version/1/4][version] -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ +## FAQ + +For answers to common questions about this code of conduct, see +[https://www.contributor-covenant.org/faq][faq] + +[homepage]: https://contributor-covenant.org +[version]: https://contributor-covenant.org/version/1/4/ +[faq]: https://www.contributor-covenant.org/faq From f01438721fb7495618e36605feec008db9855dd2 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:47:40 -0500 Subject: [PATCH 237/365] DOC: updated contributing guidelines Updated contributing guidelines to include better formatting and recent updates to style. --- CONTRIBUTING.md | 89 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c1a95415..77be3f0fa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,13 +13,27 @@ are generally held fortnightly. Short version ------------- -* Submit bug reports and feature requests at +* Submit bug reports, feature requests, and questions at [GitHub](https://github.com/pysat/pysat/issues) * Make pull requests to the ``develop`` branch +Issues +------ + +Bug reports, questions, and feature requests should all be made as GitHub +Issues. Templates are provided for each type of issue, to help you include +all the necessary information. + +Questions +^^^^^^^^^ + +Not sure how something works? Ask away! The more information you provide, the +easier the question will be to answer. You can also interact with the pysat +developers on our [slack channel](https://pysat.slack.com). + Bug reports ------------ +^^^^^^^^^^^ When [reporting a bug](https://github.com/pysat/pysat/issues) please include: @@ -31,12 +45,12 @@ include: * Detailed steps to reproduce the bug Feature requests and feedback ------------------------------ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The best way to send feedback is to file an issue at -[GitHub](https://github.com/pysat/pysat/issues). +The best way to send feedback is to file an +[issue](https://github.com/pysat/pysat/issues). -If you are proposing a feature: +If you are proposing a new feature or a change in something that already exists: * Explain in detail how it would work. @@ -66,42 +80,57 @@ To set up `pysat` for local development: Now you can make your changes locally. - Tests for new instruments are performed automatically. Tests for custom - functions should be added to the appropriately named file in ``pysat/tests``. - For example, custom functions for the time utilities are tested in - ``pysat/tests/test_utils_time.py``. If no test file exists, then you should - create one. This testing uses pytest, which will run tests on any python - file in the test directory that starts with ``test``. Classes must begin - with ``Test``, and methods must begin with ``test`` as well. + Tests for new instruments are performed automatically. See discussion + [here](https://pysat.readthedocs.io/en/main/new_instrument.html#testing-support) + for more information on triggering these standard tests. + + Tests for custom functions should be added to the appropriately named file + in ``pysat/tests``. For example, custom functions for the time utilities are + tested in ``pysat/tests/test_utils_time.py``. If no test file exists, then + you should create one. This testing uses pytest, which will run tests on + any Python file in the test directory that starts with ``test``. Classes + must begin with ``Test``, and methods must begin with ``test`` as well. 4. When you're done making changes, run all the checks to ensure that nothing is broken on your local system, as well as check for flake8 compliance: ``` - pytest -vs --flake8 pysat + pytest ``` -5. Update/add documentation (in ``docs``), if relevant +5. You should also check for flake8 style compliance: + + ``` + flake8 . --count --select=D,E,F,H,W --show-source --statistics + ``` + + Note that pysat uses the `flake-docstrings` and `hacking` packages to ensure + standards in docstring formatting. + +6. Update/add documentation (in ``docs``). Even if you don't think it's + relevant, check to see if any existing examples have changed. -6. Add your name to the .zenodo.json file as an author +7. Add your name to the .zenodo.json file as an author -7. Commit your changes: +8. Commit your changes: ``` git add . git commit -m "AAA: Brief description of your changes" ``` - Where AAA is a standard shorthand for the type of change (eg, BUG or DOC). + Where AAA is a standard shorthand for the type of change (e.g., BUG or DOC). `pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html), see the discussion there for a full list of this shorthand notation. -8. Once you are happy with the local changes, push to Github: +9. Once you are happy with the local changes, push to GitHub: ``` git push origin name-of-your-bugfix-or-feature ``` Note that each push will trigger the Continuous Integration workflow. -9. Submit a pull request through the GitHub website. Pull requests should be - made to the ``develop`` branch. +10. Submit a pull request through the GitHub website. Pull requests should be + made to the ``develop`` branch. Note that automated tests will be run on + GitHub Actions, but these must be initialized by a member of the pysat team + for first time contributors. Pull Request Guidelines @@ -114,14 +143,15 @@ For merging, you should: 1. Include an example for use 2. Add a note to ``CHANGELOG.md`` about the changes -3. Update the author list in ``zenodo.json`` if applicable -4. Ensure that all checks passed (current checks include Github Actions and Coveralls) +3. Update the author list in ``zenodo.json``, if applicable +4. Ensure that all checks passed (current checks include GitHub Actions, + Coveralls and ReadTheDocs) -If you don't have all the necessary Python versions available locally or -have trouble building all the testing environments, you can rely on -GitHub Actions to run the tests for each change you add in the pull -request. Because testing here will delay tests by other developers, -please ensure that the code passes all tests on your local system first. +If you don't have all the necessary Python versions available locally or have +trouble building all the testing environments, you can rely on GitHub Actions to +run the tests for each change you add in the pull request. Because testing here +will delay tests by other developers, please ensure that the code passes all +tests on your local system first. Project Style Guidelines @@ -151,7 +181,8 @@ These include: * All classes should have `__repr__` and `__str__` functions * Docstrings use `Note` instead of `Notes` * Try to avoid creating a try/except statement where except passes -* Use setup and teardown in test classes +* Use setup_method (or setup_class) and teardown_method (or teardown_class) in + test classes * Use pytest parametrize in test classes when appropriate * Use pysat testing utilities when appropriate * Provide testing class methods with informative failure statements and From 621b6474ddf0f5215ce7d4ac324646b6f19ea584 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:48:39 -0500 Subject: [PATCH 238/365] DOC: updated readme Updated readme to include better installation instructions. --- README.md | 72 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 8db411201..8e1f3c441 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ JGR-Space Physics [Publication](https://doi.org/10.1029/2018JA025297) [Citation Info](https://pysat.readthedocs.io/en/latest/citing.html) Come join us on Slack! An invitation to the pysat workspace is available -in the 'About' section of the [pysat GitHub Repository.](https://github.com/pysat/pysat) +in the 'About' section of the +[pysat GitHub Repository.](https://github.com/pysat/pysat) Development meetings are generally held fortnightly. # Main Features @@ -49,47 +50,52 @@ Development meetings are generally held fortnightly. instruments to pysat # Installation -## Starting from scratch -* Python and associated packages for science are freely available. Convenient - science python package setups are available from https://www.python.org/, - [Anaconda](https://www.anaconda.com/distribution/), and other locations - (some platform specific). Anaconda also includes a developer environment that - works well with pysat. Core science packages such as numpy, scipy, matplotlib, - pandas and many others may also be installed directly via pip or your - favorite package manager. - -* Installation through pip + +The following instructions provide a guide for installing pysat and give some +examples on how to use the routines. + +## Prerequisites + +pysat uses common Python modules, as well as modules developed by and for the +Space Physics community. This module officially supports Python 3.X+. + +| Common modules | Community modules | +| -------------- | ----------------- | +| dask | netCDF4 | +| numpy >= 1.12 | | +| pandas < 2.1.1 | | +| portalocker | | +| toolz | | +| xarray | | + + +## PyPi Installation ``` pip install pysat ``` -* Installation through github + +## GitHub Installation ``` git clone https://github.com/pysat/pysat.git -cd pysat -pip install . ``` -An advantage to installing through github is access to the development branches. -The latest bugfixes can be found in the `develop` branch. However, this branch -is not stable (as the name implies). We recommend using this branch in a -virtual environment or using `python setup.py develop`. + +Change directories into the repository folder and run the pyproject.toml or +setup.py file. For a local install use the "--user" flag after "install". + ``` -git clone https://github.com/pysat/pysat.git -cd pysat -git checkout develop -pip install -e . +cd pysat/ +python -m build . +pip install . ``` -* Note that pysat requires a number of packages for the install. - * dask - * netCDF4 - * numpy - * pandas - * portalocker - * scipy - * toolz - * xarray -* The first time the package is run, you will need to specify a directory to - store data. In python, run: + +# Using pysat + +* The first time pysat is run, you will need to specify a directory to store + the data. In Python, run: ``` pysat.params['data_dirs'] = 'path/to/directory/that/may/or/may/not/exist' ``` * Nominal organization of data is top_dir/platform/name/tag/inst_id/files + +Detailed examples and tutorials for using pysat are available in the +[documentation](http://pysat.readthedocs.io/en/latest/index.html). From be88ed7ca69ba56f76177649a94659b35a8f7119 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:49:07 -0500 Subject: [PATCH 239/365] MAINT: removed old citation Removed the old citation file. --- pysat/citation.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 pysat/citation.txt diff --git a/pysat/citation.txt b/pysat/citation.txt deleted file mode 100644 index 2a546499a..000000000 --- a/pysat/citation.txt +++ /dev/null @@ -1 +0,0 @@ -Stoneback, Russell, et al. (2021). pysat/pysat v3.0 (Version v3.0). Zenodo. http://doi.org/10.5281/zenodo.1199703 \ No newline at end of file From 3c8a1dd40ba74585e73d5628fe82aca83fa2bbd1 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:49:39 -0500 Subject: [PATCH 240/365] BUG: fixed metadata implementation Fixed the metadata implementation and used the resources package where possible. --- pysat/__init__.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/pysat/__init__.py b/pysat/__init__.py index f8a110a90..1fb154efc 100644 --- a/pysat/__init__.py +++ b/pysat/__init__.py @@ -34,7 +34,12 @@ """ -import importlib +try: + from importlib import metadata + from importlib import resources +except ImportError: + import importlib_metadata as metadata + resources = None import logging import os @@ -50,12 +55,7 @@ from pysat import _params # Set version -try: - __version__ = importlib.metadata.version('pysat') -except AttributeError: - # Python 3.6 requires a different version - import importlib_metadata - __version__ = importlib_metadata.version('pysat') +__version__ = metadata.version('pysat') # Get home directory home_dir = os.path.expanduser('~') @@ -64,15 +64,19 @@ pysat_dir = os.path.join(home_dir, '.pysat') # Set directory for test data -here = os.path.abspath(os.path.dirname(__file__)) -test_data_path = os.path.join(here, 'tests', 'test_data') +if resources is None: + test_data_path = os.path.join(os.path.realpath(os.path.dirname(__file__)), + 'tests', 'test_data') +else: + test_data_path = str(resources.files(__package__).joinpath('tests', + 'test_data')) # Create a .pysat directory or parameters file if one doesn't exist. # pysat_settings did not exist pre v3 thus this provides a check against # v2 users that are upgrading. Those users need the settings file plus # new internal directories. -if not os.path.isdir(pysat_dir) or \ - (not os.path.isfile(os.path.join(pysat_dir, 'pysat_settings.json'))): +settings_file = os.path.join(pysat_dir, 'pysat_settings.json') +if not os.path.isdir(pysat_dir) or not os.path.isfile(settings_file): # Make a .pysat directory if not already present if not os.path.isdir(pysat_dir): @@ -89,7 +93,7 @@ os.mkdir(os.path.join(pysat_dir, 'instruments', 'archive')) # Create parameters file - if not os.path.isfile(os.path.join(pysat_dir, 'pysat_settings.json')): + if not os.path.isfile(settings_file): params = _params.Parameters(path=pysat_dir, create_new=True) print(''.join(("\nHi there! pysat will nominally store data in a ", @@ -116,5 +120,5 @@ from pysat._constellation import Constellation __all__ = ['instruments', 'utils'] -# Cleanup -del here +# Clean up +del settings_file, resources From 0a000ab77ace80f1b0dca8b6ff2a9d547bb2b10f Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:50:06 -0500 Subject: [PATCH 241/365] MAINT: updated manifest Updated the manifest to not include old files. --- MANIFEST.in | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7c416b089..2073a9352 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,8 @@ -global-include *.py +recursive-include pysat *.py include *.md include *.txt include LICENSE -include pysat/version.txt -include pysat/citation.txt +prune pysat/tests prune docs global-exclude *.pdf global-exclude *.png From 707c53b76fb0e523e3a03914a5f5118e74d99eca Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:50:54 -0500 Subject: [PATCH 242/365] MAINT: updated requirements Updated requirements to: - not include scipy (not imported anywhere), and - move pytest to the test installation. --- docs/installation.rst | 4 ++-- pyproject.toml | 2 -- requirements.txt | 2 -- test_requirements.txt | 1 + 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 24104dfae..54db75a2e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -42,8 +42,6 @@ needed by the installer: #. numpy #. pandas #. portalocker -#. pytest -#. scipy #. toolz #. xarray @@ -57,6 +55,7 @@ pysat may also be installed directly from the source repository on github:: git clone https://github.com/pysat/pysat.git cd pysat + python -m build . pip install --user . An advantage to installing through github is access to the development branches. @@ -67,6 +66,7 @@ virtual environment and using:: git clone https://github.com/pysat/pysat.git cd pysat git checkout develop + python -m build . pip install -e . The use of `-e` in the setup command installs the code 'in-place', so any diff --git a/pyproject.toml b/pyproject.toml index eb0dbe732..24249506c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,6 @@ dependencies = [ "numpy >= 1.12", "pandas < 2.1.1", "portalocker", - "pytest", - "scipy", "toolz", "xarray" ] diff --git a/requirements.txt b/requirements.txt index 059c99271..165a2698e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,5 @@ netCDF4 numpy>=1.12 pandas<2.1.1 portalocker -pytest -scipy toolz xarray diff --git a/test_requirements.txt b/test_requirements.txt index 8a33141b8..cfbe27e5d 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -6,6 +6,7 @@ ipython m2r2 numpydoc pysatSpaceWeather +pytest pytest-cov pytest-ordering sphinx From 76ddeed750af6fbd3615f1f168d5068a3b240b0b Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:52:11 -0500 Subject: [PATCH 243/365] MAINT: address #988 TODO Address a TODO for closed issue #988, which should fix updating the data without separate calls for xarray or pandas type. --- pysat/utils/coords.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pysat/utils/coords.py b/pysat/utils/coords.py index 504dc3ff8..8c5a142ff 100644 --- a/pysat/utils/coords.py +++ b/pysat/utils/coords.py @@ -66,12 +66,7 @@ def update_longitude(inst, lon_name=None, high=180.0, low=-180.0): raise ValueError('unknown longitude variable name') new_lon = adjust_cyclic_data(inst[lon_name], high=high, low=low) - - # TODO(#988): Remove pandas/xarray logic after fixing issue in Instrument - if inst.pandas_format: - inst[lon_name] = new_lon - else: - inst.data = inst.data.update({lon_name: (inst[lon_name].dims, new_lon)}) + inst[lon_name] = new_lon return From 48f23ea596f1b1c9d21d02c628befe1316081be7 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 14:56:45 -0500 Subject: [PATCH 244/365] Revert "MAINT: removed old citation" This reverts commit be88ed7ca69ba56f76177649a94659b35a8f7119. --- pysat/citation.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 pysat/citation.txt diff --git a/pysat/citation.txt b/pysat/citation.txt new file mode 100644 index 000000000..2a546499a --- /dev/null +++ b/pysat/citation.txt @@ -0,0 +1 @@ +Stoneback, Russell, et al. (2021). pysat/pysat v3.0 (Version v3.0). Zenodo. http://doi.org/10.5281/zenodo.1199703 \ No newline at end of file From a54e439b4c137e09d610dfe1af24ad4d826edc86 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 15:00:52 -0500 Subject: [PATCH 245/365] MAINT: updated citation Updated the citation version and year. --- pysat/citation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/citation.txt b/pysat/citation.txt index 2a546499a..d6f09ccc0 100644 --- a/pysat/citation.txt +++ b/pysat/citation.txt @@ -1 +1 @@ -Stoneback, Russell, et al. (2021). pysat/pysat v3.0 (Version v3.0). Zenodo. http://doi.org/10.5281/zenodo.1199703 \ No newline at end of file +Stoneback, Russell, et al. (2023). pysat/pysat v3.1 (Version v3.1). Zenodo. http://doi.org/10.5281/zenodo.1199703 From 5b48a45f2f8bb6e9b097937e6db093c5bdaaef11 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 15:01:20 -0500 Subject: [PATCH 246/365] STY: used resouces to access citation file Use the resources module to access the citation file, if possible. --- pysat/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pysat/__init__.py b/pysat/__init__.py index 1fb154efc..beb1da02d 100644 --- a/pysat/__init__.py +++ b/pysat/__init__.py @@ -67,9 +67,12 @@ if resources is None: test_data_path = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'tests', 'test_data') + citation = os.path.join(os.path.realpath(os.path.dirname(__file__)), + 'citation.txt') else: test_data_path = str(resources.files(__package__).joinpath('tests', 'test_data')) + citation = str(resources.files(__package__).joinpath('citation.txt')) # Create a .pysat directory or parameters file if one doesn't exist. # pysat_settings did not exist pre v3 thus this provides a check against From 4c8926f24c97579c026ac2b8023dc701554f7089 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 15:01:42 -0500 Subject: [PATCH 247/365] MAINT: use new module attribute Use the new module citation attribute. --- pysat/instruments/methods/testing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 3f02ff71a..04154d2d6 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -17,8 +17,7 @@ "https://www.github.com/pysat/pysat")) # Load up citation information -with pysat.utils.NetworkLock(os.path.join(pysat.here, 'citation.txt'), 'r') as \ - locked_file: +with pysat.utils.NetworkLock(pysat.citation, 'r') as locked_file: refs = locked_file.read() From 477d8043a366901be77a640c3cfca2677c006cc2 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 15:22:36 -0500 Subject: [PATCH 248/365] REV: re-add citation to manifest Re-add the citation text file to the manifest. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 2073a9352..31a3eaf4d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ recursive-include pysat *.py include *.md include *.txt include LICENSE +include pysat/citation.txt prune pysat/tests prune docs global-exclude *.pdf From 52f9c2a19dabe4eb6fcaa93582e288f1528e1c3d Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 15:33:37 -0500 Subject: [PATCH 249/365] REV: re-added pytest requirement Re-added the pytest requirement to the main code. --- README.md | 1 + docs/installation.rst | 1 + pyproject.toml | 2 +- requirements.txt | 1 + test_requirements.txt | 1 - 5 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e1f3c441..1df45d651 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Space Physics community. This module officially supports Python 3.X+. | numpy >= 1.12 | | | pandas < 2.1.1 | | | portalocker | | +| pytest | | | toolz | | | xarray | | diff --git a/docs/installation.rst b/docs/installation.rst index 54db75a2e..3d11e03e9 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -42,6 +42,7 @@ needed by the installer: #. numpy #. pandas #. portalocker +#. pytest #. toolz #. xarray diff --git a/pyproject.toml b/pyproject.toml index 24249506c..6d24d88e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "numpy >= 1.12", "pandas < 2.1.1", "portalocker", + "pytest", "toolz", "xarray" ] @@ -58,7 +59,6 @@ test = [ "flake8-docstrings", "hacking >= 1.0", "pysatSpaceWeather", - "pytest", "pytest-cov", "pytest-ordering" ] diff --git a/requirements.txt b/requirements.txt index 165a2698e..cec8a3bfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,6 @@ netCDF4 numpy>=1.12 pandas<2.1.1 portalocker +pytest toolz xarray diff --git a/test_requirements.txt b/test_requirements.txt index cfbe27e5d..8a33141b8 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -6,7 +6,6 @@ ipython m2r2 numpydoc pysatSpaceWeather -pytest pytest-cov pytest-ordering sphinx From 8291b9f0b496efbf3f7a5f81495efd8b8273e78e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 15:55:35 -0500 Subject: [PATCH 250/365] BUG: fixed `__setitem__` xarray assignment Fixed a bug in the xarray setitem magic method that did not account for native dimensions in the data when assigning arrays as values. --- pysat/_instrument.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 1caf0d199..f3e5bf5fa 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1106,8 +1106,13 @@ def __setitem__(self, key, new_data): self.data[key] = in_data elif len(np.shape(in_data)) <= 1: # If not an xarray input, but still iterable, then we - # go through to process the 1D input - if np.shape(in_data) == np.shape(self.index): + # go through to process the input + if np.shape(in_data) == np.shape(self.data[key]): + # The ND input has the same shape as the current data + # and can be assigned directly without adjusting the + # dimensions + self.data[key] = (self.data[key].dims, in_data) + elif np.shape(in_data) == np.shape(self.index): # 1D input has the correct length for storage along # 'Epoch'. self.data[key] = (epoch_name, in_data) From 4771259384fb6c6e6e6e54298c0d0d789cf8148b Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 17:02:14 -0500 Subject: [PATCH 251/365] REV: re-added scipy Found where scipy was needed, re-adding dependency. --- README.md | 1 + docs/installation.rst | 1 + pyproject.toml | 1 + requirements.txt | 1 + 4 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 1df45d651..eef03facd 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ Space Physics community. This module officially supports Python 3.X+. | pandas < 2.1.1 | | | portalocker | | | pytest | | +| scipy | | | toolz | | | xarray | | diff --git a/docs/installation.rst b/docs/installation.rst index 3d11e03e9..f8013c5a8 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -43,6 +43,7 @@ needed by the installer: #. pandas #. portalocker #. pytest +#. scipy #. toolz #. xarray diff --git a/pyproject.toml b/pyproject.toml index 6d24d88e4..3b47b33d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "pandas < 2.1.1", "portalocker", "pytest", + "scipy", "toolz", "xarray" ] diff --git a/requirements.txt b/requirements.txt index cec8a3bfa..059c99271 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,6 @@ numpy>=1.12 pandas<2.1.1 portalocker pytest +scipy toolz xarray From 90cc089d8c0f8652195557564040aabf83dbdd39 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 17:07:56 -0500 Subject: [PATCH 252/365] BUG: fixed new data bug Fixed a bug in assigning new xarray data. --- pysat/_instrument.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index f3e5bf5fa..959a98d6d 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1107,10 +1107,11 @@ def __setitem__(self, key, new_data): elif len(np.shape(in_data)) <= 1: # If not an xarray input, but still iterable, then we # go through to process the input - if np.shape(in_data) == np.shape(self.data[key]): + if key in self.variables and ( + np.shape(in_data) == np.shape(self.data[key])): # The ND input has the same shape as the current data # and can be assigned directly without adjusting the - # dimensions + # dimensions. Only works with existing data. self.data[key] = (self.data[key].dims, in_data) elif np.shape(in_data) == np.shape(self.index): # 1D input has the correct length for storage along From 6f52301aea932d326415a6958b23a6de0f7edbc6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 15:51:54 -0500 Subject: [PATCH 253/365] ENH: add teardown_method --- pysat/tests/classes/cls_instrument_library.py | 118 ++++++++++-------- 1 file changed, 67 insertions(+), 51 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index d7bdef0a4..066ede16f 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -198,6 +198,19 @@ def teardown_class(self): del self.saved_path, self.tempdir return + def setup_method(self): + """Initialize parameters before each method.""" + + return + + def teardown_method(self): + """Clean up any instruments that were initialized.""" + + if hasattr(self, "test_inst"): + del self.test_inst + + return + def initialize_test_package(self, inst_loc, user_info=None): """Generate custom instrument lists for each category of tests. @@ -366,7 +379,7 @@ def test_download(self, inst_dict): """ - test_inst, date = initialize_test_inst_and_date(inst_dict) + self.test_inst, date = initialize_test_inst_and_date(inst_dict) # Check for username. if 'user_info' in inst_dict.keys(): @@ -374,8 +387,8 @@ def test_download(self, inst_dict): else: dl_dict = {} # Note this will download two consecutive days - test_inst.download(date, **dl_dict) - assert len(test_inst.files.files) > 0 + self.test_inst.download(date, **dl_dict) + assert len(self.test_inst.files.files) > 0 return @pytest.mark.second @@ -395,25 +408,25 @@ def test_load(self, clean_level, inst_dict): """ - test_inst, date = initialize_test_inst_and_date(inst_dict) - if len(test_inst.files.files) > 0: + self.test_inst, date = initialize_test_inst_and_date(inst_dict) + if len(self.test_inst.files.files) > 0: # Set the clean level - test_inst.clean_level = clean_level + self.test_inst.clean_level = clean_level target = 'Fake Data to be cleared' - test_inst.data = [target] + self.test_inst.data = [target] # Make sure the strict time flag doesn't interfere with # the load tests, and re-run with desired clean level - load_and_set_strict_time_flag(test_inst, date, raise_error=True, - clean_off=False) + load_and_set_strict_time_flag(self.test_inst, date, + raise_error=True, clean_off=False) # Make sure fake data is cleared - assert target not in test_inst.data + assert target not in self.test_inst.data # If cleaning not used, something should be in the file # Not used for clean levels since cleaning may remove all data if clean_level == "none": - assert not test_inst.empty + assert not self.test_inst.empty else: pytest.skip("Download data not available") @@ -434,20 +447,21 @@ def test_load_empty(self, inst_dict): """ # Get the instrument information and update the date to be in the future - test_inst, date = initialize_test_inst_and_date(inst_dict) + self.test_inst, date = initialize_test_inst_and_date(inst_dict) date = dt.datetime(dt.datetime.utcnow().year + 100, 1, 1) # Make sure the strict time flag doesn't interfere with the load test - load_and_set_strict_time_flag(test_inst, date, raise_error=True) + load_and_set_strict_time_flag(self.test_inst, date, raise_error=True) # Check the empty status - assert test_inst.empty, "Data was loaded for a far-future time" - assert test_inst.meta == pysat.Meta(), "Meta data is not empty" - if test_inst.pandas_format: - assert all(test_inst.data == pds.DataFrame()), "Data not empty" + assert self.test_inst.empty, "Data was loaded for a far-future time" + assert self.test_inst.meta == pysat.Meta(), "Meta data is not empty" + if self.test_inst.pandas_format: + assert all(self.test_inst.data == pds.DataFrame()), "Data not empty" else: - assert test_inst.data.dims == xr.Dataset().dims, "Dims not empty" - assert test_inst.data.data_vars == xr.Dataset().data_vars, \ + assert self.test_inst.data.dims == xr.Dataset().dims, \ + "Dims not empty" + assert self.test_inst.data.data_vars == xr.Dataset().data_vars, \ "Data variables not empty" return @@ -466,16 +480,17 @@ def test_load_multiple_days(self, inst_dict): """ - test_inst, date = initialize_test_inst_and_date(inst_dict) - if len(test_inst.files.files) > 0: + self.test_inst, date = initialize_test_inst_and_date(inst_dict) + if len(self.test_inst.files.files) > 0: # Make sure the strict time flag doesn't interfere with # the load tests, and re-run with desired clean level - load_and_set_strict_time_flag(test_inst, date, raise_error=True, + load_and_set_strict_time_flag(self.test_inst, date, + raise_error=True, clean_off=False, set_end_date=True) # Make sure more than one day has been loaded - assert len(np.unique(test_inst.index.day)) > 1 + assert len(np.unique(self.test_inst.index.day)) > 1 else: pytest.skip("Download data not available") @@ -507,29 +522,29 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # messages at the current clean level, specified by `clean_level` if clean_level in clean_warn.keys(): # Only need to test if there are clean warnings for this level - test_inst, date = initialize_test_inst_and_date(inst_dict) + self.test_inst, date = initialize_test_inst_and_date(inst_dict) clean_warnings = clean_warn[clean_level] # Make sure the strict time flag doesn't interfere with # the cleaning tests - load_and_set_strict_time_flag(test_inst, date) + load_and_set_strict_time_flag(self.test_inst, date) # Cycle through each of the potential cleaning messages # for this Instrument module, inst ID, tag, and clean level for (clean_method, clean_method_level, clean_method_msg, final_level) in clean_warnings: - if len(test_inst.files.files) > 0: + if len(self.test_inst.files.files) > 0: # Set the clean level - test_inst.clean_level = clean_level + self.test_inst.clean_level = clean_level target = 'Fake Data to be cleared' - test_inst.data = [target] + self.test_inst.data = [target] if clean_method == 'logger': # A logging message is expected with caplog.at_level( getattr(logging, clean_method_level), logger='pysat'): - test_inst.load(date=date) + self.test_inst.load(date=date) # Test the returned message out_msg = caplog.text @@ -539,7 +554,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): elif clean_method == 'warning': # A warning message is expected with warnings.catch_warnings(record=True) as war: - test_inst.load(date=date) + self.test_inst.load(date=date) # Test the warning output testing.eval_warnings(war, [clean_method_msg], @@ -548,7 +563,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # An error message is expected, evaluate error # and the error message testing.eval_bad_input( - test_inst.load, clean_method_level, + self.test_inst.load, clean_method_level, clean_method_msg, input_kwargs={'date': date}) else: raise AttributeError( @@ -557,12 +572,12 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # Test to see if the clean flag has the expected value # afterwards - assert test_inst.clean_level == final_level, \ + assert self.test_inst.clean_level == final_level, \ "Clean level should now be {:s}, not {:s}".format( - final_level, test_inst.clean_level) + final_level, self.test_inst.clean_level) # Make sure fake data is cleared - assert target not in test_inst.data + assert target not in self.test_inst.data else: pytest.skip("".join(["Can't test clean warnings for ", "Instrument ", @@ -619,29 +634,29 @@ def test_load_w_pad(self, pad, inst_dict): else: pad_repr = repr(pad) - test_inst, date = initialize_test_inst_and_date(inst_dict) - if len(test_inst.files.files) > 0: + self.test_inst, date = initialize_test_inst_and_date(inst_dict) + if len(self.test_inst.files.files) > 0: # Make sure the strict time flag doesn't interfere with # the load tests - load_and_set_strict_time_flag(test_inst, date, raise_error=True, - clean_off=False) + load_and_set_strict_time_flag(self.test_inst, date, + raise_error=True, clean_off=False) - if test_inst.empty: + if self.test_inst.empty: # This will be empty if this is a forecast file that doesn't # include the load date - test_inst.pad = None - load_and_set_strict_time_flag(test_inst, date, raise_error=True, - clean_off=False) - assert not test_inst.empty, "No data on {:}".format(date) - assert test_inst.index.max() < date, \ + self.test_inst.pad = None + load_and_set_strict_time_flag(self.test_inst, date, + raise_error=True, clean_off=False) + assert not self.test_inst.empty, "No data on {:}".format(date) + assert self.test_inst.index.max() < date, \ "Padding should have left data and didn't" else: # Padding was successful, evaluate the data index length - assert (test_inst.index[-1] - - test_inst.index[0]).total_seconds() < 86400.0 + assert (self.test_inst.index[-1] + - self.test_inst.index[0]).total_seconds() < 86400.0 # Evaluate the recorded pad - inst_str = test_inst.__str__() + inst_str = self.test_inst.__str__() assert inst_str.find( 'Data Padding: {:s}'.format(pad_repr)) > 0, "".join([ "bad pad value: ", pad_repr, " not in ", inst_str]) @@ -663,11 +678,11 @@ def test_remote_file_list(self, inst_dict): """ - test_inst, date = initialize_test_inst_and_date(inst_dict) - name = '_'.join((test_inst.platform, test_inst.name)) + self.test_inst, date = initialize_test_inst_and_date(inst_dict) + name = '_'.join((self.test_inst.platform, self.test_inst.name)) if hasattr(getattr(self.inst_loc, name), 'list_remote_files'): - assert callable(test_inst.remote_file_list) + assert callable(self.test_inst.remote_file_list) # Check for username if 'user_info' in inst_dict.keys(): @@ -675,7 +690,8 @@ def test_remote_file_list(self, inst_dict): else: dl_dict = {} - files = test_inst.remote_file_list(start=date, stop=date, **dl_dict) + files = self.test_inst.remote_file_list(start=date, stop=date, + **dl_dict) # If test date is correctly chosen, files should exist assert len(files) > 0 From a5afed1b80c64ad9b9206c4685429d64e523c076 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 16:24:30 -0500 Subject: [PATCH 254/365] MAINT: remove deprecated labels --- pysat/_instrument.py | 73 +------------------------------ pysat/instruments/pysat_netcdf.py | 7 +-- pysat/utils/io.py | 36 +++------------ 3 files changed, 7 insertions(+), 109 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 959a98d6d..e95ffee4b 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -90,10 +90,6 @@ class Instrument(object): of files found will be checked to ensure the filesizes are greater than zero. Empty files are removed from the stored list of files. (default=False) - labels : dict or NoneType - Dict where keys are the label attribute names and the values are tuples - that have the label values and value types in that order. If None uses - the Meta defaults. Deprecated, use `meta_kwargs` (default=None) meta_kwargs : dict or NoneType Dict to specify custom Meta initialization (default=None) custom : list or NoneType @@ -252,7 +248,7 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', orbit_info=None, inst_module=None, data_dir='', directory_format=None, file_format=None, temporary_file_list=False, strict_time_flag=True, - ignore_empty_files=False, labels=None, meta_kwargs=None, + ignore_empty_files=False, meta_kwargs=None, custom=None, **kwargs): """Initialize `pysat.Instrument` object.""" @@ -429,12 +425,6 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', # use Instrument definition of MetaLabels over the Metadata declaration. self.meta_kwargs = {} if meta_kwargs is None else meta_kwargs - if labels is not None: - warnings.warn("".join(["`labels` is deprecated, use `meta_kwargs`", - "with the 'labels' key instead. Support ", - "for `labels` will be removed in v3.2.0+"]), - DeprecationWarning, stacklevel=2) - self.meta_kwargs["labels"] = labels self.meta = pysat.Meta(**self.meta_kwargs) self.meta.mutable = False @@ -1983,30 +1973,6 @@ def _filter_netcdf4_metadata(self, mdata_dict, coltype, remove=False, # ----------------------------------------------------------------------- # Define all accessible methods - @property - def meta_labels(self): - """Provide Meta input for labels kwarg, deprecated. - - Returns - ------- - dict - Either Meta default provided locally or custom value provided - by user and stored in `meta_kwargs['labels']` - - """ - warnings.warn("".join(["Deprecated attribute, returns `meta_kwargs", - "['labels']` or Meta defaults if not set. Will", - " be removed in pysat 3.2.0+"]), - DeprecationWarning, stacklevel=2) - if 'labels' in self.meta_kwargs.keys(): - return self.meta_kwargs['labels'] - else: - return {'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', (float, int)), - 'max_val': ('value_max', (float, int)), - 'fill_val': ('fill', (float, int, str))} - @property def bounds(self): """Boundaries for iterating over instrument object by date or file. @@ -2948,43 +2914,6 @@ def rename(self, mapper, lowercase_data_labels=False): return - def generic_meta_translator(self, input_meta): - """Convert the `input_meta` metadata into a dictionary. - - .. deprecated:: 3.0.2 - `generic_meta_translator` will be removed in the 3.2.0+ release. - - Parameters - ---------- - input_meta : pysat.Meta - The metadata object to translate - - Returns - ------- - export_dict : dict - A dictionary of the metadata for each variable of an output file - - Note - ---- - Uses the translation dict, if present, at `self._meta_translation_table` - to map existing metadata labels to a list of labels used in the - returned dict. - - """ - - dstr = ''.join(['This function has been deprecated. Please see ', - '`pysat.utils.io.apply_table_translation_to_file` and ', - '`self.meta.to_dict` to get equivalent functionality.']) - warnings.warn(dstr, DeprecationWarning, stacklevel=2) - - meta_dict = input_meta.to_dict() - trans_table = self._meta_translation_table - exp_dict = pysat.utils.io.apply_table_translation_to_file(self, - meta_dict, - trans_table) - - return exp_dict - def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, end_date=None, fname=None, stop_fname=None, verifyPad=False, **kwargs): diff --git a/pysat/instruments/pysat_netcdf.py b/pysat/instruments/pysat_netcdf.py index 2f42de4a1..2196ea56b 100644 --- a/pysat/instruments/pysat_netcdf.py +++ b/pysat/instruments/pysat_netcdf.py @@ -133,7 +133,7 @@ def download(date_array, tag, inst_id, data_path=None): def load(fnames, tag='', inst_id='', strict_meta=False, file_format='NETCDF4', epoch_name=None, epoch_unit='ms', epoch_origin='unix', pandas_format=True, decode_timedelta=False, meta_kwargs=None, - load_labels=None, meta_processor=None, meta_translation=None, + meta_processor=None, meta_translation=None, drop_meta_labels=None, decode_times=None): """Load pysat-created NetCDF data and meta data. @@ -183,10 +183,6 @@ def load(fnames, tag='', inst_id='', strict_meta=False, file_format='NETCDF4', meta_kwargs : dict or NoneType Dict to specify custom Meta initialization or None to use Meta defaults (default=None) - load_labels : dict or NoneType - Dict where keys are the label attribute names and the values are tuples - that have the label values and value types in that order or None to use - Meta defaults. Deprecated, use `meta_kwargs` instead. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -228,7 +224,6 @@ def load(fnames, tag='', inst_id='', strict_meta=False, file_format='NETCDF4', pandas_format=pandas_format, decode_timedelta=decode_timedelta, meta_kwargs=meta_kwargs, - labels=load_labels, meta_processor=meta_processor, meta_translation=meta_translation, drop_meta_labels=drop_meta_labels, diff --git a/pysat/utils/io.py b/pysat/utils/io.py index 63a4692fb..9a3045266 100644 --- a/pysat/utils/io.py +++ b/pysat/utils/io.py @@ -565,7 +565,7 @@ def meta_array_expander(meta_dict): def load_netcdf(fnames, strict_meta=False, file_format='NETCDF4', epoch_name=None, epoch_unit='ms', epoch_origin='unix', pandas_format=True, decode_timedelta=False, - combine_by_coords=True, meta_kwargs=None, labels=None, + combine_by_coords=True, meta_kwargs=None, meta_processor=None, meta_translation=None, drop_meta_labels=None, decode_times=None, strict_dim_check=True): @@ -616,10 +616,6 @@ def load_netcdf(fnames, strict_meta=False, file_format='NETCDF4', meta_kwargs : dict or NoneType Dict to specify custom Meta initialization or None to use Meta defaults (default=None) - labels : dict or NoneType - Dict where keys are the label attribute names and the values are tuples - that have the label values and value types in that order. None to use - meta defaults. Deprecated, use `meta_kwargs` instead. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -678,7 +674,7 @@ def load_netcdf(fnames, strict_meta=False, file_format='NETCDF4', epoch_name=epoch_name, epoch_unit=epoch_unit, epoch_origin=epoch_origin, - meta_kwargs=meta_kwargs, labels=labels, + meta_kwargs=meta_kwargs, meta_processor=meta_processor, meta_translation=meta_translation, drop_meta_labels=drop_meta_labels) @@ -690,7 +686,7 @@ def load_netcdf(fnames, strict_meta=False, file_format='NETCDF4', epoch_origin=epoch_origin, decode_timedelta=decode_timedelta, combine_by_coords=combine_by_coords, - meta_kwargs=meta_kwargs, labels=labels, + meta_kwargs=meta_kwargs, meta_processor=meta_processor, meta_translation=meta_translation, drop_meta_labels=drop_meta_labels, @@ -702,7 +698,7 @@ def load_netcdf(fnames, strict_meta=False, file_format='NETCDF4', def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4', epoch_name='Epoch', epoch_unit='ms', epoch_origin='unix', - meta_kwargs=None, labels=None, meta_processor=None, + meta_kwargs=None, meta_processor=None, meta_translation=None, drop_meta_labels=None): """Load netCDF-3/4 file produced by pysat in a pandas format. @@ -738,10 +734,6 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4', meta_kwargs : dict or NoneType Dict to specify custom Meta initialization or None to use Meta defaults (default=None) - labels : dict or NoneType - Dict where keys are the label attribute names and the values are tuples - that have the label values and value types in that order or None to use - Meta defaults. Deprecated, use `meta_kwargs` instead. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -800,13 +792,6 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4', if meta_kwargs is None: meta_kwargs = {} - if labels is not None: - warnings.warn("".join(["`labels` is deprecated, use `meta_kwargs`", - "with the 'labels' key instead. Support ", - "for `labels` will be removed in v3.2.0+"]), - DeprecationWarning, stacklevel=2) - meta_kwargs['labels'] = labels - meta = pysat.Meta(**meta_kwargs) # Store all metadata in a dict that may be filtered before @@ -909,7 +894,7 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4', def load_netcdf_xarray(fnames, strict_meta=False, file_format='NETCDF4', epoch_name='time', epoch_unit='ms', epoch_origin='unix', decode_timedelta=False, combine_by_coords=True, - meta_kwargs=None, labels=None, meta_processor=None, + meta_kwargs=None, meta_processor=None, meta_translation=None, drop_meta_labels=None, decode_times=False, strict_dim_check=True): """Load netCDF-3/4 file produced by pysat into an xarray Dataset. @@ -953,10 +938,6 @@ def load_netcdf_xarray(fnames, strict_meta=False, file_format='NETCDF4', meta_kwargs : dict or NoneType Dict to specify custom Meta initialization or None to use Meta defaults (default=None) - labels : dict or NoneType - Dict where keys are the label attribute names and the values are tuples - that have the label values and value types in that order or None to use - Meta defaults. Deprecated, use `meta_kwargs` instead. (default=None) meta_processor : function or NoneType If not None, a dict containing all of the loaded metadata will be passed to `meta_processor` which should return a filtered version @@ -1015,13 +996,6 @@ def load_netcdf_xarray(fnames, strict_meta=False, file_format='NETCDF4', if meta_kwargs is None: meta_kwargs = {} - if labels is not None: - warnings.warn("".join(["`labels` is deprecated, use `meta_kwargs`", - "with the 'labels' key instead. Support ", - "for `labels` will be removed in v3.2.0+"]), - DeprecationWarning, stacklevel=2) - meta_kwargs['labels'] = labels - meta = pysat.Meta(**meta_kwargs) # Store all metadata in a dict that may be filtered before From 4e6fff21f613713509058c8e22d17b3917d85a92 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 16:24:44 -0500 Subject: [PATCH 255/365] TST: remove tests for deprecations --- pysat/tests/classes/cls_instrument_access.py | 25 ------- pysat/tests/test_instrument.py | 75 -------------------- pysat/tests/test_utils_io.py | 64 ----------------- 3 files changed, 164 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py index 3570eb66d..afdf9b68a 100644 --- a/pysat/tests/classes/cls_instrument_access.py +++ b/pysat/tests/classes/cls_instrument_access.py @@ -1008,28 +1008,3 @@ def test_basic_variable_renaming(self, lowercase, mapper): assert key not in self.testInst.variables assert key not in self.testInst.meta.keys() return - - def test_generic_meta_translator(self): - """Test `generic_meta_translator`.""" - - # Get default meta translation table - trans_table = pysat.utils.io.default_to_netcdf_translation_table( - self.testInst) - - # Load data - self.testInst.load(date=self.ref_time) - - # Assign table - self.testInst._meta_translation_table = trans_table - - # Apply translation - trans_meta = self.testInst.generic_meta_translator(self.testInst.meta) - - # Perform equivalent via replacement functions - meta_dict = self.testInst.meta.to_dict() - truth_meta = pysat.utils.io.apply_table_translation_to_file( - self.testInst, meta_dict, trans_table=trans_table) - - assert np.all(truth_meta == trans_meta) - - return diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 6c132fdca..4c5c39977 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -541,81 +541,6 @@ def eval_warnings(self): testing.eval_warnings(self.war, self.warn_msgs) return - def test_instrument_labels(self): - """Test deprecation of `labels` kwarg in Instrument.""" - self.in_kwargs['labels'] = { - 'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', float), 'max_val': ('value_max', float), - 'fill_val': ('fill', float)} - - # Catch the warnings - with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(**self.in_kwargs) - - self.warn_msgs = np.array(["`labels` is deprecated, use `meta_kwargs`"]) - - # Evaluate the warning output - self.eval_warnings() - - # Evaluate the performance - assert float in tinst.meta.labels.label_type['fill_val'] - return - - @pytest.mark.parametrize('use_kwargs', [True, False]) - def test_instrument_meta_labels(self, use_kwargs): - """Test deprecation of `meta_labels` attribute in Instrument. - - Parameters - ---------- - use_kwargs : bool - If True, specify labels on input. If False, use defaults. - - """ - if use_kwargs: - self.in_kwargs['meta_kwargs'] = {'labels': { - 'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', float), - 'max_val': ('value_max', float), 'fill_val': ('fill', float)}} - - # Catch the warnings - with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(**self.in_kwargs) - labels = tinst.meta_labels - - self.warn_msgs = np.array(["Deprecated attribute, returns `meta_kwarg"]) - - # Evaluate the warning output - self.eval_warnings() - - # Evaluate the performance - if not use_kwargs: - self.in_kwargs['meta_kwargs'] = {'labels': { - 'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', (float, int)), - 'max_val': ('value_max', (float, int)), - 'fill_val': ('fill', (float, int, str))}} - - assert labels == self.in_kwargs['meta_kwargs']['labels'] - return - - def test_generic_meta_translator(self): - """Test deprecation of `generic_meta_translator`.""" - - # Catch the warnings - with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(**self.in_kwargs) - tinst.generic_meta_translator(tinst.meta) - - self.warn_msgs = np.array(["".join(["This function has been deprecated", - ". Please see "])]) - - # Evaluate the warning output - self.eval_warnings() - return - def test_filter_netcdf4_metadata(self): """Test deprecation warning generated by `_filter_netcdf4_metadata`.""" diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 210be40ea..f9d462031 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -1720,67 +1720,3 @@ def teardown_method(self): del self.test_inst, self.test_date, self.out, self.meta_dict return - - -class TestIODeprecation(object): - """Unit tests for deprecation warnings in `utils.io`.""" - - def setup_method(self): - """Set up the test environment.""" - - # Create temporary directory - self.tempdir = tempfile.TemporaryDirectory() - self.saved_path = pysat.params['data_dirs'] - pysat.params['data_dirs'] = self.tempdir.name - - self.outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc') - self.in_kwargs = {'labels': { - 'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', float), 'max_val': ('value_max', float), - 'fill_val': ('fill', float)}} - - return - - def teardown_method(self): - """Clean up the test environment.""" - - pysat.params['data_dirs'] = self.saved_path - - # Remove the temporary directory - self.tempdir.cleanup() - - # Clear the attributes - del self.tempdir, self.saved_path, self.outfile, self.in_kwargs - return - - @pytest.mark.parametrize("inst_name,load_func", [ - ("testing", io.load_netcdf_pandas), - ("ndtesting", io.load_netcdf_xarray)]) - def test_load_netcdf_labels(self, inst_name, load_func): - """Test deprecation of `labels` kwarg in different load functions. - - Parameters - ---------- - inst_name : str - Instrument name for test Instrument - load_func : function - NetCDF load method with deprecation warning - - """ - - # Create a test file - testInst = pysat.Instrument(platform='pysat', name=inst_name, - num_samples=100, update_files=True) - testInst.load(date=testInst.inst_module._test_dates['']['']) - io.inst_to_netcdf(testInst, fname=self.outfile) - - # Catch the warnings - with warnings.catch_warnings(record=True) as war: - load_func(self.outfile, **self.in_kwargs) - - # Test the warnings - assert len(war) >= 1 - testing.eval_warnings(war, - ["`labels` is deprecated, use `meta_kwargs`"]) - return From fd873b77462a9705694d3f27e6ff6c4069ad37da Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 16:25:42 -0500 Subject: [PATCH 256/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd5f33ba..40bdc8aa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed deprecated `use_header` kwarg from `load` and changed default behaviour to `use_header=True` * Use temporary directories for files created during test_utils.py + * Removed deprecated `labels` kwarg for `pysat.Instrument()` [3.1.0] - 2023-05-31 -------------------- From fcb9bd29f9722669f707ab7b2114fef8a19bbdea Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 16:38:56 -0500 Subject: [PATCH 257/365] MAINT: version cap for sphinx rtd theme --- pyproject.toml | 2 +- test_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eb0dbe732..77b28770a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ doc = [ "m2r2", "numpydoc", "sphinx", - "sphinx_rtd_theme >= 1.2.2" + "sphinx_rtd_theme >= 1.2.2, < 2.0.0" ] [project.urls] diff --git a/test_requirements.txt b/test_requirements.txt index 8a33141b8..e17e0e98a 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -9,4 +9,4 @@ pysatSpaceWeather pytest-cov pytest-ordering sphinx -sphinx_rtd_theme>=1.2.2 +sphinx_rtd_theme>=1.2.2,<2.0.0 From 0f2d461a7e3bc3bb6e9e74089385484375bba995 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 16:39:21 -0500 Subject: [PATCH 258/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04680ecc9..4dba06951 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed deprecated `freq` kwarg from `download` * Removed deprecated `use_header` kwarg from `load` and changed default behaviour to `use_header=True` + * Added verion cap for sphinx_rtd_theme [3.1.0] - 2023-05-31 -------------------- From b537eb97cbe1ab4ebdd354fa5e40cfc8cd78115f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 16:52:20 -0500 Subject: [PATCH 259/365] STY: teardown date --- pysat/tests/classes/cls_instrument_library.py | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 066ede16f..0c5419e2e 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -208,6 +208,8 @@ def teardown_method(self): if hasattr(self, "test_inst"): del self.test_inst + if hasattr(self, "date"): + del self.date return @@ -379,7 +381,7 @@ def test_download(self, inst_dict): """ - self.test_inst, date = initialize_test_inst_and_date(inst_dict) + self.test_inst, self.date = initialize_test_inst_and_date(inst_dict) # Check for username. if 'user_info' in inst_dict.keys(): @@ -387,7 +389,7 @@ def test_download(self, inst_dict): else: dl_dict = {} # Note this will download two consecutive days - self.test_inst.download(date, **dl_dict) + self.test_inst.download(self.date, **dl_dict) assert len(self.test_inst.files.files) > 0 return @@ -408,7 +410,7 @@ def test_load(self, clean_level, inst_dict): """ - self.test_inst, date = initialize_test_inst_and_date(inst_dict) + self.test_inst, self.date = initialize_test_inst_and_date(inst_dict) if len(self.test_inst.files.files) > 0: # Set the clean level self.test_inst.clean_level = clean_level @@ -417,7 +419,7 @@ def test_load(self, clean_level, inst_dict): # Make sure the strict time flag doesn't interfere with # the load tests, and re-run with desired clean level - load_and_set_strict_time_flag(self.test_inst, date, + load_and_set_strict_time_flag(self.test_inst, self.date, raise_error=True, clean_off=False) # Make sure fake data is cleared @@ -447,11 +449,12 @@ def test_load_empty(self, inst_dict): """ # Get the instrument information and update the date to be in the future - self.test_inst, date = initialize_test_inst_and_date(inst_dict) - date = dt.datetime(dt.datetime.utcnow().year + 100, 1, 1) + self.test_inst, self.date = initialize_test_inst_and_date(inst_dict) + self.date = dt.datetime(dt.datetime.utcnow().year + 100, 1, 1) # Make sure the strict time flag doesn't interfere with the load test - load_and_set_strict_time_flag(self.test_inst, date, raise_error=True) + load_and_set_strict_time_flag(self.test_inst, self.date, + raise_error=True) # Check the empty status assert self.test_inst.empty, "Data was loaded for a far-future time" @@ -480,12 +483,12 @@ def test_load_multiple_days(self, inst_dict): """ - self.test_inst, date = initialize_test_inst_and_date(inst_dict) + self.test_inst, self.date = initialize_test_inst_and_date(inst_dict) if len(self.test_inst.files.files) > 0: # Make sure the strict time flag doesn't interfere with # the load tests, and re-run with desired clean level - load_and_set_strict_time_flag(self.test_inst, date, + load_and_set_strict_time_flag(self.test_inst, self.date, raise_error=True, clean_off=False, set_end_date=True) @@ -522,12 +525,13 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # messages at the current clean level, specified by `clean_level` if clean_level in clean_warn.keys(): # Only need to test if there are clean warnings for this level - self.test_inst, date = initialize_test_inst_and_date(inst_dict) + self.test_inst, self.date = initialize_test_inst_and_date( + inst_dict) clean_warnings = clean_warn[clean_level] # Make sure the strict time flag doesn't interfere with # the cleaning tests - load_and_set_strict_time_flag(self.test_inst, date) + load_and_set_strict_time_flag(self.test_inst, self.date) # Cycle through each of the potential cleaning messages # for this Instrument module, inst ID, tag, and clean level @@ -544,7 +548,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): with caplog.at_level( getattr(logging, clean_method_level), logger='pysat'): - self.test_inst.load(date=date) + self.test_inst.load(date=self.date) # Test the returned message out_msg = caplog.text @@ -554,7 +558,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): elif clean_method == 'warning': # A warning message is expected with warnings.catch_warnings(record=True) as war: - self.test_inst.load(date=date) + self.test_inst.load(date=self.date) # Test the warning output testing.eval_warnings(war, [clean_method_msg], @@ -564,7 +568,8 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): # and the error message testing.eval_bad_input( self.test_inst.load, clean_method_level, - clean_method_msg, input_kwargs={'date': date}) + clean_method_msg, + input_kwargs={'date': self.date}) else: raise AttributeError( 'unknown type of warning: {:}'.format( @@ -634,21 +639,22 @@ def test_load_w_pad(self, pad, inst_dict): else: pad_repr = repr(pad) - self.test_inst, date = initialize_test_inst_and_date(inst_dict) + self.test_inst, self.date = initialize_test_inst_and_date(inst_dict) if len(self.test_inst.files.files) > 0: # Make sure the strict time flag doesn't interfere with # the load tests - load_and_set_strict_time_flag(self.test_inst, date, + load_and_set_strict_time_flag(self.test_inst, self.date, raise_error=True, clean_off=False) if self.test_inst.empty: # This will be empty if this is a forecast file that doesn't # include the load date self.test_inst.pad = None - load_and_set_strict_time_flag(self.test_inst, date, + load_and_set_strict_time_flag(self.test_inst, self.date, raise_error=True, clean_off=False) - assert not self.test_inst.empty, "No data on {:}".format(date) - assert self.test_inst.index.max() < date, \ + assert not self.test_inst.empty, \ + "No data on {:}".format(self.date) + assert self.test_inst.index.max() < self.date, \ "Padding should have left data and didn't" else: # Padding was successful, evaluate the data index length @@ -678,7 +684,7 @@ def test_remote_file_list(self, inst_dict): """ - self.test_inst, date = initialize_test_inst_and_date(inst_dict) + self.test_inst, self.date = initialize_test_inst_and_date(inst_dict) name = '_'.join((self.test_inst.platform, self.test_inst.name)) if hasattr(getattr(self.inst_loc, name), 'list_remote_files'): @@ -690,8 +696,8 @@ def test_remote_file_list(self, inst_dict): else: dl_dict = {} - files = self.test_inst.remote_file_list(start=date, stop=date, - **dl_dict) + files = self.test_inst.remote_file_list(start=self.date, + stop=self.date, **dl_dict) # If test date is correctly chosen, files should exist assert len(files) > 0 @@ -713,10 +719,10 @@ def test_download_warning(self, inst_dict): """ - test_inst, date = initialize_test_inst_and_date(inst_dict) + test_inst, self.date = initialize_test_inst_and_date(inst_dict) with warnings.catch_warnings(record=True) as war: - test_inst.download(date, date) + test_inst.download(self.date, self.date) assert len(war) >= 1 categories = [war[j].category for j in range(0, len(war))] From 5df6516e14926bab2947d5bb5e9e781567152a6e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 16:59:16 -0500 Subject: [PATCH 260/365] MAINT: sphinx_rtd_theme cap --- pyproject.toml | 2 +- test_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3b47b33d3..cbe394866 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ doc = [ "m2r2", "numpydoc", "sphinx", - "sphinx_rtd_theme >= 1.2.2" + "sphinx_rtd_theme >= 1.2.2, < 2.0.0" ] [project.urls] diff --git a/test_requirements.txt b/test_requirements.txt index 8a33141b8..e17e0e98a 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -9,4 +9,4 @@ pysatSpaceWeather pytest-cov pytest-ordering sphinx -sphinx_rtd_theme>=1.2.2 +sphinx_rtd_theme>=1.2.2,<2.0.0 From 1af593fca3c4050d017db0167d067ecd33487be4 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 29 Nov 2023 18:09:32 -0500 Subject: [PATCH 261/365] DOC: updated core code header Added the NRL distribution statement to the core code headers. --- pysat/_constellation.py | 4 ++++ pysat/_instrument.py | 6 ++++++ pysat/_meta.py | 4 ++++ pysat/_orbits.py | 4 ++++ pysat/_params.py | 4 ++++ 5 files changed, 22 insertions(+) diff --git a/pysat/_constellation.py b/pysat/_constellation.py index 3baf1e87b..56ce88122 100644 --- a/pysat/_constellation.py +++ b/pysat/_constellation.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Class for Instrument constellations. diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 959a98d6d..d205c0c63 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -2,7 +2,13 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- +"""Class for single instruments.""" + import copy import datetime as dt import errno diff --git a/pysat/_meta.py b/pysat/_meta.py index e44c2172c..55ba88a78 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Classes for storing and managing meta data.""" diff --git a/pysat/_orbits.py b/pysat/_orbits.py index dcbc604b2..a76126eb1 100644 --- a/pysat/_orbits.py +++ b/pysat/_orbits.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- import copy diff --git a/pysat/_params.py b/pysat/_params.py index 1bc5fc906..090764ae2 100644 --- a/pysat/_params.py +++ b/pysat/_params.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- import copy From b2d593d228e38d6d5679431b50989b0461e79f43 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 29 Nov 2023 18:10:16 -0500 Subject: [PATCH 262/365] DOC: updated test const headers Updated the headers in the test constellations to include the NRL distribution statement. --- pysat/constellations/single_test.py | 9 +++++++++ pysat/constellations/testing.py | 9 +++++++++ pysat/constellations/testing_empty.py | 9 +++++++++ pysat/constellations/testing_partial.py | 9 +++++++++ 4 files changed, 36 insertions(+) diff --git a/pysat/constellations/single_test.py b/pysat/constellations/single_test.py index c080cfedd..73f610c8a 100644 --- a/pysat/constellations/single_test.py +++ b/pysat/constellations/single_test.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Create a constellation with one testing instrument. Attributes diff --git a/pysat/constellations/testing.py b/pysat/constellations/testing.py index 1d03e6833..502ee2106 100644 --- a/pysat/constellations/testing.py +++ b/pysat/constellations/testing.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Create a constellation with 5 testing instruments. Attributes diff --git a/pysat/constellations/testing_empty.py b/pysat/constellations/testing_empty.py index 8da50b08d..30673b412 100644 --- a/pysat/constellations/testing_empty.py +++ b/pysat/constellations/testing_empty.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Create an empty constellation for testing. Attributes diff --git a/pysat/constellations/testing_partial.py b/pysat/constellations/testing_partial.py index d8b8bcb1b..d7bf71acc 100644 --- a/pysat/constellations/testing_partial.py +++ b/pysat/constellations/testing_partial.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Create a constellation where not all instruments have loadable data. Attributes From 253590e62823a82520968baabc3156aa3b573403 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 29 Nov 2023 18:10:49 -0500 Subject: [PATCH 263/365] DOC: updated test inst headers Updated the test instrument headers to include the NRL publication release. --- pysat/instruments/pysat_ndtesting.py | 9 +++++++++ pysat/instruments/pysat_netcdf.py | 4 ++++ pysat/instruments/pysat_testing.py | 9 +++++++++ pysat/instruments/pysat_testmodel.py | 9 +++++++++ pysat/instruments/templates/template_instrument.py | 4 ++++ 5 files changed, 35 insertions(+) diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 63c4dfd07..b0998996c 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Produces fake instrument data for testing.""" diff --git a/pysat/instruments/pysat_netcdf.py b/pysat/instruments/pysat_netcdf.py index 2f42de4a1..f247248fd 100644 --- a/pysat/instruments/pysat_netcdf.py +++ b/pysat/instruments/pysat_netcdf.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """General Instrument for loading pysat-written netCDF files. diff --git a/pysat/instruments/pysat_testing.py b/pysat/instruments/pysat_testing.py index 2779fd8ed..51fb3a1c9 100644 --- a/pysat/instruments/pysat_testing.py +++ b/pysat/instruments/pysat_testing.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Produces fake instrument data for testing.""" diff --git a/pysat/instruments/pysat_testmodel.py b/pysat/instruments/pysat_testmodel.py index f781bdcb1..9cd5e9e78 100644 --- a/pysat/instruments/pysat_testmodel.py +++ b/pysat/instruments/pysat_testmodel.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Produces fake instrument data for testing.""" diff --git a/pysat/instruments/templates/template_instrument.py b/pysat/instruments/templates/template_instrument.py index 4890fdf77..9a2fcc0d9 100644 --- a/pysat/instruments/templates/template_instrument.py +++ b/pysat/instruments/templates/template_instrument.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Template for a pysat.Instrument support file. From 6ccf94aa5c3a74a8962e203940d355a934f18c95 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 29 Nov 2023 18:11:14 -0500 Subject: [PATCH 264/365] DOC: updated inst method headers Updated the instrument method headers to include the NRL publication release. --- pysat/instruments/methods/general.py | 9 +++++++++ pysat/instruments/methods/testing.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py index ef0e0d9a4..1799bf5ff 100644 --- a/pysat/instruments/methods/general.py +++ b/pysat/instruments/methods/general.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Provides generalized routines for integrating instruments into pysat.""" diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 04154d2d6..00aa58575 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Standard functions for the test instruments.""" import datetime as dt From 76bea2b174aa3ae6fdf81777b6fef78c88a85568 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 29 Nov 2023 18:12:12 -0500 Subject: [PATCH 265/365] DOC: updated test headers Updated the unit test file headers to include the NRL publication release statement. --- pysat/tests/classes/cls_ci.py | 4 ++++ pysat/tests/classes/cls_instrument_access.py | 9 +++++++++ pysat/tests/classes/cls_instrument_integration.py | 9 +++++++++ pysat/tests/classes/cls_instrument_iteration.py | 9 +++++++++ pysat/tests/classes/cls_instrument_library.py | 9 +++++++++ pysat/tests/classes/cls_instrument_property.py | 9 +++++++++ pysat/tests/classes/cls_registration.py | 4 ++++ pysat/tests/test_constellation.py | 4 ++++ pysat/tests/test_files.py | 9 +++++++++ pysat/tests/test_instrument.py | 9 +++++++++ pysat/tests/test_instrument_custom.py | 9 +++++++++ pysat/tests/test_instrument_index.py | 9 +++++++++ pysat/tests/test_instrument_listgen.py | 9 +++++++++ pysat/tests/test_instrument_padding.py | 9 +++++++++ pysat/tests/test_instruments.py | 9 +++++++++ pysat/tests/test_meta.py | 4 ++++ pysat/tests/test_meta_header.py | 4 ++++ pysat/tests/test_meta_labels.py | 4 ++++ pysat/tests/test_methods_general.py | 9 +++++++++ pysat/tests/test_methods_testing.py | 9 +++++++++ pysat/tests/test_orbits.py | 4 ++++ pysat/tests/test_params.py | 4 ++++ pysat/tests/test_registry.py | 4 ++++ pysat/tests/test_utils.py | 4 ++++ pysat/tests/test_utils_coords.py | 4 ++++ pysat/tests/test_utils_files.py | 4 ++++ pysat/tests/test_utils_io.py | 4 ++++ pysat/tests/test_utils_testing.py | 4 ++++ pysat/tests/test_utils_time.py | 4 ++++ 29 files changed, 186 insertions(+) diff --git a/pysat/tests/classes/cls_ci.py b/pysat/tests/classes/cls_ci.py index 8fd3285ce..60717a748 100644 --- a/pysat/tests/classes/cls_ci.py +++ b/pysat/tests/classes/cls_ci.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Class setup and teardown for unit tests that are only run in the CI env.""" diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py index 3570eb66d..809868ed1 100644 --- a/pysat/tests/classes/cls_instrument_access.py +++ b/pysat/tests/classes/cls_instrument_access.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Tests for data access and related functions in the pysat Instrument object. Includes: diff --git a/pysat/tests/classes/cls_instrument_integration.py b/pysat/tests/classes/cls_instrument_integration.py index e76a0f7e8..e066e7c1b 100644 --- a/pysat/tests/classes/cls_instrument_integration.py +++ b/pysat/tests/classes/cls_instrument_integration.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Integration tests for pysat.Instrument. Note diff --git a/pysat/tests/classes/cls_instrument_iteration.py b/pysat/tests/classes/cls_instrument_iteration.py index 2a960f758..23fed48ff 100644 --- a/pysat/tests/classes/cls_instrument_iteration.py +++ b/pysat/tests/classes/cls_instrument_iteration.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Tests for iteration in the pysat Instrument object and methods. Note diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index d7bdef0a4..6b8ddb3b9 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Standardized class and functions to test instruments for pysat libraries. Note diff --git a/pysat/tests/classes/cls_instrument_property.py b/pysat/tests/classes/cls_instrument_property.py index 5908fb720..5eb778038 100644 --- a/pysat/tests/classes/cls_instrument_property.py +++ b/pysat/tests/classes/cls_instrument_property.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Test for instrument properties in the pysat Instrument object and methods. Note diff --git a/pysat/tests/classes/cls_registration.py b/pysat/tests/classes/cls_registration.py index 7d21a4a28..537399087 100644 --- a/pysat/tests/classes/cls_registration.py +++ b/pysat/tests/classes/cls_registration.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Standardized class and functions to test registration for pysat libraries. diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py index 0712de18f..6c961625f 100644 --- a/pysat/tests/test_constellation.py +++ b/pysat/tests/test_constellation.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Unit tests for the Constellation class.""" diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py index c475b7cf9..942070f49 100644 --- a/pysat/tests/test_files.py +++ b/pysat/tests/test_files.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Test pysat Files object and code.""" import datetime as dt diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 6c132fdca..03e6a76e5 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Tests the pysat Instrument object and methods.""" diff --git a/pysat/tests/test_instrument_custom.py b/pysat/tests/test_instrument_custom.py index ab8f10caa..106049df3 100644 --- a/pysat/tests/test_instrument_custom.py +++ b/pysat/tests/test_instrument_custom.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Unit tests for the `custom_attach` methods for `pysat.Instrument`.""" import copy diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py index a36d8a68b..fc7d259f6 100644 --- a/pysat/tests/test_instrument_index.py +++ b/pysat/tests/test_instrument_index.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Unit tests for the `pysat.Instrument.index` attribute.""" import datetime as dt diff --git a/pysat/tests/test_instrument_listgen.py b/pysat/tests/test_instrument_listgen.py index 7932651b0..d208495a0 100644 --- a/pysat/tests/test_instrument_listgen.py +++ b/pysat/tests/test_instrument_listgen.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Unit tests for the list generation methods in `pysat.Instrument`.""" from importlib import reload diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py index a359916b6..5002a37db 100644 --- a/pysat/tests/test_instrument_padding.py +++ b/pysat/tests/test_instrument_padding.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Unit tests for the padding methods in `pysat.Instrument`.""" import datetime as dt diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index 4f7f87df8..1906a0944 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Unit and Integration Tests for each instrument module. Note diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 804278828..fe447a3fc 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the pysat Meta object.""" diff --git a/pysat/tests/test_meta_header.py b/pysat/tests/test_meta_header.py index 376777af9..43944ee81 100644 --- a/pysat/tests/test_meta_header.py +++ b/pysat/tests/test_meta_header.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the pysat MetaHeader object.""" diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py index 2e145f82d..db8af4174 100644 --- a/pysat/tests/test_meta_labels.py +++ b/pysat/tests/test_meta_labels.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the pysat MetaLabels object.""" diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py index 3272b5d63..e99dbf615 100644 --- a/pysat/tests/test_methods_general.py +++ b/pysat/tests/test_methods_general.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Unit tests for the general instrument methods.""" import datetime as dt diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py index 5c2818037..9332e60ad 100644 --- a/pysat/tests/test_methods_testing.py +++ b/pysat/tests/test_methods_testing.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. +# ---------------------------------------------------------------------------- """Tests the `pysat.instruments.methods.testing` methods.""" import datetime as dt diff --git a/pysat/tests/test_orbits.py b/pysat/tests/test_orbits.py index 85525ffcb..edd17ad14 100644 --- a/pysat/tests/test_orbits.py +++ b/pysat/tests/test_orbits.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Test the pysat routines for the orbits class.""" diff --git a/pysat/tests/test_params.py b/pysat/tests/test_params.py index 5ee483c24..7530a02a9 100644 --- a/pysat/tests/test_params.py +++ b/pysat/tests/test_params.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the pysat parameters storage area.""" diff --git a/pysat/tests/test_registry.py b/pysat/tests/test_registry.py index 0501f7b1b..09d9b299d 100644 --- a/pysat/tests/test_registry.py +++ b/pysat/tests/test_registry.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the registration of user-defined modules.""" diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index 3aeb87d81..845b77d74 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the pysat utils core functions.""" diff --git a/pysat/tests/test_utils_coords.py b/pysat/tests/test_utils_coords.py index d63077cc2..bcc00d3d3 100644 --- a/pysat/tests/test_utils_coords.py +++ b/pysat/tests/test_utils_coords.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the `pysat.utils.coords` functions.""" diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 48772ec38..01227a821 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the `pysat.utils.files` functions.""" diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 210be40ea..9481c0298 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the pysat utility io routines.""" import copy diff --git a/pysat/tests/test_utils_testing.py b/pysat/tests/test_utils_testing.py index f5c7fed73..574d6439b 100644 --- a/pysat/tests/test_utils_testing.py +++ b/pysat/tests/test_utils_testing.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the pysat utility testing routines.""" diff --git a/pysat/tests/test_utils_time.py b/pysat/tests/test_utils_time.py index 8b5c5714f..c12d86ee8 100644 --- a/pysat/tests/test_utils_time.py +++ b/pysat/tests/test_utils_time.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests for the pysat.utils.time functions.""" From 00eb54971c08540285cf68552f24fde0d7525c71 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 29 Nov 2023 18:12:41 -0500 Subject: [PATCH 266/365] DOC: updated utils headers Updated the headers in the utils files, including the NRL publication release statement. --- pysat/utils/_core.py | 4 ++++ pysat/utils/coords.py | 4 ++++ pysat/utils/files.py | 4 ++++ pysat/utils/io.py | 4 ++++ pysat/utils/registry.py | 4 ++++ pysat/utils/testing.py | 4 ++++ pysat/utils/time.py | 4 ++++ 7 files changed, 28 insertions(+) diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index cd32629f2..3b41f2fa0 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- import datetime as dt diff --git a/pysat/utils/coords.py b/pysat/utils/coords.py index 8c5a142ff..50ad28425 100644 --- a/pysat/utils/coords.py +++ b/pysat/utils/coords.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Coordinate transformation functions for pysat.""" diff --git a/pysat/utils/files.py b/pysat/utils/files.py index 93f3b376e..0f03e9f3c 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Utilities for file management and parsing file names.""" diff --git a/pysat/utils/io.py b/pysat/utils/io.py index 63a4692fb..be22eb58b 100644 --- a/pysat/utils/io.py +++ b/pysat/utils/io.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Input/Output utilities for pysat data.""" import copy diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py index a75a212b2..4ef41f6d6 100644 --- a/pysat/utils/registry.py +++ b/pysat/utils/registry.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """pysat user module registry utilities. diff --git a/pysat/utils/testing.py b/pysat/utils/testing.py index e654ab944..bf9e0f795 100644 --- a/pysat/utils/testing.py +++ b/pysat/utils/testing.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Utilities to perform common evaluations.""" diff --git a/pysat/utils/time.py b/pysat/utils/time.py index 6ffa84d12..a9fef7576 100644 --- a/pysat/utils/time.py +++ b/pysat/utils/time.py @@ -2,6 +2,10 @@ # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Date and time handling utilities.""" From 9e35f552597bd5dfa6f901e70e4a10384f48422e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 29 Nov 2023 18:13:41 -0500 Subject: [PATCH 267/365] DOC: updated changelog Updated the changelog to encompass this pull request. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd5f33ba..7135610db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed deprecated `use_header` kwarg from `load` and changed default behaviour to `use_header=True` * Use temporary directories for files created during test_utils.py + * Updated code file headers to be consistent and include NRL pub release [3.1.0] - 2023-05-31 -------------------- From 01401a2eb48fa1ff8af6b110008c777a27181dce Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 29 Nov 2023 20:34:29 -0500 Subject: [PATCH 268/365] Update pysat/tests/classes/cls_instrument_library.py Co-authored-by: Angeline Burrell --- pysat/tests/classes/cls_instrument_library.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 0c5419e2e..51b7bb631 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -200,16 +200,15 @@ def teardown_class(self): def setup_method(self): """Initialize parameters before each method.""" + self.test_inst = None + self.date = None return def teardown_method(self): """Clean up any instruments that were initialized.""" - if hasattr(self, "test_inst"): - del self.test_inst - if hasattr(self, "date"): - del self.date + del self.test_inst, self.date return From 955fd415ffbe283c025a9a4bd970b3dd4341d556 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 20:44:53 -0500 Subject: [PATCH 269/365] BUG: update syntax --- pysat/tests/test_meta.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 804278828..d249004f9 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -247,7 +247,8 @@ def test_init_labels_w_int_default(self): self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing', 'tag': 'default_meta', 'clean_level': 'clean', - 'labels': self.meta_labels}) + 'meta_kwargs': + {'labels': self.meta_labels}}) # Test the warning default_str = ''.join(['Metadata set to defaults, as they were', @@ -828,7 +829,7 @@ def test_assign_nonstandard_metalabels(self, inst_name): # Assign meta data with non-standard labels self.set_meta(inst_kwargs={'platform': 'pysat', 'name': inst_name, - 'labels': self.meta_labels}) + 'meta_kwargs': {'labels': self.meta_labels}}) # Test that standard attributes are missing and non-standard # attributes are present @@ -1174,7 +1175,7 @@ def test_set_wrong_case(self, num_dvals): # Set the meta object self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing', - 'labels': self.meta_labels}) + 'meta_kwargs': {'labels': self.meta_labels}}) # Set data using lower case labels dvals = self.testInst.vars_no_time[:num_dvals] From c64e72deabc38af270994ebd92e26aa2061414b9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 21:02:21 -0500 Subject: [PATCH 270/365] MAINT: remove deprecated load_netcdf4 --- pysat/tests/test_utils.py | 44 ---------------- pysat/utils/__init__.py | 1 - pysat/utils/_core.py | 103 -------------------------------------- 3 files changed, 148 deletions(-) diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index 3aeb87d81..444e792d1 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -778,50 +778,6 @@ def test_list_kwargs_passthrough(self): return -class TestDeprecation(object): - """Unit test for deprecation warnings.""" - - @pytest.mark.parametrize("kwargs,msg_inds", - [({'fnames': None}, [0, 1]), - ({'fnames': 'no_file', 'file_format': None}, - [0, 2])]) - def test_load_netcdf4(self, kwargs, msg_inds): - """Test deprecation warnings from load_netcdf4. - - Parameters - ---------- - kwargs : dict - Keyword arguments passed to `load_netcdf4` - msg_inds : list - List of indices indicating which warning message is expected - - """ - with warnings.catch_warnings(record=True) as war: - try: - # Generate relocation warning and file_format warning - utils.load_netcdf4(**kwargs) - except (FileNotFoundError, ValueError): - pass - - warn_msgs = ["".join(["function moved to `pysat.utils.io`, ", - "deprecated wrapper will be removed in ", - "pysat 3.2.0+"]), - "".join(["`fnames` as a kwarg has been deprecated, ", - "must supply a string or list of strings", - " in 3.2.0+"]), - "".join(["`file_format` must be a string value in ", - "3.2.0+, instead of None use 'NETCDF4' ", - "for same behavior."])] - - warn_msgs = [warn_msgs[ind] for ind in msg_inds] - # Ensure the minimum number of warnings were raised - assert len(war) >= len(warn_msgs) - - # Test the warning messages, ensuring each attribute is present - utils.testing.eval_warnings(war, warn_msgs) - return - - class TestMappedValue(object): """Unit tests for utility `get_mapped_value`.""" diff --git a/pysat/utils/__init__.py b/pysat/utils/__init__.py index 9badbae4e..98f33c23a 100644 --- a/pysat/utils/__init__.py +++ b/pysat/utils/__init__.py @@ -12,7 +12,6 @@ from pysat.utils._core import generate_instrument_list from pysat.utils._core import get_mapped_value from pysat.utils._core import listify -from pysat.utils._core import load_netcdf4 from pysat.utils._core import NetworkLock from pysat.utils._core import scale_units from pysat.utils._core import stringify diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index cd32629f2..333d178ad 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -190,109 +190,6 @@ def stringify(strlike): return strlike -def load_netcdf4(fnames=None, strict_meta=False, file_format='NETCDF4', - epoch_name='Epoch', epoch_unit='ms', epoch_origin='unix', - pandas_format=True, decode_timedelta=False, - labels={'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', np.float64), - 'max_val': ('value_max', np.float64), - 'fill_val': ('fill', np.float64)}): - """Load netCDF-3/4 file produced by pysat. - - .. deprecated:: 3.0.2 - Function moved to `pysat.utils.io.load_netcdf`, this wrapper will be - removed in the 3.2.0+ release. - No longer allow non-string file formats in the 3.2.0+ release. - - Parameters - ---------- - fnames : str, array_like, or NoneType - Filename(s) to load, will fail if None (default=None) - strict_meta : bool - Flag that checks if metadata across fnames is the same if True - (default=False) - file_format : str - file_format keyword passed to netCDF4 routine. Expects one of - 'NETCDF3_CLASSIC', 'NETCDF3_64BIT', 'NETCDF4_CLASSIC', or 'NETCDF4'. - (default='NETCDF4') - epoch_name : str - Data key for epoch variable. The epoch variable is expected to be an - array of integer or float values denoting time elapsed from an origin - specified by `epoch_origin` with units specified by `epoch_unit`. This - epoch variable will be converted to a `DatetimeIndex` for consistency - across pysat instruments. (default='Epoch') - epoch_unit : str - The pandas-defined unit of the epoch variable ('D', 's', 'ms', 'us', - 'ns'). (default='ms') - epoch_origin : str or timestamp-convertable - Origin of epoch calculation, following convention for - `pandas.to_datetime`. Accepts timestamp-convertable objects, as well as - two specific strings for commonly used calendars. These conversions are - handled by `pandas.to_datetime`. - If ‘unix’ (or POSIX) time; origin is set to 1970-01-01. - If ‘julian’, `epoch_unit` must be ‘D’, and origin is set to beginning of - Julian Calendar. Julian day number 0 is assigned to the day starting at - noon on January 1, 4713 BC. (default='unix') - pandas_format : bool - Flag specifying if data is stored in a pandas DataFrame (True) or - xarray Dataset (False). (default=False) - decode_timedelta : bool - Used for xarray datasets. If True, variables with unit attributes that - are 'timelike' ('hours', 'minutes', etc) are converted to - `np.timedelta64`. (default=False) - labels : dict - Dict where keys are the label attribute names and the values are tuples - that have the label values and value types in that order. - (default={'units': ('units', str), 'name': ('long_name', str), - 'notes': ('notes', str), 'desc': ('desc', str), - 'min_val': ('value_min', np.float64), - 'max_val': ('value_max', np.float64), 'fill_val': ('fill', np.float64)}) - - Returns - ------- - data : pandas.DataFrame or xarray.Dataset - Class holding file data - meta : pysat.Meta - Class holding file meta data - - Raises - ------ - ValueError - If kwargs that should be args are not set on instantiation. - KeyError - If epoch/time dimension could not be identified. - - """ - warnings.warn("".join(["function moved to `pysat.utils.io`, deprecated ", - "wrapper will be removed in pysat 3.2.0+"]), - DeprecationWarning, stacklevel=2) - - if fnames is None: - warnings.warn("".join(["`fnames` as a kwarg has been deprecated, must ", - "supply a string or list of strings in 3.2.0+"]), - DeprecationWarning, stacklevel=2) - raise ValueError("Must supply a filename/list of filenames") - - if file_format is None: - warnings.warn("".join(["`file_format` must be a string value in ", - "3.2.0+, instead of None use 'NETCDF4' for ", - "same behavior."]), - DeprecationWarning, stacklevel=2) - file_format = 'NETCDF4' - - data, meta = pysat.utils.io.load_netcdf(fnames, strict_meta=strict_meta, - file_format=file_format, - epoch_name=epoch_name, - epoch_unit=epoch_unit, - epoch_origin=epoch_origin, - pandas_format=pandas_format, - decode_timedelta=decode_timedelta, - labels=labels) - - return data, meta - - def get_mapped_value(value, mapper): """Adjust value using mapping dict or function. From 3f0dff1b12a11ebe1295e30f3b15358b4f419f37 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 21:18:30 -0500 Subject: [PATCH 271/365] MAINT: remove deprecated functions --- pysat/_instrument.py | 85 ++-------------------------------- pysat/tests/test_instrument.py | 65 -------------------------- 2 files changed, 5 insertions(+), 145 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index e95ffee4b..596b91dc3 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -252,17 +252,9 @@ def __init__(self, platform=None, name=None, tag='', inst_id='', custom=None, **kwargs): """Initialize `pysat.Instrument` object.""" - # Check for deprecated usage of None - if None in [tag, inst_id]: - warnings.warn(" ".join(["The usage of None in `tag` and `inst_id`", - "has been deprecated and will be removed", - "in 3.2.0+. Please use '' instead of", - "None."]), - DeprecationWarning, stacklevel=2) - # Set default tag, inst_id, and Instrument module - self.tag = '' if tag is None else tag.lower() - self.inst_id = '' if inst_id is None else inst_id.lower() + self.tag = tag.lower() + self.inst_id = inst_id.lower() self.inst_module = inst_module @@ -1912,64 +1904,6 @@ def _get_data_info(self, data): return data, data_type, datetime_flag - def _filter_netcdf4_metadata(self, mdata_dict, coltype, remove=False, - export_nan=None): - """Filter metadata properties to be consistent with netCDF4. - - .. deprecated:: 3.0.2 - Moved to `pysat.utils.io.filter_netcdf4_metadata. This wrapper - will be removed in 3.2.0+. - - Parameters - ---------- - mdata_dict : dict - Dictionary equivalent to Meta object info - coltype : type - Data type provided by `pysat.Instrument._get_data_info` - remove : bool - Removes FillValue and associated parameters disallowed for strings - (default=False) - export_nan : list or NoneType - Metadata parameters allowed to be NaN (default=None) - - Returns - ------- - dict - Modified as needed for netCDf4 - - Warnings - -------- - UserWarning - When data removed due to conflict between value and type - - Note - ---- - Remove forced to True if coltype consistent with a string type - - Metadata values that are NaN and not listed in export_nan are removed. - - See Also - -------- - pysat.utils.io.filter_netcdf4_metadata - - """ - warnings.warn("".join(["`pysat.Instrument._filter_netcdf4_metadata` ", - "has been deprecated and will be removed ", - "in pysat 3.2.0+. Use `pysat.utils.io.", - "filter_netcdf4_metadata` instead."]), - DeprecationWarning, stacklevel=2) - - if remove: - check_type = [self.meta.labels.fill_val, self.meta.labels.max_val, - self.meta.labels.min_val] - else: - check_type = None - - return pysat.utils.io.filter_netcdf4_metadata(self, mdata_dict, coltype, - remove=remove, - check_type=check_type, - export_nan=export_nan) - # ----------------------------------------------------------------------- # Define all accessible methods @@ -3738,20 +3672,16 @@ def download(self, start=None, stop=None, date_array=None, return - def to_netcdf4(self, fname=None, base_instrument=None, epoch_name=None, + def to_netcdf4(self, fname, base_instrument=None, epoch_name=None, zlib=False, complevel=4, shuffle=True, preserve_meta_case=False, export_nan=None, export_pysat_info=True, unlimited_time=True, modify=False): """Store loaded data into a netCDF4 file. - .. deprecated:: 3.0.2 - Changed `fname` from a kwarg to an arg of type str in the 3.2.0+ - release. - Parameters ---------- - fname : str or NoneType - Full path to save instrument object to (default=None) + fname : str + Full path to save instrument object to netCDF base_instrument : pysat.Instrument or NoneType Class used as a comparison, only attributes that are present with self and not on base_instrument are written to netCDF. Using None @@ -3805,11 +3735,6 @@ def to_netcdf4(self, fname=None, base_instrument=None, epoch_name=None, pysat.utils.io.to_netcdf """ - if fname is None: - warnings.warn("".join(["`fname` as a kwarg has been deprecated, ", - "must supply a filename 3.2.0+"]), - DeprecationWarning, stacklevel=2) - raise ValueError("Must supply an output filename") # Prepare the instrument object used to create the output file inst = self if modify else self.copy() diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 4c5c39977..c51ae895d 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -541,71 +541,6 @@ def eval_warnings(self): testing.eval_warnings(self.war, self.warn_msgs) return - def test_filter_netcdf4_metadata(self): - """Test deprecation warning generated by `_filter_netcdf4_metadata`.""" - - # Catch the warnings - with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(**self.in_kwargs) - tinst.load(date=self.ref_time) - mdata_dict = tinst.meta._data.to_dict() - tinst._filter_netcdf4_metadata(mdata_dict, - coltype='str') - - self.warn_msgs = np.array(["".join(["`pysat.Instrument.", - "_filter_netcdf4_metadata` ", - "has been deprecated and will be ", - "removed in pysat 3.2.0+. Use ", - "`pysat.utils.io.filter_netcdf4_", - "metadata` instead."])]) - - # Evaluate the warning output - self.eval_warnings() - return - - def test_to_netcdf4(self): - """Test deprecation warning generated by `to_netcdf4`.""" - - # Catch the warnings - with warnings.catch_warnings(record=True) as self.war: - tinst = pysat.Instrument(**self.in_kwargs) - try: - tinst.to_netcdf4() - except ValueError: - pass - - self.warn_msgs = np.array(["".join(["`fname` as a kwarg has been ", - "deprecated, must supply a ", - "filename 3.2.0+"])]) - - # Evaluate the warning output - self.eval_warnings() - return - - @pytest.mark.parametrize("kwargs", [{'inst_id': None}, {'tag': None}]) - def test_inst_init_with_none(self, kwargs): - """Check that instantiation with None raises a DeprecationWarning. - - Parameters - ---------- - kwargs : dict - Dictionary of optional kwargs to pass through for instrument - instantiation. - - """ - - with warnings.catch_warnings(record=True) as self.war: - pysat.Instrument('pysat', 'testing', **kwargs) - - self.warn_msgs = np.array(["".join(["The usage of None in `tag` and ", - "`inst_id` has been deprecated ", - "and will be removed in 3.2.0+. ", - "Please use '' instead of None."])]) - - # Evaluate the warning output - self.eval_warnings() - return - # TODO(#1020): Remove test when keyword `use_header` is removed. def test_load_use_header(self): """Test that user is informed of MetaHeader on load.""" From 08092e967c7199568604f45056c6ee32c73c3e53 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 29 Nov 2023 21:21:32 -0500 Subject: [PATCH 272/365] DOC: update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40bdc8aa9..6b5e09931 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,10 @@ This project adheres to [Semantic Versioning](https://semver.org/). behaviour to `use_header=True` * Use temporary directories for files created during test_utils.py * Removed deprecated `labels` kwarg for `pysat.Instrument()` + * Removed deprecated `utils.load_netcdf4` method + * Removed deprecated `_filter_netcdf4_metadata` method + * Removed deprecated usage of None for tag and inst_id + * Removed deprecated kwarg behaviour for 'fname' in `to_netCDF4` [3.1.0] - 2023-05-31 -------------------- From 0be4c4054bad7d64f34eb016002debf83521f7a4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Nov 2023 11:14:26 -0500 Subject: [PATCH 273/365] DOC: update version number --- pyproject.toml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cbe394866..645dcd67b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pysat" -version = "3.1.0" +version = "3.2.0" description = "Supports science analysis across disparate data platforms" readme = "README.md" requires-python = ">=3.6" diff --git a/setup.cfg b/setup.cfg index 84e7c49c8..25ccaf201 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pysat -version = 3.1.0 +version = 3.2.0 [flake8] max-line-length = 80 From b397b51c5a5f7a92992e2e6451f38593e29696bf Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Nov 2023 11:17:04 -0500 Subject: [PATCH 274/365] DOC: update classifiers --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 645dcd67b..e93b0458a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,9 +21,9 @@ classifiers = [ "License :: OSI Approved :: BSD License", "Natural Language :: English", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", "Operating System :: Microsoft :: Windows" From 027ea42b706586a3af7982cf3294ea9761dee267 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Nov 2023 11:30:08 -0500 Subject: [PATCH 275/365] BUG: tests required --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 31a3eaf4d..54745e981 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,7 +3,6 @@ include *.md include *.txt include LICENSE include pysat/citation.txt -prune pysat/tests prune docs global-exclude *.pdf global-exclude *.png From 1f2df6c347506b05fef0449a9370e0519205c522 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 30 Nov 2023 12:21:23 -0500 Subject: [PATCH 276/365] DOC: typos --- docs/new_instrument.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 2351f6107..091c373e8 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -914,7 +914,7 @@ but should not undergo automated download tests because it would require the user to save a password in a potentially public location. The :py:attr:`_password_req` flag is used to skip both the download tests and the download warning message tests, since a functional download routine is -present. +present. This flag is defaults to False if not specified. .. code:: python @@ -925,4 +925,4 @@ present. inst_ids = {'': ['Level_1', 'Level_2']} _test_dates = {'': {'Level_1': dt.datetime(2020, 1, 1), 'Level_2': dt.datetime(2020, 1, 1)}} - _password_req = {'': {'Level_1': False}} + _password_req = {'': {'Level_1': True}} From 75c13bb053ad53db04023557ebdc3248b0e3a40f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 30 Nov 2023 17:03:28 -0500 Subject: [PATCH 277/365] Apply suggestions from code review Co-authored-by: Angeline Burrell --- .github/workflows/stats.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index eed4565a7..74e5b09ea 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | - pip install .test + pip install .[test] pip install `awk '{print $1}' ecosystem.txt` - name: Set up pysat diff --git a/pyproject.toml b/pyproject.toml index e93b0458a..d52f2fff8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ dependencies = [ [project.optional-dependencies] test = [ - "coveralls < 3.3", + "coveralls", "flake8", "flake8-docstrings", "hacking >= 1.0", From 2f6056144733dd1afc3b6bd64103370dd3fc6448 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Dec 2023 17:34:23 -0500 Subject: [PATCH 278/365] BUG: expand pysat version support The importlib module doesn't have the `resources.files` class until version 3.9. --- pysat/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysat/__init__.py b/pysat/__init__.py index beb1da02d..ab25a4f4a 100644 --- a/pysat/__init__.py +++ b/pysat/__init__.py @@ -37,6 +37,10 @@ try: from importlib import metadata from importlib import resources + + if not hasattr(resources, 'files'): + # The `files` object was introduced in Python 3.9 + resources = None except ImportError: import importlib_metadata as metadata resources = None From 0329b936a1a2b3e86b13614c5d896ddd9b79d674 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Dec 2023 17:40:20 -0500 Subject: [PATCH 279/365] TST: skip multiple load days for forecasts If an instrument is downloading forecasted or realtime data, multiple day loads cannot be supported in unit tests using available data. --- pysat/tests/classes/cls_instrument_library.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 51b7bb631..34987c923 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -484,15 +484,21 @@ def test_load_multiple_days(self, inst_dict): self.test_inst, self.date = initialize_test_inst_and_date(inst_dict) if len(self.test_inst.files.files) > 0: - - # Make sure the strict time flag doesn't interfere with - # the load tests, and re-run with desired clean level - load_and_set_strict_time_flag(self.test_inst, self.date, - raise_error=True, - clean_off=False, set_end_date=True) - - # Make sure more than one day has been loaded - assert len(np.unique(self.test_inst.index.day)) > 1 + if self.date < self.test_inst.today(): + # Make sure the strict time flag doesn't interfere with + # the load tests, and re-run with desired clean level + load_and_set_strict_time_flag(self.test_inst, self.date, + raise_error=True, clean_off=False, + set_end_date=True) + + # Make sure more than one day has been loaded + assert hasattr(self.test_inst.index, 'day'), \ + "No data to load for {:}-{:}".format( + self.date, self.date + dt.timedelta(days=2)) + assert len(np.unique(self.test_inst.index.day)) > 1 + else: + pytest.skip("".join(["Can't download multiple days of real-", + "time or forecast data"])) else: pytest.skip("Download data not available") From bbe559c675c498983d08fc67033dcd7227a21c29 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:46:07 -0800 Subject: [PATCH 280/365] Update README.md --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index eef03facd..be085004b 100644 --- a/README.md +++ b/README.md @@ -10,16 +10,20 @@ [![Coverage Status](https://coveralls.io/repos/github/pysat/pysat/badge.svg?branch=main)](https://coveralls.io/github/pysat/pysat?branch=main) [![DOI](https://zenodo.org/badge/33449914.svg)](https://zenodo.org/badge/latestdoi/33449914) -The Python Satellite Data Analysis Toolkit (pysat) is a package providing a -simple and flexible interface for downloading, loading, cleaning, managing, -processing, and analyzing scientific measurements. Although pysat was initially -designed for in situ satellite observations, it now supports many different -types of ground- and space-based measurements. +The Python Satellite Data Analysis Toolkit (pysat) provides a simple and +flexible interface for robust data analysis from beginning to end - including +downloading, loading, cleaning, managing, processing, and analyzing data. +Pysat's plug-in design allows analysis support for any data, including user +provided data sets. The pysat team provides a variety of plug-ins to support +public scientific data sets in packages such as pysatNASA, pysatMadrigal, and +more, available as part of the general [pysat ecosystem](https://github.com/pysat). Full [Documentation](http://pysat.readthedocs.io/en/latest/index.html) JGR-Space Physics [Publication](https://doi.org/10.1029/2018JA025297) +Pysat Ecosystem [Publication](https://www.frontiersin.org/articles/10.3389/fspas.2023.1119775/full) + [Citation Info](https://pysat.readthedocs.io/en/latest/citing.html) Come join us on Slack! An invitation to the pysat workspace is available From 020b8e1cd3d742c746a4ab56c1ad572cfe6af306 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 19 Dec 2023 13:34:19 -0500 Subject: [PATCH 281/365] STY: clean up intro --- docs/introduction.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/introduction.rst b/docs/introduction.rst index 0a34fe9b8..988b3dd98 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -6,14 +6,13 @@ Introduction Every scientific instrument has unique properties, though the general process for science data analysis is platform independent. This process can by described as: finding and downloading data, writing code to load the data, cleaning the -data to an appropriate level, and applying the specific analysis for a project, -and plotting the results. The Python Satellite Data Analysis Toolkit (pysat) -provides a framework to support this general process that builds upon these -commonalities. In doing so, pysat simplifies the process of using new -instruments, reduces data management overhead, and enables the creation of -instrument independent analysis routines. Although pysat was initially designed -for `in situ` satellite measurements, pysat has grown to support both -observational and modelled space science measurements. +data to an appropriate level, and applying the specific analysis for a project. +The Python Satellite Data Analysis Toolkit (pysat) provides a framework to +support the data analysis lifecycle. In doing so, pysat simplifies the process +of using new instruments, reduces data management overhead, and enables the +creation of instrument independent analysis routines. Although pysat was +initially designed for `in situ` satellite measurements, pysat has grown to +support both observational and modelled space science measurements. The newest incarnation of pysat has been pared down to focus on the core elements of our mission: providing a framework for data management and analysis. From 529fc1497225086132b2aeb99945885a303bb015 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 19 Dec 2023 13:34:31 -0500 Subject: [PATCH 282/365] DOC: update 3.0 refs --- docs/tutorial/tutorial_v3_upgrade.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/tutorial/tutorial_v3_upgrade.rst b/docs/tutorial/tutorial_v3_upgrade.rst index aa45fcae0..3747e3078 100644 --- a/docs/tutorial/tutorial_v3_upgrade.rst +++ b/docs/tutorial/tutorial_v3_upgrade.rst @@ -3,7 +3,7 @@ Transition to v3.0 ------------------ -pysat release v3.0 introduces some backwards incompatible changes from +pysat release v3.0 introduced some backwards incompatible changes from v2.x to ensure a strong foundation for future development. Many of the changes needed to update existing pysat v2.x analysis codes are relatively trivial and relate to an updated restructuring of supporting pysat functions. However, @@ -11,7 +11,7 @@ there are some changes with how pysat stores package information as well as how pysat interacts with the local file system to find files that may require some setup work for systems with an existing pysat v2.x install. -pysat v3.0 now supports a single internal interface for storing and retrieving +pysat v3.0+ now supports a single internal interface for storing and retrieving package data that also makes it possible for users to set values for a variety of pysat defaults. pysat stores all of this information in the user's home directory under ``~/.pysat``. To get the most benefit from this internal @@ -25,12 +25,13 @@ See :ref:`tutorial-params` for more. is a string or list of strings for directories that pysat can use to store science data. -pysat v3.0 now supports more than one top-level directory to store science +pysat v3.0+ now supports more than one top-level directory to store science data as well as updates the default sub-directory structure for storing data. pysat v2.x employed an internal directory template of ``platform/name/tag`` -for organizing data while pysat v3.0 begins with a default of -``os.path.join(platform, name, tag, inst_id)``. Thus, by default, a pysat v3.0 install will -generally not find all existing data files that were managed by pysat v2.x. +for organizing data while pysat v3.0+ begins with a default of +``os.path.join(platform, name, tag, inst_id)``. Thus, by default, a pysat v3.0+ +install will generally not find all existing data files that were managed by +pysat v2.x. Additionally, support for individual instruments has been moved out of pysat and into a penumbra of supporting packages. These supporting packages must be From 3a63faf9bc31cf2de8a9fb06bcc8e87cc2bc14cf Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 19 Dec 2023 13:35:33 -0500 Subject: [PATCH 283/365] DOC: add new paper ref --- docs/citing.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/citing.rst b/docs/citing.rst index 0ff303c7f..a7c1c9467 100644 --- a/docs/citing.rst +++ b/docs/citing.rst @@ -6,7 +6,7 @@ Stoneback et al [2018] ``_ as well as the package ``_. Note that this DOI will always point to the latest version of the code. A list of DOIs for all versions can be found at the Zenodo page above. Depending on -usage, citation of the full ecosystem paper by Stoneback et al [2023] +usage, citation of the full ecosystem paper by Stoneback et al [2023] ``_ may also be appropriate. @@ -37,7 +37,7 @@ A simplified implementation of the citation. .. include:: ../pysat/citation.txt :literal: -Citing the publication: +Citing the publications: .. code:: @@ -56,6 +56,19 @@ Citing the publication: year = {2018} } + @article{Stoneback2023, + author = {Stoneback, R. A. and + Burrell, A. G. and + Klenzing, J. and + Smith, J.}, + doi = {10.3389/fspas.2023.1119775}, + journal = {Frontiers in Astronomy and Space Science}, + title = {The pysat ecosystem}, + volume = {10}, + year = {2023} + } + + To aid in scientific reproducibility, please include the version number in publications that use this code. This can be found by invoking :py:attr:`pysat.__version__`. From f2168e2456f455fff596df0596c02e9ed3becccf Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 19 Dec 2023 14:05:08 -0500 Subject: [PATCH 284/365] STY: move import related noqa to line-level rather than file level --- pysat/__init__.py | 25 ++++++++++++--------- pysat/instruments/methods/__init__.py | 4 ++-- pysat/utils/__init__.py | 32 +++++++++++++-------------- setup.cfg | 3 --- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/pysat/__init__.py b/pysat/__init__.py index ab25a4f4a..65d8a071e 100644 --- a/pysat/__init__.py +++ b/pysat/__init__.py @@ -48,6 +48,7 @@ import logging import os +# Logger needs to be initialized before other modules are imported. logger = logging.getLogger(__name__) handler = logging.StreamHandler() formatter = logging.Formatter('%(name)s %(levelname)s: %(message)s') @@ -55,8 +56,10 @@ logger.addHandler(handler) logger.setLevel(logging.WARNING) +# Import statements after this point require a noqa statement for flake8 + # Import and set user and pysat parameters object -from pysat import _params +from pysat import _params # noqa: E402 F401 # Set version __version__ = metadata.version('pysat') @@ -113,18 +116,18 @@ # Load up existing parameters file params = _params.Parameters() +# utils used by other imports, needs to be imported first. +from pysat import utils # noqa: E402 F401 -from pysat._files import Files -from pysat._instrument import Instrument -from pysat._meta import Meta -from pysat._meta import MetaHeader -from pysat._meta import MetaLabels -from pysat._orbits import Orbits -from pysat import instruments -from pysat import utils +from pysat._constellation import Constellation # noqa: E402 F401 +from pysat._files import Files # noqa: E402 F401 +from pysat._instrument import Instrument # noqa: E402 F401 +from pysat._meta import Meta # noqa: E402 F401 +from pysat._meta import MetaHeader # noqa: E402 F401 +from pysat._meta import MetaLabels # noqa: E402 F401 +from pysat._orbits import Orbits # noqa: E402 F401 +from pysat import instruments # noqa: E402 F401 -# Import constellation separately -from pysat._constellation import Constellation __all__ = ['instruments', 'utils'] # Clean up diff --git a/pysat/instruments/methods/__init__.py b/pysat/instruments/methods/__init__.py index 44278ead2..eca32e606 100644 --- a/pysat/instruments/methods/__init__.py +++ b/pysat/instruments/methods/__init__.py @@ -4,5 +4,5 @@ Each set of methods is contained within a subpackage of this set. """ -from pysat.instruments.methods import general -from pysat.instruments.methods import testing +from pysat.instruments.methods import general # noqa: F401 +from pysat.instruments.methods import testing # noqa: F401 diff --git a/pysat/utils/__init__.py b/pysat/utils/__init__.py index 98f33c23a..70d6d7a30 100644 --- a/pysat/utils/__init__.py +++ b/pysat/utils/__init__.py @@ -6,19 +6,19 @@ for the pysat data directory structure. """ -from pysat.utils._core import available_instruments -from pysat.utils._core import display_available_instruments -from pysat.utils._core import display_instrument_stats -from pysat.utils._core import generate_instrument_list -from pysat.utils._core import get_mapped_value -from pysat.utils._core import listify -from pysat.utils._core import NetworkLock -from pysat.utils._core import scale_units -from pysat.utils._core import stringify -from pysat.utils._core import update_fill_values -from pysat.utils import coords -from pysat.utils import files -from pysat.utils import io -from pysat.utils import registry -from pysat.utils import testing -from pysat.utils import time +from pysat.utils._core import available_instruments # noqa: F401 +from pysat.utils._core import display_available_instruments # noqa: F401 +from pysat.utils._core import display_instrument_stats # noqa: F401 +from pysat.utils._core import generate_instrument_list # noqa: F401 +from pysat.utils._core import get_mapped_value # noqa: F401 +from pysat.utils._core import listify # noqa: F401 +from pysat.utils._core import NetworkLock # noqa: F401 +from pysat.utils._core import scale_units # noqa: F401 +from pysat.utils._core import stringify # noqa: F401 +from pysat.utils._core import update_fill_values # noqa: F401 +from pysat.utils import coords # noqa: F401 +from pysat.utils import files # noqa: F401 +from pysat.utils import io # noqa: F401 +from pysat.utils import registry # noqa: F401 +from pysat.utils import testing # noqa: F401 +from pysat.utils import time # noqa: F401 diff --git a/setup.cfg b/setup.cfg index 25ccaf201..39623ca78 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,6 +8,3 @@ ignore = D200 D202 W503 - pysat/__init__.py E402 F401 - pysat/instruments/methods/__init__.py F401 - pysat/utils/__init__.py F401 From f5cadecf1848e56e20ab70be4f935b8323169d82 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 19 Dec 2023 14:22:50 -0500 Subject: [PATCH 285/365] MAINT: remove unused imports --- pysat/instruments/methods/general.py | 1 - pysat/instruments/pysat_netcdf.py | 1 - pysat/tests/classes/cls_instrument_integration.py | 5 ----- pysat/tests/classes/cls_registration.py | 1 - pysat/tests/test_files.py | 1 - pysat/tests/test_instrument.py | 1 - pysat/tests/test_instrument_index.py | 3 --- pysat/tests/test_instrument_padding.py | 1 - pysat/tests/test_instruments.py | 2 -- pysat/tests/test_methods_general.py | 1 - pysat/tests/test_methods_testing.py | 2 -- pysat/tests/test_utils.py | 2 -- pysat/tests/test_utils_io.py | 1 - pysat/tests/test_utils_testing.py | 2 -- pysat/utils/_core.py | 4 ---- pysat/utils/registry.py | 1 - 16 files changed, 29 deletions(-) diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py index ef0e0d9a4..fa52a90c9 100644 --- a/pysat/instruments/methods/general.py +++ b/pysat/instruments/methods/general.py @@ -4,7 +4,6 @@ import datetime as dt import numpy as np import pandas as pds -import warnings import pysat diff --git a/pysat/instruments/pysat_netcdf.py b/pysat/instruments/pysat_netcdf.py index 2196ea56b..a3e9d9776 100644 --- a/pysat/instruments/pysat_netcdf.py +++ b/pysat/instruments/pysat_netcdf.py @@ -44,7 +44,6 @@ import datetime as dt import functools -import numpy as np import warnings import pysat diff --git a/pysat/tests/classes/cls_instrument_integration.py b/pysat/tests/classes/cls_instrument_integration.py index e76a0f7e8..42c07c4fd 100644 --- a/pysat/tests/classes/cls_instrument_integration.py +++ b/pysat/tests/classes/cls_instrument_integration.py @@ -6,18 +6,13 @@ """ -import datetime as dt import logging -import numpy as np import os import tempfile -import pandas as pds import pytest -import xarray as xr import pysat -from pysat.utils import testing class InstIntegrationTests(object): diff --git a/pysat/tests/classes/cls_registration.py b/pysat/tests/classes/cls_registration.py index 7d21a4a28..8d7c8c43b 100644 --- a/pysat/tests/classes/cls_registration.py +++ b/pysat/tests/classes/cls_registration.py @@ -12,7 +12,6 @@ """ import importlib -import pytest import sys import pysat diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py index c475b7cf9..aa6084caa 100644 --- a/pysat/tests/test_files.py +++ b/pysat/tests/test_files.py @@ -8,7 +8,6 @@ import os import pandas as pds import tempfile -import time import pytest diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index c51ae895d..7b2106364 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -18,7 +18,6 @@ from pysat.tests.classes.cls_instrument_iteration import InstIterationTests from pysat.tests.classes.cls_instrument_property import InstPropertyTests from pysat.utils import testing -from pysat.utils.time import filter_datetime_input class TestBasics(InstAccessTests, InstIntegrationTests, InstIterationTests, diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py index a36d8a68b..e50ca3f11 100644 --- a/pysat/tests/test_instrument_index.py +++ b/pysat/tests/test_instrument_index.py @@ -1,9 +1,6 @@ """Unit tests for the `pysat.Instrument.index` attribute.""" -import datetime as dt from importlib import reload -import numpy as np -import warnings import pytest diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py index a359916b6..ae0377243 100644 --- a/pysat/tests/test_instrument_padding.py +++ b/pysat/tests/test_instrument_padding.py @@ -11,7 +11,6 @@ import pysat.instruments.pysat_ndtesting import pysat.instruments.pysat_testing from pysat.utils import testing -from pysat.utils.time import filter_datetime_input class TestDataPaddingbyFile(object): diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index 4f7f87df8..31cfd79ee 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -9,14 +9,12 @@ import datetime as dt import numpy as np import pandas as pds -import warnings import pytest import pysat import pysat.tests.classes.cls_instrument_library as cls_inst_lib from pysat.tests.classes.cls_instrument_library import InstLibTests -from pysat.utils import testing # Optional code to pass through user and password info to test instruments # dict, keyed by pysat instrument, with a list of usernames and passwords diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py index 3272b5d63..0a2c519e5 100644 --- a/pysat/tests/test_methods_general.py +++ b/pysat/tests/test_methods_general.py @@ -4,7 +4,6 @@ from os import path import pandas as pds import pytest -import warnings import pysat from pysat.instruments.methods import general as gen diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py index 5c2818037..b7056fc48 100644 --- a/pysat/tests/test_methods_testing.py +++ b/pysat/tests/test_methods_testing.py @@ -1,8 +1,6 @@ """Tests the `pysat.instruments.methods.testing` methods.""" import datetime as dt -from os import path -import pandas as pds import pytest import pysat diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index 444e792d1..4653b93fa 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -5,7 +5,6 @@ # ---------------------------------------------------------------------------- """Tests the pysat utils core functions.""" -import contextlib from importlib import reload import inspect import numpy as np @@ -14,7 +13,6 @@ import pytest import shutil import tempfile -import warnings import pysat from pysat.tests.classes.cls_registration import TestWithRegistration diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index f9d462031..b24358b95 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -10,7 +10,6 @@ import logging import numpy as np import os -import shutil import sys import tempfile import warnings diff --git a/pysat/tests/test_utils_testing.py b/pysat/tests/test_utils_testing.py index f5c7fed73..8c7176b48 100644 --- a/pysat/tests/test_utils_testing.py +++ b/pysat/tests/test_utils_testing.py @@ -6,9 +6,7 @@ """Tests the pysat utility testing routines.""" import numpy as np -import os import pytest -import tempfile import warnings from pysat.utils import testing diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index 333d178ad..710f9aba3 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -6,13 +6,9 @@ import datetime as dt import importlib -import netCDF4 import numpy as np import os -import pandas as pds from portalocker import Lock -import warnings -import xarray as xr import pysat diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py index a75a212b2..1afa69058 100644 --- a/pysat/utils/registry.py +++ b/pysat/utils/registry.py @@ -49,7 +49,6 @@ """ import importlib -import logging import pysat import pysat.tests.classes.cls_instrument_library as itc From 690f807b7cff35ccac64c5d77a2dbea2f7b7ebd2 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 19 Dec 2023 14:23:19 -0500 Subject: [PATCH 286/365] MAINT: update flake8 style --- docs/conf.py | 2 +- pysat/tests/test_utils.py | 3 +-- pysat/tests/test_utils_io.py | 15 +++++++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a64369fc3..6d20155b5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,6 +14,7 @@ import os import sys +import pysat # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -21,7 +22,6 @@ # sys.path.insert(0, os.path.abspath('.')) sys.path.insert(0, os.path.abspath('..')) -import pysat pysat.params['data_dirs'] = '.' diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index 4653b93fa..9c410c699 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -410,9 +410,8 @@ def test_stringify_non_str_types(self, astrlike): """ - target = type(astrlike) output = pysat.utils.stringify(astrlike) - assert type(output) == target + assert type(output) is type(astrlike) return diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index b24358b95..7c7494d5d 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -1657,6 +1657,14 @@ def test_remove_netcdf4_standards(self, caplog): # the standard name as long_name when loading, and while that would # pass the tests here as written, it would be brittle. Check everything # else. + + def assert_meta_unchanged(old_meta, filt_meta, var, key): + """Check that filtered meta value is unchanged.""" + + assert old_meta[var][key] == filt_meta[var][key], \ + 'Value changed for {}, {}'.format(var, key) + return + for var in self.meta_dict.keys(): assert var in filt_meta, 'Lost metadata variable {}'.format(var) @@ -1666,12 +1674,11 @@ def test_remove_netcdf4_standards(self, caplog): if key not in ['fill', 'value_min', 'value_max']: assert key in filt_meta[var], \ 'Lost metadata label {} for {}'.format(key, var) - assert self.meta_dict[var][key] == filt_meta[var][key],\ - 'Value changed for {}, {}'.format(var, key) + assert_meta_unchanged(self.meta_dict, filt_meta, var, key) else: if key in filt_meta: - assert self.meta_dict[var][key] == filt_meta[var][key],\ - 'Value changed for {}, {}'.format(var, key) + assert_meta_unchanged(self.meta_dict, filt_meta, var, + key) return From fd9c23075fdb7bdb7fd777cf5545244fcb2482d1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 19 Dec 2023 14:24:48 -0500 Subject: [PATCH 287/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20bf544de..131d49cdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed deprecated usage of None for tag and inst_id * Removed deprecated kwarg behaviour for 'fname' in `to_netCDF4` * Added verion cap for sphinx_rtd_theme + * Used line specific noqa statements for imports [3.1.0] - 2023-05-31 -------------------- From 4d84c5c21ae96e0eabfb81ed82533eaf5b993afc Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 19 Dec 2023 14:49:06 -0500 Subject: [PATCH 288/365] DOC: add comment --- pysat/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysat/__init__.py b/pysat/__init__.py index 65d8a071e..c5a71b8ca 100644 --- a/pysat/__init__.py +++ b/pysat/__init__.py @@ -119,6 +119,7 @@ # utils used by other imports, needs to be imported first. from pysat import utils # noqa: E402 F401 +# Import the remainder of the modules. from pysat._constellation import Constellation # noqa: E402 F401 from pysat._files import Files # noqa: E402 F401 from pysat._instrument import Instrument # noqa: E402 F401 From 9c169358a675bfef36140ee942b67dc39406d2de Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 20 Dec 2023 14:09:39 -0500 Subject: [PATCH 289/365] ENH: add flag to select if new tests are ignored --- pyproject.toml | 1 + pysat/_instrument.py | 4 ++-- pysat/tests/classes/cls_instrument_library.py | 11 +++++++++++ pysat/utils/_core.py | 10 +++++++++- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d52f2fff8..4ed0a323e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ markers = [ "download", "no_download", "load_options", + "new_tests", "first", "second" ] diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 596b91dc3..be3f9e39d 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1366,7 +1366,7 @@ def _assign_attrs(self, by_name=False): directory_format, file_format, multi_file_day, orbit_info, and pandas_format test attributes - _test_download, _test_download_ci, and _password_req + _test_download, _test_download_ci, _test_new_tests, and _password_req """ # Declare the standard Instrument methods and attributes @@ -1378,7 +1378,7 @@ def _assign_attrs(self, by_name=False): 'multi_file_day': False, 'orbit_info': None, 'pandas_format': True} test_attrs = {'_test_download': True, '_test_download_ci': True, - '_password_req': False} + '_test_new_tests': True, '_password_req': False} # Set method defaults for mname in [mm for val in inst_methods.values() for mm in val]: diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 34987c923..d6261d948 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -260,6 +260,11 @@ def initialize_test_package(self, inst_loc, user_info=None): mark = pytest.mark.parametrize("inst_name", instruments['names']) getattr(self, method).pytestmark.append(mark) + elif 'new_tests' in mark_names: + # Prioritize new test marks if present + mark = pytest.mark.parametrize("inst_dict", + instruments['new_tests']) + getattr(self, method).pytestmark.append(mark) elif 'load_options' in mark_names: # Prioritize load_options mark if present mark = pytest.mark.parametrize("inst_dict", @@ -468,8 +473,11 @@ def test_load_empty(self, inst_dict): return + # TODO (v3.3.0): remove mark.new_tests when ecosystem is fully compliant + # for this test @pytest.mark.second @pytest.mark.load_options + @pytest.mark.new_tests def test_load_multiple_days(self, inst_dict): """Test that instruments load at each cleaning level. @@ -604,8 +612,11 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): return + # TODO (v3.3.0): remove mark.new_tests when ecosystem is fully compliant + # for this test @pytest.mark.second @pytest.mark.load_options + @pytest.mark.new_tests @pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)]) def test_load_w_pad(self, pad, inst_dict): """Test that instruments load at each cleaning level. diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index 710f9aba3..4787b4cd3 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -326,6 +326,7 @@ def generate_instrument_list(inst_loc, user_info=None): instrument_download = [] instrument_optional_load = [] instrument_no_download = [] + instrument_new_tests = [] # Look through list of available instrument modules in the given location for inst_module in instrument_names: @@ -374,6 +375,8 @@ def generate_instrument_list(inst_loc, user_info=None): # Check if instrument is configured for download tests. if inst._test_download: instrument_download.append(in_dict.copy()) + if inst._test_new_tests: + instrument_new_tests.append(in_dict.copy()) if hasattr(module, '_test_load_opt'): # Add optional load tests try: @@ -385,6 +388,10 @@ def generate_instrument_list(inst_loc, user_info=None): # Append as copy so kwargs are unique. instrument_optional_load.append( in_dict.copy()) + if inst._test_new_tests: + instrument_new_tests.append( + in_dict.copy()) + except KeyError: # Option does not exist for tag/inst_id # combo @@ -400,7 +407,8 @@ def generate_instrument_list(inst_loc, user_info=None): output = {'names': instrument_names, 'download': instrument_download, 'load_options': instrument_download + instrument_optional_load, - 'no_download': instrument_no_download} + 'no_download': instrument_no_download, + 'new_tests': instrument_new_tests} return output From d9f274ef7342e724db9d7aa18cd01b0b5558d5ca Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 20 Dec 2023 14:10:31 -0500 Subject: [PATCH 290/365] DOC: update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 131d49cdd..e574ea790 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,7 +60,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed deprecated usage of None for tag and inst_id * Removed deprecated kwarg behaviour for 'fname' in `to_netCDF4` * Added verion cap for sphinx_rtd_theme - * Used line specific noqa statements for imports + * Used line specific noqa statements for imports + * Add `_test_new_tests` flag for packages to ignore select new tests [3.1.0] - 2023-05-31 -------------------- From 22bb47723b4f5cab8e73acb9940d2f0fc015f972 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 20 Dec 2023 21:39:31 -0500 Subject: [PATCH 291/365] Apply suggestions from code review --- pysat/tests/classes/cls_instrument_library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index d6261d948..0a122554c 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -473,7 +473,7 @@ def test_load_empty(self, inst_dict): return - # TODO (v3.3.0): remove mark.new_tests when ecosystem is fully compliant + # TODO(v3.3.0): remove mark.new_tests when ecosystem is fully compliant # for this test @pytest.mark.second @pytest.mark.load_options @@ -612,7 +612,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): return - # TODO (v3.3.0): remove mark.new_tests when ecosystem is fully compliant + # TODO(v3.3.0): remove mark.new_tests when ecosystem is fully compliant # for this test @pytest.mark.second @pytest.mark.load_options From 380c59c264b5e43bc4bbe810534998ab8b7963ab Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 21 Dec 2023 13:15:44 -0500 Subject: [PATCH 292/365] DOC: added dist-a to documentation Added a dist-a statement to the documentation. --- docs/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 6e8810cea..84c941b10 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,3 +23,9 @@ Welcome to pysat's documentation! roadmap.rst faq.rst release_notes.rst + + +.. admonition:: DISTRIBUTION STATEMENT A: Approved for public release. + Distribution is unlimited. + + This work was supported by the Office of Naval Research. From 164131005a0cebc26780c96f1c7684d24223aea9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 21 Dec 2023 14:20:44 -0500 Subject: [PATCH 293/365] Update docs/new_instrument.rst Co-authored-by: Angeline Burrell --- docs/new_instrument.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 091c373e8..ee248b658 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -914,7 +914,7 @@ but should not undergo automated download tests because it would require the user to save a password in a potentially public location. The :py:attr:`_password_req` flag is used to skip both the download tests and the download warning message tests, since a functional download routine is -present. This flag is defaults to False if not specified. +present. This flag is defaults to :py:val:`False` if not specified. .. code:: python From 1b461be6cc01257f8a24008d09f07305ec4fd14d Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 21 Dec 2023 17:10:11 -0500 Subject: [PATCH 294/365] BUG: fixed download test dates Fixed download test to ask for two days of data, needed for mutiple day load test. --- pysat/tests/classes/cls_instrument_library.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 34987c923..22a7f0ff2 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -387,8 +387,11 @@ def test_download(self, inst_dict): dl_dict = inst_dict['user_info'] else: dl_dict = {} - # Note this will download two consecutive days - self.test_inst.download(self.date, **dl_dict) + + # Ask to download two consecutive days + self.test_inst.download(start=self.date, + stop=self.date + dt.timedelta(days=2), + **dl_dict) assert len(self.test_inst.files.files) > 0 return From 3187d377aeaa25b24d115d127ab5dafeba781488 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Thu, 11 Jan 2024 10:08:52 -0600 Subject: [PATCH 295/365] STY: Completed funding institution list from most awesome all the way to NASA. --- pysat/_constellation.py | 3 ++- pysat/_files.py | 6 +++++- pysat/_instrument.py | 3 ++- pysat/_meta.py | 3 ++- pysat/_orbits.py | 3 ++- pysat/_params.py | 3 ++- pysat/constellations/single_test.py | 3 ++- pysat/constellations/testing.py | 3 ++- pysat/constellations/testing_empty.py | 3 ++- pysat/constellations/testing_partial.py | 3 ++- pysat/instruments/methods/general.py | 3 ++- pysat/instruments/methods/testing.py | 3 ++- pysat/instruments/pysat_ndtesting.py | 3 ++- pysat/instruments/pysat_netcdf.py | 3 ++- pysat/instruments/pysat_testing.py | 3 ++- pysat/instruments/pysat_testmodel.py | 3 ++- pysat/tests/classes/cls_ci.py | 3 ++- pysat/tests/classes/cls_instrument_access.py | 3 ++- pysat/tests/classes/cls_instrument_integration.py | 3 ++- pysat/tests/classes/cls_instrument_iteration.py | 3 ++- pysat/tests/classes/cls_instrument_library.py | 3 ++- pysat/tests/classes/cls_instrument_property.py | 3 ++- pysat/tests/classes/cls_registration.py | 3 ++- pysat/tests/test_constellation.py | 3 ++- pysat/tests/test_files.py | 3 ++- pysat/tests/test_instrument.py | 3 ++- pysat/tests/test_instrument_custom.py | 3 ++- pysat/tests/test_instrument_index.py | 3 ++- pysat/tests/test_instrument_listgen.py | 3 ++- pysat/tests/test_instrument_padding.py | 3 ++- pysat/tests/test_instruments.py | 3 ++- pysat/tests/test_meta.py | 3 ++- pysat/tests/test_meta_header.py | 3 ++- pysat/tests/test_meta_labels.py | 3 ++- pysat/tests/test_methods_general.py | 3 ++- pysat/tests/test_methods_testing.py | 3 ++- pysat/tests/test_orbits.py | 3 ++- pysat/tests/test_params.py | 3 ++- pysat/tests/test_registry.py | 3 ++- pysat/tests/test_utils.py | 3 ++- pysat/tests/test_utils_coords.py | 3 ++- pysat/tests/test_utils_files.py | 3 ++- pysat/tests/test_utils_io.py | 3 ++- pysat/tests/test_utils_testing.py | 3 ++- pysat/tests/test_utils_time.py | 3 ++- pysat/utils/_core.py | 3 ++- pysat/utils/coords.py | 3 ++- pysat/utils/files.py | 3 ++- pysat/utils/io.py | 3 ++- pysat/utils/registry.py | 3 ++- pysat/utils/testing.py | 3 ++- pysat/utils/time.py | 3 ++- 52 files changed, 107 insertions(+), 52 deletions(-) diff --git a/pysat/_constellation.py b/pysat/_constellation.py index 56ce88122..6a43b7999 100644 --- a/pysat/_constellation.py +++ b/pysat/_constellation.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Class for Instrument constellations. diff --git a/pysat/_files.py b/pysat/_files.py index 337a0243f..f17377972 100644 --- a/pysat/_files.py +++ b/pysat/_files.py @@ -1,7 +1,11 @@ -#!/usr/bin/env python # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.1199703 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- import copy diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 84ff77fbc..d5bcaed2d 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Class for single instruments.""" diff --git a/pysat/_meta.py b/pysat/_meta.py index 55ba88a78..183b163bd 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Classes for storing and managing meta data.""" diff --git a/pysat/_orbits.py b/pysat/_orbits.py index a76126eb1..b2d07c1f0 100644 --- a/pysat/_orbits.py +++ b/pysat/_orbits.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- import copy diff --git a/pysat/_params.py b/pysat/_params.py index 090764ae2..cbed4b85c 100644 --- a/pysat/_params.py +++ b/pysat/_params.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- import copy diff --git a/pysat/constellations/single_test.py b/pysat/constellations/single_test.py index 73f610c8a..b619279b6 100644 --- a/pysat/constellations/single_test.py +++ b/pysat/constellations/single_test.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Create a constellation with one testing instrument. diff --git a/pysat/constellations/testing.py b/pysat/constellations/testing.py index 502ee2106..a39b72238 100644 --- a/pysat/constellations/testing.py +++ b/pysat/constellations/testing.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Create a constellation with 5 testing instruments. diff --git a/pysat/constellations/testing_empty.py b/pysat/constellations/testing_empty.py index 30673b412..97ed4b65b 100644 --- a/pysat/constellations/testing_empty.py +++ b/pysat/constellations/testing_empty.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Create an empty constellation for testing. diff --git a/pysat/constellations/testing_partial.py b/pysat/constellations/testing_partial.py index d7bf71acc..857ef50b6 100644 --- a/pysat/constellations/testing_partial.py +++ b/pysat/constellations/testing_partial.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Create a constellation where not all instruments have loadable data. diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py index 8c312bd5e..1c79ac818 100644 --- a/pysat/instruments/methods/general.py +++ b/pysat/instruments/methods/general.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Provides generalized routines for integrating instruments into pysat.""" diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 00aa58575..6e7bf9aab 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Standard functions for the test instruments.""" diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index b0998996c..1acabd746 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Produces fake instrument data for testing.""" diff --git a/pysat/instruments/pysat_netcdf.py b/pysat/instruments/pysat_netcdf.py index 3c1db0720..6988034db 100644 --- a/pysat/instruments/pysat_netcdf.py +++ b/pysat/instruments/pysat_netcdf.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """General Instrument for loading pysat-written netCDF files. diff --git a/pysat/instruments/pysat_testing.py b/pysat/instruments/pysat_testing.py index 51fb3a1c9..d5c9b0ce9 100644 --- a/pysat/instruments/pysat_testing.py +++ b/pysat/instruments/pysat_testing.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Produces fake instrument data for testing.""" diff --git a/pysat/instruments/pysat_testmodel.py b/pysat/instruments/pysat_testmodel.py index 9cd5e9e78..3a343293c 100644 --- a/pysat/instruments/pysat_testmodel.py +++ b/pysat/instruments/pysat_testmodel.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Produces fake instrument data for testing.""" diff --git a/pysat/tests/classes/cls_ci.py b/pysat/tests/classes/cls_ci.py index 60717a748..9be8fc8e4 100644 --- a/pysat/tests/classes/cls_ci.py +++ b/pysat/tests/classes/cls_ci.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Class setup and teardown for unit tests that are only run in the CI env.""" diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py index b342ee9e7..4751c6e24 100644 --- a/pysat/tests/classes/cls_instrument_access.py +++ b/pysat/tests/classes/cls_instrument_access.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests for data access and related functions in the pysat Instrument object. diff --git a/pysat/tests/classes/cls_instrument_integration.py b/pysat/tests/classes/cls_instrument_integration.py index 0c3e7a9e8..f49efa95e 100644 --- a/pysat/tests/classes/cls_instrument_integration.py +++ b/pysat/tests/classes/cls_instrument_integration.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Integration tests for pysat.Instrument. diff --git a/pysat/tests/classes/cls_instrument_iteration.py b/pysat/tests/classes/cls_instrument_iteration.py index 23fed48ff..f8cf3f86f 100644 --- a/pysat/tests/classes/cls_instrument_iteration.py +++ b/pysat/tests/classes/cls_instrument_iteration.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests for iteration in the pysat Instrument object and methods. diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 8979a9561..339eb95bf 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Standardized class and functions to test instruments for pysat libraries. diff --git a/pysat/tests/classes/cls_instrument_property.py b/pysat/tests/classes/cls_instrument_property.py index 5eb778038..b91f63166 100644 --- a/pysat/tests/classes/cls_instrument_property.py +++ b/pysat/tests/classes/cls_instrument_property.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Test for instrument properties in the pysat Instrument object and methods. diff --git a/pysat/tests/classes/cls_registration.py b/pysat/tests/classes/cls_registration.py index b1da069be..b3669f0c1 100644 --- a/pysat/tests/classes/cls_registration.py +++ b/pysat/tests/classes/cls_registration.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Standardized class and functions to test registration for pysat libraries. diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py index 6c961625f..5a0c16d51 100644 --- a/pysat/tests/test_constellation.py +++ b/pysat/tests/test_constellation.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the Constellation class.""" diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py index 97ee25c71..0c96c7d64 100644 --- a/pysat/tests/test_files.py +++ b/pysat/tests/test_files.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Test pysat Files object and code.""" diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index e4cb2ef8f..98524b917 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Tests the pysat Instrument object and methods.""" diff --git a/pysat/tests/test_instrument_custom.py b/pysat/tests/test_instrument_custom.py index 106049df3..12fefe9ea 100644 --- a/pysat/tests/test_instrument_custom.py +++ b/pysat/tests/test_instrument_custom.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the `custom_attach` methods for `pysat.Instrument`.""" diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py index 10371199a..a02ea0373 100644 --- a/pysat/tests/test_instrument_index.py +++ b/pysat/tests/test_instrument_index.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the `pysat.Instrument.index` attribute.""" diff --git a/pysat/tests/test_instrument_listgen.py b/pysat/tests/test_instrument_listgen.py index d208495a0..40440d87e 100644 --- a/pysat/tests/test_instrument_listgen.py +++ b/pysat/tests/test_instrument_listgen.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the list generation methods in `pysat.Instrument`.""" diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py index b815f19b6..c43acb44d 100644 --- a/pysat/tests/test_instrument_padding.py +++ b/pysat/tests/test_instrument_padding.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the padding methods in `pysat.Instrument`.""" diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index 42ba8d6c8..51235804e 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit and Integration Tests for each instrument module. diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 9b1fafc7b..1cb862f78 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat Meta object.""" diff --git a/pysat/tests/test_meta_header.py b/pysat/tests/test_meta_header.py index 43944ee81..f7ddc7365 100644 --- a/pysat/tests/test_meta_header.py +++ b/pysat/tests/test_meta_header.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat MetaHeader object.""" diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py index db8af4174..77b1a3f16 100644 --- a/pysat/tests/test_meta_labels.py +++ b/pysat/tests/test_meta_labels.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat MetaLabels object.""" diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py index e0308ee47..94de6e606 100644 --- a/pysat/tests/test_methods_general.py +++ b/pysat/tests/test_methods_general.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the general instrument methods.""" diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py index 4479809a1..c58ff5310 100644 --- a/pysat/tests/test_methods_testing.py +++ b/pysat/tests/test_methods_testing.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the `pysat.instruments.methods.testing` methods.""" diff --git a/pysat/tests/test_orbits.py b/pysat/tests/test_orbits.py index edd17ad14..0208e2105 100644 --- a/pysat/tests/test_orbits.py +++ b/pysat/tests/test_orbits.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Test the pysat routines for the orbits class.""" diff --git a/pysat/tests/test_params.py b/pysat/tests/test_params.py index 7530a02a9..85dc02e4e 100644 --- a/pysat/tests/test_params.py +++ b/pysat/tests/test_params.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat parameters storage area.""" diff --git a/pysat/tests/test_registry.py b/pysat/tests/test_registry.py index 09d9b299d..6580b4b82 100644 --- a/pysat/tests/test_registry.py +++ b/pysat/tests/test_registry.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the registration of user-defined modules.""" diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index f3b188340..0aca70862 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat utils core functions.""" diff --git a/pysat/tests/test_utils_coords.py b/pysat/tests/test_utils_coords.py index bcc00d3d3..bc669a170 100644 --- a/pysat/tests/test_utils_coords.py +++ b/pysat/tests/test_utils_coords.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the `pysat.utils.coords` functions.""" diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 01227a821..5b3a71115 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the `pysat.utils.files` functions.""" diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 3bba2ae3a..9bf732a9b 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat utility io routines.""" import copy diff --git a/pysat/tests/test_utils_testing.py b/pysat/tests/test_utils_testing.py index 4588f22ad..22a096b3e 100644 --- a/pysat/tests/test_utils_testing.py +++ b/pysat/tests/test_utils_testing.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat utility testing routines.""" diff --git a/pysat/tests/test_utils_time.py b/pysat/tests/test_utils_time.py index c12d86ee8..fc43c088c 100644 --- a/pysat/tests/test_utils_time.py +++ b/pysat/tests/test_utils_time.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests for the pysat.utils.time functions.""" diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index da1f31e53..760aa924c 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- import datetime as dt diff --git a/pysat/utils/coords.py b/pysat/utils/coords.py index 50ad28425..4984352eb 100644 --- a/pysat/utils/coords.py +++ b/pysat/utils/coords.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Coordinate transformation functions for pysat.""" diff --git a/pysat/utils/files.py b/pysat/utils/files.py index 0f03e9f3c..e61b6df9e 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Utilities for file management and parsing file names.""" diff --git a/pysat/utils/io.py b/pysat/utils/io.py index cebbbfa55..c73fc2ca2 100644 --- a/pysat/utils/io.py +++ b/pysat/utils/io.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Input/Output utilities for pysat data.""" import copy diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py index b2764943a..18077abd2 100644 --- a/pysat/utils/registry.py +++ b/pysat/utils/registry.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """pysat user module registry utilities. diff --git a/pysat/utils/testing.py b/pysat/utils/testing.py index bf9e0f795..5e4822355 100644 --- a/pysat/utils/testing.py +++ b/pysat/utils/testing.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Utilities to perform common evaluations.""" diff --git a/pysat/utils/time.py b/pysat/utils/time.py index a9fef7576..34513799c 100644 --- a/pysat/utils/time.py +++ b/pysat/utils/time.py @@ -5,7 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. +# This work was supported by Cosmic Studio, the Office of Naval Research, +# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Date and time handling utilities.""" From 7f4a7aeb4ae8fb2ae0baa329b185666eac223a05 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Thu, 11 Jan 2024 10:26:06 -0600 Subject: [PATCH 296/365] STY: Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c61abea7..819f70527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). behaviour to `use_header=True` * Use temporary directories for files created during test_utils.py * Updated code file headers to be consistent and include NRL pub release + * Updated code file headers to include full institutional funding list * Removed deprecated `labels` kwarg for `pysat.Instrument()` * Removed deprecated `utils.load_netcdf4` method * Removed deprecated `_filter_netcdf4_metadata` method From 7f6260a36601cbdac8f4a4338cdae04e4576c697 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Thu, 11 Jan 2024 10:28:41 -0600 Subject: [PATCH 297/365] STY: Updated index.rst --- docs/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 84c941b10..1b84cac32 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,4 +28,5 @@ Welcome to pysat's documentation! .. admonition:: DISTRIBUTION STATEMENT A: Approved for public release. Distribution is unlimited. - This work was supported by the Office of Naval Research. + This work was supported by Cosmic Studio, the Office of Naval Research, + and the National Aeronautics and Space Administration. From a78de4c52f8bb9d93a9c7232549abd7a5fc1f5ea Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Thu, 11 Jan 2024 10:46:00 -0600 Subject: [PATCH 298/365] STY: Updated index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 1b84cac32..c650fdb44 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,4 +29,4 @@ Welcome to pysat's documentation! Distribution is unlimited. This work was supported by Cosmic Studio, the Office of Naval Research, - and the National Aeronautics and Space Administration. + and the National Aeronautics and Space Administration. From ceef62660bf9c90ffce13355d4f2eb2d01009cf0 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 16 Jan 2024 13:01:16 -0500 Subject: [PATCH 299/365] TST: updated padding test Updated padding test to not clean data, allowing forecast files full of padded data to pass the test. --- pysat/tests/classes/cls_instrument_library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 8979a9561..fe7df79b6 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -661,14 +661,14 @@ def test_load_w_pad(self, pad, inst_dict): # Make sure the strict time flag doesn't interfere with # the load tests load_and_set_strict_time_flag(self.test_inst, self.date, - raise_error=True, clean_off=False) + raise_error=True, clean_off=True) if self.test_inst.empty: # This will be empty if this is a forecast file that doesn't # include the load date self.test_inst.pad = None load_and_set_strict_time_flag(self.test_inst, self.date, - raise_error=True, clean_off=False) + raise_error=True, clean_off=True) assert not self.test_inst.empty, \ "No data on {:}".format(self.date) assert self.test_inst.index.max() < self.date, \ From 1f95cee15f643629c5eeb3eab48cf0a606da9f96 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 16 Jan 2024 15:56:51 -0500 Subject: [PATCH 300/365] MAINT: fix security issue Fixes a security issue found in RTD. --- pyproject.toml | 1 + test_requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d52f2fff8..ce55400f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ doc = [ "ipython", "m2r2", "numpydoc", + "readthedocs-sphinx-search==0.3.2", "sphinx", "sphinx_rtd_theme >= 1.2.2, < 2.0.0" ] diff --git a/test_requirements.txt b/test_requirements.txt index e17e0e98a..eade96797 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -8,5 +8,6 @@ numpydoc pysatSpaceWeather pytest-cov pytest-ordering +readthedocs-sphinx-search==0.3.2 sphinx sphinx_rtd_theme>=1.2.2,<2.0.0 From 8572352ae370bf97ffad4f8cbe4822791ab9530e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:50:44 -0500 Subject: [PATCH 301/365] MAINT: remove unused req file --- docs/requirements.txt | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index eb1cf559d..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -docutils<0.18 -ipython -m2r2 -netCDF4 -numpydoc -packaging -pandas -portalocker -pytest -readthedocs-sphinx-search==0.1.1 -sphinx_rtd_theme==1.0.0 -sphinx==4.2.0 -xarray From 4edc5e48fd197d265622680f80c9cdaa894d9c03 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:59:07 -0500 Subject: [PATCH 302/365] STY: PEP8 --- pysat/_instrument.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 945d5c96f..3c37b0aa8 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1372,7 +1372,8 @@ def _assign_attrs(self, by_name=False): directory_format, file_format, multi_file_day, orbit_info, and pandas_format test attributes - _test_download, _test_download_ci, _test_new_tests, and _password_req + _test_download, _test_download_ci, _test_new_tests, and + _password_req """ # Declare the standard Instrument methods and attributes From c3135fa09bef93043f250c65221d6f69f0d8f93f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 17 Jan 2024 15:44:37 -0500 Subject: [PATCH 303/365] DOC: update docstrings --- pysat/_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 84ff77fbc..9a944d248 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -736,7 +736,7 @@ def __str__(self): return output_str def __getitem__(self, key, data=None): - """Access data in `pysat.Instrument` object. + """Access data in `pysat.Instrument` or provided data object. Parameters ---------- From 3ce1e31e571f01f8e9a90d5f60f3dcccd0291127 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 17 Jan 2024 16:19:19 -0500 Subject: [PATCH 304/365] MAINT: use _new_tests --- CHANGELOG.md | 2 +- pysat/_instrument.py | 5 ++--- pysat/utils/_core.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90b9723c4..a9ba3c769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Removed deprecated kwarg behaviour for 'fname' in `to_netCDF4` * Added verion cap for sphinx_rtd_theme * Used line specific noqa statements for imports - * Add `_test_new_tests` flag for packages to ignore select new tests + * Add `_new_tests` flag for packages to ignore select new tests [3.1.0] - 2023-05-31 -------------------- diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 3c37b0aa8..a1714c130 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -1372,8 +1372,7 @@ def _assign_attrs(self, by_name=False): directory_format, file_format, multi_file_day, orbit_info, and pandas_format test attributes - _test_download, _test_download_ci, _test_new_tests, and - _password_req + _test_download, _test_download_ci, _new_tests, and _password_req """ # Declare the standard Instrument methods and attributes @@ -1385,7 +1384,7 @@ def _assign_attrs(self, by_name=False): 'multi_file_day': False, 'orbit_info': None, 'pandas_format': True} test_attrs = {'_test_download': True, '_test_download_ci': True, - '_test_new_tests': True, '_password_req': False} + '_new_tests': True, '_password_req': False} # Set method defaults for mname in [mm for val in inst_methods.values() for mm in val]: diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index 5a01b4980..a9e0ac6b4 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -379,7 +379,7 @@ def generate_instrument_list(inst_loc, user_info=None): # Check if instrument is configured for download tests. if inst._test_download: instrument_download.append(in_dict.copy()) - if inst._test_new_tests: + if inst._new_tests: instrument_new_tests.append(in_dict.copy()) if hasattr(module, '_test_load_opt'): # Add optional load tests @@ -392,7 +392,7 @@ def generate_instrument_list(inst_loc, user_info=None): # Append as copy so kwargs are unique. instrument_optional_load.append( in_dict.copy()) - if inst._test_new_tests: + if inst._new_tests: instrument_new_tests.append( in_dict.copy()) From 8baab0f3942248b30062d1e93186069d81d42a4d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 17 Jan 2024 16:33:38 -0500 Subject: [PATCH 305/365] DOC: update docs --- docs/new_instrument.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index ee248b658..1199c9d56 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -926,3 +926,28 @@ present. This flag is defaults to :py:val:`False` if not specified. _test_dates = {'': {'Level_1': dt.datetime(2020, 1, 1), 'Level_2': dt.datetime(2020, 1, 1)}} _password_req = {'': {'Level_1': True}} + +Updates to Instrument Suite Tests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes new standard tests are added to pysat that update compliance +requirements throughout the ecosystem. For example, pysat 3.2.0 adds new tests +for loading multiple days at a time or using a data pad. When these tests +require significant updates, an additional flag may be used to suppress these +tests temporarily for specific instruments while updates are made throughout +the ecosystem. These new tests are run by default unless specified using the +:py:attr:`_new_tests` flag. + +.. code:: python + + platform = 'newsat' + name = 'data' + tags = {'Level_1': 'Level 1 data, fully compliant', + 'Level_2': 'Level 2 data, needs updates for padding'} + inst_ids = {'': ['Level_1', 'Level_2']} + _test_dates = {'': {'Level_1': dt.datetime(2020, 1, 1), + 'Level_2': dt.datetime(2020, 1, 1)}} + _new_tests = {'': {'Level_2': False}} + +The new tests are marked with a `@pytest.mark.new_tests` statement, and will be +re-evaluated at each minor version release. From 8fe388d17cd7d0241bc7d948cc1bb55fefa4b474 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 Jan 2024 10:28:54 -0500 Subject: [PATCH 306/365] Apply suggestions from code review Co-authored-by: Angeline Burrell --- docs/new_instrument.rst | 8 ++++---- pysat/tests/classes/cls_instrument_library.py | 6 ++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index 1199c9d56..c3721dc6e 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -930,12 +930,12 @@ present. This flag is defaults to :py:val:`False` if not specified. Updates to Instrument Suite Tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Sometimes new standard tests are added to pysat that update compliance -requirements throughout the ecosystem. For example, pysat 3.2.0 adds new tests +Sometimes new standard tests are added to pysat that ensure all data handling features +work as expected throughout the ecosystem. For example, pysat 3.2.0 adds new tests for loading multiple days at a time or using a data pad. When these tests require significant updates, an additional flag may be used to suppress these -tests temporarily for specific instruments while updates are made throughout -the ecosystem. These new tests are run by default unless specified using the +tests temporarily for specific instruments while updates are made to that +instrument. These new tests are run by default unless specified using the :py:attr:`_new_tests` flag. .. code:: python diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 62d8ef472..63a1b7a19 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -485,8 +485,7 @@ def test_load_empty(self, inst_dict): return - # TODO(v3.3.0): remove mark.new_tests when ecosystem is fully compliant - # for this test + # TODO(#1172): remove mark.new_tests @pytest.mark.second @pytest.mark.load_options @pytest.mark.new_tests @@ -624,8 +623,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): return - # TODO(v3.3.0): remove mark.new_tests when ecosystem is fully compliant - # for this test + # TODO(#1172): remove mark.new_tests at v3.3.0 @pytest.mark.second @pytest.mark.load_options @pytest.mark.new_tests From 25b7b095d46a6c768ed2302e7ae43eeecb7e6bc2 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 Jan 2024 10:30:44 -0500 Subject: [PATCH 307/365] STY: char limit --- docs/new_instrument.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst index c3721dc6e..f810601a7 100644 --- a/docs/new_instrument.rst +++ b/docs/new_instrument.rst @@ -930,13 +930,13 @@ present. This flag is defaults to :py:val:`False` if not specified. Updates to Instrument Suite Tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Sometimes new standard tests are added to pysat that ensure all data handling features -work as expected throughout the ecosystem. For example, pysat 3.2.0 adds new tests -for loading multiple days at a time or using a data pad. When these tests -require significant updates, an additional flag may be used to suppress these -tests temporarily for specific instruments while updates are made to that -instrument. These new tests are run by default unless specified using the -:py:attr:`_new_tests` flag. +Sometimes new standard tests are added to pysat that ensure all data handling +features work as expected throughout the ecosystem. For example, pysat 3.2.0 +adds new tests for loading multiple days at a time or using a data pad. When +these tests require significant updates, an additional flag may be used to +suppress these tests temporarily for specific instruments while updates are +made to that instrument. These new tests are run by default unless specified +using the :py:attr:`_new_tests` flag. .. code:: python From 2563b9d569d97816ce6fdbd32c0048505fcc453a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 Jan 2024 11:30:13 -0500 Subject: [PATCH 308/365] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysat/tests/classes/cls_instrument_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 63a1b7a19..a56e23366 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -485,7 +485,7 @@ def test_load_empty(self, inst_dict): return - # TODO(#1172): remove mark.new_tests + # TODO(#1172): remove mark.new_tests at v3.3.0 @pytest.mark.second @pytest.mark.load_options @pytest.mark.new_tests From 9ebe084df6a21c04ffd080778f129e0adf035c9e Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 19 Jan 2024 14:30:00 -0600 Subject: [PATCH 309/365] STY: Moved funding institutions to single file --- acknowledgements.txt | 28 +++++++++++++++++++ pysat/_constellation.py | 2 -- pysat/_files.py | 2 -- pysat/_instrument.py | 2 -- pysat/_meta.py | 2 -- pysat/_orbits.py | 2 -- pysat/_params.py | 2 -- pysat/constellations/single_test.py | 2 -- pysat/constellations/testing.py | 2 -- pysat/constellations/testing_empty.py | 2 -- pysat/constellations/testing_partial.py | 2 -- pysat/instruments/methods/general.py | 2 -- pysat/instruments/methods/testing.py | 2 -- pysat/instruments/pysat_ndtesting.py | 2 -- pysat/instruments/pysat_netcdf.py | 2 -- pysat/instruments/pysat_testing.py | 2 -- pysat/instruments/pysat_testmodel.py | 2 -- pysat/tests/classes/cls_ci.py | 2 -- pysat/tests/classes/cls_instrument_access.py | 2 -- .../classes/cls_instrument_integration.py | 2 -- .../tests/classes/cls_instrument_iteration.py | 2 -- pysat/tests/classes/cls_instrument_library.py | 2 -- .../tests/classes/cls_instrument_property.py | 2 -- pysat/tests/classes/cls_registration.py | 2 -- pysat/tests/test_constellation.py | 2 -- pysat/tests/test_files.py | 2 -- pysat/tests/test_instrument.py | 2 -- pysat/tests/test_instrument_custom.py | 2 -- pysat/tests/test_instrument_index.py | 2 -- pysat/tests/test_instrument_listgen.py | 2 -- pysat/tests/test_instrument_padding.py | 2 -- pysat/tests/test_instruments.py | 2 -- pysat/tests/test_meta.py | 2 -- pysat/tests/test_meta_header.py | 2 -- pysat/tests/test_meta_labels.py | 2 -- pysat/tests/test_methods_general.py | 2 -- pysat/tests/test_methods_testing.py | 2 -- pysat/tests/test_orbits.py | 2 -- pysat/tests/test_params.py | 2 -- pysat/tests/test_registry.py | 2 -- pysat/tests/test_utils.py | 2 -- pysat/tests/test_utils_coords.py | 2 -- pysat/tests/test_utils_files.py | 2 -- pysat/tests/test_utils_io.py | 2 -- pysat/tests/test_utils_testing.py | 2 -- pysat/tests/test_utils_time.py | 2 -- pysat/utils/_core.py | 2 -- pysat/utils/coords.py | 2 -- pysat/utils/files.py | 2 -- pysat/utils/io.py | 2 -- pysat/utils/registry.py | 2 -- pysat/utils/testing.py | 2 -- pysat/utils/time.py | 2 -- 53 files changed, 28 insertions(+), 104 deletions(-) create mode 100644 acknowledgements.txt diff --git a/acknowledgements.txt b/acknowledgements.txt new file mode 100644 index 000000000..3d744e400 --- /dev/null +++ b/acknowledgements.txt @@ -0,0 +1,28 @@ +The following institutions, programs, and missions have provided funding +for pysat development. + +Institutions +------------ +Cosmic Studio +DARPA Defense Sciences Office +Office of Naval Research (ONR) +National Aeronautics and Space Administration (NASA) +National Oceanic and Atmospheric Administration (NOAA) +National Science Foundation (NSF) + +Missions +-------- +NOAA Constellation Observing System for Meteorology Ionosphere and Climate (COSMIC-2) +NASA Ionospheric Connections Explorer (ICON) +NASA Scintillation Observations and Response of the Ionosphere to Electrodynamics (SORTIE) +NASA Scintillation Prediction Observations Research Task (SPORT) + +Programs +-------- +NSF 125908, AGS-1651393 +NASA NNX10AT02G, NNH20ZDA001N-LWS, 80NSSC18K120, and 80NSSC21M0180 +NASA Space Precipitation Impacts (SPI) project +Naval Research Laboratory N00173191G016 and N0017322P0744 + + + diff --git a/pysat/_constellation.py b/pysat/_constellation.py index 6a43b7999..bd8cd9d87 100644 --- a/pysat/_constellation.py +++ b/pysat/_constellation.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Class for Instrument constellations. diff --git a/pysat/_files.py b/pysat/_files.py index f17377972..5e0a505f7 100644 --- a/pysat/_files.py +++ b/pysat/_files.py @@ -4,8 +4,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- import copy diff --git a/pysat/_instrument.py b/pysat/_instrument.py index d5bcaed2d..d1b98ebf5 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Class for single instruments.""" diff --git a/pysat/_meta.py b/pysat/_meta.py index 183b163bd..21694050b 100644 --- a/pysat/_meta.py +++ b/pysat/_meta.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Classes for storing and managing meta data.""" diff --git a/pysat/_orbits.py b/pysat/_orbits.py index b2d07c1f0..6a1301e50 100644 --- a/pysat/_orbits.py +++ b/pysat/_orbits.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- import copy diff --git a/pysat/_params.py b/pysat/_params.py index cbed4b85c..4dc572b3d 100644 --- a/pysat/_params.py +++ b/pysat/_params.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- import copy diff --git a/pysat/constellations/single_test.py b/pysat/constellations/single_test.py index b619279b6..4709ceacd 100644 --- a/pysat/constellations/single_test.py +++ b/pysat/constellations/single_test.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Create a constellation with one testing instrument. diff --git a/pysat/constellations/testing.py b/pysat/constellations/testing.py index a39b72238..d4b715d9c 100644 --- a/pysat/constellations/testing.py +++ b/pysat/constellations/testing.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Create a constellation with 5 testing instruments. diff --git a/pysat/constellations/testing_empty.py b/pysat/constellations/testing_empty.py index 97ed4b65b..dd9f2d59a 100644 --- a/pysat/constellations/testing_empty.py +++ b/pysat/constellations/testing_empty.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Create an empty constellation for testing. diff --git a/pysat/constellations/testing_partial.py b/pysat/constellations/testing_partial.py index 857ef50b6..351901e07 100644 --- a/pysat/constellations/testing_partial.py +++ b/pysat/constellations/testing_partial.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Create a constellation where not all instruments have loadable data. diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py index 1c79ac818..0a7cedfa0 100644 --- a/pysat/instruments/methods/general.py +++ b/pysat/instruments/methods/general.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Provides generalized routines for integrating instruments into pysat.""" diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py index 6e7bf9aab..1c40414ee 100644 --- a/pysat/instruments/methods/testing.py +++ b/pysat/instruments/methods/testing.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Standard functions for the test instruments.""" diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py index 1acabd746..1feae41f1 100644 --- a/pysat/instruments/pysat_ndtesting.py +++ b/pysat/instruments/pysat_ndtesting.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Produces fake instrument data for testing.""" diff --git a/pysat/instruments/pysat_netcdf.py b/pysat/instruments/pysat_netcdf.py index 6988034db..f989acaf9 100644 --- a/pysat/instruments/pysat_netcdf.py +++ b/pysat/instruments/pysat_netcdf.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """General Instrument for loading pysat-written netCDF files. diff --git a/pysat/instruments/pysat_testing.py b/pysat/instruments/pysat_testing.py index d5c9b0ce9..a4c6183de 100644 --- a/pysat/instruments/pysat_testing.py +++ b/pysat/instruments/pysat_testing.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Produces fake instrument data for testing.""" diff --git a/pysat/instruments/pysat_testmodel.py b/pysat/instruments/pysat_testmodel.py index 3a343293c..b0bf28105 100644 --- a/pysat/instruments/pysat_testmodel.py +++ b/pysat/instruments/pysat_testmodel.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Produces fake instrument data for testing.""" diff --git a/pysat/tests/classes/cls_ci.py b/pysat/tests/classes/cls_ci.py index 9be8fc8e4..17b6c1c4a 100644 --- a/pysat/tests/classes/cls_ci.py +++ b/pysat/tests/classes/cls_ci.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Class setup and teardown for unit tests that are only run in the CI env.""" diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py index 4751c6e24..f7c252122 100644 --- a/pysat/tests/classes/cls_instrument_access.py +++ b/pysat/tests/classes/cls_instrument_access.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests for data access and related functions in the pysat Instrument object. diff --git a/pysat/tests/classes/cls_instrument_integration.py b/pysat/tests/classes/cls_instrument_integration.py index f49efa95e..caaee66d6 100644 --- a/pysat/tests/classes/cls_instrument_integration.py +++ b/pysat/tests/classes/cls_instrument_integration.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Integration tests for pysat.Instrument. diff --git a/pysat/tests/classes/cls_instrument_iteration.py b/pysat/tests/classes/cls_instrument_iteration.py index f8cf3f86f..c820ee788 100644 --- a/pysat/tests/classes/cls_instrument_iteration.py +++ b/pysat/tests/classes/cls_instrument_iteration.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests for iteration in the pysat Instrument object and methods. diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index 339eb95bf..48187f918 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Standardized class and functions to test instruments for pysat libraries. diff --git a/pysat/tests/classes/cls_instrument_property.py b/pysat/tests/classes/cls_instrument_property.py index b91f63166..93a020d9e 100644 --- a/pysat/tests/classes/cls_instrument_property.py +++ b/pysat/tests/classes/cls_instrument_property.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Test for instrument properties in the pysat Instrument object and methods. diff --git a/pysat/tests/classes/cls_registration.py b/pysat/tests/classes/cls_registration.py index b3669f0c1..ae1da2e64 100644 --- a/pysat/tests/classes/cls_registration.py +++ b/pysat/tests/classes/cls_registration.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Standardized class and functions to test registration for pysat libraries. diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py index 5a0c16d51..b7dfe552b 100644 --- a/pysat/tests/test_constellation.py +++ b/pysat/tests/test_constellation.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the Constellation class.""" diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py index 0c96c7d64..31875f797 100644 --- a/pysat/tests/test_files.py +++ b/pysat/tests/test_files.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Test pysat Files object and code.""" diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 98524b917..3b19d1304 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Tests the pysat Instrument object and methods.""" diff --git a/pysat/tests/test_instrument_custom.py b/pysat/tests/test_instrument_custom.py index 12fefe9ea..574a72edd 100644 --- a/pysat/tests/test_instrument_custom.py +++ b/pysat/tests/test_instrument_custom.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the `custom_attach` methods for `pysat.Instrument`.""" diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py index a02ea0373..512b2bb7b 100644 --- a/pysat/tests/test_instrument_index.py +++ b/pysat/tests/test_instrument_index.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the `pysat.Instrument.index` attribute.""" diff --git a/pysat/tests/test_instrument_listgen.py b/pysat/tests/test_instrument_listgen.py index 40440d87e..71161fe42 100644 --- a/pysat/tests/test_instrument_listgen.py +++ b/pysat/tests/test_instrument_listgen.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the list generation methods in `pysat.Instrument`.""" diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py index c43acb44d..149d1610d 100644 --- a/pysat/tests/test_instrument_padding.py +++ b/pysat/tests/test_instrument_padding.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the padding methods in `pysat.Instrument`.""" diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index 51235804e..6561bf3c0 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit and Integration Tests for each instrument module. diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py index 1cb862f78..ec4ff8787 100644 --- a/pysat/tests/test_meta.py +++ b/pysat/tests/test_meta.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat Meta object.""" diff --git a/pysat/tests/test_meta_header.py b/pysat/tests/test_meta_header.py index f7ddc7365..b1f3aafde 100644 --- a/pysat/tests/test_meta_header.py +++ b/pysat/tests/test_meta_header.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat MetaHeader object.""" diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py index 77b1a3f16..ff433245e 100644 --- a/pysat/tests/test_meta_labels.py +++ b/pysat/tests/test_meta_labels.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat MetaLabels object.""" diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py index 94de6e606..1b0e403ce 100644 --- a/pysat/tests/test_methods_general.py +++ b/pysat/tests/test_methods_general.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Unit tests for the general instrument methods.""" diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py index c58ff5310..2af644cc1 100644 --- a/pysat/tests/test_methods_testing.py +++ b/pysat/tests/test_methods_testing.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the `pysat.instruments.methods.testing` methods.""" diff --git a/pysat/tests/test_orbits.py b/pysat/tests/test_orbits.py index 0208e2105..09ba52adb 100644 --- a/pysat/tests/test_orbits.py +++ b/pysat/tests/test_orbits.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Test the pysat routines for the orbits class.""" diff --git a/pysat/tests/test_params.py b/pysat/tests/test_params.py index 85dc02e4e..f51ba24c5 100644 --- a/pysat/tests/test_params.py +++ b/pysat/tests/test_params.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat parameters storage area.""" diff --git a/pysat/tests/test_registry.py b/pysat/tests/test_registry.py index 6580b4b82..3f042029a 100644 --- a/pysat/tests/test_registry.py +++ b/pysat/tests/test_registry.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the registration of user-defined modules.""" diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index 0aca70862..341a85209 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat utils core functions.""" diff --git a/pysat/tests/test_utils_coords.py b/pysat/tests/test_utils_coords.py index bc669a170..ff701bb14 100644 --- a/pysat/tests/test_utils_coords.py +++ b/pysat/tests/test_utils_coords.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the `pysat.utils.coords` functions.""" diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 5b3a71115..4e85d07b5 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the `pysat.utils.files` functions.""" diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 9bf732a9b..34dd526c8 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat utility io routines.""" import copy diff --git a/pysat/tests/test_utils_testing.py b/pysat/tests/test_utils_testing.py index 22a096b3e..5f7ef3e7c 100644 --- a/pysat/tests/test_utils_testing.py +++ b/pysat/tests/test_utils_testing.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests the pysat utility testing routines.""" diff --git a/pysat/tests/test_utils_time.py b/pysat/tests/test_utils_time.py index fc43c088c..fc9199e59 100644 --- a/pysat/tests/test_utils_time.py +++ b/pysat/tests/test_utils_time.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Tests for the pysat.utils.time functions.""" diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index 760aa924c..f721eb26c 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- import datetime as dt diff --git a/pysat/utils/coords.py b/pysat/utils/coords.py index 4984352eb..1ab471eee 100644 --- a/pysat/utils/coords.py +++ b/pysat/utils/coords.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Coordinate transformation functions for pysat.""" diff --git a/pysat/utils/files.py b/pysat/utils/files.py index e61b6df9e..56bc64a58 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Utilities for file management and parsing file names.""" diff --git a/pysat/utils/io.py b/pysat/utils/io.py index c73fc2ca2..78e419829 100644 --- a/pysat/utils/io.py +++ b/pysat/utils/io.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Input/Output utilities for pysat data.""" import copy diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py index 18077abd2..b0a1f6d9e 100644 --- a/pysat/utils/registry.py +++ b/pysat/utils/registry.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """pysat user module registry utilities. diff --git a/pysat/utils/testing.py b/pysat/utils/testing.py index 5e4822355..ff363aa93 100644 --- a/pysat/utils/testing.py +++ b/pysat/utils/testing.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Utilities to perform common evaluations.""" diff --git a/pysat/utils/time.py b/pysat/utils/time.py index 34513799c..298b29b82 100644 --- a/pysat/utils/time.py +++ b/pysat/utils/time.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by Cosmic Studio, the Office of Naval Research, -# and the National Aeronautics and Space Administration. # ---------------------------------------------------------------------------- """Date and time handling utilities.""" From eb8ad2a164f74bc905c1547ea67a8a7c368f3bbe Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 19 Jan 2024 14:32:41 -0600 Subject: [PATCH 310/365] STY: Added initial sentence pointing to funding file --- pysat/utils/_core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index f721eb26c..da9693a4a 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -5,6 +5,8 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. +# This work was supported by the institutions, missions, and programs listed in +# acknowledgements.txt. # ---------------------------------------------------------------------------- import datetime as dt From d2badfe0a67d43e4e78f4607e6bab431296b028f Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 19 Jan 2024 16:29:28 -0600 Subject: [PATCH 311/365] STY: Make .txt and .rst play nice --- docs/index.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index c650fdb44..33f2ada24 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,5 +28,6 @@ Welcome to pysat's documentation! .. admonition:: DISTRIBUTION STATEMENT A: Approved for public release. Distribution is unlimited. - This work was supported by Cosmic Studio, the Office of Naval Research, - and the National Aeronautics and Space Administration. +Funding +======= +.. include:: ../acknowledgements.txt \ No newline at end of file From 767d1060e8a20dd697033a2d30ed9e46d0d5e895 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 19 Jan 2024 16:30:23 -0600 Subject: [PATCH 312/365] STY: Make .txt and .rst play nice --- acknowledgements.txt | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/acknowledgements.txt b/acknowledgements.txt index 3d744e400..6c50ba34c 100644 --- a/acknowledgements.txt +++ b/acknowledgements.txt @@ -3,26 +3,23 @@ for pysat development. Institutions ------------ -Cosmic Studio -DARPA Defense Sciences Office -Office of Naval Research (ONR) -National Aeronautics and Space Administration (NASA) -National Oceanic and Atmospheric Administration (NOAA) -National Science Foundation (NSF) + - Cosmic Studio + - DARPA Defense Sciences Office + - Office of Naval Research (ONR) + - National Aeronautics and Space Administration (NASA) + - National Oceanic and Atmospheric Administration (NOAA) + - National Science Foundation (NSF) Missions -------- -NOAA Constellation Observing System for Meteorology Ionosphere and Climate (COSMIC-2) -NASA Ionospheric Connections Explorer (ICON) -NASA Scintillation Observations and Response of the Ionosphere to Electrodynamics (SORTIE) -NASA Scintillation Prediction Observations Research Task (SPORT) + - NOAA Constellation Observing System for Meteorology Ionosphere and Climate (COSMIC-2) + - NASA Ionospheric Connections Explorer (ICON) + - NASA Scintillation Observations and Response of the Ionosphere to Electrodynamics (SORTIE) + - NASA Scintillation Prediction Observations Research Task (SPORT) Programs -------- -NSF 125908, AGS-1651393 -NASA NNX10AT02G, NNH20ZDA001N-LWS, 80NSSC18K120, and 80NSSC21M0180 -NASA Space Precipitation Impacts (SPI) project -Naval Research Laboratory N00173191G016 and N0017322P0744 - - - + - NSF 125908, AGS-1651393 + - NASA NNX10AT02G, NNH20ZDA001N-LWS, 80NSSC18K120, and 80NSSC21M0180 + - NASA Space Precipitation Impacts (SPI) project + - Naval Research Laboratory N00173191G016 and N0017322P0744 From a4fce0256f3b24cc410a78624d4d61eb1baa98b4 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 19 Jan 2024 16:32:52 -0600 Subject: [PATCH 313/365] STY: Expanded DARPA definition --- acknowledgements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acknowledgements.txt b/acknowledgements.txt index 6c50ba34c..fc08ed423 100644 --- a/acknowledgements.txt +++ b/acknowledgements.txt @@ -1,10 +1,10 @@ -The following institutions, programs, and missions have provided funding +The following institutions, missions, and programs have provided funding for pysat development. Institutions ------------ - Cosmic Studio - - DARPA Defense Sciences Office + - Defense Advanced Research Projects Agency (DARPA) Defense Sciences Office - Office of Naval Research (ONR) - National Aeronautics and Space Administration (NASA) - National Oceanic and Atmospheric Administration (NOAA) From 117101ff8069dea79cad01747659b54652c9e8fa Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 25 Jan 2024 16:32:37 -0500 Subject: [PATCH 314/365] ENH: partial enhancement Adding type to file formatting. --- pysat/utils/files.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pysat/utils/files.py b/pysat/utils/files.py index 0f03e9f3c..efbd88bfa 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -54,6 +54,7 @@ def _init_parse_filenames(files, format_str): An output dict with the following keys: - 'search_string' (format_str with data to be parsed replaced with ?) - 'keys' (keys for data to be parsed) + - 'type' (type of data expected for each key to be parsed) - 'lengths' (string length for data to be parsed) - 'string_blocks' (the filenames are broken into fixed width segments). @@ -320,12 +321,18 @@ def parse_fixed_width_filenames(files, format_str): else: val = temp[key_str_idx[0][j]:key_str_idx[1][j]] + # Cast the data value, if possible + if search_dict['type'][j] is not None: + val = search_dict['type'][j](val) + # Save the parsed variable for this key and file if stored[key] is None: stored[key] = [val] else: stored[key].append(val) + raise RuntimeError('hi') + # Convert to numpy arrays and add additional information to output stored = _finish_parse_filenames(stored, files, format_str) @@ -476,6 +483,7 @@ def construct_searchstring_from_format(format_str, wildcard=False): An output dict with the following keys: - 'search_string' (format_str with data to be parsed replaced with ?) - 'keys' (keys for data to be parsed) + - 'type' (type of data expected for each key to be parsed) - 'lengths' (string length for data to be parsed) - 'string_blocks' (the filenames are broken into fixed width segments). @@ -498,8 +506,12 @@ def construct_searchstring_from_format(format_str, wildcard=False): """ - out_dict = {'search_string': '', 'keys': [], 'lengths': [], + out_dict = {'search_string': '', 'keys': [], 'type': [], 'lengths': [], 'string_blocks': []} + type_dict = {'s': str, 'b': np.int64, 'c': np.int64, 'd': np.int64, + 'o': np.int64, 'e': np.float64, 'E': np.float64, + 'f': np.float64, 'F': np.float64, 'g': np.float64, + 'G': np.float64} if format_str is None: raise ValueError("Must supply a filename template (format_str).") @@ -518,6 +530,15 @@ def construct_searchstring_from_format(format_str, wildcard=False): if snip[1] is not None: out_dict['keys'].append(snip[1]) + if snip[2] is None: + out_dict['type'].append(snip[2]) + else: + snip_type = snip[2][-1] + if snip_type in type_dict.keys(): + out_dict['type'].append(type_dict[snip_type]) + else: + out_dict['type'].append(None) + # Try and determine formatting width fwidths = re.findall(r'\d+', snip[2]) From 454f51e5d1d583bebfa78a18afff24974dbe18ba Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 26 Jan 2024 16:20:21 -0600 Subject: [PATCH 315/365] STY: Removed statement as requested --- pysat/utils/_core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py index da9693a4a..f721eb26c 100644 --- a/pysat/utils/_core.py +++ b/pysat/utils/_core.py @@ -5,8 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the institutions, missions, and programs listed in -# acknowledgements.txt. # ---------------------------------------------------------------------------- import datetime as dt From e8145cc902b95a0a45ad32bcead0ac3400282563 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 26 Jan 2024 16:23:23 -0600 Subject: [PATCH 316/365] STY: Moved funding to its own section --- docs/funding.rst | 3 +++ docs/index.rst | 5 +---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 docs/funding.rst diff --git a/docs/funding.rst b/docs/funding.rst new file mode 100644 index 000000000..550f270c8 --- /dev/null +++ b/docs/funding.rst @@ -0,0 +1,3 @@ +Funding +======= +.. include:: ../acknowledgements.txt diff --git a/docs/index.rst b/docs/index.rst index 33f2ada24..77e8df299 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,7 @@ Welcome to pysat's documentation! introduction.rst ecosystem.rst citing.rst + funding.rst installation.rst quickstart.rst tutorial.rst @@ -27,7 +28,3 @@ Welcome to pysat's documentation! .. admonition:: DISTRIBUTION STATEMENT A: Approved for public release. Distribution is unlimited. - -Funding -======= -.. include:: ../acknowledgements.txt \ No newline at end of file From d28fccfb463ecaa57667132cdd7fec8510922791 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 26 Jan 2024 18:58:39 -0500 Subject: [PATCH 317/365] ENH: added file key type discernment Added the ability for pysat to identify files that shouldn't be considered based on the expected value type of the identifying key. --- pysat/utils/files.py | 49 +++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/pysat/utils/files.py b/pysat/utils/files.py index efbd88bfa..8ce81f1a2 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -90,7 +90,7 @@ def _init_parse_filenames(files, format_str): return stored, search_dict -def _finish_parse_filenames(stored, files, format_str): +def _finish_parse_filenames(stored, files, format_str, bad_files): """Reshape and finalize the output for the file parsing functions. Parameters @@ -110,6 +110,9 @@ def _finish_parse_filenames(stored, files, format_str): and 'cycle' will be used for time and sorting information. For example, `instrument-{year:4d}_{month:02d}-{day:02d}_v{version:02d}.cdf`, or `*-{year:4d}_{month:02d}hithere{day:02d}_v{version:02d}.cdf` + bad_files : list + List of indices for files with names that do not fit the requested + format, or an empty list if all are good. Returns ------- @@ -125,20 +128,25 @@ def _finish_parse_filenames(stored, files, format_str): pysat.utils.files.parse_delimited_filenames """ - + # Change the bad file index list to a good file index list + good_files = [i for i in range(len(files)) if i not in bad_files] + # Convert to numpy arrays for key in stored.keys(): if stored[key] is not None: - try: - # Assume key value is numeric integer - stored[key] = np.array(stored[key]).astype(np.int64) - except ValueError: - # Store key value as string - stored[key] = np.array(stored[key]) + # Get the data type + dtype = type(stored[key][0]) + + # Cast the good data as an array of the desired type and select + # only the values with good files + stored[key] = np.array(stored[key])[good_files].astype(dtype) # Include files and file format in output - stored['files'] = files stored['format_str'] = format_str + if len(bad_files) == 0: + stored['files'] = files + else: + stored['files'] = list(np.array(files)[good_files]) return stored @@ -313,6 +321,7 @@ def parse_fixed_width_filenames(files, format_str): np.array(end_key, dtype=np.int64) - max_len] # Need to parse out dates for datetime index + bad_files = [] for i, temp in enumerate(files): for j, key in enumerate(search_dict['keys']): if key_str_idx[1][j] == 0: @@ -323,7 +332,11 @@ def parse_fixed_width_filenames(files, format_str): # Cast the data value, if possible if search_dict['type'][j] is not None: - val = search_dict['type'][j](val) + try: + val = search_dict['type'][j](val) + except ValueError: + # The type is wrong, exclude this file + bad_files.append(i) # Save the parsed variable for this key and file if stored[key] is None: @@ -331,10 +344,8 @@ def parse_fixed_width_filenames(files, format_str): else: stored[key].append(val) - raise RuntimeError('hi') - # Convert to numpy arrays and add additional information to output - stored = _finish_parse_filenames(stored, files, format_str) + stored = _finish_parse_filenames(stored, files, format_str, bad_files) return stored @@ -433,6 +444,7 @@ def parse_delimited_filenames(files, format_str, delimiter): if stored[key] is None: stored[key] = [] + bad_files = list() for temp in files: split_name = temp.split(delimiter) idx = 0 @@ -445,6 +457,15 @@ def parse_delimited_filenames(files, format_str, delimiter): val = loop_sname[sidx:sidx + search_dict['lengths'][idx]] loop_sname = loop_sname[sidx + search_dict['lengths'][idx]:] + # Cast the value as the desired data type, if not possible + # identify a bad file + if search_dict['type'][j] is not None: + try: + val = search_dict['type'][j](val) + except ValueError: + # The type is wrong, exclude this file + bad_files.append(i) + # Store parsed info and increment key index stored[search_dict['keys'][idx]].append(val) idx += 1 @@ -455,7 +476,7 @@ def parse_delimited_filenames(files, format_str, delimiter): break # Convert to numpy arrays and add additional information to output - stored = _finish_parse_filenames(stored, files, format_str) + stored = _finish_parse_filenames(stored, files, format_str, bad_files) return stored From 3a6a33f8cc435df6638737dc4c07d37bf46ed619 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 30 Jan 2024 16:34:17 -0500 Subject: [PATCH 318/365] BUG: fixed delimited implementation Fixed index use in `parse_delimited_filenames`. --- pysat/utils/files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysat/utils/files.py b/pysat/utils/files.py index 8ce81f1a2..7bd560043 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -445,7 +445,7 @@ def parse_delimited_filenames(files, format_str, delimiter): stored[key] = [] bad_files = list() - for temp in files: + for ifile, temp in enumerate(files): split_name = temp.split(delimiter) idx = 0 loop_split_idx = split_idx @@ -461,10 +461,10 @@ def parse_delimited_filenames(files, format_str, delimiter): # identify a bad file if search_dict['type'][j] is not None: try: - val = search_dict['type'][j](val) + val = search_dict['type'][idx](val) except ValueError: # The type is wrong, exclude this file - bad_files.append(i) + bad_files.append(ifile) # Store parsed info and increment key index stored[search_dict['keys'][idx]].append(val) From ad65b7ac54be359dea0c5babccfc47835b8e6c38 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 30 Jan 2024 16:34:54 -0500 Subject: [PATCH 319/365] TST: updated file tests Updated the file tests to include the new dict output and test for bad file removal by index. --- pysat/tests/test_utils_files.py | 35 ++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 01227a821..54810872c 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -46,7 +46,7 @@ def teardown_method(self): def eval_output(self): """Evaluate the output dictionary.""" - testing.assert_lists_equal(['search_string', 'keys', 'lengths', + testing.assert_lists_equal(['search_string', 'keys', 'type', 'lengths', 'string_blocks'], list(self.out_dict.keys())) @@ -337,7 +337,7 @@ def test_init_parse_filename_with_files(self): self.file_dict, sdict = futils._init_parse_filenames(file_list, fname) # Test the initalized dictionaries - testing.assert_lists_equal(['search_string', 'keys', 'lengths', + testing.assert_lists_equal(['search_string', 'keys', 'type', 'lengths', 'string_blocks'], list(sdict.keys())) for skey in sdict['keys']: @@ -351,20 +351,35 @@ def test_init_parse_filename_with_files(self): "'format_str' key set early" return - def test_finish_parsed_filenames(self): - """Test output restucturing for `_finish_parsed_filenames`.""" + @pytest.mark.parametrize("bad_files", [[], [0]]) + def test_finish_parsed_filenames(self, bad_files): + """Test output restucturing for `_finish_parsed_filenames`. + + Parameters + ---------- + bad_files : list + List of bad file indices + + """ # Format the test input fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}', '{minute:02d}', '{second:02d}', '{cycle:2s}.txt')) # Create the input file list and dict file_list = [fname.format(**kwargs) for kwargs in self.fkwargs] - self.file_dict = {'int': [1], 'none': None, 'float': [1.0], - 'str': ['hi']} + self.file_dict = {'int': [1 for fname in file_list], 'none': None, + 'float': [1.0 for fname in file_list], + 'str': ['hi' for fname in file_list]} # Get the test results self.file_dict = futils._finish_parse_filenames(self.file_dict, - file_list, fname) + file_list, fname, + bad_files) + + # Adjust the expected file output + if len(bad_files) > 0: + file_list = [fname for i, fname in enumerate(file_list) + if i not in bad_files] # Test the output for fkey in self.file_dict: @@ -376,11 +391,7 @@ def test_finish_parsed_filenames(self): assert fname == self.file_dict[fkey] else: testing.assert_isinstance(self.file_dict[fkey], np.ndarray) - - if fkey == 'str': - testing.assert_isinstance(self.file_dict[fkey][0], str) - else: - testing.assert_isinstance(self.file_dict[fkey][0], np.int64) + assert len(self.file_dict[fkey]) == len(file_list) return From 7cddfe77c874c39bc461e8e4a2c997ab3d4fa0d5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 30 Jan 2024 16:46:28 -0500 Subject: [PATCH 320/365] BUG: fixed another index Fixed another instance of the wrong index. --- pysat/utils/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/utils/files.py b/pysat/utils/files.py index 7bd560043..5e40ab85d 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -459,7 +459,7 @@ def parse_delimited_filenames(files, format_str, delimiter): # Cast the value as the desired data type, if not possible # identify a bad file - if search_dict['type'][j] is not None: + if search_dict['type'][idx] is not None: try: val = search_dict['type'][idx](val) except ValueError: From bb83b58ca4f5f43d4146e0219fdf31d31e82a880 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 30 Jan 2024 17:02:54 -0500 Subject: [PATCH 321/365] DOC: updated header Removed old line from header. --- pysat/tests/test_utils_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py index 3bba2ae3a..34dd526c8 100644 --- a/pysat/tests/test_utils_io.py +++ b/pysat/tests/test_utils_io.py @@ -5,7 +5,6 @@ # # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. -# This work was supported by the Office of Naval Research. # ---------------------------------------------------------------------------- """Tests the pysat utility io routines.""" import copy From 9b10e1c4868115dab1ca292f1f604ac73ba14a8a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 30 Jan 2024 17:03:27 -0500 Subject: [PATCH 322/365] BUG: updated default type for time parameters Set the default type for time parameters to be int, if none is supplied. --- pysat/utils/files.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pysat/utils/files.py b/pysat/utils/files.py index 5e40ab85d..af9701253 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -525,6 +525,8 @@ def construct_searchstring_from_format(format_str, wildcard=False): This is the first function employed by `pysat.Files.from_os`. + If no type is supplied for datetime parameters, int will be used. + """ out_dict = {'search_string': '', 'keys': [], 'type': [], 'lengths': [], @@ -533,6 +535,8 @@ def construct_searchstring_from_format(format_str, wildcard=False): 'o': np.int64, 'e': np.float64, 'E': np.float64, 'f': np.float64, 'F': np.float64, 'g': np.float64, 'G': np.float64} + int_keys = ['year', 'month', 'day', 'hour', 'minute', 'second', + 'microsecond'] if format_str is None: raise ValueError("Must supply a filename template (format_str).") @@ -557,6 +561,8 @@ def construct_searchstring_from_format(format_str, wildcard=False): snip_type = snip[2][-1] if snip_type in type_dict.keys(): out_dict['type'].append(type_dict[snip_type]) + elif snip[1] in int_keys: + out_dict['type'].append(np.int64) else: out_dict['type'].append(None) From 049d3ec02efe81f4c31142d4293ecfa24e6455b5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 30 Jan 2024 17:23:36 -0500 Subject: [PATCH 323/365] TST: test file removal by bad format Test file removal by bad format specifier. --- pysat/tests/test_utils_files.py | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index 54810872c..c53fc9db6 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -241,6 +241,40 @@ def test_parse_delimited_filename(self, sep_char, flead, good_kwargs): self.eval_parsed_filenames() return + @pytest.mark.parametrize("is_fixed", [True, False]) + def test_parse_filenames_all_bad(self, is_fixed): + """Test files with a bad format are removed from consideration. + + Parameters + ---------- + is_fixed : bool + True for the fixed-width function, false for delimted. + + """ + + # Format the test input + format_str = 'bad_test_{:s}.cdf'.format("_".join( + [self.kw_format[fkey] for fkey in self.fkwargs[0].keys()])) + bad_format = format_str.replace('revision:02d', 'revision:2s') + + # Create the input file list + file_list = [] + for kwargs in self.fkwargs: + kwargs['revision'] = 'aa' + file_list.append(bad_format.format(**kwargs)) + + # Get the test results + if is_fixed: + self.file_dict = futils.parse_fixed_width_filenames(file_list, + format_str) + else: + self.file_dict = futils.parse_delimited_filenames(file_list, + format_str, "_") + + # Test that all files were removed + assert len(self.file_dict['files']) == 0 + return + def test_parse_delimited_filename_empty(self): """Check ability to parse list of delimited files with no files.""" # Format the test input From 73893fb847388e2e108bc329f852f665696fbf18 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 30 Jan 2024 17:24:31 -0500 Subject: [PATCH 324/365] STY: removed whitespace Removed extra whitespace. --- pysat/tests/test_utils_files.py | 2 +- pysat/utils/files.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py index c53fc9db6..38fcff7c1 100644 --- a/pysat/tests/test_utils_files.py +++ b/pysat/tests/test_utils_files.py @@ -256,7 +256,7 @@ def test_parse_filenames_all_bad(self, is_fixed): format_str = 'bad_test_{:s}.cdf'.format("_".join( [self.kw_format[fkey] for fkey in self.fkwargs[0].keys()])) bad_format = format_str.replace('revision:02d', 'revision:2s') - + # Create the input file list file_list = [] for kwargs in self.fkwargs: diff --git a/pysat/utils/files.py b/pysat/utils/files.py index af9701253..00aacfd95 100644 --- a/pysat/utils/files.py +++ b/pysat/utils/files.py @@ -130,7 +130,7 @@ def _finish_parse_filenames(stored, files, format_str, bad_files): """ # Change the bad file index list to a good file index list good_files = [i for i in range(len(files)) if i not in bad_files] - + # Convert to numpy arrays for key in stored.keys(): if stored[key] is not None: From b6f262b79c664db179e833dcb5e5789f80ad68e5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 30 Jan 2024 17:26:07 -0500 Subject: [PATCH 325/365] DOC: updated changelog Updated the changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9ba3c769..096985e63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added test for loading multiple days of data. * Bug Fix * Fixed `utils.files.parse_fixed_width_filenames` output for empty file list + * Updated the parsing functions in `utils.files` to consider type specifiers + when identifying appropriate files in a directory * Maintenance * Update link redirects in docs. * Improved Instrument ValueError messages. From e9d6cf6055c32638423b247dff671c35dc5d25a5 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:57:15 -0500 Subject: [PATCH 326/365] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e07d0779c..a47f5a8e0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - python-version: ["3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] numpy_ver: ["latest"] test_config: ["latest"] include: From e2336a97c1298960e841c4d3290044239104fda0 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 31 Jan 2024 16:28:36 -0500 Subject: [PATCH 327/365] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5ea8636bb..52d140619 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "dask", "netCDF4", "numpy >= 1.12", - "pandas < 2.1.1", + "pandas", "portalocker", "pytest", "scipy", From 18baa4f6770e252248231ec907842057bd29cfe9 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:05:55 -0500 Subject: [PATCH 328/365] MAINT: remove pandas cap --- README.md | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be085004b..86886932a 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Space Physics community. This module officially supports Python 3.X+. | -------------- | ----------------- | | dask | netCDF4 | | numpy >= 1.12 | | -| pandas < 2.1.1 | | +| pandas | | | portalocker | | | pytest | | | scipy | | diff --git a/requirements.txt b/requirements.txt index 059c99271..6671a1449 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ dask netCDF4 numpy>=1.12 -pandas<2.1.1 +pandas portalocker pytest scipy From df82cf391a91ead4205d44c3715017aff0d5864e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:07:54 -0500 Subject: [PATCH 329/365] MAINT: update action versions --- .github/workflows/docs.yml | 4 ++-- .github/workflows/main.yml | 4 ++-- .github/workflows/pip_rc_install.yml | 4 ++-- .github/workflows/stats.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 328ad1813..266a1479c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,9 +17,9 @@ jobs: name: Documentation tests steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e07d0779c..658617d7a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,9 +30,9 @@ jobs: name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 0611a98b7..cfb33fd61 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -18,9 +18,9 @@ jobs: name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index 74e5b09ea..9a15d0ae4 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -18,9 +18,9 @@ jobs: name: Summary of instrument libraries steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 33995f9a86c972aa9104c62151c734a62448e6bc Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:08:08 -0500 Subject: [PATCH 330/365] MAINT: update python versions --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 658617d7a..99e9bc565 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,13 +12,13 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - python-version: ["3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] numpy_ver: ["latest"] test_config: ["latest"] include: # NEP29 compliance settings - python-version: "3.9" - numpy_ver: "1.21" + numpy_ver: "1.23" os: ubuntu-latest test_config: "NEP29" # Operational compliance settings From 8818edd87f35d7915db03317165df00f4956c09d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:10:36 -0500 Subject: [PATCH 331/365] DOC: update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9ba3c769..5b4820314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -[3.2.0] - 2023-xx-xx +[3.2.0] - 2024-02-xx -------------------- * New Features * Added tests for warnings, logging messages, and errors in the Instrument @@ -39,7 +39,6 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Implement pyproject to manage metadata * Updated docstring references to `pysat.utils.files` in other modules. * Remove Sphinx cap - * Add pandas cap * Update usage of whitespace and if statements (E275) * Remove hacking cap * Removed deprecated `pysat_testing2d` instrument From a6edad1984a9f2b714981891894cd5b6c03fd61c Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:14:03 -0500 Subject: [PATCH 332/365] REV: 3.12 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 99e9bc565..672e6d2a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11"] numpy_ver: ["latest"] test_config: ["latest"] include: From 9cfb9d9c7545af46e625b47231bf879c227341b3 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:40:22 -0500 Subject: [PATCH 333/365] MAINT: update error message for py 3.12 --- pysat/tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py index f3b188340..838cc4534 100644 --- a/pysat/tests/test_utils.py +++ b/pysat/tests/test_utils.py @@ -481,7 +481,7 @@ def test_neg_ncols(self): [("ncols", 0, ZeroDivisionError, "integer division or modulo by zero"), ("max_num", -10, ValueError, - "max() arg is an empty sequence")]) + "empty")]) def test_fmt_raises(self, key, val, raise_type, err_msg): """Test raises appropriate Errors for bad input values. From bb9d861f80b0b438a85cca8abb2c5e416bfafa64 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:40:42 -0500 Subject: [PATCH 334/365] DOC: update pyproject --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5ea8636bb..75d7e99e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", "Operating System :: Microsoft :: Windows" From c19a624b0a20a48a0410819342376b995d4ca0d0 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:41:08 -0500 Subject: [PATCH 335/365] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9ba3c769..8737e1c14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added verion cap for sphinx_rtd_theme * Used line specific noqa statements for imports * Add `_new_tests` flag for packages to ignore select new tests + * Add CI testing for python 3.12 [3.1.0] - 2023-05-31 -------------------- From 6efe97cef6dd4636f3c408a9709c78c606648118 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 1 Feb 2024 12:30:01 -0500 Subject: [PATCH 336/365] TST: remove unnecessary clean flag Remove a clean flag that stand in the way of this test passing. --- pysat/tests/classes/cls_instrument_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index a56e23366..8b1fe1776 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -507,7 +507,7 @@ def test_load_multiple_days(self, inst_dict): # Make sure the strict time flag doesn't interfere with # the load tests, and re-run with desired clean level load_and_set_strict_time_flag(self.test_inst, self.date, - raise_error=True, clean_off=False, + raise_error=True, clean_off=True, set_end_date=True) # Make sure more than one day has been loaded From f6dbce676ffb7a04b6996b229ace05d3252a5d46 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:33:28 -0500 Subject: [PATCH 337/365] Apply suggestions from code review Co-authored-by: Angeline Burrell --- pysat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/__init__.py b/pysat/__init__.py index c5a71b8ca..35af794dc 100644 --- a/pysat/__init__.py +++ b/pysat/__init__.py @@ -116,7 +116,7 @@ # Load up existing parameters file params = _params.Parameters() -# utils used by other imports, needs to be imported first. +# Modules used by other imports needs to be imported here first. from pysat import utils # noqa: E402 F401 # Import the remainder of the modules. From 29dff7c51f10c87361fb0165481666d75ed5f0cd Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 2 Feb 2024 15:01:10 -0600 Subject: [PATCH 338/365] Update acknowledgements.txt Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- acknowledgements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acknowledgements.txt b/acknowledgements.txt index fc08ed423..d2794ac1b 100644 --- a/acknowledgements.txt +++ b/acknowledgements.txt @@ -21,5 +21,5 @@ Programs -------- - NSF 125908, AGS-1651393 - NASA NNX10AT02G, NNH20ZDA001N-LWS, 80NSSC18K120, and 80NSSC21M0180 - - NASA Space Precipitation Impacts (SPI) project + - NASA Space Precipitation Impacts (SPI) project at Goddard Space Flight Center through the Heliophysics Internal Science Funding Model. - Naval Research Laboratory N00173191G016 and N0017322P0744 From 67e9c89270172cb3fda3cdd43574597d05cf498f Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Sun, 4 Feb 2024 13:15:58 -0600 Subject: [PATCH 339/365] Apply suggestions from code review --- acknowledgements.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/acknowledgements.txt b/acknowledgements.txt index d2794ac1b..7d0a5fe39 100644 --- a/acknowledgements.txt +++ b/acknowledgements.txt @@ -3,7 +3,8 @@ for pysat development. Institutions ------------ - - Cosmic Studio + - The Catholic University of America (CUA) + - Cosmic Studio - Defense Advanced Research Projects Agency (DARPA) Defense Sciences Office - Office of Naval Research (ONR) - National Aeronautics and Space Administration (NASA) @@ -23,3 +24,7 @@ Programs - NASA NNX10AT02G, NNH20ZDA001N-LWS, 80NSSC18K120, and 80NSSC21M0180 - NASA Space Precipitation Impacts (SPI) project at Goddard Space Flight Center through the Heliophysics Internal Science Funding Model. - Naval Research Laboratory N00173191G016 and N0017322P0744 + +Disclaimers +=========== +Any opinions or actions taken by the listed funding institutions are those of the institutions and do not necessarily reflect the views of the pysat development team or individual authors. Any opinions, findings, and conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the funding agencies. From 3687bc142a8e59c071a2e552101c8c1ca8d51abb Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Sun, 4 Feb 2024 13:19:14 -0600 Subject: [PATCH 340/365] DOC: Cleanup --- acknowledgements.txt | 6 ++++-- docs/funding.rst | 2 -- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acknowledgements.txt b/acknowledgements.txt index 7d0a5fe39..2cb4c6397 100644 --- a/acknowledgements.txt +++ b/acknowledgements.txt @@ -1,10 +1,12 @@ +Funding +======= The following institutions, missions, and programs have provided funding for pysat development. Institutions ------------ - - The Catholic University of America (CUA) - - Cosmic Studio + - The Catholic University of America (CUA) + - Cosmic Studio - Defense Advanced Research Projects Agency (DARPA) Defense Sciences Office - Office of Naval Research (ONR) - National Aeronautics and Space Administration (NASA) diff --git a/docs/funding.rst b/docs/funding.rst index 550f270c8..33c8cdcbe 100644 --- a/docs/funding.rst +++ b/docs/funding.rst @@ -1,3 +1 @@ -Funding -======= .. include:: ../acknowledgements.txt From 6d926c4e80bff2c6d493b2a417556f18a73e6b60 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Sun, 4 Feb 2024 16:24:31 -0600 Subject: [PATCH 341/365] DOC: Last transition, awesomeical -> alphabetical --- acknowledgements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acknowledgements.txt b/acknowledgements.txt index 2cb4c6397..729414ddd 100644 --- a/acknowledgements.txt +++ b/acknowledgements.txt @@ -8,10 +8,10 @@ Institutions - The Catholic University of America (CUA) - Cosmic Studio - Defense Advanced Research Projects Agency (DARPA) Defense Sciences Office - - Office of Naval Research (ONR) - National Aeronautics and Space Administration (NASA) - National Oceanic and Atmospheric Administration (NOAA) - National Science Foundation (NSF) + - Office of Naval Research (ONR) Missions -------- From cba2578cf04adaaf9de0266f10594ce50a24f48e Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Mon, 5 Feb 2024 08:03:53 -0600 Subject: [PATCH 342/365] DOC: Corrected changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 819f70527..ed09756a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). behaviour to `use_header=True` * Use temporary directories for files created during test_utils.py * Updated code file headers to be consistent and include NRL pub release - * Updated code file headers to include full institutional funding list + * Added acknowledgements.rst which includes full institutional funding list * Removed deprecated `labels` kwarg for `pysat.Instrument()` * Removed deprecated `utils.load_netcdf4` method * Removed deprecated `_filter_netcdf4_metadata` method From 448c6efef6ec1d1ad5fc72fa7d64cbcfb7e9bb4d Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Mon, 5 Feb 2024 14:34:28 -0600 Subject: [PATCH 343/365] STY: Updated filename per feedback --- CHANGELOG.md | 2 +- acknowledgements.txt => acknowledgements.md | 0 docs/funding.rst | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename acknowledgements.txt => acknowledgements.md (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed09756a4..eeb4246b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). behaviour to `use_header=True` * Use temporary directories for files created during test_utils.py * Updated code file headers to be consistent and include NRL pub release - * Added acknowledgements.rst which includes full institutional funding list + * Added acknowledgements.md which includes full institutional funding list * Removed deprecated `labels` kwarg for `pysat.Instrument()` * Removed deprecated `utils.load_netcdf4` method * Removed deprecated `_filter_netcdf4_metadata` method diff --git a/acknowledgements.txt b/acknowledgements.md similarity index 100% rename from acknowledgements.txt rename to acknowledgements.md diff --git a/docs/funding.rst b/docs/funding.rst index 33c8cdcbe..6ecbd0f7e 100644 --- a/docs/funding.rst +++ b/docs/funding.rst @@ -1 +1 @@ -.. include:: ../acknowledgements.txt +.. include:: ../acknowledgements.md From 0d4c84592671873340242a91d5a993e9c93f1b60 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Mon, 5 Feb 2024 14:46:42 -0600 Subject: [PATCH 344/365] STY: Missed capitalization on the first round --- acknowledgements.md => ACKOWLEDGEMENTS.md | 0 docs/funding.rst | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename acknowledgements.md => ACKOWLEDGEMENTS.md (100%) diff --git a/acknowledgements.md b/ACKOWLEDGEMENTS.md similarity index 100% rename from acknowledgements.md rename to ACKOWLEDGEMENTS.md diff --git a/docs/funding.rst b/docs/funding.rst index 6ecbd0f7e..24864daf9 100644 --- a/docs/funding.rst +++ b/docs/funding.rst @@ -1 +1 @@ -.. include:: ../acknowledgements.md +.. include:: ../ACKNOWLEDGEMENTS.md From 88319eff2341b3498a59f214b0c8d2179c87fd96 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 16 Feb 2024 12:42:54 -0600 Subject: [PATCH 345/365] ENH: Added check for files in load command. Error if no files. --- pysat/_instrument.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 6043a89ed..594f9687a 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -2965,6 +2965,13 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, else: use_header = True + # Provide user friendly error if there is no data + if len(self.files.files) == 0: + estr = ''.join(('No files found for Instrument. Please confirm ', + 'that data is present on the system and that ', + "pysat.params['data_dirs'] is set correctly.")) + raise OSError(estr) + # Add the load kwargs from initialization those provided on input for lkey in self.kwargs['load'].keys(): # Only use the initialized kwargs if a request hasn't been From 4e30646aca1a421b0cd2be7db1818349101617f7 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 16 Feb 2024 12:43:05 -0600 Subject: [PATCH 346/365] TST: Test for loading Instrument with no files. --- pysat/tests/test_instrument.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 3b19d1304..65f47844c 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -464,6 +464,13 @@ def test_creating_empty_instrument_object(self): assert isinstance(self.empty_inst, pysat.Instrument) return + def test_load_empty_instrument_no_files_error(self): + """Ensure error loading Instrument with no files.""" + estr = 'No files found for Instrument' + testing.eval_bad_input(self.empty_inst.load, OSError, estr) + return + + def test_empty_repr_eval(self): """Test that repr functions on empty `Instrument`.""" From 2976c24adfbf14ce756ed178517cdc10b6ad82b8 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 16 Feb 2024 12:44:42 -0600 Subject: [PATCH 347/365] DOC: Updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ada31364b..6047da24f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added ability to set Meta data using `meta['data_var', 'label'] = value` structure. * Added test for loading multiple days of data. + * Added user friendly error message when trying to load data when there are + no files to load. * Bug Fix * Fixed `utils.files.parse_fixed_width_filenames` output for empty file list * Updated the parsing functions in `utils.files` to consider type specifiers From 84d79a07b75500d0cc630faaae302f776e07ce14 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 16 Feb 2024 13:00:48 -0600 Subject: [PATCH 348/365] DOC: Corrected ACKNOWLEDGEMENTS.md name --- ACKOWLEDGEMENTS.md => ACKNOWLEDGEMENTS.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ACKOWLEDGEMENTS.md => ACKNOWLEDGEMENTS.md (100%) diff --git a/ACKOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md similarity index 100% rename from ACKOWLEDGEMENTS.md rename to ACKNOWLEDGEMENTS.md From 2324224b7a7a7a2cad330d36187db1298ac22896 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 16 Feb 2024 13:10:37 -0600 Subject: [PATCH 349/365] STY: Corrected number of empty lines in tests --- pysat/tests/test_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index 65f47844c..a8cda3516 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -466,11 +466,11 @@ def test_creating_empty_instrument_object(self): def test_load_empty_instrument_no_files_error(self): """Ensure error loading Instrument with no files.""" + estr = 'No files found for Instrument' testing.eval_bad_input(self.empty_inst.load, OSError, estr) return - def test_empty_repr_eval(self): """Test that repr functions on empty `Instrument`.""" From 7ce714166000c7b809ec3dee1825234de1cf2c35 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 21 Feb 2024 09:52:04 -0500 Subject: [PATCH 350/365] BUG: fixed clean level specification Using the `clean_off` flag doesn't change the clean level for normal loads. Changed before calling the load function and also updated docstrings. --- pysat/tests/classes/cls_instrument_library.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index b87f0753b..79cccd6c2 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -489,7 +489,7 @@ def test_load_empty(self, inst_dict): @pytest.mark.load_options @pytest.mark.new_tests def test_load_multiple_days(self, inst_dict): - """Test that instruments load at each cleaning level. + """Test that instruments load multiple days when requested. Parameters ---------- @@ -505,6 +505,7 @@ def test_load_multiple_days(self, inst_dict): if self.date < self.test_inst.today(): # Make sure the strict time flag doesn't interfere with # the load tests, and re-run with desired clean level + self.test_inst.clean_level = 'none' load_and_set_strict_time_flag(self.test_inst, self.date, raise_error=True, clean_off=True, set_end_date=True) @@ -628,7 +629,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog): @pytest.mark.new_tests @pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)]) def test_load_w_pad(self, pad, inst_dict): - """Test that instruments load at each cleaning level. + """Test that instruments load with a pad specified different ways. Parameters ---------- @@ -668,6 +669,7 @@ def test_load_w_pad(self, pad, inst_dict): if len(self.test_inst.files.files) > 0: # Make sure the strict time flag doesn't interfere with # the load tests + self.test_inst.clean = 'none' load_and_set_strict_time_flag(self.test_inst, self.date, raise_error=True, clean_off=True) From ca9a02b279367c9b2ad8168db7daeebd83936414 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 21 Feb 2024 10:02:36 -0500 Subject: [PATCH 351/365] MAINT: tidy tests --- pysat/tests/classes/cls_instrument_library.py | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py index b87f0753b..3dcf1ca57 100644 --- a/pysat/tests/classes/cls_instrument_library.py +++ b/pysat/tests/classes/cls_instrument_library.py @@ -210,13 +210,14 @@ def setup_method(self): """Initialize parameters before each method.""" self.test_inst = None self.date = None + self.module = None return def teardown_method(self): """Clean up any instruments that were initialized.""" - del self.test_inst, self.date + del self.test_inst, self.date, self.module return @@ -302,35 +303,36 @@ def test_modules_standard(self, inst_name): """ # Ensure that each module is at minimum importable - module = import_module(''.join(('.', inst_name)), - package=self.inst_loc.__name__) + self.module = import_module(''.join(('.', inst_name)), + package=self.inst_loc.__name__) # Check for presence of basic instrument module attributes for mattr in self.module_attrs: - testing.assert_hasattr(module, mattr) + testing.assert_hasattr(self.module, mattr) if mattr in self.attr_types.keys(): - testing.assert_isinstance(getattr(module, mattr), + testing.assert_isinstance(getattr(self.module, mattr), self.attr_types[mattr]) # Check for presence of required instrument attributes - for inst_id in module.inst_ids.keys(): - for tag in module.inst_ids[inst_id]: - inst = pysat.Instrument(inst_module=module, tag=tag, - inst_id=inst_id) + for inst_id in self.module.inst_ids.keys(): + for tag in self.module.inst_ids[inst_id]: + self.test_inst = pysat.Instrument(inst_module=self.module, + tag=tag, inst_id=inst_id) # Test to see that the class parameters were passed in - testing.assert_isinstance(inst, pysat.Instrument) - assert inst.platform == module.platform - assert inst.name == module.name - assert inst.inst_id == inst_id - assert inst.tag == tag - assert inst.inst_module is not None + testing.assert_isinstance(self.test_inst, pysat.Instrument) + assert self.test_inst.platform == self.module.platform + assert self.test_inst.name == self.module.name + assert self.test_inst.inst_id == inst_id + assert self.test_inst.tag == tag + assert self.test_inst.inst_module is not None # Test the required class attributes for iattr in self.inst_attrs: - testing.assert_hasattr(inst, iattr) + testing.assert_hasattr(self.test_inst, iattr) if iattr in self.attr_types: - testing.assert_isinstance(getattr(inst, iattr), + testing.assert_isinstance(getattr(self.test_inst, + iattr), self.attr_types[iattr]) return @@ -346,14 +348,14 @@ def test_standard_function_presence(self, inst_name): """ - module = import_module(''.join(('.', inst_name)), - package=self.inst_loc.__name__) + self.module = import_module(''.join(('.', inst_name)), + package=self.inst_loc.__name__) # Test for presence of all standard module functions for mcall in self.inst_callable: - if hasattr(module, mcall): + if hasattr(self.module, mcall): # If present, must be a callable function - assert callable(getattr(module, mcall)) + assert callable(getattr(self.module, mcall)) else: # If absent, must not be a required function assert mcall not in self.module_attrs @@ -371,9 +373,9 @@ def test_instrument_test_dates(self, inst_name): """ - module = import_module(''.join(('.', inst_name)), - package=self.inst_loc.__name__) - info = module._test_dates + self.module = import_module(''.join(('.', inst_name)), + package=self.inst_loc.__name__) + info = self.module._test_dates for inst_id in info.keys(): for tag in info[inst_id].keys(): testing.assert_isinstance(info[inst_id][tag], dt.datetime) @@ -744,10 +746,10 @@ def test_download_warning(self, inst_dict): """ - test_inst, self.date = initialize_test_inst_and_date(inst_dict) + self.test_inst, self.date = initialize_test_inst_and_date(inst_dict) with warnings.catch_warnings(record=True) as war: - test_inst.download(self.date, self.date) + self.test_inst.download(self.date, self.date) assert len(war) >= 1 categories = [war[j].category for j in range(0, len(war))] From 076c9769a78a32b3515ae22ac55cb9f1afe1b2ae Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 22 Feb 2024 13:27:47 -0500 Subject: [PATCH 352/365] MAINT: update test reqs --- pyproject.toml | 2 +- test_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 57cafb949..7c1f9a39e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ test = [ "flake8", "flake8-docstrings", "hacking >= 1.0", - "pysatSpaceWeather", + "pysatSpaceWeather<0.1.0", "pytest-cov", "pytest-ordering" ] diff --git a/test_requirements.txt b/test_requirements.txt index eade96797..366154df8 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -5,7 +5,7 @@ hacking>=1.0 ipython m2r2 numpydoc -pysatSpaceWeather +pysatSpaceWeather<0.1.0 pytest-cov pytest-ordering readthedocs-sphinx-search==0.3.2 From 0bdcaf2eea41934b6bd072de2ec30757052e83ff Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 22 Feb 2024 13:28:05 -0500 Subject: [PATCH 353/365] MAINT: move link out of docstring --- pysat/utils/time.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pysat/utils/time.py b/pysat/utils/time.py index 298b29b82..f4e2a8386 100644 --- a/pysat/utils/time.py +++ b/pysat/utils/time.py @@ -221,13 +221,9 @@ def freq_to_res(freq): -------- pds.offsets.DateOffset - References - ---------- - Separating alpha and numeric portions of strings, as described in: - https://stackoverflow.com/a/12409995 - """ - # Separate the alpha and numeric portions of the string + # Separate the alpha and numeric portions of the string as described in: + # https://stackoverflow.com/a/12409995 regex = re.compile(r'(\d+|\s+)') out_str = [sval for sval in regex.split(freq) if len(sval) > 0] From d7a5c322ad77e37ebe4f8b00be75e8fcac2eb936 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Tue, 27 Feb 2024 12:27:52 -0600 Subject: [PATCH 354/365] TST: Switched to raising warnings instead of Error --- pysat/_instrument.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index 594f9687a..e065b3bcf 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -2967,10 +2967,19 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # Provide user friendly error if there is no data if len(self.files.files) == 0: - estr = ''.join(('No files found for Instrument. Please confirm ', - 'that data is present on the system and that ', + # In pysat 3.3, modify this section to leave function early + # to prevent a downstream IndexError. Remove Deprecation portion + # of message above and leave as a UserWarning. + estr = ''.join(('No files found for Instrument. If files are ', + 'expected, please confirm that data is present ', + 'on the system and that ', "pysat.params['data_dirs'] is set correctly.")) - raise OSError(estr) + warnings.warn(estr, UserWarning, stacklevel=2) + estr = ''.join(("In pysat version 3.3.0+ the subsequent ", + 'IndexError will not be raised.')) + warnings.warn(estr, DeprecationWarning, stacklevel=2) + # Uncomment line below, pysat 3.3.0+ + # return # Add the load kwargs from initialization those provided on input for lkey in self.kwargs['load'].keys(): From 52f5670a9b11002fa929d4c64aca4d95102fd090 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Tue, 27 Feb 2024 12:28:14 -0600 Subject: [PATCH 355/365] TST: Added check for warnings and downstream IndexError --- pysat/tests/test_instrument.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py index a8cda3516..0ed962333 100644 --- a/pysat/tests/test_instrument.py +++ b/pysat/tests/test_instrument.py @@ -467,8 +467,14 @@ def test_creating_empty_instrument_object(self): def test_load_empty_instrument_no_files_error(self): """Ensure error loading Instrument with no files.""" - estr = 'No files found for Instrument' - testing.eval_bad_input(self.empty_inst.load, OSError, estr) + # Trying to load when there are no files produces multiple warnings + # and one Error. + with warnings.catch_warnings(record=True) as self.war: + testing.eval_bad_input(self.empty_inst.load, IndexError, + 'index 0 is out of bounds') + estr = ['No files found for Instrument', 'IndexError will not be'] + testing.eval_warnings(self.war, estr, warn_type=[UserWarning, + DeprecationWarning]) return def test_empty_repr_eval(self): From 2a8c2e673e606db89c1dcdc494325547fb335d14 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Tue, 27 Feb 2024 12:29:02 -0600 Subject: [PATCH 356/365] TST: Updated test_warnings to accept multiple warning types --- pysat/utils/testing.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pysat/utils/testing.py b/pysat/utils/testing.py index ff363aa93..492b04573 100644 --- a/pysat/utils/testing.py +++ b/pysat/utils/testing.py @@ -9,6 +9,7 @@ """Utilities to perform common evaluations.""" import numpy as np +import pysat.utils def assert_list_contains(small_list, big_list, test_nan=False, test_case=True): @@ -174,7 +175,7 @@ def eval_warnings(warns, check_msgs, warn_type=DeprecationWarning): check_msgs : list List of strings containing the expected warning messages warn_type : type - Type for the warning messages (default=DeprecationWarning) + Type or list-like for the warning messages (default=DeprecationWarning) Raises ------ @@ -183,19 +184,38 @@ def eval_warnings(warns, check_msgs, warn_type=DeprecationWarning): """ + # Ensure inputs are list-like + warn_types = pysat.utils.listify(warn_type) + check_msgs = pysat.utils.listify(check_msgs) + # Initialize the output found_msgs = [False for msg in check_msgs] + # If only one warning type provided then expand to match + # number of messages + simple_out = False + if len(warn_types) == 1: + warn_types = warn_types*len(check_msgs) + simple_out = True + # Test the warning messages, ensuring each attribute is present - for iwar in warns: + for iwar, iwartype in zip(warns, warn_types): for i, msg in enumerate(check_msgs): if str(iwar.message).find(msg) >= 0: - assert iwar.category == warn_type, \ + assert iwar.category == iwartype, \ "bad warning type for message: {:}".format(msg) found_msgs[i] = True + # If all warnings are of the same kind, we don't need to repeat the + # same type in the output string. + if simple_out: + warn_repr_str = repr(warn_type) + else: + not_found_msgs = [not msg for msg in found_msgs] + warn_repr_str = repr((np.array(warn_types)[not_found_msgs])) + assert np.all(found_msgs), "did not find {:d} expected {:}".format( - len(found_msgs) - np.sum(found_msgs), repr(warn_type)) + len(found_msgs) - np.sum(found_msgs), warn_repr_str) return From 034347ba530d27f7e0b65445d71f759aa5696034 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Tue, 27 Feb 2024 12:29:23 -0600 Subject: [PATCH 357/365] TST: Updated testing for expanded eval_warnings functionality --- pysat/tests/test_utils_testing.py | 100 +++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 10 deletions(-) diff --git a/pysat/tests/test_utils_testing.py b/pysat/tests/test_utils_testing.py index 5f7ef3e7c..8db5485e9 100644 --- a/pysat/tests/test_utils_testing.py +++ b/pysat/tests/test_utils_testing.py @@ -157,50 +157,130 @@ def test_nan_equal_bad(self, val1, val2): UserWarning, DeprecationWarning, SyntaxWarning, RuntimeWarning, FutureWarning, PendingDeprecationWarning, ImportWarning, UnicodeWarning, BytesWarning, ResourceWarning]) - def test_good_eval_warnings(self, warn_type): - """Test warning evaluation function success. + @pytest.mark.parametrize("second_type", [ + None, UserWarning, DeprecationWarning, SyntaxWarning, RuntimeWarning, + FutureWarning, PendingDeprecationWarning, ImportWarning, + UnicodeWarning, BytesWarning, ResourceWarning]) + def test_good_eval_warnings(self, warn_type, second_type): + """Test warning evaluation function success including multiple types. Parameters ---------- warn_type : Warning Warning class to be raised - + second_type : Warning or None + Optional warning class to be raised """ - warn_msg = 'test warning' + if second_type is not None: + warn_msgs = ['test warning1', 'test_warning2'] + warn_types = [warn_type, second_type] + else: + # Only test a single msg/type + warn_msgs = ['test warning'] + warn_types = [warn_type] # Raise the desired warning with warnings.catch_warnings(record=True) as war: - warnings.warn(warn_msg, warn_type) + for warn_msg, loop_warn_type in zip(warn_msgs, warn_types): + warnings.warn(warn_msg, loop_warn_type) # Evaluate the warning output - testing.eval_warnings(war, [warn_msg], warn_type) + testing.eval_warnings(war, warn_msgs, warn_types) return @pytest.mark.parametrize("warn_type", [ UserWarning, DeprecationWarning, SyntaxWarning, RuntimeWarning, FutureWarning, PendingDeprecationWarning, ImportWarning, UnicodeWarning, BytesWarning, ResourceWarning]) - def test_eval_warnings_bad_type(self, warn_type): + @pytest.mark.parametrize("second_type", [ + None, UserWarning, DeprecationWarning, SyntaxWarning, RuntimeWarning, + FutureWarning, PendingDeprecationWarning, ImportWarning, + UnicodeWarning, BytesWarning, ResourceWarning]) + def test_eval_warnings_bad_types(self, warn_type, second_type): """Test warning evaluation function failure for mismatched type. Parameters ---------- warn_type : Warning Warning class to be raised + second_type : Warning or None + Optional warning class to be raised """ warn_msg = 'test warning' bad_type = UserWarning if warn_type != UserWarning else BytesWarning + sbad_type = UserWarning if second_type != UserWarning else BytesWarning + + if second_type is not None: + warn_msgs = [warn_msg, 'test_warning2'] + warn_types = [warn_type, second_type] + bad_types = [bad_type, sbad_type] + else: + # Only test a single msg/type + warn_msgs = [warn_msg] + warn_types = [warn_type] + bad_types = [bad_type] # Raise the desired warning with warnings.catch_warnings(record=True) as war: - warnings.warn(warn_msg, warn_type) + for loop_warn_msg, loop_warn_type in zip(warn_msgs, warn_types): + warnings.warn(loop_warn_msg, loop_warn_type) + + # Catch and evaluate the expected error + with pytest.raises(AssertionError) as aerr: + testing.eval_warnings(war, warn_msgs, bad_types) + + assert str(aerr).find('bad warning type for message:') >= 0 + + return + + @pytest.mark.parametrize("warn_type", [ + UserWarning, DeprecationWarning, SyntaxWarning, RuntimeWarning, + FutureWarning, PendingDeprecationWarning, ImportWarning, + UnicodeWarning, BytesWarning, ResourceWarning]) + @pytest.mark.parametrize("second_type", [ + None, UserWarning, DeprecationWarning, SyntaxWarning, RuntimeWarning, + FutureWarning, PendingDeprecationWarning, ImportWarning, + UnicodeWarning, BytesWarning, ResourceWarning]) + def test_eval_warnings_bad_msgs(self, warn_type, second_type): + """Test warning evaluation function failure for mismatched message. + + Parameters + ---------- + warn_type : Warning + Warning class to be raised + second_type : Warning or None + Optional warning class to be raised + + """ + warn_msg = 'test warning' + bad_msg = 'not correct' + + if second_type is not None: + warn_msgs = [warn_msg, 'test_warning2'] + warn_types = [warn_type, second_type] + bad_msgs = [bad_msg, 'not_correct2'] + else: + # Only test a single msg/type + warn_msgs = [warn_msg] + warn_types = [warn_type] + bad_msgs = [bad_msg] + + # Raise the desired warning + with warnings.catch_warnings(record=True) as war: + for loop_warn_msg, loop_warn_type in zip(warn_msgs, warn_types): + warnings.warn(loop_warn_msg, loop_warn_type) # Catch and evaluate the expected error with pytest.raises(AssertionError) as aerr: - testing.eval_warnings(war, [warn_msg], bad_type) + testing.eval_warnings(war, bad_msgs, warn_types) + + assert str(aerr).find('did not find') >= 0 + + # Check for warning types in message + for loop_warn_type in warn_types: + assert str(aerr).find(repr(loop_warn_type)) >= 0 - assert str(aerr).find('bad warning type for message') >= 0 return @pytest.mark.parametrize("warn_type", [ From 0f04b6c27762122e3ac42a478ea25d02de8c2ed8 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Tue, 27 Feb 2024 13:06:48 -0600 Subject: [PATCH 358/365] BUG: Warnings don't have to be in user provided order --- pysat/utils/testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysat/utils/testing.py b/pysat/utils/testing.py index 492b04573..f6cdce6f3 100644 --- a/pysat/utils/testing.py +++ b/pysat/utils/testing.py @@ -199,8 +199,8 @@ def eval_warnings(warns, check_msgs, warn_type=DeprecationWarning): simple_out = True # Test the warning messages, ensuring each attribute is present - for iwar, iwartype in zip(warns, warn_types): - for i, msg in enumerate(check_msgs): + for iwar in warns: + for i, (msg, iwartype) in enumerate(zip(check_msgs, warn_types)): if str(iwar.message).find(msg) >= 0: assert iwar.category == iwartype, \ "bad warning type for message: {:}".format(msg) From 4372a83e1809979da3d3a8582c75083d34bae844 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Tue, 27 Feb 2024 13:16:13 -0600 Subject: [PATCH 359/365] DOC: Forgot to update the stackoverflow link --- pysat/utils/time.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pysat/utils/time.py b/pysat/utils/time.py index 298b29b82..b41529544 100644 --- a/pysat/utils/time.py +++ b/pysat/utils/time.py @@ -221,13 +221,10 @@ def freq_to_res(freq): -------- pds.offsets.DateOffset - References - ---------- - Separating alpha and numeric portions of strings, as described in: - https://stackoverflow.com/a/12409995 - """ - # Separate the alpha and numeric portions of the string + # Separate the alpha and numeric portions of the string as described in: + # https://stackoverflow.com/a/12409995 + regex = re.compile(r'(\d+|\s+)') out_str = [sval for sval in regex.split(freq) if len(sval) > 0] From 46484f3d145ef343ea8a173589f43fab4aedeb8b Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Tue, 27 Feb 2024 14:55:00 -0600 Subject: [PATCH 360/365] STY: flake8 --- pysat/utils/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysat/utils/testing.py b/pysat/utils/testing.py index f6cdce6f3..9367bc891 100644 --- a/pysat/utils/testing.py +++ b/pysat/utils/testing.py @@ -195,7 +195,7 @@ def eval_warnings(warns, check_msgs, warn_type=DeprecationWarning): # number of messages simple_out = False if len(warn_types) == 1: - warn_types = warn_types*len(check_msgs) + warn_types = warn_types * len(check_msgs) simple_out = True # Test the warning messages, ensuring each attribute is present From da54d063f99ca61216f807b662f82e4712d056d6 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Wed, 28 Feb 2024 16:11:18 -0600 Subject: [PATCH 361/365] STY: Updated changelog to reflect a warning is raised, not error. --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6047da24f..40a50f323 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,9 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added ability to set Meta data using `meta['data_var', 'label'] = value` structure. * Added test for loading multiple days of data. - * Added user friendly error message when trying to load data when there are - no files to load. + * Added user-friendly warning when trying to load data when there are + no files at all to load. Situation currently raises an IndexError. + * Expanded `eval_warnings` to support testing against multiple warning types. * Bug Fix * Fixed `utils.files.parse_fixed_width_filenames` output for empty file list * Updated the parsing functions in `utils.files` to consider type specifiers From 8e2aabc47d2cb6b66e597bfeb0278b82db783370 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Wed, 28 Feb 2024 16:14:42 -0600 Subject: [PATCH 362/365] STY: Added # TODO per review --- pysat/_instrument.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysat/_instrument.py b/pysat/_instrument.py index e065b3bcf..812d6f769 100644 --- a/pysat/_instrument.py +++ b/pysat/_instrument.py @@ -2967,9 +2967,10 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None, # Provide user friendly error if there is no data if len(self.files.files) == 0: + # TODO(#1182) - Update with pysat 3.3.0+ per directions below # In pysat 3.3, modify this section to leave function early # to prevent a downstream IndexError. Remove Deprecation portion - # of message above and leave as a UserWarning. + # of message below and leave as a UserWarning. estr = ''.join(('No files found for Instrument. If files are ', 'expected, please confirm that data is present ', 'on the system and that ', From ec7b0c410c89020b03d8e89d80e2bd9bd51a9025 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 11 Mar 2024 11:17:38 -0400 Subject: [PATCH 363/365] Apply suggestions from code review Co-authored-by: Russell Stoneback --- pysat/tests/test_constellation.py | 2 +- pysat/tests/test_instruments.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py index b7dfe552b..42712da1b 100644 --- a/pysat/tests/test_constellation.py +++ b/pysat/tests/test_constellation.py @@ -619,7 +619,7 @@ def test_delitem_one_inst(self, method): # Load the Constellation data self.const.load(date=self.ref_time) - # Delete the UTS data from all instruments + # Delete the UTS data from the pysat testing instrument dvar = "uts_pysat_testing" if method == "del": del self.const[dvar] diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py index 6561bf3c0..70a4edcf6 100644 --- a/pysat/tests/test_instruments.py +++ b/pysat/tests/test_instruments.py @@ -185,6 +185,7 @@ def test_clean_with_warnings(self, clean_level, change, warn_type, # Construct the expected warnings if warn_type == 'mult': # Note that we cannot test errors along with other warnings + # TODO(#1184) test for both warnings and errors inst_dict['inst_module']._clean_warn = { inst_dict['inst_id']: {inst_dict['tag']: {clean_level: [ ('warning', warn_level['warning'], warn_msg, final_level), From 8b19b270710a74d85db8e07d16962bae42a79704 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:21:53 -0400 Subject: [PATCH 364/365] Update .zenodo.json --- .zenodo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zenodo.json b/.zenodo.json index 8a69eee50..0fabed72f 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -17,7 +17,7 @@ ], "creators": [ { - "affiliation": "Stoneris LLC", + "affiliation": "Cosmic Studio", "name": "Stoneback, Russell", "orcid": "0000-0001-7216-4336" }, From 474ccf95d09e355068efbf832a14c5487ffaf14e Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:40:21 -0400 Subject: [PATCH 365/365] MAINT: update changelog, zenodo --- .zenodo.json | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index 0fabed72f..732c82808 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -67,7 +67,7 @@ "name": "Leite, Silvio", "orcid": "0000-0003-1707-7963" }, - { + { "affiliation": "NASA NPP", "name": "Esman, Teresa", "orcid": "0000-0003-0382-6281" diff --git a/CHANGELOG.md b/CHANGELOG.md index 40a50f323..7dc0ef87d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -[3.2.0] - 2024-02-xx +[3.2.0] - 2024-03-27 -------------------- * New Features * Added tests for warnings, logging messages, and errors in the Instrument