From 1fb947f222524a536d0841a4aaf5649247e8aa53 Mon Sep 17 00:00:00 2001
From: Tracy Esman <21086818+t-esman@users.noreply.github.com>
Date: Fri, 2 Jun 2023 14:06:43 -0400
Subject: [PATCH 001/365] Update installation.rst
---
docs/installation.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/installation.rst b/docs/installation.rst
index 423522513..afc3e029b 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -6,7 +6,7 @@ Installation
Python and associated packages for science are freely available. Convenient
science python package setups are available from ``_,
-`Anaconda `_, and other locations
+`Anaconda `_, and other locations
(some platform specific). Anaconda also includes a developer environment
that works well with pysat. Core science packages such as numpy, scipy,
matplotlib, pandas and many others may also be installed directly via the
From d80a8eb6797c4fc6b7edeaec1356628e2163c829 Mon Sep 17 00:00:00 2001
From: Tracy Esman <21086818+t-esman@users.noreply.github.com>
Date: Fri, 2 Jun 2023 14:19:37 -0400
Subject: [PATCH 002/365] Update tutorial_load.rst
---
docs/tutorial/tutorial_load.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/tutorial/tutorial_load.rst b/docs/tutorial/tutorial_load.rst
index dfd13072f..76478a28d 100644
--- a/docs/tutorial/tutorial_load.rst
+++ b/docs/tutorial/tutorial_load.rst
@@ -67,7 +67,7 @@ pysat supports the use of two different data structures. You can either use a
pandas
`DataFrame `_,
a highly capable class with labeled rows and columns, or an xarray
-`DataSet `_
+`DataSet `_
for data sets with more dimensions. The type of data class is flagged using
the attribute :py:attr:`pysat.Instrument.pandas_format`. This is set to
``True`` if a :py:class:`pandas.DataFrame` is returned by the corresponding
From 2a919f08c7b4db423b46342ae3026970aa9e1dbd Mon Sep 17 00:00:00 2001
From: Tracy Esman <21086818+t-esman@users.noreply.github.com>
Date: Fri, 2 Jun 2023 14:21:03 -0400
Subject: [PATCH 003/365] Update CHANGELOG.md
---
CHANGELOG.md | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0d34d155c..90588ccdb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,11 @@ Change Log
All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](https://semver.org/).
+[3.1.X] - 2023-xx-xx
+--------------------
+* Maintenance
+ * Update link redirects in docs.
+
[3.1.0] - 2023-05-31
--------------------
* New Features
From 73b86d41c77926d1d0463945b900b47957969c35 Mon Sep 17 00:00:00 2001
From: Tracy Esman <21086818+t-esman@users.noreply.github.com>
Date: Fri, 2 Jun 2023 14:22:58 -0400
Subject: [PATCH 004/365] Update .zenodo.json
---
.zenodo.json | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/.zenodo.json b/.zenodo.json
index bdc218349..8a69eee50 100644
--- a/.zenodo.json
+++ b/.zenodo.json
@@ -66,6 +66,11 @@
"affiliation": "@olist",
"name": "Leite, Silvio",
"orcid": "0000-0003-1707-7963"
+ },
+ {
+ "affiliation": "NASA NPP",
+ "name": "Esman, Teresa",
+ "orcid": "0000-0003-0382-6281"
}
]
}
From 066a0e9b94f27fa2caa48e6f9ee7ab84b5b085ef Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Wed, 5 Jul 2023 16:23:22 -0400
Subject: [PATCH 005/365] TST: added a test for warnings in `clean`
Added a test for logging messages, warnings, and errors that may occur in Instrument `clean` methods. Also tests to see if the clean level has been reset correctly.
---
pysat/tests/classes/cls_instrument_library.py | 136 +++++++++++++++---
1 file changed, 120 insertions(+), 16 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index bb047209e..0e32fbda0 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -32,6 +32,7 @@ class TestInstruments(InstLibTests):
import datetime as dt
from importlib import import_module
+import logging
import sys
import tempfile
import warnings
@@ -40,8 +41,7 @@ class TestInstruments(InstLibTests):
import pysat
from pysat.utils import generate_instrument_list
-from pysat.utils.testing import assert_hasattr
-from pysat.utils.testing import assert_isinstance
+from pysat.utils import testing
def initialize_test_inst_and_date(inst_dict):
@@ -66,9 +66,8 @@ def initialize_test_inst_and_date(inst_dict):
test_inst = pysat.Instrument(inst_module=inst_dict['inst_module'],
tag=inst_dict['tag'],
inst_id=inst_dict['inst_id'],
- temporary_file_list=True,
- update_files=True, use_header=True,
- **kwargs)
+ temporary_file_list=True, update_files=True,
+ use_header=True, **kwargs)
test_dates = inst_dict['inst_module']._test_dates
date = test_dates[inst_dict['inst_id']][inst_dict['tag']]
return test_inst, date
@@ -218,10 +217,10 @@ def test_modules_standard(self, inst_name):
# Check for presence of basic instrument module attributes
for mattr in self.module_attrs:
- assert_hasattr(module, mattr)
+ testing.assert_hasattr(module, mattr)
if mattr in self.attr_types.keys():
- assert_isinstance(getattr(module, mattr),
- self.attr_types[mattr])
+ testing.assert_isinstance(getattr(module, mattr),
+ self.attr_types[mattr])
# Check for presence of required instrument attributes
for inst_id in module.inst_ids.keys():
@@ -230,7 +229,7 @@ def test_modules_standard(self, inst_name):
inst_id=inst_id, use_header=True)
# Test to see that the class parameters were passed in
- assert_isinstance(inst, pysat.Instrument)
+ testing.assert_isinstance(inst, pysat.Instrument)
assert inst.platform == module.platform
assert inst.name == module.name
assert inst.inst_id == inst_id
@@ -239,10 +238,10 @@ def test_modules_standard(self, inst_name):
# Test the required class attributes
for iattr in self.inst_attrs:
- assert_hasattr(inst, iattr)
+ testing.assert_hasattr(inst, iattr)
if iattr in self.attr_types:
- assert_isinstance(getattr(inst, iattr),
- self.attr_types[iattr])
+ testing.assert_isinstance(getattr(inst, iattr),
+ self.attr_types[iattr])
return
@pytest.mark.all_inst
@@ -287,7 +286,7 @@ def test_instrument_test_dates(self, inst_name):
info = module._test_dates
for inst_id in info.keys():
for tag in info[inst_id].keys():
- assert_isinstance(info[inst_id][tag], dt.datetime)
+ testing.assert_isinstance(info[inst_id][tag], dt.datetime)
return
@pytest.mark.first
@@ -307,8 +306,10 @@ def test_download(self, inst_dict):
test_inst, date = initialize_test_inst_and_date(inst_dict)
# Check for username.
- dl_dict = inst_dict['user_info'] if 'user_info' in \
- inst_dict.keys() else {}
+ if 'user_info' in inst_dict.keys():
+ dl_dict = inst_dict['user_info']
+ else:
+ dl_dict = {}
test_inst.download(date, date, **dl_dict)
assert len(test_inst.files.files) > 0
return
@@ -352,7 +353,7 @@ def test_load(self, clean_level, inst_dict):
assert UserWarning in categories
else:
# If error message does not match, raise error anyway
- raise(verr)
+ raise ValueError(verr)
# Make sure fake data is cleared
assert target not in test_inst.data
@@ -366,6 +367,109 @@ def test_load(self, clean_level, inst_dict):
return
+ @pytest.mark.second
+ # Need to maintain download mark for backwards compatibility.
+ # Can remove once pysat 3.1.0 is released and libraries are updated.
+ @pytest.mark.load_options
+ @pytest.mark.download
+ @pytest.mark.parametrize("clean_level", ['none', 'dirty', 'dusty', 'clean'])
+ def test_clean_warn(self, clean_level, inst_dict, caplog):
+ """Test that appropriate warnings and errors are raised when cleaning.
+
+ Parameters
+ ----------
+ clean_level : str
+ Cleanliness level for loaded instrument data.
+ inst_dict : dict
+ Dictionary containing info to instantiate a specific instrument.
+ Set automatically from instruments['download'] when
+ `initialize_test_package` is run.
+ **kwags : dict
+
+ """
+ if hasattr(inst_dict['inst_module'], '_clean_warn'):
+ clean_warn = inst_dict['inst_module']._clean_warn
+
+ if clean_level in clean_warn.keys():
+ # Only need to test if there are clean warnings for this level
+ test_inst, date = initialize_test_inst_and_date(inst_dict)
+ (clean_method, clean_method_level, clean_method_msg,
+ final_level) = clean_warn[clean_level]
+
+ if len(test_inst.files.files) > 0:
+ # Set the clean level
+ test_inst.clean_level = clean_level
+ target = 'Fake Data to be cleared'
+ test_inst.data = [target]
+
+ try:
+ if clean_method == 'logger':
+ # A logging message is expected
+ with caplog.at_level(
+ getattr(logging, clean_method_level),
+ logger='pysat'):
+ test_inst.load(date=date, use_header=True)
+
+ # Test the returned message
+ out_msg = caplog.text
+ assert out_msg.find(clean_method_msg) >= 0
+ elif clean_method == 'warning':
+ # A warning message is expected
+ with warnings.catch_warnings(record=True) as war:
+ test_inst.load(date=date, use_header=True)
+
+ # Test the warning output
+ testing.eval_warnings(war, [clean_method_msg],
+ clean_method_level)
+ elif clean_method == 'error':
+ # An error message is expected, evaluate error and
+ # error message
+ try:
+ testing.eval_bad_input(
+ test_inst.load, clean_method_level,
+ clean_method_msg,
+ input_kwargs={'date': date,
+ 'use_header': True})
+ except AssertionError as aerr:
+ if str(aerr).find(
+ 'Loaded data is not unique') >= 0:
+ pytest.skip('Cannot test multiple errors')
+ else:
+ raise AssertionError(aerr)
+ else:
+ raise AttributeError(
+ 'unknown type of warning: {:}'.format(
+ clean_method))
+ except ValueError as verr:
+ # Check if instrument is failing due to strict time flag
+ if str(verr).find('Loaded data') > 0:
+ test_inst.strict_time_flag = False
+ with warnings.catch_warnings(record=True) as war:
+ test_inst.load(date=date, use_header=True)
+ assert len(war) >= 1
+ categories = [war[j].category
+ for j in range(0, len(war))]
+ assert UserWarning in categories
+ else:
+ # If error message does not match, raise error
+ raise ValueError(verr)
+
+ # Test to see if the clean flag has the expected value
+ # afterwards
+ assert test_inst.clean_level == final_level, \
+ "Clean level should now be {:s}, not {:s}".format(
+ final_level, test_inst.clean_level)
+
+ # Make sure fake data is cleared
+ assert target not in test_inst.data
+
+ # If cleaning not used, something should be in the file. Not
+ # used for clean levels since cleaning may remove all data
+ if clean_level == "none":
+ assert not test_inst.empty
+
+ return
+
@pytest.mark.download
def test_remote_file_list(self, inst_dict):
"""Test if optional list_remote_files routine exists and is callable.
From 71f6d9d3f0f09a3bd9d9369f17a5ee0d902d6321 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Wed, 5 Jul 2023 16:24:47 -0400
Subject: [PATCH 006/365] TST: expanded test Instrument cleaning
Added level resetting, logging messages, warnings, and errors as potential outputs to the clean method for test instruments. Also added a unit test for each of these elements.
---
pysat/instruments/methods/testing.py | 21 ++++++++++-
pysat/tests/test_instruments.py | 54 ++++++++++++++++++++++++++++
2 files changed, 74 insertions(+), 1 deletion(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index bf9cbeef1..95f143425 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -56,12 +56,31 @@ def clean(self, test_clean_kwarg=None):
Parameters
----------
test_clean_kwarg : any
- Testing keyword (default=None)
+ Testing keyword. If these keywords contain 'logger', 'warning', or
+ 'error', the message entered as the value to that key will be returned
+ as a logging.WARNING, UserWarning, or ValueError, respectively. If the
+ 'change' kwarg is set, the clean level will be changed to the specified
+ value. (default=None)
"""
self.test_clean_kwarg = test_clean_kwarg
+ if test_clean_kwarg is None:
+ test_clean_kwarg = {}
+
+ if 'change' in test_clean_kwarg.keys():
+ self.clean_level = test_clean_kwarg['change']
+
+ if 'logger' in test_clean_kwarg.keys():
+ pysat.logger.warning(test_clean_kwarg['logger'])
+
+ if 'warning' in test_clean_kwarg.keys():
+ warnings.warn(test_clean_kwarg['warning'], UserWarning)
+
+ if 'error' in test_clean_kwarg.keys():
+ raise ValueError(test_clean_kwarg['error'])
+
return
diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py
index 605270288..be8f27edf 100644
--- a/pysat/tests/test_instruments.py
+++ b/pysat/tests/test_instruments.py
@@ -146,6 +146,60 @@ def test_inst_max_latitude(self, inst_dict):
return
+ @pytest.mark.second
+ @pytest.mark.parametrize("clean_level", ['clean', 'dusty', 'dirty'])
+ @pytest.mark.parametrize("change", [True, False])
+ @pytest.mark.parametrize('warn_type', ['logger', 'warning', 'error'])
+ @pytest.mark.parametrize("inst_dict", instruments['download'])
+ def test_clean_with_warnings(self, clean_level, change, warn_type,
+ inst_dict, caplog):
+ """Run `test_clean_warn` with different warning behaviours.
+
+ Parameters
+ ----------
+ clean_level : str
+ Cleanliness level for loaded instrument data; must run the clean
+ routine (not include 'none').
+ change : bool
+ Specify whether or not clean level should change.
+ warn_type : str
+ Desired type of warning or error to be raised.
+ inst_dict : dict
+ One of the dictionaries returned from
+ `InstLibTests.initialize_test_package` with instruments to test.
+
+ """
+ # Set the default values
+ warn_level = {'logger': 'WARN', 'warning': UserWarning,
+ 'error': ValueError}
+ warn_msg = 'Default warning message'
+ if change:
+ final_level = 'none' if clean_level == 'clean' else 'clean'
+ else:
+ final_level = clean_level
+
+ # Construct the expected warnings
+ inst_dict['inst_module']._clean_warn = {
+ clean_level: (warn_type, warn_level[warn_type], warn_msg,
+ final_level)}
+
+ # Set the additional Instrument kwargs
+ if 'kwargs' in inst_dict.keys():
+ if 'test_clean_kwarg' in inst_dict['kwargs']:
+ inst_dict['kwargs']['test_clean_kwarg']['change'] = final_level
+ inst_dict['kwargs']['test_clean_kwarg'][warn_type] = warn_msg
+ else:
+ inst_dict['kwargs']['test_clean_kwarg'] = {
+ 'change': final_level, warn_type: warn_msg}
+ else:
+ inst_dict['kwargs'] = {'test_clean_kwarg': {'change': final_level,
+ warn_type: warn_msg}}
+
+ # Run the test
+ self.test_clean_warn(clean_level, inst_dict, caplog)
+
+ return
+
class TestDeprecation(object):
"""Unit test for deprecation warnings."""
From 6227fc92a0b57ce8da16b010498d0ecf5bf699dd Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Wed, 5 Jul 2023 16:25:22 -0400
Subject: [PATCH 007/365] DOC: updated new instrument instructions
Updated the new instrument instructions for the clean method and `_clean_warn` test variable.
---
docs/new_instrument.rst | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index 06f39603a..68b15a909 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -619,6 +619,13 @@ Cleans instrument for levels supplied in inst.clean_level.
``self`` is a :py:class:`pysat.Instrument` object. :py:func:`clean` should
modify ``self`` in-place as needed; equivalent to a custom routine.
+:py:func:`clean` is allowed to raise logger messages, warnings, and errors. If
+the routine does this, be sure to test them by assigning the necessary
+information to the :py:attr:`_clean_warn` attribute, described in Section
+rst_new_inst-test_. :py:func:`clean` may also re-assign the cleaning level if
+appropriate. If you do this, be sure to raise a logging warning, so that users
+are aware that this change is happening and why the clean level they requested
+is not appropriate.
list_remote_files
^^^^^^^^^^^^^^^^^
@@ -731,7 +738,25 @@ combinations), and runs the tests using pytestmark. By default,
routine, and will run an end-to-end test. If this is not the case, see the next
section.
+Another important test is for warnings and the re-setting of clean levels that
+may come up when cleaning data. These may be specified using the
+:py:attr:`_clean_warn` attribute, which should point to a dictionary that has a
+tuple of four elements as the value. The first element should be 'logger',
+'warning', or 'error', specifying the method through which the warning is being
+reported. The second element specifies either the logging level (as a string)
+or the warning/error type (e.g., ``ValueError``). The third element provides the
+warning message as a string and the final element provides the expected clean
+level after running the clean routine.
+.. code:: python
+
+ # ------------------------------------------
+ # Instrument test attributes
+
+ _clean_warn = {'dusty': ('logger', 'WARN', "I am a warning!", 'clean')}
+
+
+
.. _rst_test-special:
Special Test Configurations
From 665ab427c4fe6aeac055f6473e986f9359fd714d Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Wed, 5 Jul 2023 16:25:45 -0400
Subject: [PATCH 008/365] DOC: updated changelog
Updated the changelog with a description of this pull request.
---
CHANGELOG.md | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 90588ccdb..8274e2f15 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,8 +5,11 @@ This project adheres to [Semantic Versioning](https://semver.org/).
[3.1.X] - 2023-xx-xx
--------------------
+* New Features
+ * Added tests for warnings, logging messages, and errors in the Instrument
+ clean method.
* Maintenance
- * Update link redirects in docs.
+ * Update link redirects in docs.
[3.1.0] - 2023-05-31
--------------------
From b84e0052612c3f9661f0ce4b5dd8a5284cd8e33f Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Wed, 5 Jul 2023 16:37:13 -0400
Subject: [PATCH 009/365] STY: updated reference style
Updated the reference style in the docs.
---
docs/new_instrument.rst | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index 68b15a909..b95bc0306 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -621,11 +621,11 @@ Cleans instrument for levels supplied in inst.clean_level.
modify ``self`` in-place as needed; equivalent to a custom routine.
:py:func:`clean` is allowed to raise logger messages, warnings, and errors. If
the routine does this, be sure to test them by assigning the necessary
-information to the :py:attr:`_clean_warn` attribute, described in Section
-rst_new_inst-test_. :py:func:`clean` may also re-assign the cleaning level if
-appropriate. If you do this, be sure to raise a logging warning, so that users
-are aware that this change is happening and why the clean level they requested
-is not appropriate.
+information to the :py:attr:`_clean_warn` attribute, described in
+:ref:`Testing Support `_. :py:func:`clean` may also
+re-assign the cleaning level if appropriate. If you do this, be sure to raise a
+logging warning, so that users are aware that this change is happening and why
+the clean level they requested is not appropriate.
list_remote_files
^^^^^^^^^^^^^^^^^
From 67c1d8c15cbae57b845d5f3009e79971f412e264 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Wed, 5 Jul 2023 16:40:22 -0400
Subject: [PATCH 010/365] BUG: fixing reference
Fixing bug in reference format.
---
docs/new_instrument.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index b95bc0306..655a0ad06 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -622,7 +622,7 @@ modify ``self`` in-place as needed; equivalent to a custom routine.
:py:func:`clean` is allowed to raise logger messages, warnings, and errors. If
the routine does this, be sure to test them by assigning the necessary
information to the :py:attr:`_clean_warn` attribute, described in
-:ref:`Testing Support `_. :py:func:`clean` may also
+:ref:`Testing Support `. :py:func:`clean` may also
re-assign the cleaning level if appropriate. If you do this, be sure to raise a
logging warning, so that users are aware that this change is happening and why
the clean level they requested is not appropriate.
From ac2eb50f45ee26af70c1aab81ecfe8bc75d14426 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 6 Jul 2023 10:16:44 -0400
Subject: [PATCH 011/365] BUG: fixed dict evalutation
Fixed logic in clean method, allowing key search only if input is a dict.
---
pysat/instruments/methods/testing.py | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index 95f143425..fe66275bf 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -66,20 +66,18 @@ def clean(self, test_clean_kwarg=None):
self.test_clean_kwarg = test_clean_kwarg
- if test_clean_kwarg is None:
- test_clean_kwarg = {}
+ if isinstance(test_clean_kwarg, dict):
+ if 'change' in test_clean_kwarg.keys():
+ self.clean_level = test_clean_kwarg['change']
- if 'change' in test_clean_kwarg.keys():
- self.clean_level = test_clean_kwarg['change']
+ if 'logger' in test_clean_kwarg.keys():
+ pysat.logger.warning(test_clean_kwarg['logger'])
- if 'logger' in test_clean_kwarg.keys():
- pysat.logger.warning(test_clean_kwarg['logger'])
+ if 'warning' in test_clean_kwarg.keys():
+ warnings.warn(test_clean_kwarg['warning'], UserWarning)
- if 'warning' in test_clean_kwarg.keys():
- warnings.warn(test_clean_kwarg['warning'], UserWarning)
-
- if 'error' in test_clean_kwarg.keys():
- raise ValueError(test_clean_kwarg['error'])
+ if 'error' in test_clean_kwarg.keys():
+ raise ValueError(test_clean_kwarg['error'])
return
From 2b5ae2ef91b359cb2271004d4300c0cfced32b1a Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 6 Jul 2023 10:34:40 -0400
Subject: [PATCH 012/365] DOC: Update docs/new_instrument.rst
Updated example for `_clean_warn` in docs to include inst_id and tag.
---
docs/new_instrument.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index 655a0ad06..71234c454 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -753,7 +753,7 @@ level after running the clean routine.
# ------------------------------------------
# Instrument test attributes
- _clean_warn = {'dusty': ('logger', 'WARN', "I am a warning!", 'clean')}
+ _clean_warn = {inst_id: {tag: {'dusty': ('logger', 'WARN', "I am a warning!", 'clean')} for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()}
From 56dadc7707b80e19e3a32e55788df3982940bc7f Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 6 Jul 2023 11:47:05 -0400
Subject: [PATCH 013/365] ENH: removed unreachable lines
Removed lines that cannot be reached, as the clean method doesn't run if the clean level is 'none'.
---
pysat/tests/classes/cls_instrument_library.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 0e32fbda0..e9947d383 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -463,11 +463,6 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Make sure fake data is cleared
assert target not in test_inst.data
- # If cleaning not used, something should be in the file. Not
- # used for clean levels since cleaning may remove all data
- if clean_level == "none":
- assert not test_inst.empty
-
return
@pytest.mark.download
From cd75aeed01cb39fa831dbd83341989ae2653dfff Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 6 Jul 2023 12:40:39 -0400
Subject: [PATCH 014/365] MAINT: updated `_clean_warn` format
Updated the `_clean_warn` format to accept a list of tuples.
---
docs/new_instrument.rst | 24 ++-
pysat/tests/classes/cls_instrument_library.py | 143 +++++++++---------
pysat/tests/test_instruments.py | 32 ++--
3 files changed, 113 insertions(+), 86 deletions(-)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index 71234c454..d88e0e386 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -732,7 +732,7 @@ will not be present in Input/Output operations.
The standardized :py:mod:`pysat` tests are available in
:py:mod:`pysat.tests.instrument_test_class`. The test collection in
test_instruments.py imports this class, collects a list of all available
-instruments (including potential :py:data:`tag`/:py:data:`inst_id`
+instruments (including potential :py:attr:`tag`/:py:attr:`inst_id`
combinations), and runs the tests using pytestmark. By default,
:py:mod:`pysat` assumes that your instrument has a fully functional download
routine, and will run an end-to-end test. If this is not the case, see the next
@@ -741,19 +741,27 @@ section.
Another important test is for warnings and the re-setting of clean levels that
may come up when cleaning data. These may be specified using the
:py:attr:`_clean_warn` attribute, which should point to a dictionary that has a
-tuple of four elements as the value. The first element should be 'logger',
-'warning', or 'error', specifying the method through which the warning is being
-reported. The second element specifies either the logging level (as a string)
-or the warning/error type (e.g., ``ValueError``). The third element provides the
-warning message as a string and the final element provides the expected clean
-level after running the clean routine.
+list with tuples of four elements as the value. The first tuple element should
+be 'logger', 'warning', or 'error', specifying the method through which the
+warning is being reported. The second tuple element specifies either the logging
+level (as a string) or the warning/error type (e.g., ``ValueError``). The third
+tuple element provides the warning message as a string and the final element
+provides the expected clean level after running the clean routine. The list
+allows multiple types of warning messages to be tested for a given
+:py:attr:`inst_id`, :py:attr:`tag`, and :py:attr:`clean_level` combination.
.. code:: python
# ------------------------------------------
# Instrument test attributes
- _clean_warn = {inst_id: {tag: {'dusty': ('logger', 'WARN', "I am a warning!", 'clean')} for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()}
+ _clean_warn = {inst_id: {tag: {'dusty': [
+ ('logger', 'WARN', "I am a warning!", 'clean'),
+ ('warning', UserWarning,
+ 'I am a serios warning!', 'dusty'),
+ ('error, ValueError, 'I am an error', 'dusty')]}
+ for tag in inst_ids[inst_id]}
+ for inst_id in inst_ids.keys()}
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index e9947d383..5bdf6faf7 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -393,75 +393,80 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
if clean_level in clean_warn.keys():
# Only need to test if there are clean warnings for this level
test_inst, date = initialize_test_inst_and_date(inst_dict)
- (clean_method, clean_method_level, clean_method_msg,
- final_level) = clean_warn[clean_level]
-
- if len(test_inst.files.files) > 0:
- # Set the clean level
- test_inst.clean_level = clean_level
- target = 'Fake Data to be cleared'
- test_inst.data = [target]
-
- try:
- if clean_method == 'logger':
- # A logging message is expected
- with caplog.at_level(
- getattr(logging, clean_method_level),
- logger='pysat'):
- test_inst.load(date=date, use_header=True)
-
- # Test the returned message
- out_msg = caplog.text
- assert out_msg.find(clean_method_msg) >= 0
- elif clean_method == 'warning':
- # A warning message is expected
- with warnings.catch_warnings(record=True) as war:
- test_inst.load(date=date, use_header=True)
-
- # Test the warning output
- testing.eval_warnings(war, [clean_method_msg],
- clean_method_level)
- elif clean_method == 'error':
- # An error message is expected, evaluate error and
- # error message
- try:
- testing.eval_bad_input(
- test_inst.load, clean_method_level,
- clean_method_msg,
- input_kwargs={'date': date,
- 'use_header': True})
- except AssertionError as aerr:
- if str(aerr).find(
- 'Loaded data is not unique') >= 0:
- pytest.skip('Cannot test multiple errors')
- else:
- raise AssertionError(aerr)
- else:
- raise AttributeError(
- 'unknown type of warning: {:}'.format(
- clean_method))
- except ValueError as verr:
- # Check if instrument is failing due to strict time flag
- if str(verr).find('Loaded data') > 0:
- test_inst.strict_time_flag = False
- with warnings.catch_warnings(record=True) as war:
- test_inst.load(date=date, use_header=True)
- assert len(war) >= 1
- categories = [war[j].category
- for j in range(0, len(war))]
- assert UserWarning in categories
- else:
- # If error message does not match, raise error
- raise ValueError(verr)
-
- # Test to see if the clean flag has the expected value
- # afterwards
- assert test_inst.clean_level == final_level, \
- "Clean level should now be {:s}, not {:s}".format(
- final_level, test_inst.clean_level)
-
- # Make sure fake data is cleared
- assert target not in test_inst.data
+ clean_warnings = clean_warn[clean_level]
+
+ for (clean_method, clean_method_level, clean_method_msg,
+ final_level) in clean_warnings:
+ if len(test_inst.files.files) > 0:
+ # Set the clean level
+ test_inst.clean_level = clean_level
+ target = 'Fake Data to be cleared'
+ test_inst.data = [target]
+
+ try:
+ if clean_method == 'logger':
+ # A logging message is expected
+ with caplog.at_level(
+ getattr(logging, clean_method_level),
+ logger='pysat'):
+ test_inst.load(date=date, use_header=True)
+
+ # Test the returned message
+ out_msg = caplog.text
+ assert out_msg.find(clean_method_msg) >= 0
+ elif clean_method == 'warning':
+ # A warning message is expected
+ with warnings.catch_warnings(
+ record=True) as war:
+ test_inst.load(date=date, use_header=True)
+
+ # Test the warning output
+ testing.eval_warnings(war, [clean_method_msg],
+ clean_method_level)
+ elif clean_method == 'error':
+ # An error message is expected, evaluate error
+ # and the error message
+ try:
+ testing.eval_bad_input(
+ test_inst.load, clean_method_level,
+ clean_method_msg,
+ input_kwargs={'date': date,
+ 'use_header': True})
+ except AssertionError as aerr:
+ if str(aerr).find(
+ 'Loaded data is not unique') >= 0:
+ pytest.skip(
+ 'Cannot test multiple errors')
+ else:
+ raise AssertionError(aerr)
+ else:
+ raise AttributeError(
+ 'unknown type of warning: {:}'.format(
+ clean_method))
+ except ValueError as verr:
+ # Check if instrument is failing due to strict time
+ # flag
+ if str(verr).find('Loaded data') > 0:
+ test_inst.strict_time_flag = False
+ with warnings.catch_warnings(
+ record=True) as war:
+ test_inst.load(date=date, use_header=True)
+ assert len(war) >= 1
+ categories = [war[j].category
+ for j in range(0, len(war))]
+ assert UserWarning in categories
+ else:
+ # If error message does not match, raise error
+ raise ValueError(verr)
+
+ # Test to see if the clean flag has the expected value
+ # afterwards
+ assert test_inst.clean_level == final_level, \
+ "Clean level should now be {:s}, not {:s}".format(
+ final_level, test_inst.clean_level)
+
+ # Make sure fake data is cleared
+ assert target not in test_inst.data
return
diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py
index be8f27edf..37d6ed8bf 100644
--- a/pysat/tests/test_instruments.py
+++ b/pysat/tests/test_instruments.py
@@ -149,7 +149,8 @@ def test_inst_max_latitude(self, inst_dict):
@pytest.mark.second
@pytest.mark.parametrize("clean_level", ['clean', 'dusty', 'dirty'])
@pytest.mark.parametrize("change", [True, False])
- @pytest.mark.parametrize('warn_type', ['logger', 'warning', 'error'])
+ @pytest.mark.parametrize('warn_type', ['logger', 'warning', 'error',
+ 'mult'])
@pytest.mark.parametrize("inst_dict", instruments['download'])
def test_clean_with_warnings(self, clean_level, change, warn_type,
inst_dict, caplog):
@@ -179,21 +180,34 @@ def test_clean_with_warnings(self, clean_level, change, warn_type,
final_level = clean_level
# Construct the expected warnings
- inst_dict['inst_module']._clean_warn = {
- clean_level: (warn_type, warn_level[warn_type], warn_msg,
- final_level)}
+ if warn_type == 'mult':
+ inst_dict['inst_module']._clean_warn = {
+ inst_dict['inst_id']: {inst_dict['tag']: {clean_level: [
+ ('logger', warn_level['logger'], warn_msg, final_level),
+ ('warning', warn_level['warning'], warn_msg, final_level),
+ ('error', warn_level['error'], warn_msg, final_level)]}}}
+ else:
+ inst_dict['inst_module']._clean_warn = {
+ inst_dict['inst_id']: {inst_dict['tag']: {clean_level: [
+ (warn_type, warn_level[warn_type], warn_msg,
+ final_level)]}}}
# Set the additional Instrument kwargs
if 'kwargs' in inst_dict.keys():
if 'test_clean_kwarg' in inst_dict['kwargs']:
inst_dict['kwargs']['test_clean_kwarg']['change'] = final_level
- inst_dict['kwargs']['test_clean_kwarg'][warn_type] = warn_msg
else:
- inst_dict['kwargs']['test_clean_kwarg'] = {
- 'change': final_level, warn_type: warn_msg}
+ inst_dict['kwargs']['test_clean_kwarg'] = {'change':
+ final_level}
+ else:
+ inst_dict['kwargs'] = {'test_clean_kwarg': {'change': final_level}}
+
+ if warn_type == 'mult':
+ inst_dict['kwargs']['test_clean_kwarg']['logger'] = warn_msg
+ inst_dict['kwargs']['test_clean_kwarg']['warning'] = warn_msg
+ inst_dict['kwargs']['test_clean_kwarg']['error'] = warn_msg
else:
- inst_dict['kwargs'] = {'test_clean_kwarg': {'change': final_level,
- warn_type: warn_msg}}
+ inst_dict['kwargs']['test_clean_kwarg'][warn_type] = warn_msg
# Run the test
self.test_clean_warn(clean_level, inst_dict, caplog)
From cf273b527902fc311e9b8df3babedb9230dfbb4f Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 6 Jul 2023 13:22:40 -0400
Subject: [PATCH 015/365] DOC: improved documentation flow
Moved the discussion of the new clean_warn test attribute to its own section and updated the reference.
---
docs/new_instrument.rst | 24 ++++++++++++++++++------
1 file changed, 18 insertions(+), 6 deletions(-)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index d88e0e386..1712ad9a3 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -622,7 +622,7 @@ modify ``self`` in-place as needed; equivalent to a custom routine.
:py:func:`clean` is allowed to raise logger messages, warnings, and errors. If
the routine does this, be sure to test them by assigning the necessary
information to the :py:attr:`_clean_warn` attribute, described in
-:ref:`Testing Support `. :py:func:`clean` may also
+:ref:`rst_test-clean`. :py:func:`clean` may also
re-assign the cleaning level if appropriate. If you do this, be sure to raise a
logging warning, so that users are aware that this change is happening and why
the clean level they requested is not appropriate.
@@ -738,6 +738,22 @@ combinations), and runs the tests using pytestmark. By default,
routine, and will run an end-to-end test. If this is not the case, see the next
section.
+
+.. _rst_test-special:
+
+Special Test Configurations
+---------------------------
+
+The following test attributes may or may not be necessary for your new
+:py:class:`~pysat._instrument.Instrument`. The descriptions should provide
+insight into when and how they should be used.
+
+
+.. _rst_test-clean:
+
+Warnings in the Clean method
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
Another important test is for warnings and the re-setting of clean levels that
may come up when cleaning data. These may be specified using the
:py:attr:`_clean_warn` attribute, which should point to a dictionary that has a
@@ -764,11 +780,7 @@ allows multiple types of warning messages to be tested for a given
for inst_id in inst_ids.keys()}
-
-.. _rst_test-special:
-
-Special Test Configurations
----------------------------
+.. _rst_test-nodownload:
No Download Available
^^^^^^^^^^^^^^^^^^^^^
From 728153bb763ea178c8a77463168fb3a4849d1ddf Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 6 Jul 2023 13:27:12 -0400
Subject: [PATCH 016/365] MAINT: added missing quotation
Added a missing quotation mark.
---
docs/new_instrument.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index 1712ad9a3..f6d5a2cc1 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -775,7 +775,7 @@ allows multiple types of warning messages to be tested for a given
('logger', 'WARN', "I am a warning!", 'clean'),
('warning', UserWarning,
'I am a serios warning!', 'dusty'),
- ('error, ValueError, 'I am an error', 'dusty')]}
+ ('error', ValueError, 'I am an error', 'dusty')]}
for tag in inst_ids[inst_id]}
for inst_id in inst_ids.keys()}
From 150e691e1c492ffa85b7a24e9faa40dfaff6b936 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 6 Jul 2023 16:33:38 -0400
Subject: [PATCH 017/365] BUG: fixed dict access
Fixed the dict access for the expected clean warnings.
---
pysat/tests/classes/cls_instrument_library.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 5bdf6faf7..08e4238c5 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -388,7 +388,8 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
"""
if hasattr(inst_dict['inst_module'], '_clean_warn'):
- clean_warn = inst_dict['inst_module']._clean_warn
+ clean_warn = inst_dict['inst_module']._clean_warn[
+ inst_dict['inst_id']][inst_dict['tag']]
if clean_level in clean_warn.keys():
# Only need to test if there are clean warnings for this level
From 78e85aa9b5fe3d801f1d48a81c2f39f0fa6ba415 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 6 Jul 2023 18:09:13 -0400
Subject: [PATCH 018/365] BUG: handle strict time flag before messages
Ensure the strict time flag is not raising a ValueError when testing the clean warning messages.
---
pysat/tests/classes/cls_instrument_library.py | 108 +++++++++---------
1 file changed, 53 insertions(+), 55 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 08e4238c5..4f73cf428 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -396,6 +396,30 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
test_inst, date = initialize_test_inst_and_date(inst_dict)
clean_warnings = clean_warn[clean_level]
+ try:
+ test_inst.load(date=date, use_header=True)
+ except Exception as err:
+ # Catch all potential input errors, and only ensure that
+ # the one caused by the strict time flag is prevented from
+ # occurring on future load calls.
+ if str(err).find('Loaded data') > 0:
+ # Change the flags that may have caused
+ # the error to be raised, to see if it the
+ # strict time flag
+ test_inst.strict_time_flag = False
+ test_inst.clean_level = 'none'
+
+ # Evaluate the warning
+ with warnings.catch_warnings(record=True) as war:
+ test_inst.load(date=date, use_header=True)
+
+ assert len(war) >= 1
+ categories = [war[j].category for j in range(len(war))]
+ assert UserWarning in categories
+
+ # Reset the clean level
+ test_inst.clean_level = clean_level
+
for (clean_method, clean_method_level, clean_method_msg,
final_level) in clean_warnings:
if len(test_inst.files.files) > 0:
@@ -404,61 +428,35 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
target = 'Fake Data to be cleared'
test_inst.data = [target]
- try:
- if clean_method == 'logger':
- # A logging message is expected
- with caplog.at_level(
- getattr(logging, clean_method_level),
- logger='pysat'):
- test_inst.load(date=date, use_header=True)
-
- # Test the returned message
- out_msg = caplog.text
- assert out_msg.find(clean_method_msg) >= 0
- elif clean_method == 'warning':
- # A warning message is expected
- with warnings.catch_warnings(
- record=True) as war:
- test_inst.load(date=date, use_header=True)
-
- # Test the warning output
- testing.eval_warnings(war, [clean_method_msg],
- clean_method_level)
- elif clean_method == 'error':
- # An error message is expected, evaluate error
- # and the error message
- try:
- testing.eval_bad_input(
- test_inst.load, clean_method_level,
- clean_method_msg,
- input_kwargs={'date': date,
- 'use_header': True})
- except AssertionError as aerr:
- if str(aerr).find(
- 'Loaded data is not unique') >= 0:
- pytest.skip(
- 'Cannot test multiple errors')
- else:
- raise AssertionError(aerr)
- else:
- raise AttributeError(
- 'unknown type of warning: {:}'.format(
- clean_method))
- except ValueError as verr:
- # Check if instrument is failing due to strict time
- # flag
- if str(verr).find('Loaded data') > 0:
- test_inst.strict_time_flag = False
- with warnings.catch_warnings(
- record=True) as war:
- test_inst.load(date=date, use_header=True)
- assert len(war) >= 1
- categories = [war[j].category
- for j in range(0, len(war))]
- assert UserWarning in categories
- else:
- # If error message does not match, raise error
- raise ValueError(verr)
+ if clean_method == 'logger':
+ # A logging message is expected
+ with caplog.at_level(
+ getattr(logging, clean_method_level),
+ logger='pysat'):
+ test_inst.load(date=date, use_header=True)
+
+ # Test the returned message
+ out_msg = caplog.text
+ assert out_msg.find(clean_method_msg) >= 0
+ elif clean_method == 'warning':
+ # A warning message is expected
+ with warnings.catch_warnings(record=True) as war:
+ test_inst.load(date=date, use_header=True)
+
+ # Test the warning output
+ testing.eval_warnings(war, [clean_method_msg],
+ clean_method_level)
+ elif clean_method == 'error':
+ # An error message is expected, evaluate error
+ # and the error message
+ testing.eval_bad_input(
+ test_inst.load, clean_method_level,
+ clean_method_msg,
+ input_kwargs={'date': date, 'use_header': True})
+ else:
+ raise AttributeError(
+ 'unknown type of warning: {:}'.format(
+ clean_method))
# Test to see if the clean flag has the expected value
# afterwards
From 0d2c069f2dd5a8cfa4d0acd92da6de02781d2e05 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 6 Jul 2023 18:11:00 -0400
Subject: [PATCH 019/365] BUG: remove error from mult
Cannot assess other warnings when an error is raised, only test multiple non-fatal warnings.
---
pysat/tests/test_instruments.py | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py
index 37d6ed8bf..d5cede4e2 100644
--- a/pysat/tests/test_instruments.py
+++ b/pysat/tests/test_instruments.py
@@ -181,11 +181,11 @@ def test_clean_with_warnings(self, clean_level, change, warn_type,
# Construct the expected warnings
if warn_type == 'mult':
+ # Note that we cannot test errors along with other warnings
inst_dict['inst_module']._clean_warn = {
inst_dict['inst_id']: {inst_dict['tag']: {clean_level: [
- ('logger', warn_level['logger'], warn_msg, final_level),
('warning', warn_level['warning'], warn_msg, final_level),
- ('error', warn_level['error'], warn_msg, final_level)]}}}
+ ('logger', warn_level['logger'], warn_msg, final_level)]}}}
else:
inst_dict['inst_module']._clean_warn = {
inst_dict['inst_id']: {inst_dict['tag']: {clean_level: [
@@ -194,18 +194,14 @@ def test_clean_with_warnings(self, clean_level, change, warn_type,
# Set the additional Instrument kwargs
if 'kwargs' in inst_dict.keys():
- if 'test_clean_kwarg' in inst_dict['kwargs']:
- inst_dict['kwargs']['test_clean_kwarg']['change'] = final_level
- else:
- inst_dict['kwargs']['test_clean_kwarg'] = {'change':
- final_level}
+ # Ensure the test instrument cleaning kwarg is reset
+ inst_dict['kwargs']['test_clean_kwarg'] = {'change': final_level}
else:
inst_dict['kwargs'] = {'test_clean_kwarg': {'change': final_level}}
if warn_type == 'mult':
inst_dict['kwargs']['test_clean_kwarg']['logger'] = warn_msg
inst_dict['kwargs']['test_clean_kwarg']['warning'] = warn_msg
- inst_dict['kwargs']['test_clean_kwarg']['error'] = warn_msg
else:
inst_dict['kwargs']['test_clean_kwarg'][warn_type] = warn_msg
From 0c3798ace24656e82097e4922ece36e92f901b59 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 6 Jul 2023 18:59:07 -0400
Subject: [PATCH 020/365] TST: improved error message
Improved the logging error message in the clean warning test to be more useful.
---
pysat/tests/classes/cls_instrument_library.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 4f73cf428..ff14bdcf9 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -437,7 +437,9 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Test the returned message
out_msg = caplog.text
- assert out_msg.find(clean_method_msg) >= 0
+ assert out_msg.find(clean_method_msg) >= 0, \
+ "{:s} not in output: {:s}".format(
+ clean_method_msg, out_msg)
elif clean_method == 'warning':
# A warning message is expected
with warnings.catch_warnings(record=True) as war:
From c769ba87e298a1e76206f30d84ab94d3e0740734 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 10 Jul 2023 15:36:14 -0400
Subject: [PATCH 021/365] MAINT: fixed spelling
Fixed spelling and line length in the docs.
---
docs/new_instrument.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index f6d5a2cc1..1db654491 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -774,8 +774,8 @@ allows multiple types of warning messages to be tested for a given
_clean_warn = {inst_id: {tag: {'dusty': [
('logger', 'WARN', "I am a warning!", 'clean'),
('warning', UserWarning,
- 'I am a serios warning!', 'dusty'),
- ('error', ValueError, 'I am an error', 'dusty')]}
+ 'I am a serious warning!', 'dusty'),
+ ('error', ValueError, "I'm an error", 'dusty')]}
for tag in inst_ids[inst_id]}
for inst_id in inst_ids.keys()}
From 1eadc88158225a58a92ae674a9520cf2d06f52ba Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 10 Jul 2023 15:49:00 -0400
Subject: [PATCH 022/365] ENH: improved clean warning test
Improved the clean warning test by:
- Creating a function for testing/setting the strict time flag,
- Removing 'none' from the clean level parametrization,
- Fixing the docstrings,
- Adding more comments, and
- Adding pytest skip statements.
---
pysat/tests/classes/cls_instrument_library.py | 110 +++++++++++-------
1 file changed, 71 insertions(+), 39 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index ff14bdcf9..481cddc74 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -73,6 +73,50 @@ def initialize_test_inst_and_date(inst_dict):
return test_inst, date
+def set_strict_time_flag(test_inst, date, raise_error=False):
+ """Ensure the strict time flag does not interfere with other tests.
+
+ Parameters
+ ----------
+ test_inst : pysat.Instrument
+ Test instrument
+ date : dt.datetime
+ Date for loading data
+ raise_error : bool
+ Raise the load error if it is not the strict time flag error
+ (default=False)
+
+ """
+
+ try:
+ test_inst.load(date=date, use_header=True)
+ except Exception as err:
+ # Catch all potential input errors, and only ensure that the one caused
+ # by the strict time flag is prevented from occurring on future load
+ # calls.
+ if str(err).find('Loaded data') > 0:
+ # Change the flags that may have caused the error to be raised, to
+ # see if it the strict time flag
+ test_inst.strict_time_flag = False
+ orig_clean_level = str(test_inst.clean_level)
+ test_inst.clean_level = 'none'
+
+ # Evaluate the warning
+ with warnings.catch_warnings(record=True) as war:
+ test_inst.load(date=date, use_header=True)
+
+ assert len(war) >= 1
+ categories = [war[j].category for j in range(len(war))]
+ assert UserWarning in categories
+
+ # Reset the clean level
+ test_inst.clean_level = orig_clean_level
+ elif raise_error:
+ raise err
+
+ return
+
+
class InstLibTests(object):
"""Provide standardized tests for pysat instrument libraries.
@@ -340,20 +384,10 @@ def test_load(self, clean_level, inst_dict):
test_inst.clean_level = clean_level
target = 'Fake Data to be cleared'
test_inst.data = [target]
- try:
- test_inst.load(date=date, use_header=True)
- except ValueError as verr:
- # Check if instrument is failing due to strict time flag
- if str(verr).find('Loaded data') > 0:
- test_inst.strict_time_flag = False
- with warnings.catch_warnings(record=True) as war:
- test_inst.load(date=date, use_header=True)
- assert len(war) >= 1
- categories = [war[j].category for j in range(0, len(war))]
- assert UserWarning in categories
- else:
- # If error message does not match, raise error anyway
- raise ValueError(verr)
+
+ # Make sure the strict time flag doesn't interfere with
+ # the cleaning tests
+ set_strict_time_flag(test_inst, date, raise_error=True)
# Make sure fake data is cleared
assert target not in test_inst.data
@@ -372,7 +406,7 @@ def test_load(self, clean_level, inst_dict):
# Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
@pytest.mark.download
- @pytest.mark.parametrize("clean_level", ['none', 'dirty', 'dusty', 'clean'])
+ @pytest.mark.parametrize("clean_level", ['dirty', 'dusty', 'clean'])
def test_clean_warn(self, clean_level, inst_dict, caplog):
"""Test that appropriate warnings and errors are raised when cleaning.
@@ -384,42 +418,27 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
Dictionary containing info to instantiate a specific instrument.
Set automatically from instruments['download'] when
`initialize_test_package` is run.
- **kwags : dict
"""
+ # Not all Instruments have warning messages to test, only run tests
+ # when the desired test attribute is defined
if hasattr(inst_dict['inst_module'], '_clean_warn'):
clean_warn = inst_dict['inst_module']._clean_warn[
inst_dict['inst_id']][inst_dict['tag']]
+ # Cleaning warnings may vary by clean level, test the warning
+ # messages at the current clean level, specified by `clean_level`
if clean_level in clean_warn.keys():
# Only need to test if there are clean warnings for this level
test_inst, date = initialize_test_inst_and_date(inst_dict)
clean_warnings = clean_warn[clean_level]
- try:
- test_inst.load(date=date, use_header=True)
- except Exception as err:
- # Catch all potential input errors, and only ensure that
- # the one caused by the strict time flag is prevented from
- # occurring on future load calls.
- if str(err).find('Loaded data') > 0:
- # Change the flags that may have caused
- # the error to be raised, to see if it the
- # strict time flag
- test_inst.strict_time_flag = False
- test_inst.clean_level = 'none'
-
- # Evaluate the warning
- with warnings.catch_warnings(record=True) as war:
- test_inst.load(date=date, use_header=True)
-
- assert len(war) >= 1
- categories = [war[j].category for j in range(len(war))]
- assert UserWarning in categories
-
- # Reset the clean level
- test_inst.clean_level = clean_level
+ # Make sure the strict time flag doesn't interfere with
+ # the cleaning tests
+ set_strict_time_flag(test_inst, date)
+ # Cycle through each of the potential cleaning messages
+ # for this Instrument module, inst ID, tag, and clean level
for (clean_method, clean_method_level, clean_method_msg,
final_level) in clean_warnings:
if len(test_inst.files.files) > 0:
@@ -468,6 +487,19 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Make sure fake data is cleared
assert target not in test_inst.data
+ else:
+ pytest.skip("".join(["Can't test clean warnings for ",
+ "Instrument ",
+ repr(inst_dict['inst_module']),
+ " level ", clean_level,
+ " (no downloaded files)"]))
+ else:
+ pytest.skip("".join(["No clean warnings for Instrument ",
+ repr(inst_dict['inst_module']), " level ",
+ clean_level]))
+ else:
+ pytest.skip("No clean warnings for Instrument {:s}".format(
+ repr(inst_dict['inst_module'])))
return
From 35fdc51cbf04d236d57780dd279efbce7357fca6 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 11 Jul 2023 11:46:00 -0400
Subject: [PATCH 023/365] DOC: Update
pysat/tests/classes/cls_instrument_library.py
Fixed comments to be accurate.
Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
---
pysat/tests/classes/cls_instrument_library.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 481cddc74..d3e9dde11 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -386,7 +386,7 @@ def test_load(self, clean_level, inst_dict):
test_inst.data = [target]
# Make sure the strict time flag doesn't interfere with
- # the cleaning tests
+ # the load tests
set_strict_time_flag(test_inst, date, raise_error=True)
# Make sure fake data is cleared
From 31313412fa67ea201881679ab0b9f64909aa7a48 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 11 Jul 2023 14:04:49 -0400
Subject: [PATCH 024/365] TST: added clean_level input
Allow the strict flag function to run with or without the clean method.
---
pysat/tests/classes/cls_instrument_library.py | 24 ++++++++++++-------
1 file changed, 16 insertions(+), 8 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index d3e9dde11..35cad0ffb 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -73,7 +73,7 @@ def initialize_test_inst_and_date(inst_dict):
return test_inst, date
-def set_strict_time_flag(test_inst, date, raise_error=False):
+def set_strict_time_flag(test_inst, date, raise_error=False, clean_off=True):
"""Ensure the strict time flag does not interfere with other tests.
Parameters
@@ -85,6 +85,9 @@ def set_strict_time_flag(test_inst, date, raise_error=False):
raise_error : bool
Raise the load error if it is not the strict time flag error
(default=False)
+ clean_off : bool
+ Turn off the clean method when re-loading data and testing the
+ strict time flag (default=True)
"""
@@ -98,8 +101,11 @@ def set_strict_time_flag(test_inst, date, raise_error=False):
# Change the flags that may have caused the error to be raised, to
# see if it the strict time flag
test_inst.strict_time_flag = False
- orig_clean_level = str(test_inst.clean_level)
- test_inst.clean_level = 'none'
+
+ if clean_off:
+ # Turn the clean method off
+ orig_clean_level = str(test_inst.clean_level)
+ test_inst.clean_level = 'none'
# Evaluate the warning
with warnings.catch_warnings(record=True) as war:
@@ -109,8 +115,9 @@ def set_strict_time_flag(test_inst, date, raise_error=False):
categories = [war[j].category for j in range(len(war))]
assert UserWarning in categories
- # Reset the clean level
- test_inst.clean_level = orig_clean_level
+ if clean_off:
+ # Reset the clean level
+ test_inst.clean_level = orig_clean_level
elif raise_error:
raise err
@@ -380,14 +387,15 @@ def test_load(self, clean_level, inst_dict):
test_inst, date = initialize_test_inst_and_date(inst_dict)
if len(test_inst.files.files) > 0:
- # Set Clean Level
+ # Set the clean level
test_inst.clean_level = clean_level
target = 'Fake Data to be cleared'
test_inst.data = [target]
# Make sure the strict time flag doesn't interfere with
- # the load tests
- set_strict_time_flag(test_inst, date, raise_error=True)
+ # the load tests, and re-run with desired clean level
+ set_strict_time_flag(test_inst, date, raise_error=True,
+ clean_off=False)
# Make sure fake data is cleared
assert target not in test_inst.data
From a356931c7d0c68793c3da1bc42e01c716e37c14b Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 12 Jul 2023 13:26:43 -0400
Subject: [PATCH 025/365] STY: rename function
Rename the test function as per discussion.
Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
---
pysat/tests/classes/cls_instrument_library.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 35cad0ffb..b72ee011d 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -73,8 +73,8 @@ def initialize_test_inst_and_date(inst_dict):
return test_inst, date
-def set_strict_time_flag(test_inst, date, raise_error=False, clean_off=True):
- """Ensure the strict time flag does not interfere with other tests.
+def load_and_set_strict_time_flag(test_inst, date, raise_error=False, clean_off=True):
+ """Load data and set the strict time flag if needed for other tests.
Parameters
----------
@@ -394,8 +394,8 @@ def test_load(self, clean_level, inst_dict):
# Make sure the strict time flag doesn't interfere with
# the load tests, and re-run with desired clean level
- set_strict_time_flag(test_inst, date, raise_error=True,
- clean_off=False)
+ load_and_set_strict_time_flag(test_inst, date, raise_error=True,
+ clean_off=False)
# Make sure fake data is cleared
assert target not in test_inst.data
@@ -443,7 +443,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Make sure the strict time flag doesn't interfere with
# the cleaning tests
- set_strict_time_flag(test_inst, date)
+ load_and_set_strict_time_flag(test_inst, date)
# Cycle through each of the potential cleaning messages
# for this Instrument module, inst ID, tag, and clean level
From 3bacdef240707f6aaf16f67376fe7d17aa5b4247 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Wed, 12 Jul 2023 13:29:17 -0400
Subject: [PATCH 026/365] STY: pep8
---
pysat/tests/classes/cls_instrument_library.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index b72ee011d..7429c7ff6 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -73,7 +73,8 @@ def initialize_test_inst_and_date(inst_dict):
return test_inst, date
-def load_and_set_strict_time_flag(test_inst, date, raise_error=False, clean_off=True):
+def load_and_set_strict_time_flag(test_inst, date, raise_error=False,
+ clean_off=True):
"""Load data and set the strict time flag if needed for other tests.
Parameters
From b04eaac52afe8544bd017148b75d828630f51024 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Wed, 19 Jul 2023 13:12:02 -0400
Subject: [PATCH 027/365] ENH: updated ValueError message
Updated a ValueError message to be more informative, as valid tags vary with inst_id.
---
pysat/_instrument.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 744b6c302..d9964ed39 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -1396,10 +1396,12 @@ def _assign_attrs(self, by_name=False):
raise ValueError(estr)
if self.tag not in self.inst_module.inst_ids[self.inst_id]:
- tag_str = ', '.join([tkey.__repr__() for tkey
- in self.inst_module.inst_ids[self.inst_id]])
+ tag_id_str = repr(self.inst_module.inst_ids[self.inst_id]).replace(
+ "{", "'inst ID': ['tag'] combinations are: ")
+ tag_id_str = tag_id_str.replace("}", "")
estr = ''.join(("'", self.tag, "' is not one of the supported ",
- 'tags. Supported tags are: ', tag_str, '.'))
+ "tags for inst ID ['", self.inst_id, "']. ",
+ 'Supported ', tag_id_str))
raise ValueError(estr)
# Assign the Instrument methods
From 40e16dccb1a259e81948017f1b598e92406321fb Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Wed, 19 Jul 2023 13:12:30 -0400
Subject: [PATCH 028/365] TST: updated ValueError test
Updated the ValueError test message for a bad tag to reflect the changes in the message.
---
pysat/tests/classes/cls_instrument_property.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_property.py b/pysat/tests/classes/cls_instrument_property.py
index 554cd7fef..dcb0b262c 100644
--- a/pysat/tests/classes/cls_instrument_property.py
+++ b/pysat/tests/classes/cls_instrument_property.py
@@ -693,7 +693,7 @@ def test_optional_unknown_data_dir(self, caplog):
[({'inst_id': 'invalid_inst_id'},
"'invalid_inst_id' is not one of the supported inst_ids."),
({'inst_id': '', 'tag': 'bad_tag'},
- "'bad_tag' is not one of the supported tags.")])
+ "'bad_tag' is not one of the supported tags")])
def test_error_bad_instrument_object(self, kwargs, estr):
"""Ensure instantiation with invalid inst_id or tag errors.
From a1be583e49b3272ae5ae4070e0e1ecdc06a4f298 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Wed, 19 Jul 2023 13:13:15 -0400
Subject: [PATCH 029/365] DOC: updated changelog
Updated the changelog.
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8274e2f15..f920be4b3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
clean method.
* Maintenance
* Update link redirects in docs.
+ * Improved Instrument ValueError messages.
[3.1.0] - 2023-05-31
--------------------
From 37b231b22ee65fc6b94a397d067103f026f9a64a Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 20 Jul 2023 16:16:18 -0400
Subject: [PATCH 030/365] ENH: allow custom `concat_data` methods
Allow instruments to specify their own `concat_data` methods, making allowances for multiple or non-existent time indices.
---
pysat/_instrument.py | 78 +++++++++++++++++++++++++-------------------
1 file changed, 44 insertions(+), 34 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index d9964ed39..fc3056b0e 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -322,7 +322,7 @@ def __init__(self, platform=None, name=None, tag='', inst_id='',
# Expected function keywords
exp_keys = ['list_files', 'load', 'preprocess', 'download',
- 'list_remote_files', 'clean', 'init']
+ 'list_remote_files', 'clean', 'init', 'concat_data']
for fkey in exp_keys:
func_name = _kwargs_keys_to_func_name(fkey)
func = getattr(self, func_name)
@@ -568,7 +568,8 @@ def __eq__(self, other):
# required their own path for equality, string comparisons!
partial_funcs = ['_init_rtn', '_clean_rtn', '_preprocess_rtn',
'_list_files_rtn', '_download_rtn',
- '_list_remote_files_rtn', '_load_rtn']
+ '_list_remote_files_rtn', '_load_rtn',
+ '_concat_data_rtn']
# If the type is the same then check everything that is attached to
# the Instrument object. Includes attributes, methods, variables, etc.
@@ -1310,7 +1311,7 @@ def _assign_attrs(self, by_name=False):
methods
init, preprocess, and clean
functions
- load, list_files, download, and list_remote_files
+ load, list_files, download, and list_remote_files, concat_data
attributes
directory_format, file_format, multi_file_day, orbit_info, and
pandas_format
@@ -1320,7 +1321,7 @@ def _assign_attrs(self, by_name=False):
"""
# Declare the standard Instrument methods and attributes
inst_methods = {'required': ['init', 'clean'],
- 'optional': ['preprocess']}
+ 'optional': ['preprocess', 'concat_data']}
inst_funcs = {'required': ['load', 'list_files', 'download'],
'optional': ['list_remote_files']}
inst_attrs = {'directory_format': None, 'file_format': None,
@@ -2375,6 +2376,11 @@ def concat_data(self, new_data, prepend=False, **kwargs):
except if the user includes a value for dim as a keyword argument.
"""
+ # Add any concat_data kwargs
+ for ckey in self.kwargs['concat_data'].keys():
+ if ckey not in kwargs.keys():
+ kwargs[ckey] = self.kwargs['concat_data'][ckey]
+
# Order the data to be concatenated in a list
if not isinstance(new_data, list):
new_data = [new_data]
@@ -2383,36 +2389,41 @@ def concat_data(self, new_data, prepend=False, **kwargs):
new_data.append(self.data)
else:
new_data.insert(0, self.data)
-
- # Retrieve the appropriate concatenation function
- if self.pandas_format:
- # Specifically do not sort unless otherwise specified
- if 'sort' not in kwargs:
- kwargs['sort'] = False
- concat_func = pds.concat
+
+ if self._concat_data_rtn.__name__.find('_pass_method') == 0:
+ # There is no custom concat function, use the pysat standard method.
+ # Start by retrieving the appropriate concatenation function
+ if self.pandas_format:
+ # Specifically do not sort unless otherwise specified
+ if 'sort' not in kwargs:
+ kwargs['sort'] = False
+ concat_func = pds.concat
+ else:
+ # Ensure the dimensions are equal
+ equal_dims = True
+ idat = 0
+ while idat < len(new_data) - 1 and equal_dims:
+ if new_data[idat].dims != new_data[idat + 1].dims:
+ equal_dims = False
+ idat += 1
+
+ if not equal_dims:
+ # Update the dimensions, padding data where necessary
+ new_data = pysat.utils.coords.expand_xarray_dims(
+ new_data, self.meta, exclude_dims=[self.index.name])
+
+ # Specify the dimension, if not otherwise specified
+ if 'dim' not in kwargs:
+ kwargs['dim'] = self.index.name
+
+ # Set the concat function
+ concat_func = xr.concat
+
+ # Assign the concatenated data to the instrument
+ self.data = concat_func(new_data, **kwargs)
else:
- # Ensure the dimensions are equal
- equal_dims = True
- idat = 0
- while idat < len(new_data) - 1 and equal_dims:
- if new_data[idat].dims != new_data[idat + 1].dims:
- equal_dims = False
- idat += 1
-
- if not equal_dims:
- # Update the dimensions, padding data where necessary
- new_data = pysat.utils.coords.expand_xarray_dims(
- new_data, self.meta, exclude_dims=['time'])
-
- # Specify the dimension, if not otherwise specified
- if 'dim' not in kwargs:
- kwargs['dim'] = self.index.name
-
- # Set the concat function
- concat_func = xr.concat
-
- # Assign the concatenated data to the instrument
- self.data = concat_func(new_data, **kwargs)
+ self._concat_data_rtn(new_data, **kwargs)
+
return
def custom_attach(self, function, at_pos='end', args=None, kwargs=None):
@@ -3328,7 +3339,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
if not self.empty:
if (self.index[-1] == last_pad) & (not want_last_pad):
self.data = self[:-1]
-
else:
# If self.pad is False, load single day
self.data, meta = self._load_data(date=self.date, fid=self._fid,
From 1bd63537bc655eb7348af57982ee803b56f181f3 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 20 Jul 2023 16:28:52 -0400
Subject: [PATCH 031/365] DOC: update changelog
Add a description of the changes to the log.
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f920be4b3..c8b5df1c0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* New Features
* Added tests for warnings, logging messages, and errors in the Instrument
clean method.
+ * Allow Instruments to define custom `concat_data` methods.
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
From 4ea3cbe13c77bb9788ac74612b28e628604672b8 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 20 Jul 2023 16:29:19 -0400
Subject: [PATCH 032/365] DOC: update new instrument guidelines
Update the new instrument guidelines to include the `concat_data` method.
---
docs/new_instrument.rst | 28 ++++++++++++++++++++++++----
1 file changed, 24 insertions(+), 4 deletions(-)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index 1db654491..9a0255ef6 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -546,10 +546,11 @@ If provided, :py:mod:`pysat` supports the definition and use of keywords for an
instrument module so that users may define their preferred default values. A
custom keyword for an instrument module must be defined in each function that
will receive that keyword argument if provided by the user. All instrument
-functions, :py:func:`init`, :py:func:`preprocess`, :py:func:`load`,
-:py:func:`clean`, :py:func:`list_files`, :py:func:`list_remote_files`, and
-:py:func:`download` support custom keywords. The same keyword may be used in
-more than one function but the same value will be passed to each.
+functions, :py:func:`init`, :py:func:`preprocess`, :py:func:`concat_data`,
+:py:func:`load`, :py:func:`clean`, :py:func:`list_files`,
+:py:func:`list_remote_files`, and :py:func:`download` support custom keywords.
+The same keyword may be used in more than one function but the same value will
+be passed to each.
An example :py:func:`load` function definition with two custom keyword
arguments.
@@ -650,6 +651,25 @@ The user can search for subsets of files through optional keywords, such as:
inst.remote_file_list(year=2019)
inst.remote_file_list(year=2019, month=1, day=1)
+concat_data
+^^^^^^^^^^^
+
+Combines data from multiple Instruments of the same type, used internally to
+combine data from different load periods. The default method concatonates data
+using the :py:attr:`inst.index` name. However, some data sets have multiple
+different time indices along which data should be concatonated. In such cases
+(e.g., TIMED-GUVI SDR-Imaging data from :py:mod:`pysatNASA`), a custom
+:py:meth:`concat_data` method must be supplied. If available, this method
+will be used instead of the default
+:py:meth:`~pysat._instrument.Instrument.concat_data`, after the default
+method handles the prepending of the data that needs to be combined.
+
+.. code:: python
+
+ def concat_data(self, new_data, **kwargs):
+ # Perform custom concatonation here, updating self.data
+ return
+
Logging
-------
From bd82e10ad4cda7b1ca03738438b24ba69832f708 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 20 Jul 2023 16:36:13 -0400
Subject: [PATCH 033/365] MAINT: added TODO for loading
Currently, secondary time data is not down-selected in the load function. Added TODO statements in the places where data is being selected by time as reminders to address this issue.
---
pysat/_instrument.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index fc3056b0e..5fec219d1 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -3309,6 +3309,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
# __getitem__ used below to get data from instrument object.
# Details for handling pandas and xarray are different and
# handled by __getitem__.
+ # TODO: fix data selection for other time indices
self.data = self[first_pad:temp_time]
if not self.empty:
if self.index[-1] == temp_time:
@@ -3324,6 +3325,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
# Pad data using access mechanisms that work for both pandas
# and xarray
self.data = self._next_data.copy()
+ # TODO: fix data selection for other time indices
self.data = self[temp_time:last_pad]
if len(self.index) > 0:
if (self.index[0] == temp_time):
@@ -3333,6 +3335,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
self.data = stored_data
if len(self.index) > 0:
+ # TODO: fix data selection for other time indices
self.data = self[first_pad:last_pad]
# Want exclusive end slicing behavior from above
@@ -3439,6 +3442,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
# Remove the excess data padding, if any applied
if (self.pad is not None) & (not self.empty) & (not verifyPad):
+ # TODO: fix data selection for other time indices
self.data = self[first_time: last_time]
if not self.empty:
if (self.index[-1] == last_time) & (not want_last_pad):
From fdad8f4fe6e50434d66f2a2b09c4e8c12ab1bbc3 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 25 Jul 2023 11:19:12 -0400
Subject: [PATCH 034/365] BUG: fixed xarray time selection
Fixed xarray time selection for multiple time indices.
---
pysat/_instrument.py | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 5fec219d1..290c4e75e 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -868,12 +868,19 @@ def __getitem_xarray__(self, key):
"""
if 'Epoch' in self.data.indexes:
- epoch_name = 'Epoch'
+ epoch_names = ['Epoch']
elif 'time' in self.data.indexes:
- epoch_name = 'time'
+ epoch_names = ['time']
else:
return xr.Dataset(None)
+ # Find secondary time indexes that may need to be sliced
+ if len(self.data.indexes) > 1:
+ for ind in self.data.indexes.keys():
+ if(ind != epoch_names[0] and self.data.indexes[ind].dtype
+ == self.data.indexes[epoch_names[0]].dtype):
+ epoch_names.append(ind)
+
if isinstance(key, tuple):
if len(key) == 2:
# Support slicing time, variable name
@@ -885,7 +892,8 @@ def __getitem_xarray__(self, key):
data_subset = self.data[key[1]]
# If the input is a tuple, `key[0]` must be linked to the epoch.
- key_dict = {'indexers': {epoch_name: key[0]}}
+ key_dict = {'indexers': {epoch_name: key[0]
+ for epoch_name in epoch_names}}
try:
# Assume key[0] is an integer
return data_subset.isel(**key_dict)
@@ -934,7 +942,8 @@ def __getitem_xarray__(self, key):
except (TypeError, KeyError, ValueError):
# If that didn't work, likely need to use `isel` or `sel`
# Link key to the epoch.
- key_dict = {'indexers': {epoch_name: key}}
+ key_dict = {'indexers': {epoch_name: key
+ for epoch_name in epoch_names}}
try:
# Try to get all data variables, but for a subset of time
# using integer indexing
@@ -3309,7 +3318,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
# __getitem__ used below to get data from instrument object.
# Details for handling pandas and xarray are different and
# handled by __getitem__.
- # TODO: fix data selection for other time indices
self.data = self[first_pad:temp_time]
if not self.empty:
if self.index[-1] == temp_time:
@@ -3325,7 +3333,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
# Pad data using access mechanisms that work for both pandas
# and xarray
self.data = self._next_data.copy()
- # TODO: fix data selection for other time indices
self.data = self[temp_time:last_pad]
if len(self.index) > 0:
if (self.index[0] == temp_time):
@@ -3335,7 +3342,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
self.data = stored_data
if len(self.index) > 0:
- # TODO: fix data selection for other time indices
self.data = self[first_pad:last_pad]
# Want exclusive end slicing behavior from above
@@ -3442,7 +3448,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
# Remove the excess data padding, if any applied
if (self.pad is not None) & (not self.empty) & (not verifyPad):
- # TODO: fix data selection for other time indices
self.data = self[first_time: last_time]
if not self.empty:
if (self.index[-1] == last_time) & (not want_last_pad):
From d2b242041cd6b6b137c130d7c58dcd1052cd2016 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 25 Jul 2023 11:22:21 -0400
Subject: [PATCH 035/365] STY: remove whitespace
Remove extra whitespace.
---
pysat/_instrument.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 290c4e75e..ae04bb8e6 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -2398,7 +2398,7 @@ def concat_data(self, new_data, prepend=False, **kwargs):
new_data.append(self.data)
else:
new_data.insert(0, self.data)
-
+
if self._concat_data_rtn.__name__.find('_pass_method') == 0:
# There is no custom concat function, use the pysat standard method.
# Start by retrieving the appropriate concatenation function
From 35e2341d187d6b9d054982cfc9fab2e646e66e99 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 25 Jul 2023 12:08:36 -0400
Subject: [PATCH 036/365] ENH: added extra time dimension to test inst
Added an optional extra time dimension to the xarray test instrument.
---
pysat/instruments/methods/testing.py | 71 ++++++++++++++++++++++++++--
pysat/instruments/pysat_ndtesting.py | 38 +++++++++++----
2 files changed, 97 insertions(+), 12 deletions(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index fe66275bf..e60be7171 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -7,6 +7,7 @@
import pandas as pds
import time
import warnings
+import xarray as xr
import pysat
from pysat.utils import NetworkLock
@@ -82,7 +83,67 @@ def clean(self, test_clean_kwarg=None):
return
-# Optional method
+# Optional methods
+def concat_data(self, new_data, extra_time_dims=None, **kwargs):
+ """Concatonate data to self.data for extra time dimensions.
+
+ Parameters
+ ----------
+ new_data : xarray.Dataset or list of such objects
+ New data objects to be concatonated
+ extra_time_dims : list-like or NoneType
+ List of extra time dimensions that require concatonation (default=None)
+ **kwargs : dict
+ Optional keyword arguments passed to xr.concat
+
+ """
+ # Establish the time dimensions
+ time_dims = [self.index.name]
+
+ if extra_time_dims is not None:
+ time_dims.extend(list(extra_time_dims))
+
+ # Concatonate using the appropriate method for the number of time
+ # dimensions
+ if len(time_dims) == 1:
+ # There is only one time dimensions, but other dimensions may
+ # need to be adjusted
+ new_data = pysat.utils.coords.expand_xarray_dims(
+ new_data, self.meta, exclude_dims=time_dims)
+
+ # Combine the data
+ self.data = xr.combine_by_coords(new_data, **kwargs)
+ else:
+ inners = None
+ for ndata in new_data:
+ # Separate into inner datasets
+ inner_keys = {dim: [key for key in ndata.keys()
+ if dim in ndata[key].dims] for dim in time_dims}
+ inner_dat = {dim: ndata.get(inner_keys[dim]) for dim in time_dims}
+
+ # Add 'single_var's into 'time' dataset to keep track
+ sv_keys = [val.name for val in ndata.values()
+ if 'single_var' in val.dims]
+ singlevar_set = ndata.get(sv_keys)
+ inner_dat[self.index.name] = xr.merge([inner_dat[self.index.name],
+ singlevar_set])
+
+ # Concatenate along desired dimension with previous data
+ if inners is None:
+ # No previous data, assign the data separated by dimension
+ inners = dict(inner_dat)
+ else:
+ # Concatenate with existing data
+ inners = {dim: xr.concat([inners[dim], inner_dat[dim]],
+ dim=dim) for dim in time_dims}
+
+ # Combine all time dimensions
+ if inners is not None:
+ data_list = [inners[dim] for dim in time_dims]
+ self.data = xr.merge(data_list)
+ return
+
+
def preprocess(self, test_preprocess_kwarg=None):
"""Perform standard preprocessing.
@@ -96,12 +157,12 @@ def preprocess(self, test_preprocess_kwarg=None):
Testing keyword (default=None)
"""
-
self.test_preprocess_kwarg = test_preprocess_kwarg
return
+# Utility functions
def initialize_test_meta(epoch_name, data_keys):
"""Initialize meta data for test instruments.
@@ -213,8 +274,12 @@ def initialize_test_meta(epoch_name, data_keys):
'units': 'km',
'meta': alt_profile_meta}
+ # Optional and standard metadata for xarray
+ for var in data_keys:
+ if var.find('variable_profiles') == 0:
+ meta[var] = {'desc': 'Profiles with variable altitude.'}
+
# Standard metadata required for xarray.
- meta['variable_profiles'] = {'desc': 'Profiles with variable altitude.'}
meta['profile_height'] = {'value_min': 0, 'value_max': 14, 'fill': -1,
'desc': 'Altitude of profile data.'}
meta['variable_profile_height'] = {'long_name': 'Variable Profile Height'}
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 581f44446..6f8e87842 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -17,6 +17,8 @@
tags = {'': 'Regular testing data set'}
inst_ids = {'': ['']}
_test_dates = {'': {'': dt.datetime(2009, 1, 1)}}
+_test_load_opt = {'': {'': [{'num_extra_time_coords': 0},
+ {'num_extra_time_coords': 1}]}}
epoch_name = u'time'
@@ -27,12 +29,14 @@
clean = mm_test.clean
# Optional method, preprocess
+concat_data = mm_test.concat_data
preprocess = mm_test.preprocess
def load(fnames, tag='', inst_id='', non_monotonic_index=False,
non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=864, test_load_kwarg=None, max_latitude=90.):
+ num_samples=864, test_load_kwarg=None, max_latitude=90.0,
+ num_extra_time_coords=0):
"""Load the test files.
Parameters
@@ -65,7 +69,9 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
correctly. (default=None)
max_latitude : float
Latitude simulated as `max_latitude` * cos(theta(t))`, where
- theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.).
+ theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.0)
+ num_extra_time_coords : int
+ Number of extra time coordinates to include. (default=0)
Returns
-------
@@ -169,6 +175,14 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
data.coords['y'] = (('y'), np.arange(17))
data.coords['z'] = (('z'), np.arange(15))
+ # Add extra time coords
+ for i in range(num_extra_time_coords):
+ ckey = 'time{:d}'.format(i)
+ tindex = data.indexes[epoch_name][:-1 * (i + 1)]
+ data.coords[ckey] = (
+ (ckey), [itime + dt.timedelta(microseconds=1 + i)
+ for i, itime in enumerate(tindex)])
+
# Create altitude 'profile' at each location to simulate remote data
num = len(data['uts'])
data['profiles'] = (
@@ -188,14 +202,20 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
data['images'] = ((epoch_name, 'x', 'y'),
data['dummy3'].values[
:, np.newaxis, np.newaxis] * np.ones((num, 17, 17)))
- data.coords['image_lat'] = \
- ((epoch_name, 'x', 'y'),
- np.arange(17)[np.newaxis,
- np.newaxis,
- :] * np.ones((num, 17, 17)))
+ data.coords['image_lat'] = ((epoch_name, 'x', 'y'),
+ np.arange(17)[np.newaxis, np.newaxis, :]
+ * np.ones((num, 17, 17)))
data.coords['image_lon'] = ((epoch_name, 'x', 'y'),
- np.arange(17)[np.newaxis, np.newaxis,
- :] * np.ones((num, 17, 17)))
+ np.arange(17)[np.newaxis, np.newaxis, :]
+ * np.ones((num, 17, 17)))
+
+ # There may be data that depends on alternate time indices
+ for i in range(num_extra_time_coords):
+ alt_epoch = 'time{:d}'.format(i)
+ data['variable_profiles{:d}'.format(i)] = (
+ (alt_epoch, 'z'), np.full(shape=(data.coords[alt_epoch].shape[0],
+ data.coords['z'].shape[0]),
+ fill_value=100.0 + i))
meta = mm_test.initialize_test_meta(epoch_name, data.keys())
return data, meta
From fcf146236df0d9931c7a9e34e77ef2f0d3a87ef3 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 25 Jul 2023 12:59:33 -0400
Subject: [PATCH 037/365] BUG: fixed custom concat data function
Fixed a bug in the custom concat data function, where the wrong xarray function was called.
---
pysat/instruments/methods/testing.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index e60be7171..2ab416b71 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -111,8 +111,11 @@ def concat_data(self, new_data, extra_time_dims=None, **kwargs):
new_data = pysat.utils.coords.expand_xarray_dims(
new_data, self.meta, exclude_dims=time_dims)
- # Combine the data
- self.data = xr.combine_by_coords(new_data, **kwargs)
+ # Specify the dimension, if not otherwise specified
+ if 'dim' not in kwargs:
+ kwargs['dim'] = self.index.name
+
+ self.data = xr.concat(new_data, **kwargs)
else:
inners = None
for ndata in new_data:
From 78e52074a6136db166dc699d1a367dc84ba50c63 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 25 Jul 2023 13:27:56 -0400
Subject: [PATCH 038/365] BUG: fixed extra time selection with limited range
bug
Fixed a bug encountered using pysatNASA TIMED GUVI SDR-spectrograph low-res data, in which the extra time dimensions span a fraction of the day, and so the pandas DatetimeIndex index finding can't work.
---
pysat/_instrument.py | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index ae04bb8e6..e5ee7cd17 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -950,7 +950,21 @@ def __getitem_xarray__(self, key):
return self.data.isel(**key_dict)
except (KeyError, TypeError):
# Try to get a subset of time, using label based indexing
- return self.data.sel(**key_dict)
+ try:
+ return self.data.sel(**key_dict)
+ except KeyError as kerr:
+ if str(kerr).find('Timestamp') >= 0 and len(
+ epoch_names) > 0:
+ # The problem is probably coming from a limited
+ # time range in the ancillery epochs, remove them
+ # from selection
+ pysat.logger.warning(
+ ''.join(['Removing ', repr(epoch_names[1:]),
+ ' dimensions from data selection']))
+ key_dict = {'indexers': {epoch_names[0]: key}}
+ return self.data.sel(**key_dict)
+ else:
+ raise kerr
def __setitem__(self, key, new_data):
"""Set data in `pysat.Instrument` object.
From 206da7f5ab50960980b658c9522b116f3257adb4 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 25 Jul 2023 13:29:25 -0400
Subject: [PATCH 039/365] STY: removed extra whitespace
Removed extra whitespace.
---
pysat/instruments/methods/testing.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index 2ab416b71..8ce6761ea 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -114,7 +114,7 @@ def concat_data(self, new_data, extra_time_dims=None, **kwargs):
# Specify the dimension, if not otherwise specified
if 'dim' not in kwargs:
kwargs['dim'] = self.index.name
-
+
self.data = xr.concat(new_data, **kwargs)
else:
inners = None
From 251fc5a39fcb187b5059e1fa943f1e8e9b2f3c90 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 1 Aug 2023 13:55:42 -0400
Subject: [PATCH 040/365] TST: add multi-day load
---
pysat/tests/classes/cls_instrument_library.py | 53 +++++++++++++++++--
1 file changed, 49 insertions(+), 4 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 7429c7ff6..66d19aa94 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -74,7 +74,7 @@ def initialize_test_inst_and_date(inst_dict):
def load_and_set_strict_time_flag(test_inst, date, raise_error=False,
- clean_off=True):
+ clean_off=True, concat=False):
"""Load data and set the strict time flag if needed for other tests.
Parameters
@@ -89,11 +89,19 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False,
clean_off : bool
Turn off the clean method when re-loading data and testing the
strict time flag (default=True)
+ concat : bool
+ If True, load multiple days to concat. If False, load single day.
+ (default=False)
"""
+ kwargs = {'use_header': True}
+
+ if concat:
+ kwargs['end_date'] = date + dt.timedelta(days=1)
+
try:
- test_inst.load(date=date, use_header=True)
+ test_inst.load(date=date, **kwargs)
except Exception as err:
# Catch all potential input errors, and only ensure that the one caused
# by the strict time flag is prevented from occurring on future load
@@ -110,7 +118,7 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False,
# Evaluate the warning
with warnings.catch_warnings(record=True) as war:
- test_inst.load(date=date, use_header=True)
+ test_inst.load(date=date, **kwargs)
assert len(war) >= 1
categories = [war[j].category for j in range(len(war))]
@@ -362,7 +370,8 @@ def test_download(self, inst_dict):
dl_dict = inst_dict['user_info']
else:
dl_dict = {}
- test_inst.download(date, date, **dl_dict)
+ # Note this will download two consecutive days
+ test_inst.download(date, **dl_dict)
assert len(test_inst.files.files) > 0
return
@@ -415,6 +424,42 @@ def test_load(self, clean_level, inst_dict):
# Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
@pytest.mark.download
+ def test_load_multiple_days(self, inst_dict):
+ """Test that instruments load at each cleaning level.
+
+ Parameters
+ ----------
+ clean_level : str
+ Cleanliness level for loaded instrument data.
+ inst_dict : dict
+ Dictionary containing info to instantiate a specific instrument.
+ Set automatically from instruments['download'] when
+ `initialize_test_package` is run.
+
+ """
+
+ test_inst, date = initialize_test_inst_and_date(inst_dict)
+ if len(test_inst.files.files) > 0:
+ # Set the clean level
+ target = 'Fake Data to be cleared'
+ test_inst.data = [target]
+
+ # Make sure the strict time flag doesn't interfere with
+ # the load tests, and re-run with desired clean level
+ load_and_set_strict_time_flag(test_inst, date, raise_error=True,
+ clean_off=False, concat=True)
+
+ # Make sure fake data is cleared
+ assert target not in test_inst.data
+
+ assert not test_inst.empty
+ else:
+ pytest.skip("Download data not available")
+
+ return
+
+ @pytest.mark.second
+ @pytest.mark.load_options
@pytest.mark.parametrize("clean_level", ['dirty', 'dusty', 'clean'])
def test_clean_warn(self, clean_level, inst_dict, caplog):
"""Test that appropriate warnings and errors are raised when cleaning.
From b1fa4244068aada26b3a5b850737ac8eb058c7f9 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 1 Aug 2023 13:56:01 -0400
Subject: [PATCH 041/365] STY: remove old marks
---
pysat/tests/classes/cls_instrument_library.py | 6 ------
pysat/tests/test_instruments.py | 9 ---------
2 files changed, 15 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 66d19aa94..5436312eb 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -376,10 +376,7 @@ def test_download(self, inst_dict):
return
@pytest.mark.second
- # Need to maintain download mark for backwards compatibility.
- # Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
- @pytest.mark.download
@pytest.mark.parametrize("clean_level", ['none', 'dirty', 'dusty', 'clean'])
def test_load(self, clean_level, inst_dict):
"""Test that instruments load at each cleaning level.
@@ -420,10 +417,7 @@ def test_load(self, clean_level, inst_dict):
return
@pytest.mark.second
- # Need to maintain download mark for backwards compatibility.
- # Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
- @pytest.mark.download
def test_load_multiple_days(self, inst_dict):
"""Test that instruments load at each cleaning level.
diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py
index d5cede4e2..a7eaabcc0 100644
--- a/pysat/tests/test_instruments.py
+++ b/pysat/tests/test_instruments.py
@@ -266,15 +266,6 @@ def test_old_initialize_inst_and_date(self):
testing.eval_warnings(war, self.warn_msgs)
return
- def test_old_pytest_mark_presence(self):
- """Test that pytest mark is backwards compatible."""
-
- n_args = len(InstLibTests.test_load.pytestmark)
- mark_names = [InstLibTests.test_load.pytestmark[j].name
- for j in range(0, n_args)]
-
- assert "download" in mark_names
-
@pytest.mark.parametrize("inst_module", ['pysat_testing2d',
'pysat_testing_xarray',
'pysat_testing2d_xarray'])
From 17b4ec4a1ac2e81ed57f478068e9f3d151e53208 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 1 Aug 2023 13:56:59 -0400
Subject: [PATCH 042/365] DOC: update changelog
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c8b5df1c0..26c8eb1b9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Added tests for warnings, logging messages, and errors in the Instrument
clean method.
* Allow Instruments to define custom `concat_data` methods.
+ * Added test for loading multiple files and concatenating.
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
From ab2bacf8c856aa456ead56d801792b54a4fa228a Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Wed, 2 Aug 2023 10:05:02 -0400
Subject: [PATCH 043/365] BUG: end date usage
---
pysat/tests/classes/cls_instrument_library.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 5436312eb..4bbba44ec 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -98,7 +98,7 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False,
kwargs = {'use_header': True}
if concat:
- kwargs['end_date'] = date + dt.timedelta(days=1)
+ kwargs['end_date'] = date + dt.timedelta(days=2)
try:
test_inst.load(date=date, **kwargs)
From 9554dff264b05448e8396088e6d74731ea186a50 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 2 Aug 2023 15:37:22 -0400
Subject: [PATCH 044/365] BUG: changed time trigger method
Changed additional time index to trigger on different tag instead of a load option.
---
pysat/instruments/pysat_ndtesting.py | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 6f8e87842..373815c68 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -14,11 +14,9 @@
name = 'ndtesting'
pandas_format = False
-tags = {'': 'Regular testing data set'}
-inst_ids = {'': ['']}
-_test_dates = {'': {'': dt.datetime(2009, 1, 1)}}
-_test_load_opt = {'': {'': [{'num_extra_time_coords': 0},
- {'num_extra_time_coords': 1}]}}
+tags = {'': 'Regular testing data set', 'two_times': 'Two time indices'}
+inst_ids = {'': [tag for tag in tags.keys()]}
+_test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}}
epoch_name = u'time'
@@ -35,8 +33,7 @@
def load(fnames, tag='', inst_id='', non_monotonic_index=False,
non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=864, test_load_kwarg=None, max_latitude=90.0,
- num_extra_time_coords=0):
+ num_samples=864, test_load_kwarg=None, max_latitude=90.0):
"""Load the test files.
Parameters
@@ -70,8 +67,6 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
max_latitude : float
Latitude simulated as `max_latitude` * cos(theta(t))`, where
theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.0)
- num_extra_time_coords : int
- Number of extra time coordinates to include. (default=0)
Returns
-------
@@ -176,6 +171,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
data.coords['z'] = (('z'), np.arange(15))
# Add extra time coords
+ num_extra_time_coords = 1 if tag == 'two_times' else 0
for i in range(num_extra_time_coords):
ckey = 'time{:d}'.format(i)
tindex = data.indexes[epoch_name][:-1 * (i + 1)]
From 2dcd549ccb8b667a90480048b6aff33cb96d92e1 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 2 Aug 2023 18:26:45 -0400
Subject: [PATCH 045/365] BUG: fixed concat call
The concat call was incorrect, fixed it.
---
pysat/instruments/pysat_ndtesting.py | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 373815c68..efd83aa53 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -26,11 +26,26 @@
# Clean method
clean = mm_test.clean
-# Optional method, preprocess
-concat_data = mm_test.concat_data
+# Optional methods
preprocess = mm_test.preprocess
+def concat_data(self, new_data, **kwargs):
+ """Concatonate data using the appropriate method by tag.
+
+ Parameters
+ ----------
+ new_data : list-like
+ List of xarray Datasets
+
+ """
+ extra_time_dims = ['time1'] if self.tag == 'two_times' else None
+ mm_test.concat_data(self, new_data, extra_time_dims=extra_time_dims,
+ **kwargs)
+
+ return
+
+
def load(fnames, tag='', inst_id='', non_monotonic_index=False,
non_unique_index=False, malformed_index=False, start_time=None,
num_samples=864, test_load_kwarg=None, max_latitude=90.0):
From 13fa25880e9532853844aabe6279c0c5f2df1f91 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Wed, 2 Aug 2023 19:28:42 -0400
Subject: [PATCH 046/365] STY: assert multiple days
---
pysat/tests/classes/cls_instrument_library.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 4bbba44ec..8a57e451e 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -33,6 +33,7 @@ class TestInstruments(InstLibTests):
import datetime as dt
from importlib import import_module
import logging
+import numpy as np
import sys
import tempfile
import warnings
@@ -445,8 +446,8 @@ def test_load_multiple_days(self, inst_dict):
# Make sure fake data is cleared
assert target not in test_inst.data
-
- assert not test_inst.empty
+ # Make sure more than one day has been loaded
+ assert len(np.unique(test_inst.index.day)) > 1
else:
pytest.skip("Download data not available")
From d2837f1e6b168ac28888a8fcb36f0ef7d7e8ad0a Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 3 Aug 2023 11:16:25 -0400
Subject: [PATCH 047/365] TST: added new load test
Added a load test with padded data to the general instruments library of tests, to ensure that each Instrument data set runs the `concat_data` method.
---
CHANGELOG.md | 1 +
pysat/tests/classes/cls_instrument_library.py | 58 +++++++++++++++++++
2 files changed, 59 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c8b5df1c0..f887c593c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* New Features
* Added tests for warnings, logging messages, and errors in the Instrument
clean method.
+ * Added loading test with padding for Instruments.
* Allow Instruments to define custom `concat_data` methods.
* Maintenance
* Update link redirects in docs.
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 7429c7ff6..a566bf5cd 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -37,6 +37,7 @@ class TestInstruments(InstLibTests):
import tempfile
import warnings
+import pandas as pds
import pytest
import pysat
@@ -512,6 +513,63 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
return
+ @pytest.mark.second
+ # Need to maintain download mark for backwards compatibility.
+ # Can remove once pysat 3.1.0 is released and libraries are updated.
+ @pytest.mark.load_options
+ @pytest.mark.download
+ @pytest.mark.parametrize('pad', [pds.DateOffset(days=1), {'days': 1}, None,
+ dt.timedelta(days=1)])
+ def test_load_w_pad(self, pad, inst_dict):
+ """Test that instruments load at each cleaning level.
+
+ Parameters
+ ----------
+ pad : pds.DateOffset, dict, or NoneType
+ Valid pad value for initializing an instrument
+ inst_dict : dict
+ Dictionary containing info to instantiate a specific instrument.
+ Set automatically from instruments['download'] when
+ `initialize_test_package` is run.
+
+ """
+ # Update the Instrument dict with the desired pad
+ if 'kwargs' in inst_dict.keys():
+ inst_dict['kwargs']['pad'] = pad
+ else:
+ inst_dict['kwargs'] = {'pad': pad}
+
+ # Assign the expected representation
+ if type(pad) in [dict]:
+ pad_repr = repr(pds.DateOffset(days=1))
+ elif type(pad) in [dt.timedelta]:
+ pad_repr = "1 day, 0:00:00"
+ else:
+ pad_repr = repr(pad)
+
+ test_inst, date = initialize_test_inst_and_date(inst_dict)
+ if len(test_inst.files.files) > 0:
+ # Make sure the strict time flag doesn't interfere with
+ # the load tests
+ load_and_set_strict_time_flag(test_inst, date, raise_error=True,
+ clean_off=False)
+
+ assert not test_inst.empty
+
+ # Evaluate the data index length
+ assert (test_inst.index[-1]
+ - test_inst.index[0]).total_seconds() < 86400.0
+
+ # Evaluate the recorded pad
+ inst_str = test_inst.__str__()
+ assert inst_str.find(
+ 'Data Padding: {:s}'.format(pad_repr)) > 0, "".join([
+ "bad pad value: ", pad_repr, " not in ", inst_str])
+ else:
+ pytest.skip("Download data not available")
+
+ return
+
@pytest.mark.download
def test_remote_file_list(self, inst_dict):
"""Test if optional list_remote_files routine exists and is callable.
From cd51df70176cbd51a903102e474d78fd5bb5ecca Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 4 Aug 2023 10:08:33 -0400
Subject: [PATCH 048/365] TST: reduce number of tests
Reduce the number of tests by removing two of the padding options.
---
pysat/tests/classes/cls_instrument_library.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index a566bf5cd..a241659b6 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -518,8 +518,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
@pytest.mark.download
- @pytest.mark.parametrize('pad', [pds.DateOffset(days=1), {'days': 1}, None,
- dt.timedelta(days=1)])
+ @pytest.mark.parametrize('pad', [{'day': 1}, dt.timedelta(days=1)])
def test_load_w_pad(self, pad, inst_dict):
"""Test that instruments load at each cleaning level.
From 9f600fad649546f4ab173c7bb880ec60e2c39b3a Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 4 Aug 2023 14:52:39 -0400
Subject: [PATCH 049/365] BUG: fixed dict input
Fixed the dict input key name.
---
pysat/tests/classes/cls_instrument_library.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index a241659b6..3f62d442a 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -518,7 +518,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
@pytest.mark.download
- @pytest.mark.parametrize('pad', [{'day': 1}, dt.timedelta(days=1)])
+ @pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)])
def test_load_w_pad(self, pad, inst_dict):
"""Test that instruments load at each cleaning level.
From 44ecbe9640cdc0cf54dce058ee2c036e89818cfb Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 4 Aug 2023 16:17:46 -0400
Subject: [PATCH 050/365] REV: change trigger method
Change trigger method for extra time dimensions back to load options to reduce number of unit tests that need to run.
---
pysat/instruments/pysat_ndtesting.py | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index efd83aa53..71038f0f3 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -14,9 +14,11 @@
name = 'ndtesting'
pandas_format = False
-tags = {'': 'Regular testing data set', 'two_times': 'Two time indices'}
+tags = {'': 'Regular testing data set'}
inst_ids = {'': [tag for tag in tags.keys()]}
_test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}}
+_test_load_opt = {'': {'': [{'num_extra_time_coords': 0},
+ {'num_extra_time_coords': 1}]}}
epoch_name = u'time'
@@ -39,7 +41,7 @@ def concat_data(self, new_data, **kwargs):
List of xarray Datasets
"""
- extra_time_dims = ['time1'] if self.tag == 'two_times' else None
+ extra_time_dims = ['time1'] if 'time1' in self.variables else None
mm_test.concat_data(self, new_data, extra_time_dims=extra_time_dims,
**kwargs)
@@ -48,7 +50,8 @@ def concat_data(self, new_data, **kwargs):
def load(fnames, tag='', inst_id='', non_monotonic_index=False,
non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=864, test_load_kwarg=None, max_latitude=90.0):
+ num_samples=864, test_load_kwarg=None, max_latitude=90.0,
+ num_extra_time_coords=0):
"""Load the test files.
Parameters
@@ -82,6 +85,8 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
max_latitude : float
Latitude simulated as `max_latitude` * cos(theta(t))`, where
theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.0)
+ num_extra_time_coords : int
+ Number of extra time coordinates to include. (default=0)
Returns
-------
@@ -186,7 +191,6 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
data.coords['z'] = (('z'), np.arange(15))
# Add extra time coords
- num_extra_time_coords = 1 if tag == 'two_times' else 0
for i in range(num_extra_time_coords):
ckey = 'time{:d}'.format(i)
tindex = data.indexes[epoch_name][:-1 * (i + 1)]
From b62e1ab86c6d8fbcb05719079c35aa7b26edd40d Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 4 Aug 2023 17:50:04 -0400
Subject: [PATCH 051/365] BUG: fixed bug in concat method
Changed the method of selecting the extra time variable names in the concat_data method.
---
pysat/instruments/pysat_ndtesting.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 71038f0f3..63668a197 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -41,9 +41,13 @@ def concat_data(self, new_data, **kwargs):
List of xarray Datasets
"""
- extra_time_dims = ['time1'] if 'time1' in self.variables else None
- mm_test.concat_data(self, new_data, extra_time_dims=extra_time_dims,
- **kwargs)
+ # Select the extra time variable names
+ time_vars = [var for var in self.variables
+ if var.find('time') == 0 and var != 'time']
+ if len(time_vars) == 0:
+ time_vars = None
+
+ mm_test.concat_data(self, new_data, extra_time_dims=time_vars, **kwargs)
return
From bdd320970b2b5219516677201a73f9e501b93ae5 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 7 Aug 2023 13:38:22 -0400
Subject: [PATCH 052/365] STY: improved concat definition
Improved the way the custom concat method is called, simplifying it so that a wrapper is not needed. Also made sure the same kwarg is needed for this function and load, which expands the number of lines covered in the Instrument class.
---
pysat/instruments/methods/testing.py | 19 ++++++----
pysat/instruments/pysat_ndtesting.py | 54 ++++++++++++----------------
2 files changed, 35 insertions(+), 38 deletions(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index 8ce6761ea..cd3354642 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -84,24 +84,31 @@ def clean(self, test_clean_kwarg=None):
# Optional methods
-def concat_data(self, new_data, extra_time_dims=None, **kwargs):
+def concat_data(self, new_data, num_extra_time_coords=0, **kwargs):
"""Concatonate data to self.data for extra time dimensions.
Parameters
----------
new_data : xarray.Dataset or list of such objects
New data objects to be concatonated
- extra_time_dims : list-like or NoneType
- List of extra time dimensions that require concatonation (default=None)
+ num_extra_time_coords : int
+ Number of extra time dimensions that require concatonation (default=0)
**kwargs : dict
Optional keyword arguments passed to xr.concat
+ Note
+ ----
+ Expects the extra time dimensions to have a variable name that starts
+ with 'time', and no other dimensions to have a name that fits this format.
+
"""
# Establish the time dimensions
- time_dims = [self.index.name]
+ time_dims = [var for var in self.variables if var.find('time') == 0]
- if extra_time_dims is not None:
- time_dims.extend(list(extra_time_dims))
+ if len(time_dims) != num_extra_time_coords + 1:
+ raise ValueError(
+ 'unexpected number of time dimensions: len({:}) != {:d}'.format(
+ time_dims, num_extra_time_coords + 1))
# Concatonate using the appropriate method for the number of time
# dimensions
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 63668a197..7a63d54ba 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -29,29 +29,10 @@
clean = mm_test.clean
# Optional methods
+concat_data = mm_test.concat_data
preprocess = mm_test.preprocess
-def concat_data(self, new_data, **kwargs):
- """Concatonate data using the appropriate method by tag.
-
- Parameters
- ----------
- new_data : list-like
- List of xarray Datasets
-
- """
- # Select the extra time variable names
- time_vars = [var for var in self.variables
- if var.find('time') == 0 and var != 'time']
- if len(time_vars) == 0:
- time_vars = None
-
- mm_test.concat_data(self, new_data, extra_time_dims=time_vars, **kwargs)
-
- return
-
-
def load(fnames, tag='', inst_id='', non_monotonic_index=False,
non_unique_index=False, malformed_index=False, start_time=None,
num_samples=864, test_load_kwarg=None, max_latitude=90.0,
@@ -190,9 +171,9 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
dtype=np.int64))
# Add dummy coords
- data.coords['x'] = (('x'), np.arange(17))
- data.coords['y'] = (('y'), np.arange(17))
- data.coords['z'] = (('z'), np.arange(15))
+ data.coords['x'] = (('x'), np.arange(7))
+ data.coords['y'] = (('y'), np.arange(7))
+ data.coords['z'] = (('z'), np.arange(5))
# Add extra time coords
for i in range(num_extra_time_coords):
@@ -206,27 +187,36 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
num = len(data['uts'])
data['profiles'] = (
(epoch_name, 'profile_height'),
- data['dummy3'].values[:, np.newaxis] * np.ones((num, 15)))
- data.coords['profile_height'] = ('profile_height', np.arange(15))
+ data['dummy3'].values[:, np.newaxis] * np.ones(
+ (num, data.coords['z'].shape[0])))
+ data.coords['profile_height'] = ('profile_height',
+ np.arange(len(data.coords['z'])))
# Profiles that could have different altitude values
data['variable_profiles'] = (
(epoch_name, 'z'), data['dummy3'].values[:, np.newaxis]
- * np.ones((num, 15)))
+ * np.ones((num, data.coords['z'].shape[0])))
data.coords['variable_profile_height'] = (
- (epoch_name, 'z'), np.arange(15)[np.newaxis, :] * np.ones((num, 15)))
+ (epoch_name, 'z'), np.arange(data.coords['z'].shape[0])[np.newaxis, :]
+ * np.ones((num, data.coords['z'].shape[0])))
# Create fake image type data, projected to lat / lon at some location
# from satellite.
data['images'] = ((epoch_name, 'x', 'y'),
data['dummy3'].values[
- :, np.newaxis, np.newaxis] * np.ones((num, 17, 17)))
+ :, np.newaxis, np.newaxis]
+ * np.ones((num, data.coords['x'].shape[0],
+ data.coords['y'].shape[0])))
data.coords['image_lat'] = ((epoch_name, 'x', 'y'),
- np.arange(17)[np.newaxis, np.newaxis, :]
- * np.ones((num, 17, 17)))
+ np.arange(data.coords['x'].shape[0])[
+ np.newaxis, np.newaxis, :]
+ * np.ones((num, data.coords['x'].shape[0],
+ data.coords['y'].shape[0])))
data.coords['image_lon'] = ((epoch_name, 'x', 'y'),
- np.arange(17)[np.newaxis, np.newaxis, :]
- * np.ones((num, 17, 17)))
+ np.arange(data.coords['x'].shape[0])[
+ np.newaxis, np.newaxis, :]
+ * np.ones((num, data.coords['x'].shape[0],
+ data.coords['y'].shape[0])))
# There may be data that depends on alternate time indices
for i in range(num_extra_time_coords):
From 965985411e2d79d87abf5ada9f78285db38fbd68 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 7 Aug 2023 13:44:47 -0400
Subject: [PATCH 053/365] TST: added unit test for ValueError
Added a unit test for the proper-use-check in the new concat_data method.
---
pysat/tests/test_methods_testing.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py
index 0885032a7..ead66c57b 100644
--- a/pysat/tests/test_methods_testing.py
+++ b/pysat/tests/test_methods_testing.py
@@ -83,3 +83,13 @@ def test_generate_times_kwargs(self, num, kwargs, output):
delta_time = [dt.timedelta(seconds=sec) for sec in uts]
assert (index.to_pydatetime() - delta_time == dates).all
return
+
+ def test_concat_bad_kwarg(self):
+ """Test `concat_data` errors with bad kwarg input."""
+
+ testing.eval_bad_input(mm_test.concat_data, ValueError,
+ 'unexpected number of time dimensions',
+ input_args=(self.test_inst, self.test_inst),
+ input_kwargs={'num_extra_time_coords': 10})
+ return
+
From e2b1eb9eaeded55efe09330845699e6d06db9039 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 7 Aug 2023 13:47:34 -0400
Subject: [PATCH 054/365] STY: removed whitespace
Removed extra whitespace.
---
pysat/tests/test_methods_testing.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py
index ead66c57b..8ff6c2140 100644
--- a/pysat/tests/test_methods_testing.py
+++ b/pysat/tests/test_methods_testing.py
@@ -92,4 +92,3 @@ def test_concat_bad_kwarg(self):
input_args=(self.test_inst, self.test_inst),
input_kwargs={'num_extra_time_coords': 10})
return
-
From 39fa2424094bd5ec030dcb6fa28298619b570cab Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 7 Aug 2023 14:52:05 -0400
Subject: [PATCH 055/365] BUG: fixed index name handling
Fixed potential issues with the index name handling in the custom concat_data method.
---
pysat/instruments/methods/testing.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index cd3354642..e86e0550d 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -102,8 +102,11 @@ def concat_data(self, new_data, num_extra_time_coords=0, **kwargs):
with 'time', and no other dimensions to have a name that fits this format.
"""
- # Establish the time dimensions
- time_dims = [var for var in self.variables if var.find('time') == 0]
+ # Establish the time dimensions, ensuring the standard variable is included
+ # whether or not it is treated as a variable
+ time_dims = [self.index.name]
+ time_dims.extend([var for var in self.variables if var.find('time') == 0
+ and var != self.index.name])
if len(time_dims) != num_extra_time_coords + 1:
raise ValueError(
From 99b3f7bd3330b061b19e43571ab5ab524dcca5e9 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Mon, 7 Aug 2023 15:16:47 -0400
Subject: [PATCH 056/365] MAINT: move to pyproject
---
pyproject.toml | 92 +++++++++++++++++++++++++++++++++++++++++++++++
pysat/__init__.py | 17 +++++----
pysat/version.txt | 1 -
setup.cfg | 65 +--------------------------------
4 files changed, 103 insertions(+), 72 deletions(-)
create mode 100644 pyproject.toml
delete mode 100644 pysat/version.txt
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..daffc9cbe
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,92 @@
+[build-system]
+requires = ["setuptools", "pip >= 10"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pysat"
+version = "3.1.0"
+description = "Supports science analysis across disparate data platforms"
+readme = "README.md"
+requires-python = ">=3.6"
+license = {file = "LICENSE"}
+authors = [
+ {name = "Russell Stoneback, et al.", email = "pysat.developers@gmail.com"},
+]
+classifiers = [
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Science/Research",
+ "Topic :: Scientific/Engineering :: Astronomy",
+ "Topic :: Scientific/Engineering :: Physics",
+ "Topic :: Scientific/Engineering :: Atmospheric Science",
+ "License :: OSI Approved :: BSD License",
+ "Natural Language :: English",
+ "Programming Language :: Python :: 3.6",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Operating System :: MacOS :: MacOS X",
+ "Operating System :: POSIX :: Linux",
+ "Operating System :: Microsoft :: Windows"
+]
+keywords = [
+ "pysat",
+ "ionosphere",
+ "atmosphere",
+ "thermosphere",
+ "magnetosphere",
+ "heliosphere",
+ "observations",
+ "models",
+ "space",
+ "satellites",
+ "analysis"
+]
+dependencies = [
+ "dask",
+ "netCDF4",
+ "numpy >= 1.12",
+ "pandas",
+ "portalocker",
+ "pytest",
+ "scipy",
+ "toolz",
+ "xarray"
+]
+
+[project.optional-dependencies]
+test = [
+ "coveralls < 3.3",
+ "flake8",
+ "flake8-docstrings",
+ "hacking >= 1.0, 6.0",
+ "pysatSpaceWeather",
+ "pytest",
+ "pytest-cov",
+ "pytest-ordering"
+]
+doc = [
+ "extras_require",
+ "ipython",
+ "m2r2",
+ "numpydoc",
+ "sphinx<7.0",
+ "sphinx_rtd_theme"
+]
+
+[project.urls]
+Documentation = "https://pysat.readthedocs.io/en/latest/"
+Source = "https://github.com/pysat/pysat"
+
+[tool.coverage.report]
+omit = ["*/instruments/templates/"]
+
+[tool.pytest.ini_options]
+addopts = "-vs --cov=pysatNASA"
+markers = [
+ "all_inst",
+ "download",
+ "no_download",
+ "load_options",
+ "first",
+ "second"
+]
diff --git a/pysat/__init__.py b/pysat/__init__.py
index 122b541bc..f8a110a90 100644
--- a/pysat/__init__.py
+++ b/pysat/__init__.py
@@ -34,9 +34,10 @@
"""
+import importlib
+
import logging
import os
-from portalocker import Lock
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
@@ -48,9 +49,13 @@
# Import and set user and pysat parameters object
from pysat import _params
-# set version
-here = os.path.abspath(os.path.dirname(__file__))
-version_filename = os.path.join(here, 'version.txt')
+# Set version
+try:
+ __version__ = importlib.metadata.version('pysat')
+except AttributeError:
+ # Python 3.6 requires a different version
+ import importlib_metadata
+ __version__ = importlib_metadata.version('pysat')
# Get home directory
home_dir = os.path.expanduser('~')
@@ -59,6 +64,7 @@
pysat_dir = os.path.join(home_dir, '.pysat')
# Set directory for test data
+here = os.path.abspath(os.path.dirname(__file__))
test_data_path = os.path.join(here, 'tests', 'test_data')
# Create a .pysat directory or parameters file if one doesn't exist.
@@ -96,9 +102,6 @@
# Load up existing parameters file
params = _params.Parameters()
-# Load up version information
-with Lock(version_filename, 'r', params['file_timeout']) as version_file:
- __version__ = version_file.read().strip()
from pysat._files import Files
from pysat._instrument import Instrument
diff --git a/pysat/version.txt b/pysat/version.txt
deleted file mode 100644
index fd2a01863..000000000
--- a/pysat/version.txt
+++ /dev/null
@@ -1 +0,0 @@
-3.1.0
diff --git a/setup.cfg b/setup.cfg
index 579ecfbc8..84e7c49c8 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,60 +1,6 @@
[metadata]
name = pysat
-version = file: pysat/version.txt
-url = https://github.com/pysat/pysat
-author = Russell Stoneback, et al.
-author_email = pysat.developers@gmail.com
-description = 'Supports science analysis across disparate data platforms'
-keywords =
- pysat
- ionosphere
- atmosphere
- thermosphere
- magnetosphere
- heliosphere
- observations
- models
- space
- satellites
- analysis
-classifiers =
- Development Status :: 5 - Production/Stable
- Intended Audience :: Science/Research
- Topic :: Scientific/Engineering :: Astronomy
- Topic :: Scientific/Engineering :: Physics
- Topic :: Scientific/Engineering :: Atmospheric Science
- License :: OSI Approved :: BSD License
- Natural Language :: English
- Programming Language :: Python :: 3.6
- Programming Language :: Python :: 3.8
- Programming Language :: Python :: 3.9
- Programming Language :: Python :: 3.10
- Operating System :: MacOS :: MacOS X
- Operating System :: POSIX :: Linux
- Operating System :: Microsoft :: Windows
-license_file = LICENSE
-long_description = file: README.md
-long_description_content_type = text/markdown
-
-[options]
-python_requires = >= 3.6
-setup_requires = setuptools >= 38.6; pip >= 10
-include_package_data = True
-zip_safe = False
-packages = find:
-install_requires = dask
- netCDF4
- numpy
- pandas
- portalocker
- pytest
- scipy
- toolz
- xarray
-
-[coverage:report]
-omit =
- */instruments/templates/*
+version = 3.1.0
[flake8]
max-line-length = 80
@@ -65,12 +11,3 @@ ignore =
pysat/__init__.py E402 F401
pysat/instruments/methods/__init__.py F401
pysat/utils/__init__.py F401
-
-[tool:pytest]
-markers =
- all_inst: tests all instruments
- download: tests for downloadable instruments
- no_download: tests for instruments without download support
- load_options: tests for instruments including optional load kwargs
- first: first tests to run
- second: second tests to run
From 404ae8bd572b222e43457713a70b17fb64cfda93 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Mon, 7 Aug 2023 15:17:20 -0400
Subject: [PATCH 057/365] MAINT: update test install syntax
---
.github/workflows/docs.yml | 7 +++----
.github/workflows/main.yml | 33 ++++++++++++++++++++-------------
.github/workflows/stats.yml | 7 ++-----
.readthedocs.yml | 28 +++++++++++++---------------
4 files changed, 38 insertions(+), 37 deletions(-)
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 39a113b76..153cfc945 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -8,11 +8,11 @@ on: [push, pull_request]
jobs:
build:
- runs-on: ubuntu-latest
+ runs-on: "ubuntu-latest"
strategy:
fail-fast: false
matrix:
- python-version: [3.9]
+ python-version: ["3.10"]
name: Documentation tests
steps:
@@ -25,8 +25,7 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
- pip install -r test_requirements.txt
- pip install -r requirements.txt
+ pip install .[doc]
- name: Set up pysat
run: |
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 7f577605e..51489005d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,15 +11,18 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
- python-version: ["3.9", "3.10"]
+ python-version: ["3.10", "3.11"]
numpy_ver: ["latest"]
+ test_config: ["latest"]
include:
- - python-version: "3.8"
- numpy_ver: "1.20"
+ - python-version: "3.9"
+ numpy_ver: "1.21"
os: ubuntu-latest
+ test_config: "NEP29"
- python-version: "3.6.8"
numpy_ver: "1.19.5"
os: "ubuntu-20.04"
+ test_config: "Ops"
name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }}
runs-on: ${{ matrix.os }}
@@ -30,19 +33,23 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- - name: Install NEP29/Operational dependencies
- if: ${{ matrix.numpy_ver != 'latest'}}
+ - name: Install Operational dependencies
+ if: ${{ matrix.test_config == 'Ops'}}
run: |
- pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }}
+ pip install --no-cache-dir numpy==${{ matrix.numpy_ver }}
+ pip install -r requirements.txt
+ pip install -r test_requirements.txt
+ pip install .
- - name: Install standard dependencies and pysat
+ - name: Install NEP29 dependencies
+ if: ${{ matrix.test_config == 'NEP29'}}
run: |
- python -m pip install --upgrade pip
- pip install -r requirements.txt
- python setup.py install
+ pip install numpy==${{ matrix.numpy_ver }}
+ pip install --upgrade-strategy only-if-needed .[test]
- - name: Install requirements for testing setup
- run: pip install -r test_requirements.txt
+ - name: Install standard dependencies
+ if: ${{ matrix.test_config == 'latest'}}
+ run: pip install .[test]
- name: Set up pysat
run: |
@@ -61,4 +68,4 @@ jobs:
- name: Publish results to coveralls
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: coveralls --rcfile=setup.cfg --service=github
+ run: coveralls --rcfile=pyproject.toml --service=github
diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml
index 6cde36c2f..f22ad9ab3 100644
--- a/.github/workflows/stats.yml
+++ b/.github/workflows/stats.yml
@@ -13,7 +13,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- python-version: ["3.9"]
+ python-version: ["3.11"]
name: Summary of instrument libraries
steps:
@@ -25,10 +25,7 @@ jobs:
- name: Install dependencies
run: |
- pip install -r requirements.txt
- # Need to install pysat before ecosystem
- python setup.py install
- pip install -r test_requirements.txt
+ pip install .test
pip install `awk '{print $1}' ecosystem.txt`
- name: Set up pysat
diff --git a/.readthedocs.yml b/.readthedocs.yml
index defa293da..125f0b3de 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -5,22 +5,20 @@
# Required
version: 2
+build:
+ os: ubuntu-22.04
+ tools:
+ python: "3.10"
+
# Build documentation in the docs/ directory with Sphinx
sphinx:
- configuration: docs/conf.py
-
-# Build documentation with MkDocs
-#mkdocs:
-# configuration: mkdocs.yml
+ configuration: docs/conf.py
-# Optionally build your docs in additional formats such as PDF
-formats:
- - pdf
-
-# Optionally set the version of Python and requirements
-# required to build your docs
+# Declare the Python requirements required to build your docs
+# This method includes a local build of the package
python:
- version: 3.7
- install:
- - requirements: docs/requirements.txt
-
+ install:
+ - method: pip
+ path: .
+ extra_requirements:
+ - doc
From 3b86de009eb74f95879b042ea5668e4bf55042b5 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Mon, 7 Aug 2023 15:17:30 -0400
Subject: [PATCH 058/365] TST: add rc install test
---
.github/workflows/pip_rc_install.yml | 40 ++++++++++++++++++++++++++++
1 file changed, 40 insertions(+)
create mode 100644 .github/workflows/pip_rc_install.yml
diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml
new file mode 100644
index 000000000..a6dd2c0f2
--- /dev/null
+++ b/.github/workflows/pip_rc_install.yml
@@ -0,0 +1,40 @@
+# This workflow will install Python dependencies and the latest RC of pysatCDAAC from test pypi.
+# This test should be manually run before a pysatCDAAC RC is officially approved and versioned.
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Test install of latest RC from pip
+
+on: [push]
+
+jobs:
+ build:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+ python-version: ["3.11"] # Keep this version at the highest supported Python version
+
+ name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install standard dependencies
+ run: pip install -r requirements.txt
+
+ - name: Install pysat RC
+ run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat
+
+ - name: Set up pysat
+ run: |
+ mkdir pysatData
+ python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'"
+
+ - name: Check that install imports correctly
+ run: |
+ cd ..
+ python -c "import pysat; print(pysat.__version__)"
From 259c22e62614f68580ad9d8a56342246f0e7dd2b Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Mon, 7 Aug 2023 15:17:53 -0400
Subject: [PATCH 059/365] DOC: update changelog
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f920be4b3..6d2c53a8c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
+ * Implement pyproject to manage metadata
[3.1.0] - 2023-05-31
--------------------
From 853930b56456c738262f72fbeb2f4a453167ad47 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Mon, 7 Aug 2023 15:19:52 -0400
Subject: [PATCH 060/365] BUG: sphinx versions
---
pyproject.toml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index daffc9cbe..19a3e9b5e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -69,8 +69,8 @@ doc = [
"ipython",
"m2r2",
"numpydoc",
- "sphinx<7.0",
- "sphinx_rtd_theme"
+ "sphinx < 7.0",
+ "sphinx_rtd_theme >= 1.2.2"
]
[project.urls]
From de5ed552ee4ce91ab39b7ffcbec9a72724fa65d7 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Mon, 7 Aug 2023 15:21:07 -0400
Subject: [PATCH 061/365] MAINT: set to workflow dispatch
---
.github/workflows/pip_rc_install.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml
index a6dd2c0f2..c782985e0 100644
--- a/.github/workflows/pip_rc_install.yml
+++ b/.github/workflows/pip_rc_install.yml
@@ -4,7 +4,7 @@
name: Test install of latest RC from pip
-on: [push]
+on: [workflow_dispatch]
jobs:
build:
From f96541b920689430a834908351fe35d370dfa5a6 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Mon, 7 Aug 2023 15:22:33 -0400
Subject: [PATCH 062/365] BUG: dependency format
---
pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyproject.toml b/pyproject.toml
index 19a3e9b5e..ddb3e6e75 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,7 +58,7 @@ test = [
"coveralls < 3.3",
"flake8",
"flake8-docstrings",
- "hacking >= 1.0, 6.0",
+ "hacking >= 1.0, <6.0",
"pysatSpaceWeather",
"pytest",
"pytest-cov",
From 10f8da22721a35ab873eb6f9ebcc6c16bbb4a90e Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 8 Aug 2023 11:48:19 -0400
Subject: [PATCH 063/365] TST: reduced number of test samples
Changed the test frequency to 15 min instead of 100 sec.
---
pysat/instruments/pysat_ndtesting.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 7a63d54ba..c529e7af1 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -35,7 +35,7 @@
def load(fnames, tag='', inst_id='', non_monotonic_index=False,
non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=864, test_load_kwarg=None, max_latitude=90.0,
+ num_samples=96, test_load_kwarg=None, max_latitude=90.0,
num_extra_time_coords=0):
"""Load the test files.
@@ -62,7 +62,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
will begin at midnight. (default=None)
num_samples : int
Maximum number of times to generate. Data points will not go beyond the
- current day. (default=864)
+ current day. (default=96)
test_load_kwarg : any
Keyword used for pysat unit testing to ensure that functionality for
custom keywords defined in instrument support functions is working
@@ -89,8 +89,8 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
iperiod = mm_test.define_period()
drange = mm_test.define_range()
- # Using 100s frequency for compatibility with seasonal analysis unit tests
- uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='100S',
+ # Using 900s (15 min) frequency
+ uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='900S',
start_time=start_time)
# TODO(#1094): Remove in pysat 3.2.0
if malformed_index:
From d9692df06b7b0438b25f843fc9a4a07a5e6bad92 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 9 Aug 2023 14:17:19 -0400
Subject: [PATCH 064/365] MAINT: remove setup.py
---
setup.py | 16 ----------------
1 file changed, 16 deletions(-)
delete mode 100644 setup.py
diff --git a/setup.py b/setup.py
deleted file mode 100644
index dfba3800e..000000000
--- a/setup.py
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (C) 2020, Authors
-# Full license can be found in License.md
-# -----------------------------------------------------------------------------
-"""Setup routine for pysat.
-
-Note
-----
-package metadata stored in setup.cfg
-
-"""
-
-from setuptools import setup
-
-setup()
From 6be826103648d2dc795009876356c2be53334251 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 9 Aug 2023 14:17:34 -0400
Subject: [PATCH 065/365] DOC: update docs
---
README.md | 6 +++---
docs/dependency.rst | 12 ++++++------
docs/installation.rst | 14 +++++++-------
3 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/README.md b/README.md
index 30470e6cb..8db411201 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ JGR-Space Physics [Publication](https://doi.org/10.1029/2018JA025297)
[Citation Info](https://pysat.readthedocs.io/en/latest/citing.html)
-Come join us on Slack! An invitation to the pysat workspace is available
+Come join us on Slack! An invitation to the pysat workspace is available
in the 'About' section of the [pysat GitHub Repository.](https://github.com/pysat/pysat)
Development meetings are generally held fortnightly.
@@ -66,7 +66,7 @@ pip install pysat
```
git clone https://github.com/pysat/pysat.git
cd pysat
-python setup.py install
+pip install .
```
An advantage to installing through github is access to the development branches.
The latest bugfixes can be found in the `develop` branch. However, this branch
@@ -76,7 +76,7 @@ virtual environment or using `python setup.py develop`.
git clone https://github.com/pysat/pysat.git
cd pysat
git checkout develop
-python setup.py develop
+pip install -e .
```
* Note that pysat requires a number of packages for the install.
* dask
diff --git a/docs/dependency.rst b/docs/dependency.rst
index 7f0495328..f88438fba 100644
--- a/docs/dependency.rst
+++ b/docs/dependency.rst
@@ -31,8 +31,8 @@ could look something like:
| | |-- __init__.py
| | `-- test_instruments.py
| `-- __init__.py
- |-- README.md
- `-- setup.py
+ |-- pyproject.toml
+ `-- README.md
The instruments folder includes a file for each instrument object. The
@@ -53,7 +53,7 @@ suite of instrument tests to run on instruments. These are imported from the
``test_instruments.py`` file can be copied directly into the library, updating
the instrument library name as indicated.
-The ``setup.py`` file should include pysat as a dependency, as well as any
+The ``pyproject.toml`` file should include pysat as a dependency, as well as any
other packages required by the instruments.
A more complicated structure could include analysis routines,
@@ -84,8 +84,8 @@ The structure then could look like:
| | |-- compare.py
| | `-- contrast.py
| `-- __init__.py
- |-- README.md
- `-- setup.py
+ |-- pyproject.toml
+ `-- README.md
.. _pysat-dep-testinst:
@@ -330,7 +330,7 @@ leap days.
Tips and Tricks
---------------
-Remember to include pysat as a dependency in your setup.py or setup.cfg file.
+Remember to include pysat as a dependency in your pyproject.toml file.
The CI environment will also need to be configured to install pysat and its
dependencies. You may need to install pysat from github rather than pip if
diff --git a/docs/installation.rst b/docs/installation.rst
index afc3e029b..24104dfae 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -26,7 +26,7 @@ To use Anaconda's tools for creating a suitable virtual environment,
.. _inst-standard:
-
+
Standard installation
---------------------
@@ -57,7 +57,7 @@ pysat may also be installed directly from the source repository on github::
git clone https://github.com/pysat/pysat.git
cd pysat
- python setup.py install --user
+ pip install --user .
An advantage to installing through github is access to the development branches.
The latest bugfixes can be found in the ``develop`` branch. However, this
@@ -67,9 +67,9 @@ virtual environment and using::
git clone https://github.com/pysat/pysat.git
cd pysat
git checkout develop
- python setup.py develop
+ pip install -e .
-The use of `develop` rather than `install` in the setup command installs the
-code 'in-place', so any changes to the software do not have to be reinstalled
-to take effect. It is not related to changing the pysat working branch from
-``main`` to ``develop`` in the preceeding line.
+The use of `-e` in the setup command installs the code 'in-place', so any
+changes to the software do not have to be reinstalled to take effect. It is not
+related to changing the pysat working branch from ``main`` to ``develop`` in the
+preceeding line.
From 1e01596da53b513787f51a8715db6890b4cf8e31 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Thu, 10 Aug 2023 10:39:34 -0400
Subject: [PATCH 066/365] BUG: update omit
---
pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyproject.toml b/pyproject.toml
index ddb3e6e75..cb11257ed 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -78,7 +78,7 @@ Documentation = "https://pysat.readthedocs.io/en/latest/"
Source = "https://github.com/pysat/pysat"
[tool.coverage.report]
-omit = ["*/instruments/templates/"]
+omit = ["*/instruments/templates/*"]
[tool.pytest.ini_options]
addopts = "-vs --cov=pysatNASA"
From 8d6818e3d8b2eb3d3a6f221435777a5e6da51f74 Mon Sep 17 00:00:00 2001
From: jklenzing
Date: Thu, 10 Aug 2023 11:07:07 -0400
Subject: [PATCH 067/365] BUG: wrong package
---
pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyproject.toml b/pyproject.toml
index cb11257ed..745fcdc2f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -81,7 +81,7 @@ Source = "https://github.com/pysat/pysat"
omit = ["*/instruments/templates/*"]
[tool.pytest.ini_options]
-addopts = "-vs --cov=pysatNASA"
+addopts = "--cov=pysat"
markers = [
"all_inst",
"download",
From 5871b077ad4b17b55d6b2e51b97816bc95bba55f Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 7 Sep 2023 15:42:36 -0400
Subject: [PATCH 068/365] BUG: fixed load `_prev` and `_next` inputs
Ensure that kwargs passed directly into the load function are also passed into the supporting load functions: `_load_prev` and `_load_next`.
---
pysat/_instrument.py | 54 ++++++++++++++++++++++++++++++--------------
1 file changed, 37 insertions(+), 17 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index e5ee7cd17..002c57469 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -1598,7 +1598,7 @@ def _load_data(self, date=None, fid=None, inc=None, load_kwargs=None):
inc : dt.timedelta, int, or NoneType
Increment of files or dates to load, starting from the
root date or fid (default=None)
- load_kwargs : dict
+ load_kwargs : dict or NoneType
Dictionary of keywords that may be options for specific instruments.
If None, uses `self.kwargs['load']`. (default=None)
@@ -1709,9 +1709,15 @@ def _load_data(self, date=None, fid=None, inc=None, load_kwargs=None):
return data, mdata
- def _load_next(self):
+ def _load_next(self, load_kwargs=None):
"""Load the next days data (or file) without incrementing the date.
+ Parameters
+ ----------
+ load_kwargs : dict or NoneType
+ Dictionary of keywords that may be options for specific instruments.
+ If None, uses `self.kwargs['load']`. (default=None)
+
Returns
-------
data : pds.DataFrame or xr.Dataset
@@ -1728,16 +1734,24 @@ def _load_next(self):
or the file. Looks for `self._load_by_date` flag.
"""
+ load_data_kwargs = {'inc': self.load_step, 'load_kwargs': load_kwargs}
+
if self._load_by_date:
- next_date = self.date + self.load_step
- return self._load_data(date=next_date, inc=self.load_step)
+ load_data_kwargs['date'] = self.date + self.load_step
else:
- next_id = self._fid + self.load_step + 1
- return self._load_data(fid=next_id, inc=self.load_step)
+ load_data_kwargs['fid'] = self._fid + self.load_step + 1
+
+ return self._load_data(**load_data_kwargs)
- def _load_prev(self):
+ def _load_prev(self, load_kwargs=None):
"""Load the previous days data (or file) without decrementing the date.
+ Parameters
+ ----------
+ load_kwargs : dict or NoneType
+ Dictionary of keywords that may be options for specific instruments.
+ If None, uses `self.kwargs['load']`. (default=None)
+
Returns
-------
data : pds.DataFrame or xr.Dataset
@@ -1754,14 +1768,14 @@ def _load_prev(self):
or the file. Looks for `self._load_by_date` flag.
"""
- load_kwargs = {'inc': self.load_step}
+ load_data_kwargs = {'inc': self.load_step, 'load_kwargs': load_kwargs}
if self._load_by_date:
- load_kwargs['date'] = self.date - self.load_step
+ load_data_kwargs['date'] = self.date - self.load_step
else:
- load_kwargs['fid'] = self._fid - self.load_step - 1
+ load_data_kwargs['fid'] = self._fid - self.load_step - 1
- return self._load_data(**load_kwargs)
+ return self._load_data(**load_data_kwargs)
def _set_load_parameters(self, date=None, fid=None):
"""Set the necesssary load attributes.
@@ -3219,11 +3233,13 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
pysat.logger.debug('Initializing data cache.')
# Using current date or fid
- self._prev_data, self._prev_meta = self._load_prev()
+ self._prev_data, self._prev_meta = self._load_prev(
+ load_kwargs=kwargs)
self._curr_data, self._curr_meta = self._load_data(
date=self.date, fid=self._fid, inc=self.load_step,
load_kwargs=kwargs)
- self._next_data, self._next_meta = self._load_next()
+ self._next_data, self._next_meta = self._load_next(
+ load_kwargs=kwargs)
else:
if self._next_data_track == curr:
pysat.logger.debug('Using data cache. Loading next.')
@@ -3233,7 +3249,8 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
self._prev_meta = self._curr_meta
self._curr_data = self._next_data
self._curr_meta = self._next_meta
- self._next_data, self._next_meta = self._load_next()
+ self._next_data, self._next_meta = self._load_next(
+ load_kwargs=kwargs)
elif self._prev_data_track == curr:
pysat.logger.debug('Using data cache. Loading previous.')
# Moving backward in time
@@ -3242,7 +3259,8 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
self._next_meta = self._curr_meta
self._curr_data = self._prev_data
self._curr_meta = self._prev_meta
- self._prev_data, self._prev_meta = self._load_prev()
+ self._prev_data, self._prev_meta = self._load_prev(
+ load_kwargs=kwargs)
else:
# Jumped in time/or switched from filebased to date based
# access
@@ -3250,11 +3268,13 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
del self._prev_data
del self._curr_data
del self._next_data
- self._prev_data, self._prev_meta = self._load_prev()
+ self._prev_data, self._prev_meta = self._load_prev(
+ load_kwargs=kwargs)
self._curr_data, self._curr_meta = self._load_data(
date=self.date, fid=self._fid, inc=self.load_step,
load_kwargs=kwargs)
- self._next_data, self._next_meta = self._load_next()
+ self._next_data, self._next_meta = self._load_next(
+ load_kwargs=kwargs)
# Make sure datetime indices for all data is monotonic
if self.pandas_format:
From 2671e7e8f29f256bd7a1e2f7416d3d7e6eccb926 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 7 Sep 2023 15:43:37 -0400
Subject: [PATCH 069/365] STY: removed concat kwarg
Removed the concat kwarg that was used to verify correct input and the test that supported it.
---
pysat/instruments/methods/testing.py | 13 +++++--------
pysat/tests/test_methods_testing.py | 9 ---------
2 files changed, 5 insertions(+), 17 deletions(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index e86e0550d..2a4b6af15 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -84,15 +84,13 @@ def clean(self, test_clean_kwarg=None):
# Optional methods
-def concat_data(self, new_data, num_extra_time_coords=0, **kwargs):
+def concat_data(self, new_data, **kwargs):
"""Concatonate data to self.data for extra time dimensions.
Parameters
----------
new_data : xarray.Dataset or list of such objects
New data objects to be concatonated
- num_extra_time_coords : int
- Number of extra time dimensions that require concatonation (default=0)
**kwargs : dict
Optional keyword arguments passed to xr.concat
@@ -108,11 +106,6 @@ def concat_data(self, new_data, num_extra_time_coords=0, **kwargs):
time_dims.extend([var for var in self.variables if var.find('time') == 0
and var != self.index.name])
- if len(time_dims) != num_extra_time_coords + 1:
- raise ValueError(
- 'unexpected number of time dimensions: len({:}) != {:d}'.format(
- time_dims, num_extra_time_coords + 1))
-
# Concatonate using the appropriate method for the number of time
# dimensions
if len(time_dims) == 1:
@@ -292,6 +285,10 @@ def initialize_test_meta(epoch_name, data_keys):
if var.find('variable_profiles') == 0:
meta[var] = {'desc': 'Profiles with variable altitude.'}
+ if len(var) > 17:
+ tvar = 'time{:s}'.format(var[17:])
+ meta[tvar] = {'desc': 'Additional time variable.'}
+
# Standard metadata required for xarray.
meta['profile_height'] = {'value_min': 0, 'value_max': 14, 'fill': -1,
'desc': 'Altitude of profile data.'}
diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py
index 8ff6c2140..0885032a7 100644
--- a/pysat/tests/test_methods_testing.py
+++ b/pysat/tests/test_methods_testing.py
@@ -83,12 +83,3 @@ def test_generate_times_kwargs(self, num, kwargs, output):
delta_time = [dt.timedelta(seconds=sec) for sec in uts]
assert (index.to_pydatetime() - delta_time == dates).all
return
-
- def test_concat_bad_kwarg(self):
- """Test `concat_data` errors with bad kwarg input."""
-
- testing.eval_bad_input(mm_test.concat_data, ValueError,
- 'unexpected number of time dimensions',
- input_args=(self.test_inst, self.test_inst),
- input_kwargs={'num_extra_time_coords': 10})
- return
From 894f1c6cd41129e80c5defd1b40d6ae96b0bb1cf Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 7 Sep 2023 15:49:12 -0400
Subject: [PATCH 070/365] REV: reverted test instrument time step
Reverted the test instrument time step, since it didn't solve the problem and was set this way originally for compatibility with other tests.
---
pysat/instruments/pysat_ndtesting.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index c529e7af1..03fb0290f 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -89,8 +89,8 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
iperiod = mm_test.define_period()
drange = mm_test.define_range()
- # Using 900s (15 min) frequency
- uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='900S',
+ # Using 100s frequency for compatibility with seasonal analysis unit tests
+ uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='100S',
start_time=start_time)
# TODO(#1094): Remove in pysat 3.2.0
if malformed_index:
From 1ac853c39c3b94203f5e3dc18ba82db44cff9a35 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 7 Sep 2023 15:49:40 -0400
Subject: [PATCH 071/365] BUG: updated coordinate metadata
Updated the coordinate metadata to reflect the new, smaller size.
---
pysat/instruments/methods/testing.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index 2a4b6af15..8c0b2838f 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -299,13 +299,13 @@ def initialize_test_meta(epoch_name, data_keys):
'notes': 'function of image_lat and image_lon'}
meta['x'] = {'desc': 'x-value of image pixel',
'notes': 'Dummy Variable',
- 'value_min': 0, 'value_max': 17, 'fill': -1}
+ 'value_min': 0, 'value_max': 7, 'fill': -1}
meta['y'] = {'desc': 'y-value of image pixel',
'notes': 'Dummy Variable',
- 'value_min': 0, 'value_max': 17, 'fill': -1}
+ 'value_min': 0, 'value_max': 7, 'fill': -1}
meta['z'] = {'desc': 'z-value of profile height',
'notes': 'Dummy Variable',
- 'value_min': 0, 'value_max': 15, 'fill': -1}
+ 'value_min': 0, 'value_max': 5, 'fill': -1}
meta['image_lat'] = {'desc': 'Latitude of image pixel',
'notes': 'Dummy Variable',
'value_min': -90., 'value_max': 90.}
From 5adcad2a36c21b32caf566fd59c783c3d8e8fc77 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 8 Sep 2023 13:19:36 -0400
Subject: [PATCH 072/365] BUG: adding verbose flag
Temporarily adding verbose flag, to see where things are failing remotely.
---
.github/workflows/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 7f577605e..a76a98a00 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -56,7 +56,7 @@ jobs:
run: flake8 . --count --exit-zero --max-complexity=10 --statistics
- name: Test with pytest
- run: pytest --cov=pysat/
+ run: pytest -v --cov=pysat/
- name: Publish results to coveralls
env:
From 1ff5978084466c43545d1aa05ff4216be9633bec Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 10:13:14 -0400
Subject: [PATCH 073/365] MAINT: updated `to_inst` behaviour
Updated `to_inst` behaviour, only combining dimensions instead of coordinates that are not also dimensions.
---
CHANGELOG.md | 2 ++
pysat/_constellation.py | 34 ++++++++++++++++---------------
pysat/tests/test_constellation.py | 3 ---
3 files changed, 20 insertions(+), 19 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f887c593c..ebe685761 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
+ * Updated `Constellation.to_inst` method definition of coords, using dims
+ to combine common dimensions instead.
[3.1.0] - 2023-05-31
--------------------
diff --git a/pysat/_constellation.py b/pysat/_constellation.py
index 0252c7275..34e5866c9 100644
--- a/pysat/_constellation.py
+++ b/pysat/_constellation.py
@@ -670,22 +670,24 @@ def to_inst(self, common_coord=True, fill_method=None):
# Get the common coordinates needed for all data
for cinst in self.instruments:
if not cinst.pandas_format:
- for new_coord in cinst.data.coords.keys():
- if new_coord not in coords.keys():
- coords[new_coord] = cinst.data.coords[new_coord]
- elif new_coord != 'time':
- # Two instruments have the same coordinate, if they
- # are not identical, we need to establish a common
- # range and resolution. Note that this will only
- # happen if the coordinates share the same names.
- if(len(coords[new_coord])
- != len(cinst.data.coords[new_coord])
- or coords[new_coord].values
- != cinst.data.coords[new_coord].values):
- coords[new_coord] = establish_common_coord(
- [coords[new_coord].values,
- cinst.data.coords[new_coord].values],
- common=common_coord)
+ for new_coord in cinst.data.dims.keys():
+ if new_coord in cinst.data.coords.keys():
+ if new_coord not in coords.keys():
+ coords[new_coord] = cinst.data.coords[new_coord]
+ elif new_coord != 'time':
+ # Two instruments have the same coordinate, if
+ # they are not identical, we need to establish
+ # a common range and resolution. Note that
+ # this will only happen if the coordinates
+ # share the same names.
+ if(len(coords[new_coord])
+ != len(cinst.data.coords[new_coord])
+ or coords[new_coord].values
+ != cinst.data.coords[new_coord].values):
+ coords[new_coord] = establish_common_coord(
+ [coords[new_coord].values,
+ cinst.data.coords[new_coord].values],
+ common=common_coord)
data = xr.Dataset(coords=coords)
diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py
index 5dda00d34..5986b71e8 100644
--- a/pysat/tests/test_constellation.py
+++ b/pysat/tests/test_constellation.py
@@ -478,9 +478,6 @@ def test_to_inst_xarray(self, common_coord, fill_method):
testing.assert_lists_equal(self.dims, list(out_inst.data.dims.keys()))
testing.assert_list_contains(self.dims,
list(out_inst.data.coords.keys()))
- testing.assert_list_contains(['variable_profile_height', 'image_lon',
- 'image_lat'],
- list(out_inst.data.coords.keys()))
for cinst in self.const.instruments:
for var in cinst.variables:
From 37ce5d32050d3b73e77128300b2d6abd725884d0 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 10:15:08 -0400
Subject: [PATCH 074/365] REV: removed verbose flag
Removed the temporary setting of the verbosity flag.
---
.github/workflows/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a76a98a00..7f577605e 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -56,7 +56,7 @@ jobs:
run: flake8 . --count --exit-zero --max-complexity=10 --statistics
- name: Test with pytest
- run: pytest -v --cov=pysat/
+ run: pytest --cov=pysat/
- name: Publish results to coveralls
env:
From d1633c15d0765b465ba2278a542b9c66e64765d5 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 10:17:45 -0400
Subject: [PATCH 075/365] TST: added temporary skip for new test
Added a skip for the new test for Python 3.6.
---
pysat/tests/classes/cls_instrument_library.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 3f62d442a..c0c183cab 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -518,6 +518,8 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
@pytest.mark.download
+ @pytest.skipif(sys.version_info < (3, 7),
+ reason='See if everything else passes without this test')
@pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)])
def test_load_w_pad(self, pad, inst_dict):
"""Test that instruments load at each cleaning level.
From 9f854ca75ae58108fe6186fc675ceeb22e9d8787 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 10:22:48 -0400
Subject: [PATCH 076/365] REV: revert default times
Restore the old number of times, for support of other ecosystem packages.
---
pysat/instruments/pysat_ndtesting.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 03fb0290f..7a63d54ba 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -35,7 +35,7 @@
def load(fnames, tag='', inst_id='', non_monotonic_index=False,
non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=96, test_load_kwarg=None, max_latitude=90.0,
+ num_samples=864, test_load_kwarg=None, max_latitude=90.0,
num_extra_time_coords=0):
"""Load the test files.
@@ -62,7 +62,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
will begin at midnight. (default=None)
num_samples : int
Maximum number of times to generate. Data points will not go beyond the
- current day. (default=96)
+ current day. (default=864)
test_load_kwarg : any
Keyword used for pysat unit testing to ensure that functionality for
custom keywords defined in instrument support functions is working
From 1c8f3fa41545edddf06aacf7eb5be2c9376f9294 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 10:25:03 -0400
Subject: [PATCH 077/365] BUG: fixed skipif evaluation
Fixed the skipif usage.
---
pysat/tests/classes/cls_instrument_library.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index c0c183cab..8addb4730 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -518,7 +518,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
@pytest.mark.download
- @pytest.skipif(sys.version_info < (3, 7),
+ @pytest.skipif(sys.version_info.minor < 7,
reason='See if everything else passes without this test')
@pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)])
def test_load_w_pad(self, pad, inst_dict):
From 181d480e6bbb0b423ac3ebf3d2d338626eea443c Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 10:26:44 -0400
Subject: [PATCH 078/365] BUG: added missing `mark`
Added `mark` to the pytest line.
---
pysat/tests/classes/cls_instrument_library.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 8addb4730..68de0f7bd 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -518,8 +518,8 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
@pytest.mark.download
- @pytest.skipif(sys.version_info.minor < 7,
- reason='See if everything else passes without this test')
+ @pytest.mark.skipif(sys.version_info.minor < 7,
+ reason='See if everything else passes')
@pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)])
def test_load_w_pad(self, pad, inst_dict):
"""Test that instruments load at each cleaning level.
From 87c3c2a3efedc336b66c665f8b0acf6e3b96c946 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 15:38:28 -0400
Subject: [PATCH 079/365] STY: added kwarg to `__getitem__`
Added a data kwarg to `__getitem__` to streamline the data loading process.
---
pysat/_instrument.py | 125 +++++++++++++++++++++----------------------
1 file changed, 62 insertions(+), 63 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 002c57469..581ffe943 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -743,7 +743,7 @@ def __str__(self):
return output_str
- def __getitem__(self, key):
+ def __getitem__(self, key, data=None):
"""Access data in `pysat.Instrument` object.
Parameters
@@ -751,6 +751,8 @@ def __getitem__(self, key):
key : str, tuple, or dict
Data variable name, tuple with a slice, or dict used to locate
desired data.
+ data : pds.DataFrame, xr.Dataset, or NoneType
+ Desired data object to select from or None to use `data` attribute
Raises
------
@@ -789,20 +791,22 @@ def __getitem__(self, key):
inst[datetime1:datetime2, 'name1':'name2']
"""
+ if data is None:
+ data = self.data
if self.pandas_format:
if isinstance(key, str):
- return self.data[key]
+ return data[key]
elif isinstance(key, tuple):
try:
# Pass keys directly through
- return self.data.loc[key[0], key[1]]
+ return data.loc[key[0], key[1]]
except (KeyError, TypeError) as err1:
# TypeError for single integer. KeyError for list, array,
# slice of integers. Assume key[0] is integer
# (including list or slice).
try:
- return self.data.loc[self.data.index[key[0]], key[1]]
+ return data.loc[data.index[key[0]], key[1]]
except IndexError as err2:
err_message = '\n'.join(("original messages:",
str(err1), str(err2)))
@@ -812,15 +816,15 @@ def __getitem__(self, key):
else:
try:
# Integer based indexing
- return self.data.iloc[key]
+ return data.iloc[key]
except (TypeError, ValueError):
# If it's not an integer, TypeError is thrown. If it's a
# list, ValueError is thrown.
- return self.data[key]
+ return data[key]
else:
- return self.__getitem_xarray__(key)
+ return self.__getitem_xarray__(key, data=data)
- def __getitem_xarray__(self, key):
+ def __getitem_xarray__(self, key, data=None):
"""Access data in `pysat.Instrument` object with `xarray.Dataset`.
Parameters
@@ -828,6 +832,8 @@ def __getitem_xarray__(self, key):
key : str, tuple, or dict
Data variable name, tuple with a slice, or dict used to locate
desired data
+ data : xr.Dataset or NoneType
+ Desired data object to select from or None to use `data` attribute
Returns
-------
@@ -866,19 +872,21 @@ def __getitem_xarray__(self, key):
inst[datetime1:datetime2, 'name']
"""
+ if data is None:
+ data = self.data
- if 'Epoch' in self.data.indexes:
+ if 'Epoch' in data.indexes:
epoch_names = ['Epoch']
- elif 'time' in self.data.indexes:
+ elif 'time' in data.indexes:
epoch_names = ['time']
else:
return xr.Dataset(None)
# Find secondary time indexes that may need to be sliced
- if len(self.data.indexes) > 1:
- for ind in self.data.indexes.keys():
- if(ind != epoch_names[0] and self.data.indexes[ind].dtype
- == self.data.indexes[epoch_names[0]].dtype):
+ if len(data.indexes) > 1:
+ for ind in data.indexes.keys():
+ if(ind != epoch_names[0] and data.indexes[ind].dtype
+ == data.indexes[epoch_names[0]].dtype):
epoch_names.append(ind)
if isinstance(key, tuple):
@@ -886,10 +894,10 @@ def __getitem_xarray__(self, key):
# Support slicing time, variable name
if isinstance(key[1], slice):
# Extract subset of variables before epoch selection.
- data_subset = self.data[self.variables[key[1]]]
+ data_subset = data[self.variables[key[1]]]
else:
# Extract single variable before epoch selection.
- data_subset = self.data[key[1]]
+ data_subset = data[key[1]]
# If the input is a tuple, `key[0]` must be linked to the epoch.
key_dict = {'indexers': {epoch_name: key[0]
@@ -918,7 +926,7 @@ def __getitem_xarray__(self, key):
for i, dim in enumerate(self[var_name].dims):
indict[dim] = key[0][i]
- return self.data[var_name][indict]
+ return data[var_name][indict]
else:
# Multidimensional indexing where the multiple dimensions are
# not contained within another object
@@ -934,11 +942,11 @@ def __getitem_xarray__(self, key):
for i, dim in enumerate(self[var_name].dims):
indict[dim] = key[i]
- return self.data[var_name][indict]
+ return data[var_name][indict]
else:
try:
# Grab a particular variable by name
- return self.data[key]
+ return data[key]
except (TypeError, KeyError, ValueError):
# If that didn't work, likely need to use `isel` or `sel`
# Link key to the epoch.
@@ -947,11 +955,11 @@ def __getitem_xarray__(self, key):
try:
# Try to get all data variables, but for a subset of time
# using integer indexing
- return self.data.isel(**key_dict)
+ return data.isel(**key_dict)
except (KeyError, TypeError):
# Try to get a subset of time, using label based indexing
try:
- return self.data.sel(**key_dict)
+ return data.sel(**key_dict)
except KeyError as kerr:
if str(kerr).find('Timestamp') >= 0 and len(
epoch_names) > 0:
@@ -962,7 +970,7 @@ def __getitem_xarray__(self, key):
''.join(['Removing ', repr(epoch_names[1:]),
' dimensions from data selection']))
key_dict = {'indexers': {epoch_names[0]: key}}
- return self.data.sel(**key_dict)
+ return data.sel(**key_dict)
else:
raise kerr
@@ -3210,8 +3218,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
# Check for consistency between loading range and data padding, if any
if self.pad is not None:
if self._load_by_date:
- tdate = dt.datetime(2009, 1, 1)
- if tdate + self.load_step < tdate + loop_pad:
+ if date + self.load_step < date + loop_pad:
estr = ''.join(('Data padding window must be shorter than ',
'data loading window. Load a greater ',
'range of data or shorten the padding.'))
@@ -3262,7 +3269,7 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
self._prev_data, self._prev_meta = self._load_prev(
load_kwargs=kwargs)
else:
- # Jumped in time/or switched from filebased to date based
+ # Jumped in time/or switched from file based to date based
# access
pysat.logger.debug('Resetting data cache.')
del self._prev_data
@@ -3294,14 +3301,16 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
self._next_data = getattr(self._next_data,
sort_method)(*sort_args)
- # Make tracking indexes consistent with new loads
+ # Make tracking indexes consistent with new loads, as date loading
+ # and file loading have to be treated differently due to change in
+ # inclusive/exclusive range end treatment. Loading by file is
+ # inclusive.
if self._load_by_date:
+ # Arithmatic uses datetime or DateOffset objects
self._next_data_track = curr + self.load_step
self._prev_data_track = curr - self.load_step
else:
- # File and date loads have to be treated differently
- # due to change in inclusive/exclusive range end
- # treatment. Loading by file is inclusive.
+ # Aritmatic uses integers
self._next_data_track = curr + self.load_step + 1
self._prev_data_track = curr - self.load_step - 1
@@ -3341,46 +3350,36 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
"by file.")))
# Pad data based upon passed parameter
- if (not self._empty(self._prev_data)) & (not self.empty):
- stored_data = self.data # .copy()
- temp_time = copy.deepcopy(self.index[0])
-
- # Pad data using access mechanisms that works for both pandas
- # and xarray
- self.data = self._prev_data.copy()
-
- # __getitem__ used below to get data from instrument object.
- # Details for handling pandas and xarray are different and
- # handled by __getitem__.
- self.data = self[first_pad:temp_time]
- if not self.empty:
- if self.index[-1] == temp_time:
- self.data = self[:-1]
- self.concat_data(stored_data, prepend=False)
- else:
- self.data = stored_data
-
- if (not self._empty(self._next_data)) & (not self.empty):
- stored_data = self.data # .copy()
- temp_time = copy.deepcopy(self.index[-1])
-
- # Pad data using access mechanisms that work for both pandas
- # and xarray
- self.data = self._next_data.copy()
- self.data = self[temp_time:last_pad]
- if len(self.index) > 0:
- if (self.index[0] == temp_time):
- self.data = self[1:]
- self.concat_data(stored_data, prepend=True)
- else:
- self.data = stored_data
+ if not self._empty(self._prev_data) and not self.empty:
+ # __getitem__ used to handle any pandas/xarray differences in
+ # data slicing
+ pdata = self.__getitem__(slice(first_pad, self.index[0]),
+ data=self._prev_data)
+ if not self._empty(pdata):
+ if pdata[self.index.name][-1] == self.index[0]:
+ pdata = self.__getitem__(slice(-1), data=pdata)
+ self.concat_data(pdata, prepend=True)
+ del pdata
+
+ if not self._empty(self._next_data) and not self.empty:
+ # __getitem__ used to handle any pandas/xarray differences in
+ # data slicing
+
+ ndata = self.__getitem__(slice(self.index[-1], last_pad),
+ data=self._next_data)
+ if not self._empty(ndata):
+ if ndata[self.index.name][0] == self.index[-1]:
+ ndata = self.__getitem__(
+ slice(1, len(ndata[self.index.name])), data=ndata)
+ self.concat_data(ndata, prepend=False)
+ del ndata
if len(self.index) > 0:
self.data = self[first_pad:last_pad]
# Want exclusive end slicing behavior from above
if not self.empty:
- if (self.index[-1] == last_pad) & (not want_last_pad):
+ if (self.index[-1] == last_pad) and (not want_last_pad):
self.data = self[:-1]
else:
# If self.pad is False, load single day
From 0fa26b5b0959333eecbf9a945182ce16a200f659 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 15:38:50 -0400
Subject: [PATCH 080/365] TST: removed skipif
Removed the skipif, to see if the memory issue has been improved.
---
pysat/tests/classes/cls_instrument_library.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 68de0f7bd..3f62d442a 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -518,8 +518,6 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
@pytest.mark.download
- @pytest.mark.skipif(sys.version_info.minor < 7,
- reason='See if everything else passes')
@pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)])
def test_load_w_pad(self, pad, inst_dict):
"""Test that instruments load at each cleaning level.
From a92ba9422afa2a1f23f7441248881a30e48e27c6 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 15:40:44 -0400
Subject: [PATCH 081/365] STY: removed extra line
Removed an extra line that only had whitespace.
---
pysat/_instrument.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 581ffe943..3d58350d1 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -3364,7 +3364,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
if not self._empty(self._next_data) and not self.empty:
# __getitem__ used to handle any pandas/xarray differences in
# data slicing
-
ndata = self.__getitem__(slice(self.index[-1], last_pad),
data=self._next_data)
if not self._empty(ndata):
From dc5c622265380da8cc1c13c6c294235d0bc746d1 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 15:52:22 -0400
Subject: [PATCH 082/365] BUG: fixed index handling
Fixed index handling to be pandas/xarray agnostic within the load method.
---
pysat/_instrument.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 3d58350d1..9d5781e1c 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -3356,7 +3356,9 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
pdata = self.__getitem__(slice(first_pad, self.index[0]),
data=self._prev_data)
if not self._empty(pdata):
- if pdata[self.index.name][-1] == self.index[0]:
+ # Test the data index, slicing if necessary
+ pindex = self._index(data=pdata)
+ if pindex[-1] == self.index[0]:
pdata = self.__getitem__(slice(-1), data=pdata)
self.concat_data(pdata, prepend=True)
del pdata
@@ -3367,7 +3369,9 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
ndata = self.__getitem__(slice(self.index[-1], last_pad),
data=self._next_data)
if not self._empty(ndata):
- if ndata[self.index.name][0] == self.index[-1]:
+ # Test the data index, slicing if necessary
+ nindex = self._index(data=ndata)
+ if nindex[0] == self.index[-1]:
ndata = self.__getitem__(
slice(1, len(ndata[self.index.name])), data=ndata)
self.concat_data(ndata, prepend=False)
From 98c4761ed23297f578875476ac28bbe574f55670 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 15:58:21 -0400
Subject: [PATCH 083/365] DOC: updated the changelog
Added a description of the new Instrument changes to the changelog.
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f9a0fdbd7..ca78e0db3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
clean method.
* Added loading test with padding for Instruments.
* Allow Instruments to define custom `concat_data` methods.
+ * Added data kwarg to the Instrument class `__getitem__` method and reduced
+ memory usage in the `load` method.
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
From 3cb0f5aecd8e05293d6d7dc2836a40915a0ddf03 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 16:44:27 -0400
Subject: [PATCH 084/365] TST: added testing skip
Try to run new test only on one of the test instruments for Python 3.6.
---
pysat/tests/classes/cls_instrument_library.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 3f62d442a..9b6865dc5 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -532,6 +532,13 @@ def test_load_w_pad(self, pad, inst_dict):
`initialize_test_package` is run.
"""
+ # Skip all but one instrument for Python 3.6
+ if sys.version_info.minor < 7 and inst_dict[
+ 'inst_module'].__name__.find('pysat_testing') == len(
+ inst_dict['inst_module'].__name__) - len('pysat_testing'):
+ pytest.skip("skipping 'test_load_w_pad' for {:}".format(inst_dict))
+ return
+
# Update the Instrument dict with the desired pad
if 'kwargs' in inst_dict.keys():
inst_dict['kwargs']['pad'] = pad
From 77fcb2370dacc5b35e07bea9932cf756a3a80bd0 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 16:46:15 -0400
Subject: [PATCH 085/365] STY: removed whitespace
Removed extra whitespace.
---
pysat/tests/classes/cls_instrument_library.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 9b6865dc5..fe95ebb50 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -538,7 +538,7 @@ def test_load_w_pad(self, pad, inst_dict):
inst_dict['inst_module'].__name__) - len('pysat_testing'):
pytest.skip("skipping 'test_load_w_pad' for {:}".format(inst_dict))
return
-
+
# Update the Instrument dict with the desired pad
if 'kwargs' in inst_dict.keys():
inst_dict['kwargs']['pad'] = pad
From a143898ced6bfa902ad8ff065d90617bf5537935 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 11 Sep 2023 17:01:42 -0400
Subject: [PATCH 086/365] TST: add more skips
Add more skips, with information about what is being skipped.
---
pysat/tests/classes/cls_instrument_library.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index fe95ebb50..c4a380af8 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -533,10 +533,11 @@ def test_load_w_pad(self, pad, inst_dict):
"""
# Skip all but one instrument for Python 3.6
- if sys.version_info.minor < 7 and inst_dict[
- 'inst_module'].__name__.find('pysat_testing') == len(
- inst_dict['inst_module'].__name__) - len('pysat_testing'):
- pytest.skip("skipping 'test_load_w_pad' for {:}".format(inst_dict))
+ if sys.version_info.minor < 7:
+ pytest.skip("skipping 3.6 for {:} ({:} =? {:})".format(
+ inst_dict, inst_dict['inst_module'].__name__.find(
+ 'pysat_testing'), len(inst_dict['inst_module'].__name__)
+ - len('pysat_testing')))
return
# Update the Instrument dict with the desired pad
From 8e34d084652b23959b09c1c529eb785ad73a400a Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 12 Sep 2023 09:54:30 -0400
Subject: [PATCH 087/365] STY: reduced code duplication
Reduced code duplication in Instrument by adding a hidden method that finds the standard epoch name(s) in the xarray Dataset.
---
CHANGELOG.md | 2 ++
pysat/_instrument.py | 60 +++++++++++++++++++++++++++++++++-----------
2 files changed, 47 insertions(+), 15 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ca78e0db3..234631ee8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Allow Instruments to define custom `concat_data` methods.
* Added data kwarg to the Instrument class `__getitem__` method and reduced
memory usage in the `load` method.
+ * Added a hidden method the Instrument class `_get_epoch_name_from_data` to
+ reduce code duplication.
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 9d5781e1c..8f69a41d6 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -875,11 +875,9 @@ def __getitem_xarray__(self, key, data=None):
if data is None:
data = self.data
- if 'Epoch' in data.indexes:
- epoch_names = ['Epoch']
- elif 'time' in data.indexes:
- epoch_names = ['time']
- else:
+ # Find the standard epoch index name(s)
+ epoch_names = self._get_epoch_name_from_data(data=data)
+ if len(epoch_names) == 0:
return xr.Dataset(None)
# Find secondary time indexes that may need to be sliced
@@ -1089,13 +1087,16 @@ def __setitem__(self, key, new_data):
new = {'data': new}
in_data = new.pop('data')
- if 'Epoch' in self.data.indexes:
- epoch_name = 'Epoch'
- elif 'time' in self.data.indexes:
- epoch_name = 'time'
- else:
+ epoch_names = self._get_epoch_name_from_data()
+ if len(epoch_names) == 0:
raise ValueError(' '.join(('Unsupported time index name,',
'"Epoch" or "time".')))
+ else:
+ if len(epoch_names) > 1:
+ pysat.logger.error("".join(["Multiple standard time index ",
+ "names found, defaulting to ",
+ epoch_names[0]]))
+ epoch_name = epoch_names[0]
if isinstance(key, tuple):
# User provided more than one thing in assignment location
@@ -1252,6 +1253,34 @@ def __iter__(self):
# -----------------------------------------------------------------------
# Define all hidden methods
+ def _get_epoch_name_from_data(self, data=None):
+ """Get the standard epoch name used in this data object.
+
+ Parameters
+ ----------
+ data : pds.DataFrame, xr.Dataset, or NoneType
+ Desired data object to select from or None to use `data` attribute
+
+ Returns
+ -------
+ epoch_names : list
+ List of standard epoch names included in the data indexes
+
+ """
+ # Initalize output
+ epoch_names = []
+
+ # If no data is provided, use the Instrument attribute
+ if data is None:
+ data = self.data
+
+ if hasattr(data, 'indexes'):
+ for ename in ['Epoch', 'time']:
+ if ename in data.indexes:
+ epoch_names.append(ename)
+
+ return epoch_names
+
def _empty(self, data=None):
"""Determine whether or not data has been loaded.
@@ -1308,12 +1337,13 @@ def _index(self, data=None):
if self.pandas_format:
return data.index
else:
- if 'time' in data.indexes:
- return data.indexes['time']
- elif 'Epoch' in data.indexes:
- return data.indexes['Epoch']
- else:
+ epoch_names = self.get_epoch_name_from_data(data=data)
+
+ if len(epoch_names) == 0:
return pds.Index([])
+ else:
+ # Xarray preferred epoch name order is opposite
+ return data.indexes[epoch_names[-1]]
def _pass_method(*args, **kwargs):
"""Empty default method for updatable Instrument methods."""
From b562952a2b34d6566947ce1e6a16268c8a67a720 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 12 Sep 2023 09:55:09 -0400
Subject: [PATCH 088/365] TST: added TODO for skip
Added a TODO for the skip, since the solution to the current failure is out of scope for the current pull request.
---
pysat/tests/classes/cls_instrument_library.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index c4a380af8..da0dc1195 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -532,7 +532,9 @@ def test_load_w_pad(self, pad, inst_dict):
`initialize_test_package` is run.
"""
- # Skip all but one instrument for Python 3.6
+ # Skip for Python 3.6, keeping information that will allow adding
+ # or skipping particular instruments.
+ # TODO(#1136): Remove skip once memory management is improved
if sys.version_info.minor < 7:
pytest.skip("skipping 3.6 for {:} ({:} =? {:})".format(
inst_dict, inst_dict['inst_module'].__name__.find(
From 08f27646ba685f58e4f8ff8036c7e46dcc5e6163 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 12 Sep 2023 10:31:39 -0400
Subject: [PATCH 089/365] BUG: fixed typo
Fixed a typo in a method name call.
---
pysat/_instrument.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 8f69a41d6..cf6826f72 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -1337,7 +1337,7 @@ def _index(self, data=None):
if self.pandas_format:
return data.index
else:
- epoch_names = self.get_epoch_name_from_data(data=data)
+ epoch_names = self._get_epoch_name_from_data(data=data)
if len(epoch_names) == 0:
return pds.Index([])
From 16b491da068e63db26d67e4a33bf06948c384bc7 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 15 Sep 2023 12:39:46 -0400
Subject: [PATCH 090/365] MAINT: fixed spelling
Fixed spelling, because words are hard.
Co-authored-by: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com>
---
pysat/_instrument.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index cf6826f72..b5091d70f 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -3336,11 +3336,11 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
# inclusive/exclusive range end treatment. Loading by file is
# inclusive.
if self._load_by_date:
- # Arithmatic uses datetime or DateOffset objects
+ # Arithmetic uses datetime or DateOffset objects
self._next_data_track = curr + self.load_step
self._prev_data_track = curr - self.load_step
else:
- # Aritmatic uses integers
+ # Arithmetic uses integers
self._next_data_track = curr + self.load_step + 1
self._prev_data_track = curr - self.load_step - 1
From 2d72fb902c930236995be1d89587eb45cec54054 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 15 Sep 2023 13:09:17 -0400
Subject: [PATCH 091/365] BUG: pad requires time
Test for index length as well as empty data when padding.
---
pysat/_instrument.py | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index b5091d70f..aa02e0734 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -3388,9 +3388,10 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
if not self._empty(pdata):
# Test the data index, slicing if necessary
pindex = self._index(data=pdata)
- if pindex[-1] == self.index[0]:
- pdata = self.__getitem__(slice(-1), data=pdata)
- self.concat_data(pdata, prepend=True)
+ if len(pindex) > 0:
+ if pindex[-1] == self.index[0]:
+ pdata = self.__getitem__(slice(-1), data=pdata)
+ self.concat_data(pdata, prepend=True)
del pdata
if not self._empty(self._next_data) and not self.empty:
@@ -3401,10 +3402,12 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
if not self._empty(ndata):
# Test the data index, slicing if necessary
nindex = self._index(data=ndata)
- if nindex[0] == self.index[-1]:
- ndata = self.__getitem__(
- slice(1, len(ndata[self.index.name])), data=ndata)
- self.concat_data(ndata, prepend=False)
+ if len(nindex) > 0:
+ if nindex[0] == self.index[-1]:
+ ndata = self.__getitem__(
+ slice(1, len(ndata[self.index.name])),
+ data=ndata)
+ self.concat_data(ndata, prepend=False)
del ndata
if len(self.index) > 0:
From 065e77a3eb5040e70d80165effce3156b1b44591 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 15 Sep 2023 16:51:21 -0400
Subject: [PATCH 092/365] ENH: added `include` kwarg to `concat_data`
Added a kwarg to `concat_data` that allows the currently loaded data to be inserted in any location within the new data list. Applied this new kwarg to reduce `concat_data` calls in the `load` method.
---
pysat/_instrument.py | 42 +++++++++++++++++++++++++++++++-----------
1 file changed, 31 insertions(+), 11 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index aa02e0734..20f812984 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -2427,7 +2427,7 @@ def copy(self):
return inst_copy
- def concat_data(self, new_data, prepend=False, **kwargs):
+ def concat_data(self, new_data, prepend=False, include=None, **kwargs):
"""Concatonate data to self.data for xarray or pandas as needed.
Parameters
@@ -2437,6 +2437,9 @@ def concat_data(self, new_data, prepend=False, **kwargs):
prepend : bool
If True, assign new data before existing data; if False append new
data (default=False)
+ include : int or NoneType
+ Index at which `self.data` should be included in `new_data` or None
+ to use `prepend` (default=None)
**kwargs : dict
Optional keyword arguments passed to pds.concat or xr.concat
@@ -2450,6 +2453,13 @@ def concat_data(self, new_data, prepend=False, **kwargs):
For xarray, `dim=Instrument.index.name` is passed along to xarray.concat
except if the user includes a value for dim as a keyword argument.
+ Examples
+ --------
+ ::
+
+ # Concatonate data before and after the existing Instrument data
+ inst.concat_data([prev_data, next_data], include=1)
+
"""
# Add any concat_data kwargs
for ckey in self.kwargs['concat_data'].keys():
@@ -2459,11 +2469,14 @@ def concat_data(self, new_data, prepend=False, **kwargs):
# Order the data to be concatenated in a list
if not isinstance(new_data, list):
new_data = [new_data]
-
- if prepend:
- new_data.append(self.data)
+
+ if include is None:
+ if prepend:
+ new_data.append(self.data)
+ else:
+ new_data.insert(0, self.data)
else:
- new_data.insert(0, self.data)
+ new_data.insert(include, self.data)
if self._concat_data_rtn.__name__.find('_pass_method') == 0:
# There is no custom concat function, use the pysat standard method.
@@ -3380,8 +3393,10 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
"by file.")))
# Pad data based upon passed parameter
+ cdata = list()
+ include = None
if not self._empty(self._prev_data) and not self.empty:
- # __getitem__ used to handle any pandas/xarray differences in
+ # __getitem__ is used to handle any pandas/xarray differences in
# data slicing
pdata = self.__getitem__(slice(first_pad, self.index[0]),
data=self._prev_data)
@@ -3391,11 +3406,11 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
if len(pindex) > 0:
if pindex[-1] == self.index[0]:
pdata = self.__getitem__(slice(-1), data=pdata)
- self.concat_data(pdata, prepend=True)
- del pdata
+ cdata.append(pdata)
+ include = 1
if not self._empty(self._next_data) and not self.empty:
- # __getitem__ used to handle any pandas/xarray differences in
+ # __getitem__ is used to handle any pandas/xarray differences in
# data slicing
ndata = self.__getitem__(slice(self.index[-1], last_pad),
data=self._next_data)
@@ -3407,8 +3422,13 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
ndata = self.__getitem__(
slice(1, len(ndata[self.index.name])),
data=ndata)
- self.concat_data(ndata, prepend=False)
- del ndata
+ cdata.append(ndata)
+ if include is None:
+ include = 0
+
+ # Concatonate the current, previous, and next data
+ if len(cdata) > 0:
+ self.concat_data(cdata, include=include)
if len(self.index) > 0:
self.data = self[first_pad:last_pad]
From ed72675218ec67ab3a7adf6585a2855664b72b85 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 18 Sep 2023 10:10:51 -0400
Subject: [PATCH 093/365] TST: added test for `include` kwarg
Added a test for the `include` kwarg in `concat_data`.
---
pysat/tests/classes/cls_instrument_access.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py
index edc8b759a..b66b11e4a 100644
--- a/pysat/tests/classes/cls_instrument_access.py
+++ b/pysat/tests/classes/cls_instrument_access.py
@@ -574,7 +574,8 @@ def test_inequality_reduced_object(self):
@pytest.mark.parametrize("prepend, sort_dim_toggle",
[(True, True), (True, False), (False, False)])
- def test_concat_data(self, prepend, sort_dim_toggle):
+ @pytest.mark.parametrize("include", [True, False])
+ def test_concat_data(self, prepend, sort_dim_toggle, include):
"""Test `pysat.Instrument.data` concatenation.
Parameters
@@ -586,6 +587,8 @@ def test_concat_data(self, prepend, sort_dim_toggle):
If True, sort variable names in pandas before concatenation. If
False, do not sort for pandas objects. For xarray objects, rename
the epoch if True.
+ include : bool
+ Use `include` kwarg instead of `prepend` for the same behaviour.
"""
@@ -611,6 +614,9 @@ def test_concat_data(self, prepend, sort_dim_toggle):
data2 = data2.rename({self.xarray_epoch_name: 'Epoch2'})
self.testInst.data = self.testInst.data.rename(
{self.xarray_epoch_name: 'Epoch2'})
+ if include:
+ # To prepend new data, put existing data at end and vice versa
+ kwargs['include'] = 1 if prepend else 0
# Concat together
self.testInst.concat_data(data2, **kwargs)
From 2b53191a5c4355b71df57d01804858e378661178 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 18 Sep 2023 10:13:02 -0400
Subject: [PATCH 094/365] DOC: updated changelog
Added a description of the pull to the changelog.
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 234631ee8..31c5ec17e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
clean method.
* Added loading test with padding for Instruments.
* Allow Instruments to define custom `concat_data` methods.
+ * Added `include` kwarg to `Instrument.concat_data` to expand allowed inputs.
* Added data kwarg to the Instrument class `__getitem__` method and reduced
memory usage in the `load` method.
* Added a hidden method the Instrument class `_get_epoch_name_from_data` to
From 9ef05b7218e5f1360a676a6652f561b5dd0661de Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 18 Sep 2023 10:28:46 -0400
Subject: [PATCH 095/365] STY: remove excess whitespace
Removed extra whitespace.
---
pysat/_instrument.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 20f812984..5405e1ee6 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -2469,7 +2469,7 @@ def concat_data(self, new_data, prepend=False, include=None, **kwargs):
# Order the data to be concatenated in a list
if not isinstance(new_data, list):
new_data = [new_data]
-
+
if include is None:
if prepend:
new_data.append(self.data)
From 1a94949be501a340759cc89c894d5451f3e346eb Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 20 Sep 2023 17:39:39 -0400
Subject: [PATCH 096/365] ENH: improved meta delete
Added `drop` methods to MetaLabels and MetaHeader classes, as well as `__delitem__` to Meta. For consistency and ease of access, also updated `Meta.__getitem__` to access MetaHeader attributes.
---
pysat/_meta.py | 102 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 102 insertions(+)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 85d5c8d54..21fa40798 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -13,6 +13,7 @@
import pysat
import pysat.utils._core as core_utils
+from pysat.utils import listify
from pysat.utils import testing
@@ -674,6 +675,8 @@ def match_name(func, var_name, index_or_column):
meta_row.at['children'] = ''
meta_row.at['children'] = None
return meta_row
+ elif key in self.header.global_attrs:
+ return getattr(self.header, key)
else:
raise KeyError("Key '{:}' not found in MetaData".format(key))
else:
@@ -681,6 +684,55 @@ def match_name(func, var_name, index_or_column):
"{}; ".format(key.__repr__()),
"expected tuple, list, or str"]))
+ def __delitem__(self, key):
+ """Remove metadata.
+
+ Maps to pandas DataFrame methods.
+
+ Parameters
+ ----------
+ key : str, tuple, or list
+ A single variable name, a tuple, or a list
+
+ Raises
+ ------
+ KeyError
+ If a properly formatted key is not present
+
+ Examples
+ --------
+ ::
+
+ import pysat
+ inst = pysat.Instrument('pysat', 'testing2d')
+ inst.load(date=inst.inst_module._test_dates[''][''])
+ meta = inst.meta
+
+ # Any data or label of Meta, including the top-level header
+ # attributes may be removed using `del`
+ del meta['uts']
+
+ """
+
+ try:
+ # This is a data variable
+ self.drop(key)
+ except KeyError:
+ keys = listify(key)
+
+ for key in keys:
+ if key in self.data.columns:
+ # This is a metadata label
+ self.data = self.data.drop(key, axis=1)
+
+ # Also drop this from Labels
+ self.labels.drop(key)
+ elif key in self.header.global_attrs:
+ self.header.drop(key)
+ else:
+ raise KeyError("{:} not found in Meta".format(repr(key)))
+ return
+
def __contains__(self, data_var):
"""Check variable name, not distinguishing by case.
@@ -2029,6 +2081,37 @@ def default_values_from_attr(self, attr_name, data_type=None):
return default_val
+ def drop(self, names):
+ """Removes data from MetaLabels.
+
+ Parameters
+ ----------
+ names : str or list-like
+ Attribute or MetaData name(s)
+
+ Raises
+ ------
+ AttributeError or KeyError
+ If any part of `names` is missing and cannot be dropped
+
+ """
+ # Ennsure the input is list-likee
+ names = listify(names)
+
+ # Cycle through each name to drop
+ for name in names:
+ if name in self.label_attrs.keys():
+ lname = self.label_attrs[name]
+ delattr(self, lname)
+ del self.label_type[lname]
+ del self.label_attrs[name]
+ else:
+ lname = getattr(self, name)
+ delattr(self, name)
+ del self.label_type[name]
+ del self.label_attrs[lname]
+ return
+
def update(self, lattr, lname, ltype):
"""Update MetaLabels with a new label.
@@ -2207,6 +2290,25 @@ def __eq__(self, other):
return True
+ def drop(self, names):
+ """Drop variables (names) from MetaHeader.
+
+ Parameters
+ ----------
+ names : list-like
+ List of strings specifying the variable names to drop
+
+ """
+ names = listify(names)
+
+ for name in names:
+ # Delete the attribute
+ delattr(self, name)
+
+ # Remove the attribute from the attribute list
+ self.global_attrs.pop(self.global_attrs.index(name))
+ return
+
def to_dict(self):
"""Convert the header data to a dictionary.
From c22785c4a6f0204f65ac7371e844c2d11920aa51 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 20 Sep 2023 17:40:24 -0400
Subject: [PATCH 097/365] TST: added drop and del tests
Added unit and integration tests for the new `drop` and `__delitem__` methods.
---
pysat/tests/test_meta.py | 19 +++++++++++++
pysat/tests/test_meta_header.py | 47 ++++++++++++++++++++++++++++++++-
pysat/tests/test_meta_labels.py | 30 +++++++++++++++++++++
3 files changed, 95 insertions(+), 1 deletion(-)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index a4ec9aec8..fcf984b4a 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -510,6 +510,25 @@ def test_str(self, long_str, inst_kwargs):
assert out.find('ND Metadata variables:') < 0
return
+ @pytest.mark.parametrize("del_key", ["uts", "units"])
+ def test_del(self, del_key):
+ """Test deletion of Meta data.
+
+ Parameters
+ ----------
+ del_key : str
+ Key for Meta data, MetaHeater attribute, or MetaLabels label
+
+ """
+ # Set the meta data
+ self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'})
+
+ del self.meta[del_key]
+
+ assert del_key not in self.meta.data.index
+ assert del_key not in self.meta.data.columns
+ return
+
def test_self_equality(self):
"""Test Meta equality for the same object."""
diff --git a/pysat/tests/test_meta_header.py b/pysat/tests/test_meta_header.py
index 42008523a..401a34c51 100644
--- a/pysat/tests/test_meta_header.py
+++ b/pysat/tests/test_meta_header.py
@@ -121,12 +121,29 @@ def test_to_dict(self, header_data):
assert header_data == out_dict
return
+ def test_drop(self):
+ """Test the MetaHeader.drop method."""
+ # Get the current output dict
+ out_dict = self.meta_header.to_dict()
+
+ # Add an attribute
+ self.meta_header.PI = "pysat development team"
+ assert "PI" in self.meta_header.global_attrs
+ assert out_dict != self.meta_header.to_dict()
+
+ # Remove the attribute
+ self.meta_header.drop('PI')
+
+ # Ensure the current output matches the original output
+ assert out_dict == self.meta_header.to_dict()
+ return
+
# ----------------------------------------
# Test the integration with the Meta class
@pytest.mark.parametrize("header_data", [{}, {"test": "value"}])
def test_init_metaheader_in_meta(self, header_data):
- """Test changing case of meta labels after initialization.
+ """Test changing case of MetaHeader after Meta initialization.
Parameters
----------
@@ -142,3 +159,31 @@ def test_init_metaheader_in_meta(self, header_data):
# Ensure both initialization methods work the same
assert meta.header == self.meta_header
return
+
+ def test_get_metaheader_in_meta(self):
+ """Test MetaHeader attribute retrieval from Meta."""
+
+ # Initalize the header data through the meta object
+ test_attr = "PI"
+ test_val = "pysat development team"
+ meta = pysat.Meta(header_data={test_attr: test_val})
+
+ # Test value retrieval from Meta and MetaHeader
+ assert getattr(meta.header, test_attr) == test_val
+ assert meta[test_attr] == test_val
+ return
+
+ def test_delete_metaheader_in_meta(self):
+ """Test MetaHeader attribute deletion from Meta."""
+
+ # Initalize the header data through the meta object
+ test_attr = "PI"
+ test_val = "pysat development team"
+ meta = pysat.Meta(header_data={test_attr: test_val})
+
+ # Delete MetaHeader data
+ del meta[test_attr]
+
+ # Test for empty MetaHeader
+ assert meta.header.to_dict() == {}
+ return
diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py
index ee4b6658e..3678dacd9 100644
--- a/pysat/tests/test_meta_labels.py
+++ b/pysat/tests/test_meta_labels.py
@@ -11,6 +11,7 @@
import pytest
import pysat
+from pysat.utils import listify
from pysat.utils import testing
@@ -185,6 +186,24 @@ def test_default_value_from_type_int_inputs(self, in_val, comp_val):
return
+ @pytest.mark.parametrize("drop_labels", [["units", "fill_val"], "units"])
+ def test_drop(self, drop_labels):
+ """Test successfull drop from MetaLabels.
+
+ Parameters
+ ----------
+ drop_labels : str or list-like
+ Label or labels to drop
+
+ """
+ # Drop the desired label(s)
+ self.meta_labels.drop(drop_labels)
+
+ # Ensure the labels are missing
+ for dlabel in listify(drop_labels):
+ assert not hasattr(self.meta_labels, dlabel)
+ return
+
def test_update(self):
"""Test successful update of MetaLabels."""
self.meta_labels.update('new_label', 'new_name', int)
@@ -197,6 +216,17 @@ def test_update(self):
# ----------------------------------------
# Test the integration with the Meta class
+ def test_del_from_meta(self):
+ """Test successfull deletion of MetaLabels attribute from Meta."""
+ # Delete the desired label
+ del_label = list(self.meta_labels.label_type.keys())[0]
+ del self.meta[del_label]
+
+ # Ensure the label is missing
+ assert not hasattr(self.meta.labels, del_label)
+ assert del_label not in self.meta.data.columns
+ return
+
def test_change_case_of_meta_labels(self):
"""Test changing case of meta labels after initialization."""
From 16201f9aac09374cf18bc8d8715cd5384b03c657 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 20 Sep 2023 17:42:44 -0400
Subject: [PATCH 098/365] DOC: updated changelog
Added a summary of the changes to the changelog.
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 31c5ec17e..acab459a4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
memory usage in the `load` method.
* Added a hidden method the Instrument class `_get_epoch_name_from_data` to
reduce code duplication.
+ * Added `__delitem__` to Meta and `drop` to MetaHeader and MetaLabels classes.
+ * Modified Meta to allow MetaHeader attribute access directly from Meta.
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
From 6c25355cd789728f3a6160f12c2b413b5441b37c Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 20 Sep 2023 17:55:59 -0400
Subject: [PATCH 099/365] DOC: fixed docstring case
Changed the docstring case to be imperative.
---
pysat/_meta.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 21fa40798..8b764f103 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -2082,7 +2082,7 @@ def default_values_from_attr(self, attr_name, data_type=None):
return default_val
def drop(self, names):
- """Removes data from MetaLabels.
+ """Remove data from MetaLabels.
Parameters
----------
From 001e3cbe4122e218d4daf4475eac3911ccd79753 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 20 Sep 2023 17:56:20 -0400
Subject: [PATCH 100/365] STY: removed extra whitespace
Removed extra whitespace.
---
pysat/tests/test_meta_header.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/pysat/tests/test_meta_header.py b/pysat/tests/test_meta_header.py
index 401a34c51..64af8b967 100644
--- a/pysat/tests/test_meta_header.py
+++ b/pysat/tests/test_meta_header.py
@@ -162,7 +162,6 @@ def test_init_metaheader_in_meta(self, header_data):
def test_get_metaheader_in_meta(self):
"""Test MetaHeader attribute retrieval from Meta."""
-
# Initalize the header data through the meta object
test_attr = "PI"
test_val = "pysat development team"
@@ -175,7 +174,6 @@ def test_get_metaheader_in_meta(self):
def test_delete_metaheader_in_meta(self):
"""Test MetaHeader attribute deletion from Meta."""
-
# Initalize the header data through the meta object
test_attr = "PI"
test_val = "pysat development team"
From 63ba91810a6da555eeb669cd2df216f973f2e715 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 20 Sep 2023 17:56:40 -0400
Subject: [PATCH 101/365] MAINT: fixed spelling
Fixed spelling of 'successful'.
---
pysat/tests/test_meta_labels.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py
index 3678dacd9..17195f174 100644
--- a/pysat/tests/test_meta_labels.py
+++ b/pysat/tests/test_meta_labels.py
@@ -188,7 +188,7 @@ def test_default_value_from_type_int_inputs(self, in_val, comp_val):
@pytest.mark.parametrize("drop_labels", [["units", "fill_val"], "units"])
def test_drop(self, drop_labels):
- """Test successfull drop from MetaLabels.
+ """Test successful drop from MetaLabels.
Parameters
----------
@@ -217,7 +217,7 @@ def test_update(self):
# Test the integration with the Meta class
def test_del_from_meta(self):
- """Test successfull deletion of MetaLabels attribute from Meta."""
+ """Test successful deletion of MetaLabels attribute from Meta."""
# Delete the desired label
del_label = list(self.meta_labels.label_type.keys())[0]
del self.meta[del_label]
From 16df694521b3eff60fb4ff308495e419b86c845f Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 28 Sep 2023 09:06:48 -0400
Subject: [PATCH 102/365] TST: added skip
Added a skip to test memory issues.
---
pysat/tests/test_meta.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index fcf984b4a..8be80db03 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -510,6 +510,7 @@ def test_str(self, long_str, inst_kwargs):
assert out.find('ND Metadata variables:') < 0
return
+ @pytest.mark.skip("Does this test use too much memory?")
@pytest.mark.parametrize("del_key", ["uts", "units"])
def test_del(self, del_key):
"""Test deletion of Meta data.
From 93c7e41d05c1e9707437bdc9fbe2f6d32e5e34fc Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 28 Sep 2023 11:26:05 -0400
Subject: [PATCH 103/365] STY: updated drop
Changed drop to act like `__delitem__` and changed `__delitem__` to redirect user to `drop`.
---
pysat/_meta.py | 77 +++++++++++++++++++-------------------------------
1 file changed, 29 insertions(+), 48 deletions(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 8b764f103..aa1287311 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -687,51 +687,13 @@ def match_name(func, var_name, index_or_column):
def __delitem__(self, key):
"""Remove metadata.
- Maps to pandas DataFrame methods.
-
- Parameters
- ----------
- key : str, tuple, or list
- A single variable name, a tuple, or a list
-
Raises
------
- KeyError
- If a properly formatted key is not present
-
- Examples
- --------
- ::
-
- import pysat
- inst = pysat.Instrument('pysat', 'testing2d')
- inst.load(date=inst.inst_module._test_dates[''][''])
- meta = inst.meta
-
- # Any data or label of Meta, including the top-level header
- # attributes may be removed using `del`
- del meta['uts']
+ NotImplementedError
+ Redirects the user to use `drop`
"""
-
- try:
- # This is a data variable
- self.drop(key)
- except KeyError:
- keys = listify(key)
-
- for key in keys:
- if key in self.data.columns:
- # This is a metadata label
- self.data = self.data.drop(key, axis=1)
-
- # Also drop this from Labels
- self.labels.drop(key)
- elif key in self.header.global_attrs:
- self.header.drop(key)
- else:
- raise KeyError("{:} not found in Meta".format(repr(key)))
- return
+ raise NotImplementedError('`del` not supported, use `Meta.drop`')
def __contains__(self, data_var):
"""Check variable name, not distinguishing by case.
@@ -1047,18 +1009,37 @@ def drop(self, names):
Parameters
----------
- names : list-like
- List of strings specifying the variable names to drop
-
- """
+ names : str or list-like
+ String or list of strings specifying the variable names to drop
- # Drop the lower dimension data
- self.data = self._data.drop(names, axis=0)
+ Raises
+ ------
+ KeyError
+ If any of the keys provided in `names` is not found in the
+ lower dimensional data, higher dimensional data, labels, or
+ header data
- # Drop the higher dimension data
+ """
+ # Ensure the input is list-like
+ names = listify(names)
for name in names:
if name in self._ho_data:
+ # Drop the higher dimension data
self._ho_data.pop(name)
+
+ if name in self.keys():
+ # Drop the lower dimension data
+ self.data = self._data.drop(name, axis=0)
+ elif name in self.data.columns:
+ # This is a metadata label
+ self.data = self._data.drop(name, axis=1)
+
+ # Also drop this from Labels
+ self.labels.drop(name)
+ elif key in self.header.global_attrs:
+ self.header.drop(name)
+ else:
+ raise KeyError("{:} not found in Meta".format(repr(name)))
return
def keep(self, keep_names):
From 0c69fee3105063ac09a84aeda63bc2704a5e0e8e Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 28 Sep 2023 11:27:25 -0400
Subject: [PATCH 104/365] TST: added drop tests
Added drop tests for labels, string input, higher-order input, and a combination of those with normal data variables.
---
pysat/tests/test_meta.py | 69 ++++++++++++++++++++++++++++++++++------
1 file changed, 60 insertions(+), 9 deletions(-)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index 8be80db03..408e32bc7 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -510,10 +510,8 @@ def test_str(self, long_str, inst_kwargs):
assert out.find('ND Metadata variables:') < 0
return
- @pytest.mark.skip("Does this test use too much memory?")
- @pytest.mark.parametrize("del_key", ["uts", "units"])
- def test_del(self, del_key):
- """Test deletion of Meta data.
+ def test_delitem(self):
+ """Test deletion of Meta data informs user of correct path.
Parameters
----------
@@ -524,10 +522,9 @@ def test_del(self, del_key):
# Set the meta data
self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'})
- del self.meta[del_key]
-
- assert del_key not in self.meta.data.index
- assert del_key not in self.meta.data.columns
+ with pytest.raises(NotImplementedError,
+ match='`del` not supported, use `Meta.drop`'):
+ del self.meta['uts']
return
def test_self_equality(self):
@@ -1208,8 +1205,33 @@ def test_meta_merge(self):
meta_dict[label].__repr__())
return
+ @pytest.mark.parametrize("names", ['uts', ['uts', 'mlt'], 'units',
+ ['units', 'uts']])
+ def test_meta_drop(self, names):
+ """Test successful deletion of meta data for different types of data.
+
+ Parameters
+ ----------
+ names : int
+ Number of variables to drop in a single go.
+
+ """
+ # Set meta data
+ self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'})
+
+ # Drop the values
+ self.meta.drop(names)
+
+ # Test the successful deletion
+ for name in pysat.utils.listify(names):
+ if name in self.testInst.variables:
+ assert name not in self.meta.keys(), "didn't drop variable"
+ else:
+ assert name not in self.meta.data.columns, "didn't drop label"
+ return
+
@pytest.mark.parametrize("num_drop", [0, 1, 3])
- def test_meta_drop(self, num_drop):
+ def test_meta_num_drop(self, num_drop):
"""Test successful deletion of meta data for specific values.
Parameters
@@ -1547,6 +1569,35 @@ def test_update_epoch(self):
# -------------------------------
# Tests for higher order metadata
+ # TODO(#789): remove tests for higher order meta
+ @pytest.mark.parametrize("names", ['series_profiles',
+ ['uts', 'series_profiles'],
+ ['units', 'series_profiles']])
+ def test_meta_drop_higher_order(self, names):
+ """Test successful deletion of meta data for different types of data.
+
+ Parameters
+ ----------
+ names : str or list
+ Variables or labels to drop from metadata.
+
+ """
+ # Set meta data
+ self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'})
+
+ # Drop the values
+ self.meta.drop(names)
+
+ # Test the successful deletion
+ for name in pysat.utils.listify(names):
+ if name in self.testInst.variables:
+ assert name not in self.meta.keys(), "didn't drop LO variable"
+ assert name not in self.meta.ho_data.keys(), \
+ "didn't drop HO variable"
+ else:
+ assert name not in self.meta.data.columns, "didn't drop label"
+ return
+
# TODO(#789): remove tests for higher order meta
@pytest.mark.parametrize('meta_dict', [
None, {'units': 'V', 'long_name': 'test name'},
From e5b2e8ea2a0be24659e6eb4cbbe1841065bb2024 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 28 Sep 2023 11:29:40 -0400
Subject: [PATCH 105/365] BUG: fixed variable name
Fixed the variable name to reflect the drop standards.
---
pysat/_meta.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index aa1287311..cb1bd0338 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -1036,7 +1036,7 @@ def drop(self, names):
# Also drop this from Labels
self.labels.drop(name)
- elif key in self.header.global_attrs:
+ elif name in self.header.global_attrs:
self.header.drop(name)
else:
raise KeyError("{:} not found in Meta".format(repr(name)))
From 3e410f3aa537ee14db1d1eb9b7d4981c9a4f2b9e Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 28 Sep 2023 13:35:24 -0400
Subject: [PATCH 106/365] STY: remove delitem
Remove delitem completely.
---
pysat/_meta.py | 11 -----------
pysat/tests/test_meta.py | 17 -----------------
2 files changed, 28 deletions(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index cb1bd0338..70191be83 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -684,17 +684,6 @@ def match_name(func, var_name, index_or_column):
"{}; ".format(key.__repr__()),
"expected tuple, list, or str"]))
- def __delitem__(self, key):
- """Remove metadata.
-
- Raises
- ------
- NotImplementedError
- Redirects the user to use `drop`
-
- """
- raise NotImplementedError('`del` not supported, use `Meta.drop`')
-
def __contains__(self, data_var):
"""Check variable name, not distinguishing by case.
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index 408e32bc7..aa993792c 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -510,23 +510,6 @@ def test_str(self, long_str, inst_kwargs):
assert out.find('ND Metadata variables:') < 0
return
- def test_delitem(self):
- """Test deletion of Meta data informs user of correct path.
-
- Parameters
- ----------
- del_key : str
- Key for Meta data, MetaHeater attribute, or MetaLabels label
-
- """
- # Set the meta data
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'})
-
- with pytest.raises(NotImplementedError,
- match='`del` not supported, use `Meta.drop`'):
- del self.meta['uts']
- return
-
def test_self_equality(self):
"""Test Meta equality for the same object."""
From c9b104b511557b9bc3771b1da6970c30d2db1665 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 28 Sep 2023 14:33:15 -0400
Subject: [PATCH 107/365] BUG: replace del with drop
Replace del with drop in the integration tests.
---
pysat/tests/test_meta_header.py | 4 ++--
pysat/tests/test_meta_labels.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/pysat/tests/test_meta_header.py b/pysat/tests/test_meta_header.py
index 64af8b967..376777af9 100644
--- a/pysat/tests/test_meta_header.py
+++ b/pysat/tests/test_meta_header.py
@@ -172,7 +172,7 @@ def test_get_metaheader_in_meta(self):
assert meta[test_attr] == test_val
return
- def test_delete_metaheader_in_meta(self):
+ def test_drop_metaheader_in_meta(self):
"""Test MetaHeader attribute deletion from Meta."""
# Initalize the header data through the meta object
test_attr = "PI"
@@ -180,7 +180,7 @@ def test_delete_metaheader_in_meta(self):
meta = pysat.Meta(header_data={test_attr: test_val})
# Delete MetaHeader data
- del meta[test_attr]
+ meta.drop(test_attr)
# Test for empty MetaHeader
assert meta.header.to_dict() == {}
diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py
index 17195f174..2693a176c 100644
--- a/pysat/tests/test_meta_labels.py
+++ b/pysat/tests/test_meta_labels.py
@@ -216,11 +216,11 @@ def test_update(self):
# ----------------------------------------
# Test the integration with the Meta class
- def test_del_from_meta(self):
+ def test_drop_from_meta(self):
"""Test successful deletion of MetaLabels attribute from Meta."""
# Delete the desired label
del_label = list(self.meta_labels.label_type.keys())[0]
- del self.meta[del_label]
+ self.meta.drop(del_label)
# Ensure the label is missing
assert not hasattr(self.meta.labels, del_label)
From 069884ad9f47f85954c5561f1b23da683c61de81 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 28 Sep 2023 14:33:35 -0400
Subject: [PATCH 108/365] TST: added skip
Added a skip to both new meta tests.
---
pysat/tests/test_meta.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index aa993792c..502ee7bc8 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -1188,6 +1188,7 @@ def test_meta_merge(self):
meta_dict[label].__repr__())
return
+ @pytest.mark.skip('See if memory is still an issue')
@pytest.mark.parametrize("names", ['uts', ['uts', 'mlt'], 'units',
['units', 'uts']])
def test_meta_drop(self, names):
@@ -1553,6 +1554,7 @@ def test_update_epoch(self):
# Tests for higher order metadata
# TODO(#789): remove tests for higher order meta
+ @pytest.mark.skip('See if memory is still an issue')
@pytest.mark.parametrize("names", ['series_profiles',
['uts', 'series_profiles'],
['units', 'series_profiles']])
From 742902701405f3448b70de498f4201fb45909a8e Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 15:06:22 -0400
Subject: [PATCH 109/365] ENH: pass overwrite through
---
pysat/utils/registry.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py
index 97193ccbc..ac42838e1 100644
--- a/pysat/utils/registry.py
+++ b/pysat/utils/registry.py
@@ -208,7 +208,7 @@ class Foo(object):
return
-def register_by_module(module):
+def register_by_module(module, overwrite=False):
"""Register all sub-modules attached to input module.
Enables instantiation of a third-party Instrument module using
@@ -221,6 +221,9 @@ def register_by_module(module):
module : Python module
Module with one or more pysat.Instrument support modules
attached as sub-modules to the input `module`
+ overwrite : bool
+ If True, an existing registration will be updated
+ with the new module information.
Raises
------
@@ -249,7 +252,7 @@ def register_by_module(module):
module_names = [module.__name__ + '.' + mod for mod in module_names]
# Register all of the sub-modules
- register(module_names)
+ register(module_names, overwrite=overwrite)
return
From 55956dbe8e16a3e3552c46653a34b0038290bf1f Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 15:07:26 -0400
Subject: [PATCH 110/365] DOC: update links
---
pysat/_files.py | 4 ++--
pysat/_instrument.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/pysat/_files.py b/pysat/_files.py
index 21203b136..337a0243f 100644
--- a/pysat/_files.py
+++ b/pysat/_files.py
@@ -46,8 +46,8 @@ class Files(object):
`month`, `day`, etc. will be filled in as needed using python
string formatting. The default file format structure is supplied in the
instrument `list_files` routine. See
- `pysat.files.parse_delimited_filenames` and
- `pysat.files.parse_fixed_width_filenames` for more information.
+ `pysat.utils.files.parse_delimited_filenames` and
+ `pysat.utils.files.parse_fixed_width_filenames` for more information.
(default=None)
write_to_disk : bool
If true, the list of Instrument files will be written to disk.
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 5405e1ee6..5e53cce32 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -75,8 +75,8 @@ class Instrument(object):
`month`, `day`, etc. will be filled in as needed using python
string formatting. The default file format structure is supplied
in the instrument `list_files` routine. See
- `pysat.files.parse_delimited_filenames` and
- `pysat.files.parse_fixed_width_filenames` for more information.
+ `pysat.utils.files.parse_delimited_filenames` and
+ `pysat.utils.files.parse_fixed_width_filenames` for more information.
The value will be None if not specified by the user at instantiation.
(default=None)
temporary_file_list : bool
From 41b65a0b93fa7ae96ecd5c058212bfd8c6dbb290 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 15:11:22 -0400
Subject: [PATCH 111/365] DOC: update docstrings
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 31c5ec17e..441e38162 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,12 +15,14 @@ This project adheres to [Semantic Versioning](https://semver.org/).
memory usage in the `load` method.
* Added a hidden method the Instrument class `_get_epoch_name_from_data` to
reduce code duplication.
+ * Added the overwrite kwarg to `utils.registry.register_by_module`
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
* Updated `Constellation.to_inst` method definition of coords, using dims
to combine common dimensions instead.
* Implement pyproject to manage metadata
+ * Updated docstring references to `pysat.utils.files` in other modules.
[3.1.0] - 2023-05-31
--------------------
From 06aa436ee77470c966b5872d19638c3f657ffcad Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 15:17:27 -0400
Subject: [PATCH 112/365] MAINT: remove sphinx cap
---
pyproject.toml | 2 +-
test_requirements.txt | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 745fcdc2f..73277f7ba 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -69,7 +69,7 @@ doc = [
"ipython",
"m2r2",
"numpydoc",
- "sphinx < 7.0",
+ "sphinx",
"sphinx_rtd_theme >= 1.2.2"
]
diff --git a/test_requirements.txt b/test_requirements.txt
index 823ea6406..99a4e74fb 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -8,5 +8,5 @@ numpydoc
pysatSpaceWeather
pytest-cov
pytest-ordering
-sphinx<7.0
+sphinx
sphinx_rtd_theme
From a5e4030c397aa3ff51fb2f4bc6ee66ae5c66ac8c Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 15:17:48 -0400
Subject: [PATCH 113/365] DOC: update changelog
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 31c5ec17e..c09845187 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Updated `Constellation.to_inst` method definition of coords, using dims
to combine common dimensions instead.
* Implement pyproject to manage metadata
+ * Remove Sphinx cap
[3.1.0] - 2023-05-31
--------------------
From 6e64411ac4d16eeac2dfdd0526e8b887101b2b5f Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 16:02:37 -0400
Subject: [PATCH 114/365] STY: any / all
---
pysat/_instrument.py | 23 ++++++++++++-----------
pysat/_meta.py | 18 +++++++++---------
pysat/instruments/methods/general.py | 4 ++--
pysat/tests/test_utils_io.py | 5 +++--
pysat/utils/time.py | 4 ++--
5 files changed, 28 insertions(+), 26 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 5405e1ee6..21a64eb34 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -409,9 +409,9 @@ def __init__(self, platform=None, name=None, tag='', inst_id='',
# Check to make sure value is reasonable
if self.file_format is not None:
# Check if it is an iterable string
- if(not isinstance(self.file_format, str)
- or (self.file_format.find("{") < 0)
- or (self.file_format.find("}") < 0)):
+ if any([(not isinstance(self.file_format, str)),
+ (self.file_format.find("{") < 0),
+ (self.file_format.find("}") < 0)]):
raise ValueError(''.join(['file format set to default, ',
'supplied string must be iterable ',
'[{:}]'.format(self.file_format)]))
@@ -883,8 +883,9 @@ def __getitem_xarray__(self, key, data=None):
# Find secondary time indexes that may need to be sliced
if len(data.indexes) > 1:
for ind in data.indexes.keys():
- if(ind != epoch_names[0] and data.indexes[ind].dtype
- == data.indexes[epoch_names[0]].dtype):
+ if all([ind != epoch_names[0],
+ data.indexes[ind].dtype
+ == data.indexes[epoch_names[0]].dtype]):
epoch_names.append(ind)
if isinstance(key, tuple):
@@ -2270,8 +2271,8 @@ def bounds(self, value=None):
if self.files.stop_date is not None:
# Ensure the start and stop times intersect with
# the file list
- if(start <= self.files.stop_date
- and stops[i] >= self.files.start_date):
+ if all([start <= self.files.stop_date,
+ stops[i] >= self.files.start_date]):
good_bounds.append(i)
if len(good_bounds) > 0:
@@ -3843,8 +3844,8 @@ def download(self, start=None, stop=None, date_array=None,
# Get current bounds
curr_bound = self.bounds
if self._iter_type == 'date':
- if(curr_bound[0][0] == first_date
- and curr_bound[1][0] == last_date):
+ if all([curr_bound[0][0] == first_date,
+ curr_bound[1][0] == last_date]):
pysat.logger.info(' '.join(('Updating instrument',
'object bounds by date')))
self.bounds = (self.files.start_date,
@@ -3859,8 +3860,8 @@ def download(self, start=None, stop=None, date_array=None,
dsel2 = slice(last_date, last_date
+ dt.timedelta(hours=23, minutes=59,
seconds=59))
- if(curr_bound[0][0] == self.files[dsel1][0]
- and curr_bound[1][0] == self.files[dsel2][-1]):
+ if all([curr_bound[0][0] == self.files[dsel1][0],
+ curr_bound[1][0] == self.files[dsel2][-1]]):
pysat.logger.info(' '.join(('Updating instrument',
'object bounds by file')))
dsel1 = slice(self.files.start_date,
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 85d5c8d54..b04e4d7bc 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -182,8 +182,8 @@ def __init__(self, metadata=None, header_data=None,
# Set the NaN export list
self._export_nan = [] if export_nan is None else export_nan
for lvals in labels.values():
- if(lvals[0] not in self._export_nan
- and float in pysat.utils.listify(lvals[1])):
+ if all([lvals[0] not in self._export_nan,
+ float in pysat.utils.listify(lvals[1])]):
self._export_nan.append(lvals[0])
# Set the labels
@@ -427,9 +427,9 @@ def __setitem__(self, data_vars, input_dat):
to_be_set, self.labels.label_type[iattr]):
# If this is a disagreement between byte data
# and an expected str, resolve it here
- if(isinstance(to_be_set, bytes)
- and str in pysat.utils.listify(
- self.labels.label_type[iattr])):
+ if all([isinstance(to_be_set, bytes),
+ str in pysat.utils.listify(
+ self.labels.label_type[iattr])]):
to_be_set = core_utils.stringify(to_be_set)
else:
# This type is incorrect, try casting it
@@ -612,8 +612,8 @@ def match_name(func, var_name, index_or_column):
return self.data.loc[new_index, new_name]
except KeyError as kerr:
# This may instead be a child variable, check for children
- if(hasattr(self[new_index], 'children')
- and self[new_index].children is None):
+ if all([hasattr(self[new_index], 'children'),
+ self[new_index].children is None]):
raise kerr
try:
@@ -821,8 +821,8 @@ def _insert_default_values(self, data_var, data_type=None):
for i, lattr in enumerate(self.labels.label_type.keys()):
labels.append(getattr(self.labels, lattr))
lattrs.append(lattr)
- if(isinstance(self.labels.label_type[lattr], tuple)
- and data_type is not None):
+ if all([isinstance(self.labels.label_type[lattr], tuple),
+ data_type is not None]):
need_data_type[lattr] = True
else:
need_data_type[lattr] = False
diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py
index 732163702..1d6f17343 100644
--- a/pysat/instruments/methods/general.py
+++ b/pysat/instruments/methods/general.py
@@ -131,8 +131,8 @@ def list_files(tag='', inst_id='', data_path='', format_str=None,
new_out = out.asfreq('D')
for i, out_month in enumerate(out.index):
- if(out_month.month == emonth.month
- and out_month.year == emonth.year):
+ if all([out_month.month == emonth.month,
+ out_month.year == emonth.year]):
out_month = emonth
crange = pds.date_range(start=out_month, periods=2,
diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py
index 7ac9c1e11..25c769e5b 100644
--- a/pysat/tests/test_utils_io.py
+++ b/pysat/tests/test_utils_io.py
@@ -991,8 +991,9 @@ def test_filter_netcdf4_metadata(self, remove, check_type, export_nan,
assert mkey not in export_nan, \
"{:} should have been exported".format(repr(mkey))
else:
- if(mkey in export_nan and not np.issubdtype(data_type, str)
- and np.isnan(mdict[mkey])):
+ if all([mkey in export_nan,
+ not np.issubdtype(data_type, str),
+ np.isnan(mdict[mkey])]):
assert np.isnan(fdict[mkey])
else:
if mkey in check_type and fdict[mkey] != mdict[mkey]:
diff --git a/pysat/utils/time.py b/pysat/utils/time.py
index 4523aaf85..6ffa84d12 100644
--- a/pysat/utils/time.py
+++ b/pysat/utils/time.py
@@ -364,8 +364,8 @@ def filter_datetime_input(date):
if hasattr(date, '__iter__'):
out_date = []
for in_date in date:
- if(in_date.tzinfo is not None
- and in_date.utcoffset() is not None):
+ if all([in_date.tzinfo is not None,
+ in_date.utcoffset() is not None]):
in_date = in_date.astimezone(tz=dt.timezone.utc)
out_date.append(dt.datetime(in_date.year, in_date.month,
From 911ac243881e2c7600b48ef15c9bb2ec33515093 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 16:02:44 -0400
Subject: [PATCH 115/365] STY: whitespace
---
pysat/tests/test_files.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py
index be2b38daa..c50c19dd1 100644
--- a/pysat/tests/test_files.py
+++ b/pysat/tests/test_files.py
@@ -586,7 +586,8 @@ def test_instrument_has_no_files(self):
inst = pysat.Instrument(platform='pysat', name='testing',
update_files=True)
reload(pysat.instruments.pysat_testing)
- assert(inst.files.files.empty)
+ assert inst.files.files.empty
+
return
def test_instrument_has_files(self):
From 8bfac919ae424c4ce127a626fa79e08d7a75dc64 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 16:03:10 -0400
Subject: [PATCH 116/365] MAINT: consistent version limits
---
pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyproject.toml b/pyproject.toml
index 73277f7ba..d65b86913 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -70,7 +70,7 @@ doc = [
"m2r2",
"numpydoc",
"sphinx",
- "sphinx_rtd_theme >= 1.2.2"
+ "sphinx_rtd_theme"
]
[project.urls]
From 4a9d13a4b0ce552cdad5eb617ca19c9911e25c43 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 16:09:31 -0400
Subject: [PATCH 117/365] STY: any
---
pysat/_constellation.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/pysat/_constellation.py b/pysat/_constellation.py
index 34e5866c9..c51a43fd3 100644
--- a/pysat/_constellation.py
+++ b/pysat/_constellation.py
@@ -680,10 +680,10 @@ def to_inst(self, common_coord=True, fill_method=None):
# a common range and resolution. Note that
# this will only happen if the coordinates
# share the same names.
- if(len(coords[new_coord])
- != len(cinst.data.coords[new_coord])
- or coords[new_coord].values
- != cinst.data.coords[new_coord].values):
+ if any([len(cinst.data.coords[new_coord])
+ != len(coords[new_coord]),
+ cinst.data.coords[new_coord].values
+ != coords[new_coord].values]):
coords[new_coord] = establish_common_coord(
[coords[new_coord].values,
cinst.data.coords[new_coord].values],
From 2f977402b7569491d688fe6dbf6eeeaed17a3387 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 2 Oct 2023 16:19:39 -0400
Subject: [PATCH 118/365] STY: use a slice
---
pysat/_orbits.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/pysat/_orbits.py b/pysat/_orbits.py
index 93effb84d..7f68f77bd 100644
--- a/pysat/_orbits.py
+++ b/pysat/_orbits.py
@@ -465,7 +465,8 @@ def _equa_breaks(self, orbit_index_period=24.0):
# values
new_ind = []
for idx in ind:
- tidx, = np.where(lt_diff[(idx - 5):(idx + 6)]
+ sub_idx = slice((idx - 5), (idx + 6))
+ tidx, = np.where(lt_diff[sub_idx]
> 10 * typical_lt_diff)
if len(tidx) != 0:
@@ -473,9 +474,8 @@ def _equa_breaks(self, orbit_index_period=24.0):
# Iterate over samples and check.
for sub_tidx in tidx:
# Look at time change vs local time change
- if(ut_diff[idx - 5:idx + 6].iloc[sub_tidx]
- < lt_diff[idx - 5:idx + 6].iloc[sub_tidx]
- / orbit_index_period * self.orbit_period):
+ if(ut_diff[sub_idx].iloc[sub_tidx] * orbit_index_period
+ < lt_diff[sub_idx].iloc[sub_tidx] * self.orbit_period):
# The change in UT is small compared to the change
# in the orbit index this is flagged as a false
From ee79d50002be2b4f6f25c442dec904b38c8039c3 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 10:11:45 -0400
Subject: [PATCH 119/365] MAINT: sphinx_rtd_theme minimum
---
pyproject.toml | 2 +-
test_requirements.txt | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index d65b86913..73277f7ba 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -70,7 +70,7 @@ doc = [
"m2r2",
"numpydoc",
"sphinx",
- "sphinx_rtd_theme"
+ "sphinx_rtd_theme >= 1.2.2"
]
[project.urls]
diff --git a/test_requirements.txt b/test_requirements.txt
index 99a4e74fb..dc679dcce 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -9,4 +9,4 @@ pysatSpaceWeather
pytest-cov
pytest-ordering
sphinx
-sphinx_rtd_theme
+sphinx_rtd_theme>=1.2.2
From bc4cf0ee27dcb36fda61dacae30fa19a72915c1d Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 10:11:54 -0400
Subject: [PATCH 120/365] MAINT: pandas cap
---
pyproject.toml | 2 +-
requirements.txt | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 73277f7ba..9ec0b533b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,7 +45,7 @@ dependencies = [
"dask",
"netCDF4",
"numpy >= 1.12",
- "pandas",
+ "pandas < 2.1.1",
"portalocker",
"pytest",
"scipy",
diff --git a/requirements.txt b/requirements.txt
index 6671a1449..059c99271 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
dask
netCDF4
numpy>=1.12
-pandas
+pandas<2.1.1
portalocker
pytest
scipy
From 7e5977e8d57596153cf6d7dd761ea392e7b72232 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 10:34:33 -0400
Subject: [PATCH 121/365] DOC: update changelog
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c09845187..b547a3889 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
to combine common dimensions instead.
* Implement pyproject to manage metadata
* Remove Sphinx cap
+ * Add pandas cap
[3.1.0] - 2023-05-31
--------------------
From 7d00795819c57023b70cee4347e99234e56aa86d Mon Sep 17 00:00:00 2001
From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
Date: Tue, 3 Oct 2023 10:57:55 -0400
Subject: [PATCH 122/365] Apply suggestions from code review
---
pyproject.toml | 2 +-
requirements.txt | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 9ec0b533b..311c0382d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,7 +45,7 @@ dependencies = [
"dask",
"netCDF4",
"numpy >= 1.12",
- "pandas < 2.1.1",
+ "pandas < 2.1",
"portalocker",
"pytest",
"scipy",
diff --git a/requirements.txt b/requirements.txt
index 059c99271..c5577679a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
dask
netCDF4
numpy>=1.12
-pandas<2.1.1
+pandas<2.1
portalocker
pytest
scipy
From c5caa2a7ec559de583f0eca84763055abf19e53a Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 11:09:40 -0400
Subject: [PATCH 123/365] MAINT: remove pysat_testing2d
---
pysat/instruments/__init__.py | 2 +-
pysat/instruments/methods/testing.py | 33 ----
pysat/instruments/pysat_testing2d.py | 231 ---------------------------
3 files changed, 1 insertion(+), 265 deletions(-)
delete mode 100644 pysat/instruments/pysat_testing2d.py
diff --git a/pysat/instruments/__init__.py b/pysat/instruments/__init__.py
index d1fb406db..248e80f36 100644
--- a/pysat/instruments/__init__.py
+++ b/pysat/instruments/__init__.py
@@ -6,7 +6,7 @@
__all__ = ['pysat_ndtesting', 'pysat_netcdf', 'pysat_testing',
'pysat_testmodel', 'pysat_testing_xarray',
- 'pysat_testing2d', 'pysat_testing2d_xarray']
+ 'pysat_testing2d_xarray']
for inst in __all__:
exec("from pysat.instruments import {x}".format(x=inst))
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index 8c0b2838f..cb25480af 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -247,39 +247,6 @@ def initialize_test_meta(epoch_name, data_keys):
'Note the value_max is largest netCDF4 supports, ',
'but is lower than actual 64-bit int limit.'])}
- # Children metadata required for 2D pandas.
- # TODO(#789): Delete after removal of Meta children.
- series_profile_meta = pysat.Meta()
- series_profile_meta['series_profiles'] = {'desc': 'Testing series data.',
- 'value_min': 0,
- 'value_max': np.inf,
- 'units': 'm/s'}
- meta['series_profiles'] = {'meta': series_profile_meta,
- 'value_min': 0., 'value_max': 25., 'units': 'km',
- 'fill': np.nan,
- 'desc': ''.join(['Testing series profiles ',
- 'indexed by float.'])}
-
- # Children metadata required for 2D pandas.
- # TODO(#789): Delete after removal of Meta children.
- data_types = {'density': float, 'fraction': float, 'alt_profiles': float,
- 'variable_profiles': float, 'profile_height': int,
- 'variable_profile_height': int, 'images': int, 'x': int,
- 'y': int, 'z': int, 'image_lat': float, 'image_lon': float}
- alt_profile_meta = pysat.Meta()
- alt_profile_meta['density'] = {'desc': 'Simulated density values.',
- 'units': 'Log N/cc',
- 'value_min': 0, 'value_max': np.inf}
- alt_profile_meta['fraction'] = {'value_min': 0., 'value_max': 1.,
- 'desc': ''.join(['Simulated fractional O+ ',
- 'composition.'])}
- meta['alt_profiles'] = {'value_min': 0., 'value_max': 25., 'fill': np.nan,
- 'desc': ''.join([
- 'Testing profile multi-dimensional data ',
- 'indexed by float.']),
- 'units': 'km',
- 'meta': alt_profile_meta}
-
# Optional and standard metadata for xarray
for var in data_keys:
if var.find('variable_profiles') == 0:
diff --git a/pysat/instruments/pysat_testing2d.py b/pysat/instruments/pysat_testing2d.py
deleted file mode 100644
index 76819a6a4..000000000
--- a/pysat/instruments/pysat_testing2d.py
+++ /dev/null
@@ -1,231 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Produces fake instrument data for testing.
-
-.. deprecated:: 3.0.2
- Support for 2D pandas objects will be removed in 3.2.0+. This instrument
- module simulates an object that will no longer be supported.
-
-"""
-
-import datetime as dt
-import functools
-import numpy as np
-import warnings
-
-import pandas as pds
-
-import pysat
-from pysat.instruments.methods import testing as mm_test
-
-platform = 'pysat'
-name = 'testing2d'
-tags = {'': 'Regular testing data set'}
-inst_ids = {'': ['']}
-_test_dates = {'': {'': dt.datetime(2009, 1, 1)}}
-
-
-# Init method
-def init(self, test_init_kwarg=None):
- """Initialize the test instrument.
-
- Parameters
- ----------
- self : pysat.Instrument
- This object
- test_init_kwarg : any
- Testing keyword (default=None)
-
- """
-
- warnings.warn(" ".join(["The instrument module `pysat_testing2d` has been",
- "deprecated and will be removed in 3.2.0+. This",
- "module simulates an object that will no longer be",
- "supported."]),
- DeprecationWarning, stacklevel=2)
-
- mm_test.init(self, test_init_kwarg=test_init_kwarg)
- return
-
-
-# Clean method
-clean = mm_test.clean
-
-# Optional method, preprocess
-preprocess = mm_test.preprocess
-
-
-def load(fnames, tag='', inst_id='', malformed_index=False,
- start_time=None, num_samples=864, test_load_kwarg=None,
- max_latitude=90.):
- """Load the test files.
-
- Parameters
- ----------
- fnames : list
- List of filenames
- tag : str
- Tag name used to identify particular data set to be loaded.
- This input is nominally provided by pysat itself. (default='')
- inst_id : str
- Instrument ID used to identify particular data set to be loaded.
- This input is nominally provided by pysat itself. (default='')
- malformed_index : bool
- If True, the time index will be non-unique and non-monotonic.
- (default=False)
- start_time : dt.timedelta or NoneType
- Offset time of start time since midnight UT. If None, instrument data
- will begin at midnight.
- (default=None)
- num_samples : int
- Maximum number of times to generate. Data points will not go beyond the
- current day. (default=864)
- test_load_kwarg : any
- Keyword used for pysat unit testing to ensure that functionality for
- custom keywords defined in instrument support functions is working
- correctly. (default=None)
- max_latitude : float
- Latitude simulated as `max_latitude` * cos(theta(t))`, where
- theta is a linear periodic signal bounded by [0, 2 * pi) (default=90.).
-
- Returns
- -------
- data : pds.DataFrame
- Testing data
- meta : pysat.Meta
- Testing metadata
-
- """
-
- # Support keyword testing
- pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))
-
- # Create an artificial satellite data set
- iperiod = mm_test.define_period()
- drange = mm_test.define_range()
-
- # Using 100s frequency for compatibility with seasonal analysis unit tests
- uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='100S',
- start_time=start_time)
- # Seed the DataFrame with a UT array
- data = pds.DataFrame(np.mod(uts, 86400.), columns=['uts'])
-
- # Need to create simple orbits here. Have start of first orbit
- # at 2009,1, 0 UT. 14.84 orbits per day. Figure out how far in time from
- # the root start a measurement is and use that info to create a signal
- # that is continuous from that start. Going to presume there are 5820
- # seconds per orbit (97 minute period).
- time_delta = dates[0] - dt.datetime(2009, 1, 1)
-
- # MLT runs 0-24 each orbit
- data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
- period=iperiod['lt'],
- data_range=drange['lt'])
-
- # SLT, 20 second offset from `mlt`.
- data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
- uts, period=iperiod['lt'],
- data_range=drange['lt'])
-
- # Create a fake longitude, resets every 6240 seconds. Sat moves at
- # 360/5820 deg/s, Earth rotates at 360/86400, takes extra time to go
- # around full longitude.
- data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(),
- uts, period=iperiod['lon'],
- data_range=drange['lon'])
-
- # Create latitude signal for testing polar orbits
- angle = mm_test.generate_fake_data(time_delta.total_seconds(),
- uts, period=iperiod['angle'],
- data_range=drange['angle'])
- data['latitude'] = max_latitude * np.cos(angle)
-
- # Create constant altitude at 400 km
- alt0 = 400.0
- data['altitude'] = alt0 * np.ones(data['latitude'].shape)
-
- # Dummy variable data for different types
- data['string_dummy'] = ['test'] * len(data)
- data['unicode_dummy'] = [u'test'] * len(data)
- data['int8_dummy'] = np.ones(len(data), dtype=np.int8)
- data['int16_dummy'] = np.ones(len(data), dtype=np.int16)
- data['int32_dummy'] = np.ones(len(data), dtype=np.int32)
- data['int64_dummy'] = np.ones(len(data), dtype=np.int64)
-
- if malformed_index:
- mm_test._warn_malformed_kwarg()
- index = mm_test.non_monotonic_index(index)
- index = mm_test.non_unique_index(index)
-
- data.index = index
- data.index.name = 'Epoch'
-
- # Higher rate time signal (for scalar >= 2). This time signal is used
- # for 2D profiles associated with each time in main DataFrame.
- num_profiles = 50 if num_samples >= 50 else num_samples
- end_date = dates[0] + dt.timedelta(seconds=2 * num_profiles - 1)
- high_rate_template = pds.date_range(dates[0], end_date, freq='2S')
-
- # Create a few simulated profiles. This results in a pds.DataFrame at
- # each time with mixed variables.
- profiles = []
-
- # DataFrame at each time with numeric variables only
- alt_profiles = []
-
- # Series at each time, numeric data only
- series_profiles = []
-
- # Frame indexed by date times
- frame = pds.DataFrame({'density':
- data.iloc[0:num_profiles]['mlt'].values.copy(),
- 'dummy_str': ['test'] * num_profiles,
- 'dummy_ustr': [u'test'] * num_profiles},
- index=data.index[0:num_profiles],
- columns=['density', 'dummy_str', 'dummy_ustr'])
-
- # Frame indexed by float
- dd = np.arange(num_profiles) * 1.2
- ff = np.arange(num_profiles) / num_profiles
- ii = np.arange(num_profiles) * 0.5
- frame_alt = pds.DataFrame({'density': dd, 'fraction': ff},
- index=ii,
- columns=['density', 'fraction'])
-
- # Series version of storage
- series_alt = pds.Series(dd, index=ii, name='series_profiles')
-
- for time in data.index:
- frame.index = high_rate_template + (time - data.index[0])
- profiles.append(frame)
- alt_profiles.append(frame_alt)
- series_profiles.append(series_alt)
-
- # Store multiple data types into main frame
- data['profiles'] = pds.Series(profiles, index=data.index)
- data['alt_profiles'] = pds.Series(alt_profiles, index=data.index)
- data['series_profiles'] = pds.Series(series_profiles, index=data.index)
-
- # Set the meta data
- meta = mm_test.initialize_test_meta('epoch', data.keys())
-
- # Reset profiles as children meta
- profile_meta = pysat.Meta()
- profile_meta['density'] = {'long_name': 'density', 'units': 'N/cc',
- 'desc': 'Fake "density" signal for testing.',
- 'value_min': 0., 'value_max': 25.,
- 'fill': np.nan}
- profile_meta['dummy_str'] = {'long_name': 'dummy_str',
- 'desc': 'String data for testing.'}
- profile_meta['dummy_ustr'] = {'long_name': 'dummy_ustr',
- 'desc': 'Unicode string data for testing.'}
-
- # Update profiles metadata with sub-variable information
- meta['profiles'] = {'meta': profile_meta}
-
- return data, meta
-
-
-list_files = functools.partial(mm_test.list_files, test_dates=_test_dates)
-list_remote_files = functools.partial(mm_test.list_remote_files,
- test_dates=_test_dates)
-download = functools.partial(mm_test.download)
From 8ea5b51dab386f6b7f3c7ae823a726e2b27821b8 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 11:11:58 -0400
Subject: [PATCH 124/365] MAINT: remove meta children
---
pysat/_meta.py | 69 +-------------------------------------------------
1 file changed, 1 insertion(+), 68 deletions(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 85d5c8d54..4b4fe55f5 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -500,47 +500,9 @@ def __setitem__(self, data_vars, input_dat):
in_dict = input_data.to_dict()
if 'children' in in_dict:
child = in_dict.pop('children')
- if child is not None:
- # If there is data in the child object, assign it here
- self._warn_meta_children()
- self.ho_data[data_vars] = child
# Remaining items are simply assigned via recursive call
self[data_vars] = in_dict
- elif isinstance(input_data, Meta):
- # Dealing with a higher order data set
- self._warn_meta_children()
- # `data_vars` is only a single name here (by choice for support)
- if (data_vars in self._ho_data) and input_data.empty:
- # No actual metadata provided and there is already some
- # higher order metadata in self
- return
-
- # Get Meta approved variable data names
- new_item_name = self.var_case_name(data_vars)
-
- # Ensure that Meta labels of object to be assigned are
- # consistent with self. input_data accepts self's labels.
- input_data.accept_default_labels(self)
-
- # Go through and ensure Meta object to be added has variable and
- # attribute names consistent with other variables and attributes
- # this covers custom attributes not handled by default routine
- # above
- attr_names = [item for item in input_data.attrs()]
- input_data.data.columns = self.attr_case_name(attr_names)
-
- # Same thing for variables
- input_data.data.index = self.var_case_name(input_data.data.index)
-
- # Assign Meta object now that things are consistent with Meta
- # object settings, but first make sure there are lower dimension
- # metadata parameters, passing in an empty dict fills in defaults
- # if there is no existing metadata info.
- self[new_item_name] = {}
-
- # Now add to higher order data
- self._ho_data[new_item_name] = input_data
return
def __getitem__(self, key):
@@ -565,7 +527,7 @@ def __getitem__(self, key):
::
import pysat
- inst = pysat.Instrument('pysat', 'testing2d')
+ inst = pysat.Instrument('pysat', 'testing')
inst.load(date=inst.inst_module._test_dates[''][''])
meta = inst.meta
@@ -1568,26 +1530,6 @@ def to_dict(self, preserve_case=False):
for orig_key in meta_dict:
export_dict[case_key][orig_key] = meta_dict[orig_key]
- # Higher Order Data
- # TODO(#789): remove in pysat 3.2.0
- for key in self.ho_data:
- if preserve_case:
- case_key = self.var_case_name(key)
- else:
- case_key = key.lower()
-
- if case_key not in export_dict:
- export_dict[case_key] = {}
- for ho_key in self.ho_data[key].data.index:
- if preserve_case:
- case_ho_key = self.var_case_name(ho_key)
- else:
- case_ho_key = ho_key.lower()
-
- new_key = '_'.join((case_key, case_ho_key))
- export_dict[new_key] = \
- self.ho_data[key].data.loc[ho_key].to_dict()
-
return export_dict
@classmethod
@@ -1650,15 +1592,6 @@ def from_csv(cls, filename=None, col_names=None, sep=None, **kwargs):
raise ValueError(''.join(['Unable to retrieve information from ',
filename]))
- # TODO(#789): remove in pysat 3.2.0
- def _warn_meta_children(self):
- """Warn the user that higher order metadata is deprecated."""
-
- warnings.warn(" ".join(["Support for higher order metadata has been",
- "deprecated and will be removed in 3.2.0+."]),
- DeprecationWarning, stacklevel=2)
- return
-
class MetaLabels(object):
"""Store metadata labels for Instrument instance.
From 6dd81bf5e5db856ce4d03d5907c08dc670082fbe Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 11:12:30 -0400
Subject: [PATCH 125/365] MAINT: update export
---
pysat/utils/io.py | 145 +---------------------------------------------
1 file changed, 2 insertions(+), 143 deletions(-)
diff --git a/pysat/utils/io.py b/pysat/utils/io.py
index 571b6a31d..0483d1653 100644
--- a/pysat/utils/io.py
+++ b/pysat/utils/io.py
@@ -986,152 +986,11 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4',
nc_key)
full_mdict[key] = meta_dict
- # TODO(#913): Remove 2D support
- if len(data.variables[key].dimensions) == 2:
- # Part of a DataFrame to store within the main DataFrame
- two_d_keys.append(key)
- two_d_dims.append(data.variables[key].dimensions)
-
- if len(data.variables[key].dimensions) >= 3:
- raise ValueError(' '.join(('pysat only supports 1D and 2D',
+ if len(data.variables[key].dimensions) >= 2:
+ raise ValueError(' '.join(('pysat only supports 1D',
'data in pandas. Please use',
'xarray for this file.')))
- # TODO(#913): Remove 2D support
- # We now have a list of keys that need to go into a dataframe,
- # could be more than one, collect unique dimensions for 2D keys.
- for dim in set(two_d_dims):
- # First or second dimension could be epoch. Use other
- # dimension name as variable name.
- if dim[0] == epoch_name:
- obj_key = dim[1]
- elif dim[1] == epoch_name:
- obj_key = dim[0]
- else:
- estr = ''.join(['Epoch label: "', epoch_name, '"',
- ' was not found in loaded dimensions [',
- ', '.join(dim), ']'])
- raise KeyError(estr)
-
- # Collect variable names associated with dimension
- idx_bool = [dim == i for i in two_d_dims]
- idx, = np.where(np.array(idx_bool))
- obj_var_keys = []
- clean_var_keys = []
- for i in idx:
- obj_var_keys.append(two_d_keys[i])
- clean_var_keys.append(
- two_d_keys[i].split(obj_key + '_')[-1])
-
- # Figure out how to index this data, it could provide its
- # own index - or we may have to create simple integer based
- # DataFrame access. If the dimension is stored as its own
- # variable then use that info for index.
- if obj_key in obj_var_keys:
- # String used to indentify dimension also in
- # data.variables will be used as an index.
- index_key_name = obj_key
-
- # If the object index uses UNIX time, process into
- # datetime index.
- if data.variables[obj_key].getncattr(
- name_label) == epoch_name:
- # Found the name to be used in DataFrame index
- index_name = epoch_name
- time_index_flag = True
- else:
- time_index_flag = False
-
- # Label to be used in DataFrame index
- index_name = data.variables[obj_key].getncattr(
- name_label)
- else:
- # Dimension is not itself a variable
- index_key_name = None
-
- # Iterate over the variables and grab metadata
- dim_meta_data = {}
-
- # Store attributes in metadata, except for the dimension name.
- for key, clean_key in zip(obj_var_keys, clean_var_keys):
- meta_dict = {}
- for nc_key in data.variables[key].ncattrs():
- meta_dict[nc_key] = data.variables[key].getncattr(
- nc_key)
-
- dim_meta_data[clean_key] = meta_dict
-
- dim_meta_dict = {'meta': dim_meta_data}
-
- # Add top level meta
- if index_key_name is not None:
- for nc_key in data.variables[obj_key].ncattrs():
- dim_meta_dict[nc_key] = data.variables[
- obj_key].getncattr(nc_key)
- full_mdict[obj_key] = dim_meta_dict
-
- # Iterate over all variables with this dimension
- # data storage, whole shebang.
- loop_dict = {}
-
- # List holds a series of slices, parsed from dict above.
- loop_list = []
- for key, clean_key in zip(obj_var_keys, clean_var_keys):
- loop_dict[clean_key] = data.variables[
- key][:, :].flatten(order='C')
-
- # Find the number of time values
- loop_lim = data.variables[obj_var_keys[0]].shape[0]
-
- # Find the number of values per time
- step = len(data.variables[obj_var_keys[0]][0, :])
-
- # Check if there is an index we should use
- if not (index_key_name is None):
- time_var = loop_dict.pop(index_key_name)
- if time_index_flag:
- # Create datetime index from data
- time_var = pds.to_datetime(time_var, unit=epoch_unit,
- origin=epoch_origin)
- new_index = time_var
- new_index_name = index_name
- else:
- # Using integer indexing if no index identified
- new_index = np.arange((loop_lim * step),
- dtype=np.int64) % step
- new_index_name = 'index'
-
- # Load all data into frame
- if len(loop_dict.keys()) > 1:
- loop_frame = pds.DataFrame(loop_dict,
- columns=clean_var_keys)
- if obj_key in loop_frame:
- del loop_frame[obj_key]
-
- # Break massive frame into bunch of smaller frames
- for i in np.arange(loop_lim, dtype=np.int64):
- loop_list.append(loop_frame.iloc[(step * i):
- (step * (i + 1)), :])
- loop_list[-1].index = new_index[(step * i):
- (step * (i + 1))]
- loop_list[-1].index.name = new_index_name
- else:
- loop_frame = pds.Series(loop_dict[clean_var_keys[0]],
- name=obj_var_keys[0])
-
- # Break massive series into bunch of smaller series
- for i in np.arange(loop_lim, dtype=np.int64):
- loop_list.append(loop_frame.iloc[(step * i):
- (step * (i + 1))])
- loop_list[-1].index = new_index[(step * i):
- (step * (i + 1))]
- loop_list[-1].index.name = new_index_name
-
- # Add 2D object data, all based on a unique dimension within
- # netCDF, to loaded data dictionary.
- loaded_vars[obj_key] = loop_list
- del loop_list
-
# Prepare dataframe index for this netcdf file
if epoch_name not in loaded_vars.keys():
estr = ''.join(['Epoch label: "', epoch_name, '"',
From f39fa5ec8c0ff8a7b8b68878e665886fee4cf914 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 11:13:16 -0400
Subject: [PATCH 126/365] MAINT: remove 2d pandas tests
---
pysat/tests/classes/cls_instrument_access.py | 136 ----
pysat/tests/test_instrument.py | 47 --
pysat/tests/test_instrument_index.py | 3 +-
pysat/tests/test_instrument_padding.py | 1 -
pysat/tests/test_instruments.py | 3 +-
pysat/tests/test_meta.py | 642 +------------------
pysat/tests/test_meta_labels.py | 26 -
pysat/tests/test_utils_coords.py | 12 +-
pysat/tests/test_utils_io.py | 94 +--
9 files changed, 16 insertions(+), 948 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py
index b66b11e4a..5e89a7d88 100644
--- a/pysat/tests/classes/cls_instrument_access.py
+++ b/pysat/tests/classes/cls_instrument_access.py
@@ -958,142 +958,6 @@ def test_basic_variable_renaming(self, lowercase, mapper):
assert key not in self.testInst.meta.keys()
return
- @pytest.mark.parametrize("mapper", [
- {'profiles': {'density': 'ionization'}},
- {'profiles': {'density': 'mass'},
- 'alt_profiles': {'density': 'volume'}},
- str.upper])
- def test_ho_pandas_variable_renaming(self, mapper):
- """Test rename of higher order pandas variable.
-
- Parameters
- ----------
- mapper : dict or function
- A function or dict that maps how the variables will be renamed.
-
- """
- # TODO(#789): Remove when meta children support is dropped.
-
- # Initialize the testing dict
- if isinstance(mapper, dict):
- values = mapper
- else:
- values = {var: mapper(var) for var in self.testInst.variables}
-
- # Check for pysat_testing2d instrument
- if self.testInst.platform == 'pysat':
- if self.testInst.name == 'testing2d':
- self.testInst.load(self.ref_time.year, self.ref_doy,
- use_header=True)
- self.testInst.rename(mapper)
- for key in values:
- for ikey in values[key]:
- # Check column name unchanged
- assert key in self.testInst.data
- assert key in self.testInst.meta
-
- # Check for new name in HO data
- check_var = self.testInst.meta[key]['children']
-
- if isinstance(values[key], dict):
- map_val = values[key][ikey]
- else:
- map_val = mapper(ikey)
-
- assert map_val in self.testInst[0, key]
- assert map_val in check_var
-
- # Ensure old name not present
- assert ikey not in self.testInst[0, key]
- if map_val.lower() != ikey:
- assert ikey not in check_var
- return
-
- @pytest.mark.parametrize("values", [{'profiles':
- {'help': 'I need somebody'}},
- {'fake_profi':
- {'help': 'Not just anybody'}},
- {'wrong_profile':
- {'help': 'You know I need someone'},
- 'fake_profiles':
- {'Beatles': 'help!'},
- 'profiles':
- {'density': 'valid_change'}},
- {'fake_profile':
- {'density': 'valid HO change'}},
- {'Nope_profiles':
- {'density': 'valid_HO_change'}}])
- def test_ho_pandas_unknown_variable_error_renaming(self, values):
- """Test higher order pandas variable rename raises error if unknown.
-
- Parameters
- ----------
- values : dict
- Variables to be renamed. A dict where each key is the current
- variable and its value is the new variable name.
-
- """
- # TODO(#789): Remove when meta children support is dropped.
-
- # Check for pysat_testing2d instrument
- if self.testInst.platform == 'pysat':
- if self.testInst.name == 'testing2d':
- self.testInst.load(self.ref_time.year, self.ref_doy,
- use_header=True)
-
- # Check for error for unknown column or HO variable name
- testing.eval_bad_input(self.testInst.rename, ValueError,
- "cannot rename", [values])
- else:
- pytest.skip("Not implemented for this instrument")
- return
-
- @pytest.mark.parametrize("values", [{'profiles': {'density': 'Ionization'}},
- {'profiles': {'density': 'MASa'},
- 'alt_profiles':
- {'density': 'VoLuMe'}}])
- def test_ho_pandas_variable_renaming_lowercase(self, values):
- """Test rename higher order pandas variable uses lowercase.
-
- Parameters
- ----------
- values : dict
- Variables to be renamed. A dict where each key is the current
- variable and its value is the new variable name.
-
- """
- # TODO(#789): Remove when meta children support is dropped.
-
- # Check for pysat_testing2d instrument
- if self.testInst.platform == 'pysat':
- if self.testInst.name == 'testing2d':
- self.testInst.load(self.ref_time.year, self.ref_doy,
- use_header=True)
- self.testInst.rename(values)
- for key in values:
- for ikey in values[key]:
- # Check column name unchanged
- assert key in self.testInst.data
- assert key in self.testInst.meta
-
- # Check for new name in HO data
- test_val = values[key][ikey]
- assert test_val in self.testInst[0, key]
- check_var = self.testInst.meta[key]['children']
-
- # Case insensitive check
- assert values[key][ikey] in check_var
-
- # Ensure new case in there
- check_var = check_var[values[key][ikey]].name
- assert values[key][ikey] == check_var
-
- # Ensure old name not present
- assert ikey not in self.testInst[0, key]
- check_var = self.testInst.meta[key]['children']
- assert ikey not in check_var
- return
-
def test_generic_meta_translator(self):
"""Test `generic_meta_translator`."""
diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py
index 705d3eaa5..18861ca59 100644
--- a/pysat/tests/test_instrument.py
+++ b/pysat/tests/test_instrument.py
@@ -12,7 +12,6 @@
import pysat
import pysat.instruments.pysat_ndtesting
import pysat.instruments.pysat_testing
-import pysat.instruments.pysat_testing2d
import pysat.instruments.pysat_testing_xarray
from pysat.tests.classes.cls_instrument_access import InstAccessTests
@@ -228,32 +227,6 @@ def teardown_method(self):
return
-# TODO(#908): remove below class when pysat_testing2d is removed.
-class TestBasics2D(TestBasics):
- """Basic tests for 2D pandas `pysat.Instrument`."""
-
- def setup_method(self):
- """Set up the unit test environment for each method."""
-
- reload(pysat.instruments.pysat_testing2d)
- self.testInst = pysat.Instrument(platform='pysat', name='testing2d',
- num_samples=50,
- clean_level='clean',
- update_files=True,
- use_header=True,
- **self.testing_kwargs)
- self.ref_time = pysat.instruments.pysat_testing2d._test_dates['']['']
- self.ref_doy = int(self.ref_time.strftime('%j'))
- self.out = None
- return
-
- def teardown_method(self):
- """Clean up the unit test environment after each method."""
-
- del self.testInst, self.out, self.ref_time, self.ref_doy
- return
-
-
class TestBasicsNDXarray(TestBasics):
"""Basic tests for ND xarray `pysat.Instrument`.
@@ -815,23 +788,3 @@ def test_load_use_header(self):
# Evaluate the warning output
self.eval_warnings()
return
-
- def test_set_2d_pandas_data(self):
- """Check that setting 2D data for pandas raises a DeprecationWarning."""
-
- test_inst = pysat.Instrument('pysat', 'testing2d', use_header=True)
- test_date = pysat.instruments.pysat_testing2d._test_dates['']['']
- test_inst.load(date=test_date)
- with warnings.catch_warnings(record=True) as war:
- test_inst['new_profiles'] = 2 * test_inst['profiles']
-
- warn_msgs = [" ".join(["Support for 2D pandas instrument",
- "data has been deprecated and will",
- "be removed in 3.2.0+."])]
-
- # Ensure the minimum number of warnings were raised.
- assert len(war) >= len(warn_msgs)
-
- # Test the warning messages, ensuring each attribute is present.
- testing.eval_warnings(war, warn_msgs)
- return
diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py
index ea41140a5..ac87a74ce 100644
--- a/pysat/tests/test_instrument_index.py
+++ b/pysat/tests/test_instrument_index.py
@@ -107,8 +107,7 @@ def eval_warnings(self):
return
# TODO(#1094): Remove in pysat 3.2.0, potentially with class
- @pytest.mark.parametrize('name', ['testing', 'ndtesting', 'testing_xarray',
- 'testing2d'])
+ @pytest.mark.parametrize('name', ['testing', 'ndtesting', 'testing_xarray'])
def test_kwarg_malformed_index(self, name):
"""Test deprecation of `malformed_index` kwarg.
diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py
index 89610277b..59e02e687 100644
--- a/pysat/tests/test_instrument_padding.py
+++ b/pysat/tests/test_instrument_padding.py
@@ -10,7 +10,6 @@
import pysat
import pysat.instruments.pysat_ndtesting
import pysat.instruments.pysat_testing
-import pysat.instruments.pysat_testing2d
import pysat.instruments.pysat_testing_xarray
from pysat.utils import testing
from pysat.utils.time import filter_datetime_input
diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py
index d5cede4e2..b1f03a42d 100644
--- a/pysat/tests/test_instruments.py
+++ b/pysat/tests/test_instruments.py
@@ -275,8 +275,7 @@ def test_old_pytest_mark_presence(self):
assert "download" in mark_names
- @pytest.mark.parametrize("inst_module", ['pysat_testing2d',
- 'pysat_testing_xarray',
+ @pytest.mark.parametrize("inst_module", ['pysat_testing_xarray',
'pysat_testing2d_xarray'])
def test_deprecated_instruments(self, inst_module):
"""Check that instantiating old instruments raises a DeprecationWarning.
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index a4ec9aec8..9269046ea 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -201,31 +201,6 @@ def test_getitem_w_index(self):
assert str(ierr).find('expected tuple, list, or str') >= 0
return
- # TODO(#913): remove tests for 2D metadata
- @pytest.mark.parametrize("parent_child", [
- (['alt_profiles', 'profiles'], 'density'),
- (['alt_profiles', 'profiles'], ['density', 'dummy_str']),
- (['alt_profiles', 'profiles'], 'density', 'units'),
- (['alt_profiles', 'profiles'], 'density', ['units', 'long_name'])])
- def test_getitem_w_ho_child_slicing(self, parent_child):
- """Test raises NotImplementedError with parent/child slicing.
-
- Parameters
- ----------
- parent_child : list
- List of inputs with unsupported parent/child slicing options
-
- """
-
- # Set the meta object
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'})
-
- with pytest.raises(NotImplementedError) as ierr:
- self.meta[parent_child]
-
- assert str(ierr).find("retrieve child meta data from multiple") >= 0
- return
-
def test_concat_strict_w_collision(self):
"""Test raises KeyError when new meta names overlap."""
@@ -244,25 +219,6 @@ def test_concat_strict_w_collision(self):
return
- # TODO(#913): remove tests for 2d metadata
- def test_concat_strict_w_ho_collision(self):
- """Test raises KeyError when higher-order variable nams overlap."""
-
- # Set the meta object
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'})
-
- # Create a second object with the same higher-order data variables
- concat_meta = pysat.Meta()
- for dvar in self.meta.keys_nD():
- concat_meta[dvar] = self.meta[dvar]
-
- # Test the error message
- testing.eval_bad_input(
- self.meta.concat, KeyError,
- 'Duplicated keys (variable names) in Meta.keys()', [concat_meta],
- {'strict': True})
- return
-
def test_multiple_meta_assignment_error(self):
"""Test that assignment of multiple metadata raises a ValueError."""
@@ -318,20 +274,6 @@ def test_meta_csv_load_w_errors(self, bad_key, bad_val, err_msg):
input_kwargs=kwargs)
return
- # TODO(#913): remove tests for 2D metadata
- def test_meta_rename_bad_ho_input(self):
- """Test raises ValueError when treating normal data like HO data."""
-
- # Initialize the meta data
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'})
-
- # Set a bad mapping dictionary
- mapper = {'mlt': {'mlt_profile': 'mlt_density_is_not_real'}}
-
- testing.eval_bad_input(self.meta.rename, ValueError,
- "unknown mapped value at 'mlt'", [mapper])
- return
-
# -------------------------
# Test the Warning messages
@@ -459,11 +401,9 @@ def test_repr(self):
assert out.find('Meta(') >= 0
return
- # TODO(#913): remove tests for 2d metadata
@pytest.mark.parametrize('long_str', [True, False])
@pytest.mark.parametrize('inst_kwargs',
- [None, {'platform': 'pysat', 'name': 'testing'},
- {'platform': 'pysat', 'name': 'testing2d'}])
+ [None, {'platform': 'pysat', 'name': 'testing'}])
def test_str(self, long_str, inst_kwargs):
"""Test long string output with custom meta data.
@@ -501,10 +441,7 @@ def test_str(self, long_str, inst_kwargs):
else:
assert out.find('Standard Metadata variables:') < 0
- if inst_kwargs is not None and inst_kwargs['name'] == 'testing2d':
- assert out.find('ND Metadata variables:') > 0
- else:
- assert out.find('ND Metadata variables:') < 0
+ assert out.find('ND Metadata variables:') < 0
else:
assert out.find('Standard Metadata variables:') < 0
assert out.find('ND Metadata variables:') < 0
@@ -536,7 +473,7 @@ def test_equality(self):
return
# TODO(#908): remove tests for deprecated instruments
- @pytest.mark.parametrize("inst_name", ["testing", "testing2d",
+ @pytest.mark.parametrize("inst_name", ["testing",
"ndtesting", "testing_xarray",
"testmodel"])
def test_equality_w_copy(self, inst_name):
@@ -576,41 +513,6 @@ def test_inequality(self, emeta):
assert emeta != self.meta, "meta equality not detectinng differences"
return
- # TODO(#789): remove tests for higher order meta
- @pytest.mark.parametrize('val_dict', [
- {'units': 'U', 'long_name': 'HO Val', 'radn': 'raiden'},
- {'units': 'MetaU', 'long_name': 'HO Val'}])
- def test_inequality_with_higher_order_meta(self, val_dict):
- """Test inequality with higher order metadata.
-
- Parameters
- ----------
- val_dict : dict
- Information to be added to higher order metadata variable
-
- """
-
- meta_dict = {'units': {'ho_val': 'U', 'ho_prof': 'e-'},
- 'long_name': {'ho_val': 'HO Val', 'ho_prof': 'HO Profile'}}
-
- # Set the default meta object
- self.meta['ho_data'] = pysat.Meta(pds.DataFrame(meta_dict))
-
- # Set the altered meta object
- cmeta = pysat.Meta()
-
- for vkey in val_dict.keys():
- if vkey in meta_dict.keys():
- meta_dict[vkey]['ho_val'] = val_dict[vkey]
- else:
- meta_dict[vkey] = {'ho_val': val_dict[vkey]}
-
- cmeta['ho_data'] = pysat.Meta(pds.DataFrame(meta_dict))
-
- # Evaluate the inequality
- assert cmeta != self.meta
- return
-
@pytest.mark.parametrize("label_key", ["units", "name", "notes", "desc",
"min_val", "max_val", "fill_val"])
def test_value_inequality(self, label_key):
@@ -648,7 +550,7 @@ def test_value_inequality(self, label_key):
return
# TODO(#908): remove tests for deprecated instruments
- @pytest.mark.parametrize("inst_name", ["testing", "testing2d",
+ @pytest.mark.parametrize("inst_name", ["testing",
"ndtesting", "testing_xarray",
"testmodel"])
def test_pop(self, inst_name):
@@ -793,8 +695,7 @@ def test_multiple_meta_assignment(self, custom_attr, assign_type):
self.eval_meta_settings()
return
- # TODO(#913): remove tests for 2D metadata
- @pytest.mark.parametrize('inst_name', ['testing', 'testing2d'])
+ @pytest.mark.parametrize('inst_name', ['testing'])
@pytest.mark.parametrize('num_mvals', [0, 1, 3])
@pytest.mark.parametrize('num_dvals', [0, 1, 3])
def test_selected_meta_retrieval(self, inst_name, num_mvals, num_dvals):
@@ -962,8 +863,7 @@ def test_meta_immutable_at_instrument_instantiation(self):
return
- # TODO(#913): remove tests for 2D metadata
- @pytest.mark.parametrize('inst_name', ['testing', 'testing2d'])
+ @pytest.mark.parametrize('inst_name', ['testing'])
def test_assign_nonstandard_metalabels(self, inst_name):
"""Test labels do not conform to the standard values if set that way.
@@ -1524,449 +1424,6 @@ def test_update_epoch(self):
return
- # -------------------------------
- # Tests for higher order metadata
-
- # TODO(#789): remove tests for higher order meta
- @pytest.mark.parametrize('meta_dict', [
- None, {'units': 'V', 'long_name': 'test name'},
- {'units': 'V', 'long_name': 'test name',
- 'meta': pysat.Meta(metadata=pds.DataFrame(
- {'units': {'dummy_frame1': 'A', 'dummy_frame2': ''},
- 'desc': {'dummy_frame1': '',
- 'dummy_frame2': 'A filler description'},
- 'long_name': {'dummy_frame1': 'Dummy 1',
- 'dummy_frame2': 'Dummy 2'}}))},
- {'units': 'V', 'long_name': 'test name', 'bananas': 0,
- 'meta': pysat.Meta(metadata=pds.DataFrame(
- {'units': {'dummy_frame1': 'A', 'dummy_frame2': ''},
- 'desc': {'dummy_frame1': '',
- 'dummy_frame2': 'A filler description'},
- 'long_name': {'dummy_frame1': 'Dummy 1',
- 'dummy_frame2': 'Dummy 2'},
- 'bananas': {'dummy_frame1': 1, 'dummy_frame2': 2}}))}])
- def test_inst_ho_data_assignment(self, meta_dict):
- """Test the assignment of the higher order metadata.
-
- Parameters
- ----------
- meta_dict : dict or NoneType
- Dictionary used to create test metadata
-
- """
-
- # Initialize the Meta data
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'})
-
- # Alter the Meta data
- frame = pds.DataFrame({fkey: np.arange(10) for fkey in self.frame_list},
- columns=self.frame_list)
- inst_data = [frame for i in range(self.testInst.index.shape[0])]
- self.dval = 'test_val'
-
- if meta_dict is None:
- self.testInst[self.dval] = inst_data
- meta_dict = {'units': '', 'long_name': self.dval, 'desc': ''}
- else:
- meta_dict.update({'data': inst_data})
- self.testInst[self.dval] = meta_dict
-
- # Remove data key for evaluation
- del meta_dict['data']
-
- self.meta = self.testInst.meta
-
- # Test the ND metadata results
- self.eval_ho_meta_settings(meta_dict)
- return
-
- # TODO(#789): remove tests for higher order meta
- @pytest.mark.parametrize("num_ho, num_lo", [(1, 1), (2, 2)])
- def test_assign_mult_higher_order_meta_from_dict(self, num_ho, num_lo):
- """Test assign higher order metadata from dict with multiple types.
-
- Parameters
- ----------
- num_ho : int
- Number of higher order data values to initialize
- num_lo : int
- Number of lower order data valuess to initialize
-
- """
-
- # Initialize the lower order evaluation data
- self.default_val['units'] = 'U'
-
- # Initialize the higher-order meta data
- ho_meta = pysat.Meta()
- for flist in self.frame_list:
- ho_meta[flist] = {'units': 'U', 'long_name': flist}
-
- # Initialize the meta dict for setting the data values
- dvals = ['higher_{:d}'.format(i) for i in range(num_ho)]
- dvals.extend(['lower_{:d}'.format(i) for i in range(num_lo)])
-
- meta_dict = {'units': ['U' for i in range(len(dvals))],
- 'long_name': [val for val in dvals],
- 'meta': [ho_meta for i in range(num_ho)]}
- meta_dict['meta'].extend([None for i in range(num_lo)])
-
- # Assign and test the meta data
- self.meta[dvals] = meta_dict
-
- for i, self.dval in enumerate(dvals):
- if i < num_ho:
- eval_dict = {label: meta_dict[label][i]
- for label in meta_dict.keys()}
- self.eval_ho_meta_settings(eval_dict)
- else:
- self.eval_meta_settings()
- return
-
- # TODO(#789): remove tests for higher order meta
- def test_inst_ho_data_assign_meta_then_data(self):
- """Test assignment of higher order metadata before assigning data."""
-
- # Initialize the Meta data
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'})
-
- # Alter the Meta data
- frame = pds.DataFrame({fkey: np.arange(10) for fkey in self.frame_list},
- columns=self.frame_list)
- inst_data = [frame for i in range(self.testInst.index.shape[0])]
- meta_dict = {'data': inst_data, 'units': 'V', 'long_name': 'The Doors',
- 'meta': pysat.Meta(metadata=pds.DataFrame(
- {'units': {dvar: "{:d}".format(i)
- for i, dvar in enumerate(self.frame_list)},
- 'desc': {dvar: "{:s} desc".format(dvar)
- for dvar in self.frame_list},
- 'long_name': {dvar: dvar
- for dvar in self.frame_list}}))}
- self.dval = 'test_data'
-
- # Assign the metadata
- self.testInst[self.dval] = meta_dict
-
- # Alter the data
- self.testInst[self.dval] = inst_data
-
- # Test the ND metadata results
- self.meta = self.testInst.meta
- del meta_dict['data']
- self.eval_ho_meta_settings(meta_dict)
- return
-
- # TODO(#913): remove tests for 2D metadata
- def test_inst_ho_data_assign_meta_different_labels(self):
- """Test the higher order assignment of custom metadata labels."""
-
- # Initialize the Meta data
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'})
-
- # Alter the higher order metadata
- ho_meta = pysat.Meta(labels={'units': ('barrels', str),
- 'desc': ('Monkeys', str),
- 'meta': ('meta', object)})
- self.frame_list = list(
- self.testInst.meta['profiles']['children'].keys())
- for dvar in self.frame_list:
- if dvar == 'density':
- ho_meta[dvar] = {'barrels': 'A'}
- else:
- ho_meta[dvar] = {'Monkeys': 'are fun', 'bananas': 2}
-
- # The 'units', 'desc' and other labels used on self.testInst are
- # applied to the input metadata to ensure everything remains
- # consistent across the object.
- self.testInst['profiles'] = {'data': self.testInst.data['profiles'],
- 'units': 'V', 'long_name': 'The Doors',
- 'meta': ho_meta}
- self.meta = self.testInst.meta
-
- # Test the nD metadata
- assert self.testInst.meta['profiles', 'long_name'] == 'The Doors'
- testing.assert_list_contains(self.frame_list,
- self.meta.ho_data['profiles'])
- testing.assert_list_contains(self.frame_list,
- self.meta['profiles']['children'])
-
- for label in ['units', 'desc']:
- assert self.meta['profiles']['children'].hasattr_case_neutral(label)
-
- assert self.meta['profiles']['children']['density', 'units'] == 'A'
- assert self.meta['profiles']['children']['density', 'desc'] == ''
-
- for dvar in ['dummy_str', 'dummy_ustr']:
- assert self.meta['profiles']['children'][dvar, 'desc'] == 'are fun'
- assert self.meta['profiles']['children'][dvar, 'bananas'] == 2
- return
-
- # TODO(#789): remove tests for higher order meta
- def test_concat_w_ho(self):
- """Test `meta.concat` adds new meta objects with higher order data."""
-
- # Create meta data to concatenate
- meta2 = pysat.Meta()
- meta2['new3'] = {'units': 'hey3', 'long_name': 'crew_brew'}
- meta2['new4'] = pysat.Meta(pds.DataFrame({
- 'units': {'new41': 'hey4'}, 'long_name': {'new41': 'crew_brew'},
- 'bob_level': {'new41': 'max'}}))
-
- # Perform and test for successful concatenation
- self.meta = self.meta.concat(meta2)
- assert self.meta['new3'].units == 'hey3'
- assert self.meta['new4'].children['new41'].units == 'hey4'
- return
-
- # TODO(#913): remove tests for 2d metadata
- def test_concat_not_strict_w_ho_collision(self):
- """Test non-strict concat with overlapping higher-order data."""
-
- # Set the meta object
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'})
-
- # Create a second object with the same higher-order data variables
- concat_meta = pysat.Meta()
- for dvar in self.meta.keys_nD():
- concat_meta[dvar] = self.meta[dvar]
-
- # Change the units of the HO data variables
- for cvar in concat_meta[dvar].children.keys():
- # HERE FIX, DOESN'T WORK
- concat_meta[dvar].children[cvar] = {
- concat_meta.labels.units: "UpdatedUnits"}
-
- # Concatenate the data
- self.meta = self.meta.concat(concat_meta, strict=False)
-
- # Test that the Meta data kept the original values
- testing.assert_list_contains(list(concat_meta.keys_nD()),
- list(self.meta.keys_nD()))
-
- for dvar in concat_meta.keys_nD():
- testing.assert_lists_equal(list(concat_meta[dvar].children.keys()),
- list(self.meta[dvar].children.keys()))
-
- for cvar in concat_meta[dvar].children.keys():
- assert self.meta[dvar].children[
- cvar, self.meta.labels.units].find('Updated') >= 0
- return
-
- # TODO(#789): remove tests for higher order meta
- @pytest.mark.parametrize("label", ['meta_label', 'META_LABEL', 'Meta_Label',
- 'MeTa_lAbEl'])
- def test_ho_attribute_name_case(self, label):
- """Test that `meta.attribute_case_name` preserves the HO stored case.
-
- Parameters
- ----------
- label : str
- Label name
-
- """
-
- # Only set `label` in the child data variable
- self.dval = 'test_val'
- self.meta[self.dval] = self.default_val
- cmeta = pysat.Meta()
- cval = "_".join([self.dval, "child"])
- cmeta[cval] = {label: 'Test meta data for child meta label'}
- self.meta[self.dval] = cmeta
-
- # Test the meta method using different input variations
- assert self.meta.attr_case_name(label.upper()) == label
- assert self.meta.attr_case_name(label.lower()) == label
- assert self.meta.attr_case_name(label.capitalize()) == label
- assert self.meta.attr_case_name(label) == label
- return
-
- # TODO(#789): remove tests for higher order meta
- @pytest.mark.parametrize("label", ['meta_label', 'META_LABEL', 'Meta_Label',
- 'MeTa_lAbEl'])
- def test_ho_hasattr_case_neutral(self, label):
- """Test `meta.hasattr_case_neutral` identifies the HO label name.
-
- Parameters
- ----------
- label : str
- Label name
-
- """
-
- # Only set `label` in the child data variable
- self.dval = 'test_val'
- self.meta[self.dval] = self.default_val
- cmeta = pysat.Meta()
- cval = "_".join([self.dval, "child"])
- cmeta[cval] = {label: 'Test meta data for child meta label'}
- self.meta[self.dval] = cmeta
-
- # Test the meta method using different input variations
- assert self.meta[self.dval].children.hasattr_case_neutral(label.upper())
- assert self.meta[self.dval].children.hasattr_case_neutral(label.lower())
- assert self.meta[self.dval].children.hasattr_case_neutral(
- label.capitalize())
- assert self.meta[self.dval].children.hasattr_case_neutral(label)
- return
-
- # TODO(#913): remove tests for 2D metadata
- def test_ho_meta_rename_function(self):
- """Test `meta.rename` method with ho data using a function."""
-
- # Set the meta object
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'})
-
- # Rename the meta variables to be all upper case, this will differ
- # from the Instrument variables, as pysat defaults to lower case
- self.meta.rename(str.upper)
-
- for dvar in self.testInst.vars_no_time:
- mvar = dvar.upper()
-
- # Test the lower order variables
- assert dvar not in self.meta.keys(), \
- "variable not renamed: {:}".format(repr(dvar))
- assert mvar in self.meta.keys(), \
- "renamed variable missing: {:}".format(repr(mvar))
-
- if mvar in self.meta.keys_nD():
- # Get the variable names from the children
- if hasattr(self.testInst[dvar][0], 'columns'):
- columns = getattr(self.testInst[dvar][0], 'columns')
- else:
- columns = [dvar]
-
- # Test the higher order variables
- for cvar in columns:
- cmvar = cvar.upper()
- assert cmvar in self.meta[mvar].children, \
- "renamed HO variable missing: {:} ({:})".format(
- repr(cmvar), repr(mvar))
-
- return
-
- # TODO(#913): remove tests for 2D metadata
- def test_ho_meta_rename_dict(self):
- """Test `meta.rename` method with ho data using a dict."""
-
- # Set the meta object
- self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing2d'})
-
- # Create a renaming dictionary, which only changes up to four of the
- # variable names
- rename_dict = {dvar: dvar.upper()
- for i, dvar in enumerate(self.testInst.vars_no_time)
- if i < 3 or dvar == 'profiles'}
- rename_dict['profiles'] = {'density': 'DeNsItY'}
-
- # Rename the meta variables to be all upper case, this will differ
- # from the Instrument variables, as pysat defaults to lower case
- self.meta.rename(rename_dict)
-
- for dvar in self.testInst.vars_no_time:
- # Test the lower order variables
- if dvar in rename_dict.keys():
- mvar = rename_dict[dvar]
-
- if isinstance(mvar, dict):
- assert dvar in self.meta.keys_nD()
-
- # Get the variable names from the children
- if hasattr(self.testInst[dvar][0], 'columns'):
- columns = getattr(self.testInst[dvar][0], 'columns')
- else:
- columns = [dvar]
-
- # Test the higher order variables.
- for cvar in columns:
- if cvar in mvar.keys():
- cmvar = mvar[cvar]
- assert cmvar in self.meta[dvar].children.keys(), \
- "renamed HO variable missing: {:} ({:})".format(
- repr(cmvar), repr(dvar))
- else:
- assert cvar in self.meta[dvar].children.keys(), \
- "unmapped HO var altered: {:} ({:})".format(
- repr(cvar), repr(dvar))
- else:
- assert dvar not in self.meta.keys(), \
- "variable not renamed: {:}".format(repr(dvar))
- assert mvar in self.meta.keys(), \
- "renamed variable missing: {:}".format(repr(mvar))
- else:
- mvar = dvar
- assert dvar in self.meta.keys(), \
- "unmapped variable renamed: {:}".format(repr(dvar))
-
- return
-
- # TODO(#789): remove tests for higher order meta
- @pytest.mark.parametrize("label", ['meta_label', 'META_LABEL', 'Meta_Label',
- 'MeTa_lAbEl'])
- def test_get_attribute_name_case_preservation_w_higher_order_list_in(self,
- label):
- """Test that get attribute names preserves the case with ho metadata.
-
- Parameters
- ----------
- label : str
- String label used for testing metadata
-
- """
-
- # Set a meta data variable
- self.dval = 'test_val'
- self.meta[self.dval] = self.default_val
-
- # Set an attribute with case in `label`
- cval = ''.join([self.dval, '21'])
- meta2 = pysat.Meta()
- meta2[cval] = {label: 'Test meta data for meta label'}
-
- # Attach child metadata to root meta
- dval2 = ''.join([self.dval, '2'])
- self.meta[dval2] = meta2
-
- # Attempt to assign to same label at root but potentially different
- # case.
- self.meta[self.dval] = {label.lower(): 'Test meta data for meta label'}
-
- # Create inputs and get the attribute case names
- ins = [label.upper(), label.lower(), label.capitalize(),
- label]
- outputs = self.meta.attr_case_name(ins)
-
- targets = [label] * len(ins)
-
- # Confirm original input case retained.
- assert np.all(outputs == targets)
-
- return
-
- # TODO(#789): remove tests for higher order meta
- def test_ho_data_retrieval_case_insensitive(self):
- """Test that higher order data variables are case insensitive."""
-
- # Initalize the meta data
- self.dval = "test_val"
- self.meta[self.dval] = self.default_val
-
- cmeta = pysat.Meta()
- cval = '_'.join([self.dval, 'child'])
- cmeta[cval] = self.default_val
- self.meta[self.dval] = cmeta
-
- # Test that the data value is present using real key and upper-case
- # version of that key
- assert self.dval in self.meta.keys()
-
- # Test the child variable, which should only be present through the
- # children attribute. Cannot specify keys for case-insensitive look-up.
- assert cval not in self.meta.keys()
- assert cval in self.meta[self.dval].children.keys()
- assert cval.upper() in self.meta[self.dval].children
- return
-
class TestMetaImmutable(TestMeta):
"""Unit tests for immutable metadata."""
@@ -2000,8 +1457,7 @@ def teardown_method(self):
return
# TODO(#789): remove tests for higher order meta
- @pytest.mark.parametrize("prop,set_val", [('data', pds.DataFrame()),
- ('ho_data', {})])
+ @pytest.mark.parametrize("prop,set_val", [('data', pds.DataFrame())])
def test_meta_mutable_properties(self, prop, set_val):
"""Test that @properties are always mutable.
@@ -2197,73 +1653,6 @@ def test_transfer_attributes_to_instrument_leading_underscore(self):
return
-class TestDeprecation(object):
- """Unit tests for DeprecationWarnings in the Meta class."""
-
- def setup_method(self):
- """Set up the unit test environment for each method."""
-
- warnings.simplefilter("always", DeprecationWarning)
- self.meta = pysat.Meta()
- self.warn_msgs = []
- return
-
- def teardown_method(self):
- """Clean up the unit test environment after each method."""
-
- del self.meta, self.warn_msgs
- return
-
- def test_higher_order_meta_deprecation(self):
- """Test that setting higher order meta raises DeprecationWarning."""
-
- # Initialize higher-order metadata to add to the main meta object
- ho_meta = pysat.Meta()
- ho_meta['series_profiles'] = {'long_name': 'series'}
-
- # Raise and catch warnings
- with warnings.catch_warnings(record=True) as war:
- self.meta['series_profiles'] = {'meta': ho_meta,
- 'long_name': 'series'}
-
- # Evaluate warnings
- self.warn_msgs = ["Support for higher order metadata has been"]
- self.warn_msgs = np.array(self.warn_msgs)
-
- # Ensure the minimum number of warnings were raised
- assert len(war) >= len(self.warn_msgs)
-
- # Test the warning messages, ensuring each attribute is present
- testing.eval_warnings(war, self.warn_msgs)
-
- return
-
- def test_higher_order_meta_rename_deprecation(self):
- """Test that renaming higher order meta raises DeprecationWarning."""
-
- # Initialize higher-order metadata to add to the main meta object
- ho_meta = pysat.Meta()
- ho_meta['series_profiles'] = {'long_name': 'series'}
- self.meta['series_profiles'] = {'meta': ho_meta,
- 'long_name': 'series'}
-
- # Raise and catch warnings
- with warnings.catch_warnings(record=True) as war:
- self.meta.rename(str.upper)
-
- # Evaluate warnings
- self.warn_msgs = ["Support for higher order metadata has been"]
- self.warn_msgs = np.array(self.warn_msgs)
-
- # Ensure the minimum number of warnings were raised
- assert len(war) >= len(self.warn_msgs)
-
- # Test the warning messages, ensuring each attribute is present
- testing.eval_warnings(war, self.warn_msgs)
-
- return
-
-
class TestToDict(object):
"""Test `.to_dict` method using pysat test Instruments."""
@@ -2376,23 +1765,6 @@ def setup_method(self):
return
-class TestToDictPandas2D(TestToDict):
- """Test `.to_dict` methods using pysat test Instruments."""
-
- def setup_method(self):
- """Set up the unit test environment for each method."""
-
- self.testInst = pysat.Instrument('pysat', 'testing2d',
- num_samples=5, use_header=True)
- self.stime = pysat.instruments.pysat_testing2d._test_dates['']['']
- self.testInst.load(date=self.stime)
-
- # For output
- self.out = None
-
- return
-
-
class TestToDictXarrayModel(TestToDict):
"""Test `.to_dict` methods using pysat test Instruments."""
diff --git a/pysat/tests/test_meta_labels.py b/pysat/tests/test_meta_labels.py
index ee4b6658e..d1ee69b12 100644
--- a/pysat/tests/test_meta_labels.py
+++ b/pysat/tests/test_meta_labels.py
@@ -211,29 +211,3 @@ def test_change_case_of_meta_labels(self):
assert (self.meta['new2'].Units == 'hey2')
assert (self.meta['new2'].Long_Name == 'boo2')
return
-
- def test_case_change_of_meta_labels_w_ho(self):
- """Test change case of meta labels after initialization with HO data."""
-
- # Set the initial labels
- self.meta_labels = {'units': ('units', str), 'name': ('long_Name', str)}
- self.meta = pysat.Meta(labels=self.meta_labels)
- meta2 = pysat.Meta(labels=self.meta_labels)
-
- # Set meta data values
- meta2['new21'] = {'units': 'hey2', 'long_name': 'boo2'}
- self.meta['new'] = {'units': 'hey', 'long_name': 'boo'}
- self.meta['new2'] = meta2
-
- # Change the label name
- self.meta.labels.units = 'Units'
- self.meta.labels.name = 'Long_Name'
-
- # Evaluate the results in the main data
- assert (self.meta['new'].Units == 'hey')
- assert (self.meta['new'].Long_Name == 'boo')
-
- # Evaluate the results in the higher order data
- assert (self.meta['new2'].children['new21'].Units == 'hey2')
- assert (self.meta['new2'].children['new21'].Long_Name == 'boo2')
- return
diff --git a/pysat/tests/test_utils_coords.py b/pysat/tests/test_utils_coords.py
index bd08e9917..1ce7fea09 100644
--- a/pysat/tests/test_utils_coords.py
+++ b/pysat/tests/test_utils_coords.py
@@ -199,8 +199,7 @@ def test_bad_lon_name_calc_solar_local_time(self):
return
- @pytest.mark.parametrize("name", ["testmodel", "testing2d",
- "ndtesting"])
+ @pytest.mark.parametrize("name", ["testmodel", "ndtesting"])
def test_lon_broadcasting_calc_solar_local_time(self, name):
"""Test calc_solar_local_time with longitude coordinates."""
@@ -216,8 +215,7 @@ def test_lon_broadcasting_calc_solar_local_time(self, name):
assert self.py_inst['slt'].min() >= 0.0
return
- @pytest.mark.parametrize("name", ["testmodel", "testing2d",
- "ndtesting"])
+ @pytest.mark.parametrize("name", ["testmodel", "ndtesting"])
def test_lon_broadcasting_calc_solar_local_time_no_mod_multiday(self, name):
"""Test non modulated solar local time output for a 2 day range."""
@@ -235,8 +233,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod_multiday(self, name):
assert self.py_inst['slt'].min() >= 0.0
return
- @pytest.mark.parametrize("name", ["testmodel", "testing2d",
- "ndtesting"])
+ @pytest.mark.parametrize("name", ["testmodel", "ndtesting"])
def test_lon_broadcasting_calc_solar_local_time_no_mod_ref_date(self, name):
"""Test non modulated SLT output for a 2 day range with a ref date."""
@@ -256,8 +253,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod_ref_date(self, name):
assert self.py_inst['slt'].min() >= 24.0
return
- @pytest.mark.parametrize("name", ["testmodel", "testing2d",
- "ndtesting"])
+ @pytest.mark.parametrize("name", ["testmodel", "ndtesting"])
def test_lon_broadcasting_calc_solar_local_time_no_mod(self, name):
"""Test SLT calc with longitude coordinates and no modulus."""
diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py
index 7ac9c1e11..27ca83f6d 100644
--- a/pysat/tests/test_utils_io.py
+++ b/pysat/tests/test_utils_io.py
@@ -772,8 +772,8 @@ def test_read_netcdf4_with_time_meta_labels(self, kwargs, target):
"Variable {:} not loaded correctly".format(var)
return
- def test_load_netcdf_pandas_3d_error(self):
- """Test load_netcdf error with a pandas 3D file."""
+ def test_load_netcdf_pandas_2d_error(self):
+ """Test load_netcdf error with a pandas 2D file."""
# Create a bunch of files by year and doy
outfile = os.path.join(self.tempdir.name,
'pysat_test_ncdf.nc')
@@ -783,49 +783,12 @@ def test_load_netcdf_pandas_3d_error(self):
# Evaluate the error raised and the expected message
testing.eval_bad_input(
io.load_netcdf, ValueError,
- "only supports 1D and 2D data in pandas", input_args=[outfile],
+ "only supports 1D data in pandas", input_args=[outfile],
input_kwargs={"epoch_name": 'time', "pandas_format": True})
return
-class TestLoadNetCDF2DPandas(TestLoadNetCDF):
- """Unit tests for `load_netcdf` using 2d pandas data."""
-
- def setup_method(self):
- """Set up the test environment."""
-
- # Create temporary directory
- self.tempdir = tempfile.TemporaryDirectory()
- self.saved_path = pysat.params['data_dirs']
- pysat.params['data_dirs'] = self.tempdir.name
-
- self.testInst = pysat.Instrument(platform='pysat', name='testing2d',
- update_files=True, num_samples=100,
- use_header=True)
- self.stime = pysat.instruments.pysat_testing2d._test_dates['']['']
- self.epoch_name = 'time'
-
- # Initialize the loaded data object
- self.loaded_inst = None
- return
-
- def teardown_method(self):
- """Clean up the test environment."""
-
- pysat.params['data_dirs'] = self.saved_path
-
- # Clear the attributes with data in them
- del self.loaded_inst, self.testInst, self.stime, self.epoch_name
-
- # Remove the temporary directory
- self.tempdir.cleanup()
-
- # Clear the directory attributes
- del self.tempdir, self.saved_path
- return
-
-
class TestNetCDF4Integration(object):
"""Integration tests for the netCDF4 I/O utils."""
@@ -1049,17 +1012,6 @@ def test_add_netcdf4_standards_to_meta(self, missing):
assert label not in init_meta[var]
assert label in new_meta[var]
- if self.testInst.name == 'testing2D':
- assert 'Depend_1' not in init_meta[var]
-
- # Check for higher dimensional data properties
- if self.testInst.name == 'testing2D':
- for var in self.testInst.vars_no_time:
- if self.testInst.meta[var].children is not None:
- assert 'Depend_1' in new_meta[var]
- else:
- assert 'Depend_1' not in new_meta[var]
-
return
@pytest.mark.parametrize('meta_trans', [{'units': ['testingFillVal',
@@ -1335,23 +1287,6 @@ def setup_method(self):
return
-class TestNetCDF4IntegrationPandas2D(TestNetCDF4Integration):
- """Integration tests for the netCDF4 I/O utils using pandas2d Instrument."""
-
- def setup_method(self):
- """Create a testing environment."""
-
- # Create an instrument object that has a meta with some
- # variables allowed to be nan within metadata when exporting.
- self.testInst = pysat.Instrument('pysat', 'testing2d', num_samples=5,
- use_header=True)
- self.testInst.load(date=self.testInst.inst_module._test_dates[''][''],
- use_header=True)
- self.pformat = self.testInst.pandas_format
-
- return
-
-
class TestNetCDF4Integration2DXarray(TestNetCDF4Integration):
"""Integration tests for the netCDF4 I/O utils using 2dxarray Instrument."""
@@ -1846,29 +1781,6 @@ def teardown_method(self):
return
-class TestMetaTranslation2DPandas(TestMetaTranslation):
- """Test meta translation when writing/loading files testing2d Instrument."""
-
- def setup_method(self):
- """Create test environment."""
-
- self.test_inst = pysat.Instrument('pysat', 'testing2d',
- num_samples=5, use_header=True)
- self.test_date = pysat.instruments.pysat_testing2d._test_dates['']['']
- self.test_inst.load(date=self.test_date)
- self.meta_dict = self.test_inst.meta.to_dict()
- self.out = None
-
- return
-
- def teardown_method(self):
- """Cleanup test environment."""
-
- del self.test_inst, self.test_date, self.out, self.meta_dict
-
- return
-
-
class TestMetaTranslationModel(TestMetaTranslation):
"""Test meta translation when writing/loading files testmodel Instrument."""
From 11fe41aed6d4f15fce3c93004674fa0a27257667 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 11:13:44 -0400
Subject: [PATCH 127/365] DOC: remove testing2d from docs
---
docs/api.rst | 9 ---------
docs/dependency.rst | 7 -------
docs/instruments/testing_instruments.rst | 7 -------
3 files changed, 23 deletions(-)
diff --git a/docs/api.rst b/docs/api.rst
index b1768d984..d14c971a4 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -241,15 +241,6 @@ pysat_testing_xarray
:members:
-.. _api-pysat-testing2d:
-
-pysat_testing2d
-^^^^^^^^^^^^^^^
-
-.. automodule:: pysat.instruments.pysat_testing2d
- :members:
-
-
.. _api-pysat-testing2d_xarray:
pysat_testing2d_xarray
diff --git a/docs/dependency.rst b/docs/dependency.rst
index f88438fba..fa31acf32 100644
--- a/docs/dependency.rst
+++ b/docs/dependency.rst
@@ -272,13 +272,6 @@ pysat_testing_xarray
:ref:`api-pysat-testing_xarray` returns a satellite-like object with 1D data as
a function of latitude, longitude, and altitude in a xarray format.
-pysat_testing2d
-^^^^^^^^^^^^^^^
-:ref:`api-pysat-testing2d` is another satellite-like object that also returns
-profile data as a function of altitude at some distance from the satellite. It
-is similar to a Radio Occultation or other instruments that have altitude
-profiles.
-
pysat_ndtesting
^^^^^^^^^^^^^^^^^^^^^^
:ref:`api-pysat-ndtesting` is a satellite-like object that returns all
diff --git a/docs/instruments/testing_instruments.rst b/docs/instruments/testing_instruments.rst
index 0fa1299c3..2361a0765 100644
--- a/docs/instruments/testing_instruments.rst
+++ b/docs/instruments/testing_instruments.rst
@@ -21,13 +21,6 @@ and altitude in a xarray format. See :ref:`api-pysat-testing_xarray` for more
details.
-pysat_testing2d
-^^^^^^^^^^^^^^^
-An instrument with satellite-like data like :py:mod:`pysat_testing`, but with a
-2D data variable, 'profile', that is similar to Radio Occultation data. See
-:ref:`api-pysat-testing2d` for more details.
-
-
pysat_ndtesting
^^^^^^^^^^^^^^^
An instrument with satellite-like data like :py:mod:`pysat_testing` that
From b810d16f05cdaa1c574d85cd7b8227c459baaffa Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 11:31:49 -0400
Subject: [PATCH 128/365] DOC: update changelog
---
CHANGELOG.md | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b547a3889..73f76d005 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,7 @@ Change Log
All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](https://semver.org/).
-[3.1.X] - 2023-xx-xx
+[3.2.0] - 2023-xx-xx
--------------------
* New Features
* Added tests for warnings, logging messages, and errors in the Instrument
@@ -23,6 +23,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Implement pyproject to manage metadata
* Remove Sphinx cap
* Add pandas cap
+ * Remove deprecated `pysat_testing2d` instrument
+ * Remove deprecated meta children info
[3.1.0] - 2023-05-31
--------------------
From 19de5df3e44beddcbb9ac3c27aab1b4637dac0ad Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 11:39:21 -0400
Subject: [PATCH 129/365] pandas cap of 2.1.1
---
pyproject.toml | 2 +-
requirements.txt | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 311c0382d..9ec0b533b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,7 +45,7 @@ dependencies = [
"dask",
"netCDF4",
"numpy >= 1.12",
- "pandas < 2.1",
+ "pandas < 2.1.1",
"portalocker",
"pytest",
"scipy",
diff --git a/requirements.txt b/requirements.txt
index c5577679a..059c99271 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
dask
netCDF4
numpy>=1.12
-pandas<2.1
+pandas<2.1.1
portalocker
pytest
scipy
From e85dcbe33196df0a9732e1f90ef22a4c29197515 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 11:42:33 -0400
Subject: [PATCH 130/365] STY: pep8
---
pysat/_meta.py | 2 --
pysat/utils/io.py | 8 --------
2 files changed, 10 deletions(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 4b4fe55f5..274e1aefc 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -498,8 +498,6 @@ def __setitem__(self, data_vars, input_dat):
# Outputs from Meta object are a Series. Thus, this takes in input
# from a Meta object. Set data using standard assignment via a dict.
in_dict = input_data.to_dict()
- if 'children' in in_dict:
- child = in_dict.pop('children')
# Remaining items are simply assigned via recursive call
self[data_vars] = in_dict
diff --git a/pysat/utils/io.py b/pysat/utils/io.py
index 0483d1653..a935a6e98 100644
--- a/pysat/utils/io.py
+++ b/pysat/utils/io.py
@@ -928,8 +928,6 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4',
saved_meta = None
running_idx = 0
running_store = []
- two_d_keys = []
- two_d_dims = []
if meta_kwargs is None:
meta_kwargs = {}
@@ -957,12 +955,6 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4',
else:
drop_meta_labels = pysat.utils.listify(drop_meta_labels)
- # Need to use name label later to identify variables with long_name 'Epoch'
- name_label = meta.labels.name
- for key in meta_translation.keys():
- if meta.labels.name in meta_translation[key]:
- name_label = key
-
# Load data for each file
for fname in fnames:
with netCDF4.Dataset(fname, mode='r', format=file_format) as data:
From 848e573b170d41956c0dbde15598e4d9d4481695 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 12:42:39 -0400
Subject: [PATCH 131/365] STY: restore line with exception
---
pysat/_meta.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 274e1aefc..138f6beda 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -498,6 +498,8 @@ def __setitem__(self, data_vars, input_dat):
# Outputs from Meta object are a Series. Thus, this takes in input
# from a Meta object. Set data using standard assignment via a dict.
in_dict = input_data.to_dict()
+ if 'children' in in_dict:
+ child = in_dict.pop('children') # noqa: F841
# Remaining items are simply assigned via recursive call
self[data_vars] = in_dict
From 6252ed0d9045538487a89fe00007bbbbd0f13737 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 3 Oct 2023 16:03:39 -0400
Subject: [PATCH 132/365] BUG: clean up meta children
---
pysat/_instrument.py | 100 +-------------
pysat/_meta.py | 195 +++------------------------
pysat/instruments/methods/general.py | 9 --
pysat/instruments/methods/testing.py | 2 -
pysat/tests/test_meta.py | 144 ++------------------
pysat/tests/test_utils_io.py | 3 +-
6 files changed, 36 insertions(+), 417 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 5405e1ee6..31a85c254 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -1051,30 +1051,6 @@ def __setitem__(self, key, new_data):
# the rest of the keys are presumed to be metadata
in_data = new.pop('data')
- # TODO(#908): remove code below with removal of 2D pandas support.
- if hasattr(in_data, '__iter__'):
- if not isinstance(in_data, pds.DataFrame) and isinstance(
- next(iter(in_data), None), pds.DataFrame):
- # Input is a list_like of frames, denoting higher order data
- warnings.warn(" ".join(["Support for 2D pandas instrument",
- "data has been deprecated and will",
- "be removed in 3.2.0+. Please",
- "either raise an issue with the",
- "developers or modify the load",
- "statement to use an",
- "xarray.Dataset."]),
- DeprecationWarning, stacklevel=2)
-
- if ('meta' not in new) and (key not in self.meta.keys_nD()):
- # Create an empty Meta instance but with variable names.
- # This will ensure the correct defaults for all
- # subvariables. Meta can filter out empty metadata as
- # needed, the check above reduces the need to create
- # Meta instances.
- ho_meta = pysat.Meta(**self.meta_kwargs)
- ho_meta[in_data[0].columns] = {}
- self.meta[key] = ho_meta
-
# Assign data and any extra metadata
self.data[key] = in_data
self._update_data_types(key)
@@ -2915,77 +2891,13 @@ def rename(self, mapper, lowercase_data_labels=False):
map_key = pysat.utils.get_mapped_value(vkey, mapper)
if map_key is not None:
- # Treat higher-order pandas and normal pandas separately
- if vkey in self.meta.keys_nD():
- # Variable name is in higher order list
- hdict = {}
- if isinstance(map_key, dict):
- # Changing a variable name within a higher order
- # object using a dictionary. First ensure the
- # variable exist.
- for hkey in map_key.keys():
- if hkey not in self.meta[
- vkey]['children'].keys():
- estr = ' '.join(
- ('cannot rename', repr(hkey),
- 'because it is not a known ',
- 'higher-order variable under',
- repr(vkey), '.'))
- raise ValueError(estr)
- hdict = map_key
- else:
- # This is either a value or a mapping function
- for hkey in self.meta[vkey]['children'].keys():
- hmap = pysat.utils.get_mapped_value(hkey,
- mapper)
- if hmap is not None:
- hdict[hkey] = hmap
-
- pdict[vkey] = map_key
-
- # Check for lowercase flag
- change = True
- if lowercase_data_labels:
- gdict = {hkey: hdict[hkey].lower()
- for hkey in hdict.keys()
- if hkey != hdict[hkey].lower()}
-
- if len(list(gdict.keys())) == 0:
- change = False
- else:
- gdict = hdict
-
- # Change the higher-order variable names frame-by-frame
- if change:
- for i in np.arange(len(self.index)):
- if isinstance(self[i, vkey], pds.Series):
- if self[i, vkey].name in gdict:
- new_name = gdict[self[i, vkey].name]
- self[i, vkey].rename(new_name,
- inplace=True)
- else:
- tkey = list(gdict.keys())[0]
- if self[i, vkey].name != gdict[tkey]:
- estr = ' '.join(
- ('cannot rename', hkey,
- 'because, it is not a known'
- 'known higher-order ',
- 'variable under', vkey, 'at',
- 'index {:d}.'.format(i)))
- raise ValueError(estr)
- else:
- self[i, vkey].rename(columns=gdict,
- inplace=True)
-
+ # Add to the pandas renaming dictionary after accounting
+ # for the `lowercase_data_labels` flag.
+ if lowercase_data_labels:
+ if vkey != map_key.lower():
+ pdict[vkey] = map_key.lower()
else:
- # This is a normal variable. Add it to the pandas
- # renaming dictionary after accounting for the
- # `lowercase_data_labels` flag.
- if lowercase_data_labels:
- if vkey != map_key.lower():
- pdict[vkey] = map_key.lower()
- else:
- pdict[vkey] = map_key
+ pdict[vkey] = map_key
# Change variable names for attached data object
self.data.rename(columns=pdict, inplace=True)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 138f6beda..e550a69b7 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -192,9 +192,6 @@ def __init__(self, metadata=None, header_data=None,
# Set the data types, if provided
self._data_types = data_types
- # Initialize higher order (nD) data structure container, a dict
- self._ho_data = {}
-
# Use any user provided data to instantiate object with data.
# Attributes unit and name labels are called within.
if metadata is not None:
@@ -249,23 +246,17 @@ def __str__(self, long_str=True):
"""
# Get the desired variables as lists
labs = [var for var in self.attrs()]
- vdim = [var for var in self.keys() if var not in self.keys_nD()]
- nchild = {var: len([kk for kk in self[var]['children'].keys()])
- for var in self.keys_nD()}
- ndim = ["{:} -> {:d} children".format(var, nchild[var])
- for var in self.keys_nD()]
+ vdim = [var for var in self.keys()]
# Get the lengths of each list
nlabels = len(labs)
nvdim = len(vdim)
- nndim = len(ndim)
# Print the short output
out_str = "pysat Meta object\n"
out_str += "-----------------\n"
out_str += "Tracking {:d} metadata values\n".format(nlabels)
out_str += "Metadata for {:d} standard variables\n".format(nvdim)
- out_str += "Metadata for {:d} ND variables\n".format(nndim)
# Print the global meta data. `max_num` should be divisible by 2 and
# `ncol`.
@@ -283,10 +274,6 @@ def __str__(self, long_str=True):
out_str += "\nStandard Metadata variables:\n"
out_str += core_utils.fmt_output_in_cols(vdim, ncols=ncol,
max_num=max_num)
- if nndim > 0:
- out_str += "\nND Metadata variables:\n"
- out_str += core_utils.fmt_output_in_cols(ndim, ncols=ncol,
- max_num=max_num)
return out_str
@@ -540,14 +527,6 @@ def __getitem__(self, key):
meta[:, 'units']
meta[:, ['units', 'long_name']]
- # For higher order data, slicing is not supported for multiple
- # parents with any children
- meta['profiles', 'density', 'units']
- meta['profiles', 'density', ['units', 'long_name']]
- meta['profiles', ['density', 'dummy_str'], ['units', 'long_name']]
- meta['profiles', ('units', 'long_name')]
- meta[['series_profiles', 'profiles'], ('units', 'long_name')]
-
"""
# Define a local convenience function
def match_name(func, var_name, index_or_column):
@@ -567,43 +546,10 @@ def match_name(func, var_name, index_or_column):
# If tuple length is 2, index, column
new_index = match_name(self.var_case_name, key[0],
self.data.index)
- try:
- # Assume this is a label name
- new_name = match_name(self.attr_case_name, key[1],
- self.data.columns)
- return self.data.loc[new_index, new_name]
- except KeyError as kerr:
- # This may instead be a child variable, check for children
- if(hasattr(self[new_index], 'children')
- and self[new_index].children is None):
- raise kerr
-
- try:
- new_child_index = match_name(
- self.attr_case_name, key[1],
- self[new_index].children.data.index)
- return self.ho_data[new_index].data.loc[new_child_index]
- except AttributeError:
- raise NotImplementedError(
- ''.join(['Cannot retrieve child meta data ',
- 'from multiple parents']))
-
- elif len(key) == 3:
- # If tuple length is 3, index, child_index, column
- new_index = match_name(self.var_case_name, key[0],
- self.data.index)
- try:
- new_child_index = match_name(
- self.attr_case_name, key[1],
- self[new_index].children.data.index)
- except AttributeError:
- raise NotImplementedError(
- 'Cannot retrieve child meta data from multiple parents')
-
- new_name = match_name(self.attr_case_name, key[2],
+ # Assume this is a label name
+ new_name = match_name(self.attr_case_name, key[1],
self.data.columns)
- return self.ho_data[new_index].data.loc[new_child_index,
- new_name]
+ return self.data.loc[new_index, new_name]
elif isinstance(key, list):
# If key is a list, selection works as-is
@@ -627,14 +573,6 @@ def match_name(func, var_name, index_or_column):
# above and commented .copy code below have been kept. Remove
# for any subsequent releases if things are still ok.
meta_row = self.data.loc[new_key] # .copy()
- if new_key in self.keys_nD():
- meta_row.at['children'] = self.ho_data[new_key] # .copy()
- else:
- # Not higher order meta. Assign value of None. First, we
- # assign a string, and then None. Ensures column is not
- # a numeric data type.
- meta_row.at['children'] = ''
- meta_row.at['children'] = None
return meta_row
else:
raise KeyError("Key '{:}' not found in MetaData".format(key))
@@ -663,10 +601,6 @@ def __contains__(self, data_var):
if data_var.lower() in [ikey.lower() for ikey in self.keys()]:
does_contain = True
- if not does_contain:
- if data_var.lower() in [ikey.lower() for ikey in self.keys_nD()]:
- does_contain = True
-
return does_contain
def __eq__(self, other_meta):
@@ -716,39 +650,6 @@ def __eq__(self, other_meta):
other_meta[key, attr]):
return False
- # Check the higher order products. Recursive call into this function
- # didn't work, so spell out the details.
- keys1 = [key for key in self.keys_nD()]
- keys2 = [key for key in other_meta.keys_nD()]
- try:
- testing.assert_lists_equal(keys1, keys2)
- except AssertionError:
- return False
-
- # Check the higher order variables within each nD key are the same.
- # NaN is treated as equal, though mathematically NaN is not equal
- # to anything.
- for key in self.keys_nD():
- for iter1, iter2 in [(self[key].children.keys(),
- other_meta[key].children.keys()),
- (self[key].children.attrs(),
- other_meta[key].children.attrs())]:
- list1 = [value for value in iter1]
- list2 = [value for value in iter2]
-
- try:
- testing.assert_lists_equal(list1, list2)
- except AssertionError:
- return False
-
- # Check if all elements are individually equal
- for ckey in self[key].children.keys():
- for cattr in self[key].children.attrs():
- if not testing.nan_equal(
- self[key].children[ckey, cattr],
- other_meta[key].children[ckey, cattr]):
- return False
-
# If we made it this far, things are good
return True
@@ -875,12 +776,6 @@ def _label_setter(self, new_label, current_label, default_type,
'Meta instantiation.'))
pysat.logger.info(mstr)
- # Check higher order structures and recursively change labels
- for key in self.keys_nD():
- # Update children
- self.ho_data[key]._label_setter(new_label, current_label,
- default_type, use_names_default)
-
return
# -----------------------------------------------------------------------
@@ -902,22 +797,6 @@ def data(self, new_frame):
self._data = new_frame
return
- @property
- def ho_data(self):
- """Retrieve higher order data.
-
- May be set using `ho_data.setter(new_dict)`, where `new_dict` is a
- dict containing the higher order metadata.
-
- """
- return self._ho_data
-
- @ho_data.setter
- def ho_data(self, new_dict):
- # Set the ho_data property. See docstring for property above.
- self._ho_data = new_dict
- return
-
@property
def empty(self):
"""Return boolean True if there is no metadata.
@@ -962,13 +841,8 @@ def drop(self, names):
"""
- # Drop the lower dimension data
self.data = self._data.drop(names, axis=0)
- # Drop the higher dimension data
- for name in names:
- if name in self._ho_data:
- self._ho_data.pop(name)
return
def keep(self, keep_names):
@@ -991,6 +865,7 @@ def keep(self, keep_names):
# Drop names not specified in keep_names list
self.drop(drop_names)
+
return
def apply_meta_labels(self, other_meta):
@@ -1119,11 +994,6 @@ def keys(self):
for ikey in self.data.index:
yield ikey
- def keys_nD(self):
- """Yield keys for higher order metadata."""
- for ndkey in self.ho_data:
- yield ndkey
-
def attrs(self):
"""Yield metadata products stored for each variable name."""
for dcol in self.data.columns:
@@ -1191,8 +1061,8 @@ def attr_case_name(self, name):
# Create a list of all attribute names and lower case attribute names
self_keys = [key for key in self.attrs()]
- for key in list(self.keys_nD()):
- self_keys.extend(self.ho_data[key].data.columns)
+ # for key in list(self.keys_nD()):
+ # self_keys.extend(self.ho_data[key].data.columns)
lower_self_keys = [key.lower() for key in self_keys]
case_names = []
@@ -1240,37 +1110,14 @@ def rename(self, mapper):
# Update the attribute name
map_var = core_utils.get_mapped_value(var, mapper)
if map_var is not None:
- if isinstance(map_var, dict):
- if var in self.keys_nD():
- child_meta = self[var].children.copy()
- child_meta.rename(map_var)
- self.ho_data[var] = child_meta
- else:
- raise ValueError('unknown mapped value at {:}'.format(
- repr(var)))
- else:
- # Get and update the meta data
- hold_meta = self[var].copy()
- hold_meta.name = map_var
-
- # Remove the metadata under the previous variable name
- self.drop(var)
- if var in self.ho_data:
- del self.ho_data[var]
-
- # Re-add the meta data with the updated variable name
- self[map_var] = hold_meta
-
- # Determine if the attribute is present in higher order
- # structures
- if map_var in self.keys_nD():
- # The children attribute is a Meta class object.
- # Recursively call the current routine. The only way to
- # avoid Meta undoing the renaming process is to assign
- # the meta data to `ho_data`.
- child_meta = self[map_var].children.copy()
- child_meta.rename(mapper)
- self.ho_data[map_var] = child_meta
+ hold_meta = self[var].copy()
+ hold_meta.name = map_var
+
+ # Remove the metadata under the previous variable name
+ self.drop(var)
+
+ # Re-add the meta data with the updated variable name
+ self[map_var] = hold_meta
return
@@ -1314,14 +1161,10 @@ def concat(self, other_meta, strict=False):
other_meta_updated = other_meta.copy()
other_meta_updated.labels = self.labels
- # Concat 1D metadata in data frames to copy of current metadata
+ # Concat metadata in data frames to copy of current metadata
for key in other_meta_updated.keys():
mdata.data.loc[key] = other_meta.data.loc[key]
- # Combine the higher order meta data
- for key in other_meta_updated.keys_nD():
- mdata.ho_data[key] = other_meta.ho_data[key]
-
return mdata
def copy(self):
@@ -1351,8 +1194,6 @@ def pop(self, label_name):
if new_name in self.keys():
output = self[new_name]
self.data = self.data.drop(new_name, axis=0)
- else:
- output = self.ho_data.pop(new_name)
else:
raise KeyError('Key not present in metadata variables')
@@ -1661,10 +1502,6 @@ class MetaLabels(object):
but the original case is preserved. Case preservation is built in to
support writing files with a desired case to meet standards.
- Metadata for higher order data objects, those that have
- multiple products under a single variable name in a `pysat.Instrument`
- object, are stored by providing a Meta object under the single name.
-
Supports any custom metadata values in addition to the expected metadata
attributes (units, name, notes, desc, value_min, value_max, and fill).
These base attributes may be used to programatically access and set types
diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py
index 732163702..400480ddd 100644
--- a/pysat/instruments/methods/general.py
+++ b/pysat/instruments/methods/general.py
@@ -216,15 +216,6 @@ def remove_leading_text(inst, target=None):
inst.meta.data = inst.meta.data.rename(
index=lambda x: x.split(prepend_str)[-1])
- orig_keys = [kk for kk in inst.meta.keys_nD()]
-
- for keynd in orig_keys:
- if keynd.find(prepend_str) >= 0:
- new_key = keynd.split(prepend_str)[-1]
- new_meta = inst.meta.pop(keynd)
- new_meta.data = new_meta.data.rename(
- index=lambda x: x.split(prepend_str)[-1])
- inst.meta[new_key] = new_meta
return
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index cb25480af..08705398d 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -284,8 +284,6 @@ def initialize_test_meta(epoch_name, data_keys):
for var in meta.keys():
if var not in data_keys:
meta.drop(var)
- if var in meta.keys_nD():
- meta.ho_data.pop(var)
return meta
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index 9269046ea..adfaf9c6c 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -106,62 +106,8 @@ def eval_meta_settings(self, isfloat=True):
self.default_val[lkey].__repr__())
assert 'children' not in self.meta.data.columns
- assert self.dval not in self.meta.keys_nD()
return
- def eval_ho_meta_settings(self, meta_dict):
- """Test the Meta settings for higher order data.
-
- Parameters
- ----------
- meta_dict : dict
- Dict with meta data labels as keys and values to test against
-
- """
-
- # Test the ND metadata results
- testing.assert_list_contains(self.frame_list,
- list(self.meta.ho_data[self.dval].keys()))
- testing.assert_list_contains(
- self.frame_list, list(self.meta[self.dval]['children'].keys()))
-
- # Test the meta settings at the base and nD level
- for label in meta_dict.keys():
- if label == 'meta':
- testing.assert_lists_equal(
- list(self.meta[self.dval]['children'].attrs()),
- list(meta_dict[label].attrs()))
- testing.assert_lists_equal(
- list(self.meta[self.dval]['children'].keys()),
- list(meta_dict[label].keys()))
-
- for lvar in self.meta[self.dval]['children'].attrs():
- for dvar in self.meta[self.dval]['children'].keys():
- if lvar in self.default_nan:
- assert np.isnan(
- self.meta[self.dval]['children'][dvar, lvar]), \
- "{:s} child {:s} {:s} value {:} != NaN".format(
- self.dval.__repr__(), dvar.__repr__(),
- lvar.__repr__(),
- self.meta[self.dval]['children'][
- dvar, lvar].__repr__())
- else:
- assert (self.meta[self.dval]['children'][dvar, lvar]
- == meta_dict[label][dvar, lvar]), \
- "{:s} child {:s} {:s} value {:} != {:}".format(
- self.dval.__repr__(), dvar.__repr__(),
- lvar.__repr__(),
- self.meta[self.dval]['children'][
- dvar, lvar].__repr__(),
- meta_dict[label][dvar, lvar].__repr__())
- else:
- assert self.meta[self.dval]['children'].hasattr_case_neutral(
- label)
- assert self.meta[self.dval, label] == meta_dict[label], \
- "{:s} label value {:} != {:}".format(
- label, self.meta[self.dval, label].__repr__(),
- meta_dict[label].__repr__())
-
return
# -----------------------
@@ -425,7 +371,6 @@ def test_str(self, long_str, inst_kwargs):
# Evaluate the common parts of the output string
assert out.find('pysat Meta object') >= 0
assert out.find('standard variables') > 0
- assert out.find('ND variables') > 0
assert out.find('global attributes') > 0
# Evaluate the extra parts of the long output string
@@ -441,10 +386,8 @@ def test_str(self, long_str, inst_kwargs):
else:
assert out.find('Standard Metadata variables:') < 0
- assert out.find('ND Metadata variables:') < 0
else:
assert out.find('Standard Metadata variables:') < 0
- assert out.find('ND Metadata variables:') < 0
return
def test_self_equality(self):
@@ -573,7 +516,6 @@ def test_pop(self, inst_name):
# Test the popped object labels
pop_attrs = list(mpop.keys())
- pop_attrs.pop(pop_attrs.index('children'))
testing.assert_lists_equal(pop_attrs, list(self.meta.attrs()))
# Test the popped object values
@@ -581,9 +523,6 @@ def test_pop(self, inst_name):
comp_values = [mcomp[pattr] for pattr in pop_attrs]
testing.assert_lists_equal(pop_values, comp_values)
- if mpop['children'] is not None:
- assert mpop['children'] == mcomp['children']
-
# Test that the popped variable is no longer in the main object
assert dvar not in self.meta.keys(), "pop did not remove metadata"
@@ -720,20 +659,6 @@ def test_selected_meta_retrieval(self, inst_name, num_mvals, num_dvals):
mvals = [getattr(self.meta.labels, mattr)
for mattr in list(self.meta_labels.keys())[:num_mvals]]
- # If dvals is greater than zero and there is higher order data,
- # make sure at least one is included
- nd_inds = list()
- if len(dvals) > 0:
- nd_vals = [key for key in self.meta.keys_nD()]
-
- if len(nd_vals) > 0:
- nd_inds = [dvals.index(self.dval) for self.dval in nd_vals
- if self.dval in dvals]
-
- if len(nd_inds) == 0:
- dvals[0] = nd_vals[0]
- nd_inds = [0]
-
# Retrieve meta data for desired values
sel_meta = self.meta[dvals, mvals]
@@ -742,21 +667,6 @@ def test_selected_meta_retrieval(self, inst_name, num_mvals, num_dvals):
testing.assert_lists_equal(dvals, list(sel_meta.index))
testing.assert_lists_equal(mvals, list(sel_meta.columns))
- # If there is higher order data, test the retrieval
- for pind in nd_inds:
- # Get the desired number of child variables
- self.dval = dvals[pind]
- cvals = [kk for kk in self.meta[self.dval].children.keys()][
- :num_dvals]
-
- # Retrieve meta data for desired values
- sel_meta = self.meta[self.dval, cvals, mvals]
-
- # Evaluate retrieved data
- assert isinstance(sel_meta, pds.DataFrame)
- testing.assert_lists_equal(cvals, list(sel_meta.index))
- testing.assert_lists_equal(mvals, list(sel_meta.columns))
-
return
def test_replace_meta(self):
@@ -1456,32 +1366,18 @@ def teardown_method(self):
del self.mutable
return
- # TODO(#789): remove tests for higher order meta
- @pytest.mark.parametrize("prop,set_val", [('data', pds.DataFrame())])
- def test_meta_mutable_properties(self, prop, set_val):
- """Test that @properties are always mutable.
-
- Parameters
- ----------
- prop : str
- Attribute on `self.meta` to be tested
- set_val : any
- Value to be assigned to `prop` on `self.meta`
-
- """
+ def test_meta_mutable_properties(self):
+ """Test that @properties are always mutable."""
# Set anything that can be immutable to be immutable
self.meta.mutable = self.mutable
# Test that data and label values can be updated
- for prop, set_val in [('data', pds.DataFrame()), ('ho_data', {})]:
- try:
- # Pandas does not support dataframe equality
- setattr(self.meta, prop, set_val)
- except AttributeError:
- raise AssertionError(
- "Couldn't update mutable property {:}".format(
- prop.__repr__()))
+ try:
+ # Pandas does not support dataframe equality
+ setattr(self.meta, 'data', pds.DataFrame())
+ except AttributeError:
+ raise AssertionError("Couldn't update mutable property 'data'")
@pytest.mark.parametrize("label", ['units', 'name', 'desc', 'notes',
'min_val', 'max_val', 'fill_val'])
@@ -1691,40 +1587,26 @@ def test_to_dict(self, preserve_case):
# Confirm type
assert isinstance(self.out, dict)
- # Check for higher order products
- ho_vars = []
- for var in self.testInst.meta.keys():
- if 'children' in self.testInst.meta[var]:
- if self.testInst.meta[var]['children'] is not None:
- for subvar in self.testInst.meta[var]['children'].keys():
- ho_vars.append('_'.join([var, subvar]))
-
# Confirm the contents of the output for variables
for var in self.out.keys():
- if var not in ho_vars:
- for label in self.out[var]:
- assert label in self.testInst.meta.data.columns
- assert testing.nan_equal(self.out[var][label],
- self.testInst.meta[var][label]), \
- 'Differing values.'
+ for label in self.out[var]:
+ assert label in self.testInst.meta.data.columns
+ assert testing.nan_equal(self.out[var][label],
+ self.testInst.meta[var][label]), \
+ 'Differing values.'
# Confirm case
if not preserve_case:
# Outputs should all be lower case
for key in self.out.keys():
assert key == key.lower(), 'Output not lower case.'
- for key in ho_vars:
- assert key == key.lower(), 'Output not lower case.'
- assert key.lower() in self.out, 'Missing output variable.'
else:
# Case should be preserved
for key in self.out.keys():
assert key == self.testInst.meta.var_case_name(key), \
'Output case different.'
- for key in ho_vars:
- assert key in self.out, 'Output case different, or missing.'
- num_target_vars = len(ho_vars) + len(list(self.testInst.meta.keys()))
+ num_target_vars = len(list(self.testInst.meta.keys()))
assert num_target_vars == len(self.out), \
'Different number of variables.'
diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py
index 27ca83f6d..bdee5cb43 100644
--- a/pysat/tests/test_utils_io.py
+++ b/pysat/tests/test_utils_io.py
@@ -284,8 +284,7 @@ def test_inst_write_and_read_netcdf(self, add_path):
and var not in updated_attrs]
tvars = [var for var in self.testInst.meta.keys()
- if var not in self.testInst.meta.keys_nD()
- and var.lower() not in ["epoch", "time"]]
+ if var.lower() not in ["epoch", "time"]]
fvars = [var for var in netcdf_inst.meta.keys()
if var.lower() not in ["epoch", "time"]]
From 1918f73e73ee5a484e79403ec916b60169b21e7e Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 4 Oct 2023 11:08:19 -0400
Subject: [PATCH 133/365] MAINT: scrub code for higher order meta
---
pysat/_meta.py | 146 +++++++--------
pysat/tests/test_utils_io.py | 32 +---
pysat/utils/io.py | 347 +++--------------------------------
3 files changed, 92 insertions(+), 433 deletions(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index e550a69b7..83fad9c47 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -401,86 +401,72 @@ def __setitem__(self, data_vars, input_dat):
# Time to actually add the metadata
for ikey in input_data:
- if ikey not in ['children', 'meta']:
- for i, var in enumerate(data_vars):
- to_be_set = input_data[ikey][i]
- good_set = True
-
- # See if this meta data key has already been defined
- # in MetaLabels
- if ikey in self.labels.label_attrs.keys():
- iattr = self.labels.label_attrs[ikey]
- if not isinstance(
- to_be_set, self.labels.label_type[iattr]):
- # If this is a disagreement between byte data
- # and an expected str, resolve it here
- if(isinstance(to_be_set, bytes)
- and str in pysat.utils.listify(
- self.labels.label_type[iattr])):
- to_be_set = core_utils.stringify(to_be_set)
- else:
- # This type is incorrect, try casting it
- wmsg = ''.join(['Metadata with type ',
- repr(type(to_be_set)),
- ' does not match expected ',
- 'type ',
- repr(self.labels.label_type[
- iattr])])
- try:
- if hasattr(to_be_set, '__iter__'):
- if str in pysat.utils.listify(
- self.labels.label_type[
- iattr]):
- to_be_set = '\n\n'.join(
- [str(tval) for tval in
- to_be_set])
- else:
- raise TypeError("can't recast")
+ for i, var in enumerate(data_vars):
+ to_be_set = input_data[ikey][i]
+ good_set = True
+
+ # See if this meta data key has already been defined
+ # in MetaLabels
+ if ikey in self.labels.label_attrs.keys():
+ iattr = self.labels.label_attrs[ikey]
+ if not isinstance(
+ to_be_set, self.labels.label_type[iattr]):
+ # If this is a disagreement between byte data
+ # and an expected str, resolve it here
+ if(isinstance(to_be_set, bytes)
+ and str in pysat.utils.listify(
+ self.labels.label_type[iattr])):
+ to_be_set = core_utils.stringify(to_be_set)
+ else:
+ # This type is incorrect, try casting it
+ wmsg = ''.join(['Metadata with type ',
+ repr(type(to_be_set)),
+ ' does not match expected ',
+ 'type ',
+ repr(self.labels.label_type[
+ iattr])])
+ try:
+ if hasattr(to_be_set, '__iter__'):
+ if str in pysat.utils.listify(
+ self.labels.label_type[iattr]):
+ to_be_set = '\n\n'.join(
+ [str(tval) for tval in
+ to_be_set])
else:
- to_be_set = pysat.utils.listify(
- self.labels.label_type[
- iattr])[0](to_be_set)
-
- # Inform user data was recast
- pysat.logger.info(''.join((
- wmsg, '. Recasting input for ',
- repr(var), ' with key ',
- repr(ikey))))
- except (TypeError, ValueError):
- # Warn user data was dropped
- warnings.warn(''.join((
- wmsg, '. Dropping input for ',
- repr(var), ' with key ',
- repr(ikey))))
- good_set = False
- else:
- # Extend the meta labels. Ensure the attribute
- # name has no spaces and that bytes are used instead
- # of strings.
- iattr = ikey.replace(" ", "_")
- itype = type(to_be_set)
- if itype == bytes:
- itype = str
-
- # Update the MetaLabels object and the existing
- # metadata to ensure all data have all labels
- self.labels.update(iattr, ikey, itype)
- self._label_setter(ikey, ikey, type(to_be_set))
-
- # Set the data
- if good_set:
- self._data.loc[var, ikey] = to_be_set
- else:
- # Key is 'meta' or 'children', providing higher order
- # metadata. Meta inputs could be part of a larger multiple
- # parameter assignment, so not all names may actually have
- # a 'meta' object to add.
- for item, val in zip(data_vars, input_data['meta']):
- if val is not None:
- # Assign meta data, using a recursive call...
- # Heads to if Meta instance call.
- self[item] = val
-
+ raise TypeError("can't recast")
+ else:
+ to_be_set = pysat.utils.listify(
+ self.labels.label_type[
+ iattr])[0](to_be_set)
+
+ # Inform user data was recast
+ pysat.logger.info(''.join((
+ wmsg, '. Recasting input for ',
+ repr(var), ' with key ', repr(ikey))))
+ except (TypeError, ValueError):
+ # Warn user data was dropped
+ warnings.warn(''.join((
+ wmsg, '. Dropping input for ',
+ repr(var), ' with key ',
+ repr(ikey))))
+ good_set = False
+ else:
+ # Extend the meta labels. Ensure the attribute
+ # name has no spaces and that bytes are used instead
+ # of strings.
+ iattr = ikey.replace(" ", "_")
+ itype = type(to_be_set)
+ if itype == bytes:
+ itype = str
+
+ # Update the MetaLabels object and the existing
+ # metadata to ensure all data have all labels
+ self.labels.update(iattr, ikey, itype)
+ self._label_setter(ikey, ikey, type(to_be_set))
+
+ # Set the data
+ if good_set:
+ self._data.loc[var, ikey] = to_be_set
elif isinstance(input_data, pds.Series):
# Outputs from Meta object are a Series. Thus, this takes in input
# from a Meta object. Set data using standard assignment via a dict.
@@ -1061,8 +1047,6 @@ def attr_case_name(self, name):
# Create a list of all attribute names and lower case attribute names
self_keys = [key for key in self.attrs()]
- # for key in list(self.keys_nD()):
- # self_keys.extend(self.ho_data[key].data.columns)
lower_self_keys = [key.lower() for key in self_keys]
case_names = []
diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py
index bdee5cb43..47e468922 100644
--- a/pysat/tests/test_utils_io.py
+++ b/pysat/tests/test_utils_io.py
@@ -105,15 +105,7 @@ def eval_loaded_data(self, test_case=True):
# Test the data values for each variable
for dkey in keys:
lkey = dkey.lower()
- if lkey in ['profiles', 'alt_profiles', 'series_profiles']:
- # Test the loaded higher-dimension data
- for tframe, lframe in zip(self.testInst[dkey],
- self.loaded_inst[dkey]):
- assert np.all(tframe == lframe), "unequal {:s} data".format(
- dkey)
- else:
- # Test the standard data structures
- assert np.all(self.testInst[dkey] == self.loaded_inst[dkey])
+ assert np.all(self.testInst[dkey] == self.loaded_inst[dkey])
# Check that names are lower case when written
pysat.utils.testing.assert_lists_equal(keys, new_keys, test_case=False)
@@ -1232,14 +1224,6 @@ def test_meta_processor_to_from_netcdf4(self, assign_flag):
def test_missing_metadata(self):
"""Test writing file with no metadata."""
- # Collect a list of higher order meta
- ho_vars = []
- for var in self.testInst.meta.keys():
- if 'children' in self.testInst.meta[var]:
- if self.testInst.meta[var]['children'] is not None:
- for subvar in self.testInst.meta[var]['children'].keys():
- ho_vars.append((subvar, var))
-
# Drop all metadata
self.testInst.meta.keep([])
@@ -1257,12 +1241,6 @@ def test_missing_metadata(self):
# Test the warning
testing.eval_warnings(war, exp_warns, warn_type=UserWarning)
- # Test warning for higher order data as well (pandas)
- for (svar, var) in ho_vars:
- wstr = ''.join(['Unable to find MetaData for ',
- svar, ' subvariable of ', var])
- exp_warns.append(wstr)
-
# Test the warning
testing.eval_warnings(war, exp_warns, warn_type=UserWarning)
@@ -1701,8 +1679,7 @@ def test_remove_netcdf4_standards(self, caplog):
# Enforcing netcdf4 standards removes 'fill', min, and max information
# for string variables. This is not re-added by the `remove_` function
# call since, strictly speaking, we don't know what to add back in.
- # Also exepmting a check on long_name for higher order data with a time
- # index. When loading files, pysat specifically checks for 'Epoch' as
+ # When loading files, pysat specifically checks for 'Epoch' as
# the long_name. So, ensuring long_name for such variables is written
# could break loading for existent files. I could fake it, and assign
# the standard name as long_name when loading, and while that would
@@ -1712,11 +1689,6 @@ def test_remove_netcdf4_standards(self, caplog):
assert var in filt_meta, 'Lost metadata variable {}'.format(var)
for key in self.meta_dict[var].keys():
- # Creating exception for time-index of higher order data. The
- # long_name comes out differently.
- if var == 'profiles' and (key == 'long_name'):
- continue
-
# Test remaining variables accounting for possible exceptions
# for string variables.
if key not in ['fill', 'value_min', 'value_max']:
diff --git a/pysat/utils/io.py b/pysat/utils/io.py
index a935a6e98..0fb4cd4d1 100644
--- a/pysat/utils/io.py
+++ b/pysat/utils/io.py
@@ -235,142 +235,28 @@ def add_netcdf4_standards_to_metadict(inst, in_meta_dict, epoch_name,
meta_dict.update(time_meta)
- if inst[var].dtype == np.dtype('O') and coltype != str:
- # This is a Series or DataFrame, possibly with more dimensions.
- # Series and DataFrame data must be treated differently.
- try:
- # Assume it is a DataFrame and get a list of subvariables
- subvars = inst[0, var].columns
- is_frame = True
- except AttributeError:
- # Data is Series of Series, which doesn't have columns.
- subvars = [inst[0, var].name]
- is_frame = False
-
- # Get the dimensions and their names
- dims = np.shape(inst[0, var])
- obj_dim_names = []
- if len(dims) == 1:
- # Pad the dimensions so that the rest of the code works
- # for either a Series or a DataFrame.
- dims = (dims[0], 0)
-
- for dim in dims[:-1]:
- # Don't need to go over last dimension value,
- # it covers number of columns (if a DataFrame).
- obj_dim_names.append(var)
-
- # Set the base-level meta data.
- meta_dict['Depend_1'] = obj_dim_names[-1]
-
- # Cycle through each of the sub-variable, updating metadata.
- for svar in subvars:
- # Find the subvariable data within the main variable,
- # checking that this is not an empty DataFrame or
- # Series. Determine the underlying data types.
- good_data_loc = 0
- for idat in np.arange(len(inst.data)):
- if len(inst[idat, var]) > 0:
- good_data_loc = idat
- break
-
- # Get the correct location of the sub-variable based on
- # the object type.
- if is_frame:
- good_data = inst[good_data_loc, var][svar]
- else:
- good_data = inst[good_data_loc, var]
-
- # Get subvariable information
- _, sctype, sdflag = inst._get_data_info(good_data)
-
- if not sdflag:
- # Not a datetime index
- smeta_dict = {'Depend_0': epoch_name,
- 'Depend_1': obj_dim_names[-1],
- 'Display_Type': 'Multidimensional',
- 'Format': inst._get_var_type_code(sctype),
- 'Var_Type': 'data'}
- else:
- # Attach datetime index metadata
- smeta_dict = return_epoch_metadata(inst, epoch_name)
- smeta_dict.pop('MonoTon')
-
- # Construct name, variable_subvariable, and store.
- sname = '_'.join([lower_var, svar.lower()])
- if sname in out_meta_dict:
- out_meta_dict[sname].update(smeta_dict)
- else:
- warnings.warn(''.join(['Unable to find MetaData for ',
- svar, ' subvariable of ', var]))
- out_meta_dict[sname] = smeta_dict
-
- # Filter metadata
- remove = True if sctype == str else False
- out_meta_dict[sname] = filter_netcdf4_metadata(
- inst, out_meta_dict[sname], sctype, remove=remove,
- check_type=check_type, export_nan=export_nan, varname=sname)
-
- # Get information on the subvar index. This information
- # stored under primary variable name.
- sub_index = inst[good_data_loc, var].index
- _, coltype, datetime_flag = inst._get_data_info(sub_index)
- meta_dict['Format'] = inst._get_var_type_code(coltype)
-
- # Deal with index information for holding variable
- _, index_type, index_flag = inst._get_data_info(
- inst[good_data_loc, var].index)
-
- # Update metadata when a datetime index found
- if index_flag:
- update_dict = return_epoch_metadata(inst, epoch_name)
- update_dict.pop('MonoTon')
- update_dict.update(meta_dict)
- else:
- if inst[good_data_loc, var].index.name is not None:
- name = inst[good_data_loc, var].index.name
- else:
- name = var
- update_dict = {inst.meta.labels.name: name}
- update_dict.update(meta_dict)
-
- if lower_var in out_meta_dict:
- out_meta_dict[lower_var].update(update_dict)
- else:
- warnings.warn(''.join(['Unable to find MetaData for ',
- var]))
- out_meta_dict[lower_var] = update_dict
-
- # Filter metdata for other netCDF4 requirements
- remove = True if index_type == str else False
- out_meta_dict[lower_var] = filter_netcdf4_metadata(
- inst, out_meta_dict[lower_var], index_type, remove=remove,
- check_type=check_type, export_nan=export_nan, varname=lower_var)
-
+ meta_dict['Format'] = inst._get_var_type_code(coltype)
+
+ if not inst.pandas_format:
+ for i, dim in enumerate(list(inst[var].dims)):
+ meta_dict['Depend_{:1d}'.format(i)] = dim
+ num_dims = len(inst[var].dims)
+ if num_dims >= 2:
+ meta_dict['Display_Type'] = 'Multidimensional'
+
+ # Update the meta data
+ if lower_var in out_meta_dict:
+ out_meta_dict[lower_var].update(meta_dict)
else:
- # Dealing with 1D data or xarray format
- meta_dict['Format'] = inst._get_var_type_code(coltype)
-
- if not inst.pandas_format:
- for i, dim in enumerate(list(inst[var].dims)):
- meta_dict['Depend_{:1d}'.format(i)] = dim
- num_dims = len(inst[var].dims)
- if num_dims >= 2:
- meta_dict['Display_Type'] = 'Multidimensional'
-
- # Update the meta data
- if lower_var in out_meta_dict:
- out_meta_dict[lower_var].update(meta_dict)
- else:
- warnings.warn(''.join(['Unable to find MetaData for ',
- var]))
- out_meta_dict[lower_var] = meta_dict
+ warnings.warn(''.join(['Unable to find MetaData for ',
+ var]))
+ out_meta_dict[lower_var] = meta_dict
- # Filter metdata for other netCDF4 requirements
- remove = True if coltype == str else False
- out_meta_dict[lower_var] = filter_netcdf4_metadata(
- inst, out_meta_dict[lower_var], coltype, remove=remove,
- check_type=check_type, export_nan=export_nan, varname=lower_var)
+ # Filter metdata for other netCDF4 requirements
+ remove = True if coltype == str else False
+ out_meta_dict[lower_var] = filter_netcdf4_metadata(
+ inst, out_meta_dict[lower_var], coltype, remove=remove,
+ check_type=check_type, export_nan=export_nan, varname=lower_var)
return out_meta_dict
@@ -418,11 +304,6 @@ def remove_netcdf4_standards_from_meta(mdict, epoch_name, labels):
lower_sub_keys = [ckey.lower() for ckey in mdict[key].keys()]
sub_keys = list(mdict[key].keys())
- if 'meta' in lower_sub_keys:
- # Higher dimensional data, recursive treatment.
- mdict[key]['meta'] = remove_netcdf4_standards_from_meta(
- mdict[key]['meta'], '', labels)
-
# Check for presence of time information
for lval in lower_sub_keys:
if lval in lower_time_vals:
@@ -631,13 +512,6 @@ def apply_table_translation_from_file(trans_table, meta_dict):
wstr = 'Translation label "{}" not found for variable "{}".'
pysat.logger.debug(wstr.format(trans_key, var_key))
- # Check for higher order metadata
- if 'meta' in meta_dict[var_key].keys():
- # Recursive call to process metadata
- ldict = meta_dict[var_key]['meta']
- filt_dict[var_key]['meta'] = \
- apply_table_translation_from_file(trans_table, ldict)
-
return filt_dict
@@ -673,12 +547,6 @@ def meta_array_expander(meta_dict):
for key in meta_dict.keys():
loop_dict = {}
for meta_key in meta_dict[key].keys():
- # Check for higher order data from 2D pandas support
- if meta_key == 'meta':
- loop_dict[meta_key] = meta_array_expander(
- meta_dict[key][meta_key])
- continue
-
tst_array = np.asarray(meta_dict[key][meta_key])
if tst_array.shape == ():
loop_dict[meta_key] = meta_dict[key][meta_key]
@@ -1037,23 +905,7 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4',
# Assign filtered metadata to pysat.Meta instance
for key in filt_mdict:
- if 'meta' in filt_mdict[key].keys():
- # Higher order metadata
- dim_meta = pysat.Meta(**meta_kwargs)
- for skey in filt_mdict[key]['meta'].keys():
- dim_meta[skey] = filt_mdict[key]['meta'][skey]
-
- # Remove HO metdata that was just transfered elsewhere
- filt_mdict[key].pop('meta')
-
- # Assign standard metdata
- meta[key] = filt_mdict[key]
-
- # Assign HO metadata
- meta[key] = {'meta': dim_meta}
- else:
- # Standard metadata
- meta[key] = filt_mdict[key]
+ meta[key] = filt_mdict[key]
return data, meta
@@ -1489,8 +1341,6 @@ def inst_to_netcdf(inst, fname, base_instrument=None, epoch_name=None,
Stores 1-D data along dimension 'Epoch' - the date time index.
- Stores higher order data (e.g. dataframes within series) separately
-
- The name of the main variable column is used to prepend subvariable
names within netCDF, var_subvar_sub
- A netCDF4 dimension is created for each main variable column
@@ -1640,7 +1490,6 @@ def inst_to_netcdf(inst, fname, base_instrument=None, epoch_name=None,
meta_translation = copy.deepcopy(meta_translation)
# Ensure `meta_translation` has default values for items not assigned.
- # This is needed for the higher order pandas support and may be removed.
def_meta_trans = default_to_netcdf_translation_table(inst)
for key in def_meta_trans.keys():
if key not in meta_translation:
@@ -1767,156 +1616,10 @@ def inst_to_netcdf(inst, fname, base_instrument=None, epoch_name=None,
cdfkey[:] = data.values
else:
- # Still dealing with an object, not just a Series of
- # strings. Maps to `if` check on coltypes, being
- # string-based. Presuming a Series with a DataFrame or
- # Series in each location. Start by collecting some
- # basic info on dimensions sizes, names, then create
- # corresponding netCDF4 dimensions total dimensions
- # stored for object are epoch plus ones created below
- dims = np.shape(inst[key].iloc[0])
- obj_dim_names = []
-
- # Pad dimensions so that the rest of the code works
- # for either a Series or a DataFrame.
- if len(dims) == 1:
- dims = (dims[0], 0)
-
- # Don't need to go over last dimension value,
- # it covers number of columns (if a frame).
- for i, dim in enumerate(dims[:-1]):
- obj_dim_names.append(case_key)
- out_data.createDimension(obj_dim_names[-1], dim)
-
- # Create simple tuple with information needed to create
- # the right dimensions for variables that will
- # be written to file.
- var_dim = tuple([epoch_name] + obj_dim_names)
-
- # Determine whether data is in a DataFrame or Series
- try:
- # Start by assuming it is a DataFrame
- iterable = inst[key].iloc[0].columns
- is_frame = True
- except AttributeError:
- # Otherwise get sub-variables for a Series
- iterable = [inst[key].iloc[0].name]
- is_frame = False
-
- # Find the subvariable data within the main variable,
- # checking that this is not an empty DataFrame or
- # Series. Determine the underlying data types.
- good_data_loc = 0
- for idat in np.arange(len(inst.data)):
- if len(inst.data[key].iloc[0]) > 0:
- good_data_loc = idat
- break
-
- # Found a place with data, if there is one
- # now iterate over the subvariables, get data info
- # create netCDF4 variables and store the data
- # stored name is variable_subvariable.
- for col in iterable:
- if is_frame:
- # We are working with a DataFrame, so
- # multiple subvariables stored under a single
- # main variable heading.
- idx = inst[key].iloc[good_data_loc][col]
- data, coltype, _ = inst._get_data_info(idx)
-
- # netCDF4 doesn't support string compression
- if coltype == str:
- lzlib = False
- else:
- lzlib = zlib
-
- cdfkey = out_data.createVariable(
- '_'.join((case_key, col)), coltype,
- dimensions=var_dim, zlib=lzlib,
- complevel=complevel, shuffle=shuffle)
-
- # Set metadata
- lkey = '_'.join((lower_key, col.lower()))
- cdfkey.setncatts(export_meta[lkey])
-
- # Attach data. It may be slow to repeatedly
- # call the store method as well astype method
- # below collect data into a numpy array, then
- # write the full array in one go.
- temp_cdf_data = np.zeros(
- (num, dims[0])).astype(coltype)
- for i in range(num):
- temp_cdf_data[i, :] = inst[
- key].iloc[i][col].values
-
- # Write data
- cdfkey[:, :] = temp_cdf_data
- else:
- # We are dealing with a Series. Get
- # information from within the series.
- idx = inst[key].iloc[good_data_loc]
- data, coltype, _ = inst._get_data_info(idx)
-
- # netCDF4 doesn't support string compression
- if coltype == str:
- lzlib = False
- else:
- lzlib = zlib
-
- cdfkey = out_data.createVariable(
- case_key + '_data', coltype,
- dimensions=var_dim, zlib=lzlib,
- complevel=complevel, shuffle=shuffle)
-
- # Set metadata
- tempk = '_'.join([lower_key, lower_key])
- cdfkey.setncatts(export_meta[tempk])
-
- # Attach data
- temp_cdf_data = np.zeros(
- (num, dims[0]), dtype=coltype)
- for i in range(num):
- temp_cdf_data[i, :] = inst[i, key].values
-
- # Write data
- cdfkey[:, :] = temp_cdf_data
-
- # We are done storing the actual data for the given
- # higher order variable. Now we need to store the index
- # for all of that fancy data.
-
- # Get index information
- data, coltype, datetime_flag = inst._get_data_info(
- inst[key].iloc[good_data_loc].index)
-
- # Create dimension variable to store index in netCDF4
- cdfkey = out_data.createVariable(case_key, coltype,
- dimensions=var_dim,
- zlib=zlib,
- complevel=complevel,
- shuffle=shuffle)
- # Set meta data
- cdfkey.setncatts(export_meta[lower_key])
-
- # Treat time and non-time data differently
- if datetime_flag:
- # Set data
- temp_cdf_data = np.zeros((num, dims[0]),
- dtype=coltype)
- for i in range(num):
- temp_cdf_data[i, :] = inst[i, key].index.values
- cdfkey[:, :] = (temp_cdf_data * 1.0E-6).astype(
- coltype)
-
- else:
- # Set data
- temp_cdf_data = np.zeros((num, dims[0]),
- dtype=coltype)
-
- for i in range(num):
- temp_cdf_data[i, :] = inst[
- key].iloc[i].index.astype(coltype)
- cdfkey[:, :] = temp_cdf_data
+ raise ValueError(' '.join(('Recursive data detected.',
+ 'Please ensure that the',
+ 'DataFrame does not contain',
+ 'other pandas objects.')))
else:
# Attach the metadata to a separate xarray.Dataset object, ensuring
# the Instrument data object is unchanged.
From 4a2e3f2e4e42d67a45df4496510f1d363226708b Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 4 Oct 2023 11:08:32 -0400
Subject: [PATCH 134/365] DOC: update documentation refs
---
docs/tutorial/tutorial_files.rst | 2 +-
pysat/_instrument.py | 34 +++-----------------------
pysat/_meta.py | 41 +++++---------------------------
3 files changed, 10 insertions(+), 67 deletions(-)
diff --git a/docs/tutorial/tutorial_files.rst b/docs/tutorial/tutorial_files.rst
index 067518cb0..04599168e 100644
--- a/docs/tutorial/tutorial_files.rst
+++ b/docs/tutorial/tutorial_files.rst
@@ -150,7 +150,7 @@ as fill, _FillValue, and FillVal.
When writing files pysat processes metadata for both xarray and pandas before
writing the file. For xarray, pysat leverages xarray's built-in file writing
capabilities. For pandas, pysat interfaces with netCDF4 directly to translate
-both 1D and higher dimensional data into netCDF4.
+data into netCDF4.
.. _tutorial-files-meta:
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 31a85c254..9d5c5b3b7 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -2814,33 +2814,6 @@ def rename(self, mapper, lowercase_data_labels=False):
inst.rename(str.upper)
- If using a pandas-type Instrument with higher-order data and a
- dictionary mapper, the upper-level data key must contain a dictionary
- for renaming the dependent data variables. The upper-level data key
- cannot be renamed. Note that this rename will be invoked individually
- for all times in the dataset.
- ::
-
- # Applies to higher-order datasets that are loaded into pandas
- inst = pysat.Instrument('pysat', 'testing2D')
- inst.load(2009, 1)
- mapper = {'uts': 'pysat_uts',
- 'profiles': {'density': 'pysat_density'}}
- inst.rename(mapper)
- print(inst[0, 'profiles'].columns) # 'density' will be updated
-
- # To rename higher-order data at both levels using a dictionary,
- # you need two calls
- mapper2 = {'profiles': 'pysat_profile'}
- inst.rename(mapper2)
- print(inst[0, 'pysat_profile'].columns)
-
- # A function will affect both standard and higher-order data.
- # Remember this function also updates the Meta data.
- inst.rename(str.capitalize)
- print(inst.meta['Pysat_profile']['children'])
-
-
pysat supports differing case for variable labels across the data and
metadata objects attached to an Instrument. Since Meta is
case-preserving (on assignment) but case-insensitive to access, the
@@ -2850,10 +2823,9 @@ def rename(self, mapper, lowercase_data_labels=False):
::
# Example with lowercase_data_labels
- inst = pysat.Instrument('pysat', 'testing2D')
+ inst = pysat.Instrument('pysat', 'testing')
inst.load(2009, 1)
- mapper = {'uts': 'Pysat_UTS',
- 'profiles': {'density': 'PYSAT_density'}}
+ mapper = {'uts': 'Pysat_UTS'}
inst.rename(mapper, lowercase_data_labels=True)
# Note that 'Pysat_UTS' was applied to data as 'pysat_uts'
@@ -2886,7 +2858,7 @@ def rename(self, mapper, lowercase_data_labels=False):
# Initialize dict for renaming normal pandas data
pdict = {}
- # Collect normal variables and rename higher order variables
+ # Collect and rename variables
for vkey in self.variables:
map_key = pysat.utils.get_mapped_value(vkey, mapper)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 83fad9c47..25a63fac7 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -67,10 +67,6 @@ class Meta(object):
but the original case is preserved. Case preseveration is built in to
support writing files with a desired case to meet standards.
- Metadata for higher order data objects, those that have
- multiple products under a single variable name in a `pysat.Instrument`
- object, are stored by providing a Meta object under the single name.
-
Supports any custom metadata values in addition to the expected metadata
attributes (units, name, notes, desc, value_min, value_max, and fill).
These base attributes may be used to programatically access and set types
@@ -126,15 +122,6 @@ class Meta(object):
meta['var_name4'].children['name41']
meta['var_name4'].children['name42']
- # You may, of course, have a mixture of 1D and nD data
- meta = pysat.Meta()
- meta['dm'] = {'units': 'hey', 'long_name': 'boo'}
- meta['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
- meta2 = pysat.Meta()
- meta2[['higher', 'lower']] = {'meta': [meta, None],
- 'units': [None, 'boo'],
- 'long_name': [None, 'boohoo']}
-
# Meta data may be assigned from another Meta object using dict-like
# assignments
key1 = 'var_name'
@@ -794,8 +781,6 @@ def empty(self):
"""
- # Only need to check on lower data since lower data
- # is set when higher metadata assigned.
if self.data.empty:
return True
else:
@@ -813,7 +798,6 @@ def merge(self, other):
for key in other.keys():
if key not in self:
- # Copies over both lower and higher dimensional data
self[key] = other[key]
return
@@ -999,10 +983,6 @@ def hasattr_case_neutral(self, attr_name):
True if the case-insensitive check for attribute name is successful,
False if no attribute name is present.
- Note
- ----
- Does not check higher order meta objects
-
"""
if attr_name.lower() in [dcol.lower() for dcol in self.data.columns]:
@@ -1025,11 +1005,9 @@ def attr_case_name(self, name):
Note
----
- Checks first within standard attributes. If not found there, checks
- attributes for higher order data structures. If not found, returns
- supplied name as it is available for use. Intended to be used to help
- ensure that the same case is applied to all repetitions of a given
- variable name.
+ Checks first within standard attributes. If not found, returns supplied
+ name as it is available for use. Intended to be used to help ensure that
+ the same case is applied to all repetitions of a given variable name.
"""
@@ -1074,18 +1052,11 @@ def rename(self, mapper):
Dictionary with old names as keys and new names as variables or
a function to apply to all names
- Raises
- ------
- ValueError
- When normal data is treated like higher-order data in dict mapping.
-
Note
----
- Checks first within standard attributes. If not found there, checks
- attributes for higher order data structures. If not found, returns
- supplied name as it is available for use. Intended to be used to help
- ensure that the same case is applied to all repetitions of a given
- variable name.
+ Checks first within standard attributes. If not found, returns supplied
+ name as it is available for use. Intended to be used to help ensure that
+ the same case is applied to all repetitions of a given variable name.
"""
From bf6ed856dfaafe05c1a41f700659741653cc88e8 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 4 Oct 2023 11:31:52 -0400
Subject: [PATCH 135/365] BUG: remove unused variable
---
pysat/tests/test_utils_io.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py
index 47e468922..cb3dd7b87 100644
--- a/pysat/tests/test_utils_io.py
+++ b/pysat/tests/test_utils_io.py
@@ -104,7 +104,6 @@ def eval_loaded_data(self, test_case=True):
# Test the data values for each variable
for dkey in keys:
- lkey = dkey.lower()
assert np.all(self.testInst[dkey] == self.loaded_inst[dkey])
# Check that names are lower case when written
From 294fe4c8defb68271336c1527d8cfec7af77fdc7 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 4 Oct 2023 13:34:37 -0400
Subject: [PATCH 136/365] STY: no children
---
pysat/_meta.py | 13 -------------
pysat/tests/test_meta.py | 3 ---
pysat/utils/io.py | 4 ----
3 files changed, 20 deletions(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 25a63fac7..bf14905e6 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -117,11 +117,6 @@ class Meta(object):
meta2['var_name42'] = {'long_name': 'name2of4', 'units': 'Units2'}
meta['var_name4'] = {'meta': meta2}
- # An alternative method to achieve the same result is:
- meta['var_name4'] = meta2
- meta['var_name4'].children['name41']
- meta['var_name4'].children['name42']
-
# Meta data may be assigned from another Meta object using dict-like
# assignments
key1 = 'var_name'
@@ -454,14 +449,6 @@ def __setitem__(self, data_vars, input_dat):
# Set the data
if good_set:
self._data.loc[var, ikey] = to_be_set
- elif isinstance(input_data, pds.Series):
- # Outputs from Meta object are a Series. Thus, this takes in input
- # from a Meta object. Set data using standard assignment via a dict.
- in_dict = input_data.to_dict()
- if 'children' in in_dict:
- child = in_dict.pop('children') # noqa: F841
- # Remaining items are simply assigned via recursive call
- self[data_vars] = in_dict
return
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index adfaf9c6c..de3bf2c6e 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -105,9 +105,6 @@ def eval_meta_settings(self, isfloat=True):
lkey.__repr__(), self.meta[self.dval, lkey].__repr__(),
self.default_val[lkey].__repr__())
- assert 'children' not in self.meta.data.columns
- return
-
return
# -----------------------
diff --git a/pysat/utils/io.py b/pysat/utils/io.py
index 0fb4cd4d1..2b972ca4a 100644
--- a/pysat/utils/io.py
+++ b/pysat/utils/io.py
@@ -882,10 +882,6 @@ def load_netcdf_pandas(fnames, strict_meta=False, file_format='NETCDF4',
for label in drop_meta_labels:
if label in full_mdict[var]:
full_mdict[var].pop(label)
- if 'meta' in full_mdict[var]:
- for var2 in full_mdict[var]['meta'].keys():
- if label in full_mdict[var]['meta'][var2]:
- full_mdict[var]['meta'][var2].pop(label)
# Second, remove some items pysat added for netcdf compatibility.
filt_mdict = remove_netcdf4_standards_from_meta(full_mdict, epoch_name,
From df91d43564eea3bdee2ac50ebec6fd61a89ba2c4 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
Date: Thu, 5 Oct 2023 11:10:53 -0400
Subject: [PATCH 137/365] Update pysat/utils/registry.py
Co-authored-by: Angeline Burrell
---
pysat/utils/registry.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py
index ac42838e1..fd0beec76 100644
--- a/pysat/utils/registry.py
+++ b/pysat/utils/registry.py
@@ -223,7 +223,7 @@ def register_by_module(module, overwrite=False):
attached as sub-modules to the input `module`
overwrite : bool
If True, an existing registration will be updated
- with the new module information.
+ with the new module information. (default=False)
Raises
------
From 63afe6ec14ee039a6159fddde21d3f254b3b4716 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
Date: Thu, 5 Oct 2023 11:11:46 -0400
Subject: [PATCH 138/365] Update registry.py
---
pysat/utils/registry.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py
index fd0beec76..77c1f8338 100644
--- a/pysat/utils/registry.py
+++ b/pysat/utils/registry.py
@@ -92,7 +92,7 @@ def register(module_names, overwrite=False):
specify package name and instrument modules
overwrite : bool
If True, an existing registration will be updated
- with the new module information.
+ with the new module information. (default=False)
Raises
------
From ad4509296907006f1013b5ef1ccfdad3bf0f0fb6 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
Date: Thu, 5 Oct 2023 12:17:41 -0400
Subject: [PATCH 139/365] Apply suggestions from code review
Co-authored-by: Angeline Burrell
---
pysat/_meta.py | 1 +
pysat/tests/test_meta.py | 4 ++++
2 files changed, 5 insertions(+)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index bf14905e6..ca2fd982a 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -506,6 +506,7 @@ def match_name(func, var_name, index_or_column):
# If tuple length is 2, index, column
new_index = match_name(self.var_case_name, key[0],
self.data.index)
+
# Assume this is a label name
new_name = match_name(self.attr_case_name, key[1],
self.data.columns)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index de3bf2c6e..77b19437a 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -1373,8 +1373,12 @@ def test_meta_mutable_properties(self):
try:
# Pandas does not support dataframe equality
setattr(self.meta, 'data', pds.DataFrame())
+
+ # Test that data is empty
+ assert self.meta.data.empty, "`meta.data` not updated correctly"
except AttributeError:
raise AssertionError("Couldn't update mutable property 'data'")
+ return
@pytest.mark.parametrize("label", ['units', 'name', 'desc', 'notes',
'min_val', 'max_val', 'fill_val'])
From 25053cd65f4cbeb77501cc2df1e6729227bb2806 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 5 Oct 2023 12:18:40 -0400
Subject: [PATCH 140/365] BUG: allow input as series
---
pysat/_meta.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index bf14905e6..fb094baa3 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -449,6 +449,10 @@ def __setitem__(self, data_vars, input_dat):
# Set the data
if good_set:
self._data.loc[var, ikey] = to_be_set
+ elif isinstance(input_data, pds.Series):
+ # Outputs from Meta object are a Series. Thus, this takes in input
+ # from a Meta object. Set data using standard assignment via a dict.
+ self[data_vars] = input_data.to_dict()
return
From 2d631d94d5cfade83f01ec40a0da5db9ef3f67d4 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 5 Oct 2023 12:19:43 -0400
Subject: [PATCH 141/365] STY: whitespace
---
pysat/tests/test_meta.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index 77b19437a..5e3b1699a 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -1373,7 +1373,7 @@ def test_meta_mutable_properties(self):
try:
# Pandas does not support dataframe equality
setattr(self.meta, 'data', pds.DataFrame())
-
+
# Test that data is empty
assert self.meta.data.empty, "`meta.data` not updated correctly"
except AttributeError:
From 2f29f0ecbe1499a34f7b56e10ab62ab86bf054e6 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 5 Oct 2023 12:30:18 -0400
Subject: [PATCH 142/365] STY: e275
---
pysat/_orbits.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/pysat/_orbits.py b/pysat/_orbits.py
index 7f68f77bd..dcbc604b2 100644
--- a/pysat/_orbits.py
+++ b/pysat/_orbits.py
@@ -474,8 +474,11 @@ def _equa_breaks(self, orbit_index_period=24.0):
# Iterate over samples and check.
for sub_tidx in tidx:
# Look at time change vs local time change
- if(ut_diff[sub_idx].iloc[sub_tidx] * orbit_index_period
- < lt_diff[sub_idx].iloc[sub_tidx] * self.orbit_period):
+ false_alarm = (
+ ut_diff[sub_idx].iloc[sub_tidx] * orbit_index_period
+ < lt_diff[sub_idx].iloc[sub_tidx]
+ * self.orbit_period)
+ if false_alarm:
# The change in UT is small compared to the change
# in the orbit index this is flagged as a false
From 8a505c5e69eeb31c936829ffc6c2fd92d8ff801c Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 5 Oct 2023 13:44:44 -0400
Subject: [PATCH 143/365] MAINT: remove deprecated instruments
---
pysat/instruments/__init__.py | 3 +-
pysat/instruments/pysat_testing2d_xarray.py | 66 ------
pysat/instruments/pysat_testing_xarray.py | 223 --------------------
3 files changed, 1 insertion(+), 291 deletions(-)
delete mode 100644 pysat/instruments/pysat_testing2d_xarray.py
delete mode 100644 pysat/instruments/pysat_testing_xarray.py
diff --git a/pysat/instruments/__init__.py b/pysat/instruments/__init__.py
index 248e80f36..882080c8b 100644
--- a/pysat/instruments/__init__.py
+++ b/pysat/instruments/__init__.py
@@ -5,8 +5,7 @@
"""
__all__ = ['pysat_ndtesting', 'pysat_netcdf', 'pysat_testing',
- 'pysat_testmodel', 'pysat_testing_xarray',
- 'pysat_testing2d_xarray']
+ 'pysat_testmodel']
for inst in __all__:
exec("from pysat.instruments import {x}".format(x=inst))
diff --git a/pysat/instruments/pysat_testing2d_xarray.py b/pysat/instruments/pysat_testing2d_xarray.py
deleted file mode 100644
index 334821385..000000000
--- a/pysat/instruments/pysat_testing2d_xarray.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Produces fake instrument data for testing.
-
-.. deprecated:: 3.1.0
- This module has been renamed pysat_ndtesting. A copy inheriting the
- routines from the new location is maintained here for backwards-
- compatibility. This instrument will be removed in 3.2.0+ to reduce
- redundancy.
-
-"""
-
-import datetime as dt
-import functools
-import numpy as np
-import warnings
-
-import xarray as xr
-
-import pysat
-from pysat.instruments.methods import testing as mm_test
-from pysat.instruments import pysat_ndtesting
-
-platform = 'pysat'
-name = 'testing2d_xarray'
-
-tags = pysat_ndtesting.tags
-inst_ids = pysat_ndtesting.inst_ids
-pandas_format = pysat_ndtesting.pandas_format
-_test_dates = pysat_ndtesting._test_dates
-
-
-# Init method
-def init(self, test_init_kwarg=None):
- """Initialize the test instrument.
-
- Parameters
- ----------
- self : pysat.Instrument
- This object
- test_init_kwarg : any
- Testing keyword (default=None)
-
- """
-
- warnings.warn(" ".join(["The instrument module `pysat_testing2d_xarray`",
- "has been deprecated and will be removed in",
- "3.2.0+. Please use `pysat_ndtesting` instead."]),
- DeprecationWarning, stacklevel=2)
-
- mm_test.init(self, test_init_kwarg=test_init_kwarg)
- return
-
-
-# Clean method
-clean = pysat_ndtesting.clean
-
-# Optional method, preprocess
-preprocess = pysat_ndtesting.preprocess
-
-load = pysat_ndtesting.load
-
-list_files = functools.partial(pysat_ndtesting.list_files,
- test_dates=_test_dates)
-list_remote_files = functools.partial(pysat_ndtesting.list_remote_files,
- test_dates=_test_dates)
-download = functools.partial(pysat_ndtesting.download)
diff --git a/pysat/instruments/pysat_testing_xarray.py b/pysat/instruments/pysat_testing_xarray.py
deleted file mode 100644
index 0edd7365b..000000000
--- a/pysat/instruments/pysat_testing_xarray.py
+++ /dev/null
@@ -1,223 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Produces fake instrument data for testing.
-
-.. deprecated:: 3.0.2
- All data present in this instrument is duplicated in pysat_ndtesting.
- This instrument will be removed in 3.2.0+ to reduce redundancy.
-
-"""
-
-import datetime as dt
-import functools
-import numpy as np
-import warnings
-
-import xarray as xr
-
-import pysat
-from pysat.instruments.methods import testing as mm_test
-
-# pysat required parameters
-platform = 'pysat'
-name = 'testing_xarray'
-
-# Dictionary of data 'tags' and corresponding description
-tags = {'': 'Regular testing data set'}
-
-# Dictionary of satellite IDs, list of corresponding tags
-inst_ids = {'': ['']}
-_test_dates = {'': {'': dt.datetime(2009, 1, 1)}}
-pandas_format = False
-
-epoch_name = u'time'
-
-
-# Init method
-def init(self, test_init_kwarg=None):
- """Initialize the test instrument.
-
- Parameters
- ----------
- self : pysat.Instrument
- This object
- test_init_kwarg : any
- Testing keyword (default=None)
-
- """
-
- warnings.warn(" ".join(["The instrument module `pysat_testing_xarray` has",
- "been deprecated and will be removed in 3.2.0+."]),
- DeprecationWarning, stacklevel=2)
-
- mm_test.init(self, test_init_kwarg=test_init_kwarg)
- return
-
-
-# Clean method
-clean = mm_test.clean
-
-# Optional method, preprocess
-preprocess = mm_test.preprocess
-
-
-def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
- sim_multi_file_left=False, non_monotonic_index=False,
- non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=86400, test_load_kwarg=None, max_latitude=90.):
- """Load the test files.
-
- Parameters
- ----------
- fnames : list
- List of filenames.
- tag : str
- Tag name used to identify particular data set to be loaded.
- This input is nominally provided by pysat itself. (default='')
- inst_id : str
- Instrument ID used to identify particular data set to be loaded.
- This input is nominally provided by pysat itself. (default='')
- sim_multi_file_right : bool
- Adjusts date range to be 12 hours in the future or twelve hours beyond
- `root_date`. (default=False)
- sim_multi_file_left : bool
- Adjusts date range to be 12 hours in the past or twelve hours before
- `root_date`. (default=False)
- non_monotonic_index : bool
- If True, time index will be non-monotonic (default=False)
- non_unique_index : bool
- If True, time index will be non-unique (default=False)
- malformed_index : bool
- If True, the time index will be non-unique and non-monotonic. Deprecated
- and scheduled for removal in pysat 3.2.0. (default=False)
- start_time : dt.timedelta or NoneType
- Offset time of start time since midnight UT. If None, instrument data
- will begin at midnight. (default=None)
- num_samples : int
- Maximum number of times to generate. Data points will not go beyond the
- current day. (default=86400)
- test_load_kwarg : any
- Keyword used for pysat unit testing to ensure that functionality for
- custom keywords defined in instrument support functions is working
- correctly. (default=None)
- max_latitude : float
- Latitude simulated as `max_latitude` * cos(theta(t))`, where
- theta is a linear periodic signal bounded by [0, 2 * pi). (default=90.)
-
- Returns
- -------
- data : xr.Dataset
- Testing data
- meta : pysat.Meta
- Metadata
-
- """
-
- # Support keyword testing
- pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))
-
- # Create an artificial satellite data set
- iperiod = mm_test.define_period()
- drange = mm_test.define_range()
-
- uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='1S',
- start_time=start_time)
-
- if sim_multi_file_right:
- root_date = dt.datetime(2009, 1, 1, 12)
- elif sim_multi_file_left:
- root_date = dt.datetime(2008, 12, 31, 12)
- else:
- root_date = dt.datetime(2009, 1, 1)
-
- # TODO(#1094): Remove in pysat 3.2.0
- if malformed_index:
- # Warn that kwarg is deprecated and set new kwargs.
- mm_test._warn_malformed_kwarg()
- non_monotonic_index = True
- non_unique_index = True
-
- if non_monotonic_index:
- index = mm_test.non_monotonic_index(index)
- if non_unique_index:
- index = mm_test.non_unique_index(index)
-
- data = xr.Dataset({'uts': ((epoch_name), uts)},
- coords={epoch_name: index})
-
- # Need to create simple orbits here. Have start of first orbit
- # at 2009,1, 0 UT. 14.84 orbits per day. Figure out how far in time from
- # the root start a measurement is and use that info to create a signal
- # that is continuous from that start. Going to presume there are 5820
- # seconds per orbit (97 minute period).
- time_delta = dates[0] - root_date
- mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
- period=iperiod['lt'],
- data_range=drange['lt'])
- data['mlt'] = ((epoch_name), mlt)
-
- # SLT, 20 second offset from `mlt`.
- slt = mm_test.generate_fake_data(time_delta.total_seconds() + 20, uts,
- period=iperiod['lt'],
- data_range=drange['lt'])
- data['slt'] = ((epoch_name), slt)
-
- # Create a fake longitude, resets every 6240 seconds. Sat moves at
- # 360/5820 deg/s, Earth rotates at 360/86400, takes extra time to go
- # around full longitude.
- longitude = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
- period=iperiod['lon'],
- data_range=drange['lon'])
- data['longitude'] = ((epoch_name), longitude)
-
- # Create latitude area for testing polar orbits
- angle = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
- period=iperiod['angle'],
- data_range=drange['angle'])
- latitude = max_latitude * np.cos(angle)
- data['latitude'] = ((epoch_name), latitude)
-
- # Create constant altitude at 400 km
- alt0 = 400.0
- altitude = alt0 * np.ones(data['latitude'].shape)
- data['altitude'] = ((epoch_name), altitude)
-
- # Fake orbit number
- fake_delta = dates[0] - dt.datetime(2008, 1, 1)
- orbit_num = mm_test.generate_fake_data(fake_delta.total_seconds(),
- uts, period=iperiod['lt'],
- cyclic=False)
-
- data['orbit_num'] = ((epoch_name), orbit_num)
-
- # Create some fake data to support testing of averaging routines
- mlt_int = data['mlt'].astype(int).data
- long_int = (data['longitude'] / 15.).astype(int).data
- data['dummy1'] = ((epoch_name), mlt_int)
- data['dummy2'] = ((epoch_name), long_int)
- data['dummy3'] = ((epoch_name), mlt_int + long_int * 1000.)
- data['dummy4'] = ((epoch_name), uts)
- data['string_dummy'] = ((epoch_name),
- ['test'] * len(data.indexes[epoch_name]))
- data['unicode_dummy'] = ((epoch_name),
- [u'test'] * len(data.indexes[epoch_name]))
- data['int8_dummy'] = ((epoch_name),
- np.ones(len(data.indexes[epoch_name]), dtype=np.int8))
- data['int16_dummy'] = ((epoch_name),
- np.ones(len(data.indexes[epoch_name]),
- dtype=np.int16))
- data['int32_dummy'] = ((epoch_name),
- np.ones(len(data.indexes[epoch_name]),
- dtype=np.int32))
- data['int64_dummy'] = ((epoch_name),
- np.ones(len(data.indexes[epoch_name]),
- dtype=np.int64))
-
- # Set the meta data
- meta = mm_test.initialize_test_meta(epoch_name, data.keys())
- return data, meta
-
-
-list_files = functools.partial(mm_test.list_files, test_dates=_test_dates)
-list_remote_files = functools.partial(mm_test.list_remote_files,
- test_dates=_test_dates)
-download = functools.partial(mm_test.download)
From 0054093077a7a802c332727c8db96e5ce39b726b Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 5 Oct 2023 13:52:55 -0400
Subject: [PATCH 144/365] ENH: add capability to ndtesting to mirror deprecated
instrument
---
pysat/instruments/pysat_ndtesting.py | 27 ++++++++++++++++++++++-----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 7a63d54ba..f10be34a4 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -33,10 +33,11 @@
preprocess = mm_test.preprocess
-def load(fnames, tag='', inst_id='', non_monotonic_index=False,
+def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
+ sim_multi_file_left=False, non_monotonic_index=False,
non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=864, test_load_kwarg=None, max_latitude=90.0,
- num_extra_time_coords=0):
+ num_samples=864, sample_rate='100S', test_load_kwarg=None,
+ max_latitude=90.0, num_extra_time_coords=0):
"""Load the test files.
Parameters
@@ -49,6 +50,12 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
inst_id : str
Instrument ID used to identify particular data set to be loaded.
This input is nominally provided by pysat itself. (default='')
+ sim_multi_file_right : bool
+ Adjusts date range to be 12 hours in the future or twelve hours beyond
+ `root_date`. (default=False)
+ sim_multi_file_left : bool
+ Adjusts date range to be 12 hours in the past or twelve hours before
+ `root_date`. (default=False)
non_monotonic_index : bool
If True, time index will be non-monotonic (default=False)
non_unique_index : bool
@@ -63,6 +70,8 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
num_samples : int
Maximum number of times to generate. Data points will not go beyond the
current day. (default=864)
+ sample_rate : str
+ Frequency of data points, using pandas conventions. (default='100s')
test_load_kwarg : any
Keyword used for pysat unit testing to ensure that functionality for
custom keywords defined in instrument support functions is working
@@ -90,8 +99,16 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
drange = mm_test.define_range()
# Using 100s frequency for compatibility with seasonal analysis unit tests
- uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='100S',
+ uts, index, dates = mm_test.generate_times(fnames, num_samples,
+ freq=sample_rate,
start_time=start_time)
+ if sim_multi_file_right:
+ root_date = dt.datetime(2009, 1, 1, 12)
+ elif sim_multi_file_left:
+ root_date = dt.datetime(2008, 12, 31, 12)
+ else:
+ root_date = dt.datetime(2009, 1, 1)
+
# TODO(#1094): Remove in pysat 3.2.0
if malformed_index:
# Warn that kwarg is deprecated and set new kwargs.
@@ -112,7 +129,7 @@ def load(fnames, tag='', inst_id='', non_monotonic_index=False,
# the root start a measurement is and use that info to create a signal
# that is continuous from that start. Going to presume there are 5820
# seconds per orbit (97 minute period).
- time_delta = dates[0] - dt.datetime(2009, 1, 1)
+ time_delta = dates[0] - root_date
# MLT runs 0-24 each orbit
mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
From f68067fa57b6b973e39a66856f36afe428909de0 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 5 Oct 2023 14:09:24 -0400
Subject: [PATCH 145/365] MAINT: replace deprecated instruments in tests
---
pysat/tests/test_instrument.py | 31 +-------------
pysat/tests/test_instrument_custom.py | 4 +-
pysat/tests/test_instrument_index.py | 2 +-
pysat/tests/test_instrument_padding.py | 57 +++++++++++++++++---------
pysat/tests/test_instruments.py | 29 -------------
pysat/tests/test_meta.py | 29 ++-----------
pysat/tests/test_orbits.py | 24 +++++------
pysat/tests/test_utils_coords.py | 9 ++--
pysat/tests/test_utils_io.py | 47 ++-------------------
9 files changed, 64 insertions(+), 168 deletions(-)
diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py
index 18861ca59..aeb1b8cdb 100644
--- a/pysat/tests/test_instrument.py
+++ b/pysat/tests/test_instrument.py
@@ -12,7 +12,6 @@
import pysat
import pysat.instruments.pysat_ndtesting
import pysat.instruments.pysat_testing
-import pysat.instruments.pysat_testing_xarray
from pysat.tests.classes.cls_instrument_access import InstAccessTests
from pysat.tests.classes.cls_instrument_integration import InstIntegrationTests
@@ -199,34 +198,6 @@ def teardown_method(self):
return
-# TODO(#908): remove below class when pysat_testing_xarray is removed.
-class TestBasicsXarray(TestBasics):
- """Basic tests for xarray `pysat.Instrument`."""
-
- def setup_method(self):
- """Set up the unit test environment for each method."""
-
- reload(pysat.instruments.pysat_testing_xarray)
- self.testInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
- num_samples=10,
- clean_level='clean',
- update_files=True,
- use_header=True,
- **self.testing_kwargs)
- self.ref_time = pysat.instruments.pysat_testing_xarray._test_dates[
- '']['']
- self.ref_doy = int(self.ref_time.strftime('%j'))
- self.out = None
- return
-
- def teardown_method(self):
- """Clean up the unit test environment after each method."""
-
- del self.testInst, self.out, self.ref_time, self.ref_doy
- return
-
-
class TestBasicsNDXarray(TestBasics):
"""Basic tests for ND xarray `pysat.Instrument`.
@@ -548,7 +519,7 @@ def test_eq_different_object(self):
num_samples=10, clean_level='clean',
update_files=True, use_header=True)
- obj2 = pysat.Instrument(platform='pysat', name='testing_xarray',
+ obj2 = pysat.Instrument(platform='pysat', name='ndtesting',
num_samples=10, clean_level='clean',
update_files=True, use_header=True)
assert not (obj1 == obj2)
diff --git a/pysat/tests/test_instrument_custom.py b/pysat/tests/test_instrument_custom.py
index 2c3f5ba30..b7888c7ac 100644
--- a/pysat/tests/test_instrument_custom.py
+++ b/pysat/tests/test_instrument_custom.py
@@ -235,10 +235,10 @@ class TestBasicsXarray(TestBasics):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
num_samples=10, clean_level='clean',
use_header=True)
- self.load_date = pysat.instruments.pysat_testing_xarray._test_dates
+ self.load_date = pysat.instruments.pysat_ndtesting._test_dates
self.load_date = self.load_date['']['']
self.testInst.load(date=self.load_date)
self.custom_args = [2]
diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py
index ac87a74ce..451598019 100644
--- a/pysat/tests/test_instrument_index.py
+++ b/pysat/tests/test_instrument_index.py
@@ -107,7 +107,7 @@ def eval_warnings(self):
return
# TODO(#1094): Remove in pysat 3.2.0, potentially with class
- @pytest.mark.parametrize('name', ['testing', 'ndtesting', 'testing_xarray'])
+ @pytest.mark.parametrize('name', ['testing', 'ndtesting'])
def test_kwarg_malformed_index(self, name):
"""Test deprecation of `malformed_index` kwarg.
diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py
index 59e02e687..ad30f55ac 100644
--- a/pysat/tests/test_instrument_padding.py
+++ b/pysat/tests/test_instrument_padding.py
@@ -10,7 +10,6 @@
import pysat
import pysat.instruments.pysat_ndtesting
import pysat.instruments.pysat_testing
-import pysat.instruments.pysat_testing_xarray
from pysat.utils import testing
from pysat.utils.time import filter_datetime_input
@@ -143,9 +142,11 @@ class TestDataPaddingbyFileXarray(TestDataPaddingbyFile):
def setup_method(self):
"""Set up the unit test environment for each method."""
- reload(pysat.instruments.pysat_testing_xarray)
+ reload(pysat.instruments.pysat_ndtesting)
self.testInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
+ num_samples=86400,
+ sample_rate='1s',
clean_level='clean',
pad={'minutes': 5},
update_files=True,
@@ -153,7 +154,9 @@ def setup_method(self):
self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile')
self.rawInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
+ num_samples=86400,
+ sample_rate='1s',
clean_level='clean',
update_files=True,
use_header=True)
@@ -205,16 +208,20 @@ class TestOffsetRightFileDataPaddingBasicsXarray(TestDataPaddingbyFile):
def setup_method(self):
"""Set up the unit test environment for each method."""
- reload(pysat.instruments.pysat_testing_xarray)
+ reload(pysat.instruments.pysat_ndtesting)
self.testInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
+ num_samples=86400,
+ sample_rate='1s',
clean_level='clean',
update_files=True,
sim_multi_file_right=True,
pad={'minutes': 5},
use_header=True)
self.rawInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
+ num_samples=86400,
+ sample_rate='1s',
clean_level='clean',
update_files=True,
sim_multi_file_right=True,
@@ -499,9 +506,11 @@ class TestDataPaddingXArray(TestDataPadding):
def setup_method(self):
"""Set up the unit test environment for each method."""
- reload(pysat.instruments.pysat_testing_xarray)
+ reload(pysat.instruments.pysat_ndtesting)
self.testInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
+ num_samples=86400,
+ sample_rate='1s',
clean_level='clean',
pad={'minutes': 5},
update_files=True,
@@ -524,9 +533,11 @@ class TestDataPaddingXArrayNonMonotonic(TestDataPadding):
def setup_method(self):
"""Set up the unit test environment for each method."""
- reload(pysat.instruments.pysat_testing_xarray)
+ reload(pysat.instruments.pysat_ndtesting)
self.testInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
+ num_samples=86400,
+ sample_rate='1s',
clean_level='clean',
pad={'minutes': 5},
non_monotonic_index=True,
@@ -603,9 +614,11 @@ class TestMultiFileRightDataPaddingBasicsXarray(TestDataPadding):
def setup_method(self):
"""Set up the unit test environment for each method."""
- reload(pysat.instruments.pysat_testing_xarray)
+ reload(pysat.instruments.pysat_ndtesting)
self.testInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
+ num_samples=86400,
+ sample_rate='1s',
clean_level='clean',
update_files=True,
sim_multi_file_right=True,
@@ -630,9 +643,11 @@ class TestMultiFileRightDataPaddingBasicsXarrayNonMonotonic(TestDataPadding):
def setup_method(self):
"""Set up the unit test environment for each method."""
- reload(pysat.instruments.pysat_testing_xarray)
+ reload(pysat.instruments.pysat_ndtesting)
self.testInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
+ num_samples=86400,
+ sample_rate='1s',
clean_level='clean',
update_files=True,
sim_multi_file_right=True,
@@ -713,9 +728,11 @@ class TestMultiFileLeftDataPaddingBasicsXarray(TestDataPadding):
def setup_method(self):
"""Set up the unit test environment for each method."""
- reload(pysat.instruments.pysat_testing_xarray)
+ reload(pysat.instruments.pysat_ndtesting)
self.testInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
+ num_samples=86400,
+ sample_rate='1s',
clean_level='clean',
update_files=True,
sim_multi_file_left=True,
@@ -740,11 +757,13 @@ class TestMultiFileLeftDataPaddingBasicsXarrayNonMonotonic(TestDataPadding):
def setup_method(self):
"""Set up the unit test environment for each method."""
- reload(pysat.instruments.pysat_testing_xarray)
+ reload(pysat.instruments.pysat_ndtesting)
self.testInst = pysat.Instrument(platform='pysat',
- name='testing_xarray',
+ name='ndtesting',
clean_level='clean',
update_files=True,
+ num_samples=86400,
+ sample_rate='1s',
sim_multi_file_left=True,
non_monotonic_index=True,
pad={'minutes': 5},
diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py
index b1f03a42d..470a83278 100644
--- a/pysat/tests/test_instruments.py
+++ b/pysat/tests/test_instruments.py
@@ -274,32 +274,3 @@ def test_old_pytest_mark_presence(self):
for j in range(0, n_args)]
assert "download" in mark_names
-
- @pytest.mark.parametrize("inst_module", ['pysat_testing_xarray',
- 'pysat_testing2d_xarray'])
- def test_deprecated_instruments(self, inst_module):
- """Check that instantiating old instruments raises a DeprecationWarning.
-
- Parameters
- ----------
- inst_module : str
- name of deprecated module.
-
- """
-
- with warnings.catch_warnings(record=True) as war:
- pysat.Instrument(inst_module=getattr(pysat.instruments,
- inst_module),
- use_header=True)
-
- warn_msgs = [" ".join(["The instrument module",
- "`{:}`".format(inst_module),
- "has been deprecated and will be removed",
- "in 3.2.0+."])]
-
- # Ensure the minimum number of warnings were raised.
- assert len(war) >= len(warn_msgs)
-
- # Test the warning messages, ensuring each attribute is present.
- testing.eval_warnings(war, warn_msgs)
- return
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index 5e3b1699a..9de618392 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -412,10 +412,7 @@ def test_equality(self):
assert cmeta == self.meta, "identical meta objects differ"
return
- # TODO(#908): remove tests for deprecated instruments
- @pytest.mark.parametrize("inst_name", ["testing",
- "ndtesting", "testing_xarray",
- "testmodel"])
+ @pytest.mark.parametrize("inst_name", ["testing", "ndtesting", "testmodel"])
def test_equality_w_copy(self, inst_name):
"""Test that meta remains the same when copied.
@@ -489,10 +486,7 @@ def test_value_inequality(self, label_key):
"differences not detected in label {:s}".format(label_key)
return
- # TODO(#908): remove tests for deprecated instruments
- @pytest.mark.parametrize("inst_name", ["testing",
- "ndtesting", "testing_xarray",
- "testmodel"])
+ @pytest.mark.parametrize("inst_name", ["testing", "ndtesting", "testmodel"])
def test_pop(self, inst_name):
"""Test meta attributes are retained when extracted using pop.
@@ -1614,23 +1608,6 @@ def test_to_dict(self, preserve_case):
return
-class TestToDictXarray(TestToDict):
- """Test `.to_dict` methods using pysat test Instruments."""
-
- def setup_method(self):
- """Set up the unit test environment for each method."""
-
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
- num_samples=5, use_header=True)
- self.stime = pysat.instruments.pysat_testing_xarray._test_dates['']['']
- self.testInst.load(date=self.stime)
-
- # For output
- self.out = None
-
- return
-
-
class TestToDictXarrayND(TestToDict):
"""Test `.to_dict` methods using pysat test Instruments."""
@@ -1639,7 +1616,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'ndtesting',
num_samples=5, use_header=True)
- self.stime = pysat.instruments.pysat_testing_xarray._test_dates['']['']
+ self.stime = pysat.instruments.pysat_ndtesting._test_dates['']['']
self.testInst.load(date=self.stime)
# For output
diff --git a/pysat/tests/test_orbits.py b/pysat/tests/test_orbits.py
index 28d3173f6..c849d29a9 100644
--- a/pysat/tests/test_orbits.py
+++ b/pysat/tests/test_orbits.py
@@ -589,12 +589,12 @@ class TestGeneralOrbitsMLTxarray(TestGeneralOrbitsMLT):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'mlt'},
update_files=True,
use_header=True)
- self.stime = pysat.instruments.pysat_testing_xarray._test_dates['']['']
+ self.stime = pysat.instruments.pysat_ndtesting._test_dates['']['']
return
def teardown_method(self):
@@ -701,7 +701,7 @@ class TestGeneralOrbitsLongXarray(TestGeneralOrbitsMLT):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'longitude',
'kind': 'longitude'},
@@ -745,13 +745,13 @@ class TestGeneralOrbitsOrbitNumberXarray(TestGeneralOrbitsMLT):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'orbit_num',
'kind': 'orbit'},
update_files=True,
use_header=True)
- self.stime = pysat.instruments.pysat_testing_xarray._test_dates['']['']
+ self.stime = pysat.instruments.pysat_ndtesting._test_dates['']['']
return
def teardown_method(self):
@@ -789,13 +789,13 @@ class TestGeneralOrbitsLatitudeXarray(TestGeneralOrbitsMLT):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'latitude',
'kind': 'polar'},
update_files=True,
use_header=True)
- self.stime = pysat.instruments.pysat_testing_xarray._test_dates['']['']
+ self.stime = pysat.instruments.pysat_ndtesting._test_dates['']['']
return
def teardown_method(self):
@@ -874,7 +874,7 @@ class TestOrbitsGappyDataXarray(TestOrbitsGappyData):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'mlt'},
update_files=True,
@@ -946,7 +946,7 @@ class TestOrbitsGappyData2Xarray(TestOrbitsGappyData2):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'mlt'},
use_header=True)
@@ -991,7 +991,7 @@ class TestOrbitsGappyLongDataXarray(TestOrbitsGappyData):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'longitude',
'kind': 'longitude'},
@@ -1037,7 +1037,7 @@ class TestOrbitsGappyOrbitNumDataXarray(TestOrbitsGappyData):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'orbit_num',
'kind': 'orbit'},
@@ -1084,7 +1084,7 @@ class TestOrbitsGappyOrbitLatDataXarray(TestOrbitsGappyData):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'latitude',
'kind': 'polar'},
diff --git a/pysat/tests/test_utils_coords.py b/pysat/tests/test_utils_coords.py
index 1ce7fea09..8edd4b113 100644
--- a/pysat/tests/test_utils_coords.py
+++ b/pysat/tests/test_utils_coords.py
@@ -63,8 +63,7 @@ def teardown_method(self):
del self.py_inst, self.inst_time
return
- @pytest.mark.parametrize("name", ["testing", "testing_xarray",
- "ndtesting", "testmodel"])
+ @pytest.mark.parametrize("name", ["testing", "ndtesting", "testmodel"])
def test_update_longitude(self, name):
"""Test `update_longitude` successful run."""
@@ -119,7 +118,7 @@ def teardown_method(self):
del self.py_inst, self.inst_time
return
- @pytest.mark.parametrize("name", ["testing", "testing_xarray"])
+ @pytest.mark.parametrize("name", ["testing", "ndtesting"])
def test_calc_solar_local_time(self, name):
"""Test SLT calculation with longitudes from 0-360 deg for 0 UTH."""
@@ -143,7 +142,7 @@ def test_calc_solar_local_time(self, name):
assert np.min(np.abs(cos_diff)) > 1.0 - 1.0e-6
return
- @pytest.mark.parametrize("name", ["testing", "testing_xarray"])
+ @pytest.mark.parametrize("name", ["testing", "ndtesting"])
def test_calc_solar_local_time_inconsistent_keywords(self, name, caplog):
"""Test that ref_date only works when apply_modulus=False."""
@@ -274,7 +273,7 @@ def test_single_lon_calc_solar_local_time(self):
"""Test calc_solar_local_time with a single longitude value."""
# Instantiate instrument and load data
- self.py_inst = pysat.Instrument(platform='pysat', name="testing_xarray",
+ self.py_inst = pysat.Instrument(platform='pysat', name="ndtesting",
use_header=True)
self.py_inst.load(date=self.inst_time)
lon_name = 'lon2'
diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py
index cb3dd7b87..f41cf2e79 100644
--- a/pysat/tests/test_utils_io.py
+++ b/pysat/tests/test_utils_io.py
@@ -1247,24 +1247,7 @@ def test_missing_metadata(self):
class TestNetCDF4IntegrationXarray(TestNetCDF4Integration):
- """Integration tests for the netCDF4 I/O utils using xarray data."""
-
- def setup_method(self):
- """Create a testing environment."""
-
- # Create an instrument object that has a meta with some
- # variables allowed to be nan within metadata when exporting.
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
- num_samples=5, use_header=True)
- self.testInst.load(date=self.testInst.inst_module._test_dates[''][''],
- use_header=True)
- self.pformat = self.testInst.pandas_format
-
- return
-
-
-class TestNetCDF4Integration2DXarray(TestNetCDF4Integration):
- """Integration tests for the netCDF4 I/O utils using 2dxarray Instrument."""
+ """Integration tests for the netCDF4 I/O utils using xarray Instrument."""
def setup_method(self):
"""Create a testing environment."""
@@ -1305,7 +1288,7 @@ def setup_method(self):
# Create an instrument object that has a meta with some
# variables allowed to be nan within metadata when exporting.
- self.testInst = pysat.Instrument('pysat', 'testing_xarray',
+ self.testInst = pysat.Instrument('pysat', 'ndtesting',
num_samples=5, use_header=True)
self.testInst.load(date=self.testInst.inst_module._test_dates[''][''],
use_header=True)
@@ -1706,36 +1689,12 @@ def test_remove_netcdf4_standards(self, caplog):
class TestMetaTranslationXarray(TestMetaTranslation):
"""Test meta translation when writing/loading files xarray Instrument."""
- def setup_method(self):
- """Create test environment."""
-
- self.test_inst = pysat.Instrument('pysat', 'testing_xarray',
- num_samples=5, use_header=True)
- self.test_date = pysat.instruments.pysat_testing_xarray._test_dates
- self.test_date = self.test_date['']['']
- self.test_inst.load(date=self.test_date)
- self.meta_dict = self.test_inst.meta.to_dict()
- self.out = None
-
- return
-
- def teardown_method(self):
- """Cleanup test environment."""
-
- del self.test_inst, self.test_date, self.out, self.meta_dict
-
- return
-
-
-class TestMetaTranslation2DXarray(TestMetaTranslation):
- """Test meta translation when writing/loading files xarray2d Instrument."""
-
def setup_method(self):
"""Create test environment."""
self.test_inst = pysat.Instrument('pysat', 'ndtesting',
num_samples=5, use_header=True)
- self.test_date = pysat.instruments.pysat_testing_xarray._test_dates
+ self.test_date = pysat.instruments.pysat_ndtesting._test_dates
self.test_date = self.test_date['']['']
self.test_inst.load(date=self.test_date)
self.meta_dict = self.test_inst.meta.to_dict()
From b021068fc715bb39301994ef92b6c5495406e84e Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 5 Oct 2023 14:09:36 -0400
Subject: [PATCH 146/365] ENH: add orbit number to ndtesting
---
pysat/instruments/pysat_ndtesting.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index f10be34a4..b12a09f9e 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -5,6 +5,7 @@
import functools
import numpy as np
+import pandas as pds
import xarray as xr
import pysat
@@ -164,6 +165,13 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
altitude = alt0 * np.ones(data['latitude'].shape)
data['altitude'] = ((epoch_name), altitude)
+ # Fake orbit number
+ fake_delta = dates[0] - (_test_dates[''][''] - pds.DateOffset(years=1))
+ data['orbit_num'] = ((epoch_name),
+ mm_test.generate_fake_data(fake_delta.total_seconds(),
+ uts, period=iperiod['lt'],
+ cyclic=False))
+
# Create some fake data to support testing of averaging routines
mlt_int = data['mlt'].astype(int).data
long_int = (data['longitude'] / 15.).astype(int).data
From 4b05449ab02eff8139687beb1e86ee4e24191304 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 5 Oct 2023 14:10:07 -0400
Subject: [PATCH 147/365] DOC: remove deprecated instruments from docs
---
docs/api.rst | 18 ------------------
docs/dependency.rst | 5 -----
docs/instruments/testing_instruments.rst | 14 --------------
docs/quickstart.rst | 2 +-
4 files changed, 1 insertion(+), 38 deletions(-)
diff --git a/docs/api.rst b/docs/api.rst
index d14c971a4..89c4a4ddb 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -232,24 +232,6 @@ pysat_testing
:members:
-.. _api-pysat-testing_xarray:
-
-pysat_testing_xarray
-^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: pysat.instruments.pysat_testing_xarray
- :members:
-
-
-.. _api-pysat-testing2d_xarray:
-
-pysat_testing2d_xarray
-^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: pysat.instruments.pysat_testing2d_xarray
- :members:
-
-
.. _api-pysat-testmodel:
pysat_testmodel
diff --git a/docs/dependency.rst b/docs/dependency.rst
index fa31acf32..4b0251cb3 100644
--- a/docs/dependency.rst
+++ b/docs/dependency.rst
@@ -267,11 +267,6 @@ pysat_testing
object with 1D data as a function of latitude, longitude, and altitude in a
pandas format. Most similar to in situ data.
-pysat_testing_xarray
-^^^^^^^^^^^^^^^^^^^^
-:ref:`api-pysat-testing_xarray` returns a satellite-like object with 1D data as
-a function of latitude, longitude, and altitude in a xarray format.
-
pysat_ndtesting
^^^^^^^^^^^^^^^^^^^^^^
:ref:`api-pysat-ndtesting` is a satellite-like object that returns all
diff --git a/docs/instruments/testing_instruments.rst b/docs/instruments/testing_instruments.rst
index 2361a0765..f92558e33 100644
--- a/docs/instruments/testing_instruments.rst
+++ b/docs/instruments/testing_instruments.rst
@@ -14,13 +14,6 @@ An instrument with satellite-like data as a function of latitude, longitude,
and altitude in a pandas format. See :ref:`api-pysat-testing` for more details.
-pysat_testing_xarray
-^^^^^^^^^^^^^^^^^^^^
-An instrument with satellite-like data as a function of latitude, longitude,
-and altitude in a xarray format. See :ref:`api-pysat-testing_xarray` for more
-details.
-
-
pysat_ndtesting
^^^^^^^^^^^^^^^
An instrument with satellite-like data like :py:mod:`pysat_testing` that
@@ -28,13 +21,6 @@ also has an imager-like 3D data variable. See :ref:`api-pysat-ndtesting`
for more details.
-pysat_testing2d_xarray
-^^^^^^^^^^^^^^^^^^^^^^
-An instrument with satellite-like data like :py:mod:`pysat_testing_xarray` that
-also has an imager-like 3D data variable. See :ref:`api-pysat-testing2d_xarray`
-for more details.
-
-
pysat_testmodel
^^^^^^^^^^^^^^^
An instrument with model-like data that returns a 4D object as a function of
diff --git a/docs/quickstart.rst b/docs/quickstart.rst
index eecb860a5..f1796aaea 100644
--- a/docs/quickstart.rst
+++ b/docs/quickstart.rst
@@ -67,7 +67,7 @@ installations.
print(inst.data)
# Testing out the xarray installation
- inst = pysat.Instrument('pysat', 'testing_xarray')
+ inst = pysat.Instrument('pysat', 'ndtesting')
inst.load(2009, 1)
print(inst.data)
From 7b5396e48ca8dbb56d9086b41b04c646061bbf32 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 5 Oct 2023 14:12:30 -0400
Subject: [PATCH 148/365] DOC: update changelog
---
CHANGELOG.md | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 73f76d005..db8d3fe9c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,9 @@ This project adheres to [Semantic Versioning](https://semver.org/).
memory usage in the `load` method.
* Added a hidden method the Instrument class `_get_epoch_name_from_data` to
reduce code duplication.
+ * Added options to customize `pysat_ndtesting` instrument with sample rate,
+ shift in time.
+ * Added orbit number to `pysat_ndtesting`.
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
@@ -25,6 +28,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Add pandas cap
* Remove deprecated `pysat_testing2d` instrument
* Remove deprecated meta children info
+ * Remove deprecated `pysat_testing_xarray` instrument
+ * Remove deprecated `pysat_testing2d_xarray` instrument
[3.1.0] - 2023-05-31
--------------------
From 74e7101c8fa135d3e4e319c72f16903dcf8bb6e4 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
Date: Thu, 5 Oct 2023 16:10:51 -0400
Subject: [PATCH 149/365] Update pysat/tests/test_meta.py
Co-authored-by: Angeline Burrell
---
pysat/tests/test_meta.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index 5e3b1699a..4f7bf9777 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -1375,7 +1375,7 @@ def test_meta_mutable_properties(self):
setattr(self.meta, 'data', pds.DataFrame())
# Test that data is empty
- assert self.meta.data.empty, "`meta.data` not updated correctly"
+ assert self.meta.empty, "`meta.data` not updated correctly"
except AttributeError:
raise AssertionError("Couldn't update mutable property 'data'")
return
From 65cc8daf7a4fbf3569e85d385e3f12b5e80e88e6 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 12:25:22 -0400
Subject: [PATCH 150/365] STY: remove unaccessible error
---
pysat/utils/io.py | 38 ++++++++++++++------------------------
1 file changed, 14 insertions(+), 24 deletions(-)
diff --git a/pysat/utils/io.py b/pysat/utils/io.py
index 2b972ca4a..63a4692fb 100644
--- a/pysat/utils/io.py
+++ b/pysat/utils/io.py
@@ -1588,34 +1588,24 @@ def inst_to_netcdf(inst, fname, base_instrument=None, epoch_name=None,
# Not datetime data, just store as is.
cdfkey[:] = data.values.astype(coltype)
else:
- # It is a Series of objects. First, figure out what the
- # individual object types are. Then, act as needed.
-
- # Use info in coltype to get real datatype of object
- if coltype == str:
- if '_FillValue' in export_meta[lower_key].keys():
- str_fill = export_meta[lower_key]['_FillValue']
- del export_meta[lower_key]['_FillValue']
- else:
- str_fill = ''
+ if '_FillValue' in export_meta[lower_key].keys():
+ str_fill = export_meta[lower_key]['_FillValue']
+ del export_meta[lower_key]['_FillValue']
+ else:
+ str_fill = ''
- cdfkey = out_data.createVariable(case_key, coltype,
- dimensions=epoch_name,
- complevel=complevel,
- shuffle=shuffle,
- fill_value=str_fill)
+ cdfkey = out_data.createVariable(case_key, coltype,
+ dimensions=epoch_name,
+ complevel=complevel,
+ shuffle=shuffle,
+ fill_value=str_fill)
- # Set metadata
- cdfkey.setncatts(export_meta[lower_key])
+ # Set metadata
+ cdfkey.setncatts(export_meta[lower_key])
- # Time to actually write the data now
- cdfkey[:] = data.values
+ # Time to actually write the data now
+ cdfkey[:] = data.values
- else:
- raise ValueError(' '.join(('Recursive data detected.',
- 'Please ensure that the',
- 'DataFrame does not contain',
- 'other pandas objects.')))
else:
# Attach the metadata to a separate xarray.Dataset object, ensuring
# the Instrument data object is unchanged.
From b25e836d21ae5b447b0388564bf552a7fd9bc564 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 12:45:39 -0400
Subject: [PATCH 151/365] BUG: nest if
---
pysat/tests/test_utils_io.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py
index 25c769e5b..cf6d03695 100644
--- a/pysat/tests/test_utils_io.py
+++ b/pysat/tests/test_utils_io.py
@@ -992,9 +992,9 @@ def test_filter_netcdf4_metadata(self, remove, check_type, export_nan,
"{:} should have been exported".format(repr(mkey))
else:
if all([mkey in export_nan,
- not np.issubdtype(data_type, str),
- np.isnan(mdict[mkey])]):
- assert np.isnan(fdict[mkey])
+ not np.issubdtype(data_type, str)]):
+ if np.isnan(mdict[mkey]):
+ assert np.isnan(fdict[mkey])
else:
if mkey in check_type and fdict[mkey] != mdict[mkey]:
assert fdict[mkey] == data_type(mdict[mkey]), \
From 6395f80839e14af76317cc5377d2e7852c148c65 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 12:48:46 -0400
Subject: [PATCH 152/365] MAINT: remove hacking cap
---
pyproject.toml | 2 +-
test_requirements.txt | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 9ec0b533b..a5a48123d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,7 +58,7 @@ test = [
"coveralls < 3.3",
"flake8",
"flake8-docstrings",
- "hacking >= 1.0, <6.0",
+ "hacking >= 1.0,
"pysatSpaceWeather",
"pytest",
"pytest-cov",
diff --git a/test_requirements.txt b/test_requirements.txt
index dc679dcce..8a33141b8 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -1,7 +1,7 @@
coveralls<3.3
flake8
flake8-docstrings
-hacking>=1.0,<6.0
+hacking>=1.0
ipython
m2r2
numpydoc
From 2d6e94304b76404e729851cd758033969ab59952 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 12:54:20 -0400
Subject: [PATCH 153/365] BUG: "
---
pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyproject.toml b/pyproject.toml
index a5a48123d..eb0dbe732 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,7 +58,7 @@ test = [
"coveralls < 3.3",
"flake8",
"flake8-docstrings",
- "hacking >= 1.0,
+ "hacking >= 1.0",
"pysatSpaceWeather",
"pytest",
"pytest-cov",
From a55d4f730c8c2947993147b66dcd5d1e7c96ba76 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 12:56:08 -0400
Subject: [PATCH 154/365] DOC: update changelog
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b547a3889..ed07e4687 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Implement pyproject to manage metadata
* Remove Sphinx cap
* Add pandas cap
+ * Update usage of whitespace and if statements (E275)
+ * Remove hacking cap
[3.1.0] - 2023-05-31
--------------------
From e4eb7ea782a9b2dde453d5c98fe717b760ca25db Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 13:33:38 -0400
Subject: [PATCH 155/365] MAINT: remove instrument test class
---
pysat/tests/instrument_test_class.py | 78 ----------------------------
1 file changed, 78 deletions(-)
delete mode 100644 pysat/tests/instrument_test_class.py
diff --git a/pysat/tests/instrument_test_class.py b/pysat/tests/instrument_test_class.py
deleted file mode 100644
index c35e0ff3a..000000000
--- a/pysat/tests/instrument_test_class.py
+++ /dev/null
@@ -1,78 +0,0 @@
-"""Standardized class and functions to test instruments for pysat libraries.
-
-Note
-----
-Not directly called by pytest, but imported as part of test_instruments.py.
-Can be imported directly for external instrument libraries of pysat instruments.
-
-"""
-
-import warnings
-
-import pysat.tests.classes.cls_instrument_library as cls_inst_lib
-
-
-def initialize_test_inst_and_date(inst_dict):
- """Initialize the instrument object to test and date.
-
- .. deprecated:: 3.0.2
- `initialize_test_inst_and_date` will be removed in pysat 3.2.0, it is
- moved to `pysat.tests.classes.cls_instrument_library`.
-
- Parameters
- ----------
- inst_dict : dict
- Dictionary containing specific instrument info, generated by
- generate_instrument_list
-
- Returns
- -------
- test_inst : pysat.Instrument
- instrument object to be tested
- date : dt.datetime
- test date from module
-
- """
-
- warnings.warn(" ".join(["`initialize_test_inst_and_date` has been moved to",
- "`pysat.tests.classes.cls_instrument_library`.",
- "The link here will be removed in 3.2.0+."]),
- DeprecationWarning, stacklevel=2)
- return cls_inst_lib.initialize_test_inst_and_date(inst_dict)
-
-
-class InstTestClass(cls_inst_lib.InstLibTests):
- """Provide standardized tests for pysat instrument libraries.
-
- .. deprecated:: 3.0.2
- `InstTestClass` will be removed in pysat 3.2.0, it is replaced by
- `pysat.tests.classes.cls_instrument_library.InstLibTests`.
-
- Note
- ----
- Uses class level setup and teardown so that all tests use the same
- temporary directory. We do not want to geneate a new tempdir for each test,
- as the load tests need to be the same as the download tests.
-
- Not directly run by pytest, but inherited through test_instruments.py
-
- Users will need to run `apply_marks_to_tests` before setting up the test
- class.
-
- """
-
- def __init_subclass__(self):
- """Throw a warning if used as a subclass."""
-
- warnings.warn(" ".join(
- ["`InstTestClass` has been deprecated and will be removed in",
- "3.2.0+. Please update code to use the `InstLibTests` class",
- "under `pysat.tests.classes.cls_instrument_library`."]),
- DeprecationWarning, stacklevel=2)
- warnings.warn(" ".join(
- ["`test_load` now uses `@pytest.mark.load_options` in place",
- "of `@pytest.mark.download`. The old behavior will be removed in",
- "3.2.0+. Please update code or use the new"
- "`InstLibTests.initialize_test_package` function",
- "under `pysat.tests.classes.cls_instrument_library`."]),
- DeprecationWarning, stacklevel=2)
From 162fa2450b0e977813d8818a2d477dc49befda79 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 13:34:03 -0400
Subject: [PATCH 156/365] MAINT: remove dep tests
---
pysat/tests/test_instruments.py | 66 ---------------------------------
1 file changed, 66 deletions(-)
diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py
index 470a83278..e118d0a52 100644
--- a/pysat/tests/test_instruments.py
+++ b/pysat/tests/test_instruments.py
@@ -16,7 +16,6 @@
import pysat
import pysat.tests.classes.cls_instrument_library as cls_inst_lib
from pysat.tests.classes.cls_instrument_library import InstLibTests
-import pysat.tests.instrument_test_class as itc
from pysat.utils import testing
# Optional code to pass through user and password info to test instruments
@@ -209,68 +208,3 @@ def test_clean_with_warnings(self, clean_level, change, warn_type,
self.test_clean_warn(clean_level, inst_dict, caplog)
return
-
-
-class TestDeprecation(object):
- """Unit test for deprecation warnings."""
-
- def setup_method(self):
- """Set up the unit test environment for each method."""
-
- warnings.simplefilter("always", DeprecationWarning)
- return
-
- def teardown_method(self):
- """Clean up the unit test environment after each method."""
-
- return
-
- def test_subclass_inst_test_class(self):
- """Check that subclass of old instrument library tests is deprecated."""
-
- with warnings.catch_warnings(record=True) as war:
-
- class OldClass(itc.InstTestClass):
- """Dummy subclass."""
-
- pass
-
- self.warn_msgs = ["`InstTestClass` has been deprecated",
- "`test_load` now uses `@pytest.mark.load_options`"]
- self.warn_msgs = np.array(self.warn_msgs)
-
- # Ensure the minimum number of warnings were raised
- assert len(war) >= len(self.warn_msgs)
-
- # Test the warning messages, ensuring each attribute is present
- testing.eval_warnings(war, self.warn_msgs)
- return
-
- def test_old_initialize_inst_and_date(self):
- """Check that subclass of old instrument library tests is deprecated."""
-
- with warnings.catch_warnings(record=True) as war:
- try:
- itc.initialize_test_inst_and_date({})
- except KeyError:
- # empty dict produces KeyError
- pass
-
- self.warn_msgs = ["`initialize_test_inst_and_date` has been moved to"]
- self.warn_msgs = np.array(self.warn_msgs)
-
- # Ensure the minimum number of warnings were raised
- assert len(war) >= len(self.warn_msgs)
-
- # Test the warning messages, ensuring each attribute is present
- testing.eval_warnings(war, self.warn_msgs)
- return
-
- def test_old_pytest_mark_presence(self):
- """Test that pytest mark is backwards compatible."""
-
- n_args = len(InstLibTests.test_load.pytestmark)
- mark_names = [InstLibTests.test_load.pytestmark[j].name
- for j in range(0, n_args)]
-
- assert "download" in mark_names
From 3adadc7061a8a5debdd26980b6437766ce298b1e Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 13:34:13 -0400
Subject: [PATCH 157/365] MAINT: update usage
---
pysat/utils/registry.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pysat/utils/registry.py b/pysat/utils/registry.py
index 97193ccbc..b0e24001a 100644
--- a/pysat/utils/registry.py
+++ b/pysat/utils/registry.py
@@ -52,7 +52,7 @@
import logging
import pysat
-import pysat.tests.instrument_test_class as itc
+import pysat.tests.classes.cls_instrument_library as itc
def load_saved_modules():
@@ -147,7 +147,7 @@ def register(module_names, overwrite=False):
raise
# Second, check that module is itself pysat compatible
- validate = itc.InstTestClass()
+ validate = itc.InstLibTests()
# Work with test code, create dummy structure to make things work
class Foo(object):
From c272e4a88dbf5bc4d84911aaa128752fd26e29a2 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 13:34:23 -0400
Subject: [PATCH 158/365] DOC: update comments
---
pysat/tests/test_instrument_listgen.py | 2 +-
pysat/utils/_core.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/pysat/tests/test_instrument_listgen.py b/pysat/tests/test_instrument_listgen.py
index 8414f9fbb..7932651b0 100644
--- a/pysat/tests/test_instrument_listgen.py
+++ b/pysat/tests/test_instrument_listgen.py
@@ -45,7 +45,7 @@ def test_for_missing_test_date(self):
# If an instrument does not have the _test_dates attribute, it should
# still be added to the list for other checks to be run.
- # This will be caught later by InstTestClass.test_instrument_test_dates.
+ # This will be caught later by InstLibTests.test_instrument_test_dates.
assert not hasattr(self.test_library.pysat_testing, '_test_dates')
inst_list = generate_instrument_list(self.test_library)
assert 'pysat_testing' in inst_list['names']
diff --git a/pysat/utils/_core.py b/pysat/utils/_core.py
index 1241d34ef..cd32629f2 100644
--- a/pysat/utils/_core.py
+++ b/pysat/utils/_core.py
@@ -443,7 +443,7 @@ def generate_instrument_list(inst_loc, user_info=None):
# If this can't be imported, we can't pull out the info for the
# download / no_download tests. Leaving in basic tests for all
# instruments, but skipping the rest. The import error will be
- # caught as part of the pytest.mark.all_inst tests in InstTestClass
+ # caught as part of the pytest.mark.all_inst tests in InstLibTests
pass
else:
# try to grab basic information about the module so we
@@ -453,7 +453,7 @@ def generate_instrument_list(inst_loc, user_info=None):
except AttributeError:
# If a module does not have a test date, add it anyway for
# other tests. This will be caught later by
- # InstTestClass.test_instrument_test_dates
+ # InstLibTests.test_instrument_test_dates
info = {}
info[''] = {'': dt.datetime(2009, 1, 1)}
module._test_dates = info
From ce56952eacffc4ea9fd3c50ee84a499969cd6972 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 13:34:50 -0400
Subject: [PATCH 159/365] DOC: update changelog
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index db8d3fe9c..93f3160f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Remove deprecated meta children info
* Remove deprecated `pysat_testing_xarray` instrument
* Remove deprecated `pysat_testing2d_xarray` instrument
+ * Remove deprecated `instrument_test_class`
[3.1.0] - 2023-05-31
--------------------
From e3a1fcc47bf02f38b9930e4348371cf1451f03fd Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 14:18:41 -0400
Subject: [PATCH 160/365] MAINT: remove #1094
---
pysat/instruments/methods/testing.py | 10 -----
pysat/instruments/pysat_ndtesting.py | 17 ++------
pysat/instruments/pysat_testing.py | 14 +------
pysat/tests/test_instrument_index.py | 62 ----------------------------
4 files changed, 5 insertions(+), 98 deletions(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index 08705398d..3f02ff71a 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -770,13 +770,3 @@ def non_unique_index(index):
new_index = pds.to_datetime(new_index)
return new_index
-
-
-def _warn_malformed_kwarg():
- """Warn user that kwarg has been deprecated."""
-
- dstr = ' '.join(['The kwarg malformed_index has been deprecated and',
- 'will be removed in pysat 3.2.0+. Please use',
- 'non_monotonic_index or non_unique_index to specify',
- 'desired behaviour.'])
- warnings.warn(dstr, DeprecationWarning, stacklevel=2)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index b12a09f9e..d455eebcb 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -36,9 +36,9 @@
def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
sim_multi_file_left=False, non_monotonic_index=False,
- non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=864, sample_rate='100S', test_load_kwarg=None,
- max_latitude=90.0, num_extra_time_coords=0):
+ non_unique_index=False, start_time=None, num_samples=864,
+ sample_rate='100S', test_load_kwarg=None, max_latitude=90.0,
+ num_extra_time_coords=0):
"""Load the test files.
Parameters
@@ -61,10 +61,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
If True, time index will be non-monotonic (default=False)
non_unique_index : bool
If True, time index will be non-unique (default=False)
- malformed_index : bool
- If True, the time index will be non-unique and non-monotonic. Deprecated
- and scheduled for removal in pysat 3.2.0.
- (default=False)
start_time : dt.timedelta or NoneType
Offset time of start time since midnight UT. If None, instrument data
will begin at midnight. (default=None)
@@ -110,13 +106,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
else:
root_date = dt.datetime(2009, 1, 1)
- # TODO(#1094): Remove in pysat 3.2.0
- if malformed_index:
- # Warn that kwarg is deprecated and set new kwargs.
- mm_test._warn_malformed_kwarg()
- non_monotonic_index = True
- non_unique_index = True
-
if non_monotonic_index:
index = mm_test.non_monotonic_index(index)
if non_unique_index:
diff --git a/pysat/instruments/pysat_testing.py b/pysat/instruments/pysat_testing.py
index c96dccdfe..90592c754 100644
--- a/pysat/instruments/pysat_testing.py
+++ b/pysat/instruments/pysat_testing.py
@@ -40,8 +40,8 @@
def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
sim_multi_file_left=False, root_date=None, non_monotonic_index=False,
- non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=86400, test_load_kwarg=None, max_latitude=90.):
+ non_unique_index=False,start_time=None, num_samples=86400,
+ test_load_kwarg=None, max_latitude=90.):
"""Load the test files.
Parameters
@@ -67,9 +67,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
If True, time index will be non-monotonic (default=False)
non_unique_index : bool
If True, time index will be non-unique (default=False)
- malformed_index : bool
- If True, the time index will be non-unique and non-monotonic. Deprecated
- and scheduled for removal in pysat 3.2.0. (default=False)
start_time : dt.timedelta or NoneType
Offset time of start time since midnight UT. If None, instrument data
will begin at midnight. (default=None)
@@ -164,13 +161,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
data['int32_dummy'] = np.ones(len(data), dtype=np.int32)
data['int64_dummy'] = np.ones(len(data), dtype=np.int64)
- # TODO(#1094): Remove in pysat 3.2.0
- if malformed_index:
- # Warn that kwarg is deprecated and set new kwargs.
- mm_test._warn_malformed_kwarg()
- non_monotonic_index = True
- non_unique_index = True
-
# Activate if non-monotonic index is needed.
if np.any([non_monotonic_index, (tag == 'non_strict')]):
index = mm_test.non_monotonic_index(index)
diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py
index 451598019..5c33c65b2 100644
--- a/pysat/tests/test_instrument_index.py
+++ b/pysat/tests/test_instrument_index.py
@@ -76,65 +76,3 @@ def teardown_method(self):
del self.ref_time, self.name
return
-
-
-class TestDeprecation(object):
- """Unit test for deprecation warnings for index."""
-
- def setup_method(self):
- """Set up the unit test environment for each method."""
-
- warnings.simplefilter("always", DeprecationWarning)
- self.ref_time = pysat.instruments.pysat_testing._test_dates['']['']
- self.warn_msgs = []
- self.war = ""
- return
-
- def teardown_method(self):
- """Clean up the unit test environment after each method."""
-
- del self.ref_time, self.warn_msgs, self.war
- return
-
- def eval_warnings(self):
- """Evaluate the number and message of the raised warnings."""
-
- # Ensure the minimum number of warnings were raised.
- assert len(self.war) >= len(self.warn_msgs)
-
- # Test the warning messages, ensuring each attribute is present.
- testing.eval_warnings(self.war, self.warn_msgs)
- return
-
- # TODO(#1094): Remove in pysat 3.2.0, potentially with class
- @pytest.mark.parametrize('name', ['testing', 'ndtesting'])
- def test_kwarg_malformed_index(self, name):
- """Test deprecation of `malformed_index` kwarg.
-
- Parameters
- ----------
- name : str
- name of instrument that uses the deprecated `malformed_index` kwarg.
-
- """
-
- test_inst = pysat.Instrument(platform='pysat',
- name=name,
- strict_time_flag=False,
- use_header=True,
- malformed_index=True)
-
- # Catch the warnings
- with warnings.catch_warnings(record=True) as self.war:
- test_inst.load(date=self.ref_time)
-
- self.warn_msgs = np.array([" ".join(["The kwarg malformed_index has",
- "been deprecated"])])
-
- # Evaluate the warning output
- self.eval_warnings()
-
- # Check that resulting index is both non-monotonic and non-unique
- assert not test_inst.index.is_monotonic_increasing
- assert not test_inst.index.is_unique
- return
From 799f613f8b6a66c8643c0cc8335a6339f0339ca5 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 14:19:49 -0400
Subject: [PATCH 161/365] MAINT: remove #947
---
pysat/instruments/methods/general.py | 37 ----------------------------
pysat/tests/test_methods_general.py | 35 --------------------------
2 files changed, 72 deletions(-)
diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py
index 400480ddd..a9afcdf8e 100644
--- a/pysat/instruments/methods/general.py
+++ b/pysat/instruments/methods/general.py
@@ -148,43 +148,6 @@ def list_files(tag='', inst_id='', data_path='', format_str=None,
return out
-def convert_timestamp_to_datetime(inst, sec_mult=1.0, epoch_name='time'):
- """Use datetime instead of timestamp for Epoch.
-
- .. deprecated:: 3.0.2
- This routine has been deprecated with the addition of the kwargs
- `epoch_unit` and `epoch_origin` to `pysat.utils.io.load_netcdf4`.
- This routing will be removed in 3.2.0.
-
- Parameters
- ----------
- inst : pysat.Instrument
- associated pysat.Instrument object
- sec_mult : float
- Multiplier needed to convert epoch time to seconds (default=1.0)
- epoch_name : str
- variable name for instrument index (default='Epoch')
-
- Note
- ----
- If the variable represented by epoch_name is not a float64, data is passed
- through unchanged.
-
- """
-
- warnings.warn(" ".join(["New kwargs added to `pysat.utils.io.load_netCDF4`",
- "for generalized handling, deprecated",
- "function will be removed in pysat 3.2.0+"]),
- DeprecationWarning, stacklevel=2)
-
- if inst.data[epoch_name].dtype == 'float64':
- inst.data[epoch_name] = pds.to_datetime(
- [dt.datetime.utcfromtimestamp(int(np.floor(epoch_time * sec_mult)))
- for epoch_time in inst.data[epoch_name]])
-
- return
-
-
def remove_leading_text(inst, target=None):
"""Remove leading text on variable names.
diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py
index d08108f72..a9724553e 100644
--- a/pysat/tests/test_methods_general.py
+++ b/pysat/tests/test_methods_general.py
@@ -297,38 +297,3 @@ def test_load_file_with_kwargs(self):
self.eval_data_cols()
assert len(self.data.columns) == len(self.data_cols)
return
-
-
-class TestDeprecation(object):
- """Unit tests for deprecated methods."""
-
- def setup_method(self):
- """Set up the unit test environment for each method."""
-
- warnings.simplefilter("always", DeprecationWarning)
- return
-
- def teardown_method(self):
- """Clean up the unit test environment after each method."""
-
- return
-
- def test_convert_timestamp_to_datetime(self):
- """Test that convert_timestamp_to_datetime is deprecated."""
-
- warn_msgs = [" ".join(
- ["New kwargs added to `pysat.utils.io.load_netCDF4`",
- "for generalized handling, deprecated",
- "function will be removed in pysat 3.2.0+"])]
-
- test = pysat.Instrument('pysat', 'testing', use_header=True)
- test.load(2009, 1)
- with warnings.catch_warnings(record=True) as war:
- gen.convert_timestamp_to_datetime(test, epoch_name='uts')
-
- # Ensure the minimum number of warnings were raised
- assert len(war) >= len(warn_msgs)
-
- # Test the warning messages, ensuring each attribute is present
- pysat.utils.testing.eval_warnings(war, warn_msgs)
- return
From 365829a1e818f7ab2cbff6a45090a678a90563f3 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 14:20:57 -0400
Subject: [PATCH 162/365] MAINT: remove #807
---
pysat/_instrument.py | 18 ------------------
pysat/tests/test_instrument.py | 23 -----------------------
2 files changed, 41 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 9d5c5b3b7..1e90ab233 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -1518,24 +1518,6 @@ def _assign_attrs(self, by_name=False):
else:
missing.append(iattr)
- # Check and see if this instrument has deprecated _test_download_travis
- # TODO(#807): Remove this check once _test_download_travis is removed.
- if hasattr(self.inst_module, '_test_download_travis'):
- local_attr = getattr(self.inst_module, '_test_download_travis')
-
- # Test to see that this attribute is set for the desired
- # `inst_id` and `tag`.
- if self.inst_id in local_attr.keys():
- if self.tag in local_attr[self.inst_id].keys():
- # Update the test attribute value
- setattr(self, '_test_download_ci',
- local_attr[self.inst_id][self.tag])
- warnings.warn(" ".join(["`_test_download_travis` has been",
- "deprecated and will be replaced",
- "by `_test_download_ci` in",
- "3.2.0+"]),
- DeprecationWarning, stacklevel=2)
-
if len(missing) > 0:
pysat.logger.debug(' '.join(['These Instrument test attributes',
'kept their default values:',
diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py
index aeb1b8cdb..2ef0adda8 100644
--- a/pysat/tests/test_instrument.py
+++ b/pysat/tests/test_instrument.py
@@ -649,29 +649,6 @@ def test_download_freq_kwarg(self):
self.eval_warnings()
return
- def test_download_travis_attr(self):
- """Test deprecation of instrument attribute `_test_download_travis`."""
-
- inst_module = pysat.instruments.pysat_testing
- # Add deprecated attribute.
- inst_module._test_download_travis = {'': {'': False}}
-
- self.warn_msgs = np.array([" ".join(["`_test_download_travis` has been",
- "deprecated and will be replaced",
- "by `_test_download_ci` in",
- "3.2.0+"])])
-
- # Catch the warnings.
- with warnings.catch_warnings(record=True) as self.war:
- tinst = pysat.Instrument(inst_module=inst_module, use_header=True)
-
- # Ensure attributes set properly.
- assert tinst._test_download_ci is False
-
- # Evaluate the warning output
- self.eval_warnings()
- return
-
def test_filter_netcdf4_metadata(self):
"""Test deprecation warning generated by `_filter_netcdf4_metadata`."""
From 54b886577859b2bba9d2f94010dca57083a75d11 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 14:55:13 -0400
Subject: [PATCH 163/365] MAINT: remove #788
---
pysat/_instrument.py | 18 +++---------------
pysat/tests/test_instrument.py | 17 -----------------
2 files changed, 3 insertions(+), 32 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 1e90ab233..ae720567c 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -3597,10 +3597,6 @@ def download(self, start=None, stop=None, date_array=None,
**kwargs):
"""Download data for given Instrument object from start to stop.
- .. deprecated:: 3.2.0
- `freq`, which sets the step size for downloads, will be removed in
- the 3.2.0+ release.
-
Parameters
----------
start : pandas.datetime or NoneType
@@ -3635,17 +3631,9 @@ def download(self, start=None, stop=None, date_array=None,
pandas.DatetimeIndex
"""
- # Test for deprecated kwargs
- if 'freq' in kwargs.keys():
- warnings.warn("".join(["`pysat.Instrument.download` kwarg `freq` ",
- "has been deprecated and will be removed ",
- "in pysat 3.2.0+. Use `date_array` for ",
- "non-daily frequencies instead."]),
- DeprecationWarning, stacklevel=2)
- freq = kwargs['freq']
- del kwargs['freq']
- else:
- freq = 'D'
+
+ # Set frequency to daily.
+ freq = 'D'
# Make sure directories are there, otherwise create them
try:
diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py
index 2ef0adda8..95fbf75b2 100644
--- a/pysat/tests/test_instrument.py
+++ b/pysat/tests/test_instrument.py
@@ -632,23 +632,6 @@ def test_generic_meta_translator(self):
self.eval_warnings()
return
- def test_download_freq_kwarg(self):
- """Test deprecation of download kwarg `freq`."""
-
- # Catch the warnings
- with warnings.catch_warnings(record=True) as self.war:
- tinst = pysat.Instrument(use_header=True, **self.in_kwargs)
- tinst.download(start=self.ref_time, freq='D')
-
- self.warn_msgs = np.array(["".join(["`pysat.Instrument.download` kwarg",
- " `freq` has been deprecated and ",
- "will be removed in pysat ",
- "3.2.0+"])])
-
- # Evaluate the warning output
- self.eval_warnings()
- return
-
def test_filter_netcdf4_metadata(self):
"""Test deprecation warning generated by `_filter_netcdf4_metadata`."""
From a633a6a9b5978f74cea3d2d8b34fd203be200a9f Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 14:57:37 -0400
Subject: [PATCH 164/365] DOC: update changelog
---
CHANGELOG.md | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 93f3160f1..15743291d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,11 +26,15 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Implement pyproject to manage metadata
* Remove Sphinx cap
* Add pandas cap
- * Remove deprecated `pysat_testing2d` instrument
- * Remove deprecated meta children info
- * Remove deprecated `pysat_testing_xarray` instrument
- * Remove deprecated `pysat_testing2d_xarray` instrument
- * Remove deprecated `instrument_test_class`
+ * Removed deprecated `pysat_testing2d` instrument
+ * Removed deprecated meta children info
+ * Removed deprecated `pysat_testing_xarray` instrument
+ * Removed deprecated `pysat_testing2d_xarray` instrument
+ * Removed deprecated `instrument_test_class`
+ * Removed deprecated `malformed_index` kwarg in test instrumennts
+ * Removed deprecated `convert_timestamp_to_datetime` function
+ * Removed deprecated `_test_download_travis` flag
+ * Removed deprecated `freq` kwarg from `download`
[3.1.0] - 2023-05-31
--------------------
From fc27c60caf46f84098c7699901c0c84358839d64 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 6 Oct 2023 17:49:35 -0400
Subject: [PATCH 165/365] STY: fix whitespace
---
pysat/_meta.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 7034d44c4..e51b06af4 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -395,9 +395,9 @@ def __setitem__(self, data_vars, input_dat):
to_be_set, self.labels.label_type[iattr]):
# If this is a disagreement between byte data
# and an expected str, resolve it here
- if(isinstance(to_be_set, bytes)
- and str in pysat.utils.listify(
- self.labels.label_type[iattr])):
+ if all([isinstance(to_be_set, bytes),
+ str in pysat.utils.listify(
+ self.labels.label_type[iattr])]):
to_be_set = core_utils.stringify(to_be_set)
else:
# This type is incorrect, try casting it
From 1608f4a9103a556b366d3b6f4960680490fce12e Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 10 Oct 2023 10:59:04 -0400
Subject: [PATCH 166/365] BUG: fix logic
---
pysat/_instrument.py | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 461a4f3f5..306f19c33 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -409,11 +409,17 @@ def __init__(self, platform=None, name=None, tag='', inst_id='',
# Check to make sure value is reasonable
if self.file_format is not None:
# Check if it is an iterable string
- if any([(not isinstance(self.file_format, str)),
- (self.file_format.find("{") < 0),
- (self.file_format.find("}") < 0)]):
+ if isinstance(self.file_format, str):
+ if any([self.file_format.find("{") < 0,
+ self.file_format.find("}") < 0]):
+ raise ValueError(''.join(['file format set to default, ',
+ 'supplied string must be ',
+ 'iterable [{',
+ self.file_format, '}]']))
+ else:
raise ValueError(''.join(['file format set to default, ',
- 'supplied string must be iterable ',
+ 'supplied format must be iterable ',
+ 'string',
'[{:}]'.format(self.file_format)]))
# Set up empty data and metadata.
From 6fe60e46c1b7da5e7ab2897534f57d2299087dc6 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
Date: Tue, 10 Oct 2023 11:15:52 -0400
Subject: [PATCH 167/365] Update pysat/tests/test_utils_io.py
Co-authored-by: Angeline Burrell
---
pysat/tests/test_utils_io.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py
index f41cf2e79..14b53dfdb 100644
--- a/pysat/tests/test_utils_io.py
+++ b/pysat/tests/test_utils_io.py
@@ -1694,8 +1694,7 @@ def setup_method(self):
self.test_inst = pysat.Instrument('pysat', 'ndtesting',
num_samples=5, use_header=True)
- self.test_date = pysat.instruments.pysat_ndtesting._test_dates
- self.test_date = self.test_date['']['']
+ self.test_date = pysat.instruments.pysat_ndtesting._test_dates['']['']
self.test_inst.load(date=self.test_date)
self.meta_dict = self.test_inst.meta.to_dict()
self.out = None
From 9268edec8293581837c134c63ef7614dfddadd4c Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 10 Oct 2023 11:16:23 -0400
Subject: [PATCH 168/365] STY: update root_date
---
pysat/instruments/pysat_ndtesting.py | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index b12a09f9e..ab354f974 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -35,7 +35,7 @@
def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
- sim_multi_file_left=False, non_monotonic_index=False,
+ sim_multi_file_left=False, root_date=None, non_monotonic_index=False,
non_unique_index=False, malformed_index=False, start_time=None,
num_samples=864, sample_rate='100S', test_load_kwarg=None,
max_latitude=90.0, num_extra_time_coords=0):
@@ -57,6 +57,9 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
sim_multi_file_left : bool
Adjusts date range to be 12 hours in the past or twelve hours before
`root_date`. (default=False)
+ root_date : NoneType
+ Optional central date, uses _test_dates if not specified.
+ (default=None)
non_monotonic_index : bool
If True, time index will be non-monotonic (default=False)
non_unique_index : bool
@@ -103,12 +106,15 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
uts, index, dates = mm_test.generate_times(fnames, num_samples,
freq=sample_rate,
start_time=start_time)
+
+ # Specify the date tag locally and determine the desired date range
+ pds_offset = dt.timedelta(hours=12)
if sim_multi_file_right:
- root_date = dt.datetime(2009, 1, 1, 12)
+ root_date = root_date or _test_dates[''][''] + pds_offset
elif sim_multi_file_left:
- root_date = dt.datetime(2008, 12, 31, 12)
+ root_date = root_date or _test_dates[''][''] - pds_offset
else:
- root_date = dt.datetime(2009, 1, 1)
+ root_date = root_date or _test_dates['']['']
# TODO(#1094): Remove in pysat 3.2.0
if malformed_index:
From 7ea9a3d0bb891280caa87ed956c0ab26b5f54eb1 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 10 Oct 2023 14:06:51 -0400
Subject: [PATCH 169/365] STY: improve error message
---
pysat/_instrument.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 306f19c33..1edc9a299 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -414,7 +414,8 @@ def __init__(self, platform=None, name=None, tag='', inst_id='',
self.file_format.find("}") < 0]):
raise ValueError(''.join(['file format set to default, ',
'supplied string must be ',
- 'iterable [{',
+ 'iterable string with key ',
+ 'formatting [{',
self.file_format, '}]']))
else:
raise ValueError(''.join(['file format set to default, ',
From 2ca3e2fd42cf72b92496ee51f7be57ce29f7bfb1 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 10 Oct 2023 16:54:09 -0400
Subject: [PATCH 170/365] BUG: fixed bad merge
---
pysat/instruments/pysat_ndtesting.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 7c4de51d9..63c4dfd07 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -39,10 +39,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
non_unique_index=False, start_time=None, num_samples=864,
sample_rate='100S', test_load_kwarg=None, max_latitude=90.0,
num_extra_time_coords=0):
- sim_multi_file_left=False, root_date=None, non_monotonic_index=False,
- non_unique_index=False, malformed_index=False, start_time=None,
- num_samples=864, sample_rate='100S', test_load_kwarg=None,
- max_latitude=90.0, num_extra_time_coords=0):
"""Load the test files.
Parameters
From 7fb5ced53dde270ec3648532b5e964b5107f428f Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 10 Oct 2023 16:57:05 -0400
Subject: [PATCH 171/365] STY: whitespace
---
pysat/instruments/pysat_testing.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/instruments/pysat_testing.py b/pysat/instruments/pysat_testing.py
index 90592c754..6bbc08b02 100644
--- a/pysat/instruments/pysat_testing.py
+++ b/pysat/instruments/pysat_testing.py
@@ -40,7 +40,7 @@
def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
sim_multi_file_left=False, root_date=None, non_monotonic_index=False,
- non_unique_index=False,start_time=None, num_samples=86400,
+ non_unique_index=False, start_time=None, num_samples=86400,
test_load_kwarg=None, max_latitude=90.):
"""Load the test files.
From e4079f5ae7b83d85c9021a4c46775b9fa0c44e7f Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 11 Oct 2023 15:40:08 -0400
Subject: [PATCH 172/365] STY: clarify error message
---
pysat/_instrument.py | 8 +++-----
pysat/tests/test_files.py | 2 +-
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 1edc9a299..b9153ce49 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -412,15 +412,13 @@ def __init__(self, platform=None, name=None, tag='', inst_id='',
if isinstance(self.file_format, str):
if any([self.file_format.find("{") < 0,
self.file_format.find("}") < 0]):
- raise ValueError(''.join(['file format set to default, ',
- 'supplied string must be ',
+ raise ValueError(''.join(['Supplied format string must be ',
'iterable string with key ',
'formatting [{',
self.file_format, '}]']))
else:
- raise ValueError(''.join(['file format set to default, ',
- 'supplied format must be iterable ',
- 'string',
+ raise ValueError(''.join(['Supplied format string must be ',
+ 'iterable string',
'[{:}]'.format(self.file_format)]))
# Set up empty data and metadata.
diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py
index c50c19dd1..cdc5039a0 100644
--- a/pysat/tests/test_files.py
+++ b/pysat/tests/test_files.py
@@ -1087,7 +1087,7 @@ def test_files_non_standard_file_format_template_fail(self, file_format):
'temporary_file_list': self.temporary_file_list}
testing.eval_bad_input(pysat.Instrument, ValueError,
- 'file format set to default',
+ 'Supplied format string',
input_kwargs=in_kwargs)
return
From 75be1817970f31ebf83d76183f25227c2c17504d Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 12 Oct 2023 11:46:21 -0400
Subject: [PATCH 173/365] MAINT: dropped HO support
Dropped higher order support in `Meta.drop`. Also adjusted method to reduce number of drop calls and allow partially good data to be dropped without raising and error.
---
pysat/_meta.py | 52 ++++++++++++++++++++++++++++++++++----------------
1 file changed, 36 insertions(+), 16 deletions(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 79b6f9ca2..668530550 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -807,31 +807,51 @@ def drop(self, names):
Raises
------
KeyError
- If any of the keys provided in `names` is not found in the
- lower dimensional data, higher dimensional data, labels, or
- header data
+ If all of the keys provided in `names` is not found in the
+ standard metadata, labels, or header metadata. If a subset is
+ missing, a logger warning is issued instead.
"""
# Ensure the input is list-like
names = listify(names)
- for name in names:
- if name in self._ho_data:
- # Drop the higher dimension data
- self._ho_data.pop(name)
+ # Divide the names by category
+ data_names = []
+ label_names = []
+ header_names = []
+ bad_names = []
+ for name in names:
if name in self.keys():
- # Drop the lower dimension data
- self.data = self._data.drop(name, axis=0)
+ data_names.append(name)
elif name in self.data.columns:
- # This is a metadata label
- self.data = self._data.drop(name, axis=1)
-
- # Also drop this from Labels
- self.labels.drop(name)
+ label_names.append(name)
elif name in self.header.global_attrs:
- self.header.drop(name)
+ header_names.append(name)
+ else:
+ bad_names.append(name)
+
+ # Drop the data
+ if len(data_names) > 0:
+ # Drop the lower dimension data
+ self.data = self._data.drop(data_names, axis=0)
+
+ if len(label_names) > 0:
+ # This is a metadata label
+ self.data = self._data.drop(label_names, axis=1)
+
+ # Also drop this from Labels
+ self.labels.drop(label_names)
+
+ if len(header_names) > 0:
+ # There is header metadata to drop
+ self.header.drop(header_names)
+
+ if len(bad_names) > 0:
+ estr = "{:} not found in Meta".format(repr(bad_names))
+ if len(data_names) + len(label_names) + len(header_names) == 0:
+ raise KeyError(estr)
else:
- raise KeyError("{:} not found in Meta".format(repr(name)))
+ pysat.logger.warning(estr)
return
def keep(self, keep_names):
From aa34e9634a10e4dbfae655549ad28de0716897d5 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 12 Oct 2023 11:46:50 -0400
Subject: [PATCH 174/365] TST: added drop error tests
Added error and logging tests for the Meta.drop method.
---
pysat/tests/test_meta.py | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index 3b1088fa1..38be65eec 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -126,6 +126,13 @@ def test_pop_w_bad_key(self):
input_args=['not_a_key'])
return
+ def test_drop_w_bad_name(self):
+ """Test that a bad name will raise a KeyError for `meta.drop`."""
+
+ testing.eval_bad_input(self.meta.drop, KeyError, 'not found in Meta',
+ input_args=['not_a_name'])
+ return
+
def test_getitem_w_bad_key(self):
"""Test that a bad key will raise a KeyError in meta access."""
@@ -280,6 +287,24 @@ def test_set_meta_with_wrong_type_drop(self, bad_val):
# -------------------------
# Test the logging messages
+ def test_drop_with_some_bad_names(self, caplog):
+ """Test a logger warning is raised if not all names can be dropped."""
+
+ with caplog.at_level(logging.WARN, logger='pysat'):
+ self.meta.drop(['uts', 'units', 'fake_var'])
+
+ # Test the warning
+ captured = caplog.text
+ estr = "missing expected message in: {:}".format(captured)
+ assert captured.find('not found in Meta') >= 0, estr
+
+ # Check that correct meta data and labels were dropped
+ assert 'uts' not in self.meta.keys(), 'Did not drop metadata'
+ assert not hasattr(self.meta.labels, 'units'), 'Did not drop MetaLabel'
+ assert 'units' not in self.meta.data.columns, 'Did not drop meta label'
+
+ return
+
@pytest.mark.parametrize('bad_val', [[1, 2], 1, 2.0, True, None])
def test_set_meta_with_wrong_type_cast(self, bad_val, caplog):
"""Test that setting meta as recastable type raises appropriate warning.
From f1db3af3ee93b9eeef1f59e0f515af690f8a26d7 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 12 Oct 2023 15:50:05 -0400
Subject: [PATCH 175/365] ENH: added `__delitem__`
Re-added `__delitem__`, which now just calls `drop`.
---
pysat/_meta.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 668530550..51751f982 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -571,6 +571,24 @@ def __contains__(self, data_var):
return does_contain
+ def __delitem__(self, key):
+ """Delete a key by calling `drop` method.
+
+ Parameters
+ ----------
+ key : str or list-like
+ A meta data variable, label, or MetaHeader attribute; which are
+ considered in that order.
+
+ Raises
+ ------
+ KeyError
+ If all key values are unavailable
+
+ """
+ self.drop(key)
+ return
+
def __eq__(self, other_meta):
"""Check equality between Meta instances.
From 3a008eb08b7403bea19284d4257d8ded6e6fa573 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 12 Oct 2023 15:50:37 -0400
Subject: [PATCH 176/365] TST: removed skip
Removed the skip statement as tests are passing and added a parameter to test drop and delete together.
---
pysat/tests/test_meta.py | 21 ++++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index 38be65eec..e5cb68c64 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -1014,23 +1014,28 @@ def test_meta_merge(self):
meta_dict[label].__repr__())
return
- @pytest.mark.skip('See if memory is still an issue')
@pytest.mark.parametrize("names", ['uts', ['uts', 'mlt'], 'units',
['units', 'uts']])
- def test_meta_drop(self, names):
+ @pytest.mark.parametrize("is_drop", [True, False])
+ def test_meta_drop(self, names, is_drop):
"""Test successful deletion of meta data for different types of data.
Parameters
----------
names : int
Number of variables to drop in a single go.
+ is_drop : bool
+ Use `drop` if True, use `del` if False.
"""
# Set meta data
self.set_meta(inst_kwargs={'platform': 'pysat', 'name': 'testing'})
# Drop the values
- self.meta.drop(names)
+ if is_drop:
+ self.meta.drop(names)
+ else:
+ del self.meta[names]
# Test the successful deletion
for name in pysat.utils.listify(names):
@@ -1041,13 +1046,16 @@ def test_meta_drop(self, names):
return
@pytest.mark.parametrize("num_drop", [0, 1, 3])
- def test_meta_num_drop(self, num_drop):
+ @pytest.mark.parametrize("is_drop", [True, False])
+ def test_meta_num_drop(self, num_drop, is_drop):
"""Test successful deletion of meta data for specific values.
Parameters
----------
num_drop : int
Number of variables to drop in a single go.
+ is_drop : bool
+ Use `drop` if True, use `del` if False.
"""
@@ -1060,7 +1068,10 @@ def test_meta_num_drop(self, num_drop):
[val for val in self.meta.keys()])
# Drop the values
- self.meta.drop(self.dval)
+ if is_drop:
+ self.meta.drop(self.dval)
+ else:
+ del self.meta[self.dval]
# Test the successful deletion
meta_vals = [val for val in self.meta.keys()]
From ecd2933d69192ac5c207c107905f0509fc6f3706 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 12 Oct 2023 15:50:51 -0400
Subject: [PATCH 177/365] DOC: updated changelog
Improved the changelog description.
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ada664d21..dc43ecbcb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
reduce code duplication.
* Added `__delitem__` to Meta and `drop` to MetaHeader and MetaLabels classes.
* Modified Meta to allow MetaHeader attribute access directly from Meta.
+ * Expanded `Meta.drop` to traverse attached MetaLabel and MetaHeader data.
* Added options to customize `pysat_ndtesting` instrument with sample rate,
shift in time.
* Added orbit number to `pysat_ndtesting`.
From ce0f75206e61899dff6944b813156d5561f156a9 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 17 Oct 2023 10:21:33 -0400
Subject: [PATCH 178/365] MAINT: removed `use_header`
Removed `use_header` as a standard kwarg for `load`. Adjusted deprecation warning to notify user of the end of support for the optional kwarg.
---
pysat/_instrument.py | 33 +++++++++++++++++++--------------
1 file changed, 19 insertions(+), 14 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index d66ae92b8..4ffa28a1d 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -331,7 +331,7 @@ def __init__(self, platform=None, name=None, tag='', inst_id='',
default_kwargs = _get_supported_keywords(func)
# Expand the dict to include method keywords for load.
- # TODO(#1020): Remove this if statement for the 3.2.0+ release
+ # TODO(#1020): Remove this if statement when `use_header` is removed
if fkey == 'load':
meth = getattr(self, fkey)
default_kwargs.update(_get_supported_keywords(meth))
@@ -2920,7 +2920,7 @@ def generic_meta_translator(self, input_meta):
def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
end_date=None, fname=None, stop_fname=None, verifyPad=False,
- use_header=False, **kwargs):
+ **kwargs):
"""Load the instrument data and metadata.
Parameters
@@ -2957,9 +2957,6 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
verifyPad : bool
If True, padding data not removed for debugging. Padding
parameters are provided at Instrument instantiation. (default=False)
- use_header : bool
- If True, moves custom Meta attributes to MetaHeader instead of
- Instrument (default=False)
**kwargs : dict
Dictionary of keywords that may be options for specific instruments.
@@ -3017,6 +3014,22 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
inst.load(fname=inst.files[0], stop_fname=inst.files[1])
"""
+ # If the `use_header` kwarg is included, set it here. Otherwise set
+ # it to True.
+ # TODO(#1020): removed this logic after kwarg not supported.
+ if 'use_header' in kwargs.keys():
+ use_header = kwargs['use_header']
+ warnings.warn(''.join(['Meta now contains a class for global ',
+ 'metadata (MetaHeader). Allowing attachment',
+ ' of global attributes to Instrument ',
+ 'through `use_header=False` will be ',
+ 'Deprecated in pysat 3.3.0+. Remove ',
+ '`use_header` kwarg (now same as ',
+ '`use_header=True`) to stop this warning.']),
+ DeprecationWarning, stacklevel=2)
+ else:
+ use_header = True
+
# Add the load kwargs from initialization those provided on input
for lkey in self.kwargs['load'].keys():
# Only use the initialized kwargs if a request hasn't been
@@ -3368,19 +3381,11 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
# Transfer any extra attributes in meta to the Instrument object.
# Metadata types need to be initialized before preprocess is run.
- # TODO(#1020): Change the way this kwarg is handled
+ # TODO(#1020): Remove warning and logic when kwarg is removed
if use_header or ('use_header' in self.kwargs['load']
and self.kwargs['load']['use_header']):
self.meta.transfer_attributes_to_header()
else:
- warnings.warn(''.join(['Meta now contains a class for global ',
- 'metadata (MetaHeader). Default attachment ',
- 'of global attributes to Instrument will ',
- 'be Deprecated in pysat 3.2.0+. Set ',
- '`use_header=True` in this load call or ',
- 'on Instrument instantiation to remove this',
- ' warning.']), DeprecationWarning,
- stacklevel=2)
self.meta.transfer_attributes_to_instrument(self)
# Transfer loaded data types to meta.
From 588a735033570fc203fec7d1ec4de4f95d686161 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 17 Oct 2023 10:22:18 -0400
Subject: [PATCH 179/365] DOC: updated basic tutorial
Updated basic tutorial by changing the description of the old `use_header` kwarg.
---
docs/tutorial/tutorial_basics.rst | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/docs/tutorial/tutorial_basics.rst b/docs/tutorial/tutorial_basics.rst
index 63fcb1bdb..59ab6d4a0 100644
--- a/docs/tutorial/tutorial_basics.rst
+++ b/docs/tutorial/tutorial_basics.rst
@@ -518,15 +518,16 @@ the instrument PI, etc. Previous versions of pysat stored this data as custom
attributes attached to the :py:class:`pysat.Instrument`, instead of keeping all
metadata in the :py:class:`pysat.Meta` object.
-To avoid breaking existing workflows, global metadata is only loaded into
-:py:class:`pysat.MetaHeader` through completely internal pysat processes or
-after setting the :py:data:`use_header` keyword argument.
+Global metadata is loaded into the :py:class:`pysat.MetaHeader` by default, but
+to avoid breaking existing workflows, loading this metadata directly into the
+:py:class:`~pysat._instrument.Instrument` by setting the :py:data:`use_header`
+keyword argument.
.. code:: python
- # This will show: Metadata for 0 global attributes
- dmsp.load(date=start, use_header=True)
- print(dmsp.meta.header)
+ # This will raise a warning that future releases will require use of
+ # the MetaHeader class
+ dmsp.load(date=start, use_header=False)
You can manually add global metadata the same way you would assign an attribute.
From 28cfb88c3204b33611204719dec707e83719abc8 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 17 Oct 2023 10:23:06 -0400
Subject: [PATCH 180/365] TST: removed `use_header` kwarg
Removed the `use_header` kwarg from the test suite and updated the deprecation test.
---
pysat/constellations/testing.py | 6 +-
pysat/constellations/testing_partial.py | 4 +-
pysat/tests/classes/cls_instrument_access.py | 105 ++++++++--------
pysat/tests/classes/cls_instrument_library.py | 15 ++-
.../tests/classes/cls_instrument_property.py | 22 ++--
pysat/tests/test_constellation.py | 44 ++++---
pysat/tests/test_files.py | 5 +-
pysat/tests/test_instrument.py | 85 ++++++-------
pysat/tests/test_instrument_custom.py | 17 +--
pysat/tests/test_instrument_index.py | 10 +-
pysat/tests/test_instrument_padding.py | 115 ++++++------------
pysat/tests/test_instruments.py | 12 +-
pysat/tests/test_meta.py | 14 +--
pysat/tests/test_methods_general.py | 8 +-
pysat/tests/test_methods_testing.py | 2 +-
pysat/tests/test_orbits.py | 64 +++-------
pysat/tests/test_utils.py | 4 +-
pysat/tests/test_utils_coords.py | 37 +++---
pysat/tests/test_utils_files.py | 10 +-
pysat/tests/test_utils_io.py | 74 +++++------
20 files changed, 258 insertions(+), 395 deletions(-)
diff --git a/pysat/constellations/testing.py b/pysat/constellations/testing.py
index 1b7917e80..1d03e6833 100644
--- a/pysat/constellations/testing.py
+++ b/pysat/constellations/testing.py
@@ -13,8 +13,8 @@
import pysat
instruments = [pysat.Instrument('pysat', 'testing', clean_level='clean',
- num_samples=10, use_header=True),
+ num_samples=10),
pysat.Instrument('pysat', 'ndtesting', clean_level='clean',
- num_samples=16, use_header=True),
+ num_samples=16),
pysat.Instrument('pysat', 'testmodel', clean_level='clean',
- num_samples=18, use_header=True)]
+ num_samples=18)]
diff --git a/pysat/constellations/testing_partial.py b/pysat/constellations/testing_partial.py
index 800e9850c..d8b8bcb1b 100644
--- a/pysat/constellations/testing_partial.py
+++ b/pysat/constellations/testing_partial.py
@@ -9,6 +9,6 @@
import pysat
instruments = [pysat.Instrument('pysat', 'testing', clean_level='clean',
- num_samples=10, use_header=True),
+ num_samples=10),
pysat.Instrument('pysat', 'testing', tag='no_download',
- clean_level='clean', use_header=True)]
+ clean_level='clean')]
diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py
index 5e89a7d88..e32d244f1 100644
--- a/pysat/tests/classes/cls_instrument_access.py
+++ b/pysat/tests/classes/cls_instrument_access.py
@@ -47,8 +47,7 @@ def test_instrument_complete_by_init(self):
"""Test Instrument object fully complete by self._init_rtn()."""
# Create a base instrument to compare against
- inst_copy = pysat.Instrument(inst_module=self.testInst.inst_module,
- use_header=True)
+ inst_copy = pysat.Instrument(inst_module=self.testInst.inst_module)
# Get instrument module and init funtcion
inst_mod = self.testInst.inst_module
@@ -71,7 +70,7 @@ def temp_init(inst, test_init_kwarg=None):
# Instantiate instrument with test module which invokes needed test
# code in the background
- pysat.Instrument(inst_module=inst_mod, use_header=True)
+ pysat.Instrument(inst_module=inst_mod)
# Restore nominal init function
inst_mod.init = inst_mod_init
@@ -125,8 +124,7 @@ def test_basic_instrument_load(self, kwargs):
"""
# Load data by year and day of year
- self.testInst.load(self.ref_time.year, self.ref_doy, **kwargs,
- use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy, **kwargs)
# Test that the loaded date range is correct
self.eval_successful_load()
@@ -152,7 +150,7 @@ def test_basic_instrument_load_no_data(self, caplog, pad):
# Test doesn't check against loading by filename since that produces
# an error if there is no file. Loading by yr, doy no different
# than date in this case.
- self.testInst.load(date=no_data_d, pad=pad, use_header=True)
+ self.testInst.load(date=no_data_d, pad=pad)
# Confirm by checking against caplog that metadata was
# not assigned.
@@ -182,7 +180,7 @@ def test_basic_instrument_load_two_days(self):
end_date = self.ref_time + dt.timedelta(days=2)
end_doy = int(end_date.strftime("%j"))
self.testInst.load(self.ref_time.year, self.ref_doy, end_date.year,
- end_doy, use_header=True)
+ end_doy)
# Test that the loaded date range is correct
self.eval_successful_load(end_date=end_date)
@@ -195,8 +193,7 @@ def test_basic_instrument_bad_keyword_at_load(self):
testing.eval_bad_input(self.testInst.load, TypeError,
"load() got an unexpected keyword",
input_kwargs={'date': self.ref_time,
- 'unsupported_keyword': True,
- 'use_header': True})
+ 'unsupported_keyword': True})
return
def test_basic_instrument_load_yr_no_doy(self):
@@ -205,7 +202,7 @@ def test_basic_instrument_load_yr_no_doy(self):
# Check that the correct error is raised
estr = 'Unknown or incomplete input combination.'
testing.eval_bad_input(self.testInst.load, TypeError, estr,
- [self.ref_time.year], {'use_header': True})
+ [self.ref_time.year])
return
@pytest.mark.parametrize('doy', [0, 367, 1000, -1, -10000])
@@ -221,7 +218,7 @@ def test_basic_instrument_load_yr_bad_doy(self, doy):
estr = 'Day of year (doy) is only valid between and '
testing.eval_bad_input(self.testInst.load, ValueError, estr,
- [self.ref_time.year, doy], {'use_header': True})
+ [self.ref_time.year, doy])
return
@pytest.mark.parametrize('end_doy', [0, 367, 1000, -1, -10000])
@@ -239,7 +236,7 @@ def test_basic_instrument_load_yr_bad_end_doy(self, end_doy):
testing.eval_bad_input(self.testInst.load, ValueError, estr,
[self.ref_time.year, 1],
{'end_yr': self.ref_time.year,
- 'end_doy': end_doy, 'use_header': True})
+ 'end_doy': end_doy})
return
def test_basic_instrument_load_yr_no_end_doy(self):
@@ -248,7 +245,7 @@ def test_basic_instrument_load_yr_no_end_doy(self):
estr = 'Both end_yr and end_doy must be set'
testing.eval_bad_input(self.testInst.load, ValueError, estr,
[self.ref_time.year, self.ref_doy,
- self.ref_time.year], {'use_header': True})
+ self.ref_time.year])
return
@pytest.mark.parametrize("kwargs", [{'yr': 2009, 'doy': 1,
@@ -277,7 +274,6 @@ def test_basic_instrument_load_mixed_inputs(self, kwargs):
"""
- kwargs['use_header'] = True
estr = 'An inconsistent set of inputs have been'
testing.eval_bad_input(self.testInst.load, ValueError, estr,
input_kwargs=kwargs)
@@ -286,7 +282,7 @@ def test_basic_instrument_load_mixed_inputs(self, kwargs):
def test_basic_instrument_load_no_input(self):
"""Test that `.load()` loads all data."""
- self.testInst.load(use_header=True)
+ self.testInst.load()
assert (self.testInst.index[0] == self.testInst.files.start_date)
assert (self.testInst.index[-1] >= self.testInst.files.stop_date)
assert (self.testInst.index[-1] <= self.testInst.files.stop_date
@@ -316,7 +312,6 @@ def test_instrument_load_errors_with_multifile(self, load_in, verr):
else:
load_kwargs = dict()
- load_kwargs['use_header'] = True
testing.eval_bad_input(self.testInst.load, ValueError, verr,
input_kwargs=load_kwargs)
return
@@ -324,7 +319,7 @@ def test_instrument_load_errors_with_multifile(self, load_in, verr):
def test_basic_instrument_load_by_date(self):
"""Test loading by date."""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
self.eval_successful_load()
return
@@ -332,8 +327,7 @@ def test_basic_instrument_load_by_dates(self):
"""Test date range loading, `date` and `end_date`."""
end_date = self.ref_time + dt.timedelta(days=2)
- self.testInst.load(date=self.ref_time, end_date=end_date,
- use_header=True)
+ self.testInst.load(date=self.ref_time, end_date=end_date)
self.eval_successful_load(end_date=end_date)
return
@@ -341,15 +335,14 @@ def test_basic_instrument_load_by_date_with_extra_time(self):
"""Ensure `.load(date=date)` only uses date portion of datetime."""
# Put in a date that has more than year, month, day
- self.testInst.load(date=(self.ref_time + dt.timedelta(minutes=71)),
- use_header=True)
+ self.testInst.load(date=(self.ref_time + dt.timedelta(minutes=71)))
self.eval_successful_load()
return
def test_basic_instrument_load_data(self):
"""Test that correct day loads (checking down to the sec)."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.eval_successful_load()
return
@@ -361,7 +354,7 @@ def test_basic_instrument_load_leap_year(self):
self.ref_time = dt.datetime(2008, 12, 31)
self.ref_doy = 366
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.eval_successful_load()
return
@@ -398,7 +391,7 @@ def test_file_load_bad_start_file(self, operator):
"""
- self.testInst.load(fname=self.testInst.files[1], use_header=True)
+ self.testInst.load(fname=self.testInst.files[1])
# Set new bounds that do not include this date.
self.testInst.bounds = (self.testInst.files[0], self.testInst.files[2],
@@ -418,7 +411,7 @@ def test_file_load_bad_start_date(self, operator):
"""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
# Set new bounds that do not include this date.
self.testInst.bounds = (self.ref_time + dt.timedelta(days=1),
@@ -435,7 +428,7 @@ def test_basic_fname_instrument_load(self):
# If mangle_file_date is true, index will not match exactly.
# Find the closest point instead.
ind = np.argmin(abs(self.testInst.files.files.index - self.ref_time))
- self.testInst.load(fname=self.testInst.files[ind], use_header=True)
+ self.testInst.load(fname=self.testInst.files[ind])
self.eval_successful_load()
return
@@ -455,7 +448,7 @@ def test_fname_load_default(self, operator, direction):
# If mangle_file_date is true, index will not match exactly.
# Find the closest point.
ind = np.argmin(abs(self.testInst.files.files.index - self.ref_time))
- self.testInst.load(fname=self.testInst.files[ind], use_header=True)
+ self.testInst.load(fname=self.testInst.files[ind])
getattr(self.testInst, operator)()
# Modify ref time since iterator changes load date.
@@ -468,8 +461,7 @@ def test_fname_load_default(self, operator, direction):
def test_filename_load(self):
"""Test if file is loadable by filename with no path."""
- self.testInst.load(fname=self.ref_time.strftime('%Y-%m-%d.nofile'),
- use_header=True)
+ self.testInst.load(fname=self.ref_time.strftime('%Y-%m-%d.nofile'))
self.eval_successful_load()
return
@@ -481,7 +473,7 @@ def test_filenames_load(self):
stop_fname = self.ref_time + foff
stop_fname = stop_fname.strftime('%Y-%m-%d.nofile')
self.testInst.load(fname=self.ref_time.strftime('%Y-%m-%d.nofile'),
- stop_fname=stop_fname, use_header=True)
+ stop_fname=stop_fname)
assert self.testInst.index[0] == self.ref_time
assert self.testInst.index[-1] >= self.ref_time + foff
assert self.testInst.index[-1] <= self.ref_time + (2 * foff)
@@ -499,8 +491,7 @@ def test_filenames_load_out_of_order(self):
testing.eval_bad_input(self.testInst.load, ValueError, estr,
input_kwargs={'fname': stop_fname,
- 'stop_fname': check_fname,
- 'use_header': True})
+ 'stop_fname': check_fname})
return
def test_eq_no_data(self):
@@ -513,7 +504,7 @@ def test_eq_no_data(self):
def test_eq_both_with_data(self):
"""Test equality when the same object with loaded data."""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
inst_copy = self.testInst.copy()
assert inst_copy == self.testInst
return
@@ -521,7 +512,7 @@ def test_eq_both_with_data(self):
def test_eq_one_with_data(self):
"""Test equality when the same objects but only one with loaded data."""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
inst_copy = self.testInst.copy()
inst_copy.data = self.testInst._null_data
assert inst_copy != self.testInst
@@ -530,7 +521,7 @@ def test_eq_one_with_data(self):
def test_eq_different_data_type(self):
"""Test equality different data type."""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
inst_copy = self.testInst.copy()
# Can only change data types if Instrument empty
@@ -596,12 +587,12 @@ def test_concat_data(self, prepend, sort_dim_toggle, include):
ref_time2 = self.ref_time + pds.tseries.frequencies.to_offset(
self.testInst.files.files.index.freqstr)
doy2 = int(ref_time2.strftime('%j'))
- self.testInst.load(ref_time2.year, doy2, use_header=True)
+ self.testInst.load(ref_time2.year, doy2)
data2 = self.testInst.data
len2 = len(self.testInst.index)
# Load a different data set into the instrument
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
len1 = len(self.testInst.index)
# Set the keyword arguments
@@ -655,7 +646,7 @@ def test_empty_flag_data_empty(self):
def test_empty_flag_data_not_empty(self):
"""Test the status of the empty flag for loaded data."""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
assert not self.testInst.empty
return
@@ -666,7 +657,7 @@ def test_index_attribute(self):
assert isinstance(self.testInst.index, pds.Index)
# Test an index is present with data loaded in an Instrument
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
assert isinstance(self.testInst.index, pds.Index)
return
@@ -674,7 +665,7 @@ def test_index_return(self):
"""Test that the index is returned in the proper format."""
# Load data
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
# Ensure we get the index back
if self.testInst.pandas_format:
@@ -697,7 +688,7 @@ def test_basic_data_access_by_name(self, labels):
"""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
assert np.all((self.testInst[labels]
== self.testInst.data[labels]).values)
return
@@ -716,7 +707,7 @@ def test_data_access_by_indices_and_name(self, index):
"""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
assert np.all(self.testInst[index, 'mlt']
== self.testInst.data['mlt'][index])
return
@@ -724,7 +715,7 @@ def test_data_access_by_indices_and_name(self, index):
def test_data_access_by_row_slicing(self):
"""Check that each variable is downsampled."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
result = self.testInst[0:10]
for variable, array in result.items():
assert len(array) == len(self.testInst.data[variable].values[0:10])
@@ -737,7 +728,7 @@ def test_data_access_by_row_slicing_and_name_slicing(self):
if not self.testInst.pandas_format:
pytest.skip("name slicing not implemented for xarray")
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
result = self.testInst[0:10, 'uts':'mlt']
for variable, array in result.items():
assert len(array) == len(self.testInst.data[variable].values[0:10])
@@ -747,7 +738,7 @@ def test_data_access_by_row_slicing_and_name_slicing(self):
def test_data_access_by_datetime_and_name(self):
"""Check that datetime can be used to access data."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.out = dt.datetime(2009, 1, 1, 0, 0, 0)
assert np.all(self.testInst[self.out, 'uts']
== self.testInst.data['uts'].values[0])
@@ -756,7 +747,7 @@ def test_data_access_by_datetime_and_name(self):
def test_data_access_by_datetime_slicing_and_name(self):
"""Check that a slice of datetimes can be used to access data."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
time_step = (self.testInst.index[1]
- self.testInst.index[0]).value / 1.E9
offset = dt.timedelta(seconds=(10 * time_step))
@@ -769,7 +760,7 @@ def test_data_access_by_datetime_slicing_and_name(self):
def test_setting_data_by_name(self):
"""Test setting data by name."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst['doubleMLT'] = 2. * self.testInst['mlt']
assert np.all(self.testInst['doubleMLT'] == 2. * self.testInst['mlt'])
return
@@ -777,7 +768,7 @@ def test_setting_data_by_name(self):
def test_setting_series_data_by_name(self):
"""Test setting series data by name."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst['doubleMLT'] = 2. * pds.Series(
self.testInst['mlt'].values, index=self.testInst.index)
assert np.all(self.testInst['doubleMLT'] == 2. * self.testInst['mlt'])
@@ -789,7 +780,7 @@ def test_setting_series_data_by_name(self):
def test_setting_pandas_dataframe_by_names(self):
"""Test setting pandas dataframe by name."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst[['doubleMLT', 'tripleMLT']] = pds.DataFrame(
{'doubleMLT': 2. * self.testInst['mlt'].values,
'tripleMLT': 3. * self.testInst['mlt'].values},
@@ -801,7 +792,7 @@ def test_setting_pandas_dataframe_by_names(self):
def test_setting_data_by_name_single_element(self):
"""Test setting data by name for a single element."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst['doubleMLT'] = 2.
assert np.all(self.testInst['doubleMLT'] == 2.)
assert len(self.testInst['doubleMLT']) == len(self.testInst.index)
@@ -813,7 +804,7 @@ def test_setting_data_by_name_single_element(self):
def test_setting_data_by_name_with_meta(self):
"""Test setting data by name with meta."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst['doubleMLT'] = {'data': 2. * self.testInst['mlt'],
'units': 'hours',
'long_name': 'double trouble'}
@@ -825,7 +816,7 @@ def test_setting_data_by_name_with_meta(self):
def test_setting_partial_data(self):
"""Test setting partial data by index."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.out = self.testInst
if self.testInst.pandas_format:
self.testInst[0:3] = 0
@@ -860,7 +851,7 @@ def test_setting_partial_data_by_inputs(self, changed, fixed):
"""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst['doubleMLT'] = 2. * self.testInst['mlt']
self.testInst[changed, 'doubleMLT'] = 0
assert (self.testInst[fixed, 'doubleMLT']
@@ -871,7 +862,7 @@ def test_setting_partial_data_by_inputs(self, changed, fixed):
def test_modifying_data_inplace(self):
"""Test modification of data inplace."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst['doubleMLT'] = 2. * self.testInst['mlt']
self.testInst['doubleMLT'] += 100
assert (self.testInst['doubleMLT']
@@ -890,7 +881,7 @@ def test_getting_all_data_by_index(self, index):
"""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
inst_subset = self.testInst[index]
if self.testInst.pandas_format:
assert len(inst_subset) == len(index)
@@ -913,7 +904,7 @@ def test_unknown_variable_error_renaming(self, values):
"""
# Check for error for unknown variable name
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
# Capture the ValueError and message
testing.eval_bad_input(self.testInst.rename, ValueError,
@@ -944,7 +935,7 @@ def test_basic_variable_renaming(self, lowercase, mapper):
values = {var: mapper(var) for var in self.testInst.variables}
# Test single variable
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst.rename(mapper, lowercase_data_labels=lowercase)
for key in values:
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index da0dc1195..030500179 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -68,7 +68,7 @@ def initialize_test_inst_and_date(inst_dict):
tag=inst_dict['tag'],
inst_id=inst_dict['inst_id'],
temporary_file_list=True, update_files=True,
- use_header=True, **kwargs)
+ **kwargs)
test_dates = inst_dict['inst_module']._test_dates
date = test_dates[inst_dict['inst_id']][inst_dict['tag']]
return test_inst, date
@@ -94,7 +94,7 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False,
"""
try:
- test_inst.load(date=date, use_header=True)
+ test_inst.load(date=date)
except Exception as err:
# Catch all potential input errors, and only ensure that the one caused
# by the strict time flag is prevented from occurring on future load
@@ -111,7 +111,7 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False,
# Evaluate the warning
with warnings.catch_warnings(record=True) as war:
- test_inst.load(date=date, use_header=True)
+ test_inst.load(date=date)
assert len(war) >= 1
categories = [war[j].category for j in range(len(war))]
@@ -279,7 +279,7 @@ def test_modules_standard(self, inst_name):
for inst_id in module.inst_ids.keys():
for tag in module.inst_ids[inst_id]:
inst = pysat.Instrument(inst_module=module, tag=tag,
- inst_id=inst_id, use_header=True)
+ inst_id=inst_id)
# Test to see that the class parameters were passed in
testing.assert_isinstance(inst, pysat.Instrument)
@@ -462,7 +462,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
with caplog.at_level(
getattr(logging, clean_method_level),
logger='pysat'):
- test_inst.load(date=date, use_header=True)
+ test_inst.load(date=date)
# Test the returned message
out_msg = caplog.text
@@ -472,7 +472,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
elif clean_method == 'warning':
# A warning message is expected
with warnings.catch_warnings(record=True) as war:
- test_inst.load(date=date, use_header=True)
+ test_inst.load(date=date)
# Test the warning output
testing.eval_warnings(war, [clean_method_msg],
@@ -482,8 +482,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
# and the error message
testing.eval_bad_input(
test_inst.load, clean_method_level,
- clean_method_msg,
- input_kwargs={'date': date, 'use_header': True})
+ clean_method_msg, input_kwargs={'date': date})
else:
raise AttributeError(
'unknown type of warning: {:}'.format(
diff --git a/pysat/tests/classes/cls_instrument_property.py b/pysat/tests/classes/cls_instrument_property.py
index dcb0b262c..5908fb720 100644
--- a/pysat/tests/classes/cls_instrument_property.py
+++ b/pysat/tests/classes/cls_instrument_property.py
@@ -378,7 +378,7 @@ def test_inst_attributes_not_overwritten(self):
greeting = '... listen!'
self.testInst.hei = greeting
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
assert self.testInst.hei == greeting
return
@@ -416,15 +416,11 @@ def test_str_w_orbit(self):
"""Test string output with Orbit data."""
reload(pysat.instruments.pysat_testing)
- orbit_info = {'index': 'mlt',
- 'kind': 'local time',
+ orbit_info = {'index': 'mlt', 'kind': 'local time',
'period': np.timedelta64(97, 'm')}
testInst = pysat.Instrument(platform='pysat', name='testing',
- num_samples=10,
- clean_level='clean',
- update_files=True,
- orbit_info=orbit_info,
- use_header=True)
+ num_samples=10, clean_level='clean',
+ update_files=True, orbit_info=orbit_info)
self.out = testInst.__str__()
@@ -434,7 +430,7 @@ def test_str_w_orbit(self):
assert self.out.find('Loaded Orbit Number: 0') > 0
# Activate orbits, check that message has changed
- testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ testInst.load(self.ref_time.year, self.ref_doy)
testInst.orbits.next()
self.out = testInst.__str__()
assert self.out.find('Loaded Orbit Number: 1') > 0
@@ -462,7 +458,7 @@ def passfunc(self):
def test_str_w_load_lots_data(self):
"""Test string output with loaded data with many variables."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.out = self.testInst.__str__()
assert self.out.find('Number of variables:') > 0
assert self.out.find('...') > 0
@@ -472,7 +468,7 @@ def test_str_w_load_less_data(self):
"""Test string output with loaded data with few (4) variables."""
# Load the test data
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
# Ensure the desired data variable is present and delete all others
# 4-6 variables are needed to test all lines; choose the lesser limit
@@ -548,7 +544,7 @@ def test_instrument_function_keywords(self, caplog, func, kwarg, val):
with caplog.at_level(logging.INFO, logger='pysat'):
# Trigger load functions
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
# Refresh files to trigger other functions
self.testInst.files.refresh()
@@ -607,7 +603,7 @@ def test_instrument_function_keyword_liveness(self, caplog, func, kwarg):
with caplog.at_level(logging.INFO, logger='pysat'):
# Trigger load functions
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
# Refresh files to trigger other functions
self.testInst.files.refresh()
diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py
index 5986b71e8..608251db2 100644
--- a/pysat/tests/test_constellation.py
+++ b/pysat/tests/test_constellation.py
@@ -43,8 +43,7 @@ def test_construct_constellation(self, ikeys, ivals, ilen):
# Initialize the Constellation using the desired kwargs
const = pysat.Constellation(
- **{ikey: ivals[i] for i, ikey in enumerate(ikeys)},
- use_header=True)
+ **{ikey: ivals[i] for i, ikey in enumerate(ikeys)})
# Test that the appropriate number of Instruments were loaded. Each
# fake Instrument has 5 tags and 1 inst_id.
@@ -72,7 +71,7 @@ def test_some_bad_construct_constellation(self, caplog):
# Load the Constellation and capture log output
with caplog.at_level(logging.WARNING, logger='pysat'):
const = pysat.Constellation(platforms=['Executor', 'platname1'],
- tags=[''], use_header=True)
+ tags=[''])
# Test the partial Constellation initialization
assert len(const.instruments) == 2
@@ -164,7 +163,7 @@ def test_getitem(self):
"""Test Constellation iteration through instruments attribute."""
self.in_kwargs['const_module'] = None
- self.const = pysat.Constellation(**self.in_kwargs, use_header=True)
+ self.const = pysat.Constellation(**self.in_kwargs)
tst_get_inst = self.const[:]
pysat.utils.testing.assert_lists_equal(self.instruments, tst_get_inst)
return
@@ -173,7 +172,7 @@ def test_repr_w_inst(self):
"""Test Constellation string output with instruments loaded."""
self.in_kwargs['const_module'] = None
- self.const = pysat.Constellation(**self.in_kwargs, use_header=True)
+ self.const = pysat.Constellation(**self.in_kwargs)
out_str = self.const.__repr__()
assert out_str.find("Constellation(instruments") >= 0
@@ -183,7 +182,7 @@ def test_str_w_inst(self):
"""Test Constellation string output with instruments loaded."""
self.in_kwargs['const_module'] = None
- self.const = pysat.Constellation(**self.in_kwargs, use_header=True)
+ self.const = pysat.Constellation(**self.in_kwargs)
out_str = self.const.__str__()
assert out_str.find("pysat Constellation ") >= 0
@@ -216,7 +215,7 @@ def test_str_with_data(self, common_index, cstr):
self.in_kwargs["common_index"] = common_index
self.const = pysat.Constellation(**self.in_kwargs)
- self.const.load(date=self.ref_time, use_header=True)
+ self.const.load(date=self.ref_time)
out_str = self.const.__str__()
assert out_str.find("pysat Constellation ") >= 0
@@ -239,7 +238,7 @@ def double_mlt(inst):
# Add the custom function
self.const.custom_attach(double_mlt, at_pos='end')
- self.const.load(date=self.ref_time, use_header=True)
+ self.const.load(date=self.ref_time)
# Test the added value
for inst in self.const:
@@ -255,7 +254,7 @@ def setup_method(self):
"""Set up the unit test environment for each method."""
self.inst = list(constellations.testing.instruments)
- self.const = pysat.Constellation(instruments=self.inst, use_header=True)
+ self.const = pysat.Constellation(instruments=self.inst)
self.ref_time = pysat.instruments.pysat_testing._test_dates['']['']
self.attrs = ["platforms", "names", "tags", "inst_ids", "instruments",
"bounds", "empty", "empty_partial", "index_res",
@@ -342,7 +341,7 @@ def test_empty_flag_data_empty_partial_load(self):
"""Test the status of the empty flag for partially loaded data."""
self.const = pysat.Constellation(
- const_module=constellations.testing_partial, use_header=True)
+ const_module=constellations.testing_partial)
self.const.load(date=self.ref_time)
assert self.const.empty_partial
assert not self.const.empty
@@ -352,7 +351,7 @@ def test_empty_flag_data_not_empty_partial_load(self):
"""Test the alt status of the empty flag for partially loaded data."""
self.const = pysat.Constellation(
- const_module=constellations.testing_partial, use_header=True)
+ const_module=constellations.testing_partial)
self.const.load(date=self.ref_time)
assert not self.const._empty(all_inst=False)
return
@@ -361,7 +360,7 @@ def test_empty_flag_data_not_empty(self):
"""Test the status of the empty flag for loaded data."""
# Load data and test the status flag
- self.const.load(date=self.ref_time, use_header=True)
+ self.const.load(date=self.ref_time)
assert not self.const.empty
return
@@ -372,7 +371,7 @@ def test_full_data_index(self, ikwarg):
# Test the attribute with loaded data
self.const = pysat.Constellation(instruments=self.inst, **ikwarg)
- self.const.load(date=self.ref_time, use_header=True)
+ self.const.load(date=self.ref_time)
assert isinstance(self.const.index, pds.Index)
assert self.const.index[0] == self.ref_time
@@ -394,7 +393,7 @@ def test_full_data_date(self):
"""Test the date property when no data is loaded."""
# Test the attribute with loaded data
- self.const.load(date=self.ref_time, use_header=True)
+ self.const.load(date=self.ref_time)
assert self.const.date == self.ref_time
return
@@ -403,7 +402,7 @@ def test_full_variables(self):
"""Test the variables property when no data is loaded."""
# Test the attribute with loaded data
- self.const.load(date=self.ref_time, use_header=True)
+ self.const.load(date=self.ref_time)
assert len(self.const.variables) > 0
assert 'uts_pysat_testing' in self.const.variables
@@ -501,10 +500,9 @@ def test_to_inst_pandas_w_pad(self):
"""
# Redefine the Instrument and constellation
self.inst = pysat.Instrument(
- inst_module=pysat.instruments.pysat_testing, use_header=True,
- pad=pds.DateOffset(hours=1), num_samples=10)
- self.const = pysat.Constellation(instruments=[self.inst],
- use_header=True)
+ inst_module=pysat.instruments.pysat_testing, num_samples=10,
+ pad=pds.DateOffset(hours=1))
+ self.const = pysat.Constellation(instruments=[self.inst])
# Load the data
self.inst.load(date=self.ref_time)
@@ -539,11 +537,11 @@ def test_to_inst_mult_pad_clean(self):
pad = pds.DateOffset(hours=1)
self.inst = [
pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
- use_header=True, pad=pad, num_samples=10),
+ pad=pad, num_samples=10),
pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
- use_header=True, pad=2 * pad,
- clean_level=clean_level, num_samples=10)]
- self.const = pysat.Constellation(instruments=self.inst, use_header=True)
+ pad=2 * pad, clean_level=clean_level,
+ num_samples=10)]
+ self.const = pysat.Constellation(instruments=self.inst)
# Load the Instrument and Constellation data
self.inst[-1].load(date=self.ref_time)
diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py
index cdc5039a0..0933d2ed7 100644
--- a/pysat/tests/test_files.py
+++ b/pysat/tests/test_files.py
@@ -148,8 +148,7 @@ def setup_method(self):
self.testInst = pysat.Instrument(
inst_module=pysat.instruments.pysat_testing, clean_level='clean',
- temporary_file_list=self.temporary_file_list, update_files=True,
- use_header=True)
+ temporary_file_list=self.temporary_file_list, update_files=True)
# Create instrument directories in tempdir
create_dir(self.testInst)
@@ -215,7 +214,7 @@ def test_equality_with_copy(self):
def test_equality_with_copy_with_data(self):
"""Test that copy is the same as original, loaded `inst.data`."""
# Load data
- self.testInst.load(date=self.start, use_header=True)
+ self.testInst.load(date=self.start)
# Make copy
self.out = self.testInst.files.copy()
diff --git a/pysat/tests/test_instrument.py b/pysat/tests/test_instrument.py
index 95fbf75b2..6c132fdca 100644
--- a/pysat/tests/test_instrument.py
+++ b/pysat/tests/test_instrument.py
@@ -49,10 +49,8 @@ def setup_method(self):
reload(pysat.instruments.pysat_testing)
self.testInst = pysat.Instrument(platform='pysat', name='testing',
- num_samples=10,
- clean_level='clean',
+ num_samples=10, clean_level='clean',
update_files=True,
- use_header=True,
**self.testing_kwargs)
self.ref_time = pysat.instruments.pysat_testing._test_dates['']['']
self.ref_doy = int(self.ref_time.strftime('%j'))
@@ -90,10 +88,8 @@ def setup_method(self):
self.ref_time + pds.DateOffset(years=2)
- pds.DateOffset(days=1), freq=self.freq)
self.testInst = pysat.Instrument(platform='pysat', name='testing',
- num_samples=10,
- clean_level='clean',
+ num_samples=10, clean_level='clean',
update_files=True,
- use_header=True,
file_date_range=date_range,
**self.testing_kwargs)
self.ref_doy = int(self.ref_time.strftime('%j'))
@@ -122,10 +118,8 @@ def setup_method(self):
+ pds.DateOffset(years=2, days=-1),
freq=self.freq)
self.testInst = pysat.Instrument(platform='pysat', name='testing',
- num_samples=10,
- clean_level='clean',
+ num_samples=10, clean_level='clean',
update_files=True,
- use_header=True,
file_date_range=date_range,
**self.testing_kwargs)
self.ref_doy = int(self.ref_time.strftime('%j'))
@@ -155,10 +149,8 @@ def setup_method(self):
+ pds.DateOffset(years=5, days=-1),
freq=self.freq)
self.testInst = pysat.Instrument(platform='pysat', name='testing',
- num_samples=10,
- clean_level='clean',
+ num_samples=10, clean_level='clean',
update_files=True,
- use_header=True,
file_date_range=date_range,
**self.testing_kwargs)
self.ref_doy = int(self.ref_time.strftime('%j'))
@@ -180,11 +172,8 @@ def setup_method(self):
reload(pysat.instruments.pysat_testing)
imod = pysat.instruments.pysat_testing
- self.testInst = pysat.Instrument(inst_module=imod,
- num_samples=10,
- clean_level='clean',
- update_files=True,
- use_header=True,
+ self.testInst = pysat.Instrument(inst_module=imod, num_samples=10,
+ clean_level='clean', update_files=True,
**self.testing_kwargs)
self.ref_time = imod._test_dates['']['']
self.ref_doy = int(self.ref_time.strftime('%j'))
@@ -211,12 +200,9 @@ def setup_method(self):
"""Set up the unit test environment for each method."""
reload(pysat.instruments.pysat_ndtesting)
- self.testInst = pysat.Instrument(platform='pysat',
- name='ndtesting',
- num_samples=10,
- clean_level='clean',
+ self.testInst = pysat.Instrument(platform='pysat', name='ndtesting',
+ num_samples=10, clean_level='clean',
update_files=True,
- use_header=True,
**self.testing_kwargs)
self.ref_time = pysat.instruments.pysat_ndtesting._test_dates['']['']
self.ref_doy = int(self.ref_time.strftime('%j'))
@@ -232,7 +218,7 @@ def teardown_method(self):
def test_setting_data_as_tuple(self):
"""Test setting data as a tuple."""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst['doubleMLT'] = ('time', 2. * self.testInst['mlt'].values)
assert np.all(self.testInst['doubleMLT'] == 2. * self.testInst['mlt'])
return
@@ -267,7 +253,7 @@ def test_data_access_by_2d_indices_and_name(self, index):
"""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
assert np.all(self.testInst[index, index, 'profiles']
== self.testInst.data['profiles'][index, index])
return
@@ -275,7 +261,7 @@ def test_data_access_by_2d_indices_and_name(self, index):
def test_data_access_by_2d_tuple_indices_and_name(self):
"""Check that variables and be accessed by multi-dim tuple index."""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
index = ([0, 1, 2, 3], [0, 1, 2, 3])
assert np.all(self.testInst[index, 'profiles']
== self.testInst.data['profiles'][index[0], index[1]])
@@ -284,7 +270,7 @@ def test_data_access_by_2d_tuple_indices_and_name(self):
def test_data_access_bad_dimension_tuple(self):
"""Test raises ValueError for mismatched tuple index and data dims."""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
index = ([0, 1, 2, 3], [0, 1, 2, 3], [0, 1, 2, 3])
with pytest.raises(ValueError) as verr:
@@ -297,7 +283,7 @@ def test_data_access_bad_dimension_tuple(self):
def test_data_access_bad_dimension_for_multidim(self):
"""Test raises ValueError for mismatched index and data dimensions."""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
index = [0, 1, 2, 3]
with pytest.raises(ValueError) as verr:
@@ -324,7 +310,7 @@ def test_setting_partial_data_by_2d_indices_and_name(self, changed, fixed):
"""
- self.testInst.load(self.ref_time.year, self.ref_doy, use_header=True)
+ self.testInst.load(self.ref_time.year, self.ref_doy)
self.testInst['doubleProfile'] = 2. * self.testInst['profiles']
self.testInst[changed, changed, 'doubleProfile'] = 0
assert np.all(np.all(self.testInst[fixed, fixed, 'doubleProfile']
@@ -373,7 +359,7 @@ def test_set_xarray_single_value_warnings(self, val, warn_msg):
warnings.simplefilter("always")
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
with warnings.catch_warnings(record=True) as self.war:
self.testInst["new_val"] = val
@@ -391,7 +377,7 @@ def test_set_xarray_single_value_errors(self):
"""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
self.testInst.data = self.testInst.data.assign_coords(
{'preset_val': np.array([1.0, 2.0])})
@@ -413,7 +399,7 @@ def test_set_xarray_single_value_broadcast(self, new_val):
"""
- self.testInst.load(date=self.ref_time, use_header=True)
+ self.testInst.load(date=self.ref_time)
self.testInst.data = self.testInst.data.assign_coords(
{'preset_val': 1.0})
@@ -433,12 +419,10 @@ def setup_method(self):
reload(pysat.instruments.pysat_testing)
self.testInst = pysat.Instrument(platform='pysat', name='testing',
- num_samples=10,
- clean_level='clean',
+ num_samples=10, clean_level='clean',
update_files=True,
mangle_file_dates=True,
strict_time_flag=True,
- use_header=True,
**self.testing_kwargs)
self.ref_time = pysat.instruments.pysat_testing._test_dates['']['']
self.ref_doy = int(self.ref_time.strftime('%j'))
@@ -517,11 +501,11 @@ def test_eq_different_object(self):
obj1 = pysat.Instrument(platform='pysat', name='testing',
num_samples=10, clean_level='clean',
- update_files=True, use_header=True)
+ update_files=True)
obj2 = pysat.Instrument(platform='pysat', name='ndtesting',
num_samples=10, clean_level='clean',
- update_files=True, use_header=True)
+ update_files=True)
assert not (obj1 == obj2)
return
@@ -567,7 +551,7 @@ def test_instrument_labels(self):
# Catch the warnings
with warnings.catch_warnings(record=True) as self.war:
- tinst = pysat.Instrument(use_header=True, **self.in_kwargs)
+ tinst = pysat.Instrument(**self.in_kwargs)
self.warn_msgs = np.array(["`labels` is deprecated, use `meta_kwargs`"])
@@ -597,7 +581,7 @@ def test_instrument_meta_labels(self, use_kwargs):
# Catch the warnings
with warnings.catch_warnings(record=True) as self.war:
- tinst = pysat.Instrument(use_header=True, **self.in_kwargs)
+ tinst = pysat.Instrument(**self.in_kwargs)
labels = tinst.meta_labels
self.warn_msgs = np.array(["Deprecated attribute, returns `meta_kwarg"])
@@ -622,7 +606,7 @@ def test_generic_meta_translator(self):
# Catch the warnings
with warnings.catch_warnings(record=True) as self.war:
- tinst = pysat.Instrument(use_header=True, **self.in_kwargs)
+ tinst = pysat.Instrument(**self.in_kwargs)
tinst.generic_meta_translator(tinst.meta)
self.warn_msgs = np.array(["".join(["This function has been deprecated",
@@ -637,8 +621,8 @@ def test_filter_netcdf4_metadata(self):
# Catch the warnings
with warnings.catch_warnings(record=True) as self.war:
- tinst = pysat.Instrument(use_header=True, **self.in_kwargs)
- tinst.load(date=self.ref_time, use_header=True)
+ tinst = pysat.Instrument(**self.in_kwargs)
+ tinst.load(date=self.ref_time)
mdata_dict = tinst.meta._data.to_dict()
tinst._filter_netcdf4_metadata(mdata_dict,
coltype='str')
@@ -659,7 +643,7 @@ def test_to_netcdf4(self):
# Catch the warnings
with warnings.catch_warnings(record=True) as self.war:
- tinst = pysat.Instrument(use_header=True, **self.in_kwargs)
+ tinst = pysat.Instrument(**self.in_kwargs)
try:
tinst.to_netcdf4()
except ValueError:
@@ -686,7 +670,7 @@ def test_inst_init_with_none(self, kwargs):
"""
with warnings.catch_warnings(record=True) as self.war:
- pysat.Instrument('pysat', 'testing', use_header=True, **kwargs)
+ pysat.Instrument('pysat', 'testing', **kwargs)
self.warn_msgs = np.array(["".join(["The usage of None in `tag` and ",
"`inst_id` has been deprecated ",
@@ -697,19 +681,20 @@ def test_inst_init_with_none(self, kwargs):
self.eval_warnings()
return
+ # TODO(#1020): Remove test when keyword `use_header` is removed.
def test_load_use_header(self):
"""Test that user is informed of MetaHeader on load."""
# Determine the expected warnings
self.warn_msgs = np.array(["".join(['Meta now contains a class for ',
'global metadata (MetaHeader). ',
- 'Default attachment of global ',
- 'attributes to Instrument will be',
- ' Deprecated in pysat 3.2.0+. Set ',
- '`use_header=True` in this load ',
- 'call or on Instrument ',
- 'instantiation to remove this',
- ' warning.'])])
+ 'Allowing attachment of global ',
+ 'attributes to Instrument through',
+ ' `use_header=False` will be ',
+ 'Deprecated in pysat 3.3.0+. ',
+ 'Remove `use_header` kwarg (now ',
+ 'same as `use_header=True`) to ',
+ 'stop this warning.'])])
# Capture the warnings
with warnings.catch_warnings(record=True) as self.war:
diff --git a/pysat/tests/test_instrument_custom.py b/pysat/tests/test_instrument_custom.py
index b7888c7ac..ab8f10caa 100644
--- a/pysat/tests/test_instrument_custom.py
+++ b/pysat/tests/test_instrument_custom.py
@@ -43,7 +43,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'testing', num_samples=10,
clean_level='clean',
- update_files=False, use_header=True)
+ update_files=False)
self.out = ''
return
@@ -73,9 +73,7 @@ def setup_method(self):
"""Set up the unit test environment for each method."""
self.testInst = pysat.Instrument('pysat', 'testing', num_samples=10,
- clean_level='clean',
- update_files=True,
- use_header=True)
+ clean_level='clean', update_files=True)
self.load_date = pysat.instruments.pysat_testing._test_dates['']['']
self.testInst.load(date=self.load_date)
self.custom_args = [2]
@@ -236,8 +234,7 @@ def setup_method(self):
"""Set up the unit test environment for each method."""
self.testInst = pysat.Instrument('pysat', 'ndtesting',
- num_samples=10, clean_level='clean',
- use_header=True)
+ num_samples=10, clean_level='clean')
self.load_date = pysat.instruments.pysat_ndtesting._test_dates
self.load_date = self.load_date['']['']
self.testInst.load(date=self.load_date)
@@ -259,8 +256,7 @@ def setup_method(self):
self.testConst = pysat.Constellation(instruments=[
pysat.Instrument('pysat', 'testing', num_samples=10,
- clean_level='clean', update_files=True,
- use_header=True)
+ clean_level='clean', update_files=True)
for i in range(5)])
self.load_date = pysat.instruments.pysat_testing._test_dates['']['']
self.testConst.load(date=self.load_date)
@@ -371,8 +367,7 @@ def test_custom_inst_keyword_instantiation(self):
{'function': mult_data, 'args': self.custom_args}]
testConst2 = pysat.Constellation(instruments=[
pysat.Instrument('pysat', 'testing', num_samples=10,
- clean_level='clean', custom=custom,
- use_header=True)
+ clean_level='clean', custom=custom)
for i in range(5)])
# Ensure all instruments within both constellations have the same
@@ -400,7 +395,7 @@ def test_custom_const_keyword_instantiation(self):
'apply_inst': False}]
testConst2 = pysat.Constellation(
instruments=[pysat.Instrument('pysat', 'testing', num_samples=10,
- clean_level='clean', use_header=True)
+ clean_level='clean')
for i in range(5)], custom=custom)
# Ensure both constellations have the same custom_* attributes
diff --git a/pysat/tests/test_instrument_index.py b/pysat/tests/test_instrument_index.py
index 5c33c65b2..a36d8a68b 100644
--- a/pysat/tests/test_instrument_index.py
+++ b/pysat/tests/test_instrument_index.py
@@ -47,13 +47,9 @@ def test_index_error_messages(self, kwargs, msg):
"""
- test_inst = pysat.Instrument(platform='pysat',
- name=self.name,
- num_samples=10,
- clean_level='clean',
- update_files=True,
- strict_time_flag=True,
- use_header=True,
+ test_inst = pysat.Instrument(platform='pysat', name=self.name,
+ num_samples=10, clean_level='clean',
+ update_files=True, strict_time_flag=True,
**kwargs)
year, doy = pysat.utils.time.getyrdoy(self.ref_time)
testing.eval_bad_input(test_inst.load, ValueError, msg,
diff --git a/pysat/tests/test_instrument_padding.py b/pysat/tests/test_instrument_padding.py
index ad30f55ac..a359916b6 100644
--- a/pysat/tests/test_instrument_padding.py
+++ b/pysat/tests/test_instrument_padding.py
@@ -23,15 +23,11 @@ def setup_method(self):
reload(pysat.instruments.pysat_testing)
self.testInst = pysat.Instrument(platform='pysat', name='testing',
clean_level='clean',
- pad={'minutes': 5},
- update_files=True,
- use_header=True)
+ pad={'minutes': 5}, update_files=True)
self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile')
self.rawInst = pysat.Instrument(platform='pysat', name='testing',
- clean_level='clean',
- update_files=True,
- use_header=True)
+ clean_level='clean', update_files=True)
self.rawInst.bounds = self.testInst.bounds
self.delta = dt.timedelta(seconds=0)
return
@@ -143,23 +139,15 @@ def setup_method(self):
"""Set up the unit test environment for each method."""
reload(pysat.instruments.pysat_ndtesting)
- self.testInst = pysat.Instrument(platform='pysat',
- name='ndtesting',
- num_samples=86400,
- sample_rate='1s',
+ self.testInst = pysat.Instrument(platform='pysat', name='ndtesting',
+ num_samples=86400, sample_rate='1s',
clean_level='clean',
- pad={'minutes': 5},
- update_files=True,
- use_header=True)
+ pad={'minutes': 5}, update_files=True)
self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile')
- self.rawInst = pysat.Instrument(platform='pysat',
- name='ndtesting',
- num_samples=86400,
- sample_rate='1s',
- clean_level='clean',
- update_files=True,
- use_header=True)
+ self.rawInst = pysat.Instrument(platform='pysat', name='ndtesting',
+ num_samples=86400, sample_rate='1s',
+ clean_level='clean', update_files=True)
self.rawInst.bounds = self.testInst.bounds
self.delta = dt.timedelta(seconds=0)
return
@@ -182,14 +170,11 @@ def setup_method(self):
clean_level='clean',
update_files=True,
sim_multi_file_right=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.rawInst = pysat.Instrument(platform='pysat', name='testing',
- tag='',
- clean_level='clean',
+ tag='', clean_level='clean',
update_files=True,
- sim_multi_file_right=True,
- use_header=True)
+ sim_multi_file_right=True)
self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile')
self.rawInst.bounds = self.testInst.bounds
self.delta = dt.timedelta(seconds=0)
@@ -216,16 +201,11 @@ def setup_method(self):
clean_level='clean',
update_files=True,
sim_multi_file_right=True,
- pad={'minutes': 5},
- use_header=True)
- self.rawInst = pysat.Instrument(platform='pysat',
- name='ndtesting',
- num_samples=86400,
- sample_rate='1s',
- clean_level='clean',
- update_files=True,
- sim_multi_file_right=True,
- use_header=True)
+ pad={'minutes': 5})
+ self.rawInst = pysat.Instrument(platform='pysat', name='ndtesting',
+ num_samples=86400, sample_rate='1s',
+ clean_level='clean', update_files=True,
+ sim_multi_file_right=True)
self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile')
self.rawInst.bounds = self.testInst.bounds
self.delta = dt.timedelta(seconds=0)
@@ -246,16 +226,12 @@ def setup_method(self):
reload(pysat.instruments.pysat_testing)
self.testInst = pysat.Instrument(platform='pysat', name='testing',
- clean_level='clean',
- update_files=True,
+ clean_level='clean', update_files=True,
sim_multi_file_left=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.rawInst = pysat.Instrument(platform='pysat', name='testing',
- clean_level='clean',
- update_files=True,
- sim_multi_file_left=True,
- use_header=True)
+ clean_level='clean', update_files=True,
+ sim_multi_file_left=True)
self.testInst.bounds = ('2008-01-01.nofile', '2010-12-31.nofile')
self.rawInst.bounds = self.testInst.bounds
self.delta = dt.timedelta(seconds=0)
@@ -278,8 +254,7 @@ def setup_method(self):
self.testInst = pysat.Instrument(platform='pysat', name='testing',
clean_level='clean',
pad={'minutes': 5},
- update_files=True,
- use_header=True)
+ update_files=True)
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
self.delta = dt.timedelta(minutes=5)
@@ -319,8 +294,7 @@ def test_data_padding_offset_instantiation(self, pad):
self.testInst = pysat.Instrument(platform='pysat', name='testing',
clean_level='clean',
pad=pad,
- update_files=True,
- use_header=True)
+ update_files=True)
self.testInst.load(self.ref_time.year, self.ref_doy, verifyPad=True)
self.eval_index_start_end()
return
@@ -354,8 +328,7 @@ def test_padding_exceeds_load_window(self):
self.testInst = pysat.Instrument(platform='pysat', name='testing',
clean_level='clean',
pad={'days': 2},
- update_files=True,
- use_header=True)
+ update_files=True)
testing.eval_bad_input(self.testInst.load, ValueError,
'Data padding window must be shorter than ',
@@ -486,8 +459,7 @@ def setup_method(self):
clean_level='clean',
pad={'minutes': 5},
non_monotonic_index=True,
- update_files=True,
- use_header=True)
+ update_files=True)
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
self.delta = dt.timedelta(minutes=5)
@@ -513,8 +485,7 @@ def setup_method(self):
sample_rate='1s',
clean_level='clean',
pad={'minutes': 5},
- update_files=True,
- use_header=True)
+ update_files=True)
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
self.delta = dt.timedelta(minutes=5)
@@ -541,8 +512,7 @@ def setup_method(self):
clean_level='clean',
pad={'minutes': 5},
non_monotonic_index=True,
- update_files=True,
- use_header=True)
+ update_files=True)
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
self.delta = dt.timedelta(minutes=5)
@@ -566,8 +536,7 @@ def setup_method(self):
clean_level='clean',
update_files=True,
sim_multi_file_right=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.testInst.multi_file_day = True
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
@@ -593,8 +562,7 @@ def setup_method(self):
update_files=True,
sim_multi_file_right=True,
non_monotonic_index=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.testInst.multi_file_day = True
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
@@ -622,8 +590,7 @@ def setup_method(self):
clean_level='clean',
update_files=True,
sim_multi_file_right=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.testInst.multi_file_day = True
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
@@ -652,8 +619,7 @@ def setup_method(self):
update_files=True,
sim_multi_file_right=True,
non_monotonic_index=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.testInst.multi_file_day = True
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
@@ -679,8 +645,7 @@ def setup_method(self):
clean_level='clean',
update_files=True,
sim_multi_file_left=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.testInst.multi_file_day = True
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
@@ -707,8 +672,7 @@ def setup_method(self):
update_files=True,
sim_multi_file_left=True,
non_monotonic_index=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.testInst.multi_file_day = True
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
@@ -736,8 +700,7 @@ def setup_method(self):
clean_level='clean',
update_files=True,
sim_multi_file_left=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.testInst.multi_file_day = True
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
@@ -758,16 +721,12 @@ def setup_method(self):
"""Set up the unit test environment for each method."""
reload(pysat.instruments.pysat_ndtesting)
- self.testInst = pysat.Instrument(platform='pysat',
- name='ndtesting',
- clean_level='clean',
- update_files=True,
- num_samples=86400,
- sample_rate='1s',
+ self.testInst = pysat.Instrument(platform='pysat', name='ndtesting',
+ clean_level='clean', update_files=True,
+ num_samples=86400, sample_rate='1s',
sim_multi_file_left=True,
non_monotonic_index=True,
- pad={'minutes': 5},
- use_header=True)
+ pad={'minutes': 5})
self.testInst.multi_file_day = True
self.ref_time = dt.datetime(2009, 1, 2)
self.ref_doy = 2
diff --git a/pysat/tests/test_instruments.py b/pysat/tests/test_instruments.py
index e118d0a52..4f7f87df8 100644
--- a/pysat/tests/test_instruments.py
+++ b/pysat/tests/test_instruments.py
@@ -64,11 +64,10 @@ def test_inst_start_time(self, inst_dict, kwarg, output):
_, date = cls_inst_lib.initialize_test_inst_and_date(inst_dict)
if kwarg:
self.test_inst = pysat.Instrument(
- inst_module=inst_dict['inst_module'], start_time=kwarg,
- use_header=True)
+ inst_module=inst_dict['inst_module'], start_time=kwarg)
else:
self.test_inst = pysat.Instrument(
- inst_module=inst_dict['inst_module'], use_header=True)
+ inst_module=inst_dict['inst_module'])
self.test_inst.load(date=date)
@@ -92,7 +91,7 @@ def test_inst_num_samples(self, inst_dict):
num = 10
_, date = cls_inst_lib.initialize_test_inst_and_date(inst_dict)
self.test_inst = pysat.Instrument(inst_module=inst_dict['inst_module'],
- num_samples=num, use_header=True)
+ num_samples=num)
self.test_inst.load(date=date)
assert len(self.test_inst['uts']) == num
@@ -115,7 +114,7 @@ def test_inst_file_date_range(self, inst_dict):
_, date = cls_inst_lib.initialize_test_inst_and_date(inst_dict)
self.test_inst = pysat.Instrument(inst_module=inst_dict['inst_module'],
file_date_range=file_date_range,
- update_files=True, use_header=True)
+ update_files=True)
file_list = self.test_inst.files.files
assert all(file_date_range == file_list.index)
@@ -134,8 +133,7 @@ def test_inst_max_latitude(self, inst_dict):
"""
_, date = cls_inst_lib.initialize_test_inst_and_date(inst_dict)
- self.test_inst = pysat.Instrument(inst_module=inst_dict['inst_module'],
- use_header=True)
+ self.test_inst = pysat.Instrument(inst_module=inst_dict['inst_module'])
if self.test_inst.name != 'testmodel':
self.test_inst.load(date=date, max_latitude=10.)
assert np.all(np.abs(self.test_inst['latitude']) <= 10.)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index e5cb68c64..0bee40e27 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -67,7 +67,7 @@ class `mutable` attribute. (default=False)
"""
if inst_kwargs is not None:
# Load the test Instrument
- self.testInst = pysat.Instrument(**inst_kwargs, use_header=True)
+ self.testInst = pysat.Instrument(**inst_kwargs)
stime = self.testInst.inst_module._test_dates['']['']
self.testInst.load(date=stime)
@@ -1476,8 +1476,7 @@ class TestMetaMutable(object):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument(platform='pysat', name='testing',
- use_header=True)
+ self.testInst = pysat.Instrument(platform='pysat', name='testing')
self.testInst.load(date=self.testInst.inst_module._test_dates[''][''])
self.meta = self.testInst.meta
self.meta.mutable = True
@@ -1612,8 +1611,7 @@ class TestToDict(object):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testing', num_samples=5,
- use_header=True)
+ self.testInst = pysat.Instrument('pysat', 'testing', num_samples=5)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.testInst.load(date=self.stime)
@@ -1676,8 +1674,7 @@ class TestToDictXarrayND(TestToDict):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'ndtesting',
- num_samples=5, use_header=True)
+ self.testInst = pysat.Instrument('pysat', 'ndtesting', num_samples=5)
self.stime = pysat.instruments.pysat_ndtesting._test_dates['']['']
self.testInst.load(date=self.stime)
@@ -1693,8 +1690,7 @@ class TestToDictXarrayModel(TestToDict):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.testInst = pysat.Instrument('pysat', 'testmodel',
- num_samples=5, use_header=True)
+ self.testInst = pysat.Instrument('pysat', 'testmodel', num_samples=5)
self.stime = pysat.instruments.pysat_testmodel._test_dates['']['']
self.testInst.load(date=self.stime)
diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py
index a9724553e..38a4bd0b4 100644
--- a/pysat/tests/test_methods_general.py
+++ b/pysat/tests/test_methods_general.py
@@ -114,7 +114,7 @@ def setup_method(self):
# Load a test instrument
self.testInst = pysat.Instrument('pysat', 'testing', num_samples=12,
- clean_level='clean', use_header=True)
+ clean_level='clean')
self.testInst.load(2009, 1)
self.npts = len(self.testInst['uts'])
return
@@ -199,10 +199,8 @@ def setup_method(self):
"""Set up the unit test environment for each method."""
# Load a test instrument
- self.testInst = pysat.Instrument('pysat', 'ndtesting',
- num_samples=12,
- clean_level='clean',
- use_header=True)
+ self.testInst = pysat.Instrument('pysat', 'ndtesting', num_samples=12,
+ clean_level='clean')
self.testInst.load(2009, 1)
self.npts = len(self.testInst['uts'])
return
diff --git a/pysat/tests/test_methods_testing.py b/pysat/tests/test_methods_testing.py
index 0885032a7..5c2818037 100644
--- a/pysat/tests/test_methods_testing.py
+++ b/pysat/tests/test_methods_testing.py
@@ -16,7 +16,7 @@ class TestMethodsTesting(object):
def setup_method(self):
"""Set up the unit test environment for each method."""
- self.test_inst = pysat.Instrument('pysat', 'testing', use_header=True)
+ self.test_inst = pysat.Instrument('pysat', 'testing')
# Get list of filenames.
self.fnames = [self.test_inst.files.files.values[0]]
diff --git a/pysat/tests/test_orbits.py b/pysat/tests/test_orbits.py
index c849d29a9..85525ffcb 100644
--- a/pysat/tests/test_orbits.py
+++ b/pysat/tests/test_orbits.py
@@ -140,7 +140,6 @@ def test_orbit_w_bad_orbit_info(self, info):
"""
self.in_kwargs['orbit_info'] = info
- self.in_kwargs['use_header'] = True
self.testInst = pysat.Instrument(*self.in_args, **self.in_kwargs)
self.testInst.load(date=self.stime)
@@ -168,7 +167,6 @@ def test_orbit_polar_w_missing_orbit_index(self, info):
"""
self.in_kwargs['orbit_info'] = info
- self.in_kwargs['use_header'] = True
self.testInst = pysat.Instrument(*self.in_args, **self.in_kwargs)
# Force index to None beforee loading and iterating
@@ -182,7 +180,6 @@ def test_orbit_repr(self):
"""Test the Orbit representation."""
self.in_kwargs['orbit_info'] = {'index': 'mlt'}
- self.in_kwargs['use_header'] = True
self.testInst = pysat.Instrument(*self.in_args, **self.in_kwargs)
out_str = self.testInst.orbits.__repr__()
@@ -193,7 +190,6 @@ def test_orbit_str(self):
"""Test the Orbit string representation with data."""
self.in_kwargs['orbit_info'] = {'index': 'mlt'}
- self.in_kwargs['use_header'] = True
self.testInst = pysat.Instrument(*self.in_args, **self.in_kwargs)
self.testInst.load(date=self.stime)
out_str = self.testInst.orbits.__str__()
@@ -212,8 +208,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'testing',
clean_level='clean',
orbit_info={'index': 'mlt'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.inc_min = 97
self.etime = None
@@ -391,8 +386,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'testing',
clean_level='clean',
orbit_info={'index': 'mlt'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
return
@@ -592,8 +586,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'mlt'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_ndtesting._test_dates['']['']
return
@@ -620,8 +613,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'testing',
clean_level='clean',
orbit_info={'index': 'mlt'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.testInst.bounds = (self.testInst.files.files.index[0],
self.testInst.files.files.index[11],
'2D', dt.timedelta(days=3))
@@ -683,8 +675,7 @@ def setup_method(self):
clean_level='clean',
orbit_info={'index': 'longitude',
'kind': 'longitude'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
return
@@ -705,8 +696,7 @@ def setup_method(self):
clean_level='clean',
orbit_info={'index': 'longitude',
'kind': 'longitude'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
return
@@ -727,8 +717,7 @@ def setup_method(self):
clean_level='clean',
orbit_info={'index': 'orbit_num',
'kind': 'orbit'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
return
@@ -749,8 +738,7 @@ def setup_method(self):
clean_level='clean',
orbit_info={'index': 'orbit_num',
'kind': 'orbit'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_ndtesting._test_dates['']['']
return
@@ -771,8 +759,7 @@ def setup_method(self):
clean_level='clean',
orbit_info={'index': 'latitude',
'kind': 'polar'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
return
@@ -793,8 +780,7 @@ def setup_method(self):
clean_level='clean',
orbit_info={'index': 'latitude',
'kind': 'polar'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_ndtesting._test_dates['']['']
return
@@ -835,8 +821,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'testing',
clean_level='clean',
orbit_info={'index': 'mlt'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.gaps = self.stime + self.deltime
self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps})
@@ -877,8 +862,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'mlt'},
- update_files=True,
- use_header=True)
+ update_files=True)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.gaps = self.stime + self.deltime
self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps})
@@ -919,8 +903,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'testing',
clean_level='clean',
- orbit_info={'index': 'mlt'},
- use_header=True)
+ orbit_info={'index': 'mlt'})
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.testInst.custom_attach(filter_data, kwargs={'times': self.times})
return
@@ -948,8 +931,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
- orbit_info={'index': 'mlt'},
- use_header=True)
+ orbit_info={'index': 'mlt'})
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.testInst.custom_attach(filter_data, kwargs={'times': self.times})
return
@@ -970,8 +952,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'testing',
clean_level='clean',
orbit_info={'index': 'longitude',
- 'kind': 'longitude'},
- use_header=True)
+ 'kind': 'longitude'})
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.gaps = self.stime + self.deltime
@@ -994,8 +975,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'longitude',
- 'kind': 'longitude'},
- use_header=True)
+ 'kind': 'longitude'})
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.gaps = self.stime + self.deltime
self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps})
@@ -1017,8 +997,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'testing',
clean_level='clean',
orbit_info={'index': 'orbit_num',
- 'kind': 'orbit'},
- use_header=True)
+ 'kind': 'orbit'})
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.gaps = self.stime + self.deltime
self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps})
@@ -1040,8 +1019,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'orbit_num',
- 'kind': 'orbit'},
- use_header=True)
+ 'kind': 'orbit'})
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.gaps = self.stime + self.deltime
self.testInst.custom_attach(filter_data, kwargs={'times': self.gaps})
@@ -1063,8 +1041,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'testing',
clean_level='clean',
orbit_info={'index': 'latitude',
- 'kind': 'polar'},
- use_header=True)
+ 'kind': 'polar'})
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.gaps = self.stime + self.deltime
@@ -1087,8 +1064,7 @@ def setup_method(self):
self.testInst = pysat.Instrument('pysat', 'ndtesting',
clean_level='clean',
orbit_info={'index': 'latitude',
- 'kind': 'polar'},
- use_header=True)
+ 'kind': 'polar'})
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.gaps = self.stime + self.deltime
diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py
index a6869bf4c..b88c961f6 100644
--- a/pysat/tests/test_utils.py
+++ b/pysat/tests/test_utils.py
@@ -50,7 +50,7 @@ def test_update_fill_values_numbers(self, name, variables):
"""
# Initalize the instrument
- inst = pysat.Instrument('pysat', name, use_header=True)
+ inst = pysat.Instrument('pysat', name)
inst.load(date=self.ref_time)
# Ensure there are fill values to check
@@ -82,7 +82,7 @@ def test_update_fill_values_by_type(self, name):
"""
# Initalize the instrument
- inst = pysat.Instrument('pysat', name, use_header=True)
+ inst = pysat.Instrument('pysat', name)
inst.load(date=self.ref_time)
# Ensure there are fill values to check
diff --git a/pysat/tests/test_utils_coords.py b/pysat/tests/test_utils_coords.py
index 8edd4b113..d63077cc2 100644
--- a/pysat/tests/test_utils_coords.py
+++ b/pysat/tests/test_utils_coords.py
@@ -67,8 +67,7 @@ def teardown_method(self):
def test_update_longitude(self, name):
"""Test `update_longitude` successful run."""
- self.py_inst = pysat.Instrument(platform='pysat', name=name,
- use_header=True)
+ self.py_inst = pysat.Instrument(platform='pysat', name=name)
self.py_inst.load(date=self.inst_time)
# Test instruments initially define longitude between 0-360 deg
@@ -85,8 +84,7 @@ def test_update_longitude(self, name):
def test_bad_lon_name_update_longitude(self):
"""Test update_longitude with a bad longitude name."""
- self.py_inst = pysat.Instrument(platform='pysat', name="testing",
- use_header=True)
+ self.py_inst = pysat.Instrument(platform='pysat', name="testing")
self.py_inst.load(date=self.inst_time)
testing.eval_bad_input(coords.update_longitude, ValueError,
@@ -124,7 +122,7 @@ def test_calc_solar_local_time(self, name):
# Instantiate instrument and load data
self.py_inst = pysat.Instrument(platform='pysat', name=name,
- num_samples=1, use_header=True)
+ num_samples=1)
self.py_inst.load(date=self.inst_time)
coords.calc_solar_local_time(self.py_inst, lon_name="longitude",
@@ -148,7 +146,7 @@ def test_calc_solar_local_time_inconsistent_keywords(self, name, caplog):
# Instantiate instrument and load data
self.py_inst = pysat.Instrument(platform='pysat', name=name,
- num_samples=1, use_header=True)
+ num_samples=1)
self.py_inst.load(date=self.inst_time)
# Apply solar local time method and capture logging output
@@ -167,8 +165,7 @@ def test_calc_solar_local_time_w_neg_longitude(self):
"""Test calc_solar_local_time with longitudes from -180 to 180 deg."""
# Instantiate instrument and load data
- self.py_inst = pysat.Instrument(platform='pysat', name="testing",
- use_header=True)
+ self.py_inst = pysat.Instrument(platform='pysat', name="testing")
self.py_inst.load(date=self.inst_time)
coords.calc_solar_local_time(self.py_inst, lon_name="longitude",
@@ -186,8 +183,7 @@ def test_bad_lon_name_calc_solar_local_time(self):
"""Test raises ValueError with a bad longitude name."""
# Instantiate instrument and load data
- self.py_inst = pysat.Instrument(platform='pysat', name="testing",
- use_header=True)
+ self.py_inst = pysat.Instrument(platform='pysat', name="testing")
self.py_inst.load(date=self.inst_time)
# Test that the correct Exception and error message are raised
@@ -203,8 +199,7 @@ def test_lon_broadcasting_calc_solar_local_time(self, name):
"""Test calc_solar_local_time with longitude coordinates."""
# Instantiate instrument and load data
- self.py_inst = pysat.Instrument(platform='pysat', name=name,
- use_header=True)
+ self.py_inst = pysat.Instrument(platform='pysat', name=name)
self.py_inst.load(date=self.inst_time)
coords.calc_solar_local_time(self.py_inst, lon_name="longitude",
slt_name='slt')
@@ -219,8 +214,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod_multiday(self, name):
"""Test non modulated solar local time output for a 2 day range."""
# Instantiate instrument and load data
- self.py_inst = pysat.Instrument(platform='pysat', name=name,
- use_header=True)
+ self.py_inst = pysat.Instrument(platform='pysat', name=name)
self.py_inst.load(date=self.inst_time,
end_date=self.inst_time + dt.timedelta(days=2))
coords.calc_solar_local_time(self.py_inst, lon_name="longitude",
@@ -237,8 +231,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod_ref_date(self, name):
"""Test non modulated SLT output for a 2 day range with a ref date."""
# Instantiate instrument and load data
- self.py_inst = pysat.Instrument(platform='pysat', name=name,
- use_header=True)
+ self.py_inst = pysat.Instrument(platform='pysat', name=name)
self.py_inst.load(date=self.inst_time, end_date=self.inst_time
+ dt.timedelta(days=2))
coords.calc_solar_local_time(self.py_inst, lon_name="longitude",
@@ -257,8 +250,7 @@ def test_lon_broadcasting_calc_solar_local_time_no_mod(self, name):
"""Test SLT calc with longitude coordinates and no modulus."""
# Instantiate instrument and load data
- self.py_inst = pysat.Instrument(platform='pysat', name=name,
- use_header=True)
+ self.py_inst = pysat.Instrument(platform='pysat', name=name)
self.py_inst.load(date=self.inst_time)
coords.calc_solar_local_time(self.py_inst, lon_name="longitude",
slt_name='slt', apply_modulus=False)
@@ -273,8 +265,7 @@ def test_single_lon_calc_solar_local_time(self):
"""Test calc_solar_local_time with a single longitude value."""
# Instantiate instrument and load data
- self.py_inst = pysat.Instrument(platform='pysat', name="ndtesting",
- use_header=True)
+ self.py_inst = pysat.Instrument(platform='pysat', name="ndtesting")
self.py_inst.load(date=self.inst_time)
lon_name = 'lon2'
@@ -362,7 +353,7 @@ class TestExpandXarrayDims(object):
def setup_method(self):
"""Set up the unit test environment."""
self.test_inst = pysat.Instrument(
- inst_module=pysat.instruments.pysat_ndtesting, use_header=True)
+ inst_module=pysat.instruments.pysat_ndtesting)
self.start_time = pysat.instruments.pysat_ndtesting._test_dates['']['']
self.data_list = []
self.out = None
@@ -397,12 +388,12 @@ def set_data_meta(self, dims_equal):
# Load a second data set with half the time samples
self.test_inst = pysat.Instrument(
inst_module=self.test_inst.inst_module,
- num_samples=num_samples, use_header=True)
+ num_samples=num_samples)
else:
# Load a second data set with different dimensions apart from time
self.test_inst = pysat.Instrument(
inst_module=pysat.instruments.pysat_testmodel,
- num_samples=num_samples, use_header=True)
+ num_samples=num_samples)
self.test_inst.load(date=self.start_time + dt.timedelta(days=1))
self.data_list.append(self.test_inst.data)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index 87ba21683..b8dc67667 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -152,15 +152,13 @@ def setup_method(self):
self.insts_kwargs = []
# Data by day, ACE SIS data
- self.insts.append(pysat.Instrument('ace', 'sis', tag='historic',
- use_header=True))
+ self.insts.append(pysat.Instrument('ace', 'sis', tag='historic'))
test_dates = pysatSpaceWeather.instruments.ace_sis._test_dates
self.insts_dates.append([test_dates['']['historic']] * 2)
self.insts_kwargs.append({})
# Data with date mangling, regular F10.7 data, stored monthly
- self.insts.append(pysat.Instrument('sw', 'f107', tag='historic',
- use_header=True))
+ self.insts.append(pysat.Instrument('sw', 'f107', tag='historic'))
test_dates = pysatSpaceWeather.instruments.sw_f107._test_dates
self.insts_dates.append([test_dates['']['historic'],
test_dates['']['historic']
@@ -265,7 +263,7 @@ def test_updating_directories(self, capsys):
# Refresh inst with the old directory template set to get now 'old'
# path information.
inst2 = pysat.Instrument(inst.platform, inst.name, tag=inst.tag,
- inst_id=inst.inst_id, use_header=True)
+ inst_id=inst.inst_id)
# Check that directories with simpler platform org were NOT removed.
assert os.path.isdir(inst2.files.data_path)
@@ -326,7 +324,7 @@ def setup_method(self):
self.testInst = pysat.Instrument(
inst_module=pysat.instruments.pysat_testing, clean_level='clean',
- update_files=True, use_header=True)
+ update_files=True)
# Create instrument directories in tempdir
pysat.utils.files.check_and_make_path(self.testInst.files.data_path)
diff --git a/pysat/tests/test_utils_io.py b/pysat/tests/test_utils_io.py
index 14b9a5c94..210be40ea 100644
--- a/pysat/tests/test_utils_io.py
+++ b/pysat/tests/test_utils_io.py
@@ -61,8 +61,7 @@ def setup_method(self):
pysat.params['data_dirs'] = self.tempdir.name
self.testInst = pysat.Instrument(platform='pysat', name='testing',
- num_samples=100, update_files=True,
- use_header=True)
+ num_samples=100, update_files=True)
self.stime = pysat.instruments.pysat_testing._test_dates['']['']
self.epoch_name = 'time'
@@ -114,7 +113,7 @@ def test_basic_write_and_read_netcdf_mixed_case_data_format(self):
"""Test basic netCDF4 read/write with mixed case data variables."""
# Create a bunch of files by year and doy
outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
# Modify data names in data
if self.testInst.pandas_format:
@@ -157,7 +156,7 @@ def test_basic_write_and_read_netcdf_mixed_case_meta_format(self):
"""Test basic netCDF4 read/write with mixed case metadata variables."""
# Create a bunch of files by year and doy
outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
# Modify data and metadata names in data
self.testInst.meta.rename(str.upper)
@@ -200,7 +199,7 @@ def test_basic_write_and_read_netcdf_export_pysat_info(self, kwargs,
"""
# Create a bunch of files by year and doy
outfile = os.path.join(self.tempdir.name, 'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
io.inst_to_netcdf(self.testInst, fname=outfile, preserve_meta_case=True,
epoch_name=default_epoch_name, **kwargs)
@@ -235,7 +234,7 @@ def test_inst_write_and_read_netcdf(self, add_path):
'pysat_test_ncdf_%Y%j.nc'))
# Load and write the test instrument data
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
self.testInst.to_netcdf4(fname=outfile, epoch_name=default_epoch_name)
# Load the written file directly into an Instrument
@@ -244,7 +243,7 @@ def test_inst_write_and_read_netcdf(self, add_path):
netcdf_inst = pysat.Instrument(
'pysat', 'netcdf', data_dir=file_path, update_files=True,
file_format=file_root, pandas_format=self.testInst.pandas_format,
- use_header=True, epoch_name=default_epoch_name, **tkwargs)
+ epoch_name=default_epoch_name, **tkwargs)
# Confirm data path is correct
assert os.path.normpath(netcdf_inst.files.data_path) \
@@ -253,7 +252,7 @@ def test_inst_write_and_read_netcdf(self, add_path):
# Deleting the test file here via os.remove(...) does work
# Load data
- netcdf_inst.load(date=self.stime, use_header=True)
+ netcdf_inst.load(date=self.stime)
# Test the loaded Instrument data
self.loaded_inst = netcdf_inst.data
@@ -316,7 +315,7 @@ def test_write_netcdf4_duplicate_variable_names(self):
# Create a bunch of files by year and doy
outfile = os.path.join(self.tempdir.name,
'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
self.testInst['MLT'] = 1
# Evaluate the expected error and message
@@ -348,7 +347,7 @@ def test_read_netcdf4_bad_epoch_name(self, write_epoch, err_msg, err_type):
# Load data
outfile = os.path.join(self.tempdir.name,
'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
# Write file
io.inst_to_netcdf(self.testInst, fname=outfile, epoch_name=write_epoch)
@@ -392,7 +391,7 @@ def test_read_netcdf4_epoch_not_xarray_dimension(self, caplog, write_epoch,
# Load data
outfile = os.path.join(self.tempdir.name,
'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
# Write file
io.inst_to_netcdf(self.testInst, outfile, epoch_name=write_epoch)
@@ -430,7 +429,7 @@ def test_write_and_read_netcdf4_w_kwargs(self, wkwargs, lkwargs):
# Create a new file based on loaded test data
outfile = os.path.join(self.tempdir.name,
'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
if 'epoch_name' not in wkwargs.keys():
wkwargs['epoch_name'] = default_epoch_name
io.inst_to_netcdf(self.testInst, fname=outfile, **wkwargs)
@@ -471,7 +470,7 @@ def test_read_netcdf4_w_epoch_kwargs(self, kwargs, target):
# Create a bunch of files by year and doy
outfile = os.path.join(self.tempdir.name,
'pysat_{:}_ncdf.nc'.format(self.testInst.name))
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
io.inst_to_netcdf(self.testInst, fname=outfile,
epoch_name=default_epoch_name)
@@ -523,7 +522,7 @@ def test_read_netcdf4_w_epoch_kwargs(self, kwargs, target):
def test_netcdf_prevent_attribute_override(self):
"""Test that attributes will not be overridden by default."""
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
# Test that `bespoke` attribute is initially missing
assert not hasattr(self.testInst, 'bespoke')
@@ -543,7 +542,7 @@ def test_netcdf_prevent_attribute_override(self):
def test_netcdf_attribute_override(self):
"""Test that attributes in the netCDF file may be overridden."""
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
self.testInst.meta.mutable = True
self.testInst.meta.bespoke = True
@@ -583,7 +582,7 @@ def test_decode_times(self, decode_times):
# Create a file
outfile = os.path.join(self.tempdir.name,
'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
io.inst_to_netcdf(self.testInst, fname=outfile,
epoch_name=default_epoch_name)
@@ -633,7 +632,7 @@ def test_drop_labels(self, drop_labels):
# Create a file with additional metadata
outfile = os.path.join(self.tempdir.name,
'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
# Add additional metadata
self.testInst.meta['mlt'] = {drop_label: 1.}
@@ -687,8 +686,7 @@ def setup_method(self):
self.testInst = pysat.Instrument(platform='pysat',
name='ndtesting',
- update_files=True, num_samples=100,
- use_header=True)
+ update_files=True, num_samples=100)
self.stime = pysat.instruments.pysat_ndtesting._test_dates[
'']['']
self.epoch_name = 'time'
@@ -737,7 +735,7 @@ def test_read_netcdf4_with_time_meta_labels(self, kwargs, target):
# Prepare output test data
outfile = os.path.join(self.tempdir.name,
'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
# Modify the variable attributes directly before writing to file
self.testInst.meta['uts'] = {'units': 'seconds'}
@@ -767,7 +765,7 @@ def test_load_netcdf_pandas_2d_error(self):
# Create a bunch of files by year and doy
outfile = os.path.join(self.tempdir.name,
'pysat_test_ncdf.nc')
- self.testInst.load(date=self.stime, use_header=True)
+ self.testInst.load(date=self.stime)
io.inst_to_netcdf(self.testInst, fname=outfile)
# Evaluate the error raised and the expected message
@@ -801,10 +799,8 @@ def setup_method(self):
# Create an instrument object that has a meta with some
# variables allowed to be nan within metadata when exporting.
- self.testInst = pysat.Instrument('pysat', 'testing', num_samples=5,
- use_header=True)
- self.testInst.load(date=self.testInst.inst_module._test_dates[''][''],
- use_header=True)
+ self.testInst = pysat.Instrument('pysat', 'testing', num_samples=5)
+ self.testInst.load(date=self.testInst.inst_module._test_dates[''][''])
self.pformat = self.testInst.pandas_format
return
@@ -1256,9 +1252,8 @@ def setup_method(self):
# Create an instrument object that has a meta with some
# variables allowed to be nan within metadata when exporting.
self.testInst = pysat.Instrument('pysat', 'ndtesting',
- num_samples=5, use_header=True)
- self.testInst.load(date=self.testInst.inst_module._test_dates[''][''],
- use_header=True)
+ num_samples=5)
+ self.testInst.load(date=self.testInst.inst_module._test_dates[''][''])
self.pformat = self.testInst.pandas_format
return
@@ -1272,10 +1267,8 @@ def setup_method(self):
# Create an instrument object that has a meta with some
# variables allowed to be nan within metadata when exporting.
- self.testInst = pysat.Instrument('pysat', 'testmodel', num_samples=5,
- use_header=True)
- self.testInst.load(date=self.testInst.inst_module._test_dates[''][''],
- use_header=True)
+ self.testInst = pysat.Instrument('pysat', 'testmodel', num_samples=5)
+ self.testInst.load(date=self.testInst.inst_module._test_dates[''][''])
self.pformat = self.testInst.pandas_format
return
@@ -1289,10 +1282,8 @@ def setup_method(self):
# Create an instrument object that has a meta with some
# variables allowed to be nan within metadata when exporting.
- self.testInst = pysat.Instrument('pysat', 'ndtesting',
- num_samples=5, use_header=True)
- self.testInst.load(date=self.testInst.inst_module._test_dates[''][''],
- use_header=True)
+ self.testInst = pysat.Instrument('pysat', 'ndtesting', num_samples=5)
+ self.testInst.load(date=self.testInst.inst_module._test_dates[''][''])
self.epoch_name = 'time'
return
@@ -1411,8 +1402,7 @@ class TestMetaTranslation(object):
def setup_method(self):
"""Create test environment."""
- self.test_inst = pysat.Instrument('pysat', 'testing', num_samples=5,
- use_header=True)
+ self.test_inst = pysat.Instrument('pysat', 'testing', num_samples=5)
self.test_date = pysat.instruments.pysat_testing._test_dates['']['']
self.test_inst.load(date=self.test_date)
self.meta_dict = self.test_inst.meta.to_dict()
@@ -1694,7 +1684,7 @@ def setup_method(self):
"""Create test environment."""
self.test_inst = pysat.Instrument('pysat', 'ndtesting',
- num_samples=5, use_header=True)
+ num_samples=5)
self.test_date = pysat.instruments.pysat_ndtesting._test_dates['']['']
self.test_inst.load(date=self.test_date)
self.meta_dict = self.test_inst.meta.to_dict()
@@ -1716,8 +1706,7 @@ class TestMetaTranslationModel(TestMetaTranslation):
def setup_method(self):
"""Create test environment."""
- self.test_inst = pysat.Instrument('pysat', 'testmodel',
- num_samples=5, use_header=True)
+ self.test_inst = pysat.Instrument('pysat', 'testmodel', num_samples=5)
self.test_date = pysat.instruments.pysat_testmodel._test_dates['']['']
self.test_inst.load(date=self.test_date)
self.meta_dict = self.test_inst.meta.to_dict()
@@ -1782,8 +1771,7 @@ def test_load_netcdf_labels(self, inst_name, load_func):
# Create a test file
testInst = pysat.Instrument(platform='pysat', name=inst_name,
- num_samples=100, update_files=True,
- use_header=True)
+ num_samples=100, update_files=True)
testInst.load(date=testInst.inst_module._test_dates[''][''])
io.inst_to_netcdf(testInst, fname=self.outfile)
From 2f4f07e21a8536faaaf05254796c2c4eca89bc54 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 17 Oct 2023 10:23:24 -0400
Subject: [PATCH 181/365] DOC: updated changelog
Added a description of the changes to the changelog.
---
CHANGELOG.md | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b7afe9f3b..30fe2f368 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,7 +28,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Updated `Constellation.to_inst` method definition of coords, using dims
to combine common dimensions instead.
* Implement pyproject to manage metadata
-* Updated docstring references to `pysat.utils.files` in other modules.
+ * Updated docstring references to `pysat.utils.files` in other modules.
* Remove Sphinx cap
* Add pandas cap
* Update usage of whitespace and if statements (E275)
@@ -42,6 +42,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Removed deprecated `convert_timestamp_to_datetime` function
* Removed deprecated `_test_download_travis` flag
* Removed deprecated `freq` kwarg from `download`
+ * Removed deprecated `use_header` kwarg from `load` and changed default
+ behaviour to `use_header=True`
[3.1.0] - 2023-05-31
--------------------
From 9ca8e683621b4bbfb69663703775ab08a6f1efa8 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 17 Oct 2023 16:47:02 -0400
Subject: [PATCH 182/365] BUG: `use_header` can be used at init
Make sure the `use_header` kwarg is allowed for `load` at initialization.
---
pysat/_instrument.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 4ffa28a1d..111322b25 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -333,8 +333,7 @@ def __init__(self, platform=None, name=None, tag='', inst_id='',
# Expand the dict to include method keywords for load.
# TODO(#1020): Remove this if statement when `use_header` is removed
if fkey == 'load':
- meth = getattr(self, fkey)
- default_kwargs.update(_get_supported_keywords(meth))
+ default_kwargs['use_header'] = True
# Confirm there are no reserved keywords present
for kwarg in kwargs.keys():
From 6c1d784af4ab547304aff2527a53a48310da8c66 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 18 Oct 2023 10:21:46 -0400
Subject: [PATCH 183/365] STY: removed duplication in parsing
Reduced duplication in file parsing code, ensuring both routines have the same type of output in the same situations.
---
pysat/utils/files.py | 189 +++++++++++++++++++++++++++++--------------
1 file changed, 127 insertions(+), 62 deletions(-)
diff --git a/pysat/utils/files.py b/pysat/utils/files.py
index ce0c18727..ed473b5fd 100644
--- a/pysat/utils/files.py
+++ b/pysat/utils/files.py
@@ -126,6 +126,122 @@ def process_parsed_filenames(stored, two_digit_year_break=None):
return pds.Series(None, dtype='object')
+def init_parse_filenames(files, format_str):
+ """Initalize the output for the file parsing functions.
+
+ Parameters
+ ----------
+ files : list
+ List of files, typically provided by
+ `pysat.utils.files.search_local_system_formatted_filename`.
+ format_str : str
+ Provides the naming pattern of the instrument files and the
+ locations of date information so an ordered list may be produced.
+ Supports all provided string formatting codes though only 'year',
+ 'month', 'day', 'hour', 'minute', 'second', 'version', 'revision',
+ and 'cycle' will be used for time and sorting information. For example,
+ `instrument-{year:4d}_{month:02d}-{day:02d}_v{version:02d}.cdf`, or
+ `*-{year:4d}_{month:02d}hithere{day:02d}_v{version:02d}.cdf`
+
+ Returns
+ -------
+ stored : collections.OrderedDict
+ Information parsed from filenames that includes: 'year', 'month', 'day',
+ 'hour', 'minute', 'second', 'version', 'revision', and 'cycle', as
+ well as any other user provided template variables. Also
+ includes `files`, an input list of files, and `format_str`.
+ search_dict : dict
+ An output dict with the following keys:
+ - 'search_string' (format_str with data to be parsed replaced with ?)
+ - 'keys' (keys for data to be parsed)
+ - 'lengths' (string length for data to be parsed)
+ - 'string_blocks' (the filenames are broken into fixed width segments).
+
+ See Also
+ --------
+ pysat.utils.files.parse_fixed_width_filenames
+ pysat.utils.files.parse_delimited_filenames
+ pysat.utils.files.construct_searchstring_from_format
+
+ """
+ # Create storage for data to be parsed from filenames
+ ordered_keys = ['year', 'month', 'day', 'hour', 'minute', 'second',
+ 'version', 'revision', 'cycle']
+ stored = collections.OrderedDict({kk: None for kk in ordered_keys})
+
+ # Only define search dictionary if there are files to search
+ if len(files) == 0:
+ # Include keys that should only be added at the end, if there are no
+ # files to process
+ stored['format_str'] = format_str
+ stored['files'] = []
+ search_dict = dict()
+ else:
+ # Parse format string to get information needed to parse filenames
+ search_dict = construct_searchstring_from_format(format_str,
+ wildcard=False)
+
+ # Add non-standard keys
+ for key in search_dict['keys']:
+ if key not in stored:
+ stored[key] = None
+
+ return stored, search_dict
+
+
+def finish_parse_filenames(stored, files, format_str):
+ """Reshape and finalize the output for the file parsing functions.
+
+ Parameters
+ ----------
+ stored : collections.OrderedDict
+ Information parsed from filenames that includes: 'year', 'month', 'day',
+ 'hour', 'minute', 'second', 'version', 'revision', and 'cycle', as
+ well as any other user provided template variables.
+ files : list
+ List of files, typically provided by
+ `pysat.utils.files.search_local_system_formatted_filename`.
+ format_str : str
+ Provides the naming pattern of the instrument files and the
+ locations of date information so an ordered list may be produced.
+ Supports all provided string formatting codes though only 'year',
+ 'month', 'day', 'hour', 'minute', 'second', 'version', 'revision',
+ and 'cycle' will be used for time and sorting information. For example,
+ `instrument-{year:4d}_{month:02d}-{day:02d}_v{version:02d}.cdf`, or
+ `*-{year:4d}_{month:02d}hithere{day:02d}_v{version:02d}.cdf`
+
+ Returns
+ -------
+ stored : collections.OrderedDict
+ Information parsed from filenames that includes: 'year', 'month', 'day',
+ 'hour', 'minute', 'second', 'version', 'revision', and 'cycle', as
+ well as any other user provided template variables. Also
+ includes `files`, an input list of files, and `format_str`.
+
+ See Also
+ --------
+ pysat.utils.files.parse_fixed_width_filenames
+ pysat.utils.files.parse_delimited_filenames
+
+ """
+
+ # Convert to numpy arrays
+ for key in stored.keys():
+ if stored[key] is not None:
+ try:
+ # Assume key value is numeric integer
+ stored[key] = np.array(stored[key]).astype(np.int64)
+ except ValueError:
+ # Store key value as string
+ stored[key] = np.array(stored[key])
+
+ # Include files and file format in output
+ stored['files'] = files
+ stored['format_str'] = format_str
+
+ return stored
+
+
def parse_fixed_width_filenames(files, format_str):
"""Extract specified info from a list of files with a fixed name width.
@@ -165,24 +281,11 @@ def parse_fixed_width_filenames(files, format_str):
"""
# Create storage for data to be parsed from filenames
- ordered_keys = ['year', 'month', 'day', 'hour', 'minute', 'second',
- 'version', 'revision', 'cycle']
- stored = collections.OrderedDict({kk: list() for kk in ordered_keys})
+ stored, search_dict = init_parse_filenames(files, format_str)
if len(files) == 0:
- stored['files'] = []
- # Include format string as convenience for later functions
- stored['format_str'] = format_str
return stored
- # Parse format string to get information needed to parse filenames
- search_dict = construct_searchstring_from_format(format_str)
-
- # Add non-standard keys
- for key in search_dict['keys']:
- if key not in stored:
- stored[key] = []
-
# Determine the locations the date/version information in a filename is
# stored and use these indices to slice out date from filenames.
idx = 0
@@ -208,25 +311,15 @@ def parse_fixed_width_filenames(files, format_str):
val = temp[key_str_idx[0][j]:]
else:
val = temp[key_str_idx[0][j]:key_str_idx[1][j]]
- stored[key].append(val)
-
- # Convert to numpy arrays
- for key in stored.keys():
- if len(stored[key]) == 0:
- stored[key] = None
- else:
- try:
- # Assume key value is numeric integer
- stored[key] = np.array(stored[key]).astype(np.int64)
- except ValueError:
- # Store key value as string
- stored[key] = np.array(stored[key])
- # Include files in output
- stored['files'] = files
+ # Save the parsed variable for this key and file
+ if stored[key] is None:
+ stored[key] = [val]
+ else:
+ stored[key].append(val)
- # Include format string as convenience for later functions
- stored['format_str'] = format_str
+ # Convert to numpy arrays and add additional information to output
+ stored = finish_parse_filenames(stored, files, format_str)
return stored
@@ -281,26 +374,11 @@ def parse_delimited_filenames(files, format_str, delimiter):
"""
# Create storage for data to be parsed from filenames
- ordered_keys = ['year', 'month', 'day', 'hour', 'minute', 'second',
- 'version', 'revision', 'cycle']
- stored = collections.OrderedDict({kk: None for kk in ordered_keys})
+ stored, search_dict = init_parse_filenames(files, format_str)
- # Exit early if there are no files
if len(files) == 0:
- stored['files'] = []
-
- # Include format string as convenience for later functions
- stored['format_str'] = format_str
return stored
- # Parse format string to get information needed to parse filenames
- search_dict = construct_searchstring_from_format(format_str, wildcard=False)
-
- # Add non-standard keys
- for key in search_dict['keys']:
- if key not in stored:
- stored[key] = None
-
# Going to parse the string on the delimiter. It is possible that other
# regions have the delimiter but aren't going to be parsed out.
# Reconstruct string from `snips` and use `{}` in place of `keys` and
@@ -361,21 +439,8 @@ def parse_delimited_filenames(files, format_str, delimiter):
loop_split_idx = loop_split_idx[j + 1:]
break
- # Convert to numpy arrays
- for key in stored.keys():
- if stored[key] is not None:
- try:
- # Assume key value is numeric integer
- stored[key] = np.array(stored[key]).astype(np.int64)
- except ValueError:
- # Store key value as string
- stored[key] = np.array(stored[key])
-
- # Include files in output
- stored['files'] = files
-
- # Include format string as convenience for later functions
- stored['format_str'] = format_str
+ # Convert to numpy arrays and add additional information to output
+ stored = finish_parse_filenames(stored, files, format_str)
return stored
From 87abad2932870058f387cae8dd54c3585b1fdb76 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 18 Oct 2023 10:22:42 -0400
Subject: [PATCH 184/365] TST: added file parsing unit tests
Added unit tests for `parse_fixed_width_filenames` and the new supporting functions created to reduce code duplication.
---
pysat/tests/test_utils_files.py | 137 ++++++++++++++++++++++++++++++--
1 file changed, 132 insertions(+), 5 deletions(-)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index 87ba21683..f63a2d647 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -21,8 +21,8 @@
from pysat.utils import testing
-class TestParseDelimitedFilenames(object):
- """Unit tests for the `parse_delimited_filename` function."""
+class TestParseFilenames(object):
+ """Unit tests for the file parsing functions."""
def setup_method(self):
"""Set up the unit test environment for each method."""
@@ -46,7 +46,7 @@ def teardown_method(self):
del self.fkwargs, self.file_dict, self.kw_format
del self.temporary_file_list
- def eval_parse_delimited_filename(self):
+ def eval_parsed_filenames(self):
"""Evaluate the output of a `parse_delimited_filename` unit test.
Returns
@@ -106,7 +106,7 @@ def test_parse_delimited_filename(self, sep_char, flead, good_kwargs):
sep_char)
# Test each of the return values
- assert self.eval_parse_delimited_filename()
+ assert self.eval_parsed_filenames()
return
def test_parse_delimited_filename_empty(self):
@@ -121,7 +121,134 @@ def test_parse_delimited_filename_empty(self):
self.file_dict = futils.parse_delimited_filenames([], fname, sep_char)
# Test each of the return values
- assert self.eval_parse_delimited_filename()
+ assert self.eval_parsed_filenames()
+ return
+
+ @pytest.mark.parametrize("sep_char,flead,good_kwargs", [
+ ("_", "*_", ['year', 'month', 'day', 'hour', 'minute', 'version']),
+ ('?', "test", ['year', 'day', 'hour', 'minute', 'second', 'cycle',
+ 'revision']), ('fun', '*', [])])
+ def test_parse_fixed_filename(self, sep_char, flead, good_kwargs):
+ """Check ability to parse list of fixed width files.
+
+ Parameters
+ ----------
+ sep_char : str
+ Separation character to use in joining the filename
+ flead : str
+ File prefix
+ good_kwargs : list
+ List of kwargs to include in the file format
+
+ """
+ # Format the test input
+ fname = '{:s}{:s}.cdf'.format(flead, sep_char.join(
+ [self.kw_format[fkey] for fkey in good_kwargs]))
+
+ # Adjust the test input/comparison data for this run
+ bad_kwargs = [fkey for fkey in self.fkwargs[0]
+ if fkey not in good_kwargs]
+
+ for kwargs in self.fkwargs:
+ for fkey in bad_kwargs:
+ del kwargs[fkey]
+
+ # Create the input file list
+ file_list = [fname.format(**kwargs) for kwargs in self.fkwargs]
+
+ # Get the test results
+ self.file_dict = futils.parse_fixed_width_filenames(file_list, fname)
+
+ # Test each of the return values
+ assert self.eval_parsed_filenames()
+ return
+
+ def test_parse_fixed_width_filename_empty(self):
+ """Check ability to parse list of fixed-width files with no files."""
+ # Format the test input
+ fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}',
+ '{minute:02d}', '{second:02d}', '{cycle:2s}.txt'))
+ self.fkwargs = []
+
+ # Get the test results
+ self.file_dict = futils.parse_fixed_width_filenames([], fname)
+
+ # Test each of the return values
+ assert self.eval_parsed_filenames()
+ return
+
+ def test_init_parse_filename_empty(self):
+ """Check the `init_parse_filenames` output with no files."""
+ # Format the test input
+ fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}',
+ '{minute:02d}', '{second:02d}', '{cycle:2s}.txt'))
+ self.fkwargs = []
+
+ # Get the test results
+ self.file_dict, sdict = futils.init_parse_filenames([], fname)
+
+ # Test each of the return values
+ assert self.eval_parsed_filenames()
+ assert len(sdict.keys()) == 0, "Search dict was defined unnecessarily"
+ return
+
+ def test_init_parse_filename_with_files(self):
+ """Check the `init_parse_filenames` output with files."""
+ # Format the test input
+ fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}',
+ '{minute:02d}', '{second:02d}', '{cycle:2s}.txt'))
+
+ # Create the input file list
+ file_list = [fname.format(**kwargs) for kwargs in self.fkwargs]
+
+ # Get the test results
+ self.file_dict, sdict = futils.init_parse_filenames(file_list, fname)
+
+ # Test the initalized dictionaries
+ testing.assert_lists_equal(['search_string', 'keys', 'lengths',
+ 'string_blocks'], list(sdict.keys()))
+
+ for skey in sdict['keys']:
+ assert skey in self.file_dict.keys(), "Missing key {:}".format(skey)
+
+ for fkey in self.file_dict.keys():
+ assert self.file_dict[fkey] is None, "File dict not initalized"
+
+ assert "files" not in self.file_dict.keys(), "'files' key set early"
+ assert "format_str" not in self.file_dict.keys(), \
+ "'format_str' key set early"
+ return
+
+ def test_finish_parsed_filenames(self):
+ """Test output restucturing for `finish_parsed_filenames`."""
+ # Format the test input
+ fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}',
+ '{minute:02d}', '{second:02d}', '{cycle:2s}.txt'))
+
+ # Create the input file list and dict
+ file_list = [fname.format(**kwargs) for kwargs in self.fkwargs]
+ self.file_dict = {'int': [1], 'none': None, 'float': [1.0],
+ 'str': ['hi']}
+
+ # Get the test results
+ self.file_dict = futils.finish_parse_filenames(self.file_dict,
+ file_list, fname)
+
+ # Test the output
+ for fkey in self.file_dict:
+ if fkey == 'none':
+ assert self.file_dict[fkey] is None
+ elif fkey == 'files':
+ testing.assert_lists_equal(file_list, self.file_dict[fkey])
+ elif fkey == 'format_str':
+ assert fname == self.file_dict[fkey]
+ else:
+ testing.assert_isinstance(self.file_dict[fkey], np.ndarray)
+
+ if fkey == 'str':
+ testing.assert_isinstance(self.file_dict[fkey][0], str)
+ else:
+ testing.assert_isinstance(self.file_dict[fkey][0], np.int64)
return
From 2ad73cba048c797ce1d41166b083fa4f01324ad2 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 18 Oct 2023 10:30:18 -0400
Subject: [PATCH 185/365] STY: renamed support functions
Renamed the support functions to have hidden names, in order to not clutter the file utility namespace.
---
pysat/tests/test_utils_files.py | 14 +-
pysat/utils/files.py | 226 ++++++++++++++++----------------
2 files changed, 122 insertions(+), 118 deletions(-)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index f63a2d647..2c1cfb552 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -178,14 +178,14 @@ def test_parse_fixed_width_filename_empty(self):
return
def test_init_parse_filename_empty(self):
- """Check the `init_parse_filenames` output with no files."""
+ """Check the `_init_parse_filenames` output with no files."""
# Format the test input
fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}',
'{minute:02d}', '{second:02d}', '{cycle:2s}.txt'))
self.fkwargs = []
# Get the test results
- self.file_dict, sdict = futils.init_parse_filenames([], fname)
+ self.file_dict, sdict = futils._init_parse_filenames([], fname)
# Test each of the return values
assert self.eval_parsed_filenames()
@@ -193,7 +193,7 @@ def test_init_parse_filename_empty(self):
return
def test_init_parse_filename_with_files(self):
- """Check the `init_parse_filenames` output with files."""
+ """Check the `_init_parse_filenames` output with files."""
# Format the test input
fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}',
'{minute:02d}', '{second:02d}', '{cycle:2s}.txt'))
@@ -202,7 +202,7 @@ def test_init_parse_filename_with_files(self):
file_list = [fname.format(**kwargs) for kwargs in self.fkwargs]
# Get the test results
- self.file_dict, sdict = futils.init_parse_filenames(file_list, fname)
+ self.file_dict, sdict = futils._init_parse_filenames(file_list, fname)
# Test the initalized dictionaries
testing.assert_lists_equal(['search_string', 'keys', 'lengths',
@@ -220,7 +220,7 @@ def test_init_parse_filename_with_files(self):
return
def test_finish_parsed_filenames(self):
- """Test output restucturing for `finish_parsed_filenames`."""
+ """Test output restucturing for `_finish_parsed_filenames`."""
# Format the test input
fname = ''.join(('test*', '{year:04d}', '{day:03d}', '{hour:02d}',
'{minute:02d}', '{second:02d}', '{cycle:2s}.txt'))
@@ -231,8 +231,8 @@ def test_finish_parsed_filenames(self):
'str': ['hi']}
# Get the test results
- self.file_dict = futils.finish_parse_filenames(self.file_dict,
- file_list, fname)
+ self.file_dict = futils._finish_parse_filenames(self.file_dict,
+ file_list, fname)
# Test the output
for fkey in self.file_dict:
diff --git a/pysat/utils/files.py b/pysat/utils/files.py
index ed473b5fd..3e5ebd49b 100644
--- a/pysat/utils/files.py
+++ b/pysat/utils/files.py
@@ -19,114 +19,10 @@
from pysat.utils._core import listify
from pysat.utils.time import create_datetime_index
+# Define hidden support functions
-def process_parsed_filenames(stored, two_digit_year_break=None):
- """Create a Files pandas Series of filenames from a formatted dict.
-
- Parameters
- ----------
- stored : collections.orderedDict
- Ordered dictionary produced by `parse_fixed_width_filenames` or
- `parse_delimited_filenames`, containing date, time, version, and
- other information extracted from the filenames.
- two_digit_year_break : int or NoneType
- If filenames only store two digits for the year, then
- '1900' will be added for years >= two_digit_year_break
- and '2000' will be added for years < two_digit_year_break.
- If None, then four-digit years are assumed. (default=None)
-
- Returns
- -------
- pds.Series
- Series, indexed by datetime, with file strings
-
- Note
- ----
- If two files have the same date and time information in the filename then
- the file with the higher version/revision/cycle is used. Series returned
- only has one file per datetime. Version is required for this filtering,
- revision and cycle are optional.
-
- See Also
- --------
- pysat.utils.files.parse_fixed_width_filenames,
- pysat.utils.files.parse_delimited_filenames
-
- """
-
- search_dict = construct_searchstring_from_format(stored['format_str'])
- keys = search_dict['keys']
-
- if len(stored['files']) > 0:
- # Deal with the possibility of two digit years. Years above
- # or equal to break are considered to be 1900+, while
- # years below break are considered to be 2000+
- if two_digit_year_break is not None:
- idx, = np.where(np.array(stored['year'])
- >= two_digit_year_break)
- stored['year'][idx] = stored['year'][idx] + 1900
- idx, = np.where(np.array(stored['year']) < two_digit_year_break)
- stored['year'][idx] = stored['year'][idx] + 2000
-
- # Need to sort the information for things to work
- rec_arr = [stored[key] for key in keys]
- rec_arr.append(stored['files'])
-
- # Sort all arrays by creating a sortable records array
- # withs keys corresponding to the files
- val_keys = keys + ['files']
- rec_arr = np.rec.fromarrays(rec_arr, names=val_keys)
- rec_arr.sort(order=val_keys, axis=0)
-
- # Pull out sorted info
- for key in keys:
- stored[key] = rec_arr[key]
- files = rec_arr['files']
- # Add hour and minute information to 'second'
- if stored['second'] is None:
- stored['second'] = np.zeros(len(files))
- if stored['hour'] is not None:
- stored['second'] += 3600 * stored['hour']
- if stored['minute'] is not None:
- stored['second'] += 60 * stored['minute']
-
- # The version shouldn't be set to zero, it is required to remove
- # duplicate datetimes
- if stored['revision'] is None:
- stored['revision'] = np.zeros(len(files))
- if stored['cycle'] is None:
- stored['cycle'] = np.zeros(len(files))
-
- index = create_datetime_index(year=stored['year'],
- month=stored['month'],
- day=stored['day'],
- uts=stored['second'])
-
- # If version, revision, and cycle are supplied, use these parameters
- # to weed out files that have been replaced with updated versions.
- # First, check for duplicate index times
- dups = index[index.duplicated()].unique()
- if (len(dups) > 0) and (stored['version'] is not None):
- # We have duplicates, keep the highest version/revision combo
- version = pds.Series(stored['version'], index=index)
- revision = pds.Series(stored['revision'], index=index)
- cycle = pds.Series(stored['cycle'], index=index)
- revive = version * 100000. + revision + cycle * 1e-5
- frame = pds.DataFrame({'files': files, 'revive': revive,
- 'time': index}, index=index)
- frame = frame.sort_values(by=['time', 'revive'],
- ascending=[True, False])
- frame = frame.drop_duplicates(subset='time', keep='first')
-
- return frame['files']
- else:
- return pds.Series(files, index=index).sort_index()
- else:
- return pds.Series(None, dtype='object')
-
-
-def init_parse_filenames(files, format_str):
+def _init_parse_filenames(files, format_str):
"""Initalize the output for the file parsing functions.
Parameters
@@ -189,7 +85,7 @@ def init_parse_filenames(files, format_str):
return stored, search_dict
-def finish_parse_filenames(stored, files, format_str):
+def _finish_parse_filenames(stored, files, format_str):
"""Reshape and finalize the output for the file parsing functions.
Parameters
@@ -242,6 +138,114 @@ def finish_parse_filenames(stored, files, format_str):
return stored
+# Define file utility functions
+
+def process_parsed_filenames(stored, two_digit_year_break=None):
+ """Create a Files pandas Series of filenames from a formatted dict.
+
+ Parameters
+ ----------
+ stored : collections.orderedDict
+ Ordered dictionary produced by `parse_fixed_width_filenames` or
+ `parse_delimited_filenames`, containing date, time, version, and
+ other information extracted from the filenames.
+ two_digit_year_break : int or NoneType
+ If filenames only store two digits for the year, then
+ '1900' will be added for years >= two_digit_year_break
+ and '2000' will be added for years < two_digit_year_break.
+ If None, then four-digit years are assumed. (default=None)
+
+ Returns
+ -------
+ pds.Series
+ Series, indexed by datetime, with file strings
+
+ Note
+ ----
+ If two files have the same date and time information in the filename then
+ the file with the higher version/revision/cycle is used. Series returned
+ only has one file per datetime. Version is required for this filtering,
+ revision and cycle are optional.
+
+ See Also
+ --------
+ pysat.utils.files.parse_fixed_width_filenames,
+ pysat.utils.files.parse_delimited_filenames
+
+ """
+
+ search_dict = construct_searchstring_from_format(stored['format_str'])
+ keys = search_dict['keys']
+
+ if len(stored['files']) > 0:
+ # Deal with the possibility of two digit years. Years above
+ # or equal to break are considered to be 1900+, while
+ # years below break are considered to be 2000+
+ if two_digit_year_break is not None:
+ idx, = np.where(np.array(stored['year'])
+ >= two_digit_year_break)
+ stored['year'][idx] = stored['year'][idx] + 1900
+ idx, = np.where(np.array(stored['year']) < two_digit_year_break)
+ stored['year'][idx] = stored['year'][idx] + 2000
+
+ # Need to sort the information for things to work
+ rec_arr = [stored[key] for key in keys]
+ rec_arr.append(stored['files'])
+
+ # Sort all arrays by creating a sortable records array
+ # withs keys corresponding to the files
+ val_keys = keys + ['files']
+ rec_arr = np.rec.fromarrays(rec_arr, names=val_keys)
+ rec_arr.sort(order=val_keys, axis=0)
+
+ # Pull out sorted info
+ for key in keys:
+ stored[key] = rec_arr[key]
+ files = rec_arr['files']
+
+ # Add hour and minute information to 'second'
+ if stored['second'] is None:
+ stored['second'] = np.zeros(len(files))
+ if stored['hour'] is not None:
+ stored['second'] += 3600 * stored['hour']
+ if stored['minute'] is not None:
+ stored['second'] += 60 * stored['minute']
+
+ # The version shouldn't be set to zero, it is required to remove
+ # duplicate datetimes
+ if stored['revision'] is None:
+ stored['revision'] = np.zeros(len(files))
+ if stored['cycle'] is None:
+ stored['cycle'] = np.zeros(len(files))
+
+ index = create_datetime_index(year=stored['year'],
+ month=stored['month'],
+ day=stored['day'],
+ uts=stored['second'])
+
+ # If version, revision, and cycle are supplied, use these parameters
+ # to weed out files that have been replaced with updated versions.
+ # First, check for duplicate index times
+ dups = index[index.duplicated()].unique()
+ if (len(dups) > 0) and (stored['version'] is not None):
+ # We have duplicates, keep the highest version/revision combo
+ version = pds.Series(stored['version'], index=index)
+ revision = pds.Series(stored['revision'], index=index)
+ cycle = pds.Series(stored['cycle'], index=index)
+ revive = version * 100000. + revision + cycle * 1e-5
+ frame = pds.DataFrame({'files': files, 'revive': revive,
+ 'time': index}, index=index)
+ frame = frame.sort_values(by=['time', 'revive'],
+ ascending=[True, False])
+ frame = frame.drop_duplicates(subset='time', keep='first')
+
+ return frame['files']
+ else:
+ return pds.Series(files, index=index).sort_index()
+ else:
+ return pds.Series(None, dtype='object')
+
+
def parse_fixed_width_filenames(files, format_str):
"""Extract specified info from a list of files with a fixed name width.
@@ -281,7 +285,7 @@ def parse_fixed_width_filenames(files, format_str):
"""
# Create storage for data to be parsed from filenames
- stored, search_dict = init_parse_filenames(files, format_str)
+ stored, search_dict = _init_parse_filenames(files, format_str)
if len(files) == 0:
return stored
@@ -319,7 +323,7 @@ def parse_fixed_width_filenames(files, format_str):
stored[key].append(val)
# Convert to numpy arrays and add additional information to output
- stored = finish_parse_filenames(stored, files, format_str)
+ stored = _finish_parse_filenames(stored, files, format_str)
return stored
@@ -374,7 +378,7 @@ def parse_delimited_filenames(files, format_str, delimiter):
"""
# Create storage for data to be parsed from filenames
- stored, search_dict = init_parse_filenames(files, format_str)
+ stored, search_dict = _init_parse_filenames(files, format_str)
if len(files) == 0:
return stored
@@ -440,7 +444,7 @@ def parse_delimited_filenames(files, format_str, delimiter):
break
# Convert to numpy arrays and add additional information to output
- stored = finish_parse_filenames(stored, files, format_str)
+ stored = _finish_parse_filenames(stored, files, format_str)
return stored
From 99755802963589c710cba1c2abf292318200ddae Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 18 Oct 2023 10:32:23 -0400
Subject: [PATCH 186/365] DOC: updated changelog
Added a summary of the bug fixes, tests, and file restructuring to the changelog.
---
CHANGELOG.md | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b7afe9f3b..b3e982414 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,14 +21,19 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Added options to customize `pysat_ndtesting` instrument with sample rate,
shift in time.
* Added orbit number to `pysat_ndtesting`.
- * Added the overwrite kwarg to `utils.registry.register_by_module`
+ * Added the overwrite kwarg to `utils.registry.register_by_module`.
+ * Added unit tests for all file parsing functions in `utils.files`.
+ * Reduced code duplication in the `utils.files.parse_fixed_width_filenames`
+ and `utils.files.parse_delimited_filenames` functions
+* Bug Fix
+ * Fixed `utils.files.parse_fixed_width_filenames` output for empty file list
* Maintenance
* Update link redirects in docs.
* Improved Instrument ValueError messages.
* Updated `Constellation.to_inst` method definition of coords, using dims
to combine common dimensions instead.
* Implement pyproject to manage metadata
-* Updated docstring references to `pysat.utils.files` in other modules.
+ * Updated docstring references to `pysat.utils.files` in other modules.
* Remove Sphinx cap
* Add pandas cap
* Update usage of whitespace and if statements (E275)
From 11c949c3e8f969830d2ea0bec8a27527678cf2c3 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 18 Oct 2023 10:32:43 -0400
Subject: [PATCH 187/365] DOC: udpated docstrings
Updated an incomplete test docstring.
---
pysat/tests/test_utils_files.py | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index 2c1cfb552..38b4deecb 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -85,7 +85,18 @@ def eval_parsed_filenames(self):
('-', "test", ['year', 'day', 'hour', 'minute', 'second', 'cycle',
'revision']), ('fun', 'test', [])])
def test_parse_delimited_filename(self, sep_char, flead, good_kwargs):
- """Check ability to parse list of delimited files."""
+ """Check ability to parse list of delimited files.
+
+ Parameters
+ ----------
+ sep_char : str
+ Separation character to use in joining the filename
+ flead : str
+ File prefix
+ good_kwargs : list
+ List of kwargs to include in the file format
+
+ """
# Format the test input
fname = '{:s}{:s}.cdf'.format(flead, sep_char.join(
[self.kw_format[fkey] for fkey in good_kwargs]))
From 449db5ba568344df4f58f5f561ea467653a8231b Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 18 Oct 2023 10:38:31 -0400
Subject: [PATCH 188/365] DOC: update wording in docstring
Parser didn't recognize imperative mood.
---
pysat/utils/files.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/utils/files.py b/pysat/utils/files.py
index 3e5ebd49b..01177aff7 100644
--- a/pysat/utils/files.py
+++ b/pysat/utils/files.py
@@ -23,7 +23,7 @@
def _init_parse_filenames(files, format_str):
- """Initalize the output for the file parsing functions.
+ """Set the initial output for the file parsing functions.
Parameters
----------
From 5dc4572369bf4c60ae7e0a22a91108525517dd0b Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 18 Oct 2023 16:40:11 -0400
Subject: [PATCH 189/365] TST: added `process_parsed_filenames` unit tests
Added unit tests for the `process_parsed_filenames` function. Also improved parsing evaluation method to assert a minimum number of keys and improve coverage results.
---
pysat/tests/test_utils_files.py | 142 ++++++++++++++++++++++++++++----
1 file changed, 126 insertions(+), 16 deletions(-)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index 0b62a497d..5a2b2535b 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -6,6 +6,7 @@
"""Tests the `pysat.utils.files` functions."""
import datetime as dt
+from collections import OrderedDict
from importlib import reload
import numpy as np
import os
@@ -47,17 +48,9 @@ def teardown_method(self):
del self.temporary_file_list
def eval_parsed_filenames(self):
- """Evaluate the output of a `parse_delimited_filename` unit test.
-
- Returns
- -------
- bool
- True if there is data to evalute, False if data dict is empty
-
- """
+ """Evaluate the output of a `parse_delimited_filename` unit test."""
# Evaluate the returned data dict
- if len(self.file_dict.keys()) < 2:
- return False
+ assert len(self.file_dict.keys()) >= 2, "insufficient keys in file dict"
# Extract the test lists
if len(self.fkwargs) > 0:
@@ -78,7 +71,7 @@ def eval_parsed_filenames(self):
assert self.file_dict[fkey] is None, \
"unused format key has a value"
- return True
+ return
@pytest.mark.parametrize("sep_char,flead,good_kwargs", [
("_", "test_", ['year', 'month', 'day', 'hour', 'minute', 'version']),
@@ -117,7 +110,7 @@ def test_parse_delimited_filename(self, sep_char, flead, good_kwargs):
sep_char)
# Test each of the return values
- assert self.eval_parsed_filenames()
+ self.eval_parsed_filenames()
return
def test_parse_delimited_filename_empty(self):
@@ -132,7 +125,7 @@ def test_parse_delimited_filename_empty(self):
self.file_dict = futils.parse_delimited_filenames([], fname, sep_char)
# Test each of the return values
- assert self.eval_parsed_filenames()
+ self.eval_parsed_filenames()
return
@pytest.mark.parametrize("sep_char,flead,good_kwargs", [
@@ -171,7 +164,7 @@ def test_parse_fixed_filename(self, sep_char, flead, good_kwargs):
self.file_dict = futils.parse_fixed_width_filenames(file_list, fname)
# Test each of the return values
- assert self.eval_parsed_filenames()
+ self.eval_parsed_filenames()
return
def test_parse_fixed_width_filename_empty(self):
@@ -185,7 +178,7 @@ def test_parse_fixed_width_filename_empty(self):
self.file_dict = futils.parse_fixed_width_filenames([], fname)
# Test each of the return values
- assert self.eval_parsed_filenames()
+ self.eval_parsed_filenames()
return
def test_init_parse_filename_empty(self):
@@ -199,7 +192,7 @@ def test_init_parse_filename_empty(self):
self.file_dict, sdict = futils._init_parse_filenames([], fname)
# Test each of the return values
- assert self.eval_parsed_filenames()
+ self.eval_parsed_filenames()
assert len(sdict.keys()) == 0, "Search dict was defined unnecessarily"
return
@@ -263,6 +256,123 @@ def test_finish_parsed_filenames(self):
return
+class TestProcessParsedFilenames(object):
+ """Unit tests for `process_parsed_filenames` function."""
+
+ def setup_method(self):
+ """Set up the unit test environment for each method."""
+ self.stored = OrderedDict({'year': np.full(shape=3, fill_value=2001),
+ 'month': np.full(shape=3, fill_value=2),
+ 'day': np.ones(shape=3, dtype=np.int64),
+ 'hour': np.zeros(shape=3, dtype=np.int64),
+ 'minute': np.zeros(shape=3, dtype=np.int64),
+ 'second': np.arange(0, 3, 1),
+ 'version': np.arange(0, 3, 1),
+ 'revision': np.arange(3, 0, -1),
+ 'cycle': np.array([1, 3, 2])})
+ self.format_str = "_".join(["test", "{year:04d}", "{month:02d}",
+ "{day:02d}", "{hour:02d}", "{minute:02d}",
+ "{second:02d}", "v{version:02d}",
+ "r{revision:02d}", "c{cycle:02d}.cdf"])
+
+ return
+
+ def teardown_method(self):
+ """Clean up the unit test environment for each method."""
+
+ del self.stored, self.format_str
+ return
+
+ def complete_stored(self):
+ """Add the 'files' and 'format_str' kwargs to the `stored` dict."""
+
+ file_list = []
+ for ind in range(len(self.stored['year'])):
+ ind_dict = {skey: self.stored[skey][ind]
+ for skey in self.stored.keys()}
+ file_list.append(self.format_str.format(**ind_dict))
+
+ self.stored['files'] = file_list
+ self.stored['format_str'] = self.format_str
+ return
+
+ @pytest.mark.parametrize("year_break", [0, 50])
+ def test_two_digit_years(self, year_break):
+ """Test the results of using different year breaks for YY formats."""
+ # Complete the ordered dict of file information
+ self.stored['year'] -= 2000
+ self.format_str = self.format_str.replace('year:04', 'year:02')
+ self.complete_stored()
+
+ # Get the file series
+ series = futils.process_parsed_filenames(
+ self.stored, two_digit_year_break=year_break)
+
+ # Test the series year
+ test_year = series.index.max().year
+ century = 1900 if year_break == 0 else 2000
+ assert test_year - century < 100, "year break caused wrong century"
+
+ # Test that the series length is correct and all filenames are unique
+ assert series.shape == self.stored['year'].shape
+ assert np.unique(series.values).shape == self.stored['year'].shape
+ return
+
+ def test_version_selection(self):
+ """Test version selection dominates when time is the same."""
+ # Complete the ordered dict of file information
+ self.stored['second'] = np.zeros(shape=self.stored['year'].shape,
+ dtype=np.int64)
+ self.complete_stored()
+
+ # Get the file series
+ series = futils.process_parsed_filenames(self.stored)
+
+ # Ensure there is only one file and that it has the highest version
+ ver_num = "v{:02d}".format(self.stored['version'].max())
+ assert series.shape == (1, )
+ assert series.values[0].find(ver_num) > 0
+ return
+
+ def test_revision_selection(self):
+ """Test revision selection dominates after time and version."""
+ # Complete the ordered dict of file information
+ self.stored['second'] = np.zeros(shape=self.stored['year'].shape,
+ dtype=np.int64)
+ self.stored['version'] = np.zeros(shape=self.stored['year'].shape,
+ dtype=np.int64)
+ self.complete_stored()
+
+ # Get the file series
+ series = futils.process_parsed_filenames(self.stored)
+
+ # Ensure there is only one file and that it has the highest version
+ rev_num = "r{:02d}".format(self.stored['revision'].max())
+ assert series.shape == (1, )
+ assert series.values[0].find(rev_num) > 0
+ return
+
+ def test_cycle_selection(self):
+ """Test cycle selection dominates after time, version, and revision."""
+ # Complete the ordered dict of file information
+ self.stored['second'] = np.zeros(shape=self.stored['year'].shape,
+ dtype=np.int64)
+ self.stored['version'] = np.zeros(shape=self.stored['year'].shape,
+ dtype=np.int64)
+ self.stored['revision'] = np.zeros(shape=self.stored['year'].shape,
+ dtype=np.int64)
+ self.complete_stored()
+
+ # Get the file series
+ series = futils.process_parsed_filenames(self.stored)
+
+ # Ensure there is only one file and that it has the highest version
+ cyc_num = "c{:02d}".format(self.stored['cycle'].max())
+ assert series.shape == (1, )
+ assert series.values[0].find(cyc_num) > 0
+ return
+
+
class TestFileDirectoryTranslations(CICleanSetup):
"""Unit tests for file directory setup."""
From 95d4646fbcf9c2b86209f8af4f0959dbe02be67d Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 18 Oct 2023 17:20:16 -0400
Subject: [PATCH 190/365] DOC: add missing default values
Add the missing default values to the docstring. Also improve the error message to be more informative.
---
pysat/utils/files.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/pysat/utils/files.py b/pysat/utils/files.py
index 01177aff7..93f3b376e 100644
--- a/pysat/utils/files.py
+++ b/pysat/utils/files.py
@@ -19,8 +19,8 @@
from pysat.utils._core import listify
from pysat.utils.time import create_datetime_index
-# Define hidden support functions
+# Define hidden support functions
def _init_parse_filenames(files, format_str):
"""Set the initial output for the file parsing functions.
@@ -464,7 +464,7 @@ def construct_searchstring_from_format(format_str, wildcard=False):
`instrument_{year:04d}{month:02d}{day:02d}_v{version:02d}.cdf`
wildcard : bool
If True, replaces each '?' sequence that would normally
- be returned with a single '*'.
+ be returned with a single '*'. (default=False)
Returns
-------
@@ -531,10 +531,10 @@ def construct_searchstring_from_format(format_str, wildcard=False):
out_dict['search_string'] += '*'
break
else:
- estr = ''.join(["Couldn't determine formatting width. ",
- "This may be due to the use of unsupported ",
- "wildcard characters."])
- raise ValueError(estr)
+ raise ValueError(
+ ''.join(["Couldn't determine formatting width, check ",
+ "formatting length specification (e.g., ",
+ "{day:03d} for day of year)."]))
return out_dict
From 411d63fc38d6702c34b2073a3e0bdc58736a77ea Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 18 Oct 2023 17:20:54 -0400
Subject: [PATCH 191/365] TST: add searchstring unit tests
Added unit tests for the `construct_searchstring_from_format` file utility function.
---
pysat/tests/test_utils_files.py | 124 ++++++++++++++++++++++++++++++++
1 file changed, 124 insertions(+)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index 5a2b2535b..8fae3eaf0 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -22,6 +22,130 @@
from pysat.utils import testing
+class TestConstructSearchstring(object):
+ """Unit tests for the `construct_searchstring_from_format` function."""
+
+ def setup_method(self):
+ """Set up the unit test environment for each method."""
+ self.out_dict = {}
+ self.num_fmt = None
+ self.str_len = None
+ self.fill_len = None
+ return
+
+ def teardown_method(self):
+ """Clean up the unit test environment after each method."""
+
+ del self.out_dict, self.num_fmt, self.str_len, self.fill_len
+ return
+
+ def eval_output(self):
+ """Evaluate the output dictionary."""
+
+ testing.assert_lists_equal(['search_string', 'keys', 'lengths',
+ 'string_blocks'],
+ list(self.out_dict.keys()))
+
+ assert len(self.out_dict['keys']) == self.num_fmt
+ assert len(''.join(self.out_dict['string_blocks'])) == self.str_len
+ assert sum(self.out_dict['lengths']) == self.fill_len
+
+ if self.out_dict['search_string'].find('*') < 0:
+ assert len(
+ self.out_dict['search_string']) == self.fill_len + self.str_len
+ else:
+ assert len(
+ self.out_dict['search_string']) <= self.fill_len + self.str_len
+ return
+
+ @pytest.mark.parametrize("format_str,nfmt,slen,flen", [
+ ("", 0, 0, 0), ("test", 0, 4, 0), ("{year:02d}{month:02d}", 2, 0, 4),
+ ("test_{year:04d}.ext", 1, 9, 4)])
+ def test_searchstring_success(self, format_str, nfmt, slen, flen):
+ """Test successful construction of a searchable string.
+
+ Parameters
+ ----------
+ format_str : str
+ The naming pattern of the instrument files and the locations of
+ date/version/revision/cycle information needed to create an ordered
+ list.
+ nfmt : int
+ Number of formatting options included in the format string
+ slen : int
+ Length of non-formatted string segments
+ flen : int
+ Length of formatted segments
+
+ """
+ # Set the evaluation criteria
+ self.num_fmt = nfmt
+ self.str_len = slen
+ self.fill_len = flen
+
+ # Get the streachstring dictionary
+ self.out_dict = futils.construct_searchstring_from_format(format_str)
+
+ # Evaluate the output
+ self.eval_output()
+ return
+
+ @pytest.mark.parametrize("format_str,nfmt,slen,flen, nwc", [
+ ("", 0, 0, 0, 0), ("test", 0, 4, 0, 0),
+ ("{year:02d}{month:02d}", 2, 0, 4, 2),
+ ("test_{year:04d}_{month:02d}.ext", 2, 10, 6, 2)])
+ def test_searchstring_w_wildcard(self, format_str, nfmt, slen, flen, nwc):
+ """Test successful construction of a searchable string with wildcards.
+
+ Parameters
+ ----------
+ format_str : str
+ The naming pattern of the instrument files and the locations of
+ date/version/revision/cycle information needed to create an ordered
+ list.
+ nfmt : int
+ Number of formatting options included in the format string
+ slen : int
+ Length of non-formatted string segments
+ flen : int
+ Length of formatted segments
+ nwc : int
+ Number of wildcard (*) symbols
+
+ """
+ # Set the evaluation criteria
+ self.num_fmt = nfmt
+ self.str_len = slen
+ self.fill_len = flen
+
+ # Get the streachstring dictionary
+ self.out_dict = futils.construct_searchstring_from_format(format_str,
+ True)
+
+ # Evaluate the output
+ self.eval_output()
+ assert len(self.out_dict['search_string'].split('*')) == nwc + 1
+ return
+
+ def test_searchstring_noformat(self):
+ """Test failure if the input argument is NoneType."""
+
+ testing.eval_bad_input(futils.construct_searchstring_from_format,
+ ValueError,
+ 'Must supply a filename template (format_str).',
+ input_args=[None])
+ return
+
+ def test_searchstring_bad_wildcard(self):
+ """Test failure if unsupported wildcard use is encountered."""
+
+ testing.eval_bad_input(futils.construct_searchstring_from_format,
+ ValueError,
+ "Couldn't determine formatting width, check",
+ input_args=["test{year:02d}{month:d}.txt"])
+ return
+
+
class TestParseFilenames(object):
"""Unit tests for the file parsing functions."""
From c19fa8518990667703e08a161db6827528be0ffd Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 19 Oct 2023 14:16:06 -0400
Subject: [PATCH 192/365] BUG: fixed bug in loading with pad
Fixed a bug in loading with a pad, where an empty slice could be attempted.
---
pysat/_instrument.py | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 111322b25..7e3318fa5 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -1307,7 +1307,7 @@ def _index(self, data=None):
Returns
-------
- pds.Series
+ index : pds.Series
Series containing the time indices for the Instrument data
"""
@@ -1316,15 +1316,17 @@ def _index(self, data=None):
data = self.data
if self.pandas_format:
- return data.index
+ index = data.index
else:
epoch_names = self._get_epoch_name_from_data(data=data)
if len(epoch_names) == 0:
- return pds.Index([])
+ index = pds.Index([])
else:
# Xarray preferred epoch name order is opposite
- return data.indexes[epoch_names[-1]]
+ index = data.indexes[epoch_names[-1]]
+
+ return index
def _pass_method(*args, **kwargs):
"""Empty default method for updatable Instrument methods."""
@@ -3301,11 +3303,10 @@ def load(self, yr=None, doy=None, end_yr=None, end_doy=None, date=None,
if not self._empty(ndata):
# Test the data index, slicing if necessary
nindex = self._index(data=ndata)
- if len(nindex) > 0:
+ if len(nindex) > 1:
if nindex[0] == self.index[-1]:
ndata = self.__getitem__(
- slice(1, len(ndata[self.index.name])),
- data=ndata)
+ slice(1, len(nindex)), data=ndata)
cdata.append(ndata)
if include is None:
include = 0
From dc285e64fcd427f5403f0d4f77e494b9b4d3a660 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 19 Oct 2023 14:17:01 -0400
Subject: [PATCH 193/365] ENH: add index name to CSV loading
If there is no time index name when loading CSV data into a pandas DataFrame, include the default name 'Epoch'.
---
pysat/instruments/methods/general.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/pysat/instruments/methods/general.py b/pysat/instruments/methods/general.py
index 5dee2195b..ef0e0d9a4 100644
--- a/pysat/instruments/methods/general.py
+++ b/pysat/instruments/methods/general.py
@@ -256,5 +256,12 @@ def load_csv_data(fnames, read_csv_kwargs=None):
for fname in fnames:
fdata.append(pds.read_csv(fname, **read_csv_kwargs))
- data = pds.DataFrame() if len(fdata) == 0 else pds.concat(fdata, axis=0)
+ if len(fdata) == 0:
+ data = pds.DataFrame()
+ else:
+ data = pds.concat(fdata, axis=0)
+
+ if data.index.name is None:
+ data.index.name = "Epoch"
+
return data
From 2dcd4345e6f82ae898535517db0cf7c6d03996b7 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 19 Oct 2023 14:18:41 -0400
Subject: [PATCH 194/365] TST: fixed bug in padding test
Fixed a bug in the padding test, so that datasets that only load data before their load date (e.g., some forecast/nowcast files) are allowed to produce empty datasets after the time selection that happens with padding data.
---
pysat/tests/classes/cls_instrument_library.py | 30 ++++++++++++-------
1 file changed, 19 insertions(+), 11 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 030500179..316f2fdde 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -562,17 +562,25 @@ def test_load_w_pad(self, pad, inst_dict):
load_and_set_strict_time_flag(test_inst, date, raise_error=True,
clean_off=False)
- assert not test_inst.empty
-
- # Evaluate the data index length
- assert (test_inst.index[-1]
- - test_inst.index[0]).total_seconds() < 86400.0
-
- # Evaluate the recorded pad
- inst_str = test_inst.__str__()
- assert inst_str.find(
- 'Data Padding: {:s}'.format(pad_repr)) > 0, "".join([
- "bad pad value: ", pad_repr, " not in ", inst_str])
+ if test_inst.empty:
+ # This will be empty if this is a forecast file that doesn't
+ # include the load date
+ test_inst.pad = None
+ load_and_set_strict_time_flag(test_inst, date, raise_error=True,
+ clean_off=False)
+ assert not test_inst.empty, "No data on {:}".format(date)
+ assert test_inst.index.max() < date, \
+ "Padding should have left data and didn't"
+ else:
+ # Padding was successful, evaluate the data index length
+ assert (test_inst.index[-1]
+ - test_inst.index[0]).total_seconds() < 86400.0
+
+ # Evaluate the recorded pad
+ inst_str = test_inst.__str__()
+ assert inst_str.find(
+ 'Data Padding: {:s}'.format(pad_repr)) > 0, "".join([
+ "bad pad value: ", pad_repr, " not in ", inst_str])
else:
pytest.skip("Download data not available")
From 9e0a910750ae1adef2bf390d89d2b18c354caccb Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 19 Oct 2023 18:10:51 -0400
Subject: [PATCH 195/365] TST: added tests for general methods
Added tests for uncovered lines in the general methods submodule.
---
pysat/tests/test_methods_general.py | 24 +++++++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py
index 38a4bd0b4..3c3202c72 100644
--- a/pysat/tests/test_methods_general.py
+++ b/pysat/tests/test_methods_general.py
@@ -11,6 +11,15 @@
from pysat.utils import testing
+def test_filename_creator():
+ """Test the `filename_creator` placeholder."""
+
+ testing.eval_bad_input(gen.filename_creator, NotImplementedError,
+ 'This feature has not been implemented yet',
+ input_args=[0.0])
+ return
+
+
class TestListFiles(object):
"""Unit tests for `pysat.instrument.methods.general.list_files`."""
@@ -268,7 +277,7 @@ def eval_data_cols(self):
return
def test_load_single_file(self):
- """Test the CVS data load with a single file."""
+ """Test the CSV data load with a single file."""
self.data = gen.load_csv_data(self.csv_file)
assert isinstance(self.data.index, pds.RangeIndex)
@@ -277,7 +286,7 @@ def test_load_single_file(self):
return
def test_load_file_list(self):
- """Test the CVS data load with multiple files."""
+ """Test the CSV data load with multiple files."""
self.data = gen.load_csv_data([self.csv_file, self.csv_file])
assert self.data.index.dtype == 'int64'
@@ -286,7 +295,7 @@ def test_load_file_list(self):
return
def test_load_file_with_kwargs(self):
- """Test the CVS data load with kwargs."""
+ """Test the CSV data load with kwargs."""
self.data = gen.load_csv_data([self.csv_file],
read_csv_kwargs={"parse_dates": True,
@@ -295,3 +304,12 @@ def test_load_file_with_kwargs(self):
self.eval_data_cols()
assert len(self.data.columns) == len(self.data_cols)
return
+
+ def test_load_empty_filelist(self):
+ """Test the CSV data loading with an empty file list."""
+
+ self.data = gen.load_csv_data([])
+
+ # Evaluate the empty output
+ assert self.data.empty
+ return
From 868cee259a6bf89a86905e58c6b3d73c34bd4c67 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 19 Oct 2023 18:13:48 -0400
Subject: [PATCH 196/365] STY: fixed indent
Fixed the indentation to be correct.
---
pysat/tests/test_methods_general.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pysat/tests/test_methods_general.py b/pysat/tests/test_methods_general.py
index 3c3202c72..3272b5d63 100644
--- a/pysat/tests/test_methods_general.py
+++ b/pysat/tests/test_methods_general.py
@@ -15,8 +15,8 @@ def test_filename_creator():
"""Test the `filename_creator` placeholder."""
testing.eval_bad_input(gen.filename_creator, NotImplementedError,
- 'This feature has not been implemented yet',
- input_args=[0.0])
+ 'This feature has not been implemented yet',
+ input_args=[0.0])
return
From 58f50f133ad464a8bd05803a81279bbe0df881c2 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 23 Oct 2023 18:06:35 -0400
Subject: [PATCH 197/365] TST: added file utils unit tests
Added file utility unit tests for the `search_local_system_formatted_filename` and `update_data_directory_structure` functions.
---
pysat/tests/test_utils_files.py | 34 +++++++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index 8fae3eaf0..ff3269218 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -709,6 +709,40 @@ def teardown_method(self):
del self.testInst, self.out, self.tempdir, self.start, self.stop
return
+ def test_updating_directories_no_registration(self, capsys):
+ """Test directory structure update method without registered insts."""
+ # New Template
+ templ = '{platform}'
+
+ # Convert directories to simpler platform structure, to get output
+ futils.update_data_directory_structure(new_template=templ,
+ full_breakdown=True)
+
+ # Capture printouts and test the results
+ captured = capsys.readouterr()
+ assert captured.find("No registered instruments detected.") >= 0, \
+ "Expected output not captured in: {:}".format(captured)
+ return
+
+ def test_search_local_system_formatted_filename(self):
+ """Test `search_local_system_formatted_filename` success."""
+ # Create a temporary file with a unique, searchable name
+ prefix = "test_me"
+ suffix = "tstfile"
+ searchstr = "*".join([prefix, suffix])
+ with tempfile.NamedTemporaryFile(
+ dir=self.testInst.files.data_path, prefix=prefix,
+ suffix=suffix) as temp:
+ files = futils.search_local_system_formatted_filename(
+ self.testInst.files.data_path, searchstr)
+
+ assert len(files) == 1, "unexpected number of files in search results"
+ assert files[0].find(
+ prefix) >= 0, "unexpected file prefix in search results"
+ assert files[0].find(
+ suffix) > 0, "unexpected file extension in search results"
+ return
+
def test_get_file_information(self):
"""Test `utils.files.get_file_information` success with existing files.
From 262accfcbb7a2c65dd0cb83385de823f69be92c3 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 23 Oct 2023 18:06:59 -0400
Subject: [PATCH 198/365] DOC: improved changelog description
Updated the changelog description to be more correct.
---
CHANGELOG.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a9de1d907..942d2eca5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,7 +22,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
shift in time.
* Added orbit number to `pysat_ndtesting`.
* Added the overwrite kwarg to `utils.registry.register_by_module`.
- * Added unit tests for all file parsing functions in `utils.files`.
+ * Added unit tests for all functions in `utils.files`.
* Reduced code duplication in the `utils.files.parse_fixed_width_filenames`
and `utils.files.parse_delimited_filenames` functions
* Bug Fix
From 30e0314b366825eaf7945e62f97668e7dbe930a8 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 23 Oct 2023 18:12:09 -0400
Subject: [PATCH 199/365] STY: removed unused variable
Removed an unused variable definition.
---
pysat/tests/test_utils_files.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index ff3269218..de5069304 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -730,9 +730,8 @@ def test_search_local_system_formatted_filename(self):
prefix = "test_me"
suffix = "tstfile"
searchstr = "*".join([prefix, suffix])
- with tempfile.NamedTemporaryFile(
- dir=self.testInst.files.data_path, prefix=prefix,
- suffix=suffix) as temp:
+ with tempfile.NamedTemporaryFile(dir=self.testInst.files.data_path,
+ prefix=prefix, suffix=suffix):
files = futils.search_local_system_formatted_filename(
self.testInst.files.data_path, searchstr)
From a64e4e4ba27f7b29e68e68430a635a194f13906c Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 23 Oct 2023 18:14:37 -0400
Subject: [PATCH 200/365] STY: changed import order
Changed import order to make flake8 happy.
---
pysat/tests/test_utils_files.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index de5069304..54c59c5d3 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -6,9 +6,9 @@
"""Tests the `pysat.utils.files` functions."""
import datetime as dt
-from collections import OrderedDict
from importlib import reload
import numpy as np
+from collections import OrderedDict
import os
import pandas as pds
import platform
From 3664b6112a0a72fbca758cd4e61d281ef30beedb Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Mon, 23 Oct 2023 18:20:46 -0400
Subject: [PATCH 201/365] STY: import order
Attempt two at fixing the import order.
---
pysat/tests/test_utils_files.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index 54c59c5d3..240f601cf 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -5,10 +5,10 @@
# ----------------------------------------------------------------------------
"""Tests the `pysat.utils.files` functions."""
+from collections import OrderedDict
import datetime as dt
from importlib import reload
import numpy as np
-from collections import OrderedDict
import os
import pandas as pds
import platform
From ad1d1b6824503e02c15cdf018dad0343ee353bd1 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Tue, 24 Oct 2023 10:50:22 -0400
Subject: [PATCH 202/365] TST: fixed bug in capsys
Fixed bug in capsys call to get STDOUT. Also added skip for local systems, as test is not expected to pass there.
---
pysat/tests/test_utils_files.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pysat/tests/test_utils_files.py b/pysat/tests/test_utils_files.py
index 240f601cf..48772ec38 100644
--- a/pysat/tests/test_utils_files.py
+++ b/pysat/tests/test_utils_files.py
@@ -709,19 +709,19 @@ def teardown_method(self):
del self.testInst, self.out, self.tempdir, self.start, self.stop
return
+ @pytest.mark.skipif(os.environ.get('CI') != 'true', reason="CI test only")
def test_updating_directories_no_registration(self, capsys):
"""Test directory structure update method without registered insts."""
- # New Template
- templ = '{platform}'
-
# Convert directories to simpler platform structure, to get output
+ templ = '{platform}'
futils.update_data_directory_structure(new_template=templ,
full_breakdown=True)
# Capture printouts and test the results
captured = capsys.readouterr()
- assert captured.find("No registered instruments detected.") >= 0, \
- "Expected output not captured in: {:}".format(captured)
+ captxt = captured.out
+ assert captxt.find("No registered instruments detected.") >= 0, \
+ "Expected output not captured in STDOUT: {:}".format(captxt)
return
def test_search_local_system_formatted_filename(self):
From 8398f284d37d8cf977505ef6902c009e11a020aa Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 26 Oct 2023 15:05:02 -0400
Subject: [PATCH 203/365] ENH: allow direct assignment to meta
Allow direct assignment of data to Meta (e.g., meta['var', 'label'] = value).
---
pysat/_meta.py | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 4501300f9..5850dbb4d 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -318,13 +318,17 @@ def __setitem__(self, data_vars, input_dat):
Parameters
----------
- data_vars : str, list
+ data_vars : str, list, tuple
Data variable names for the input metadata
input_dat : dict, pds.Series, or Meta
Input metadata to be assigned
- """
+ Raises
+ ------
+ ValueError
+ For unexpected input type that does not allow metadata to be set.
+ """
input_data = deepcopy(input_dat)
if isinstance(input_data, dict):
@@ -454,6 +458,14 @@ def __setitem__(self, data_vars, input_dat):
# Outputs from Meta object are a Series. Thus, this takes in input
# from a Meta object. Set data using standard assignment via a dict.
self[data_vars] = input_data.to_dict()
+ else:
+ # The input data is a value, this only works if `data_vars` is
+ # a tuple that contains the data variable and the metadata label
+ if isinstance(data_vars, tuple) and len(data_vars) == 2:
+ self[data_vars[0]] = {data_vars[1]: input_data}
+ else:
+ raise ValueError(
+ "unexpected input combination, can't set metadata")
return
From ff28aa8c6832971e275afabb2736b6f24545ffc3 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 26 Oct 2023 15:05:20 -0400
Subject: [PATCH 204/365] DOC: updated docs
Added a direct assignment example to the docs.
---
docs/tutorial/tutorial_basics.rst | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/docs/tutorial/tutorial_basics.rst b/docs/tutorial/tutorial_basics.rst
index 59ab6d4a0..1e1311801 100644
--- a/docs/tutorial/tutorial_basics.rst
+++ b/docs/tutorial/tutorial_basics.rst
@@ -439,13 +439,13 @@ Explorer `(ICON) `_.
# Retrieve units using general labels
dmsp.meta['ti', dmsp.meta.labels.units]
- # Update units for ion temperature
- dmsp.meta['ti'] = {dmsp.meta.labels.units: 'Kelvin'}
+ # Update units for ion temperature using direct assignment
+ dmsp.meta['ti', dmsp.meta.labels.units] = 'Kelvin'
- # Update display name for ion temperature, using LaTeX notation
+ # Update display name for ion temp, using dict assignment and LaTeX notation
dmsp.meta['ti'] = {dmsp.meta.labels.name: 'T$_i$'}
- # Add new meta data
+ # Add new meta data for multiple labels using dict assignment
dmsp.meta['new'] = {dmsp.meta.labels.units: 'unitless',
dmsp.meta.labels.name: 'New display name'}
From b2fe06128303da7e03972700c96547f5ceee9cf8 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 26 Oct 2023 15:05:34 -0400
Subject: [PATCH 205/365] DOC: updated changelog
Added a summary of the PR to the changelog.
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a9de1d907..683cf85ed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Added unit tests for all file parsing functions in `utils.files`.
* Reduced code duplication in the `utils.files.parse_fixed_width_filenames`
and `utils.files.parse_delimited_filenames` functions
+ * Added ability to set Meta data using `meta['data_var', 'label'] = value`
+ structure
* Bug Fix
* Fixed `utils.files.parse_fixed_width_filenames` output for empty file list
* Maintenance
From df1eec4ed234d3ea9c5bb3844233c289081d902f Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Thu, 26 Oct 2023 15:05:49 -0400
Subject: [PATCH 206/365] TST: added unit tests
Added unit tests for the new meta setting behaviour.
---
pysat/tests/test_meta.py | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/pysat/tests/test_meta.py b/pysat/tests/test_meta.py
index 0bee40e27..804278828 100644
--- a/pysat/tests/test_meta.py
+++ b/pysat/tests/test_meta.py
@@ -142,6 +142,16 @@ def test_getitem_w_bad_key(self):
assert str(kerr).find('not found in MetaData') >= 0
return
+ def test_setitem_w_bad_input_combo(self):
+ """Test that bad input calls will raise ValueError when setting data."""
+
+ with pytest.raises(ValueError) as verr:
+ self.meta[['uts', 'units']] = 'seconds'
+
+ assert str(verr).find(
+ "unexpected input combination, can't set metadata") >= 0
+ return
+
def test_getitem_w_index(self):
"""Test raises NotImplementedError with an integer index."""
@@ -578,6 +588,22 @@ def test_accept_default_labels(self):
return
+ def test_meta_assign_single_val(self):
+ """Test basic assignment of a single metadata value."""
+ # Ensure the data has not been set already
+ data_name = 'special_data'
+ label_name = self.meta.labels.notes
+ meta_val = "test me"
+ assert data_name not in self.meta.keys(), "bad testing set up"
+
+ # Assign notes metadata
+ self.meta[data_name, label_name] = meta_val
+
+ # Test the assigned metadata
+ assert data_name in self.meta.keys()
+ assert self.meta[data_name, label_name] == meta_val
+ return
+
@pytest.mark.parametrize("custom_attr", [None, 'custom_meta'])
@pytest.mark.parametrize("assign_type", [dict, pds.Series])
def test_meta_assignment(self, custom_attr, assign_type):
From b4f9680bb04358961b7638c30a19ea363c3b017a Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 26 Oct 2023 15:11:21 -0400
Subject: [PATCH 207/365] DOC: updated docstring
Updated the allowed `input_dat` type.
---
pysat/_meta.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/_meta.py b/pysat/_meta.py
index 5850dbb4d..e44c2172c 100644
--- a/pysat/_meta.py
+++ b/pysat/_meta.py
@@ -320,7 +320,7 @@ def __setitem__(self, data_vars, input_dat):
----------
data_vars : str, list, tuple
Data variable names for the input metadata
- input_dat : dict, pds.Series, or Meta
+ input_dat : dict, pds.Series, Meta, int, float, str, bool, or NoneType
Input metadata to be assigned
Raises
From b64619a361662207de338130bdbb888bc8549973 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 27 Oct 2023 11:55:10 -0400
Subject: [PATCH 208/365] ENH: added `drop` and `delitem`
Added the `drop` and `__delitem__` methods to the Instrument class, which delete data variables from the `data` and `Meta` objects.
---
pysat/_instrument.py | 60 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 60 insertions(+)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 7e3318fa5..1caf0d199 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -1164,6 +1164,24 @@ def __setitem__(self, key, new_data):
return
+ def __delitem__(self, key):
+ """Delete a key by calling `drop` method.
+
+ Parameters
+ ----------
+ key : str or list-like
+ A meta data variable, label, or MetaHeader attribute; which are
+ considered in that order.
+
+ Raises
+ ------
+ KeyError
+ If all key values are unavailable
+
+ """
+ self.drop(key)
+ return
+
def __iter__(self):
"""Load data for subsequent days or files.
@@ -2580,6 +2598,48 @@ def custom_clear(self):
self.custom_kwargs = []
return
+ def drop(self, names):
+ """Drop variables from Instrument.
+
+ Parameters
+ ----------
+ names : str or list-like
+ String or list of strings specifying the variables names to drop
+
+ Raises
+ ------
+ KeyError
+ If all of the variable names provided in `names` are not found
+ in the variable list. If a subset is missing, a logger warning is
+ issued instead.
+
+ """
+ # Ensure the input is list-like
+ names = pysat.utils.listify(names)
+
+ # Ensure the names are present in the list of variables
+ good_names = [name for name in names if name in self.variables]
+
+ if len(good_names) > 0:
+ # Drop the Instrument data using the appropriate methods
+ if self.pandas_format:
+ self.data = self.data.drop(columns=good_names)
+ else:
+ self.data = self.data.drop_vars(good_names)
+
+ # Drop the meta data associated with this variable
+ self.meta.drop(good_names)
+
+ if len(good_names) < len(names):
+ if len(good_names) == 0:
+ raise KeyError("{:} not found in Instrument variables".format(
+ names))
+ else:
+ pysat.logger.warning(
+ "{:} not found in Instrument variables".format(
+ [name for name in names if name not in good_names]))
+ return
+
def today(self):
"""Get today's date (UTC), with no hour, minute, second, etc.
From 19c39d3850a5e071cc730c2917fbef18fdc41774 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 27 Oct 2023 11:56:22 -0400
Subject: [PATCH 209/365] ENH: added `drop` and `delitem` to const
Added the `drop` and `__delitem__` methods to the Constellation class, which delete data variables from the `data` and `Meta` objects in all the appropriate Instruments. May use the names from the individual Instruments or from the Constellation `variables` attribute.
---
pysat/_constellation.py | 72 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 72 insertions(+)
diff --git a/pysat/_constellation.py b/pysat/_constellation.py
index c51a43fd3..6dc87a8e4 100644
--- a/pysat/_constellation.py
+++ b/pysat/_constellation.py
@@ -378,6 +378,23 @@ def is_plural(num):
return output_str
+ def __delitem__(self, key):
+ """Delete a key by calling `drop` method.
+
+ Parameters
+ ----------
+ key : str or list-like
+ A Constellation variable or list of variables.
+
+ Raises
+ ------
+ KeyError
+ If all key values are unavailable
+
+ """
+ self.drop(key)
+ return
+
# -----------------------------------------------------------------------
# Define all hidden methods
@@ -592,6 +609,61 @@ def date(self):
else:
return None
+ def drop(self, names):
+ """Drop variables (names) from metadata.
+
+ Parameters
+ ----------
+ names : str or list-like
+ String or list of strings specifying the variable names to drop
+
+ Raises
+ ------
+ KeyError
+ If all of the keys provided in `names` is not found in the
+ standard metadata, labels, or header metadata. If a subset is
+ missing, a logger warning is issued instead.
+
+ """
+ # Ensure the input is list-like
+ names = pysat.utils.listify(names)
+
+ # Divide the names by instrument
+ good_inst_names = [list() for inst in self.instruments]
+ bad_names = list()
+ inst_strs = ['_'.join([attr for attr in [inst.platform, inst.name,
+ inst.tag, inst.inst_id]
+ if len(attr) > 0]) for inst in self.instruments]
+ for name in names:
+ got_name = False
+ for i, inst in enumerate(self.instruments):
+ if name in inst.variables:
+ good_inst_names[i].append(name)
+ got_name = True
+ elif name in self.variables and name.find(inst_strs[i]) > 0:
+ good_inst_names[i].append(name.split("_{:s}".format(
+ inst_strs[i]))[0])
+ got_name = True
+
+ if not got_name:
+ bad_names.append(name)
+
+ # If there are no good names, raise a KeyError
+ if len(bad_names) == len(names):
+ raise KeyError('{:} not found in Constellation'.format(names))
+
+ # Drop names by instrument
+ for i, inst in enumerate(self.instruments):
+ if len(good_inst_names[i]) > 0:
+ inst.drop(good_inst_names[i])
+
+ # If there are some bad names, raise a logging warning
+ if len(bad_names) > 0:
+ pysat.logger.warning('{:} not found in Constellation'.format(
+ bad_names))
+
+ return
+
@property
def empty(self):
"""Boolean flag reflecting lack of data.
From b91adaaf540aba11b666cb806769c92323bb8f7f Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 27 Oct 2023 11:56:47 -0400
Subject: [PATCH 210/365] TST: added Instrument del tests
Added unit tests for the Instrument del and drop methods.
---
pysat/tests/classes/cls_instrument_access.py | 60 ++++++++++++++++++++
1 file changed, 60 insertions(+)
diff --git a/pysat/tests/classes/cls_instrument_access.py b/pysat/tests/classes/cls_instrument_access.py
index e32d244f1..3570eb66d 100644
--- a/pysat/tests/classes/cls_instrument_access.py
+++ b/pysat/tests/classes/cls_instrument_access.py
@@ -130,6 +130,66 @@ def test_basic_instrument_load(self, kwargs):
self.eval_successful_load()
return
+ @pytest.mark.parametrize("method", ["del", "drop"])
+ @pytest.mark.parametrize("del_all", [True, False])
+ def test_basic_instrument_del(self, method, del_all):
+ """Test that data can be deleted from an Instrument.
+
+ Parameters
+ ----------
+ method : str
+ String specifying the deletion method
+ del_all : bool
+ Delete a single variable if False, delete all if True
+
+ """
+
+ # Load data by year and day of year
+ self.testInst.load(self.ref_time.year, self.ref_doy)
+
+ # Get a variable name(s) to delete
+ var = self.testInst.variables if del_all else self.testInst.variables[0]
+
+ # Delete the variable
+ if method == 'del':
+ del self.testInst[var]
+ else:
+ self.testInst.drop(var)
+
+ # Test that the absence of the desired variable(s)
+ if del_all:
+ assert self.testInst.empty
+ assert len(self.testInst.variables) == 0
+ else:
+ assert var not in self.testInst.variables
+ return
+
+ def test_basic_instrument_bad_var_drop(self):
+ """Check for error when deleting absent data variable."""
+ # Load data by year and day of year
+ self.testInst.load(self.ref_time.year, self.ref_doy)
+
+ # Test that the correct error is raised
+ testing.eval_bad_input(self.testInst.drop, KeyError,
+ "not found in Instrument variables",
+ input_args=["not_a_data_variable"])
+ return
+
+ def test_basic_instrument_partial_bad_var_drop(self, caplog):
+ """Check for log warning when deleting present and absent variables."""
+ # Load data by year and day of year
+ self.testInst.load(self.ref_time.year, self.ref_doy)
+
+ dvars = [self.testInst.variables[0], "not_a_data_var"]
+
+ # Test that the correct warning is raised
+ with caplog.at_level(logging.INFO, logger='pysat'):
+ self.testInst.drop(dvars)
+
+ captured = caplog.text
+ assert captured.find("not found in Instrument variables") > 0
+ return
+
@pytest.mark.parametrize('pad', [None, dt.timedelta(days=1)])
def test_basic_instrument_load_no_data(self, caplog, pad):
"""Test Instrument load with no data for appropriate log messages.
From f44f1c5948354aec01d76ee5421eff7c477ff573 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 27 Oct 2023 11:57:07 -0400
Subject: [PATCH 211/365] TST: added Constellation del tests
Added unit tests for the Constellation delete and drop methods.
---
pysat/tests/test_constellation.py | 88 +++++++++++++++++++++++++++++++
1 file changed, 88 insertions(+)
diff --git a/pysat/tests/test_constellation.py b/pysat/tests/test_constellation.py
index 608251db2..0712de18f 100644
--- a/pysat/tests/test_constellation.py
+++ b/pysat/tests/test_constellation.py
@@ -572,3 +572,91 @@ def test_to_inst_mult_pad_clean(self):
list(self.inst[1].index))
return
+
+ @pytest.mark.parametrize("method", ["del", "drop"])
+ def test_delitem_all_inst(self, method):
+ """Test Constellation deletion of data variables from all instruments.
+
+ Parameters
+ ----------
+ method : str
+ String specifying the deletion method
+
+ """
+ # Load the Constellation data
+ self.const.load(date=self.ref_time)
+
+ # Delete the UTS data from all instruments
+ dvar = "uts"
+ if method == "del":
+ del self.const[dvar]
+ else:
+ self.const.drop(dvar)
+
+ # Test that this variable is gone from all Instruments
+ for inst in self.const.instruments:
+ assert dvar not in inst.variables
+
+ # Test that the constellation variable list has been updated
+ for var in self.const.variables:
+ assert var.find(dvar) != 0
+
+ return
+
+ @pytest.mark.parametrize("method", ["del", "drop"])
+ def test_delitem_one_inst(self, method):
+ """Test Constellation deletion of data variables from one instrument.
+
+ Parameters
+ ----------
+ method : str
+ String specifying the deletion method
+
+ """
+ # Load the Constellation data
+ self.const.load(date=self.ref_time)
+
+ # Delete the UTS data from all instruments
+ dvar = "uts_pysat_testing"
+ if method == "del":
+ del self.const[dvar]
+ else:
+ self.const.drop(dvar)
+
+ # Test that this variable is gone from only the desired Instrument
+ for inst in self.const.instruments:
+ if inst.platform == "pysat" and inst.name == "testing":
+ assert "uts" not in inst.variables
+ else:
+ assert "uts" in inst.variables
+
+ # Test that the constellation variable list has been updated
+ assert dvar not in self.const.variables
+
+ return
+
+ def test_bad_var_drop(self):
+ """Check for error when deleting absent data variable."""
+ # Load the Constellation data
+ self.const.load(date=self.ref_time)
+
+ # Test that the correct error is raised
+ testing.eval_bad_input(self.const.drop, KeyError,
+ "not found in Constellation",
+ input_args=["not_a_data_variable"])
+ return
+
+ def test_partial_bad_var_drop(self, caplog):
+ """Check for log warning when deleting present and absent variables."""
+ # Load the Constellation data
+ self.const.load(date=self.ref_time)
+
+ dvars = [self.const.variables[0], "not_a_data_var"]
+
+ # Test that the correct warning is raised
+ with caplog.at_level(logging.INFO, logger='pysat'):
+ self.const.drop(dvars)
+
+ captured = caplog.text
+ assert captured.find("not found in Constellation") > 0
+ return
From 2aea6a363a436af9a1ccbc179417bca10cef3cc5 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 27 Oct 2023 11:57:23 -0400
Subject: [PATCH 212/365] DOC: updated changelog
Added a summary of changes to the changelog.
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a9de1d907..3c0455f16 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Added `__delitem__` to Meta and `drop` to MetaHeader and MetaLabels classes.
* Modified Meta to allow MetaHeader attribute access directly from Meta.
* Expanded `Meta.drop` to traverse attached MetaLabel and MetaHeader data.
+ * Added `__delitem__` and `drop` to Instrument and Constellation classes.
* Added options to customize `pysat_ndtesting` instrument with sample rate,
shift in time.
* Added orbit number to `pysat_ndtesting`.
From d203ec7e32a9e9b258f42b30ed90b9420eaeb168 Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Fri, 27 Oct 2023 12:05:22 -0400
Subject: [PATCH 213/365] STY: fixed indent
Fixed a visual indent.
---
pysat/_constellation.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/_constellation.py b/pysat/_constellation.py
index 6dc87a8e4..3baf1e87b 100644
--- a/pysat/_constellation.py
+++ b/pysat/_constellation.py
@@ -632,7 +632,7 @@ def drop(self, names):
good_inst_names = [list() for inst in self.instruments]
bad_names = list()
inst_strs = ['_'.join([attr for attr in [inst.platform, inst.name,
- inst.tag, inst.inst_id]
+ inst.tag, inst.inst_id]
if len(attr) > 0]) for inst in self.instruments]
for name in names:
got_name = False
From 74188e32b2f12d0f67c68b1a906af4e15e08ce15 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 31 Oct 2023 15:00:49 -0400
Subject: [PATCH 214/365] STY: reduce load calc in `pysat_testing.py`
Reduce the amount of unnecessary calculations performed in the pysat_testing.py load method.
---
pysat/instruments/pysat_testing.py | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/pysat/instruments/pysat_testing.py b/pysat/instruments/pysat_testing.py
index 6bbc08b02..2779fd8ed 100644
--- a/pysat/instruments/pysat_testing.py
+++ b/pysat/instruments/pysat_testing.py
@@ -93,6 +93,10 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
# Support keyword testing
pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))
+ # If no download should be simulated, return empty `data` and `meta` objects
+ if tag == 'no_download':
+ return pds.DataFrame(), pysat.Meta()
+
# Create an artificial satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
@@ -172,16 +176,14 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
data.index = index
data.index.name = 'Epoch'
+ # If we only want data and not metadata stop now
+ if tag == 'default_meta':
+ return data, pysat.Meta()
+
# Set the meta data
meta = mm_test.initialize_test_meta('Epoch', data.keys())
- # TODO(#1120): Move logic up so that empty data is returned first.
- if tag == 'default_meta':
- return data, pysat.Meta()
- elif tag == 'no_download':
- return pds.DataFrame(), pysat.Meta()
- else:
- return data, meta
+ return data, meta
list_files = functools.partial(mm_test.list_files, test_dates=_test_dates)
From c71884c59f21f3c4d939a1010163219c664c17c5 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 31 Oct 2023 15:01:30 -0400
Subject: [PATCH 215/365] ENH: added 'no_download' support for xarray
Added the 'no_download' test for xarray.
---
pysat/instruments/pysat_ndtesting.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 63c4dfd07..d4c24c6a6 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -15,9 +15,11 @@
name = 'ndtesting'
pandas_format = False
-tags = {'': 'Regular testing data set'}
+tags = {'': 'Regular testing data set',
+ 'no_download': 'simulate an instrument without download support'}
inst_ids = {'': [tag for tag in tags.keys()]}
_test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}}
+_test_download = {'': {'no_download': False}}
_test_load_opt = {'': {'': [{'num_extra_time_coords': 0},
{'num_extra_time_coords': 1}]}}
@@ -94,6 +96,10 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
# Support keyword testing
pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))
+ # If no download should be simulated, return empty `data` and `meta` objects
+ if tag == 'no_download':
+ return xr.Dataset(), pysat.Meta()
+
# Create an artificial satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
From e6830a15da7f3b8622e4226733dcaaedbff8ba16 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 31 Oct 2023 15:02:03 -0400
Subject: [PATCH 216/365] DOC: updated new instrument instructions
Added the expected output for loading no data to the new instrument instructions.
---
docs/new_instrument.rst | 3 +++
1 file changed, 3 insertions(+)
diff --git a/docs/new_instrument.rst b/docs/new_instrument.rst
index 9a0255ef6..2351f6107 100644
--- a/docs/new_instrument.rst
+++ b/docs/new_instrument.rst
@@ -414,6 +414,9 @@ The load module method signature should appear as:
commmonly specify the data set to be loaded
- The :py:func:`load` routine should return a tuple with :py:attr:`data` as the
first element and a :py:class:`pysat.Meta` object as the second element.
+ If there is no data to load, :py:attr:`data` should return an empty
+ :py:class:`pandas.DataFrame` or :py:class:`xarray.Dataset` and :py:attr:`meta`
+ should return an empty :py:class:`pysat.Meta` object.
- For simple time-series data sets, :py:attr:`data` is a
:py:class:`pandas.DataFrame`, column names are the data labels, rows are
indexed by :py:class:`datetime.datetime` objects.
From 260eb227e6955a83a45d915e7f825dff38ee8e3c Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 31 Oct 2023 15:02:19 -0400
Subject: [PATCH 217/365] TST: added empty load test
Added a test for loading times without data.
---
pysat/tests/classes/cls_instrument_library.py | 34 +++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 316f2fdde..a40f64f77 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -39,6 +39,7 @@ class TestInstruments(InstLibTests):
import pandas as pds
import pytest
+import xarray as xr
import pysat
from pysat.utils import generate_instrument_list
@@ -411,6 +412,39 @@ def test_load(self, clean_level, inst_dict):
return
+ @pytest.mark.second
+ @pytest.mark.load_options
+ def test_load_empty(self, inst_dict):
+ """Test that instruments load empty objects if no data is available.
+
+ Parameters
+ ----------
+ inst_dict : dict
+ Dictionary containing info to instantiate a specific instrument.
+ Set automatically from instruments['download'] when
+ `initialize_test_package` is run.
+
+ """
+
+ # Get the instrument information and update the date to be in the future
+ test_inst, date = initialize_test_inst_and_date(inst_dict)
+ date = dt.datetime(dt.datetime.utcnow().year + 100, 1, 1)
+
+ # Make sure the strict time flag doesn't interfere with the load test
+ load_and_set_strict_time_flag(test_inst, date, raise_error=True)
+
+ # Check the empty status
+ assert test_inst.empty, "Data was loaded for a far-future time"
+ assert test_inst.meta == pysat.Meta(), "Meta data is not empty"
+ if test_inst.pandas_format:
+ assert all(test_inst.data == pds.DataFrame()), "Data not empty"
+ else:
+ assert test_inst.data.dims == xr.Dataset().dims, "Dims not empty"
+ assert test_inst.data.data_vars == xr.Dataset().data_vars, \
+ "Data variables not empty"
+
+ return
+
@pytest.mark.second
# Need to maintain download mark for backwards compatibility.
# Can remove once pysat 3.1.0 is released and libraries are updated.
From fb0ff24ed39bca7dbef1cd8226527c52fd73bf7e Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 31 Oct 2023 15:02:33 -0400
Subject: [PATCH 218/365] DOC: updated changelog
Added a summary of the changes to the changelog.
---
CHANGELOG.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 67364ca24..388222f20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* New Features
* Added tests for warnings, logging messages, and errors in the Instrument
clean method.
- * Added loading test with padding for Instruments.
+ * Added Instrument loading test with padding and for times without data.
* Allow Instruments to define custom `concat_data` methods.
* Added `include` kwarg to `Instrument.concat_data` to expand allowed inputs.
* Added data kwarg to the Instrument class `__getitem__` method and reduced
From d4fa58f209cbd26590ff604b7e112c01f9b71574 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Tue, 31 Oct 2023 15:08:34 -0400
Subject: [PATCH 219/365] STY: removed trailing whitespace
Removed trailing whitespace.
---
pysat/instruments/pysat_ndtesting.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index d4c24c6a6..3d2815170 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -98,7 +98,7 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
# If no download should be simulated, return empty `data` and `meta` objects
if tag == 'no_download':
- return xr.Dataset(), pysat.Meta()
+ return xr.Dataset(), pysat.Meta()
# Create an artificial satellite data set
iperiod = mm_test.define_period()
From 8d20e8633e405a375a27a2045c0aff938b48f434 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
Date: Tue, 31 Oct 2023 15:49:11 -0400
Subject: [PATCH 220/365] Apply suggestions from code review
Co-authored-by: Angeline Burrell
---
pysat/tests/classes/cls_instrument_library.py | 7 -------
1 file changed, 7 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index e737baf87..c51ed79e8 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -425,8 +425,6 @@ def test_load_multiple_days(self, inst_dict):
Parameters
----------
- clean_level : str
- Cleanliness level for loaded instrument data.
inst_dict : dict
Dictionary containing info to instantiate a specific instrument.
Set automatically from instruments['download'] when
@@ -436,17 +434,12 @@ def test_load_multiple_days(self, inst_dict):
test_inst, date = initialize_test_inst_and_date(inst_dict)
if len(test_inst.files.files) > 0:
- # Set the clean level
- target = 'Fake Data to be cleared'
- test_inst.data = [target]
# Make sure the strict time flag doesn't interfere with
# the load tests, and re-run with desired clean level
load_and_set_strict_time_flag(test_inst, date, raise_error=True,
clean_off=False, concat=True)
- # Make sure fake data is cleared
- assert target not in test_inst.data
# Make sure more than one day has been loaded
assert len(np.unique(test_inst.index.day)) > 1
else:
From acc29f985942b8c9367b1653f814f65eb8bd445e Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 31 Oct 2023 15:49:45 -0400
Subject: [PATCH 221/365] MAINT: remove old marks
---
pysat/tests/classes/cls_instrument_library.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index e737baf87..03389a360 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -553,10 +553,7 @@ def test_clean_warn(self, clean_level, inst_dict, caplog):
return
@pytest.mark.second
- # Need to maintain download mark for backwards compatibility.
- # Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
- @pytest.mark.download
@pytest.mark.parametrize('pad', [{'days': 1}, dt.timedelta(days=1)])
def test_load_w_pad(self, pad, inst_dict):
"""Test that instruments load at each cleaning level.
From 2fa067021376adf6af8d76126b4099db30c69214 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 31 Oct 2023 15:52:19 -0400
Subject: [PATCH 222/365] Apply suggestions from review
---
pysat/tests/classes/cls_instrument_library.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index af1ae1f78..6d21afe56 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -76,7 +76,7 @@ def initialize_test_inst_and_date(inst_dict):
def load_and_set_strict_time_flag(test_inst, date, raise_error=False,
- clean_off=True, concat=False):
+ clean_off=True, set_end_date=False):
"""Load data and set the strict time flag if needed for other tests.
Parameters
@@ -91,15 +91,15 @@ def load_and_set_strict_time_flag(test_inst, date, raise_error=False,
clean_off : bool
Turn off the clean method when re-loading data and testing the
strict time flag (default=True)
- concat : bool
- If True, load multiple days to concat. If False, load single day.
+ set_end_date : bool
+ If True, load with setting the end date. If False, load single day.
(default=False)
"""
- kwargs = {'use_header': True}
+ kwargs = {}
- if concat:
+ if set_end_date:
kwargs['end_date'] = date + dt.timedelta(days=2)
try:
From 6cecdedba75524840683490a0f05bcb565ba8b61 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Tue, 31 Oct 2023 18:59:00 -0400
Subject: [PATCH 223/365] BUG: kwarg
---
pysat/tests/classes/cls_instrument_library.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 6d21afe56..6b661dc3b 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -438,7 +438,7 @@ def test_load_multiple_days(self, inst_dict):
# Make sure the strict time flag doesn't interfere with
# the load tests, and re-run with desired clean level
load_and_set_strict_time_flag(test_inst, date, raise_error=True,
- clean_off=False, concat=True)
+ clean_off=False, set_end_date=True)
# Make sure more than one day has been loaded
assert len(np.unique(test_inst.index.day)) > 1
From 1afbd203bdb02990a31099cb4504a815b7eefa6e Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Wed, 1 Nov 2023 16:17:05 -0400
Subject: [PATCH 224/365] REV: remove 'no_download' from ndxarray
Remove the 'no_download' option from the ndxarray test Instrument.
Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
---
pysat/instruments/pysat_ndtesting.py | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/pysat/instruments/pysat_ndtesting.py b/pysat/instruments/pysat_ndtesting.py
index 3d2815170..63c4dfd07 100644
--- a/pysat/instruments/pysat_ndtesting.py
+++ b/pysat/instruments/pysat_ndtesting.py
@@ -15,11 +15,9 @@
name = 'ndtesting'
pandas_format = False
-tags = {'': 'Regular testing data set',
- 'no_download': 'simulate an instrument without download support'}
+tags = {'': 'Regular testing data set'}
inst_ids = {'': [tag for tag in tags.keys()]}
_test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}}
-_test_download = {'': {'no_download': False}}
_test_load_opt = {'': {'': [{'num_extra_time_coords': 0},
{'num_extra_time_coords': 1}]}}
@@ -96,10 +94,6 @@ def load(fnames, tag='', inst_id='', sim_multi_file_right=False,
# Support keyword testing
pysat.logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))
- # If no download should be simulated, return empty `data` and `meta` objects
- if tag == 'no_download':
- return xr.Dataset(), pysat.Meta()
-
# Create an artificial satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
From 478ba17403e23609ece8b993f2c133705aeed7bc Mon Sep 17 00:00:00 2001
From: Angeline Burrell
Date: Thu, 2 Nov 2023 12:47:55 -0400
Subject: [PATCH 225/365] DOC: remove old comments
Remove comments that were referring to code that was removed previously.
Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com>
---
pysat/tests/classes/cls_instrument_library.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/pysat/tests/classes/cls_instrument_library.py b/pysat/tests/classes/cls_instrument_library.py
index 92a6ae3d5..d7bdef0a4 100644
--- a/pysat/tests/classes/cls_instrument_library.py
+++ b/pysat/tests/classes/cls_instrument_library.py
@@ -453,8 +453,6 @@ def test_load_empty(self, inst_dict):
return
@pytest.mark.second
- # Need to maintain download mark for backwards compatibility.
- # Can remove once pysat 3.1.0 is released and libraries are updated.
@pytest.mark.load_options
def test_load_multiple_days(self, inst_dict):
"""Test that instruments load at each cleaning level.
From 5c6b52aa98a191525d7d579a40af3a8a9be11306 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 22 Nov 2023 20:51:40 -0500
Subject: [PATCH 226/365] DOC: remove todo
---
pysat/tests/test_files.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/pysat/tests/test_files.py b/pysat/tests/test_files.py
index 0933d2ed7..c475b7cf9 100644
--- a/pysat/tests/test_files.py
+++ b/pysat/tests/test_files.py
@@ -1295,8 +1295,6 @@ def teardown_method(self):
temporary_file_list=self.temporary_file_list)
pysat.params['data_dirs'] = self.data_paths
- # TODO(#871): This needs to be replaced or expanded based on the tests that
- # portalocker uses
def test_race_condition(self):
"""Test that multiple instances of pysat instrument creation run."""
processes = 5
From 7a8b5b409419c2d1efbfbccb78652c6f4bdd3310 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 22 Nov 2023 20:55:08 -0500
Subject: [PATCH 227/365] STY: use temporary directory
---
pysat/tests/test_utils.py | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py
index b88c961f6..4e819b2eb 100644
--- a/pysat/tests/test_utils.py
+++ b/pysat/tests/test_utils.py
@@ -13,6 +13,7 @@
import portalocker
import pytest
import shutil
+import sys
import tempfile
import warnings
@@ -624,19 +625,22 @@ class TestNetworkLock(object):
def setup_method(self):
"""Set up the unit test environment."""
+ # Use a temporary directory so that the user's setup is not altered.
+ self.temp_dir = tempfile.TemporaryDirectory()
+
# Create and write a temporary file
- self.fname = 'temp_lock_file.txt'
+ self.fname = os.path.join(self.temp_dir.name, 'temp_lock_file.txt')
with open(self.fname, 'w') as fh:
fh.write('spam and eggs')
return
def teardown_method(self):
"""Clean up the unit test environment."""
- # Remove the temporary file
- os.remove(self.fname)
+ # Remove the temporary directory.
+ self.temp_dir.cleanup()
# Delete the test class attributes
- del self.fname
+ del self.fname, self.temp_dir
return
def test_with_timeout(self):
From 33199c795d9c7b3bdbee77bee3bc815ea132170c Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Wed, 22 Nov 2023 20:55:15 -0500
Subject: [PATCH 228/365] DOC: update changelog
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 04680ecc9..2acb2c1ce 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -53,6 +53,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
* Removed deprecated `freq` kwarg from `download`
* Removed deprecated `use_header` kwarg from `load` and changed default
behaviour to `use_header=True`
+ * Use temporary directories for files created during test_utils.py
[3.1.0] - 2023-05-31
--------------------
From a84bd0f5c0762edf412bdc2ac800bc642b26f20f Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Thu, 23 Nov 2023 09:38:17 -0500
Subject: [PATCH 229/365] STY: pep8
---
pysat/tests/test_utils.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pysat/tests/test_utils.py b/pysat/tests/test_utils.py
index 4e819b2eb..3aeb87d81 100644
--- a/pysat/tests/test_utils.py
+++ b/pysat/tests/test_utils.py
@@ -13,7 +13,6 @@
import portalocker
import pytest
import shutil
-import sys
import tempfile
import warnings
From e9a119dc98c2648b36fb79c5c2242e82e72db07f Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 24 Nov 2023 12:14:20 -0500
Subject: [PATCH 230/365] TST: continue-on-error for coveralls
---
.github/workflows/main.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 51489005d..845985ebd 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -69,3 +69,4 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: coveralls --rcfile=pyproject.toml --service=github
+ continue-on-error: true
From 6ce10e17f2500aa4b8e28d089d2fdb35ef50460a Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Fri, 24 Nov 2023 14:04:52 -0500
Subject: [PATCH 231/365] TST: use coveralls app
---
.github/workflows/main.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 845985ebd..2f47b9ab2 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -68,5 +68,4 @@ jobs:
- name: Publish results to coveralls
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: coveralls --rcfile=pyproject.toml --service=github
- continue-on-error: true
+ uses: coverallsapp/github-action@v2
From 685648112fe2d7c67cd6baeeba9ba82f1abe40d4 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing
Date: Mon, 27 Nov 2023 13:40:51 -0500
Subject: [PATCH 232/365] MAINT: update coveralls
---
.github/workflows/main.yml | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 2f47b9ab2..8b814363a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -68,4 +68,17 @@ jobs:
- name: Publish results to coveralls
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: coverallsapp/github-action@v2
+ COVERALLS_PARALLEL: true
+ run: coveralls --rcfile=pyproject.toml --service=github
+
+ finish:
+ name: Finish Coverage Analysis
+ needs: build
+ runs-on: ubuntu-latest
+ steps:
+ - name: Coveralls Finished
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ pip install --upgrade coveralls
+ coveralls --service=github --finish
From 669186909864175191ceaf0c7b3efc609d0eec8a Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:45:29 -0500
Subject: [PATCH 233/365] DOC: updated issue forms
Updated the issue forms to be up to current standards.
---
.github/ISSUE_TEMPLATE/bug_report.md | 26 +++++++++++------------
.github/ISSUE_TEMPLATE/feature_request.md | 24 +++++++++++++++------
.github/ISSUE_TEMPLATE/question.md | 19 +++++++++++++++++
3 files changed, 49 insertions(+), 20 deletions(-)
create mode 100644 .github/ISSUE_TEMPLATE/question.md
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 2764ce0c5..c29622e00 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -1,29 +1,29 @@
---
name: Bug report
about: Create a report to help us improve
+labels: bug
+title: "BUG: "
---
-**Describe the bug**
-A clear and concise description of what the bug is.
+# Description
+A clear and concise description of what the bug is, including a description
+of what you expected the outcome to be.
-**To Reproduce**
+# To Reproduce this bug:
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
-**Expected behavior**
-A clear and concise description of what you expected to happen.
+Consider including images or test files to help others reproduce the bug and
+solve the problem.
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Version [e.g. 22]
+## Test configuration
+ - OS: [e.g., Hal]
+ - Version [e.g., Python 3.47]
- Other details about your setup that could be relevant
-**Additional context**
-Add any other context about the problem here.
+# Additional context
+Add any other context about the problem here, including expected behaviour.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index 066b2d920..d02da2ef4 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -1,17 +1,27 @@
---
name: Feature request
about: Suggest an idea for this project
+title: "ENH: "
+labels: enhancement
---
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+# Description
+A clear and concise description of the new feature or behaviour you would like.
-**Describe the solution you'd like**
+## Potential impact
+
+- Is the feature related to an existing problem?
+- How critical is this feature to your workflow?
+- How wide of an impact to you anticipate this enhancement having?
+- Would this break any existing functionality?
+
+## Potential solution(s)
A clear and concise description of what you want to happen.
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
+# Alternatives
+A clear description of any alternative solutions or features you've considered.
-**Additional context**
-Add any other context or screenshots about the feature request here.
+# Additional context
+Add any other context or screenshots about the feature request here, potentially
+including your operational configuration.
diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md
new file mode 100644
index 000000000..463725bae
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/question.md
@@ -0,0 +1,19 @@
+---
+name: Question
+about: A question about this project
+title: "QUEST: "
+labels: question
+
+---
+
+# Description
+A clear and concise summary of your query
+
+## Example code (optional)
+If relevant, include sample code, images, or files so that others can understand
+the full context of your question.
+
+## Configuration
+ - OS: [e.g., Hal]
+ - Version: [e.g., Python 3.47]
+ - Other details about your setup that could be relevant
From 9edbe901aa2996ec71d2bbb83b4cc391bfb90e5e Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:46:03 -0500
Subject: [PATCH 234/365] DOC: updated pull request template
Updated the pull request template to include all current check items.
---
.github/pull_request_template.md | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 692a10c51..75ee10d47 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,6 +1,6 @@
# Description
-Addresses # (issue)
+Addresses #(issue)
Please include a summary of the change and which issue is fixed. Please also
include relevant motivation and context. List any dependencies that are required
@@ -23,7 +23,10 @@ instructions so we can reproduce. Please also list any relevant details for
your test configuration
- Test A
-- Test B
+
+```
+Test B
+```
**Test Configuration**:
* Operating system: Hal
@@ -35,6 +38,7 @@ your test configuration
- [ ] Make sure you are merging into the ``develop`` (not ``main``) branch
- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my own code
+- [ ] I have linted the files updated in this pull request
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
@@ -42,6 +46,8 @@ your test configuration
- [ ] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged and published in downstream modules
- [ ] Add a note to ``CHANGELOG.md``, summarizing the changes
+- [ ] Update zenodo.json file for new code contributors
-If this is a release PR, replace the first item of the above checklist with the release
-checklist on the wiki: https://github.com/pysat/pysat/wiki/Checklist-for-Release
+If this is a release PR, replace the first item of the above checklist with the
+release checklist on the wiki:
+https://github.com/pysat/pysat/wiki/Checklist-for-Release
From 79c82cbcce370700477d06b76cdc8af10f100ada Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:46:50 -0500
Subject: [PATCH 235/365] TST: updated CI yamls
Updated the CI yamls to be better formatted and contain the new coveralls logic.
---
.github/workflows/docs.yml | 11 +++++------
.github/workflows/main.yml | 26 +++++++++++++++++++++-----
.github/workflows/pip_rc_install.yml | 7 ++++---
.github/workflows/stats.yml | 7 ++++---
4 files changed, 34 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 153cfc945..328ad1813 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -1,5 +1,6 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+# This workflow will install Python dependencies, run tests and lint with a
+# variety of Python versions. For more information see:
+# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Documentation Check
@@ -8,7 +9,7 @@ on: [push, pull_request]
jobs:
build:
- runs-on: "ubuntu-latest"
+ runs-on: ["ubuntu-latest"]
strategy:
fail-fast: false
matrix:
@@ -23,9 +24,7 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install .[doc]
+ run: pip install .[doc]
- name: Set up pysat
run: |
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 51489005d..e07d0779c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -1,5 +1,6 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+# This workflow will install Python dependencies, run tests and lint with a
+# variety of Python versions. For more information see:
+# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Pytest with Flake8
@@ -10,15 +11,17 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [ubuntu-latest, windows-latest]
+ os: ["ubuntu-latest", "windows-latest", "macos-latest"]
python-version: ["3.10", "3.11"]
numpy_ver: ["latest"]
test_config: ["latest"]
include:
+ # NEP29 compliance settings
- python-version: "3.9"
numpy_ver: "1.21"
os: ubuntu-latest
test_config: "NEP29"
+ # Operational compliance settings
- python-version: "3.6.8"
numpy_ver: "1.19.5"
os: "ubuntu-20.04"
@@ -36,7 +39,7 @@ jobs:
- name: Install Operational dependencies
if: ${{ matrix.test_config == 'Ops'}}
run: |
- pip install --no-cache-dir numpy==${{ matrix.numpy_ver }}
+ pip install numpy==${{ matrix.numpy_ver }}
pip install -r requirements.txt
pip install -r test_requirements.txt
pip install .
@@ -63,9 +66,22 @@ jobs:
run: flake8 . --count --exit-zero --max-complexity=10 --statistics
- name: Test with pytest
- run: pytest --cov=pysat/
+ run: pytest
- name: Publish results to coveralls
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ COVERALLS_PARALLEL: true
run: coveralls --rcfile=pyproject.toml --service=github
+
+ finish:
+ name: Finish Coverage Analysis
+ needs: build
+ runs-on: ubuntu-latest
+ steps:
+ - name: Coveralls Finished
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ pip install --upgrade coveralls
+ coveralls --service=github --finish
diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml
index c782985e0..0611a98b7 100644
--- a/.github/workflows/pip_rc_install.yml
+++ b/.github/workflows/pip_rc_install.yml
@@ -1,6 +1,7 @@
-# This workflow will install Python dependencies and the latest RC of pysatCDAAC from test pypi.
-# This test should be manually run before a pysatCDAAC RC is officially approved and versioned.
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+# This workflow will install Python dependencies and the latest RC of pysat from
+# test pypi. This test should be manually run before a pysat RC is officially
+# approved and versioned. For more information see:
+# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Test install of latest RC from pip
diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml
index f22ad9ab3..eed4565a7 100644
--- a/.github/workflows/stats.yml
+++ b/.github/workflows/stats.yml
@@ -1,5 +1,6 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+# This workflow will install Python dependencies, run tests and lint with a
+# variety of Python versions. For more information see:
+# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Statistics of supported instruments
@@ -9,7 +10,7 @@ on:
jobs:
build:
- runs-on: ubuntu-latest
+ runs-on: ["ubuntu-latest"]
strategy:
fail-fast: false
matrix:
From 1c229a36c79f5524ab79c8fabb9030f1387b5786 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:47:17 -0500
Subject: [PATCH 236/365] DOC: updated code of conduct
Updated code of conduct to comply with style standards.
---
CODE_OF_CONDUCT.md | 62 ++++++++++++++++++++++++++++++++++++----------
1 file changed, 49 insertions(+), 13 deletions(-)
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index bb9195cc8..b6d84d819 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -2,11 +2,17 @@
## Our Pledge
-In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age,
+body size, disability, ethnicity, gender identity and expression, level of
+experience, nationality, personal appearance, race, religion, or sexual
+identity and orientation.
## Our Standards
-Examples of behavior that contributes to creating a positive environment include:
+Examples of behavior that contributes to creating a positive environment
+include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
@@ -16,31 +22,61 @@ Examples of behavior that contributes to creating a positive environment include
Examples of unacceptable behavior by participants include:
-* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
-* Publishing others' private information, such as a physical or electronic address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a professional setting
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
## Our Responsibilities
-Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
-Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
## Scope
-This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an
+appointed representative at an online or offline event. Representation of a
+project may be further defined and clarified by project maintainers.
## Enforcement
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at pysat.developers@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at pysat.developers@gmail.com. The
+pysat project team will review and investigate all complaints, and will respond
+in a way that it deems appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an
+incident. Further details of specific enforcement policies may be posted
+separately.
-Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
## Attribution
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.4, available at
+[https://contributor-covenant.org/version/1/4][version]
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/
+## FAQ
+
+For answers to common questions about this code of conduct, see
+[https://www.contributor-covenant.org/faq][faq]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/
+[faq]: https://www.contributor-covenant.org/faq
From f01438721fb7495618e36605feec008db9855dd2 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:47:40 -0500
Subject: [PATCH 237/365] DOC: updated contributing guidelines
Updated contributing guidelines to include better formatting and recent updates to style.
---
CONTRIBUTING.md | 89 +++++++++++++++++++++++++++++++++----------------
1 file changed, 60 insertions(+), 29 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4c1a95415..77be3f0fa 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -13,13 +13,27 @@ are generally held fortnightly.
Short version
-------------
-* Submit bug reports and feature requests at
+* Submit bug reports, feature requests, and questions at
[GitHub](https://github.com/pysat/pysat/issues)
* Make pull requests to the ``develop`` branch
+Issues
+------
+
+Bug reports, questions, and feature requests should all be made as GitHub
+Issues. Templates are provided for each type of issue, to help you include
+all the necessary information.
+
+Questions
+^^^^^^^^^
+
+Not sure how something works? Ask away! The more information you provide, the
+easier the question will be to answer. You can also interact with the pysat
+developers on our [slack channel](https://pysat.slack.com).
+
Bug reports
------------
+^^^^^^^^^^^
When [reporting a bug](https://github.com/pysat/pysat/issues) please
include:
@@ -31,12 +45,12 @@ include:
* Detailed steps to reproduce the bug
Feature requests and feedback
------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-The best way to send feedback is to file an issue at
-[GitHub](https://github.com/pysat/pysat/issues).
+The best way to send feedback is to file an
+[issue](https://github.com/pysat/pysat/issues).
-If you are proposing a feature:
+If you are proposing a new feature or a change in something that already exists:
* Explain in detail how it would work.
@@ -66,42 +80,57 @@ To set up `pysat` for local development:
Now you can make your changes locally.
- Tests for new instruments are performed automatically. Tests for custom
- functions should be added to the appropriately named file in ``pysat/tests``.
- For example, custom functions for the time utilities are tested in
- ``pysat/tests/test_utils_time.py``. If no test file exists, then you should
- create one. This testing uses pytest, which will run tests on any python
- file in the test directory that starts with ``test``. Classes must begin
- with ``Test``, and methods must begin with ``test`` as well.
+ Tests for new instruments are performed automatically. See discussion
+ [here](https://pysat.readthedocs.io/en/main/new_instrument.html#testing-support)
+ for more information on triggering these standard tests.
+
+ Tests for custom functions should be added to the appropriately named file
+ in ``pysat/tests``. For example, custom functions for the time utilities are
+ tested in ``pysat/tests/test_utils_time.py``. If no test file exists, then
+ you should create one. This testing uses pytest, which will run tests on
+ any Python file in the test directory that starts with ``test``. Classes
+ must begin with ``Test``, and methods must begin with ``test`` as well.
4. When you're done making changes, run all the checks to ensure that nothing
is broken on your local system, as well as check for flake8 compliance:
```
- pytest -vs --flake8 pysat
+ pytest
```
-5. Update/add documentation (in ``docs``), if relevant
+5. You should also check for flake8 style compliance:
+
+ ```
+ flake8 . --count --select=D,E,F,H,W --show-source --statistics
+ ```
+
+ Note that pysat uses the `flake-docstrings` and `hacking` packages to ensure
+ standards in docstring formatting.
+
+6. Update/add documentation (in ``docs``). Even if you don't think it's
+ relevant, check to see if any existing examples have changed.
-6. Add your name to the .zenodo.json file as an author
+7. Add your name to the .zenodo.json file as an author
-7. Commit your changes:
+8. Commit your changes:
```
git add .
git commit -m "AAA: Brief description of your changes"
```
- Where AAA is a standard shorthand for the type of change (eg, BUG or DOC).
+ Where AAA is a standard shorthand for the type of change (e.g., BUG or DOC).
`pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html),
see the discussion there for a full list of this shorthand notation.
-8. Once you are happy with the local changes, push to Github:
+9. Once you are happy with the local changes, push to GitHub:
```
git push origin name-of-your-bugfix-or-feature
```
Note that each push will trigger the Continuous Integration workflow.
-9. Submit a pull request through the GitHub website. Pull requests should be
- made to the ``develop`` branch.
+10. Submit a pull request through the GitHub website. Pull requests should be
+ made to the ``develop`` branch. Note that automated tests will be run on
+ GitHub Actions, but these must be initialized by a member of the pysat team
+ for first time contributors.
Pull Request Guidelines
@@ -114,14 +143,15 @@ For merging, you should:
1. Include an example for use
2. Add a note to ``CHANGELOG.md`` about the changes
-3. Update the author list in ``zenodo.json`` if applicable
-4. Ensure that all checks passed (current checks include Github Actions and Coveralls)
+3. Update the author list in ``zenodo.json``, if applicable
+4. Ensure that all checks passed (current checks include GitHub Actions,
+ Coveralls and ReadTheDocs)
-If you don't have all the necessary Python versions available locally or
-have trouble building all the testing environments, you can rely on
-GitHub Actions to run the tests for each change you add in the pull
-request. Because testing here will delay tests by other developers,
-please ensure that the code passes all tests on your local system first.
+If you don't have all the necessary Python versions available locally or have
+trouble building all the testing environments, you can rely on GitHub Actions to
+run the tests for each change you add in the pull request. Because testing here
+will delay tests by other developers, please ensure that the code passes all
+tests on your local system first.
Project Style Guidelines
@@ -151,7 +181,8 @@ These include:
* All classes should have `__repr__` and `__str__` functions
* Docstrings use `Note` instead of `Notes`
* Try to avoid creating a try/except statement where except passes
-* Use setup and teardown in test classes
+* Use setup_method (or setup_class) and teardown_method (or teardown_class) in
+ test classes
* Use pytest parametrize in test classes when appropriate
* Use pysat testing utilities when appropriate
* Provide testing class methods with informative failure statements and
From 621b6474ddf0f5215ce7d4ac324646b6f19ea584 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:48:39 -0500
Subject: [PATCH 238/365] DOC: updated readme
Updated readme to include better installation instructions.
---
README.md | 72 ++++++++++++++++++++++++++++++-------------------------
1 file changed, 39 insertions(+), 33 deletions(-)
diff --git a/README.md b/README.md
index 8db411201..8e1f3c441 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,8 @@ JGR-Space Physics [Publication](https://doi.org/10.1029/2018JA025297)
[Citation Info](https://pysat.readthedocs.io/en/latest/citing.html)
Come join us on Slack! An invitation to the pysat workspace is available
-in the 'About' section of the [pysat GitHub Repository.](https://github.com/pysat/pysat)
+in the 'About' section of the
+[pysat GitHub Repository.](https://github.com/pysat/pysat)
Development meetings are generally held fortnightly.
# Main Features
@@ -49,47 +50,52 @@ Development meetings are generally held fortnightly.
instruments to pysat
# Installation
-## Starting from scratch
-* Python and associated packages for science are freely available. Convenient
- science python package setups are available from https://www.python.org/,
- [Anaconda](https://www.anaconda.com/distribution/), and other locations
- (some platform specific). Anaconda also includes a developer environment that
- works well with pysat. Core science packages such as numpy, scipy, matplotlib,
- pandas and many others may also be installed directly via pip or your
- favorite package manager.
-
-* Installation through pip
+
+The following instructions provide a guide for installing pysat and give some
+examples on how to use the routines.
+
+## Prerequisites
+
+pysat uses common Python modules, as well as modules developed by and for the
+Space Physics community. This module officially supports Python 3.X+.
+
+| Common modules | Community modules |
+| -------------- | ----------------- |
+| dask | netCDF4 |
+| numpy >= 1.12 | |
+| pandas < 2.1.1 | |
+| portalocker | |
+| toolz | |
+| xarray | |
+
+
+## PyPi Installation
```
pip install pysat
```
-* Installation through github
+
+## GitHub Installation
```
git clone https://github.com/pysat/pysat.git
-cd pysat
-pip install .
```
-An advantage to installing through github is access to the development branches.
-The latest bugfixes can be found in the `develop` branch. However, this branch
-is not stable (as the name implies). We recommend using this branch in a
-virtual environment or using `python setup.py develop`.
+
+Change directories into the repository folder and run the pyproject.toml or
+setup.py file. For a local install use the "--user" flag after "install".
+
```
-git clone https://github.com/pysat/pysat.git
-cd pysat
-git checkout develop
-pip install -e .
+cd pysat/
+python -m build .
+pip install .
```
-* Note that pysat requires a number of packages for the install.
- * dask
- * netCDF4
- * numpy
- * pandas
- * portalocker
- * scipy
- * toolz
- * xarray
-* The first time the package is run, you will need to specify a directory to
- store data. In python, run:
+
+# Using pysat
+
+* The first time pysat is run, you will need to specify a directory to store
+ the data. In Python, run:
```
pysat.params['data_dirs'] = 'path/to/directory/that/may/or/may/not/exist'
```
* Nominal organization of data is top_dir/platform/name/tag/inst_id/files
+
+Detailed examples and tutorials for using pysat are available in the
+[documentation](http://pysat.readthedocs.io/en/latest/index.html).
From be88ed7ca69ba56f76177649a94659b35a8f7119 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:49:07 -0500
Subject: [PATCH 239/365] MAINT: removed old citation
Removed the old citation file.
---
pysat/citation.txt | 1 -
1 file changed, 1 deletion(-)
delete mode 100644 pysat/citation.txt
diff --git a/pysat/citation.txt b/pysat/citation.txt
deleted file mode 100644
index 2a546499a..000000000
--- a/pysat/citation.txt
+++ /dev/null
@@ -1 +0,0 @@
-Stoneback, Russell, et al. (2021). pysat/pysat v3.0 (Version v3.0). Zenodo. http://doi.org/10.5281/zenodo.1199703
\ No newline at end of file
From 3c8a1dd40ba74585e73d5628fe82aca83fa2bbd1 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:49:39 -0500
Subject: [PATCH 240/365] BUG: fixed metadata implementation
Fixed the metadata implementation and used the resources package where possible.
---
pysat/__init__.py | 32 ++++++++++++++++++--------------
1 file changed, 18 insertions(+), 14 deletions(-)
diff --git a/pysat/__init__.py b/pysat/__init__.py
index f8a110a90..1fb154efc 100644
--- a/pysat/__init__.py
+++ b/pysat/__init__.py
@@ -34,7 +34,12 @@
"""
-import importlib
+try:
+ from importlib import metadata
+ from importlib import resources
+except ImportError:
+ import importlib_metadata as metadata
+ resources = None
import logging
import os
@@ -50,12 +55,7 @@
from pysat import _params
# Set version
-try:
- __version__ = importlib.metadata.version('pysat')
-except AttributeError:
- # Python 3.6 requires a different version
- import importlib_metadata
- __version__ = importlib_metadata.version('pysat')
+__version__ = metadata.version('pysat')
# Get home directory
home_dir = os.path.expanduser('~')
@@ -64,15 +64,19 @@
pysat_dir = os.path.join(home_dir, '.pysat')
# Set directory for test data
-here = os.path.abspath(os.path.dirname(__file__))
-test_data_path = os.path.join(here, 'tests', 'test_data')
+if resources is None:
+ test_data_path = os.path.join(os.path.realpath(os.path.dirname(__file__)),
+ 'tests', 'test_data')
+else:
+ test_data_path = str(resources.files(__package__).joinpath('tests',
+ 'test_data'))
# Create a .pysat directory or parameters file if one doesn't exist.
# pysat_settings did not exist pre v3 thus this provides a check against
# v2 users that are upgrading. Those users need the settings file plus
# new internal directories.
-if not os.path.isdir(pysat_dir) or \
- (not os.path.isfile(os.path.join(pysat_dir, 'pysat_settings.json'))):
+settings_file = os.path.join(pysat_dir, 'pysat_settings.json')
+if not os.path.isdir(pysat_dir) or not os.path.isfile(settings_file):
# Make a .pysat directory if not already present
if not os.path.isdir(pysat_dir):
@@ -89,7 +93,7 @@
os.mkdir(os.path.join(pysat_dir, 'instruments', 'archive'))
# Create parameters file
- if not os.path.isfile(os.path.join(pysat_dir, 'pysat_settings.json')):
+ if not os.path.isfile(settings_file):
params = _params.Parameters(path=pysat_dir, create_new=True)
print(''.join(("\nHi there! pysat will nominally store data in a ",
@@ -116,5 +120,5 @@
from pysat._constellation import Constellation
__all__ = ['instruments', 'utils']
-# Cleanup
-del here
+# Clean up
+del settings_file, resources
From 0a000ab77ace80f1b0dca8b6ff2a9d547bb2b10f Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:50:06 -0500
Subject: [PATCH 241/365] MAINT: updated manifest
Updated the manifest to not include old files.
---
MANIFEST.in | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/MANIFEST.in b/MANIFEST.in
index 7c416b089..2073a9352 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,9 +1,8 @@
-global-include *.py
+recursive-include pysat *.py
include *.md
include *.txt
include LICENSE
-include pysat/version.txt
-include pysat/citation.txt
+prune pysat/tests
prune docs
global-exclude *.pdf
global-exclude *.png
From 707c53b76fb0e523e3a03914a5f5118e74d99eca Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:50:54 -0500
Subject: [PATCH 242/365] MAINT: updated requirements
Updated requirements to:
- not include scipy (not imported anywhere), and
- move pytest to the test installation.
---
docs/installation.rst | 4 ++--
pyproject.toml | 2 --
requirements.txt | 2 --
test_requirements.txt | 1 +
4 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/docs/installation.rst b/docs/installation.rst
index 24104dfae..54db75a2e 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -42,8 +42,6 @@ needed by the installer:
#. numpy
#. pandas
#. portalocker
-#. pytest
-#. scipy
#. toolz
#. xarray
@@ -57,6 +55,7 @@ pysat may also be installed directly from the source repository on github::
git clone https://github.com/pysat/pysat.git
cd pysat
+ python -m build .
pip install --user .
An advantage to installing through github is access to the development branches.
@@ -67,6 +66,7 @@ virtual environment and using::
git clone https://github.com/pysat/pysat.git
cd pysat
git checkout develop
+ python -m build .
pip install -e .
The use of `-e` in the setup command installs the code 'in-place', so any
diff --git a/pyproject.toml b/pyproject.toml
index eb0dbe732..24249506c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -47,8 +47,6 @@ dependencies = [
"numpy >= 1.12",
"pandas < 2.1.1",
"portalocker",
- "pytest",
- "scipy",
"toolz",
"xarray"
]
diff --git a/requirements.txt b/requirements.txt
index 059c99271..165a2698e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,5 @@ netCDF4
numpy>=1.12
pandas<2.1.1
portalocker
-pytest
-scipy
toolz
xarray
diff --git a/test_requirements.txt b/test_requirements.txt
index 8a33141b8..cfbe27e5d 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -6,6 +6,7 @@ ipython
m2r2
numpydoc
pysatSpaceWeather
+pytest
pytest-cov
pytest-ordering
sphinx
From 76ddeed750af6fbd3615f1f168d5068a3b240b0b Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:52:11 -0500
Subject: [PATCH 243/365] MAINT: address #988 TODO
Address a TODO for closed issue #988, which should fix updating the data without separate calls for xarray or pandas type.
---
pysat/utils/coords.py | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/pysat/utils/coords.py b/pysat/utils/coords.py
index 504dc3ff8..8c5a142ff 100644
--- a/pysat/utils/coords.py
+++ b/pysat/utils/coords.py
@@ -66,12 +66,7 @@ def update_longitude(inst, lon_name=None, high=180.0, low=-180.0):
raise ValueError('unknown longitude variable name')
new_lon = adjust_cyclic_data(inst[lon_name], high=high, low=low)
-
- # TODO(#988): Remove pandas/xarray logic after fixing issue in Instrument
- if inst.pandas_format:
- inst[lon_name] = new_lon
- else:
- inst.data = inst.data.update({lon_name: (inst[lon_name].dims, new_lon)})
+ inst[lon_name] = new_lon
return
From 48f23ea596f1b1c9d21d02c628befe1316081be7 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 14:56:45 -0500
Subject: [PATCH 244/365] Revert "MAINT: removed old citation"
This reverts commit be88ed7ca69ba56f76177649a94659b35a8f7119.
---
pysat/citation.txt | 1 +
1 file changed, 1 insertion(+)
create mode 100644 pysat/citation.txt
diff --git a/pysat/citation.txt b/pysat/citation.txt
new file mode 100644
index 000000000..2a546499a
--- /dev/null
+++ b/pysat/citation.txt
@@ -0,0 +1 @@
+Stoneback, Russell, et al. (2021). pysat/pysat v3.0 (Version v3.0). Zenodo. http://doi.org/10.5281/zenodo.1199703
\ No newline at end of file
From a54e439b4c137e09d610dfe1af24ad4d826edc86 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 15:00:52 -0500
Subject: [PATCH 245/365] MAINT: updated citation
Updated the citation version and year.
---
pysat/citation.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pysat/citation.txt b/pysat/citation.txt
index 2a546499a..d6f09ccc0 100644
--- a/pysat/citation.txt
+++ b/pysat/citation.txt
@@ -1 +1 @@
-Stoneback, Russell, et al. (2021). pysat/pysat v3.0 (Version v3.0). Zenodo. http://doi.org/10.5281/zenodo.1199703
\ No newline at end of file
+Stoneback, Russell, et al. (2023). pysat/pysat v3.1 (Version v3.1). Zenodo. http://doi.org/10.5281/zenodo.1199703
From 5b48a45f2f8bb6e9b097937e6db093c5bdaaef11 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 15:01:20 -0500
Subject: [PATCH 246/365] STY: used resouces to access citation file
Use the resources module to access the citation file, if possible.
---
pysat/__init__.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/pysat/__init__.py b/pysat/__init__.py
index 1fb154efc..beb1da02d 100644
--- a/pysat/__init__.py
+++ b/pysat/__init__.py
@@ -67,9 +67,12 @@
if resources is None:
test_data_path = os.path.join(os.path.realpath(os.path.dirname(__file__)),
'tests', 'test_data')
+ citation = os.path.join(os.path.realpath(os.path.dirname(__file__)),
+ 'citation.txt')
else:
test_data_path = str(resources.files(__package__).joinpath('tests',
'test_data'))
+ citation = str(resources.files(__package__).joinpath('citation.txt'))
# Create a .pysat directory or parameters file if one doesn't exist.
# pysat_settings did not exist pre v3 thus this provides a check against
From 4c8926f24c97579c026ac2b8023dc701554f7089 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 15:01:42 -0500
Subject: [PATCH 247/365] MAINT: use new module attribute
Use the new module citation attribute.
---
pysat/instruments/methods/testing.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/pysat/instruments/methods/testing.py b/pysat/instruments/methods/testing.py
index 3f02ff71a..04154d2d6 100644
--- a/pysat/instruments/methods/testing.py
+++ b/pysat/instruments/methods/testing.py
@@ -17,8 +17,7 @@
"https://www.github.com/pysat/pysat"))
# Load up citation information
-with pysat.utils.NetworkLock(os.path.join(pysat.here, 'citation.txt'), 'r') as \
- locked_file:
+with pysat.utils.NetworkLock(pysat.citation, 'r') as locked_file:
refs = locked_file.read()
From 477d8043a366901be77a640c3cfca2677c006cc2 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 15:22:36 -0500
Subject: [PATCH 248/365] REV: re-add citation to manifest
Re-add the citation text file to the manifest.
---
MANIFEST.in | 1 +
1 file changed, 1 insertion(+)
diff --git a/MANIFEST.in b/MANIFEST.in
index 2073a9352..31a3eaf4d 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,6 +2,7 @@ recursive-include pysat *.py
include *.md
include *.txt
include LICENSE
+include pysat/citation.txt
prune pysat/tests
prune docs
global-exclude *.pdf
From 52f9c2a19dabe4eb6fcaa93582e288f1528e1c3d Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 15:33:37 -0500
Subject: [PATCH 249/365] REV: re-added pytest requirement
Re-added the pytest requirement to the main code.
---
README.md | 1 +
docs/installation.rst | 1 +
pyproject.toml | 2 +-
requirements.txt | 1 +
test_requirements.txt | 1 -
5 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index 8e1f3c441..1df45d651 100644
--- a/README.md
+++ b/README.md
@@ -65,6 +65,7 @@ Space Physics community. This module officially supports Python 3.X+.
| numpy >= 1.12 | |
| pandas < 2.1.1 | |
| portalocker | |
+| pytest | |
| toolz | |
| xarray | |
diff --git a/docs/installation.rst b/docs/installation.rst
index 54db75a2e..3d11e03e9 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -42,6 +42,7 @@ needed by the installer:
#. numpy
#. pandas
#. portalocker
+#. pytest
#. toolz
#. xarray
diff --git a/pyproject.toml b/pyproject.toml
index 24249506c..6d24d88e4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -47,6 +47,7 @@ dependencies = [
"numpy >= 1.12",
"pandas < 2.1.1",
"portalocker",
+ "pytest",
"toolz",
"xarray"
]
@@ -58,7 +59,6 @@ test = [
"flake8-docstrings",
"hacking >= 1.0",
"pysatSpaceWeather",
- "pytest",
"pytest-cov",
"pytest-ordering"
]
diff --git a/requirements.txt b/requirements.txt
index 165a2698e..cec8a3bfa 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,5 +3,6 @@ netCDF4
numpy>=1.12
pandas<2.1.1
portalocker
+pytest
toolz
xarray
diff --git a/test_requirements.txt b/test_requirements.txt
index cfbe27e5d..8a33141b8 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -6,7 +6,6 @@ ipython
m2r2
numpydoc
pysatSpaceWeather
-pytest
pytest-cov
pytest-ordering
sphinx
From 8291b9f0b496efbf3f7a5f81495efd8b8273e78e Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 15:55:35 -0500
Subject: [PATCH 250/365] BUG: fixed `__setitem__` xarray assignment
Fixed a bug in the xarray setitem magic method that did not account for native dimensions in the data when assigning arrays as values.
---
pysat/_instrument.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index 1caf0d199..f3e5bf5fa 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -1106,8 +1106,13 @@ def __setitem__(self, key, new_data):
self.data[key] = in_data
elif len(np.shape(in_data)) <= 1:
# If not an xarray input, but still iterable, then we
- # go through to process the 1D input
- if np.shape(in_data) == np.shape(self.index):
+ # go through to process the input
+ if np.shape(in_data) == np.shape(self.data[key]):
+ # The ND input has the same shape as the current data
+ # and can be assigned directly without adjusting the
+ # dimensions
+ self.data[key] = (self.data[key].dims, in_data)
+ elif np.shape(in_data) == np.shape(self.index):
# 1D input has the correct length for storage along
# 'Epoch'.
self.data[key] = (epoch_name, in_data)
From 4771259384fb6c6e6e6e54298c0d0d789cf8148b Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 17:02:14 -0500
Subject: [PATCH 251/365] REV: re-added scipy
Found where scipy was needed, re-adding dependency.
---
README.md | 1 +
docs/installation.rst | 1 +
pyproject.toml | 1 +
requirements.txt | 1 +
4 files changed, 4 insertions(+)
diff --git a/README.md b/README.md
index 1df45d651..eef03facd 100644
--- a/README.md
+++ b/README.md
@@ -66,6 +66,7 @@ Space Physics community. This module officially supports Python 3.X+.
| pandas < 2.1.1 | |
| portalocker | |
| pytest | |
+| scipy | |
| toolz | |
| xarray | |
diff --git a/docs/installation.rst b/docs/installation.rst
index 3d11e03e9..f8013c5a8 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -43,6 +43,7 @@ needed by the installer:
#. pandas
#. portalocker
#. pytest
+#. scipy
#. toolz
#. xarray
diff --git a/pyproject.toml b/pyproject.toml
index 6d24d88e4..3b47b33d3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,6 +48,7 @@ dependencies = [
"pandas < 2.1.1",
"portalocker",
"pytest",
+ "scipy",
"toolz",
"xarray"
]
diff --git a/requirements.txt b/requirements.txt
index cec8a3bfa..059c99271 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,5 +4,6 @@ numpy>=1.12
pandas<2.1.1
portalocker
pytest
+scipy
toolz
xarray
From 90cc089d8c0f8652195557564040aabf83dbdd39 Mon Sep 17 00:00:00 2001
From: "Angeline G. Burrell"
Date: Mon, 27 Nov 2023 17:07:56 -0500
Subject: [PATCH 252/365] BUG: fixed new data bug
Fixed a bug in assigning new xarray data.
---
pysat/_instrument.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pysat/_instrument.py b/pysat/_instrument.py
index f3e5bf5fa..959a98d6d 100644
--- a/pysat/_instrument.py
+++ b/pysat/_instrument.py
@@ -1107,10 +1107,11 @@ def __setitem__(self, key, new_data):
elif len(np.shape(in_data)) <= 1:
# If not an xarray input, but still iterable, then we
# go through to process the input
- if np.shape(in_data) == np.shape(self.data[key]):
+ if key in self.variables and (
+ np.shape(in_data) == np.shape(self.data[key])):
# The ND input has the same shape as the current data
# and can be assigned directly without adjusting the
- # dimensions
+ # dimensions. Only works with existing data.
self.data[key] = (self.data[key].dims, in_data)
elif np.shape(in_data) == np.shape(self.index):
# 1D input has the correct length for storage along
From 6f52301aea932d326415a6958b23a6de0f7edbc6 Mon Sep 17 00:00:00 2001
From: Jeff Klenzing