From 8875a0fc7f168dc0ea7a1626e0be801d4ec477f1 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 2 Mar 2023 16:47:15 -0500 Subject: [PATCH 001/139] BUG: fixed download stop time Fixed the definition of the download stop time, not assuming it has a daily cadence. --- pysatMadrigal/instruments/methods/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index bc26563..cd1d21a 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -883,7 +883,7 @@ def download(date_array, inst_code=None, kindat=None, data_path=None, start = date_array.min() stop = date_array.max() if start == stop: - stop += dt.timedelta(days=1) + stop = date_array.shift().max() # Initialize the connection to Madrigal logger.info('Connecting to Madrigal') From b9582f9795824b893f838f8d44a3fe96cdccff2d Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 2 Mar 2023 16:47:44 -0500 Subject: [PATCH 002/139] ENH: added DMSP SSJ instrument Added an instrument for the DMSP SSJ auroral boundary data. --- pysatMadrigal/instruments/__init__.py | 3 +- pysatMadrigal/instruments/dmsp_ssj.py | 270 ++++++++++++++++++++++ pysatMadrigal/instruments/methods/dmsp.py | 15 +- 3 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 pysatMadrigal/instruments/dmsp_ssj.py diff --git a/pysatMadrigal/instruments/__init__.py b/pysatMadrigal/instruments/__init__.py index 3a8698b..a4bf71a 100644 --- a/pysatMadrigal/instruments/__init__.py +++ b/pysatMadrigal/instruments/__init__.py @@ -1,5 +1,6 @@ # Import Madrigal instruments from pysatMadrigal.instruments import dmsp_ivm +from pysatMadrigal.instruments import dmsp_ssj from pysatMadrigal.instruments import gnss_tec from pysatMadrigal.instruments import jro_isr from pysatMadrigal.instruments import madrigal_pandas @@ -8,4 +9,4 @@ from pysatMadrigal.instruments import methods # noqa F401 # Define variable name with all available instruments -__all__ = ['dmsp_ivm', 'gnss_tec', 'jro_isr', 'madrigal_pandas'] +__all__ = ['dmsp_ivm', 'dmsp_ssj', 'gnss_tec', 'jro_isr', 'madrigal_pandas'] diff --git a/pysatMadrigal/instruments/dmsp_ssj.py b/pysatMadrigal/instruments/dmsp_ssj.py new file mode 100644 index 0000000..d73f883 --- /dev/null +++ b/pysatMadrigal/instruments/dmsp_ssj.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# ---------------------------------------------------------------------------- +# -*- coding: utf-8 -*- +"""Supports the Special Sensor-J (SSJ) instrument and derived products from the +Defense Meteorological Satellite Program (DMSP). + +The SSJ measures precipitating particles using spectrometery. The Auroral +Boundary Index (ABI) is automatically computed from this data set and marks the +midnight equatorward boundary in each hemisphere. + +Further questions can be addressed to: + Gordon Wilson + Air Force Research Lab + RVBXP + 3550 Aberdeen Avenue SE, Bldg 570 + Kirtland Air Force Base, NM 87117-5776 + Phone: 505-853-2027 + e-mail: gordon.wilson@kirtland.af.mil +or + Ernie Holeman (ernestholeman7408@comcast.net) + +Please send a copy of all publications that use the Auroral Boundary Index +(ABI) to Dr. Gordon Wilson at the above address. + + +Properties +---------- +platform + 'dmsp' +name + 'ssj' +tag + 'abi' +inst_id + 'f11' + +Example +------- +:: + + import pysat + dmsp = pysat.Instrument('dmsp', 'ssj', 'abi', clean_level='clean') + dmsp.download(dt.datetime(2017, 12, 30), dt.datetime(2017, 12, 31), + user='Firstname+Lastname', password='email@address.com') + dmsp.load(2017, 363) + +Note +---- +Please provide name and email when downloading data with this routine. + +""" + +import datetime as dt +import functools +import numpy as np +import pandas as pds +import warnings + +from pysat import logger +from pysat.utils.time import create_date_range + +from pysatMadrigal.instruments.methods import general, dmsp + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'dmsp' +name = 'ssj' +tags = {'abi': 'Midnight Auroral Boundary Index'} +inst_ids = {'': [tag for tag in tags.keys()]} + +pandas_format = True + +# Madrigal tags +madrigal_inst_code = 180 +madrigal_tag = {'': {'abi': '17110'}} + +# Local attributes +dmsp_fname = general.madrigal_file_format_str(madrigal_inst_code) +supported_tags = {inst_id: {tag: dmsp_fname for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} +remote_tags = { + inst_id: {tag: supported_tags[inst_id][tag].format(file_type='hdf5') + for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'abi': dt.datetime(1982, 12, 30)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +def init(self): + """Initialize the Instrument object with values specific to DMSP IVM.""" + + self.acknowledgements = ''.join([ + general.cedar_rules(), '\nThe Air Force Research Laboratory Auroral ', + 'Boundary Index (ABI) was provided by the United States Air Force ', + 'Research Laboratory, Kirtland Air Force Base, New Mexico.']) + + logger.info(self.acknowledgements) + self.references = dmsp.references(self.name) + return + + +def clean(self): + """Clean DMSP IVM data cleaned to the specified level. + + Note + ---- + Supports 'clean' and 'dusty' + + 'clean' QC == 1 + 'dusty' QC <= 2 + 'dirty' and 'none' allow all QC flags (QC <= 3) + + When called directly by pysat, a clean level of 'none' causes pysat to skip + this routine. + + Warnings + -------- + UserWarning + If the 'dirty' level is invoked (same as no cleaning) + + """ + + if self.clean_level == 'clean': + iclean, = np.where(self['EQB_QC_FL'] <= 1) + elif self.clean_level == 'dusty': + iclean, = np.where(self['EQB_QC_FL'] <= 2) + else: + warnings.warn('No quality control level "dirty", using "none"') + iclean = None + + # Downselect data based upon cleaning conditions above + if iclean is not None: + self.data = self[iclean] + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default Madrigal and pysat methods + +# Support listing the local files +list_files = functools.partial(general.list_files, + supported_tags=supported_tags, + file_cadence=pds.DateOffset(years=1)) + +# Set the list_remote_files routine +list_remote_files = functools.partial(general.list_remote_files, + inst_code=madrigal_inst_code, + kindats=madrigal_tag, + supported_tags=remote_tags) + + +# Set the load routine +def load(fnames, tag='', inst_id=''): + """Loads data from Madrigal after accounting for date tags. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + tag : str + Tag name used to identify particular data set to be loaded. This input + is nominally provided by pysat itself and is not used here. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself, and is not used here. + (default='') + + Returns + ------- + data : pds.DataFrame or xr.Dataset + A pandas DataFrame or xarray Dataset holding the data from the file + meta : pysat.Meta + Metadata from the file, as well as default values from pysat + + Raises + ------ + ValueError + If data columns expected to create the time index are missing or if + coordinates are not supplied for all data columns. + + Note + ---- + Currently HDF5 reading breaks if a different file type was used previously + + This routine is called as needed by pysat. It is not intended + for direct user interaction. + + """ + # The ABI data has a yearly cadance, extract the unique filenames to load + load_fnames = list() + file_dates = list() + for fname in fnames: + file_dates.append(dt.datetime.strptime(fname[-10:], '%Y-%m-%d')) + if fname[0:-11] not in load_fnames: + load_fnames.append(fname[0:-11]) + + # Load the data and metadata + data, meta = general.load(load_fnames, tag=tag, inst_id=inst_id) + + # If there is a date range, downselect here + if len(file_dates) > 0: + idx, = np.where((data.index >= min(file_dates)) + & (data.index < max(file_dates) + dt.timedelta(days=1))) + data = data.iloc[idx, :] + + return data, meta + + +def download(date_array, tag='', inst_id='', data_path=None, user=None, + password=None, file_type='hdf5'): + """Downloads data from Madrigal. + + Parameters + ---------- + date_array : array-like + list of datetimes to download data for. The sequence of dates need not + be contiguous. + tag : str + Tag identifier used for particular dataset. This input is provided by + pysat. (default='') + inst_id : str + Satellite ID string identifier used for particular dataset. This input + is provided by pysat. (default='') + data_path : str + Path to directory to download data to. (default=None) + user : str + User string input used for download. Provided by user and passed via + pysat. If an account is required for downloads this routine here must + error if user not supplied. (default=None) + password : str + Password for data download. (default=None) + file_type : str + File format for Madrigal data. (default='hdf5') + + Note + ---- + The user's names should be provided in field user. Ritu Karidhal should + be entered as Ritu+Karidhal + + The password field should be the user's email address. These parameters + are passed to Madrigal when downloading. + + The affiliation field is set to pysat to enable tracking of pysat + downloads. + + """ + + # Ensure the date range is correct + if date_array.freq not in ['AS-JAN', 'YS', 'AS']: + date_array = create_date_range( + dt.datetime(date_array[0].year, 1, 1), date_array[-1], freq='YS') + + # Download the remote files + general.download(date_array, inst_code=str(madrigal_inst_code), + kindat=madrigal_tag[inst_id][tag], data_path=data_path, + user=user, password=password, file_type=file_type) + return diff --git a/pysatMadrigal/instruments/methods/dmsp.py b/pysatMadrigal/instruments/methods/dmsp.py index 8ee5771..48ce86a 100644 --- a/pysatMadrigal/instruments/methods/dmsp.py +++ b/pysatMadrigal/instruments/methods/dmsp.py @@ -32,7 +32,20 @@ def references(name): 'SSIES-3) on Spacecraft of the Defense', 'Meteorological Satellite Program (Air Force', 'Phillips Laboratory, Hanscom AFB, MA, 1994),', - 'Vol. 1, p. 25.'))} + 'Vol. 1, p. 25.')), + 'ssj': ''.join(['Gussenhoven, M. S., D. A. Hardy and W. J. Burke, ', + 'DMSP/F2 electron observations of equatorward ', + 'auroral boundaries and their relationship to ', + 'magnetospheric electric fields, J. Geophys. Res.,', + ' 86, 768-778, 1981.\nGussenhoven, M. S., D. A. ', + 'Hardy and N. Heinemann, Systematics of the ', + 'equatorward diffuse auroral boundary, J. Geophys.', + ' Res., 88, 5692-5708, 1983.\nHardy, D. A., E. ', + 'G. Holeman, W. J. Burke, L. C. Gentile and K. H. ', + 'Bounar (2008), Probability distributions of ', + 'electron precipitation at high magnetic latitudes', + ', Journal of Geophysical Research, Volume 113, ', + 'Issue A6, doi10.1029/2007JA012746.'])} return refs[name] From fb9113e9c1b11761fa506b75c5e96e981b799539 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 2 Mar 2023 16:48:03 -0500 Subject: [PATCH 003/139] DOC: updated the supported instruments Added the DMSP SSJ instrument to the docs. --- docs/supported_instruments.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index ca45d41..6db4917 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -11,6 +11,16 @@ Meter (IVM) Madrigal data. .. automodule:: pysatMadrigal.instruments.dmsp_ivm :members: +DMSP_SSJ +-------- + +Supports the Defense Meteorological Satelitte Program (DMSP) Special Sensor J +(SSJ) Madrigal data. + + +.. automodule:: pysatMadrigal.instruments.dmsp_ssj + :members: + GNSS_TEC -------- From 255c052fe8b86d7f8c60d6f04c8f87ce6f9e6459 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 2 Mar 2023 16:48:18 -0500 Subject: [PATCH 004/139] DOC: updated changelog Added a summary of the changes to the changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc12568..9d8cf09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). between '.' delimiters, required for some Madrigal file formats * Standardized the Instrument method kwarg defaults * Added 'site' tag to the GNSS TEC Instrument + * Added a 'dmsp_ssj' instrument for Auroral Boundary Index data * Added support for varied use of `two_digit_year_break` to `methods.general.list_remote_files` * Implemented `two_digit_year_break` support for `vtec` GNSS TEC Instrument @@ -29,6 +30,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added quick-fail for main pytest command * Bug * Fixed bugs in the coordinate conversion functions + * Fixed bug in the general download function that sets the stop date * Maintenance * Updated GitHub action and NEP29 versions * Updated the minimum Madrigal version to allow HDF4 downloads From 06ad0c9bffff8ea7a58fb43b4454e31a1a1ccace Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 13 Mar 2023 12:54:25 -0400 Subject: [PATCH 005/139] MAINT: removed temporary code Removed code needed to handle failure of pysat to allow local definitions of start/stop times. --- pysatMadrigal/instruments/methods/general.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index bc26563..09ecd07 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -1165,13 +1165,6 @@ def list_remote_files(tag, inst_id, inst_code=None, kindats=None, user=None, format_str = supported_tags[inst_id][tag] kindat = kindats[inst_id][tag] - # TODO(#1022, pysat) Note default of `pysat.Instrument.remote_file_list` - # for start and stop is None. Setting defaults needed for Madrigal. - if start is None: - start = dt.datetime(1900, 1, 1) - if stop is None: - stop = dt.datetime.utcnow() - # Retrieve remote file experiment list files = get_remote_filenames(inst_code=inst_code, kindat=kindat, user=user, password=password, url=url, start=start, From 96d3e891cbf95ace5a9357c5a461681e2a313a77 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 13 Mar 2023 12:54:44 -0400 Subject: [PATCH 006/139] DOC: updated CHANGELOG Updated changelog with a description of this pull. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc12568..511e25e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated the minimum Madrigal version to allow HDF4 downloads * Update pysat instrument testing suite, pytest syntax * Add manual GitHub Actions tests for pysat RC + * Removed code needed to work around pysat bugs [0.0.4] - 2021-06-11 -------------------- From dbf8b60688bf91455d26ef07b3083d2e4978f55e Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 16 Mar 2023 17:03:37 -0400 Subject: [PATCH 007/139] MAINT: updated pysat version limit Updated the pysat version limit to the next release that will include all the necessary changes for pysatMadrigal to work as expected. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 10b7915..063b76f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ madrigalWeb>=2.6 numpy packaging pandas -pysat>=3.0.3 +pysat>=3.1.0 xarray From dc99c8eaf30c81d2bcde1db564628a288598f21a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 2 May 2023 10:14:12 -0400 Subject: [PATCH 008/139] TST: add rc install workflow --- .github/workflows/pip_rc_install.yml | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/pip_rc_install.yml diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml new file mode 100644 index 0000000..8a74d6b --- /dev/null +++ b/.github/workflows/pip_rc_install.yml @@ -0,0 +1,30 @@ +# This workflow will install Python dependencies and the latest RC of pysatNASA from test pypi. +# This test should be manually run before a pysatMadrigal RC is officially approved and versioned. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Test install of latest RC from pip + +on: [workflow_dispatch] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.10"] # Keep this version at the highest supported Python version + + name: Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install standard dependencies + run: pip install -r requirements.txt + + - name: Install pysatMadrigal RC + run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysatMadrigal From 22437e3c917cf33ac5fc91ebd3c430c3361b6f02 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 2 May 2023 10:15:21 -0400 Subject: [PATCH 009/139] MAINT: update pip syntax --- .github/workflows/pysat_rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 2e4ce4e..6d82336 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -24,7 +24,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install pysat RC - run: pip install --no-deps -i https://test.pypi.org/simple/ pysat + run: pip install --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat - name: Install standard dependencies run: | From aece8aac924401fcb17b164071a0b0ec1b69c6b0 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 2 May 2023 10:16:34 -0400 Subject: [PATCH 010/139] DOC: update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36c26b8..865a723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +[0.1.X] - 2023-XX-XX +-------------------- +* Maintenance + * Add manual GitHub Actions tests for pysatMadrigal RC + * Update GitHub Actions workflows for newer versions of pip + [0.1.0] - 2023-04-11 -------------------- * Enhancements From 120699d2e271eae2ab1f21d0d56fd295ab41b603 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 2 May 2023 10:16:54 -0400 Subject: [PATCH 011/139] MAINT: set to push --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 8a74d6b..27eb670 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -4,7 +4,7 @@ name: Test install of latest RC from pip -on: [workflow_dispatch] +on: [push] jobs: build: From d7395462160da75342b941f6d023912908f3d0cc Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 2 May 2023 10:23:40 -0400 Subject: [PATCH 012/139] TST: set to workflow_dispatch --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 27eb670..8a74d6b 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -4,7 +4,7 @@ name: Test install of latest RC from pip -on: [push] +on: [workflow_dispatch] jobs: build: From bd6c6c29ee933079bf38c966517c730d5eff17b8 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 20:56:49 +0900 Subject: [PATCH 013/139] Update pip_rc_install.yml --- .github/workflows/pip_rc_install.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 8a74d6b..dc77bbe 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -4,7 +4,7 @@ name: Test install of latest RC from pip -on: [workflow_dispatch] +on: [push] jobs: build: @@ -28,3 +28,13 @@ jobs: - name: Install pysatMadrigal RC run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysatMadrigal + + - name: Set up pysat + run: | + mkdir pysatData + python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" + + - name: Check that install imports correctly + run: | + cd .. + python -c "import pysatMadrigal; print(pysatMadrigal.__version__)" From 6e7658ac398d1a61497925ab26282aaaf812d21d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 18 May 2023 20:58:29 +0900 Subject: [PATCH 014/139] Update test_requirements.txt --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index fc59dc8..9893985 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -6,5 +6,5 @@ m2r2 numpydoc pytest-cov pytest-ordering -sphinx +sphinx<7.0 sphinx_rtd_theme From 87fb3cacaeca62c9e19edc665c7094d020d8d416 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Tue, 30 May 2023 13:29:19 -0400 Subject: [PATCH 015/139] ENH: add version to package --- pysatMadrigal/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pysatMadrigal/__init__.py b/pysatMadrigal/__init__.py index c1a628f..908e3ad 100644 --- a/pysatMadrigal/__init__.py +++ b/pysatMadrigal/__init__.py @@ -1,2 +1,10 @@ +import os from pysatMadrigal import instruments # noqa F401 from pysatMadrigal import utils # noqa F401 + +# set version +here = os.path.abspath(os.path.dirname(__file__)) +version_filename = os.path.join(here, 'version.txt') +with open(version_filename, 'r') as version_file: + __version__ = version_file.read().strip() +del here, version_filename, version_file From c18d806db9ef9c13cacd1c0fed1575d3f492a245 Mon Sep 17 00:00:00 2001 From: jklenzing Date: Tue, 30 May 2023 14:49:52 -0400 Subject: [PATCH 016/139] DOC: add docstring --- pysatMadrigal/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pysatMadrigal/__init__.py b/pysatMadrigal/__init__.py index 908e3ad..654bab7 100644 --- a/pysatMadrigal/__init__.py +++ b/pysatMadrigal/__init__.py @@ -1,3 +1,10 @@ +"""Core library for pysatMadrigal. + +This is a library of `pysat` instrument modules and methods designed to support +instruments archived at the Madrigal portal. + +""" + import os from pysatMadrigal import instruments # noqa F401 from pysatMadrigal import utils # noqa F401 From 76b7021f2330efd91dc8d77d9d018224182d62d7 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 31 May 2023 16:40:45 -0400 Subject: [PATCH 017/139] MAINT: remove ssj from general pandas Because DMSP SSJ requires annual loading, it cannot be handled correctly by the general pandas instrument. --- pysatMadrigal/instruments/madrigal_pandas.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index 57fb6bd..793753d 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -75,7 +75,9 @@ name = 'pandas' tags = dict() pandas_format = True -excluded_tags = ['8105'] # Pandas-style data that requires special support + +# Pandas-style data that requires special support +excluded_tags = ['8105', '180'] # Assign only tags with pysat-compatible file format strings pandas_codes = general.known_madrigal_inst_codes(pandas_format=True) @@ -102,9 +104,8 @@ # Need to sort out test day setting for unit testing, maybe through a remote # function tag_dates = {'120': dt.datetime(1963, 11, 27), '170': dt.datetime(1998, 7, 1), - '180': dt.datetime(2000, 1, 1), '210': dt.datetime(1950, 1, 1), - '211': dt.datetime(1978, 1, 1), '212': dt.datetime(1957, 1, 1), - '7800': dt.datetime(2009, 11, 10)} + '210': dt.datetime(1950, 1, 1), '211': dt.datetime(1978, 1, 1), + '212': dt.datetime(1957, 1, 1), '7800': dt.datetime(2009, 11, 10)} _test_dates = {'': {tag: tag_dates[tag] if tag in tag_dates.keys() else tag_dates['7800'] for tag in tags.keys()}} _test_download = {'': {tag: True for tag in tags.keys()}} From 8c707831ea1184b7ba7a32e6132847f24cc52bb2 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 31 May 2023 17:52:02 -0400 Subject: [PATCH 018/139] BUG: add sphinx version limit Fix CI docs tests with a sphinx version limit. --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 6d4240d..5b48ef4 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -4,5 +4,5 @@ m2r2 numpydoc pytest-cov pytest-ordering -sphinx +sphinx<7.0 sphinx_rtd_theme From 673c6681073e184a9b13281e16f02ac057b7f320 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 1 Jun 2023 09:04:21 -0400 Subject: [PATCH 019/139] STY: made flake8 updates Updated the docstring and import style. --- pysatMadrigal/instruments/dmsp_ssj.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pysatMadrigal/instruments/dmsp_ssj.py b/pysatMadrigal/instruments/dmsp_ssj.py index d73f883..f1c587e 100644 --- a/pysatMadrigal/instruments/dmsp_ssj.py +++ b/pysatMadrigal/instruments/dmsp_ssj.py @@ -4,7 +4,7 @@ # DOI:10.5281/zenodo.3824979 # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- -"""Supports the Special Sensor-J (SSJ) instrument and derived products from the +"""Support the Special Sensor-J (SSJ) instrument and derived products from the Defense Meteorological Satellite Program (DMSP). The SSJ measures precipitating particles using spectrometery. The Auroral @@ -62,7 +62,8 @@ from pysat import logger from pysat.utils.time import create_date_range -from pysatMadrigal.instruments.methods import general, dmsp +from pysatMadrigal.instruments.methods import dmsp +from pysatMadrigal.instruments.methods import general # ---------------------------------------------------------------------------- # Instrument attributes @@ -97,7 +98,6 @@ def init(self): """Initialize the Instrument object with values specific to DMSP IVM.""" - self.acknowledgements = ''.join([ general.cedar_rules(), '\nThe Air Force Research Laboratory Auroral ', 'Boundary Index (ABI) was provided by the United States Air Force ', @@ -128,7 +128,6 @@ def clean(self): If the 'dirty' level is invoked (same as no cleaning) """ - if self.clean_level == 'clean': iclean, = np.where(self['EQB_QC_FL'] <= 1) elif self.clean_level == 'dusty': @@ -163,7 +162,7 @@ def clean(self): # Set the load routine def load(fnames, tag='', inst_id=''): - """Loads data from Madrigal after accounting for date tags. + """Load DMSP SSJ4 data from Madrigal after accounting for date tags. Parameters ---------- @@ -221,7 +220,7 @@ def load(fnames, tag='', inst_id=''): def download(date_array, tag='', inst_id='', data_path=None, user=None, password=None, file_type='hdf5'): - """Downloads data from Madrigal. + """Download DMSP SSJ4 data from Madrigal. Parameters ---------- @@ -257,7 +256,6 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None, downloads. """ - # Ensure the date range is correct if date_array.freq not in ['AS-JAN', 'YS', 'AS']: date_array = create_date_range( From 8b30f43dcad7e7cafe75b9cf9cea8218c619fdd1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 1 Jun 2023 12:38:23 -0400 Subject: [PATCH 020/139] STY: update flake8 Attempt to update flake8 so PEP8 tests pass. --- pysatMadrigal/instruments/dmsp_ssj.py | 11 ++++++----- setup.cfg | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pysatMadrigal/instruments/dmsp_ssj.py b/pysatMadrigal/instruments/dmsp_ssj.py index f1c587e..5d4c707 100644 --- a/pysatMadrigal/instruments/dmsp_ssj.py +++ b/pysatMadrigal/instruments/dmsp_ssj.py @@ -4,12 +4,13 @@ # DOI:10.5281/zenodo.3824979 # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- -"""Support the Special Sensor-J (SSJ) instrument and derived products from the -Defense Meteorological Satellite Program (DMSP). +"""Support the DMSP Special Sensor-J (SSJ) instrument and derived products. -The SSJ measures precipitating particles using spectrometery. The Auroral -Boundary Index (ABI) is automatically computed from this data set and marks the -midnight equatorward boundary in each hemisphere. + +The Defense Meteorological Satellite Program (DMSP) SSJ measures precipitating +particles using spectrometery. The Auroral Boundary Index (ABI) is automatically +computed from this data set and marks the midnight equatorward boundary in each +hemisphere. Further questions can be addressed to: Gordon Wilson diff --git a/setup.cfg b/setup.cfg index 1f3d15c..2b03414 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,6 +56,8 @@ omit = */instruments/templates/* [flake8] max-line-length = 80 ignore = W503 + D200 + D202 [tool:pytest] markers = From 06d472d131e5bb73517d42fc35751a2a83e3cf5a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 1 Jun 2023 15:18:41 -0400 Subject: [PATCH 021/139] STY: removed extra blank line Removed an extra blank line in a docstring. --- pysatMadrigal/instruments/dmsp_ssj.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysatMadrigal/instruments/dmsp_ssj.py b/pysatMadrigal/instruments/dmsp_ssj.py index 5d4c707..8ae3ea3 100644 --- a/pysatMadrigal/instruments/dmsp_ssj.py +++ b/pysatMadrigal/instruments/dmsp_ssj.py @@ -6,7 +6,6 @@ # -*- coding: utf-8 -*- """Support the DMSP Special Sensor-J (SSJ) instrument and derived products. - The Defense Meteorological Satellite Program (DMSP) SSJ measures precipitating particles using spectrometery. The Auroral Boundary Index (ABI) is automatically computed from this data set and marks the midnight equatorward boundary in each From 1185105ef19774045892a46f086d346fb45de7c3 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 2 Jun 2023 10:48:01 -0400 Subject: [PATCH 022/139] BUG: fixed variable case Fixed variable names to lowercase, to comply with pysat standards. --- pysatMadrigal/instruments/dmsp_ssj.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatMadrigal/instruments/dmsp_ssj.py b/pysatMadrigal/instruments/dmsp_ssj.py index 8ae3ea3..9a2f263 100644 --- a/pysatMadrigal/instruments/dmsp_ssj.py +++ b/pysatMadrigal/instruments/dmsp_ssj.py @@ -129,9 +129,9 @@ def clean(self): """ if self.clean_level == 'clean': - iclean, = np.where(self['EQB_QC_FL'] <= 1) + iclean, = np.where(self['eqb_qc_fl'] <= 1) elif self.clean_level == 'dusty': - iclean, = np.where(self['EQB_QC_FL'] <= 2) + iclean, = np.where(self['eqb_qc_fl'] <= 2) else: warnings.warn('No quality control level "dirty", using "none"') iclean = None From 01591fc19678d58d54deb13d341c2571f6f835e8 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 8 Jun 2023 15:37:14 -0400 Subject: [PATCH 023/139] TST: added windows to CI Added Windows to CI, may or may not work. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e9cd42c..58767c5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] # TODO(76): add windows-latest + os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.9", "3.10"] numpy_ver: ["latest"] include: From 1e2039950f2a931612d97256da0e736cf609a45d Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 21 Jun 2023 14:24:52 -0400 Subject: [PATCH 024/139] TST: added close statement Added a close statement to help Windows OS. --- pysatMadrigal/tests/test_methods_general.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index 5fc36eb..8ab8cf4 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -601,6 +601,9 @@ def test_load_netcdf(self, nfiles): # Evaluate the loaded data self.eval_dataset_meta_output() + # Close for Windows OS + self.data.close() + return def test_load_netcdf_extra_xarray_coord(self): @@ -617,6 +620,9 @@ def test_load_netcdf_extra_xarray_coord(self): # Evaluate the loaded data self.eval_dataset_meta_output() + # Close for Windows OS + self.data.close() + return From 7335346d02d9efd9b9a061a579d7854ad984b570 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 22 Jun 2023 12:20:36 -0400 Subject: [PATCH 025/139] BUG: removed the AE data Removed the AE data from the general madrigal instrument, as they have a very large file and by default have duplicated times. --- pysatMadrigal/instruments/madrigal_pandas.py | 42 ++++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index 6837a0a..292b780 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -77,7 +77,7 @@ pandas_format = True # Pandas-style data that requires special support -excluded_tags = ['8105', '180'] +excluded_tags = ['8105', '180', '211'] # Assign only tags with pysat-compatible file format strings pandas_codes = general.known_madrigal_inst_codes(pandas_format=True) @@ -100,15 +100,12 @@ # ---------------------------------------------------------------------------- # Instrument test attributes - -# Need to sort out test day setting for unit testing, maybe through a remote -# function tag_dates = {'120': dt.datetime(1963, 11, 27), '170': dt.datetime(1998, 7, 1), - '210': dt.datetime(1950, 1, 1), '211': dt.datetime(1978, 1, 1), - '212': dt.datetime(1957, 1, 1), '7800': dt.datetime(2009, 11, 10)} + '210': dt.datetime(1950, 1, 1), '212': dt.datetime(1957, 1, 1), + '7800': dt.datetime(2009, 11, 10)} _test_dates = {'': {tag: tag_dates[tag] if tag in tag_dates.keys() else tag_dates['7800'] for tag in tags.keys()}} -_test_download = {'': {tag: True for tag in tags.keys()}} +_test_download = {'': {tag: tag in tag_dates.keys() for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods @@ -236,27 +233,25 @@ def list_files(tag, inst_id, data_path, kindat='', format_str=None, return out -def download(date_array, tag='', inst_id='', data_path=None, user=None, - password=None, file_type='hdf5', kindat=''): +def download(date_array, tag, inst_id, data_path, user=None, password=None, + file_type='hdf5', kindat=''): """Download data from Madrigal. Parameters ---------- date_array : array-like - list of datetimes to download data for. The sequence of dates need not + List of datetimes to download data for. The sequence of dates need not be contiguous. tag : str - Madrigal Instrument code cast as a string. (default='') + Madrigal Instrument code cast as a string. inst_id : str - Satellite ID string identifier used for particular dataset. (default='') + Instrument ID used for particular dataset. data_path : str - Path to directory to download data to. (default=None) - user : str - User string input used for download. Provided by user and passed via - pysat. If an account is required for dowloads this routine here must - error if user not supplied. (default=None) - password : str - Password for data download. (default=None) + Path to directory to download data to. + user : str or NoneType + User name, raises an error if user not supplied. (default=None) + password : str or NoneType + User email, raises an error if not supplied. (default=None) file_type : str File format for Madrigal data. (default='hdf5') kindat : str @@ -264,8 +259,13 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None, instrument. May be a single value, blank (all), or a comma-delimited list. (defaut='') - Notes - ----- + Raises + ------ + ValueError + If user or password are not supplied + + Note + ---- The user's names should be provided in field user. Maria Goeppert Mayer should be entered as "Maria Goeppert Mayer" From 6b1f0c69d0700bde831f6a167706bbe2c6c1b4dc Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 22 Jun 2023 12:21:49 -0400 Subject: [PATCH 026/139] BUG: allow any kwarg input to general download Allow any kwarg input to the general download function, improving ease of use with functools. --- pysatMadrigal/instruments/methods/general.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index 5910330..d12a69c 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -819,7 +819,7 @@ def load(fnames, tag='', inst_id='', xarray_coords=None): def download(date_array, inst_code=None, kindat=None, data_path=None, user=None, password=None, url="http://cedar.openmadrigal.org", - file_type='hdf5'): + file_type='hdf5', **kwargs): """Download data from Madrigal. Parameters @@ -847,6 +847,9 @@ def download(date_array, inst_code=None, kindat=None, data_path=None, File format for Madrigal data. Load routines currently only accepts 'hdf5' and 'netCDF4', but any of the Madrigal options may be used here. (default='hdf5') + **kwargs : dict + Additional kwarg catch, allows general use when tag/inst_id are not + needed for a given instrument. Raises ------ From c9c6903aa5f1b493f0c1d43ebb728a349b795a57 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 22 Jun 2023 12:22:47 -0400 Subject: [PATCH 027/139] ENH: added the NGDC AE instrument Added an instrument for the NGDC AE data. This should fix the windows memory issue, as it only loads the desired portion of the data (which is all contained in a single file). --- pysatMadrigal/instruments/__init__.py | 4 +- pysatMadrigal/instruments/ngdc_ae.py | 304 ++++++++++++++++++++++++++ 2 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 pysatMadrigal/instruments/ngdc_ae.py diff --git a/pysatMadrigal/instruments/__init__.py b/pysatMadrigal/instruments/__init__.py index e4b6a84..de82e34 100644 --- a/pysatMadrigal/instruments/__init__.py +++ b/pysatMadrigal/instruments/__init__.py @@ -10,9 +10,11 @@ from pysatMadrigal.instruments import gnss_tec from pysatMadrigal.instruments import jro_isr from pysatMadrigal.instruments import madrigal_pandas +from pysatMadrigal.instruments import ngdc_ae # Import Madrigal methods from pysatMadrigal.instruments import methods # noqa F401 # Define variable name with all available instruments -__all__ = ['dmsp_ivm', 'dmsp_ssj', 'gnss_tec', 'jro_isr', 'madrigal_pandas'] +__all__ = ['dmsp_ivm', 'dmsp_ssj', 'gnss_tec', 'jro_isr', 'madrigal_pandas', + 'ngdc_ae'] diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py new file mode 100644 index 0000000..334a78f --- /dev/null +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# ---------------------------------------------------------------------------- +# -*- coding: utf-8 -*-. +"""Supports access to taped data of AE from the World Data Center A (Boulder) + +Properties +---------- +platform + 'ngdc' +name + 'ae' +tag + None supported +inst_id + None supported + +Note +---- +Please provide name (user) and email (password) when downloading data with this +routine. + +Warnings +-------- +The entire data set (1 Jan 1978 through 31 Dec 1987) is provided in a single +file on Madrigal. The download method will break this file up by year for +easier access. + +Examples +-------- +:: + + + import datetime as dt + import pysat + import pysatMadrigal as py_mad + + # Download DMSP data from Madrigal + aei = pysat.Instrument(inst_module=py_mad.instruments.ngdc_ae) + aei.download(start=py_mad.instruments.ngdc_ae.madrigal_start, + user='Firstname+Lastname', password='email@address.com') + aei.load(date=dt.datetime(1981, 1, 1)) + +""" + +import datetime as dt +import functools +import numpy as np +import pandas as pds + +import h5py +import pysat + +from pysatMadrigal.instruments.methods import general + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'ngdc' +name = 'ae' +tags = {'': ''} +inst_ids = {'': list(tags.keys())} +pandas_format = True + +# Madrigal tags and limits +madrigal_inst_code = 211 +madrigal_tag = {'': {'': "30008"}} +madrigal_start = dt.datetime(1978, 1, 1) +madrigal_end = dt.datetime(1988, 1, 1) + +# Local attributes +# +# Need a way to get the filename strings for a particular instrument unless +# wildcards start working +supported_tags = { + inst_id: {tag: general.madrigal_file_format_str(madrigal_inst_code, + verbose=False) + for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} +remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5') + for kk in inst_ids[ss]} for ss in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {inst_id: {tag: madrigal_start for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} +_test_download = {inst_id: {tag: True for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +def init(self): + """Initialize the Instrument object in support of Madrigal access. + + Parameters + ---------- + kindat : str + Madrigal instrument experiment code(s). (default='') + + """ + # Set the standard pysat attributes + self.acknowledgements = general.cedar_rules() + self.references = ''.join(['Davis, T. Neil and Masahisa Sugiura. “Auroral', + ' electrojet activity index AE and its ', + 'universal time variations.” Journal of ', + 'Geophysical Research 71 (1966): 785-801.']) + + # Remind the user of the Rules of the Road + pysat.logger.info(self.acknowledgements) + return + + +def clean(self): + """Raise warning that cleaning is not possible for general data. + + Note + ---- + Supports 'clean', 'dusty', 'dirty' in the sense that it prints + a message noting there is no cleaning. + 'None' is also supported as it signifies no cleaning. + + Routine is called by pysat, and not by the end user directly. + + """ + + warned = False + for dvar in self.variables: + if self.meta[dvar, self.meta.labels.units].find('nT') >= 0: + # The 'clean', 'dusty', and 'dirty' levels all replace the missing + # parameter value of -32766 with NaN + mask = self[dvar] == self.meta[dvar, self.meta.labels.fill_val] + self[dvar][mask] == np.nan + self.meta[dvar] = {self.meta.labels.fill_val: np.nan} + + if self.clean_level in ['clean', 'dusty']: + if self.clean_level == 'dusty' and not warned: + pysat.logger.warning( + "The NGDC AE 'dusty' and 'clean' levels are the same.") + warned = True + + # The 'clean' and 'dusty' levels replace the parameter error + # value of -32766 with NaN + self[dvar][self[dvar] == -32766] = np.nan + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default Madrigal and pysat methods +file_cadence = madrigal_end - madrigal_start +two_digit_year_break = 50 + +# Set the download routine +download = functools.partial(general.download, + inst_code=str(madrigal_inst_code), + kindat=madrigal_tag[''][''], file_type='hdf5') + +# Set the list routine +list_files = functools.partial(general.list_files, + supported_tags=supported_tags, + file_cadence=file_cadence, + two_digit_year_break=two_digit_year_break) + +# Set list_remote_files routine +list_remote_files = functools.partial(general.list_remote_files, + supported_tags=remote_tags, + inst_code=madrigal_inst_code, + kindats=madrigal_tag, + two_digit_year_break=two_digit_year_break) + + +def load(fnames, tag='', inst_id=''): + """Load the NGDC AE data. + + Parameters + ----------- + fnames : list + List of filenames + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + + Returns + -------- + data : pds.DataFrame + Object containing satellite data + meta : pysat.Meta + Object containing metadata such as column names and units + + Raises + ------ + ValueError + Unexpected time variable names + + """ + # Initialize the output + meta = pysat.Meta() + labels = [] + data = None + fill_val = -32767 + notes = "".join(["Assumed parameters error values are assigned a value ", + "of -32766 for clean levels of 'dirty' or 'none'"]) + + # Cycle through all the filenames, getting the desired start and stop times + fstart = None + fstop = None + for fname_date in fnames: + # Split the date from the filename + fname = fname_date[:-11] + fdate = dt.datetime.strptime(fname_date[-10:], '%Y-%m-%d') + + if fstart is None: + fstart = fdate + + if fstop is None: + fstop = fdate + + # There is only one file for this Instrument + with h5py.File(fname, 'r') as filed: + file_data = filed['Data']['Table Layout'] + file_meta = filed['Metadata']['Data Parameters'] + + # Load available info into pysat.Meta if this is the first file + if len(labels) == 0: + for item in file_meta: + name_string = item[0].decode('UTF-8') + unit_string = item[3].decode('UTF-8') + desc_string = item[1].decode('UTF-8') + labels.append(name_string) + + # Only update metadata if necessary + if name_string.lower() not in meta: + meta_dict = {meta.labels.name: name_string, + meta.labels.units: unit_string, + meta.labels.desc: desc_string} + + if unit_string.find('nT') > 0: + # Fill and error values only apply to index values + meta_dict[meta.labels.fill_val] = fill_val + meta_dict[meta.labels.notes] = notes + + meta[name_string.lower()] = meta_dict + + # Add additional metadata notes. Custom attributes attached to + # meta are attached to corresponding Instrument object when + # pysat receives data and meta from this routine + for key in filed['Metadata']: + if key != 'Data Parameters': + setattr(meta, key.replace(' ', '_'), filed['Metadata'][key][:]) + + # Extended processing is the same for simple and HDF5 files + # + # Construct datetime index from times + time_keys = np.array(['year', 'month', 'day', 'hour', 'min', 'sec']) + lower_labels = [ll.lower() for ll in labels] + time_keys = [key for key in time_keys if key not in lower_labels] + if len(time_keys) > 0: + raise ValueError(' '.join(["unable to construct time index, ", + "missing {:}".format(time_keys)])) + + # Get the date information + year = file_data[:]['year'] + month = file_data[:]['month'] + day = file_data[:]['day'] + fdate = pysat.utils.time.create_datetime_index(year=year, month=month, + day=day) + + # Get the data mask + dmask = (fdate >= fstart) & (fdate <= fstop) + + # Construct the time index + hour = file_data[dmask]['hour'] + minute = (file_data[dmask]['hm'] / 100.0 - hour) * 100.0 + uts = 3600.0 * hour + 60.0 * minute + file_data[dmask]['hmi'] + + tindex = pysat.utils.time.create_datetime_index( + year=year[dmask], month=month[dmask], day=day[dmask], uts=uts) + + # Load the data into a pandas DataFrame + data = pds.DataFrame.from_records(file_data[dmask], columns=labels, + index=tindex) + + # Raise a logging warning if there are duplicate times. This + # means the data should be stored as an xarray Dataset + if np.any(tindex.duplicated()): + pysat.logger.warning(''.join(["duplicated time indices, consider ", + "specifing additional coordinates ", + "and storing the data as an xarray ", + "Dataset"])) + + # Ensure that data is at least an empty Dataset + if data is None: + data = pds.DataFrame(dtype=np.float64) + + return data, meta From 90e371d0a2f364b2da92a58704f9da2a6355506b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 22 Jun 2023 12:23:02 -0400 Subject: [PATCH 028/139] DOC: updated changelog Added a summary of the changes to the changelog. --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9782d53..d336c5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ This project adheres to [Semantic Versioning](https://semver.org/). [0.1.X] - 2023-XX-XX -------------------- +* Enhancements + * Moved the NGDC AE index Instrument from the general Madrigal Pandas + instrument to it's own, fixing the Windows memory issue and a problem + with duplicated times * Maintenance * Add manual GitHub Actions tests for pysatMadrigal RC * Update GitHub Actions workflows for newer versions of pip From c03dbc5c1781e1d6c9a5d6daa62b0277b8e8a009 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 22 Jun 2023 12:28:22 -0400 Subject: [PATCH 029/139] DOC: updated supported instruments Added the new NGDC AE instrument to the documentation. --- docs/supported_instruments.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 6db4917..ffb04f7 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -59,4 +59,17 @@ support (e.g., cleaning methods, experiment acknowledgements, and references). :members: +NGDC_AE +------- + +An instrument for the Geophysical indices from NGDC, which include AE AL, AU, +and AO. The :py:attr:`name` is AE due to the Madrigal naming conventions. The +data set spans the years of 1978 through 1987, will all data saved in a single +file. Because of this, you only need to download the data once and any desired +time period may be loaded. + +.. automodule:: pysatMadrigal.instruments.ngdc_ae + :members: + + From 0ba7971eebec30d3d4bbacbc07a748e496ded996 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 22 Jun 2023 12:33:59 -0400 Subject: [PATCH 030/139] STY: added missing period Added a missing period the the module docstring. --- pysatMadrigal/instruments/ngdc_ae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py index 334a78f..6ce82f0 100644 --- a/pysatMadrigal/instruments/ngdc_ae.py +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -4,7 +4,7 @@ # DOI:10.5281/zenodo.3824979 # ---------------------------------------------------------------------------- # -*- coding: utf-8 -*-. -"""Supports access to taped data of AE from the World Data Center A (Boulder) +"""Supports access to taped data of AE from the World Data Center A (Boulder). Properties ---------- From 2248fb86df36306b89ca41465c77cb49cba640a7 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 22 Jun 2023 14:29:35 -0400 Subject: [PATCH 031/139] BUG: clearer logic statement Make the logic statement clearer. --- pysatMadrigal/instruments/madrigal_pandas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index 292b780..8977782 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -105,7 +105,7 @@ '7800': dt.datetime(2009, 11, 10)} _test_dates = {'': {tag: tag_dates[tag] if tag in tag_dates.keys() else tag_dates['7800'] for tag in tags.keys()}} -_test_download = {'': {tag: tag in tag_dates.keys() for tag in tags.keys()}} +_test_download = {'': {tag: True if tag in tag_dates.keys() else False for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods From 2040e2dfc426d0ec3650dacfd6fe9aea12c77c66 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 22 Jun 2023 14:30:56 -0400 Subject: [PATCH 032/139] STY: fixed line length Fixed the line length for a GitHub suggestion. --- pysatMadrigal/instruments/madrigal_pandas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index 8977782..c53abe4 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -105,7 +105,8 @@ '7800': dt.datetime(2009, 11, 10)} _test_dates = {'': {tag: tag_dates[tag] if tag in tag_dates.keys() else tag_dates['7800'] for tag in tags.keys()}} -_test_download = {'': {tag: True if tag in tag_dates.keys() else False for tag in tags.keys()}} +_test_download = {'': {tag: True if tag in tag_dates.keys() else False + for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods From b81cb176683ceb003a63c1c5e6ddf76c9a07e25b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 22 Jun 2023 16:29:37 -0400 Subject: [PATCH 033/139] BUG: fixed test bug Fixed the test bug introduced by trying to be clever. --- pysatMadrigal/instruments/madrigal_pandas.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index c53abe4..b4804b3 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -105,8 +105,7 @@ '7800': dt.datetime(2009, 11, 10)} _test_dates = {'': {tag: tag_dates[tag] if tag in tag_dates.keys() else tag_dates['7800'] for tag in tags.keys()}} -_test_download = {'': {tag: True if tag in tag_dates.keys() else False - for tag in tags.keys()}} +_test_download = {'': {tag: True for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods From 57c902a9589562c8c885e4c1b102552b6db4c633 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:37:49 -0400 Subject: [PATCH 034/139] Update pip_rc_install.yml --- .github/workflows/pip_rc_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index dc77bbe..6b9d2d6 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -4,7 +4,7 @@ name: Test install of latest RC from pip -on: [push] +on: [workflow_dispatch] jobs: build: From 651f0c908ecee55d7a0ada8dfc1e1ecff155661a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 23 Jun 2023 09:06:47 -0400 Subject: [PATCH 035/139] BUG: fixed string evalutation Fixed a string evaluation that should have included an equality. Also removed a logger warning for duplicate times, as this should be caught by pysat. --- pysatMadrigal/instruments/ngdc_ae.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py index 6ce82f0..d8b3525 100644 --- a/pysatMadrigal/instruments/ngdc_ae.py +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -243,7 +243,7 @@ def load(fnames, tag='', inst_id=''): meta.labels.units: unit_string, meta.labels.desc: desc_string} - if unit_string.find('nT') > 0: + if unit_string.find('nT') >= 0: # Fill and error values only apply to index values meta_dict[meta.labels.fill_val] = fill_val meta_dict[meta.labels.notes] = notes @@ -289,14 +289,6 @@ def load(fnames, tag='', inst_id=''): data = pds.DataFrame.from_records(file_data[dmask], columns=labels, index=tindex) - # Raise a logging warning if there are duplicate times. This - # means the data should be stored as an xarray Dataset - if np.any(tindex.duplicated()): - pysat.logger.warning(''.join(["duplicated time indices, consider ", - "specifing additional coordinates ", - "and storing the data as an xarray ", - "Dataset"])) - # Ensure that data is at least an empty Dataset if data is None: data = pds.DataFrame(dtype=np.float64) From c41d2c3351e54b6476e3749960965d571e36f2bf Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 23 Jun 2023 13:18:28 -0400 Subject: [PATCH 036/139] BUG: test the correct variable names Changed from testing 'min' to 'hm' and 'sec' to 'hmi', as these are the time variables needed to create a uniquely defined timestamp for this file. --- pysatMadrigal/instruments/ngdc_ae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py index d8b3525..dd8404d 100644 --- a/pysatMadrigal/instruments/ngdc_ae.py +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -260,7 +260,7 @@ def load(fnames, tag='', inst_id=''): # Extended processing is the same for simple and HDF5 files # # Construct datetime index from times - time_keys = np.array(['year', 'month', 'day', 'hour', 'min', 'sec']) + time_keys = np.array(['year', 'month', 'day', 'hour', 'hm', 'hmi']) lower_labels = [ll.lower() for ll in labels] time_keys = [key for key in time_keys if key not in lower_labels] if len(time_keys) > 0: From dcc7e9311978590947c1e8de7a5cca26ee72f7cc Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 27 Jun 2023 14:21:27 -0400 Subject: [PATCH 037/139] BUG: added try/except loop Added a pass-catch for the Windows PermissionError. --- pysatMadrigal/tests/test_methods_general.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index 8ab8cf4..b9033eb 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +ea#!/usr/bin/env python # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 @@ -502,7 +502,10 @@ def teardown_method(self): # Remove the temporary directory and file for tfile in self.temp_files: if os.path.isfile(tfile): - os.remove(tfile) + try: + os.remove(tfile) + except PermissionError: + pass # Windows is annoying self.data_path.cleanup() del self.data_path, self.temp_files, self.xarray_coords, self.data @@ -649,7 +652,10 @@ def teardown_method(self): # Remove the temporary file, if it exists for tfile in self.temp_files: if os.path.isfile(tfile): - os.remove(tfile) + try: + os.remove(tfile) + except PermissionError: + pass # Windows hates everything del self.inst, self.temp_files, self.supported_tags return From e5a146e87b146a25d225dd3eae51f5dec4822f36 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 27 Jun 2023 14:24:02 -0400 Subject: [PATCH 038/139] BUG: bad keyboard Removed extra letters caused by my laptop's broken keyboard. --- pysatMadrigal/tests/test_methods_general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index b9033eb..0dc48f4 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -1,4 +1,4 @@ -ea#!/usr/bin/env python +#!/usr/bin/env python # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 From b7f325c5812b2315a8243598632cd65bd92c0047 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 27 Jun 2023 17:00:15 -0400 Subject: [PATCH 039/139] BUG: changed try/except location Moved the try/except loop. --- pysatMadrigal/tests/test_methods_general.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index 0dc48f4..9b0a72f 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -502,11 +502,14 @@ def teardown_method(self): # Remove the temporary directory and file for tfile in self.temp_files: if os.path.isfile(tfile): - try: - os.remove(tfile) - except PermissionError: - pass # Windows is annoying - self.data_path.cleanup() + os.remove(tfile) + + try: + self.data_path.cleanup() + except PermissionError: + # Windows fix until `ignore_cleanup_errors=True` can be used in all + # supported versions + pass del self.data_path, self.temp_files, self.xarray_coords, self.data del self.meta @@ -652,10 +655,7 @@ def teardown_method(self): # Remove the temporary file, if it exists for tfile in self.temp_files: if os.path.isfile(tfile): - try: - os.remove(tfile) - except PermissionError: - pass # Windows hates everything + os.remove(tfile) del self.inst, self.temp_files, self.supported_tags return From 74a34af145f35363a12c664cef0f8403598a8d71 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 28 Jun 2023 11:16:14 -0400 Subject: [PATCH 040/139] BUG: file error popping up again Windows just hates closing files. Added more close and try/except statements. --- pysatMadrigal/tests/test_methods_general.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index 9b0a72f..739d6fb 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -502,13 +502,19 @@ def teardown_method(self): # Remove the temporary directory and file for tfile in self.temp_files: if os.path.isfile(tfile): - os.remove(tfile) + if hasattr(tfile, 'close'): + tfile.close() + + try: + os.remove(tfile) + except PermissionError: + pass # Windows thinks files are always open try: self.data_path.cleanup() except PermissionError: - # Windows fix until `ignore_cleanup_errors=True` can be used in all - # supported versions + # TODO (#https://github.com/pysat/pysat/issues/974): Windows fix + # until `ignore_cleanup_errors=True` can be used (3.10 is lowest) pass del self.data_path, self.temp_files, self.xarray_coords, self.data From b4c5d2f74766e703c90133634ecf3a382ba42468 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 28 Jun 2023 11:19:32 -0400 Subject: [PATCH 041/139] STY: fixed TODO comment Fixed the TODO formatting. --- pysatMadrigal/tests/test_methods_general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index 739d6fb..c80f58a 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -513,7 +513,7 @@ def teardown_method(self): try: self.data_path.cleanup() except PermissionError: - # TODO (#https://github.com/pysat/pysat/issues/974): Windows fix + # TODO(#https://github.com/pysat/pysat/issues/974): Windows fix # until `ignore_cleanup_errors=True` can be used (3.10 is lowest) pass From 005ed290f9c9896642df67cd867ca18097b0bd0f Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 28 Jun 2023 13:32:35 -0400 Subject: [PATCH 042/139] BUG: allow many types of errors Windows will throw many things at you to avoid removing data. --- pysatMadrigal/tests/test_methods_general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index c80f58a..dd6f859 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -512,7 +512,7 @@ def teardown_method(self): try: self.data_path.cleanup() - except PermissionError: + except Exception: # TODO(#https://github.com/pysat/pysat/issues/974): Windows fix # until `ignore_cleanup_errors=True` can be used (3.10 is lowest) pass From 780df451b01b5c061c4d1e270c4eaa6b6e6fc678 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 29 Jun 2023 11:45:20 -0400 Subject: [PATCH 043/139] TST: remove unused close call Files do not have the `close` method, remove useless call. --- pysatMadrigal/tests/test_methods_general.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index dd6f859..6663189 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -502,9 +502,6 @@ def teardown_method(self): # Remove the temporary directory and file for tfile in self.temp_files: if os.path.isfile(tfile): - if hasattr(tfile, 'close'): - tfile.close() - try: os.remove(tfile) except PermissionError: From f7381f2c1e7f7fb129341056c034e3823722e71a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 5 Jul 2023 13:34:23 -0400 Subject: [PATCH 044/139] STY: changed wording Changed wording without any change in meaning or clarity to satisfy @jklenzing. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d336c5a..e34ce08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). -------------------- * Enhancements * Moved the NGDC AE index Instrument from the general Madrigal Pandas - instrument to it's own, fixing the Windows memory issue and a problem + instrument to a new one, fixing the Windows memory issue and a problem with duplicated times * Maintenance * Add manual GitHub Actions tests for pysatMadrigal RC From 8297f42c3aa24822ad1fe466d9e085241ebe0f19 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 12:19:12 -0400 Subject: [PATCH 045/139] MAINT: added Windows to configuration Added the fact that we now support Windows to the setup.cfg file. --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index f209109..6e68fba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,6 +33,8 @@ classifiers = Programming Language :: Python :: 3.10 Operating System :: MacOS :: MacOS X Operating System :: POSIX :: Linux + Operating System :: POSIX :: Linux + Operating System :: Microsoft :: Windows license_file = LICENSE long_description = file: README.md long_description_content_type = text/markdown From 405fd639fb212ffa0151ccf21587ced78a7d1ae7 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 12:19:46 -0400 Subject: [PATCH 046/139] TST: added `_clean_warn` attribute Added the `_clean_warn` attribute to all existing Instruments. --- pysatMadrigal/instruments/dmsp_ivm.py | 7 +++++++ pysatMadrigal/instruments/dmsp_ssj.py | 4 ++++ pysatMadrigal/instruments/gnss_tec.py | 8 ++++++++ pysatMadrigal/instruments/jro_isr.py | 15 +++++++++++++++ pysatMadrigal/instruments/madrigal_pandas.py | 5 +++++ pysatMadrigal/instruments/ngdc_ae.py | 5 +++++ 6 files changed, 44 insertions(+) diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index 6ee5764..1855e5a 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -104,6 +104,13 @@ 'f17': {tag: dt.datetime(2009, 1, 1) for tag in inst_ids['f17']}, 'f18': {tag: dt.datetime(2017, 12, 30) for tag in inst_ids['f18']}} +_clean_warn = {inst_id: {tag: {clean_level: [('logger', 'WARN', + 'this level 1 data has no qual', + clean_level)] + for clean_level in ['clean', 'dusty', 'dirty']} + for tag in inst_ids[inst_id] if inst_id != 'utd'} + for inst_id in inst_ids.keys()} + # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatMadrigal/instruments/dmsp_ssj.py b/pysatMadrigal/instruments/dmsp_ssj.py index 9a2f263..21c4cb6 100644 --- a/pysatMadrigal/instruments/dmsp_ssj.py +++ b/pysatMadrigal/instruments/dmsp_ssj.py @@ -92,6 +92,10 @@ _test_dates = {'': {'abi': dt.datetime(1982, 12, 30)}} +_clean_warn = {'': {'abi': {'dirty': [('warning', UserWarning, + 'No quality control level "dirty"', + 'dirty')]}}} + # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index fbeb339..6e66f29 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -75,6 +75,14 @@ _test_dates = {'': {'vtec': dt.datetime(2017, 11, 19), 'site': dt.datetime(2001, 1, 1)}} +_clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', + 'Data provided at a clean level' + if tag != 'vtec' else + 'further cleaning may be performed', + clean_level)] + for clean_level in ['clean', 'dusty', 'dirty']} + for tag in inst_ids['']}} + # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatMadrigal/instruments/jro_isr.py b/pysatMadrigal/instruments/jro_isr.py index 47361c1..b500a72 100644 --- a/pysatMadrigal/instruments/jro_isr.py +++ b/pysatMadrigal/instruments/jro_isr.py @@ -88,6 +88,21 @@ 'oblique_rand': dt.datetime(2000, 11, 9), 'oblique_long': dt.datetime(2010, 4, 12)}} +_clean_warn = {'': {tag: {clean_level: [('logger', 'WARN', + 'this level 2 data has no quality ', + clean_level)] + for clean_level in ['clean', 'dusty']} + for tag in inst_ids['']}} + +for tag in inst_ids['']: + if tag.find('oblique') == 0: + _clean_warn[''][tag]['dirty'] = [('logger', 'WARN', + 'this level 2 data has no quality ', + 'dirty')] + for clean_level in ['clean', 'dusty', 'dirty']: + _clean_warn[''][tag][clean_level].append( + ('logger', 'INFO', 'The double pulse, coded ', clean_level)) + # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index b4804b3..f1a6fe9 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -106,6 +106,11 @@ _test_dates = {'': {tag: tag_dates[tag] if tag in tag_dates.keys() else tag_dates['7800'] for tag in tags.keys()}} _test_download = {'': {tag: True for tag in tags.keys()}} +_clean_warn = {'': {tag: {clean_level: [('logger', 'WARN', + "can't support instrument-specific", + clean_level)] + for clean_level in ['clean', 'dusty', 'dirty']} + for tag in inst_ids['']}} # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py index dd8404d..a87ba69 100644 --- a/pysatMadrigal/instruments/ngdc_ae.py +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -88,6 +88,11 @@ for inst_id in inst_ids.keys()} _test_download = {inst_id: {tag: True for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} +_clean_warn = {inst_id: {tag: {'dusty': [('logger', 'WARN', + "'dusty' and 'clean' are the same", + 'dusty')]} + for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} # ---------------------------------------------------------------------------- # Instrument methods From 1c9244c3a195bf0c53674eb9c069dc4e7d08673e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 6 Jul 2023 12:20:02 -0400 Subject: [PATCH 047/139] DOC: update Changelog Updated the changelog with a description of the PR. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e34ce08..4775c9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Maintenance * Add manual GitHub Actions tests for pysatMadrigal RC * Update GitHub Actions workflows for newer versions of pip + * Added clean warning test attributes to all Instruments [0.1.0] - 2023-04-11 -------------------- From 59b7c1710656b6d9f5c4ee3e293a01dcbf026955 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 10:11:01 -0400 Subject: [PATCH 048/139] ENH: removed IMF data from general inst Removed the IMF data from the general pandas instrument, since it is all contained in one file and requires special handling. --- pysatMadrigal/instruments/madrigal_pandas.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index f1a6fe9..f436304 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -77,7 +77,7 @@ pandas_format = True # Pandas-style data that requires special support -excluded_tags = ['8105', '180', '211'] +excluded_tags = ['8105', '180', '211', '120'] # Assign only tags with pysat-compatible file format strings pandas_codes = general.known_madrigal_inst_codes(pandas_format=True) @@ -100,9 +100,8 @@ # ---------------------------------------------------------------------------- # Instrument test attributes -tag_dates = {'120': dt.datetime(1963, 11, 27), '170': dt.datetime(1998, 7, 1), - '210': dt.datetime(1950, 1, 1), '212': dt.datetime(1957, 1, 1), - '7800': dt.datetime(2009, 11, 10)} +tag_dates = {'170': dt.datetime(1998, 7, 1), '210': dt.datetime(1950, 1, 1), + '212': dt.datetime(1957, 1, 1), '7800': dt.datetime(2009, 11, 10)} _test_dates = {'': {tag: tag_dates[tag] if tag in tag_dates.keys() else tag_dates['7800'] for tag in tags.keys()}} _test_download = {'': {tag: True for tag in tags.keys()}} From 4f545b846e4993732f07c92eae0ffc517705258b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 10:20:29 -0400 Subject: [PATCH 049/139] DOC: updated supported instruments page Added the OMNI IMF instrument to the supported instruments page. --- docs/supported_instruments.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index ffb04f7..7653de0 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -50,7 +50,7 @@ A general instrument for Madrigal time-series data. This :py:class:`pysat.Instrument` uses Madrigal instrument codes and kindats to support the use of any of the Madrigal time-series data sets. There are some further constraints in that the data set's Madrigal naming convention must be -parsable by pysat. Currently nine Madrigal instrument codes are supported by +parsable by pysat. Currently four Madrigal instrument codes are supported by this :py:class:`pysat.Instrument`. When possible, using a specific instrument module is recommended, since that instrument module will have additional support (e.g., cleaning methods, experiment acknowledgements, and references). @@ -72,4 +72,15 @@ time period may be loaded. :members: - +OMNI2_IMF +--------- + +An instrument for the interplanetary magnetic field (IMF) data from Omni 2. The +data starts in 1963 and the entire data set is contained in a single file. The +file is occasionally updated, and so obtaining the most recent data means that +all historic data must also be downloaded (or re-downloaded). OMNI data may +also be obtained more directly using +`pysatNASA `_. + +.. automodule:: pysatMadrigal.instruments.omni2_imf + :members: From 776eae2ab2e174b187b03d38a5308ace98da3bcc Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 10:20:49 -0400 Subject: [PATCH 050/139] DOC: updated changelog Added a description of the current changes to the log. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4775c9c..856a6dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). [0.1.X] - 2023-XX-XX -------------------- * Enhancements + * Moved the OMNI-2 IMF index Instrument from the general Madrigal Pandas + instrument into a new one * Moved the NGDC AE index Instrument from the general Madrigal Pandas instrument to a new one, fixing the Windows memory issue and a problem with duplicated times From d1282d12ded0a5730409a26fb37c6d986f4f05f1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 11:01:30 -0400 Subject: [PATCH 051/139] DOC: update AE docstring Fixed the AE module docstring so that the example doesn't wrongly reference DMSP. --- pysatMadrigal/instruments/ngdc_ae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py index a87ba69..b9eb595 100644 --- a/pysatMadrigal/instruments/ngdc_ae.py +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -37,7 +37,7 @@ import pysat import pysatMadrigal as py_mad - # Download DMSP data from Madrigal + # Download AE data from Madrigal aei = pysat.Instrument(inst_module=py_mad.instruments.ngdc_ae) aei.download(start=py_mad.instruments.ngdc_ae.madrigal_start, user='Firstname+Lastname', password='email@address.com') From 27eb37c3233d83ce1947c9087b07810e4fa513e1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 13:44:26 -0400 Subject: [PATCH 052/139] ENH: added index instruments Added Instruments for the Dst, IMF, and geomagnetic indices provided by Madrigal. --- pysatMadrigal/instruments/__init__.py | 5 +- pysatMadrigal/instruments/madrigal_dst.py | 209 +++++++++++++++++ pysatMadrigal/instruments/madrigal_geoind.py | 225 ++++++++++++++++++ pysatMadrigal/instruments/omni2_imf.py | 232 +++++++++++++++++++ 4 files changed, 670 insertions(+), 1 deletion(-) create mode 100644 pysatMadrigal/instruments/madrigal_dst.py create mode 100644 pysatMadrigal/instruments/madrigal_geoind.py create mode 100644 pysatMadrigal/instruments/omni2_imf.py diff --git a/pysatMadrigal/instruments/__init__.py b/pysatMadrigal/instruments/__init__.py index de82e34..1aefc97 100644 --- a/pysatMadrigal/instruments/__init__.py +++ b/pysatMadrigal/instruments/__init__.py @@ -10,11 +10,14 @@ from pysatMadrigal.instruments import gnss_tec from pysatMadrigal.instruments import jro_isr from pysatMadrigal.instruments import madrigal_pandas +from pysatMadrigal.instruments import madrigal_dst +from pysatMadrigal.instruments import madrigal_geoind from pysatMadrigal.instruments import ngdc_ae +from pysatMadrigal.instruments import omni2_imf # Import Madrigal methods from pysatMadrigal.instruments import methods # noqa F401 # Define variable name with all available instruments __all__ = ['dmsp_ivm', 'dmsp_ssj', 'gnss_tec', 'jro_isr', 'madrigal_pandas', - 'ngdc_ae'] + 'madrigal_dst', 'madrigal_geoind', 'ngdc_ae', 'omni2_imf'] diff --git a/pysatMadrigal/instruments/madrigal_dst.py b/pysatMadrigal/instruments/madrigal_dst.py new file mode 100644 index 0000000..8dea5a5 --- /dev/null +++ b/pysatMadrigal/instruments/madrigal_dst.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# ---------------------------------------------------------------------------- +# -*- coding: utf-8 -*-. +"""Supports access to Dst data archieved at Madrigal. + +Properties +---------- +platform + 'madrigal' +name + 'dst' +tag + None supported +inst_id + None supported + +Note +---- +Please provide name (user) and email (password) when downloading data with this +routine. + +Warnings +-------- +The entire data set (1 Jan 1597 through a period close to the recent day) is +provided in a single file on Madrigal. + +Examples +-------- +:: + + + import datetime as dt + import pysat + import pysatMadrigal as py_mad + + # Download Dst data from Madrigal + dst = pysat.Instrument(inst_module=py_mad.instruments.madrigal_dst) + dst.download(start=py_mad.instruments.madrigal_dst.madrigal_start, + user='Firstname+Lastname', password='email@address.com') + dst.load(date=dt.datetime(1981, 1, 1)) + +""" + +import datetime as dt +import functools + +import pysat + +from pysatMadrigal.instruments.methods import general + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'madrigal' +name = 'dst' +tags = {'': ''} +inst_ids = {'': list(tags.keys())} +pandas_format = True + +# Madrigal tags and limits +madrigal_inst_code = 212 +madrigal_tag = {'': {'': "30006"}} +madrigal_start = dt.datetime(1957, 1, 1) +madrigal_end = dt.datetime.utcnow() + +# Local attributes +# +# Need a way to get the filename strings for a particular instrument unless +# wildcards start working +supported_tags = { + inst_id: {tag: general.madrigal_file_format_str(madrigal_inst_code, + verbose=False) + for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} +remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5') + for kk in inst_ids[ss]} for ss in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {inst_id: {tag: madrigal_start for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} +_test_download = {inst_id: {tag: True for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} +_clean_warn = {inst_id: {tag: {clvl: [('logger', 'WARN', + "No cleaning available", clvl)] + for clvl in ['clean', 'dusty', 'dirty']} + for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +def init(self): + """Initialize the Instrument object in support of Madrigal access. + + Parameters + ---------- + kindat : str + Madrigal instrument experiment code(s). (default='') + + """ + # Set the standard pysat attributes + self.acknowledgements = general.cedar_rules() + self.references = ''.join(['See referenece list and publication at: ', + 'Sugiura M. and T. Kamei, http://', + 'wdc.kugi.kyoto-u.ac.jp/dstdir/dst2/', + 'onDstindex.html, last updated June 1991, ', + 'accessed Dec 2020']) + + # Remind the user of the Rules of the Road + pysat.logger.info(self.acknowledgements) + return + + +def clean(self): + """Raise warning that cleaning is not needed for the OMNI2 data. + + Note + ---- + Supports 'clean', 'dusty', 'dirty' in the sense that it prints + a message noting there is no cleaning. + 'None' is also supported as it signifies no cleaning. + + Routine is called by pysat, and not by the end user directly. + + """ + pysat.logger.warning("No cleaning available for the Madrigal Dst") + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default Madrigal and pysat methods +file_cadence = madrigal_end - madrigal_start +two_digit_year_break = 50 + +# Set the download routine +download = functools.partial(general.download, + inst_code=str(madrigal_inst_code), + kindat=madrigal_tag['']['']) + +# Set the list routine +list_files = functools.partial(general.list_files, + supported_tags=supported_tags, + file_cadence=file_cadence, + two_digit_year_break=two_digit_year_break) + +# Set list_remote_files routine +list_remote_files = functools.partial(general.list_remote_files, + supported_tags=remote_tags, + inst_code=madrigal_inst_code, + kindats=madrigal_tag, + two_digit_year_break=two_digit_year_break) + + +def load(fnames, tag='', inst_id=''): + """Load the Madrigal Dst data. + + Parameters + ----------- + fnames : list + List of filenames + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + + Returns + -------- + data : pds.DataFrame + Object containing IMF data + meta : pysat.Meta + Object containing metadata such as column names and units + + Raises + ------ + ValueError + Unexpected time variable names + + """ + # Cycle through all the filenames, getting the desired start and stop times + fstart = None + fstop = None + for fname_date in fnames: + # Split the date from the filename + fname = fname_date[:-11] + fdate = dt.datetime.strptime(fname_date[-10:], '%Y-%m-%d') + fstop = fdate + + if fstart is None: + fstart = fdate + + fstop += dt.timedelta(days=1) + + # There is only one file for this Instrument + data, meta = general.load([fname], tag=tag, inst_id=inst_id) + + # Select the data for the desired time period + data = data[fstart:fstop] + + return data, meta diff --git a/pysatMadrigal/instruments/madrigal_geoind.py b/pysatMadrigal/instruments/madrigal_geoind.py new file mode 100644 index 0000000..0cf19d3 --- /dev/null +++ b/pysatMadrigal/instruments/madrigal_geoind.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# ---------------------------------------------------------------------------- +# -*- coding: utf-8 -*-. +"""Supports access to F10.7, Kp, and Ap data archieved at Madrigal. + +Properties +---------- +platform + 'madrigal' +name + 'geoind' +tag + None supported +inst_id + None supported + +Note +---- +Please provide name (user) and email (password) when downloading data with this +routine. + +Warnings +-------- +The entire data set (1 Jan 1950 through 31 Dec 1987) is provided in a single +file on Madrigal. + +Examples +-------- +:: + + + import datetime as dt + import pysat + import pysatMadrigal as py_mad + + # Download geophysical indices from Madrigal + gind = pysat.Instrument(inst_module=py_mad.instruments.madrigal_geoind) + gind.download(start=py_mad.instruments.madrigal_geoind.madrigal_start, + user='Firstname+Lastname', password='email@address.com') + gind.load(date=dt.datetime(1981, 1, 1)) + +""" + +import datetime as dt +import functools + +import pysat + +from pysatMadrigal.instruments.methods import general + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'madrigal' +name = 'geoind' +tags = {'': ''} +inst_ids = {'': list(tags.keys())} +pandas_format = True + +# Madrigal tags and limits +madrigal_inst_code = 210 +madrigal_tag = {'': {'': "30007"}} +madrigal_start = dt.datetime(1950, 1, 1) +madrigal_end = dt.datetime.utcnow() + +# Local attributes +# +# Need a way to get the filename strings for a particular instrument unless +# wildcards start working +supported_tags = { + inst_id: {tag: general.madrigal_file_format_str(madrigal_inst_code, + verbose=False) + for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} +remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5') + for kk in inst_ids[ss]} for ss in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {inst_id: {tag: madrigal_start for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} +_test_download = {inst_id: {tag: True for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} +_clean_warn = {inst_id: {tag: {clvl: [('logger', 'WARN', + "No cleaning available", clvl)] + for clvl in ['clean', 'dusty', 'dirty']} + for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +def init(self): + """Initialize the Instrument object in support of Madrigal access. + + Parameters + ---------- + kindat : str + Madrigal instrument experiment code(s). (default='') + + """ + # Set the standard pysat attributes + self.acknowledgements = general.cedar_rules() + self.references = "\n".join( + [''.join(["Covington, A.E. (1948), Solar noise observations on 10.7 ", + "centimeters Solar noise observations on 10.7 centimeters,", + " Proceedings of the IRE, 36(44), p 454-457."]), + ''.join(["J. Bartels, The technique of scaling indices K and Q of ", + "geomagnetic activity, Ann. Intern. Geophys. Year 4, ", + "215-226, 1957."]), + ''.join(["J. Bartels, The geomagnetic measures for the time-", + "variations of solar corpuscular radiation, described for ", + "use in correlation studies in other geophysical fields, ", + "Ann. Intern. Geophys. Year 4, 227-236, 1957."]), + ''.join(["P.N. Mayaud, Derivation, Meaning and Use of Geomagnetic ", + "Indices, Geophysical Monograph 22, Am. Geophys. Union, ", + "Washington D.C., 1980."]), + ''.join(["G.K. Rangarajan, Indices of magnetic activity, in ", + "Geomagnetism, edited by I.A. Jacobs, Academic, San Diego,", + " 1989."]), + ''.join(["M. Menvielle and A. Berthelier, The K-derived planetary ", + "indices: description and availability, Rev. Geophys. 29, ", + "3, 415-432, 1991."])]) + + # Remind the user of the Rules of the Road + pysat.logger.info(self.acknowledgements) + return + + +def clean(self): + """Raise warning that cleaning is not needed for the OMNI2 data. + + Note + ---- + Supports 'clean', 'dusty', 'dirty' in the sense that it prints + a message noting there is no cleaning. + 'None' is also supported as it signifies no cleaning. + + Routine is called by pysat, and not by the end user directly. + + """ + pysat.logger.warning( + "No cleaning available for the Madrigal geophysical indices") + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default Madrigal and pysat methods +file_cadence = madrigal_end - madrigal_start +two_digit_year_break = 50 + +# Set the download routine +download = functools.partial(general.download, + inst_code=str(madrigal_inst_code), + kindat=madrigal_tag['']['']) + +# Set the list routine +list_files = functools.partial(general.list_files, + supported_tags=supported_tags, + file_cadence=file_cadence, + two_digit_year_break=two_digit_year_break) + +# Set list_remote_files routine +list_remote_files = functools.partial(general.list_remote_files, + supported_tags=remote_tags, + inst_code=madrigal_inst_code, + kindats=madrigal_tag, + two_digit_year_break=two_digit_year_break) + + +def load(fnames, tag='', inst_id=''): + """Load the Madrigal Dst data. + + Parameters + ----------- + fnames : list + List of filenames + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + + Returns + -------- + data : pds.DataFrame + Object containing IMF data + meta : pysat.Meta + Object containing metadata such as column names and units + + Raises + ------ + ValueError + Unexpected time variable names + + """ + # Cycle through all the filenames, getting the desired start and stop times + fstart = None + fstop = None + for fname_date in fnames: + # Split the date from the filename + fname = fname_date[:-11] + fdate = dt.datetime.strptime(fname_date[-10:], '%Y-%m-%d') + fstop = fdate + + if fstart is None: + fstart = fdate + + fstop += dt.timedelta(days=1) + + # There is only one file for this Instrument + data, meta = general.load([fname], tag=tag, inst_id=inst_id) + + # Select the data for the desired time period + data = data[fstart:fstop] + + return data, meta diff --git a/pysatMadrigal/instruments/omni2_imf.py b/pysatMadrigal/instruments/omni2_imf.py new file mode 100644 index 0000000..d06296e --- /dev/null +++ b/pysatMadrigal/instruments/omni2_imf.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# ---------------------------------------------------------------------------- +# -*- coding: utf-8 -*-. +"""Supports access to OMNI 2 IMF data archieved at Madrigal. + +Properties +---------- +platform + 'omni2' +name + 'imf' +tag + None supported +inst_id + None supported + +Note +---- +Please provide name (user) and email (password) when downloading data with this +routine. + +Glenn Campbell and Bill Rideout completely rebuilt the Madrigal interplanetary +magnetic field data using data from: +ftp://nssdcftp.gsfc.nasa.gov/spacecraft_data/omni/. The old file had +originally come from Cedar and had gaps even in places where there was data +available. This new Madrigal file is based on the Omni 2 data set, described +at http://nssdc.gsfc.nasa.gov/omniweb/. (4 May 2004, brideout@haystack.mit.edu) + +The OMNI data may be directly downloaded using pysatNASA and is now described +at: https://omniweb.gsfc.nasa.gov/html/ow_data.html + +Warnings +-------- +The entire data set (27 Nov 1963 through 3 Jun 2019) is provided in a single +file on Madrigal. The download method will break this file up by year for +easier access. + +Examples +-------- +:: + + + import datetime as dt + import pysat + import pysatMadrigal as py_mad + + # Download IMF data from Madrigal + imf = pysat.Instrument(inst_module=py_mad.instruments.omni2_imf) + imf.download(start=py_mad.instruments.omni2_imf.madrigal_start, + user='Firstname+Lastname', password='email@address.com') + imf.load(date=dt.datetime(1981, 1, 1)) + +""" + +import datetime as dt +import functools + +import pysat + +from pysatMadrigal.instruments.methods import general + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'omni2' +name = 'imf' +tags = {'': ''} +inst_ids = {'': list(tags.keys())} +pandas_format = True + +# Madrigal tags and limits +madrigal_inst_code = 120 +madrigal_tag = {'': {'': "30012"}} +madrigal_start = dt.datetime(1963, 11, 27) +madrigal_end = dt.datetime(2019, 6, 4) + +# Local attributes +# +# Need a way to get the filename strings for a particular instrument unless +# wildcards start working +supported_tags = { + inst_id: {tag: general.madrigal_file_format_str(madrigal_inst_code, + verbose=False) + for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} +remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5') + for kk in inst_ids[ss]} for ss in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {inst_id: {tag: madrigal_start for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} +_test_download = {inst_id: {tag: True for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} +_clean_warn = {inst_id: {tag: {clvl: [('logger', 'WARN', + "No cleaning available", clvl)] + for clvl in ['clean', 'dusty', 'dirty']} + for tag in inst_ids[inst_id]} + for inst_id in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +def init(self): + """Initialize the Instrument object in support of Madrigal access. + + Parameters + ---------- + kindat : str + Madrigal instrument experiment code(s). (default='') + + """ + # Set the standard pysat attributes + self.acknowledgements = ''.join([general.cedar_rules(), '\nFor full ', + 'acknowledgement info, please see: ', + 'https://omniweb.gsfc.nasa.gov/html/', + 'citing.html']) + self.references = ' '.join(('J.H. King and N.E. Papitashvili, Solar', + 'wind spatial scales in and comparisons', + 'of hourly Wind and ACE plasma and', + 'magnetic field data, J. Geophys. Res.,', + 'Vol. 110, No. A2, A02209,', + '10.1029/2004JA010649.')) + + # Remind the user of the Rules of the Road + pysat.logger.info(self.acknowledgements) + return + + +def clean(self): + """Raise warning that cleaning is not needed for the OMNI2 data. + + Note + ---- + Supports 'clean', 'dusty', 'dirty' in the sense that it prints + a message noting there is no cleaning. + 'None' is also supported as it signifies no cleaning. + + Routine is called by pysat, and not by the end user directly. + + """ + pysat.logger.warning("No cleaning available for the collected Omni 2 IMF") + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default Madrigal and pysat methods +file_cadence = madrigal_end - madrigal_start +two_digit_year_break = 50 + +# Set the download routine +download = functools.partial(general.download, + inst_code=str(madrigal_inst_code), + kindat=madrigal_tag['']['']) + +# Set the list routine +list_files = functools.partial(general.list_files, + supported_tags=supported_tags, + file_cadence=file_cadence, + two_digit_year_break=two_digit_year_break) + +# Set list_remote_files routine +list_remote_files = functools.partial(general.list_remote_files, + supported_tags=remote_tags, + inst_code=madrigal_inst_code, + kindats=madrigal_tag, + two_digit_year_break=two_digit_year_break) + + +def load(fnames, tag='', inst_id=''): + """Load the OMNI2 IMF data. + + Parameters + ----------- + fnames : list + List of filenames + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + + Returns + -------- + data : pds.DataFrame + Object containing IMF data + meta : pysat.Meta + Object containing metadata such as column names and units + + Raises + ------ + ValueError + Unexpected time variable names + + """ + # Cycle through all the filenames, getting the desired start and stop times + fstart = None + fstop = None + for fname_date in fnames: + # Split the date from the filename + fname = fname_date[:-11] + fdate = dt.datetime.strptime(fname_date[-10:], '%Y-%m-%d') + fstop = fdate + + if fstart is None: + fstart = fdate + + fstop += dt.timedelta(days=1) + + # There is only one file for this Instrument + data, meta = general.load([fname], tag=tag, inst_id=inst_id) + + # Test to see if there is data beyond the expected file end + if data.index[-1] > madrigal_end: + pysat.logger.critical(''.join(['There is data beyond ', + '{:}'.format(madrigal_end), ' in the ', + 'Omni2 IMF file, please notify the ', + 'pysatMadrigal developers so that they ', + 'can update this Instrument'])) + + # Select the data for the desired time period + data = data[fstart:fstop] + + return data, meta From 4306dac46f6b96ba898b965dbc9525d8ded28c51 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 13:45:08 -0400 Subject: [PATCH 053/139] MAINT: removed support for single-file data sets Removed general support for the index data sets, which require special processing because all data is contained in a single file. --- pysatMadrigal/instruments/madrigal_pandas.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index f436304..1a2c283 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -77,7 +77,7 @@ pandas_format = True # Pandas-style data that requires special support -excluded_tags = ['8105', '180', '211', '120'] +excluded_tags = ['120', '180', '210', '211', '212', '8105'] # Assign only tags with pysat-compatible file format strings pandas_codes = general.known_madrigal_inst_codes(pandas_format=True) @@ -100,8 +100,7 @@ # ---------------------------------------------------------------------------- # Instrument test attributes -tag_dates = {'170': dt.datetime(1998, 7, 1), '210': dt.datetime(1950, 1, 1), - '212': dt.datetime(1957, 1, 1), '7800': dt.datetime(2009, 11, 10)} +tag_dates = {'170': dt.datetime(1998, 7, 1), '7800': dt.datetime(2009, 11, 10)} _test_dates = {'': {tag: tag_dates[tag] if tag in tag_dates.keys() else tag_dates['7800'] for tag in tags.keys()}} _test_download = {'': {tag: True for tag in tags.keys()}} From 7520cb7de89c18d92a7e7b6e43aff9ec1ff6e135 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 13:45:41 -0400 Subject: [PATCH 054/139] BUG: fixed file times Fixed the file time range determination in the load function. Also updated the docstrings to be more correct. --- pysatMadrigal/instruments/ngdc_ae.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py index b9eb595..7cb1e54 100644 --- a/pysatMadrigal/instruments/ngdc_ae.py +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -25,8 +25,7 @@ Warnings -------- The entire data set (1 Jan 1978 through 31 Dec 1987) is provided in a single -file on Madrigal. The download method will break this file up by year for -easier access. +file on Madrigal. Examples -------- @@ -222,12 +221,12 @@ def load(fnames, tag='', inst_id=''): # Split the date from the filename fname = fname_date[:-11] fdate = dt.datetime.strptime(fname_date[-10:], '%Y-%m-%d') + fstop = fdate if fstart is None: fstart = fdate - if fstop is None: - fstop = fdate + fstop += dt.timedelta(days=1) # There is only one file for this Instrument with h5py.File(fname, 'r') as filed: @@ -280,7 +279,7 @@ def load(fnames, tag='', inst_id=''): day=day) # Get the data mask - dmask = (fdate >= fstart) & (fdate <= fstop) + dmask = (fdate >= fstart) & (fdate < fstop) # Construct the time index hour = file_data[dmask]['hour'] From 02f7e9381a0f11c5e684fb0d9235965830e7f809 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 13:46:00 -0400 Subject: [PATCH 055/139] DOC: added new instruments to docs Added the new index instruments to the docs. --- docs/supported_instruments.rst | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 7653de0..53f986d 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -43,6 +43,32 @@ magnetic equator over Peru. :members: +Madrigal_Dst +------------ + +An instrument for the Madrigal Dst index data. This data set spans the years +of 1957 through a period close to today, with all data saved in a single file. +Because of this, you only need to download the data once and any desired time +period may be loaded (unless you require a time between your last download and +now). + +.. automodule:: pysatMadrigal.instruments.madrigal_dst + :members: + + +Madrigal_Geoind +--------------- + +An instrument for the Madrigal geomagnetic index data. This data set spans the +years of 1950 through a period close to today, with all data saved in a single +file. Because of this, you only need to download the data once and any desired +time period may be loaded (unless you require a time between your last download +and now). + +.. automodule:: pysatMadrigal.instruments.madrigal_geoind + :members: + + Madrigal_Pandas --------------- @@ -50,7 +76,7 @@ A general instrument for Madrigal time-series data. This :py:class:`pysat.Instrument` uses Madrigal instrument codes and kindats to support the use of any of the Madrigal time-series data sets. There are some further constraints in that the data set's Madrigal naming convention must be -parsable by pysat. Currently four Madrigal instrument codes are supported by +parsable by pysat. Currently three Madrigal instrument codes are supported by this :py:class:`pysat.Instrument`. When possible, using a specific instrument module is recommended, since that instrument module will have additional support (e.g., cleaning methods, experiment acknowledgements, and references). From f56238c684a4a152f04d715b0aa56c185fb18f82 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 13:46:15 -0400 Subject: [PATCH 056/139] DOC: updated changelog Added the other instruments to the changelog. --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 856a6dd..e6fd2d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). [0.1.X] - 2023-XX-XX -------------------- * Enhancements - * Moved the OMNI-2 IMF index Instrument from the general Madrigal Pandas - instrument into a new one + * Moved the OMNI-2 IMF, Dst, and Geomagnetic index Instruments from the + general Madrigal Pandas instrument into new ones * Moved the NGDC AE index Instrument from the general Madrigal Pandas instrument to a new one, fixing the Windows memory issue and a problem with duplicated times From 1b0e17e14ed9c20306495eed9fffade9d30b86fa Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 31 Jul 2023 13:51:31 -0400 Subject: [PATCH 057/139] STY: fixed import order Fixed the import order to be alphabetical. --- pysatMadrigal/instruments/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatMadrigal/instruments/__init__.py b/pysatMadrigal/instruments/__init__.py index 1aefc97..814cc06 100644 --- a/pysatMadrigal/instruments/__init__.py +++ b/pysatMadrigal/instruments/__init__.py @@ -9,9 +9,9 @@ from pysatMadrigal.instruments import dmsp_ssj from pysatMadrigal.instruments import gnss_tec from pysatMadrigal.instruments import jro_isr -from pysatMadrigal.instruments import madrigal_pandas from pysatMadrigal.instruments import madrigal_dst from pysatMadrigal.instruments import madrigal_geoind +from pysatMadrigal.instruments import madrigal_pandas from pysatMadrigal.instruments import ngdc_ae from pysatMadrigal.instruments import omni2_imf @@ -19,5 +19,5 @@ from pysatMadrigal.instruments import methods # noqa F401 # Define variable name with all available instruments -__all__ = ['dmsp_ivm', 'dmsp_ssj', 'gnss_tec', 'jro_isr', 'madrigal_pandas', - 'madrigal_dst', 'madrigal_geoind', 'ngdc_ae', 'omni2_imf'] +__all__ = ['dmsp_ivm', 'dmsp_ssj', 'gnss_tec', 'jro_isr', 'madrigal_dst', + 'madrigal_geoind', 'madrigal_pandas', 'ngdc_ae', 'omni2_imf'] From b670b2b2bf20c68c0184b45ae97fd71eb2900367 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 1 Aug 2023 10:30:12 -0400 Subject: [PATCH 058/139] MAINT: updated JRO link Updates the link to the Jicamarca Radio Observatory. --- docs/supported_instruments.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 53f986d..3708be5 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -35,9 +35,9 @@ JRO_ISR ------- The incoherent scatter radar (ISR) at the -`Jicamarca Radio Observatory `_ regularly -measures the velocity, density, and other ionospheric characteristics near the -magnetic equator over Peru. +`Jicamarca Radio Observatory `_ +regularly measures the velocity, density, and other ionospheric characteristics +near the magnetic equator over Peru. .. automodule:: pysatMadrigal.instruments.jro_isr :members: From 34cdb01c24dcbaccb96a32e5587b78cfb57cb2fd Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:10:37 -0400 Subject: [PATCH 059/139] DOC: update issue templates Fixed typos in the issue templates. --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/question.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a7f6ad6..5764018 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -26,4 +26,4 @@ solve the problem. - Other details about your setup that could be relevant # Additional context -Add any other context about the problem here. +Add any other context about the problem here, including expected behaviour. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index 2073086..da43edc 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -15,5 +15,5 @@ the full context of your question. ## Configuration - OS: [e.g. Hal] - - Version [e.g. Python 3.47] + - Version: [e.g. Python 3.47] - Other details about your setup that could be relevant From ba34e52b4712a4085673e87e07b7c58a2820aaff Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:11:01 -0400 Subject: [PATCH 060/139] DOC: updated PR template Added new item for the PR checklist. --- .github/pull_request_template.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5fa8548..24fa713 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,6 @@ # Description -Addresses # (issue) +Addresses #(issue) Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required @@ -47,6 +47,7 @@ Test B - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules - [ ] Add a note to ``CHANGELOG.md``, summarizing the changes +- [ ] Update zenodo.json file for new code contributors If this is a release PR, replace the first item of the above checklist with the release checklist on the pysat wiki: From 0628aa93d21fafbf09d65d31b8e84fe69c1ceba7 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:11:32 -0400 Subject: [PATCH 061/139] TST: updated test yamls Updated the test yamls to use pyproject.toml and cycle the version support. --- .github/workflows/docs.yml | 16 +++++++--------- .github/workflows/main.yml | 24 ++++++++++++++++++------ .github/workflows/pip_rc_install.yml | 9 +++++---- .github/workflows/pysat_rc.yml | 16 +++++++--------- 4 files changed, 37 insertions(+), 28 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0c02453..ec7a09e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,5 +1,6 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow will install Python dependencies, run tests and lint with a +# variety of Python versions. For more information see: +# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Documentation Check @@ -8,11 +9,11 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ["ubuntu-latest"] strategy: fail-fast: false matrix: - python-version: [3.9] + python-version: ["3.9"] name: Documentation tests steps: @@ -21,11 +22,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r test_requirements.txt - pip install -r requirements.txt + - name: Install with dependencies + run: pip install .[doc] - name: Set up pysat run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 58767c5..cb9a144 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,15 +13,19 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.9", "3.10"] + python-version: ["3.10", "3.11"] numpy_ver: ["latest"] include: - - python-version: "3.8" + # NEP29 compliance settings + - python-version: "3.9" numpy_ver: "1.21" os: ubuntu-latest + test_config: "NEP29" + # Operational compliance settings - python-version: "3.6.8" numpy_ver: "1.19.5" os: "ubuntu-20.04" + test_config: "Ops" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} @@ -32,15 +36,23 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install standard dependencies + - name: Install Operational dependencies + if: ${{ matrix.test_config == 'Ops'}} run: | + pip install numpy==${{ matrix.numpy_ver }} + pip install -r requirements.txt pip install -r test_requirements.txt - pip install -r requirements.txt; + pip install . - name: Install NEP29 dependencies - if: ${{ matrix.numpy_ver != 'latest'}} + if: ${{ matrix.test_config == 'NEP29'}} run: | pip install numpy==${{ matrix.numpy_ver }} + pip install --upgrade-strategy only-if-needed .[test] + + - name: Install standard dependencies + if: ${{ matrix.test_config == 'latest'}} + run: pip install .[test] - name: Set up pysat run: | @@ -54,7 +66,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: pytest -x --cov=pysatMadrigal/ + run: pytest - name: Publish results to coveralls env: diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index 6b9d2d6..d9d3f41 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -1,6 +1,7 @@ -# This workflow will install Python dependencies and the latest RC of pysatNASA from test pypi. -# This test should be manually run before a pysatMadrigal RC is officially approved and versioned. -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow will install Python dependencies and the latest RC of +# pysatMadrigal from test pypi. This test should be manually run before an RC is +# officially approved and versioned. For more information see: +# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Test install of latest RC from pip @@ -12,7 +13,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10"] # Keep this version at the highest supported Python version + python-version: ["3.11"] # Keep this version at the highest supported Python version name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 6d82336..924ebbb 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -1,5 +1,6 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow will install Python dependencies, run tests and lint with a +# variety of Python versions. For more information see: +# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Test with latest pysat RC @@ -10,8 +11,7 @@ jobs: strategy: fail-fast: false matrix: -# TODO(#76): add windows tests when bugs are sorted - os: [ubuntu-latest, macos-latest] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.10"] name: Python ${{ matrix.python-version }} on ${{ matrix.os }} @@ -26,10 +26,8 @@ jobs: - name: Install pysat RC run: pip install --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat - - name: Install standard dependencies - run: | - pip install -r requirements.txt - pip install -r test_requirements.txt + - name: Install standard dependencies and package + run: run pip install .[test] - name: Set up pysat run: | @@ -37,7 +35,7 @@ jobs: python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" - name: Test with pytest - run: pytest -vs -x --cov=pysatMadrigal/ + run: pytest - name: Publish results to coveralls env: From bd7400f2a1bcf08f687fc9e6c2e5bbcd43472b9e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:11:50 -0400 Subject: [PATCH 062/139] ENH: added readthedocs yaml Added a readthedocs yaml. --- .readthedocs.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..e166702 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,25 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required version of readthedocs +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + + +# Declare the Python requirements required to build your docs +python: + install: + - method: pip + path: . + extra_requirements: + - doc From 725e2d796cd345ef6d6a6dbf49126457957c53ad Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:12:20 -0400 Subject: [PATCH 063/139] DOC: updated code of conduct Added missing section to the code of conduct. --- CODE_OF_CONDUCT.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 483ea9a..ee4a9e7 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -56,9 +56,9 @@ project may be further defined and clarified by project maintainers. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at pysat.developers@gmail.com. The -project team will review and investigate all complaints, and will respond in a -way that it deems appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an +pysatMadrigal project team will review and investigate all complaints, and will +respond in a way that it deems appropriate to the circumstances. The project +team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. @@ -69,7 +69,14 @@ members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 1.4, available at [https://contributor-covenant.org/version/1/4][version] +version 1.4, available at +[https://contributor-covenant.org/version/1/4][version] + +## FAQ + +For answers to common questions about this code of conduct, see +[https://www.contributor-covenant.org/faq][faq] [homepage]: https://contributor-covenant.org [version]: https://contributor-covenant.org/version/1/4/ +[faq]: https://www.contributor-covenant.org/faq From 89bd7cdd50239108b10af268e3c73e9fd86778c5 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:12:50 -0400 Subject: [PATCH 064/139] DOC: updated contributing guidelines Fixed format and updated style requirements in the contributing guidelines. --- CONTRIBUTING.md | 136 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 93 insertions(+), 43 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a81865e..2c0b357 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,15 +5,21 @@ Bug reports, feature suggestions and other contributions are greatly appreciated! pysat and pysatMadrigal are community-driven projects that welcome both feedback and contributions. +Come join us on Slack! An invitation to the pysat workspace is available +in the 'About' section of the +[pysat GitHub Repository.](https://github.com/pysat/pysat) Development meetings +are generally held fortnightly. + Short version ------------- * Submit bug reports, feature requests, and questions at -`GitHub Issues `_ +[GitHub](https://github.com/pysat/pysatMadrigal/issues) + * Make pull requests to the ``develop`` branch -More about Issues ------------------ +Issues +------ Bug reports, questions, and feature requests should all be made as GitHub Issues. Templates are provided for each type of issue, to help you include @@ -24,73 +30,112 @@ Questions Not sure how something works? Ask away! The more information you provide, the easier the question will be to answer. You can also interact with the pysat -developers on our `slack channel `_. +developers on our [slack channel](https://pysat.slack.com). Bug reports ^^^^^^^^^^^ -When reporting a bug please include: +When [reporting a bug](https://github.com/pysat/pysatMadrigal/issues) please +include: * Your operating system name and version + * Any details about your local setup that might be helpful in troubleshooting + * Detailed steps to reproduce the bug -Feature requests -^^^^^^^^^^^^^^^^ +Feature requests and feedback +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The best way to send feedback is to file an issue at +[GitHub](https://github.com/pysat/pysatMadrigal/issues). If you are proposing a new feature or a change in something that already exists: * Explain in detail how it would work. + * Keep the scope as narrow as possible, to make it easier to implement. + * Remember that this is a volunteer-driven project, and that code contributions are welcome :) -More about Development ----------------------- +Development +----------- To set up `pysatMadrigal` for local development: -1. Fork pysatMadrigal on - `GitHub `_. -2. Clone your fork locally:: +1. Fork pysatMadrigal on [GitHub](https://github.com/pysat/pysatMadrigal/fork). + +2. Clone your fork locally: + ``` git clone git@github.com:your_name_here/pysatMadrigal.git + ``` -3. Create a branch for local development:: +3. Create a branch for local development: + ``` git checkout -b name-of-your-bugfix-or-feature + ``` -4. Make your changes locally. Tests for new instruments are performed - automatically. Tests for custom functions should be added to the - appropriately named file in ``pysatMadrigal/tests``. For example, - Jicamarca methods containined in ``pysatMadrigal/instruments/methods/jro.py`` - should be named ``pysatMadrigal/tests/test_methods_jro.py``. If no test - file exists, then you should create one. This testing uses pytest, which - will run tests on any python file in the test directory that starts with - ``test``. Test classes must begin with ``Test``, and test methods must also - begin with ``test``. + Now you can make your changes locally. -5. When you're done making changes, run all the checks to ensure that nothing - is broken on your local system:: + Tests for new instruments are performed automatically. See discussion + [here](https://pysat.readthedocs.io/en/main/new_instrument.html#testing-support) + for more information on triggering these standard tests. - pytest -vs pysatMadrigal + Tests for custom functions should be added to the appropriately named file + in ``pysatMadrigal/tests``. For example, custom functions for the time + utilities are tested in ``pysat/tests/test_utils_time.py``. If no test file + exists, then you should create one. This testing uses pytest, which will run + tests on any python file in the test directory that starts with ``test``. + Classes must begin with ``Test``, and methods must begin with ``test`` as + well. + +4. When you're done making changes, run all the checks to ensure that nothing + is broken on your local system: + + ``` + pytest + + ``` + +5. You should also check for flake8 style compliance: + + ``` + flake8 . --count --select=D,E,F,H,W --show-source --statistics + ``` + + Note that pysat uses the `flake-docstrings` and `hacking` packages to ensure + standards in docstring formatting. 6. Update/add documentation (in ``docs``). Even if you don't think it's relevant, check to see if any existing examples have changed. 7. Add your name to the .zenodo.json file as an author -8. Commit your changes and push your branch to GitHub:: +8. Commit your changes: + ``` + git add . + git commit -m "AAA: Brief description of your changes" + ``` + Where AAA is a standard shorthand for the type of change (eg, BUG or DOC). + `pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html), + see the discussion there for a full list of this shorthand notation. - git add . - git commit -m "Brief description of your changes" - git push origin name-of-your-bugfix-or-feature +9. Once you are happy with the local changes, push to GitHub: + ``` + git push origin name-of-your-bugfix-or-feature + ``` + Note that each push will trigger the Continuous Integration workflow. + +10. Submit a pull request through the GitHub website. Pull requests should be + made to the ``develop`` branch. Note that automated tests will be run on + GitHub Actions, but these must be initialized by a member of the pysat team. -9. Submit a pull request through the GitHub website. Pull requests should be - made to the ``develop`` branch. Pull Request Guidelines -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- If you need some code review or feedback while you're developing the code, just make a pull request. Pull requests should be made to the ``develop`` branch. @@ -99,17 +144,18 @@ For merging, you should: 1. Include an example for use 2. Add a note to ``CHANGELOG.md`` about the changes -3. Ensure that all checks passed (current checks include Travis-CI - and Coveralls) [1]_ +3. Ensure that all checks passed (current checks include GitHub Actions and + Coveralls) + +If you don't have all the necessary Python versions available locally or have +trouble building all the testing environments, you can rely on GitHub Actions to +run the tests for each change you add in the pull request. Because testing here +will delay tests by other developers, please ensure that the code passes all +tests on your local system first. -.. [1] If you don't have all the necessary Python versions available locally or - have trouble building all the testing environments, you can rely on - Travis to run the tests for each change you add in the pull request. - Because testing here will delay tests by other developers, please ensure - that the code passes all tests on your local system first. Project Style Guidelines -^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------ In general, pysat follows PEP8 and numpydoc guidelines. Pytest runs the unit and integration tests, flake8 checks for style, and sphinx-build performs @@ -130,15 +176,19 @@ These include: * `import numpy as np` * `import pandas as pds` * `import xarray as xr` +* When incrementing a timestamp, use `dt.timedelta` instead of `pds.DateOffset` + when possible to reduce program runtime * All classes should have `__repr__` and `__str__` functions * Docstrings use `Note` instead of `Notes` * Try to avoid creating a try/except statement where except passes -* Use setup and teardown in test classes +* Use setup_method (or setup_class) and teardown_method (or teardown_class) in + test classes * Use pytest parametrize in test classes when appropriate +* Use pysat testing utilities when appropriate * Provide testing class methods with informative failure statements and descriptive, one-line docstrings * Block and inline comments should use proper English grammar and punctuation with the exception of single sentences in a block, which may then omit the final period -* When casting is necessary, use `np.int64` and `np.float64` to ensure operating - system agnosticism +* When casting is necessary, use `np.int64` and `np.float64` to ensure operating + system agnosticism From fa5af6079d42cd44da2349fa66d26aca4a464850 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:13:12 -0400 Subject: [PATCH 065/139] MAINT: added pyproject.toml Replaced the setup.py with pyproject.toml. --- pyproject.toml | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 65 +----------------------------------- setup.py | 13 -------- 3 files changed, 90 insertions(+), 77 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..15c87cc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,89 @@ +[build-system] +requires = ["setuptools >= 38.6", "pip >= 10"] +build-backend = "setuptools.build_meta" + +[project] +name = "pysatMadrigal" +version = "0.1.0" +description = 'Madrigal instrument support for the pysat ecosystem' +readme = "README.md" +requires-python = ">=3.6" +license = {file = "LICENSE"} +authors = [ + {name = "Angeline G. Burrell, et al.", email = "pysat.developers@gmail.com"} +] +classifiers = [ + "Development Status :: 4 - Beta", + "Topic :: Scientific/Engineering :: Physics", + "Topic :: Scientific/Engineering :: Atmospheric Science", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows" +] +keywords = [ + "pysat", + "ionosphere", + "Madrigal", + "CEDAR", + "thermosphere", + "GPS", + "GNSS", + "TEC", + "Jicamarca", + "DMSP", + "ISR", + "Incoherent scatter radar", +] +dependencies = [ + "h5py", + "madrigalWeb", + "numpy", + "packaging", + "pandas", + "pysat>=3.1.0", + "xarray"] + +[project.optional-dependencies] +test = [ + "coveralls < 3.3", + "flake8", + "flake8-docstrings", + "hacking >= 1.0", + "pytest", + "pytest-cov", + "pytest-ordering" +] +doc = [ + "extras_require", + "ipython", + "m2r2", + "numpydoc", + "pyproject_parser", + "sphinx", + "sphinx_rtd_theme >= 1.2.2" +] + +[project.urls] +Documentation = "https://pysatmadrigal.readthedocs.io/en/latest/" +Source = "https://github.com/pysat/pysatMadrigal" + +[tool.coverage.report] + +[tool.pytest.ini_options] +addopts = "-x --cov=pysatMadrigal" +markers = [ + "all_inst", + "download", + "no_download", + "load_options", + "first", + "second" +] diff --git a/setup.cfg b/setup.cfg index 6e68fba..a2f4510 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,72 +1,9 @@ [metadata] name = pysatMadrigal -version = file: pysatMadrigal/version.txt -url = https://github.com/pysat/pysatMadrigal -author = Angeline G. Burrell, et al. -author_email = pysat.developers@gmail.com -description = 'Madrigal instrument support for the pysat ecosystem' -keywords = - pysat - ionosphere - Madrigal - CEDAR - thermosphere - GPS - GNSS - TEC - Jicamarca - DMSP - ISR - Incoherent scatter radar -classifiers = - Development Status :: 4 - Beta - Topic :: Scientific/Engineering :: Physics - Topic :: Scientific/Engineering :: Atmospheric Science - Intended Audience :: Science/Research - License :: OSI Approved :: BSD License - Natural Language :: English - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Operating System :: MacOS :: MacOS X - Operating System :: POSIX :: Linux - Operating System :: POSIX :: Linux - Operating System :: Microsoft :: Windows -license_file = LICENSE -long_description = file: README.md -long_description_content_type = text/markdown - -[options] -python_requires = >= 3.6 -setup_requires = setuptools >= 38.6; pip >= 10 -include_package_data = True -zip_safe = False -packages = find: -install_requires = h5py - madrigalWeb - numpy - packaging - pandas - pysat - xarray - -[coverage:report] -omit = */instruments/templates/* +version = 0.1.0 [flake8] max-line-length = 80 ignore = W503 D200 D202 - -[tool:pytest] -markers = - all_inst: tests all instruments - download: tests for downloadable instruments - no_download: tests for instruments without download support - load_options: tests for instruments with additional options - first: first tests to run - second: second tests to run diff --git a/setup.py b/setup.py deleted file mode 100644 index 2e9b3e7..0000000 --- a/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2020, Authors -# Full license can be found in License.md and AUTHORS.md -# ----------------------------------------------------------------------------- -"""Package setup.""" - -from setuptools import setup - - -# Run setup. Setuptools will look for parameters in [metadata] section of -# setup.cfg -setup() From c18dce743860eeec861dc590dd3db2834b87c361 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:13:35 -0400 Subject: [PATCH 066/139] DOC: updated README Fixed broken links and updated dependencies in the README. --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 00b7599..0ae27ba 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ pysatMadrigal -# pysatMadrigal +# pysatMadrigal: pysat support for Madrigal data sets [![Documentation Status](https://readthedocs.org/projects/pysatmadrigal/badge/?version=latest)](https://pysatmadrigal.readthedocs.io/en/latest/?badge=latest) -[![Build Status](https://github.com/github/docs/actions/workflows/main.yml/badge.svg)](https://github.com/github/docs/actions/workflows/main.yml/badge.svg) +[![Pytest with Flake8](https://github.com/pysat/pysatMadrigal/actions/workflows/main.yml/badge.svg)](https://github.com/pysat/pysatMadrigal/actions/workflows/main.yml) [![Coverage Status](https://coveralls.io/repos/github/pysat/pysatMadrigal/badge.svg?branch=main)](https://coveralls.io/github/pysat/pysatMadrigal?branch=main) [![DOI](https://zenodo.org/badge/258384773.svg)](https://zenodo.org/badge/latestdoi/258384773) [![PyPI version](https://badge.fury.io/py/pysatMadrigal.svg)](https://badge.fury.io/py/pysatMadrigal) @@ -27,7 +27,7 @@ for the Space Physics community. This module officially supports Python 3.7+. | Common modules | Community modules | | -------------- | ----------------- | | h5py | madrigalWeb>=2.6 | -| numpy | pysat >= 3.0.3 | +| numpy | pysat >= 3.1.0 | | pandas | | | xarray | | @@ -69,3 +69,6 @@ Afterward, pysat will identify them using the `platform` and `name` keywords. pysat.utils.registry.register('pysatMadrigal.instruments.dmsp_ivm') dst = pysat.Instrument('dmsp', 'ivm', tag='utd', inst_id='f15') ``` + +The package also include analysis tools. Detailed examples are in the +[documentation](https://pysatmadrigal.readthedocs.io/en/latest/?badge=latest). From 527f68cc734406901f3629c9a30506aa7de77223 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:13:55 -0400 Subject: [PATCH 067/139] MAINT: updated version support Updated the version support to use pyproject.toml. --- pysatMadrigal/__init__.py | 13 ++++++------- pysatMadrigal/version.txt | 1 - 2 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 pysatMadrigal/version.txt diff --git a/pysatMadrigal/__init__.py b/pysatMadrigal/__init__.py index 654bab7..3c72456 100644 --- a/pysatMadrigal/__init__.py +++ b/pysatMadrigal/__init__.py @@ -5,13 +5,12 @@ """ -import os +try: + from importlib import metadata +except ImportError: + import importlib_metadata as metadata + from pysatMadrigal import instruments # noqa F401 from pysatMadrigal import utils # noqa F401 -# set version -here = os.path.abspath(os.path.dirname(__file__)) -version_filename = os.path.join(here, 'version.txt') -with open(version_filename, 'r') as version_file: - __version__ = version_file.read().strip() -del here, version_filename, version_file +__version__ = metadata.version('pysatMadrigal') diff --git a/pysatMadrigal/version.txt b/pysatMadrigal/version.txt deleted file mode 100644 index 6e8bf73..0000000 --- a/pysatMadrigal/version.txt +++ /dev/null @@ -1 +0,0 @@ -0.1.0 From bb889a395ba4c486eb04aac879d2ac89f9a91029 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 13 Sep 2023 17:15:10 -0400 Subject: [PATCH 068/139] DOC: updated changelog Added a summary of the changes to the changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6fd2d0..66456a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Add manual GitHub Actions tests for pysatMadrigal RC * Update GitHub Actions workflows for newer versions of pip * Added clean warning test attributes to all Instruments + * Updated documentation to comply with current Ecosystem guidelines + * Replaced setup.py with pyproject.toml [0.1.0] - 2023-04-11 -------------------- From 88c8781c1f676d51f1f26531ebb7038d4eea08b2 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 14 Sep 2023 09:49:44 -0400 Subject: [PATCH 069/139] BUG: fixed if statement in main.yml Fixed matrix variables so that new if statement is triggered. --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cb9a144..07f4e21 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ jobs: matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.10", "3.11"] - numpy_ver: ["latest"] + test_config: ["latest"] include: # NEP29 compliance settings - python-version: "3.9" @@ -27,7 +27,7 @@ jobs: os: "ubuntu-20.04" test_config: "Ops" - name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} + name: ${{ matrix.test_config }} Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 From 6e400febbe739a9fdc76609ebe61519fc745c8fe Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 14 Sep 2023 09:56:05 -0400 Subject: [PATCH 070/139] DOC: updated conf.py Updated version access in conf.py. --- docs/conf.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a1063ec..64aff85 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,7 +19,9 @@ import json import os +from pyproject_parser import PyProject import sys + sys.path.insert(0, os.path.abspath('..')) # -- General configuration ------------------------------------------------ @@ -55,20 +57,21 @@ master_doc = 'index' # General information about the project. +info = PyProject.load("../pyproject.toml") + project = 'pysatMadrigal' title = '{:s} Documentation'.format(project) zenodo = json.loads(open('../.zenodo.json').read()) author = ', '.join([creator['name'] for creator in zenodo['creators']]) description = ''.join(['Tools for accessing and analyzing data from the ', 'Madrigal database']) -copyright = ', '.join(['2021', author]) +category = 'Space Physics' +copyright = ', '.join(['2023', author]) # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. -doc_dir = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(doc_dir, "..", project, "version.txt"), "r") as fin: - version = fin.read().strip() +version = info.project['version'].base_version release = '{:s}-beta'.format(version) # Include alpha/beta/rc tags. # The language for content autogenerated by Sphinx. Refer to documentation @@ -76,7 +79,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -143,7 +146,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [(master_doc, project, title, author, project, - description, 'Space Physics')] + description, category)] # -- Options for Epub output ---------------------------------------------- From 8e2deb8805b90703351d05a22880857c7a482ff5 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 14 Sep 2023 10:02:37 -0400 Subject: [PATCH 071/139] DOC: removed duplicate links Removed duplicate explicit links. --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2c0b357..956b6f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,8 +47,8 @@ include: Feature requests and feedback ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The best way to send feedback is to file an issue at -[GitHub](https://github.com/pysat/pysatMadrigal/issues). +The best way to send feedback is to file an +[issue](https://github.com/pysat/pysatMadrigal/issues). If you are proposing a new feature or a change in something that already exists: @@ -64,7 +64,7 @@ Development To set up `pysatMadrigal` for local development: -1. Fork pysatMadrigal on [GitHub](https://github.com/pysat/pysatMadrigal/fork). +1. Fork [pysatMadrigal on GitHub](https://github.com/pysat/pysatMadrigal/fork). 2. Clone your fork locally: From a664c2155677da5d4470b90efbdce69b0b8b3303 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 14 Sep 2023 10:02:54 -0400 Subject: [PATCH 072/139] DOC: updated example index Included a missing example in the example index. --- docs/examples.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/examples.rst b/docs/examples.rst index 8dfafff..f1f482e 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -7,4 +7,5 @@ tools .. toctree:: examples/ex_init.rst examples/ex_gnss_tec.rst - examples/ex_jro_isr_beam.rst \ No newline at end of file + examples/ex_jro_isr_beam.rst + examples/ex_dmsp_methods.rst From 424649ad24a23382de6e0f838760a08ca95f52c7 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 14 Sep 2023 10:03:15 -0400 Subject: [PATCH 073/139] BUG: removed comma Removed comma from authors. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 15c87cc..c2a2051 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" requires-python = ">=3.6" license = {file = "LICENSE"} authors = [ - {name = "Angeline G. Burrell, et al.", email = "pysat.developers@gmail.com"} + {name = "Angeline G. Burrell et al.", email = "pysat.developers@gmail.com"} ] classifiers = [ "Development Status :: 4 - Beta", From bd32988e2efd98cb14cceb4228f296232d4d99ac Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 14 Sep 2023 10:08:22 -0400 Subject: [PATCH 074/139] DOC: updated installation guide Update the installation guide to reflect the change to pyproject.toml. --- docs/installation.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 4ecc22a..1c922eb 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -37,17 +37,18 @@ You may either install pysatMadrigal via pip or by cloning the git repository pip install pysatMadrigal -2. Clone the git repository and use the ``setup.py`` file to install +2. Clone the git repository and use the ``pyproject.toml`` file to install :: git clone https://github.com/pysat/pysatMadrigal.git - # Install on the system (root privileges required) - sudo python3 setup.py install + # Move into the pysatMadrigal directory. Then build the wheel + python -m build . + - # Install at the user level - python3 setup.py install --user + # Install at the user or system level, depending on privledges + pip install . - # Install at the user level with the intent to develop locally - python3 setup.py develop --user + # Install with the intent to develop locally + pip install -e . From e27b21deae344c3ed5e4f6a047fdb77f23ff5555 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Thu, 14 Sep 2023 16:53:12 -0400 Subject: [PATCH 075/139] DOC: updated Manifest Removed the old version.txt file from the manifest. --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index d24537a..6e2fb91 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,7 +5,6 @@ recursive-include pysatMadrigal *.py include *.md include *.txt include LICENSE -include pysatMadrigal/version.txt prune pysatMadrigal/tests prune docs prune demo From b56c3adc2fcba19011800117eea528702f888f98 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 20 Nov 2023 17:16:12 -0500 Subject: [PATCH 076/139] STY: refactored general load function Extracted useful elements of the `load` function to allow their use elsewhere without code duplication. --- pysatMadrigal/instruments/methods/general.py | 379 ++++++++++++------- 1 file changed, 237 insertions(+), 142 deletions(-) diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index d12a69c..f4013c0 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -19,7 +19,6 @@ from madrigalWeb import madrigalWeb -logger = pysat.logger file_types = {'hdf5': 'hdf5', 'netCDF4': 'netCDF4', 'simple': 'simple.gz'} @@ -481,13 +480,232 @@ def madrigal_file_format_str(inst_code, strict=False, verbose=True): if strict: raise ValueError(msg) elif verbose: - logger.warning(msg) + pysat.logger.warning(msg) fstr += "{file_type}" return fstr +def sort_file_formats(fnames): + """Separate filenames by file format type. + + Parameters + ---------- + fnames : array-like + Iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + + Returns + ------- + load_file_types : dict + A dictionary with file types as keys and a list of filenames for + each file type. + + """ + # Sort the files by file format type + load_file_types = {ftype: [] for ftype in file_types.keys()} + for fname in fnames: + for ftype in file_types.keys(): + if fname.find(ftype) > 0: + load_file_types[ftype].append(fname) + break + + if fname not in load_file_types[ftype]: + # Raise a logger warning if a file with an unknown extension + # is encountered + pysat.logger.warning( + 'file with unknown file type: {:}'.format(fname)) + + return load_file_types + + +def update_meta_with_hdf5(file_ptr, meta): + """Get meta data from a Madrigal HDF5 file. + + Parameters + ---------- + file_ptr : h5py._hl.files.File + Pointer to an open HDF5 file + meta : pysat.Meta + Existing Meta class to be updated + + Returns + ------- + file_labels : list + List of metadata available + + """ + # Get the Madrigal metadata + file_meta = file_ptr['Metadata']['Data Parameters'] + file_labels = list() + + # Load available info into pysat.Meta if not already present + for item in file_meta: + name_string = item[0].decode('UTF-8') + unit_string = item[3].decode('UTF-8') + desc_string = item[1].decode('UTF-8') + file_labels.append(name_string) + + # Only update metadata if necessary + if name_string.lower() not in meta: + meta[name_string.lower()] = {meta.labels.name: name_string, + meta.labels.units: unit_string, + meta.labels.desc: desc_string} + + + # Add additional metadata notes. Custom attributes attached to meta are + # attached to the MetaHeader object later + for key in file_ptr['Metadata']: + if key != 'Data Parameters': + setattr(meta, key.replace(' ', '_'), file_ptr['Metadata'][key][:]) + + return file_labels + + +def build_madrigal_datetime_index(mad_data): + """Create a datetime index using standard Madrigal variables. + + Parameters + ---------- + mad_data : pds.DataFrame + Madrigal data, expects time variables 'year', 'month', 'day', 'hour', + 'min', and 'sec' + + Returns + ------- + data_time : + Datetime index for use by pysat + + Raises + ------ + ValueError + If expected time variables are missing + + """ + # Set the standard time keys + time_keys = np.array(['year', 'month', 'day', 'hour', 'min', 'sec']) + + # Construct datetime index from times + if not np.all([key in mad_data.columns for key in time_keys]): + time_keys = [key for key in time_keys if key not in mad_data.columns] + raise ValueError(' '.join(["unable to construct time index, missing ", + repr(time_keys)])) + + # Get the UT seconds of day and build the datetime index + uts = 3600.0 * mad_data.loc[:, 'hour'] + 60.0 * mad_data.loc[:, 'min'] \ + + mad_data.loc[:, 'sec'] + data_time = pysat.utils.time.create_datetime_index( + year=mad_data.loc[:, 'year'], month=mad_data.loc[:, 'month'], + day=mad_data.loc[:, 'day'], uts=uts) + + return data_time + + +def convert_pandas_to_xarray(xarray_coords, data, time_ind): + """Convert Madrigal HDF5/simple data from pandas to xarray. + + Parameters + ---------- + xarray_coords : list or NoneType + List of keywords to use as coordinates if xarray output is desired + instead of a Pandas DataFrame. Can build an xarray Dataset + that have different coordinate dimensions by providing a dict + inside the list instead of coordinate variable name strings. Each dict + will have a tuple of coordinates as the key and a list of variable + strings as the value. Empty list if None. For example, + xarray_coords=[{('time',): ['year', 'doy'], + ('time', 'gdalt'): ['data1', 'data2']}]. (default=None) + data : pds.DataFrame + Data to be converted into the xarray format + time_ind : pds.DatetimeIndex + Time index for the data + + Returns + ------- + + """ + # If a list was provided, recast as a dict and grab the data columns + if not isinstance(xarray_coords, dict): + xarray_coords = {tuple(xarray_coords): [col for col in data.columns + if col not in xarray_coords]} + + # Determine the order in which the keys should be processed: + # Greatest to least number of dimensions + len_dict = {len(xcoords): xcoords for xcoords in xarray_coords.keys()} + coord_order = [len_dict[xkey] for xkey in sorted( + [lkey for lkey in len_dict.keys()], reverse=True)] + + # Append time to the data frame + data = data.assign(time=pds.Series(time_ind, index=data.index)) + + # Cycle through each of the coordinate dimensions + xdatasets = list() + for xcoords in coord_order: + if not np.all([xkey.lower() in data.columns for xkey in xcoords]): + raise ValueError(''.join(['unknown coordinate key in [', + repr(xcoords), '], use only: ', + repr(data.columns)])) + + if not np.all([xkey.lower() in data.columns + for xkey in xarray_coords[xcoords]]): + good_ind = [i for i, xkey in enumerate(xarray_coords[xcoords]) + if xkey.lower() in data.columns] + + if len(good_ind) == 0: + raise ValueError('All data variables {:} are unknown.'.format( + xarray_coords[xcoords])) + elif len(good_ind) < len(xarray_coords[xcoords]): + # Remove the coordinates that aren't present. + temp = np.array(xarray_coords[xcoords])[good_ind] + + # Warn user, some of this may be due to a file format change. + bad_ind = [i for i in range(len(xarray_coords[xcoords])) + if i not in good_ind] + pysat.logger.warning( + 'unknown data variable(s) {:}, using only: {:}'.format( + np.array(xarray_coords[xcoords])[bad_ind], temp)) + + # Assign good data as a list. + xarray_coords[xcoords] = list(temp) + + # Select the desired data values + sel_data = data[list(xcoords) + xarray_coords[xcoords]] + + # Remove duplicates before indexing, to ensure data with the same values + # at different locations are kept + sel_data = sel_data.drop_duplicates() + + # Set the indices + sel_data = sel_data.set_index(list(xcoords)) + + # Recast as an xarray + xdatasets.append(sel_data.to_xarray()) + + # Get the necessary information to test the data + lcols = data.columns + len_data = len(lcols) + + # Merge all of the datasets + data = xr.merge(xdatasets) + test_variables = [xkey for xkey in data.variables.keys()] + ltest = len(test_variables) + + # Test to see that all data was retrieved + if ltest != len_data: + if ltest < len_data: + estr = 'missing: {:}'.format(' '.join([ + dvar for dvar in lcols if dvar not in test_variables])) + else: + estr = 'have extra: {:}'.format(' '.join([ + tvar for tvar in test_variables if tvar not in lcols])) + raise ValueError(''.join([ + 'coordinates not supplied for all data columns', + ': {:d} != {:d}; '.format(ltest, len_data), estr])) + + return data + + def load(fnames, tag='', inst_id='', xarray_coords=None): """Load data from Madrigal into Pandas or XArray. @@ -534,13 +752,8 @@ def load(fnames, tag='', inst_id='', xarray_coords=None): for direct user interaction. """ - # Test the file formats - load_file_types = {ftype: [] for ftype in file_types.keys()} - for fname in fnames: - for ftype in file_types.keys(): - if fname.find(ftype) > 0: - load_file_types[ftype].append(fname) - break + # Test and sort the file formats + load_file_types = sort_file_formats(fnames) # Initialize xarray coordinates, if needed if xarray_coords is None: @@ -637,30 +850,10 @@ def load(fnames, tag='', inst_id='', xarray_coords=None): # Open the specified file and get the data and metadata filed = h5py.File(fname, 'r') file_data = filed['Data']['Table Layout'] - file_meta = filed['Metadata']['Data Parameters'] - # Load available info into pysat.Meta if this is the first file + new_labels = update_meta_with_hdf5(filed, meta) if len(labels) == 0: - for item in file_meta: - name_string = item[0].decode('UTF-8') - unit_string = item[3].decode('UTF-8') - desc_string = item[1].decode('UTF-8') - labels.append(name_string) - - # Only update metadata if necessary - if name_string.lower() not in meta: - meta[name_string.lower()] = { - meta.labels.name: name_string, - meta.labels.units: unit_string, - meta.labels.desc: desc_string} - - # Add additional metadata notes. Custom attributes attached to - # meta are attached to corresponding Instrument object when - # pysat receives data and meta from this routine - for key in filed['Metadata']: - if key != 'Data Parameters': - setattr(meta, key.replace(' ', '_'), - filed['Metadata'][key][:]) + labels = new_labels # Load data into frame, with labels from metadata ldata = pds.DataFrame.from_records(file_data, columns=labels) @@ -671,109 +864,11 @@ def load(fnames, tag='', inst_id='', xarray_coords=None): # Extended processing is the same for simple and HDF5 files # # Construct datetime index from times - time_keys = np.array(['year', 'month', 'day', 'hour', 'min', 'sec']) - if not np.all([key in ldata.columns for key in time_keys]): - time_keys = [key for key in time_keys - if key not in ldata.columns] - raise ValueError(' '.join(["unable to construct time index, ", - "missing {:}".format(time_keys)])) - - uts = 3600.0 * ldata.loc[:, 'hour'] + 60.0 * ldata.loc[:, 'min'] \ - + ldata.loc[:, 'sec'] - time = pysat.utils.time.create_datetime_index( - year=ldata.loc[:, 'year'], month=ldata.loc[:, 'month'], - day=ldata.loc[:, 'day'], uts=uts) + time = build_madrigal_datetime_index(ldata) # Declare index or recast as xarray if coord_len > 0: - # If a list was provided, recast as a dict and grab the data - # columns - if not isinstance(xarray_coords, dict): - xarray_coords = {tuple(xarray_coords): - [col for col in ldata.columns - if col not in xarray_coords]} - - # Determine the order in which the keys should be processed: - # Greatest to least number of dimensions - len_dict = {len(xcoords): xcoords - for xcoords in xarray_coords.keys()} - coord_order = [len_dict[xkey] for xkey in sorted( - [lkey for lkey in len_dict.keys()], reverse=True)] - - # Append time to the data frame - ldata = ldata.assign(time=pds.Series(time, index=ldata.index)) - - # Cycle through each of the coordinate dimensions - xdatasets = list() - for xcoords in coord_order: - if not np.all([xkey.lower() in ldata.columns - for xkey in xcoords]): - raise ValueError(''.join(['unknown coordinate key ', - 'in [{:}'.format(xcoords), - '], use only: {:}'.format( - ldata.columns)])) - if not np.all([xkey.lower() in ldata.columns - for xkey in xarray_coords[xcoords]]): - good_ind = [ - i for i, xkey in enumerate(xarray_coords[xcoords]) - if xkey.lower() in ldata.columns] - - if len(good_ind) == 0: - raise ValueError(''.join([ - 'All data variables {:} are unknown.'.format( - xarray_coords[xcoords])])) - elif len(good_ind) < len(xarray_coords[xcoords]): - # Remove the coordinates that aren't present. - temp = np.array(xarray_coords[xcoords])[good_ind] - - # Warn user, some of this may be due to a file - # format update or change. - bad_ind = [i for i in - range(len(xarray_coords[xcoords])) - if i not in good_ind] - logger.warning(''.join([ - 'unknown data variable(s) {:}, '.format( - np.array(xarray_coords[xcoords])[bad_ind]), - 'using only: {:}'.format(temp)])) - - # Assign good data as a list. - xarray_coords[xcoords] = list(temp) - - # Select the desired data values - sel_data = ldata[list(xcoords) + xarray_coords[xcoords]] - - # Remove duplicates before indexing, to ensure data with - # the same values at different locations are kept - sel_data = sel_data.drop_duplicates() - - # Set the indices - sel_data = sel_data.set_index(list(xcoords)) - - # Recast as an xarray - xdatasets.append(sel_data.to_xarray()) - - # Get the necessary information to test the data - lcols = ldata.columns - len_data = len(lcols) - - # Merge all of the datasets - ldata = xr.merge(xdatasets) - test_variables = [xkey for xkey in ldata.variables.keys()] - ltest = len(test_variables) - - # Test to see that all data was retrieved - if ltest != len_data: - if ltest < len_data: - estr = 'missing: {:}'.format( - ' '.join([dvar for dvar in lcols - if dvar not in test_variables])) - else: - estr = 'have extra: {:}'.format( - ' '.join([tvar for tvar in test_variables - if tvar not in lcols])) - raise ValueError(''.join([ - 'coordinates not supplied for all data columns', - ': {:d} != {:d}; '.format(ltest, len_data), estr])) + ldata = convert_pandas_to_xarray(xarray_coords, ldata, time) else: # Set the index to time ldata.index = time @@ -781,10 +876,10 @@ def load(fnames, tag='', inst_id='', xarray_coords=None): # Raise a logging warning if there are duplicate times. This # means the data should be stored as an xarray Dataset if np.any(time.duplicated()): - logger.warning(''.join(["duplicated time indices, ", - "consider specifing additional", - " coordinates and storing the ", - "data as an xarray Dataset"])) + pysat.logger.warning(''.join([ + "duplicated time indices, consider specifing ", + "additional coordinates and storing the data as an ", + "xarray Dataset"])) # Compile a list of the data objects fdata.append(ldata) @@ -884,9 +979,9 @@ def download(date_array, inst_code=None, kindat=None, data_path=None, stop = date_array.shift().max() # Initialize the connection to Madrigal - logger.info('Connecting to Madrigal') + pysat.logger.info('Connecting to Madrigal') web_data = madrigalWeb.MadrigalData(url) - logger.info('Connection established.') + pysat.logger.info('Connection established.') files = get_remote_filenames(inst_code=inst_code, kindat=kindat, user=user, password=password, @@ -905,12 +1000,12 @@ def download(date_array, inst_code=None, kindat=None, data_path=None, if not os.path.isfile(local_file): fstr = ''.join(('Downloading data for ', local_file)) - logger.info(fstr) + pysat.logger.info(fstr) web_data.downloadFile(mad_file.name, local_file, user, password, "pysat", format=file_type) else: estr = ''.join((local_file, ' already exists. Skipping.')) - logger.info(estr) + pysat.logger.info(estr) return @@ -1018,7 +1113,7 @@ def get_remote_filenames(inst_code=None, kindat='', user=None, password=None, files = list() istr = "Found {:d} Madrigal experiments between {:s} and {:s}".format( len(exp_list), start.strftime('%d %B %Y'), stop.strftime('%d %B %Y')) - logger.info(istr) + pysat.logger.info(istr) for exp in exp_list: if good_exp(exp, date_array=date_array): file_list = web_data.getExperimentFiles(exp.id) @@ -1175,7 +1270,7 @@ def list_remote_files(tag, inst_id, inst_code=None, kindats=None, user=None, format_str = format_str.replace('.hdf5', '.h5') # Parse these filenames to grab out the ones we want - logger.info("Parsing filenames") + pysat.logger.info("Parsing filenames") if format_str.find('*') < 0: stored = pysat.utils.files.parse_fixed_width_filenames(filenames, format_str) @@ -1184,7 +1279,7 @@ def list_remote_files(tag, inst_id, inst_code=None, kindats=None, user=None, format_str, '.') # Process the parsed filenames and return a properly formatted Series - logger.info("Processing filenames") + pysat.logger.info("Processing filenames") return pysat.utils.files.process_parsed_filenames(stored, two_digit_year_break) From 6e636d2a004fd1faf3d5937de996b5a965d5c8a1 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 20 Nov 2023 17:16:49 -0500 Subject: [PATCH 077/139] ENH: added gnss load functions Added type-specific GNSS load functions and exploratory file functions for the line-of-sight files. --- pysatMadrigal/instruments/methods/gnss.py | 254 +++++++++++++++++++++- 1 file changed, 253 insertions(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/methods/gnss.py b/pysatMadrigal/instruments/methods/gnss.py index 2aa3df2..51cad84 100644 --- a/pysatMadrigal/instruments/methods/gnss.py +++ b/pysatMadrigal/instruments/methods/gnss.py @@ -6,6 +6,14 @@ # -*- coding: utf-8 -*- """Methods supporting the Global Navigation Satellite System platform.""" +import h5py +import numpy as np +import pandas as pds + +import pysat + +from pysatMadrigal.instruments.methods import general + def acknowledgements(name): """Provide the acknowledgements for different GNSS instruments. @@ -63,6 +71,250 @@ def references(name): Suggested Instrument reference(s) """ - refs = {'tec': "Rideout and Coster (2006) doi:10.1007/s10291-006-0029-5"} + refs = {'tec': "\n".join([ + "Rideout and Coster (2006) doi:10.1007/s10291-006-0029-5", + "Vierinen et al., (2016) doi:10.5194/amt-9-1303-2016"])} return refs[name] + + +def load_vtec(fnames): + """Load the GNSS vertical TEC data. + + Parameters + ---------- + fnames : list + List of filenames + + Returns + ------- + data : xarray.Dataset + Object containing satellite data + meta : pysat.Meta + Object containing metadata such as column names and units + lat_key : list + Latitude key names + lon_key : list + Longitude key names + + """ + # Define the xarray coordinate dimensions and lat/lon keys + xcoords = {('time', 'gdlat', 'glon', 'kindat', 'kinst'): + ['gdalt', 'tec', 'dtec'], + ('time', ): ['year', 'month', 'day', 'hour', 'min', + 'sec', 'ut1_unix', 'ut2_unix', 'recno']} + lat_keys = ['gdlat'] + lon_keys = ['glon'] + + # Load the specified data + data, meta = general.load(fnames, 'vtec', '', xarray_coords=xcoords) + + # Fix the units for tec and dtec + meta['tec'] = {meta.labels.units: 'TECU', meta.labels.min_val: 0.0, + meta.labels.max_val: np.nan} + meta['dtec'] = {meta.labels.units: 'TECU', meta.labels.min_val: 0.0, + meta.labels.max_val: np.nan} + + # Get the maximum and minimum values for altiutde, along with lat/lon keys + meta['gdalt'] = {meta.labels.min_val: 0.0, meta.labels.max_val: np.nan} + + return data, meta, lat_keys, lon_keys + + +def load_site(fnames): + """Load the GNSS TEC site data. + + Parameters + ---------- + fnames : list + List of filenames + + Returns + ------- + data : xarray.Dataset + Object containing satellite data + meta : pysat.Meta + Object containing metadata such as column names and units + lat_keys : list + Latitude key names + lon_keys : list + Longitude key names + + """ + # Define the xarray coordinate dimensions and lat/lon keys + xcoords = {('time', 'gps_site'): ['gdlatr', 'gdlonr']} + lat_keys = ['gdlatr'] + lon_keys = ['gdlonr'] + + # Load the specified data + data, meta = general.load(fnames, 'site', '', xarray_coords=xcoords) + + return data, meta, lat_keys, lon_keys + + +def load_los(fnames, los_method, los_value, gnss_network='all'): + """Load the GNSS slant TEC data. + + Parameters + ---------- + fnames : list + List of filenames + los_method : str + For 'los' tag only, load data for a unique GNSS receiver site ('site') + or at a unique time ('time') + los_value : str or dt.datetime + For 'los' tag only, load data at this unique site or time + gnss_network : bool + Limit data by GNSS network, if not 'all'. Currently supports 'all', + 'gps', and 'glonass' (default='all') + + Returns + ------- + data : xarray.Dataset + Object containing satellite data + meta : pysat.Meta + Object containing metadata such as column names and units + lat_keys : list + Latitude key names + lon_keys : list + Longitude key names + + """ + # Define the xarray coordinate dimensions and lat/lon keys + xcoords = {('time', 'gps_site', 'sat_id', 'kindat', 'kinst'): + ['pierce_alt', 'los_tec', 'dlos_tec', 'tec', 'azm', 'elm', + 'gdlat', 'glon', 'rec_bias', 'drec_bias'], + ('time', ): ['year', 'month', 'day', 'hour', 'min', + 'sec', 'ut1_unix', 'ut2_unix', 'recno'], + ('time', 'sat_id', ): ['gnss_type'], + ('time', 'gps_site', ): ['gdlatr', 'gdlonr']} + lat_keys = ['gdlatr', 'gdlat'] + lon_keys = ['gdlonr', 'glon'] + + # Sort and test the desired filenames by file format + load_file_types = general.sort_file_formats(fnames) + + for ftype in load_file_types.keys(): + if ftype != 'hdf5' and len(load_file_types[ftype]) > 0: + pysat.logger.warning( + 'unable to load non-HDF5 slant TEC files: {:}'.format( + load_fyle_types[ftype])) + + # Initalize the meta data + meta = pysat.Meta() + + # Load the data using the desired method + if los_method.lower() == 'site': + sel_key = 'gps_site' + + # Convert the site to bytes + los_value = np.bytes_(los_value) + elif los_method.lower() == 'time': + sel_key = 'ut1_unix' + + # Convert the input datetime to UNIX seconds + los_value = (los_value - dt.datetime(1970, 1, 1)).total_seconds() + else: + raise ValueError('unsupported selection type: {:}'.format(los_method)) + + # Load the data by desired method + data = list() + for fname in load_file_types['hdf5']: + with h5py.File(fname, 'r') as fin: + sel_arr = fin['Data']['Table Layout'][sel_key] + sel_mask = sel_arr == los_value + + if gnss_network.lower() != 'all': + # Redefine the selection mask to include network as well + gnss_val = '{:8s}'.format(gnss_network.upper()) + try: + net_arr = fin['Data']['Table Layout']['gnss_type'][sel_mask] + sel_mask[sel_mask] = net_arr.astype(str) == gnss_val + except ValueError: + # If the 'gnss_type' is not available, all data is GPS + if gnss_network.lower() != 'gps': + sel_mask[sel_mask] = False + + # Save the output for the desired slice + if sel_mask.any(): + data.extend(list(fin['Data']['Table Layout'][sel_mask])) + + # Save the meta data + labels = general.update_meta_with_hdf5(fin, meta) + + # If this is time selection, only need to load from one file + if len(data) > 0: + break + + # Load data into frame, with labels from metadata + data = pds.DataFrame.from_records(data, columns=labels) + + # Enforce lowercase variable names + data.columns = [item.lower() for item in data.columns] + + # Convert the data to an xarray Dataset + time_ind = general.build_madrigal_datetime_index(data) + + # Convert the output to xarray + data = general.convert_pandas_to_xarray(xcoords, data, time_ind) + + return data, meta, lat_keys, lon_keys + + +def get_los_receiver_sites(los_fnames): + """Retrieve an array of unique receiver names for the desired LoS files. + + Parameters + ---------- + los_fnames : list + List of filenames + + Returns + ------- + sites : np.array + Array of strings containing GNSS receiver names with data in the files + + """ + los_fnames = pysat.utils.listify(los_fnames) + sites = list() + + # Get all of the potential sites + for fname in los_fnames: + with h5py.File(fname, 'r') as fin: + site_arr = fin['Data']['Table Layout']['gps_site'] + sites.extend(list(site_arr.astype(str))) + + # Find the unique sites + sites = np.unique(sites) + return sites + + +def get_los_times(los_fnames): + """Retrieve an array of unique times for the desired LoS files. + + Parameters + ---------- + los_fnames : list + List of filenames + + Returns + ------- + all_times : np.array + Array of datetime objects with data in the files + + """ + los_fnames = pysat.utils.listify(los_fnames) + all_times = list() + + # Get all of the potential sites + for fname in los_fnames: + with h5py.File(fname, 'r') as fin: + time_arr = fin['Data']['Table Layout']['ut1_unix'] + + # Convert from unix time to a datetime object + all_times.extend([dt.datetime(1970, 1, 1) + + dt.timedelta(seconds=int(time_val)) + for time_val in time_arr]) + + return all_times + From 8117a8c07aa9d42bcdc82bdaadff7c9c9bf05bee Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 20 Nov 2023 17:17:20 -0500 Subject: [PATCH 078/139] ENH: added 'los' tag to gnss_tec Added support for the line-of-sight (los) TEC data. --- pysatMadrigal/instruments/gnss_tec.py | 111 +++++++++++++++++--------- 1 file changed, 72 insertions(+), 39 deletions(-) diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index 6e66f29..ace24cf 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -3,7 +3,7 @@ The Global Navigation Satellite System (GNSS) is used in conjunction with a world-wide receiver network to produce total electron content (TEC) data -products, including vertical and line-of-sight TEC. +products, including vertical and line-of-sight (or slant) TEC. Downloads data from the MIT Haystack Madrigal Database. @@ -14,7 +14,9 @@ name 'tec' tag - 'vtec', 'site' + 'vtec', 'site', 'los' +inst_id + '' (not used) Examples -------- @@ -24,16 +26,30 @@ import pysat import pysatMadrigal as pymad + # Get and load all vertical TEC for 19 Nov 2017 vtec = pysat.Instrument(inst_module=pymad.instruments.gnss_tec, tag='vtec') vtec.download(dt.datetime(2017, 11, 19), dt.datetime(2017, 11, 20), user='Firstname+Lastname', password='email@address.com') vtec.load(date=dt.datetime(2017, 11, 19)) + # Get and load the GLONASS slant TEC from the zzon site on 1 Jan 2023 + stec = pysat.Instrument(inst_module=pymad.instruments.gnss_tec, tag='los') + stec.download(start=dt.datetime(2023, 1, 1), user='Firstname+Lastname', + password='email@address.com') + stec.load(2023, 1, los_method='site', los_value='zzon', + gnss_network='glonass') Note ---- Please provide name and email when downloading data with this routine. +The line-of-sight data is too large to load an entire file at once. Data may be +loaded by individual receiver site for any number of days (recommended to load +one day) or a given time. To discover the available sites and times (exact times +are required for selection), you may use the +`pysatMadrigal.instruments.methods.gnss.get_los_times` and +`pysatMadrigal.instruments.methods.gnss.get_los_receiver_sites` functions. + """ import datetime as dt @@ -49,20 +65,22 @@ platform = 'gnss' name = 'tec' -tags = {'vtec': 'vertical TEC', 'site': 'Sites used in daily TEC data'} +tags = {'vtec': 'vertical TEC', 'site': 'Sites used in daily TEC data', + 'los': 'line-of-sight TEC'} inst_ids = {'': [tag for tag in tags.keys()]} pandas_format = False # Madrigal tags madrigal_inst_code = 8000 -madrigal_tag = {'': {'vtec': '3500', 'site': '3506'}} -# TODO(#12): `, 'los': '3505'}}` +madrigal_tag = {'': {'vtec': '3500', 'site': '3506', 'los': '3505'}} # Local attributes fname = general.madrigal_file_format_str(madrigal_inst_code, verbose=False).split("*") supported_tags = {ss: {'vtec': ''.join(['gps', fname[1], 'g', fname[2]]), + 'los': ''.join(['los_{{year:04d}}{{month:02d}}', + '{{day:02d}}', fname[2]]), 'site': ''.join(['site_{{year:04d}}{{month:02d}}', '{{day:02d}}', fname[2]])} for ss in inst_ids.keys()} @@ -73,7 +91,8 @@ # Instrument test attributes _test_dates = {'': {'vtec': dt.datetime(2017, 11, 19), - 'site': dt.datetime(2001, 1, 1)}} + 'site': dt.datetime(2001, 1, 1), + 'los': dt.datetime(2023, 1, 1)}} _clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', 'Data provided at a clean level' @@ -226,7 +245,8 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None, return -def load(fnames, tag='', inst_id=''): +def load(fnames, tag='', inst_id='', los_method='site', los_value=None, + gnss_network='all'): """Load the GNSS TEC data. Parameters @@ -239,6 +259,14 @@ def load(fnames, tag='', inst_id=''): inst_id : str Instrument ID used to identify particular data set to be loaded. This input is nominally provided by pysat itself. (default='') + los_method : str + For 'los' tag only, load data for a unique GNSS receiver site ('site') + or at a unique time ('time') (default='site') + los_value : str, dt.datetime, or NoneType + For 'los' tag only, load data at this unique site or time (default=None) + gnss_nework : bool + For 'los' tag only, limit data by GNSS network if not 'all'. Currently + supports 'all', 'gps', and 'glonass' (default='all') Returns ------- @@ -247,17 +275,34 @@ def load(fnames, tag='', inst_id=''): meta : pysat.Meta Object containing metadata such as column names and units - """ - # Define the xarray coordinate dimensions (apart from time) - # Not needed for netCDF - xcoords = {'vtec': {('time', 'gdlat', 'glon', 'kindat', 'kinst'): - ['gdalt', 'tec', 'dtec'], - ('time', ): ['year', 'month', 'day', 'hour', 'min', - 'sec', 'ut1_unix', 'ut2_unix', 'recno']}, - 'site': {('time', 'gps_site'): ['gdlatr', 'gdlonr']}} + Raises + ------ + ValueError + If tag is 'los' and no valid 'los_value' is provided or unknown tag + + Note + ---- + The line-of-sight data is too large to load an entire file at once. Data + may be loaded by individual receiver site for any number of days + (recommended to load one day) or a given time. To discover the available + sites and times (exact times are required for selection), you may use the + `pysatMadrigal.instruments.methods.gnss.get_los_times` and + `pysatMadrigal.instruments.methods.gnss.get_los_receiver_sites` functions. + """ # Load the specified data - data, meta = general.load(fnames, tag, inst_id, xarray_coords=xcoords[tag]) + if tag == 'vtec': + data, meta, lat_keys, lon_keys = gnss.load_vtec(fnames) + elif tag == 'site': + data, meta, lat_keys, lon_keys = gnss.load_site(fnames) + elif tag == 'los': + if los_value is None: + raise ValueError('must specify a valid {:}'.format(los_method)) + + data, meta, lat_keys, lon_keys = gnss.load_los(fnames, los_method, + los_value, gnss_network) + else: + raise ValueError('unknown tag provided for GNSS TEC Instrument') # Squeeze the kindat and kinst 'coordinates', but keep them as floats squeeze_dims = np.array(['kindat', 'kinst']) @@ -265,29 +310,17 @@ def load(fnames, tag='', inst_id=''): if np.any(squeeze_mask): data = data.squeeze(dim=squeeze_dims[squeeze_mask]) - # Fix the units for tec and dtec - if tag == 'vtec': - meta['tec'] = {meta.labels.units: 'TECU', meta.labels.min_val: 0.0, - meta.labels.max_val: np.nan} - meta['dtec'] = {meta.labels.units: 'TECU', meta.labels.min_val: 0.0, - meta.labels.max_val: np.nan} - - # Get the maximum and minimum values for time, latitude, longitude, - # and altitude + # Get the maximum and minimum values for time, latitude, and longitude meta['time'] = {meta.labels.notes: data['time'].values.dtype.__doc__, meta.labels.min_val: np.nan, meta.labels.max_val: np.nan} - if tag == 'vtec': - meta['gdalt'] = {meta.labels.min_val: 0.0, meta.labels.max_val: np.nan} - lat_key = 'gdlat' - lon_key = 'glon' - else: - lat_key = 'gdlatr' - lon_key = 'gdlonr' - meta[lat_key] = {meta.labels.min_val: -90.0, meta.labels.max_val: 90.0} - min_lon = 0.0 if data[lon_key].values.min() >= 0.0 else -180.0 - meta[lon_key] = {meta.labels.min_val: min_lon, - meta.labels.max_val: min_lon + 360.0} + for lat_key in lat_keys: + meta[lat_key] = {meta.labels.min_val: -90.0, meta.labels.max_val: 90.0} + + for lon_key in lon_keys: + min_lon = 0.0 if data[lon_key].values.min() >= 0.0 else -180.0 + meta[lon_key] = {meta.labels.min_val: min_lon, + meta.labels.max_val: min_lon + 360.0} return data, meta @@ -328,10 +361,10 @@ def list_remote_files(tag, inst_id, start=dt.datetime(1998, 10, 15), pysatMadrigal.instruments.methods.general.list_remote_files """ - if tag == 'site': - two_break = None - elif tag == 'vtec': + if tag == 'vtec': two_break = 99 + else: + two_break = None files = general.list_remote_files( tag, inst_id, supported_tags=remote_tags, From dcd81ead4fc618aca444ffebc3d901ef6fb14e84 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 20 Nov 2023 17:17:53 -0500 Subject: [PATCH 079/139] DOC: updated GNSS method description Improved the GNSS instrument method description to encompass the new scope. --- docs/methods.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/methods.rst b/docs/methods.rst index 8a21645..f8f0131 100644 --- a/docs/methods.rst +++ b/docs/methods.rst @@ -26,7 +26,8 @@ GNSS ---- Supports the Global Navigation Satellite System instruments by providing -reference and acknowledgement information. +reference and acknowledgement information, specialised load functions, and +supporting information for probing the line-of-sight (LoS) files. .. automodule:: pysatMadrigal.instruments.methods.gnss From fe6107c79a2894defdf15878fcd39d8b82ae65be Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 20 Nov 2023 17:18:08 -0500 Subject: [PATCH 080/139] DOC: updated changelog Added a summary of the changes to the changelog. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66456a0..094d7b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Moved the NGDC AE index Instrument from the general Madrigal Pandas instrument to a new one, fixing the Windows memory issue and a problem with duplicated times + * Added slant TEC (tag of 'los') to the gnss_tec Instrument + * Refactored general load function to extract useful parts of the code that + were used for specific load functions * Maintenance * Add manual GitHub Actions tests for pysatMadrigal RC * Update GitHub Actions workflows for newer versions of pip From dcbdafdee21264c817008acac8d652b1dd135798 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 20 Nov 2023 17:26:32 -0500 Subject: [PATCH 081/139] STY: fixed PEP8 violations Removed excess lines, fixed typos, added missing imports. --- pysatMadrigal/instruments/methods/general.py | 1 - pysatMadrigal/instruments/methods/gnss.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index f4013c0..ac163da 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -553,7 +553,6 @@ def update_meta_with_hdf5(file_ptr, meta): meta.labels.units: unit_string, meta.labels.desc: desc_string} - # Add additional metadata notes. Custom attributes attached to meta are # attached to the MetaHeader object later for key in file_ptr['Metadata']: diff --git a/pysatMadrigal/instruments/methods/gnss.py b/pysatMadrigal/instruments/methods/gnss.py index 51cad84..a5d6cb9 100644 --- a/pysatMadrigal/instruments/methods/gnss.py +++ b/pysatMadrigal/instruments/methods/gnss.py @@ -6,6 +6,7 @@ # -*- coding: utf-8 -*- """Methods supporting the Global Navigation Satellite System platform.""" +import datetime as dt import h5py import numpy as np import pandas as pds @@ -198,7 +199,7 @@ def load_los(fnames, los_method, los_value, gnss_network='all'): if ftype != 'hdf5' and len(load_file_types[ftype]) > 0: pysat.logger.warning( 'unable to load non-HDF5 slant TEC files: {:}'.format( - load_fyle_types[ftype])) + load_file_types[ftype])) # Initalize the meta data meta = pysat.Meta() @@ -317,4 +318,3 @@ def get_los_times(los_fnames): for time_val in time_arr]) return all_times - From 297c5b8d3aacb9d0b3b991592084faa29ff05dcf Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 20 Nov 2023 17:28:35 -0500 Subject: [PATCH 082/139] DOC: updated docstring Added missing return value to the docstring. --- pysatMadrigal/instruments/methods/general.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index ac163da..fdedf7f 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -622,6 +622,8 @@ def convert_pandas_to_xarray(xarray_coords, data, time_ind): Returns ------- + data : xr.Dataset + Data in the dataset format. """ # If a list was provided, recast as a dict and grab the data columns From 656671d48c0e61ae9fec3c06087ac1338838dddd Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 20 Nov 2023 17:39:50 -0500 Subject: [PATCH 083/139] BUG: added load options for los data Added two load options for the GNSS TEC LoS tests. --- pysatMadrigal/instruments/gnss_tec.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index ace24cf..df348a1 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -93,6 +93,10 @@ _test_dates = {'': {'vtec': dt.datetime(2017, 11, 19), 'site': dt.datetime(2001, 1, 1), 'los': dt.datetime(2023, 1, 1)}} +_test_load_opt = {'': {'los': [{'los_method': 'site', 'los_value': 'zzon', + 'gnss_network': 'glonass'}, + {'los_method': 'time', + 'los_value': dt.datetime(2023, 1, 1)}]}} _clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', 'Data provided at a clean level' From 04db5e8a9b714c5c4bf84387d7a9df0743e785c9 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 21 Nov 2023 10:15:39 -0500 Subject: [PATCH 084/139] BUG: allow empty data Allow creation of an empty Dataset from an empty DataFrame. --- pysatMadrigal/instruments/methods/general.py | 50 +++++++++++--------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index fdedf7f..28b5e9b 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -617,8 +617,8 @@ def convert_pandas_to_xarray(xarray_coords, data, time_ind): ('time', 'gdalt'): ['data1', 'data2']}]. (default=None) data : pds.DataFrame Data to be converted into the xarray format - time_ind : pds.DatetimeIndex - Time index for the data + time_ind : pds.DatetimeIndex or NoneType + Time index for the data or None for no time index Returns ------- @@ -637,18 +637,20 @@ def convert_pandas_to_xarray(xarray_coords, data, time_ind): coord_order = [len_dict[xkey] for xkey in sorted( [lkey for lkey in len_dict.keys()], reverse=True)] - # Append time to the data frame - data = data.assign(time=pds.Series(time_ind, index=data.index)) + # Append time to the data frame, if provided + if time_ind is not None: + data = data.assign(time=pds.Series(time_ind, index=data.index)) # Cycle through each of the coordinate dimensions xdatasets = list() for xcoords in coord_order: - if not np.all([xkey.lower() in data.columns for xkey in xcoords]): + if data.empty: + break + elif not np.all([xkey.lower() in data.columns for xkey in xcoords]): raise ValueError(''.join(['unknown coordinate key in [', repr(xcoords), '], use only: ', repr(data.columns)])) - - if not np.all([xkey.lower() in data.columns + elif not np.all([xkey.lower() in data.columns for xkey in xarray_coords[xcoords]]): good_ind = [i for i, xkey in enumerate(xarray_coords[xcoords]) if xkey.lower() in data.columns] @@ -688,21 +690,25 @@ def convert_pandas_to_xarray(xarray_coords, data, time_ind): len_data = len(lcols) # Merge all of the datasets - data = xr.merge(xdatasets) - test_variables = [xkey for xkey in data.variables.keys()] - ltest = len(test_variables) - - # Test to see that all data was retrieved - if ltest != len_data: - if ltest < len_data: - estr = 'missing: {:}'.format(' '.join([ - dvar for dvar in lcols if dvar not in test_variables])) - else: - estr = 'have extra: {:}'.format(' '.join([ - tvar for tvar in test_variables if tvar not in lcols])) - raise ValueError(''.join([ - 'coordinates not supplied for all data columns', - ': {:d} != {:d}; '.format(ltest, len_data), estr])) + if len(xdatasets) > 0: + data = xr.merge(xdatasets) + test_variables = [xkey for xkey in data.variables.keys()] + ltest = len(test_variables) + + # Test to see that all data was retrieved + if ltest != len_data: + if ltest < len_data: + estr = 'missing: {:}'.format(' '.join([ + dvar for dvar in lcols if dvar not in test_variables])) + else: + estr = 'have extra: {:}'.format(' '.join([ + tvar for tvar in test_variables if tvar not in lcols])) + raise ValueError(''.join([ + 'coordinates not supplied for all data columns', + ': {:d} != {:d}; '.format(ltest, len_data), estr])) + else: + # Return an empty object + data = xr.Dataset() return data From c5ccdaa22bb8328f695e0d90bbda7e1b6c8ab390 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 21 Nov 2023 10:16:14 -0500 Subject: [PATCH 085/139] BUG: allow empty file list Allow an empty filelist to be provided and create an empty data object. --- pysatMadrigal/instruments/methods/gnss.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pysatMadrigal/instruments/methods/gnss.py b/pysatMadrigal/instruments/methods/gnss.py index a5d6cb9..eb3b620 100644 --- a/pysatMadrigal/instruments/methods/gnss.py +++ b/pysatMadrigal/instruments/methods/gnss.py @@ -220,6 +220,7 @@ def load_los(fnames, los_method, los_value, gnss_network='all'): # Load the data by desired method data = list() + labels = list() for fname in load_file_types['hdf5']: with h5py.File(fname, 'r') as fin: sel_arr = fin['Data']['Table Layout'][sel_key] @@ -250,14 +251,17 @@ def load_los(fnames, los_method, los_value, gnss_network='all'): # Load data into frame, with labels from metadata data = pds.DataFrame.from_records(data, columns=labels) - # Enforce lowercase variable names - data.columns = [item.lower() for item in data.columns] + if not data.empty: + # Enforce lowercase variable names + data.columns = [item.lower() for item in data.columns] - # Convert the data to an xarray Dataset - time_ind = general.build_madrigal_datetime_index(data) + # Convert the data to an xarray Dataset + time_ind = general.build_madrigal_datetime_index(data) + else: + time_ind = None - # Convert the output to xarray - data = general.convert_pandas_to_xarray(xcoords, data, time_ind) + # Convert the output to xarray + data = general.convert_pandas_to_xarray(xcoords, data, time_ind) return data, meta, lat_keys, lon_keys From acb83c5a58fab68e0baf2f1a0b9efd0c8ce98e86 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 21 Nov 2023 10:16:43 -0500 Subject: [PATCH 086/139] BUG: allow empty gnss load Allow the GNSS TEC load to work with an empty file list. --- pysatMadrigal/instruments/gnss_tec.py | 39 ++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index df348a1..32b83b7 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -97,7 +97,7 @@ 'gnss_network': 'glonass'}, {'los_method': 'time', 'los_value': dt.datetime(2023, 1, 1)}]}} - +_test_download = {'': {'los': False}} # Download is too large to test _clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', 'Data provided at a clean level' if tag != 'vtec' else @@ -308,23 +308,26 @@ def load(fnames, tag='', inst_id='', los_method='site', los_value=None, else: raise ValueError('unknown tag provided for GNSS TEC Instrument') - # Squeeze the kindat and kinst 'coordinates', but keep them as floats - squeeze_dims = np.array(['kindat', 'kinst']) - squeeze_mask = [sdim in data.coords for sdim in squeeze_dims] - if np.any(squeeze_mask): - data = data.squeeze(dim=squeeze_dims[squeeze_mask]) - - # Get the maximum and minimum values for time, latitude, and longitude - meta['time'] = {meta.labels.notes: data['time'].values.dtype.__doc__, - meta.labels.min_val: np.nan, meta.labels.max_val: np.nan} - - for lat_key in lat_keys: - meta[lat_key] = {meta.labels.min_val: -90.0, meta.labels.max_val: 90.0} - - for lon_key in lon_keys: - min_lon = 0.0 if data[lon_key].values.min() >= 0.0 else -180.0 - meta[lon_key] = {meta.labels.min_val: min_lon, - meta.labels.max_val: min_lon + 360.0} + if len(data.dims.keys()) > 0: + # Squeeze the kindat and kinst 'coordinates', but keep them as floats + squeeze_dims = np.array(['kindat', 'kinst']) + squeeze_mask = [sdim in data.coords for sdim in squeeze_dims] + if np.any(squeeze_mask): + data = data.squeeze(dim=squeeze_dims[squeeze_mask]) + + # Get the maximum and minimum values for time, latitude, and longitude + meta['time'] = {meta.labels.notes: data['time'].values.dtype.__doc__, + meta.labels.min_val: np.nan, + meta.labels.max_val: np.nan} + + for lat_key in lat_keys: + meta[lat_key] = {meta.labels.min_val: -90.0, + meta.labels.max_val: 90.0} + + for lon_key in lon_keys: + min_lon = 0.0 if data[lon_key].values.min() >= 0.0 else -180.0 + meta[lon_key] = {meta.labels.min_val: min_lon, + meta.labels.max_val: min_lon + 360.0} return data, meta From 666ae484d0f69500ae190d68ed50bf8e827ad1f1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 21 Nov 2023 10:17:08 -0500 Subject: [PATCH 087/139] TST: added load tests Added load tests for the LoS logger warning and ValueError. --- pysatMadrigal/tests/test_methods_gnss.py | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/pysatMadrigal/tests/test_methods_gnss.py b/pysatMadrigal/tests/test_methods_gnss.py index cbe9d72..ee0239b 100644 --- a/pysatMadrigal/tests/test_methods_gnss.py +++ b/pysatMadrigal/tests/test_methods_gnss.py @@ -5,8 +5,11 @@ # ---------------------------------------------------------------------------- """Test methods for `pysatMadrigal.instruments.methods.gnss`.""" +import logging import pytest +from pysat.utils.testing import eval_bad_input + from pysatMadrigal.instruments.methods import gnss @@ -31,3 +34,39 @@ def test_ref_output(self, func, comp_str, in_args): self.out = getattr(gnss, func)(*in_args) assert self.out.find(comp_str) >= 0 return + + +class TestGNSSBadLoad(object): + """Test GNSS load warnings and errors.""" + + def setup_method(self): + """Run before every method to create a clean testing setup.""" + self.bad_fnames = ['los_20230101.simple.gz', 'los_20230102.netCDF4'] + return + + def teardown_method(self): + """Run after every method to clean up previous testing.""" + del self.bad_fnames + return + + def test_bad_file_type_warning(self, caplog): + """Test logger warning for unsupported file types loading LoS data.""" + + # Get the output and raise the logging warning + with caplog.at_level(logging.WARN, logger='pysat'): + out = gnss.load_los(self.bad_fnames, "site", "zzon") + + # Test the logger warning + assert len(caplog.records) == 2, "unexpected number of warnings" + + for record in caplog.records: + assert record.levelname == "WARNING" + assert record.message.find("unable to load non-HDF5 slant TEC") >= 0 + return + + def test_bad_sel_type(self): + """Test ValueError raised for an unknown LoS down-selection type.""" + + eval_bad_input(gnss.load_los, ValueError, "unsupported selection type", + input_args=[self.bad_fnames, "bad_sel", "bad_val"]) + return From 7ef3225797f95fb27bae6e5be8189ca19e7c1f6a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 21 Nov 2023 10:21:17 -0500 Subject: [PATCH 088/139] STY: fixed PEP8 errors Removed unused variable and fixed indentation. --- pysatMadrigal/instruments/methods/general.py | 2 +- pysatMadrigal/tests/test_methods_gnss.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index 28b5e9b..7eb04fa 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -651,7 +651,7 @@ def convert_pandas_to_xarray(xarray_coords, data, time_ind): repr(xcoords), '], use only: ', repr(data.columns)])) elif not np.all([xkey.lower() in data.columns - for xkey in xarray_coords[xcoords]]): + for xkey in xarray_coords[xcoords]]): good_ind = [i for i, xkey in enumerate(xarray_coords[xcoords]) if xkey.lower() in data.columns] diff --git a/pysatMadrigal/tests/test_methods_gnss.py b/pysatMadrigal/tests/test_methods_gnss.py index ee0239b..ff571ec 100644 --- a/pysatMadrigal/tests/test_methods_gnss.py +++ b/pysatMadrigal/tests/test_methods_gnss.py @@ -54,7 +54,7 @@ def test_bad_file_type_warning(self, caplog): # Get the output and raise the logging warning with caplog.at_level(logging.WARN, logger='pysat'): - out = gnss.load_los(self.bad_fnames, "site", "zzon") + gnss.load_los(self.bad_fnames, "site", "zzon") # Test the logger warning assert len(caplog.records) == 2, "unexpected number of warnings" From cdfe14ea574ba87482cf81b205135da26579deb8 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 21 Nov 2023 20:09:09 -0500 Subject: [PATCH 089/139] TST: skip los CI download tests Corrected the download test flag. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pysatMadrigal/instruments/gnss_tec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index 32b83b7..efb9ad1 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -97,7 +97,7 @@ 'gnss_network': 'glonass'}, {'los_method': 'time', 'los_value': dt.datetime(2023, 1, 1)}]}} -_test_download = {'': {'los': False}} # Download is too large to test +_test_download_ci = {'': {'los': False}} # Download is too large to test _clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', 'Data provided at a clean level' if tag != 'vtec' else From c95f5388caa1b316c4a42d54d13b6601e141d426 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 22 Nov 2023 09:40:37 -0500 Subject: [PATCH 090/139] BUG: fixed clean method Fixed the clean method to have an informative message for the LoS data. Also removed inaccessable line in the load method. --- pysatMadrigal/instruments/gnss_tec.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index efb9ad1..6d10620 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -100,7 +100,7 @@ _test_download_ci = {'': {'los': False}} # Download is too large to test _clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', 'Data provided at a clean level' - if tag != 'vtec' else + if tag == 'site' else 'further cleaning may be performed', clean_level)] for clean_level in ['clean', 'dusty', 'dirty']} @@ -129,12 +129,14 @@ def clean(self): `clean_level` is None. """ - if self.tag in ["vtec", "site"]: - msg = "Data provided at a clean level" - if self.tag == "vtec": - msg = "".join([msg, ", further cleaning may be performed using ", - "the measurement error 'dtec'"]) - logger.info(msg) + msg = "Data provided at a clean level" + if self.tag == "vtec": + msg = "".join([msg, ", further cleaning may be performed using ", + "the measurement error 'dtec'"]) + elif self.tag == "los": + msg = "".join([msg, ", further cleaning may be performed using ", + "the measurement error 'dlos_tec'"]) + logger.info(msg) return @@ -305,8 +307,6 @@ def load(fnames, tag='', inst_id='', los_method='site', los_value=None, data, meta, lat_keys, lon_keys = gnss.load_los(fnames, los_method, los_value, gnss_network) - else: - raise ValueError('unknown tag provided for GNSS TEC Instrument') if len(data.dims.keys()) > 0: # Squeeze the kindat and kinst 'coordinates', but keep them as floats From ecf0044a07c25683965a02db864904681325a11f Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 22 Nov 2023 09:41:13 -0500 Subject: [PATCH 091/139] TST: added gnss_tec load error test Added a test for the gnss_tec load method ValueError raised if a bad `los_value` is supplied. --- pysatMadrigal/tests/test_instruments.py | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pysatMadrigal/tests/test_instruments.py b/pysatMadrigal/tests/test_instruments.py index d3ed072..9903b3e 100644 --- a/pysatMadrigal/tests/test_instruments.py +++ b/pysatMadrigal/tests/test_instruments.py @@ -5,8 +5,12 @@ # ---------------------------------------------------------------------------- """Unit tests for the Instruments.""" +import datetime as dt + # Import the test classes from pysat +import pysat from pysat.tests.classes import cls_instrument_library as clslib +from pysat.utils.testing import eval_bad_input import pysatMadrigal @@ -34,3 +38,27 @@ class TestInstruments(clslib.InstLibTests): instrument test class. """ + + +class TestInstrumentLoadError(object): + """Class for unit testing errors raised when loading data.""" + + def setup_method(self): + """Run before every method to create a clean testing setup.""" + self.inst_kwargs = [{'inst_module': pysatMadrigal.instruments.gnss_tec, + 'tag': 'los', 'los_method': 'site'}] + self.load_time = dt.datetime(2001, 1, 1) + return + + def teardown_method(self): + """Run after every method to clean up previous testing.""" + del self.inst_kwargs, self.load_time + return + + def test_bad_los_value(self): + """Test ValueError when the `los_value` is omitted.""" + inst = pysat.Instrument(**self.inst_kwargs) + + eval_bad_input(inst.load, ValueError, "must specify a valid", + input_kwargs={'date': self.load_time}) + return From 410a81869400f60af7a3c931bfe166b8fff3b0f6 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 22 Nov 2023 09:41:59 -0500 Subject: [PATCH 092/139] TST: added general unit tests Added unit tests for uncovered lines in the general instruments methods sub-module. Also improved style for Error tests. --- pysatMadrigal/tests/test_methods_general.py | 71 ++++++++++++++------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index 6663189..8d817c7 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -17,6 +17,7 @@ import netCDF4 as nc import pandas as pds import pysat +from pysat.utils.testing import eval_bad_input import pytest import xarray as xr @@ -42,6 +43,25 @@ def test_acknowledgements(self): assert self.out.find("CEDAR 'Rules of the Road'") >= 0 return + def test_sort_file_format(self, caplog): + """Test successful sorting of file names by extension.""" + # Get the output and raise the logging warning + with caplog.at_level(logging.WARN, logger='pysat'): + self.out = general.sort_file_formats(['test.hdf5', 'test.netCDF4', + 'test.simple.gz', 'test.bad']) + + # Evaluate the output + assert isinstance(self.out, dict) + pysat.utils.testing.assert_lists_equal(list(self.out.keys()), + ['hdf5', 'netCDF4', 'simple']) + + # Evaluate the logger warning + assert len(caplog.records) == 1, "unexpected number of warnings" + assert caplog.records[0].levelname == "WARNING" + assert caplog.records[0].message.find( + "file with unknown file type") >= 0 + return + @pytest.mark.parametrize("xarray_coords", [None, ["lat"]]) def test_empty_load(self, xarray_coords): """Test the general load function with no data files.""" @@ -244,11 +264,9 @@ def test_check_madrigal_params_no_code(self, inst_code): del self.kwargs['kindats'], self.kwargs['supported_tags'] self.kwargs['inst_code'] = inst_code - # Get the expected error message and evaluate it - with pytest.raises(ValueError) as verr: - general._check_madrigal_params(**self.kwargs) - - assert str(verr).find("Unknown Madrigal instrument code") >= 0 + eval_bad_input(general._check_madrigal_params, ValueError, + "Unknown Madrigal instrument code", + input_kwargs=self.kwargs) return @pytest.mark.parametrize("bad_val", [None, 17, False, 12.34]) @@ -269,10 +287,9 @@ def test_check_madrigal_params_bad_input(self, bad_val, test_key): self.kwargs[test_key] = bad_val # Get the expected error message and evaluate it - with pytest.raises(ValueError) as verr: - general._check_madrigal_params(**self.kwargs) - - assert str(verr).find("The madrigal database requries a username") >= 0 + eval_bad_input(general._check_madrigal_params, ValueError, + "The madrigal database requries a username", + input_kwargs=self.kwargs) return @pytest.mark.parametrize("del_val", ['kindats', 'supported_tags']) @@ -289,19 +306,17 @@ def test_list_remote_files_bad_kwargs(self, del_val): del self.kwargs[del_val] # Get the expected error message and evaluate it - with pytest.raises(ValueError) as verr: - general.list_remote_files('testing', 'tag', **self.kwargs) - - assert str(verr).find("Must supply supported_tags and kindats") >= 0 + eval_bad_input(general.list_remote_files, ValueError, + "Must supply supported_tags and kindats", + input_args=['testing', 'tag'], input_kwargs=self.kwargs) return def test_list_remote_files_bad_tag_inst_id(self): """Test that an error is thrown if None is passed through.""" # Get the expected error message and evaluate it - with pytest.raises(KeyError) as kerr: - general.list_remote_files('testing', 'not_tag', **self.kwargs) - - assert str(kerr).find('not_tag') >= 0 + eval_bad_input(general.list_remote_files, KeyError, "not_tag", + input_args=['testing', 'not_tag'], + input_kwargs=self.kwargs) return @pytest.mark.parametrize("in_key, in_val, test_verr", [ @@ -323,10 +338,9 @@ def test_download_valueerror(self, in_key, in_val, test_verr): del self.kwargs['supported_tags'], self.kwargs['kindats'] self.kwargs[in_key] = in_val - with pytest.raises(ValueError) as verr: - general.download([], **self.kwargs) - - assert str(verr).find(test_verr) >= 0 + # Get the expected error message and evaluate it + eval_bad_input(general.download, ValueError, test_verr, + input_args=[[]], input_kwargs=self.kwargs) return def test_get_remote_filenames_bad_date_array(self): @@ -334,10 +348,19 @@ def test_get_remote_filenames_bad_date_array(self): del self.kwargs['supported_tags'], self.kwargs['kindats'] self.kwargs['date_array'] = [] - with pytest.raises(ValueError) as verr: - general.get_remote_filenames(**self.kwargs) + # Get the expected error message and evaluate it + eval_bad_input(general.get_remote_filenames, ValueError, + "unknown date_array supplied", input_kwargs=self.kwargs) + return - assert str(verr).find("unknown date_array supplied") >= 0 + def test_convert_pandas_to_xarray_bad_data_vars(self): + """Test raises ValueError for unexpected date_array input.""" + self.kwargs = [{('time', ): ['bad_var']}, pds.DataFrame([0]), + pds.DatetimeIndex([dt.datetime(2001, 1, 1)])] + + # Get the expected error message and evaluate it + eval_bad_input(general.convert_pandas_to_xarray, ValueError, + "All data variables", input_args=self.kwargs) return From b30b2dbc9bc5a38bb04b80987de31d1e3134d4d6 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 22 Nov 2023 09:42:36 -0500 Subject: [PATCH 093/139] TST: added empty load test Added a test for loading no data in the `load_los` function, additionally testing the time selection set-up. --- pysatMadrigal/tests/test_methods_gnss.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pysatMadrigal/tests/test_methods_gnss.py b/pysatMadrigal/tests/test_methods_gnss.py index ff571ec..e50f1cf 100644 --- a/pysatMadrigal/tests/test_methods_gnss.py +++ b/pysatMadrigal/tests/test_methods_gnss.py @@ -5,6 +5,7 @@ # ---------------------------------------------------------------------------- """Test methods for `pysatMadrigal.instruments.methods.gnss`.""" +import datetime as dt import logging import pytest @@ -70,3 +71,13 @@ def test_bad_sel_type(self): eval_bad_input(gnss.load_los, ValueError, "unsupported selection type", input_args=[self.bad_fnames, "bad_sel", "bad_val"]) return + + def test_empty_los_load(self): + """Test the returned dataset is empty for a LoS load.""" + data, meta, lats, lons = gnss.load_los(self.bad_fnames, "time", + dt.datetime(2023, 1, 1)) + + assert len(data.dims.keys()) == 0 + assert len(lats) == 2 + assert len(lons) == 2 + return From 42735b144d8a7f8c45ee2654a1f74fec327cafb7 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 22 Nov 2023 09:43:37 -0500 Subject: [PATCH 094/139] TST: added coveralls line Added a coveralls service job environment variable. --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 07f4e21..4a8fcc1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,4 +71,5 @@ jobs: - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_SERVICE_JOB_ID: ${{ github.job }} run: coveralls --rcfile=setup.cfg --service=github From 13562edf062fe31beb5280abeb0446fc36efb168 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 22 Nov 2023 16:13:15 -0500 Subject: [PATCH 095/139] BUG: accessed correct instrument kwargs Needed to get the right dict from the setup. --- pysatMadrigal/tests/test_instruments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/tests/test_instruments.py b/pysatMadrigal/tests/test_instruments.py index 9903b3e..171ca61 100644 --- a/pysatMadrigal/tests/test_instruments.py +++ b/pysatMadrigal/tests/test_instruments.py @@ -57,7 +57,7 @@ def teardown_method(self): def test_bad_los_value(self): """Test ValueError when the `los_value` is omitted.""" - inst = pysat.Instrument(**self.inst_kwargs) + inst = pysat.Instrument(**self.inst_kwargs[0]) eval_bad_input(inst.load, ValueError, "must specify a valid", input_kwargs={'date': self.load_time}) From 1a62f4e32870fb50e662d0a15e571cf241888a39 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 27 Nov 2023 09:24:00 -0500 Subject: [PATCH 096/139] BUG: add fake files Add a fake file to test the load failures. --- pysatMadrigal/tests/test_instruments.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pysatMadrigal/tests/test_instruments.py b/pysatMadrigal/tests/test_instruments.py index 171ca61..e34d09e 100644 --- a/pysatMadrigal/tests/test_instruments.py +++ b/pysatMadrigal/tests/test_instruments.py @@ -6,6 +6,8 @@ """Unit tests for the Instruments.""" import datetime as dt +import pathlib +import os # Import the test classes from pysat import pysat @@ -48,17 +50,33 @@ def setup_method(self): self.inst_kwargs = [{'inst_module': pysatMadrigal.instruments.gnss_tec, 'tag': 'los', 'los_method': 'site'}] self.load_time = dt.datetime(2001, 1, 1) + self.fake_file = '' return def teardown_method(self): """Run after every method to clean up previous testing.""" - del self.inst_kwargs, self.load_time + if os.path.isfile(self.fake_file): + os.remove(self.fake_file) + + del self.inst_kwargs, self.load_time, self.fake_file return def test_bad_los_value(self): """Test ValueError when the `los_value` is omitted.""" inst = pysat.Instrument(**self.inst_kwargs[0]) + # Ensure a file is available + if self.load_time not in inst.files.files.keys(): + self.fake_file = os.path.join( + inst.files.data_path, + self.inst_kwargs['inst_module'].supported_tags[inst.inst_id][ + inst.tag].format(file_type='hdf5').format( + year=self.load_time.year, month=self.load_time.month, + day=self.load_time.day, version=1)) + pysat.utils.files.check_and_make_path(inst.files.data_path) + pathlib.Path(self.fake_file).touch() + inst = pysat.Instrument(**self.inst_kwargs[0]) + eval_bad_input(inst.load, ValueError, "must specify a valid", input_kwargs={'date': self.load_time}) return From be25ef56b9b810d355fdba1413c860d7d3214148 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 13:00:19 -0500 Subject: [PATCH 097/139] STY: fixed import order Fixed import order to be alphabetical. --- pysatMadrigal/tests/test_instruments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/tests/test_instruments.py b/pysatMadrigal/tests/test_instruments.py index e34d09e..7bb6d8d 100644 --- a/pysatMadrigal/tests/test_instruments.py +++ b/pysatMadrigal/tests/test_instruments.py @@ -6,8 +6,8 @@ """Unit tests for the Instruments.""" import datetime as dt -import pathlib import os +import pathlib # Import the test classes from pysat import pysat From fedd62c1f15dbad84a8d3ddbd107274672f9ae24 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 13:00:32 -0500 Subject: [PATCH 098/139] TST: updated coveralls Updated coveralls to run in parallel. --- .github/workflows/main.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4a8fcc1..42c802e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,5 +71,17 @@ jobs: - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_SERVICE_JOB_ID: ${{ github.job }} - run: coveralls --rcfile=setup.cfg --service=github + COVERALLS_PARALLEL: true + run: coveralls --rcfile=pyproject.toml --service=github + + finish: + name: Finish Coverage Analysis + needs: build + runs-on: ubuntu-latest + steps: + - name: Coveralls Finished + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip install --upgrade coveralls + coveralls --service=github --finish From 45b22e5e7c6d6689bdb7490fb0bd935a8ebb3658 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Mon, 27 Nov 2023 13:05:52 -0500 Subject: [PATCH 099/139] TST: disabled local LoS download tests Disabled local LoS download tests, also added a warning for users attempting downloads. --- pysatMadrigal/instruments/gnss_tec.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index 6d10620..fb7d2bc 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -55,7 +55,7 @@ import datetime as dt import numpy as np -from pysat import logger +import pysat from pysatMadrigal.instruments.methods import general from pysatMadrigal.instruments.methods import gnss @@ -98,6 +98,7 @@ {'los_method': 'time', 'los_value': dt.datetime(2023, 1, 1)}]}} _test_download_ci = {'': {'los': False}} # Download is too large to test +_password_req = {'': {'los': True}} # Disable locally, download is too large _clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', 'Data provided at a clean level' if tag == 'site' else @@ -115,7 +116,7 @@ def init(self): self.acknowledgements = '\n'.join([gnss.acknowledgements(self.name), general.cedar_rules()]) self.references = gnss.references(self.name) - logger.info(self.acknowledgements) + pysat.logger.info(self.acknowledgements) return @@ -136,7 +137,7 @@ def clean(self): elif self.tag == "los": msg = "".join([msg, ", further cleaning may be performed using ", "the measurement error 'dlos_tec'"]) - logger.info(msg) + pysat.logger.info(msg) return @@ -244,6 +245,10 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None, downloads. """ + if tag == 'los': + pysat.logger.warning( + 'LoS download is very large and succeptible to failure.') + general.download(date_array, inst_code=str(madrigal_inst_code), kindat=madrigal_tag[inst_id][tag], data_path=data_path, user=user, password=password, file_type=file_type, url=url) From 9b8919b5311d837f384b86a1a1c9c2b702a7b29e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 29 Nov 2023 13:51:59 -0500 Subject: [PATCH 100/139] TST: removed old test line Removed old test line. --- pysatMadrigal/instruments/gnss_tec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index fb7d2bc..4c53f6c 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -97,7 +97,6 @@ 'gnss_network': 'glonass'}, {'los_method': 'time', 'los_value': dt.datetime(2023, 1, 1)}]}} -_test_download_ci = {'': {'los': False}} # Download is too large to test _password_req = {'': {'los': True}} # Disable locally, download is too large _clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', 'Data provided at a clean level' From 5335437dfe839dd8665639da9d6aaaa5fceba024 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 29 Nov 2023 13:56:33 -0500 Subject: [PATCH 101/139] MAINT: updated version caps Updated the version caps for coveralls, sphinx, and sphinx_rtd_theme. --- pyproject.toml | 4 ++-- test_requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c2a2051..74cdda8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ [project.optional-dependencies] test = [ - "coveralls < 3.3", + "coveralls", "flake8", "flake8-docstrings", "hacking >= 1.0", @@ -68,7 +68,7 @@ doc = [ "numpydoc", "pyproject_parser", "sphinx", - "sphinx_rtd_theme >= 1.2.2" + "sphinx_rtd_theme >= 1.2.2, < 2.0.0" ] [project.urls] diff --git a/test_requirements.txt b/test_requirements.txt index 9893985..3bfdc26 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,4 @@ -coveralls<3.3 +coveralls flake8 flake8-docstrings hacking>=1.0 @@ -6,5 +6,5 @@ m2r2 numpydoc pytest-cov pytest-ordering -sphinx<7.0 -sphinx_rtd_theme +sphinx +sphinx_rtd_theme>=1.2.2,<2.0.0 From d78742cabf9f1f9a8c40382be60001b92bfd94f5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Dec 2023 18:21:40 -0500 Subject: [PATCH 102/139] Revert "TST: removed old test line" This reverts commit 9b8919b5311d837f384b86a1a1c9c2b702a7b29e. --- pysatMadrigal/instruments/gnss_tec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index 4c53f6c..fb7d2bc 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -97,6 +97,7 @@ 'gnss_network': 'glonass'}, {'los_method': 'time', 'los_value': dt.datetime(2023, 1, 1)}]}} +_test_download_ci = {'': {'los': False}} # Download is too large to test _password_req = {'': {'los': True}} # Disable locally, download is too large _clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', 'Data provided at a clean level' From ba6ddba94cff0256bf8a6d028e749829b65ad4bd Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Dec 2023 18:24:50 -0500 Subject: [PATCH 103/139] BUG: removed bad test variable Removed a test variable that was being used for an incorrect purpose. --- pysatMadrigal/instruments/gnss_tec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index fb7d2bc..a51b496 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -98,7 +98,6 @@ {'los_method': 'time', 'los_value': dt.datetime(2023, 1, 1)}]}} _test_download_ci = {'': {'los': False}} # Download is too large to test -_password_req = {'': {'los': True}} # Disable locally, download is too large _clean_warn = {'': {tag: {clean_level: [('logger', 'INFO', 'Data provided at a clean level' if tag == 'site' else From 95216c6e0ae6503aef938dce82b4b7ff422a769c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Dec 2023 18:33:48 -0500 Subject: [PATCH 104/139] BUG: fixed workflow Removed extra 'run' from workflow. --- .github/workflows/pysat_rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 924ebbb..bed6da0 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -27,7 +27,7 @@ jobs: run: pip install --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat - name: Install standard dependencies and package - run: run pip install .[test] + run: pip install .[test] - name: Set up pysat run: | From 757133bd3edde244e95cf5c58acd514523594355 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Dec 2023 18:47:45 -0500 Subject: [PATCH 105/139] MAINT: updated coveralls Updated the coveralls line in the RC CI test. --- .github/workflows/pysat_rc.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index bed6da0..176b59e 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -40,4 +40,17 @@ jobs: - name: Publish results to coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls --rcfile=setup.cfg --service=github + COVERALLS_PARALLEL: true + run: coveralls --rcfile=pyproject.toml --service=github + + finish: + name: Finish Coverage Analysis + needs: build + runs-on: ubuntu-latest + steps: + - name: Coveralls Finished + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip install --upgrade coveralls + coveralls --service=github --finish From d549e2cbfa6b8e1e2031fab988f0e71d128a16a1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 18 Dec 2023 17:30:51 -0500 Subject: [PATCH 106/139] BUG: updated DMSP url Added the 'url' kwarg to the DMSP IVM download method, since UTD downloads were failing at the central site. --- pysatMadrigal/instruments/dmsp_ivm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index 1855e5a..8c7bf24 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -178,7 +178,8 @@ def clean(self): def download(date_array, tag='', inst_id='', data_path=None, user=None, - password=None, file_type='hdf5'): + password=None, file_type='hdf5', + url="http://millstonehill.haystack.mit.edu/"): """Download data from Madrigal. Parameters @@ -202,6 +203,9 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None, Password for data download. (default=None) file_type : str File format for Madrigal data. (default='hdf5') + url : str + URL for the specific Madrigal data site. + (default='http://millstonehill.haystack.mit.edu/') Note ---- @@ -217,5 +221,5 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None, """ general.download(date_array, inst_code=str(madrigal_inst_code), kindat=madrigal_tag[inst_id][tag], data_path=data_path, - user=user, password=password, file_type=file_type) + user=user, password=password, file_type=file_type, url=url) return From d173e8a5b788f0b1d7f176bc8778f719e708617f Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 18 Dec 2023 17:52:12 -0500 Subject: [PATCH 107/139] BUG: removed url kwarg Removed the URL kwarg, local tests didn't need the url change. --- pysatMadrigal/instruments/dmsp_ivm.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index 8c7bf24..1855e5a 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -178,8 +178,7 @@ def clean(self): def download(date_array, tag='', inst_id='', data_path=None, user=None, - password=None, file_type='hdf5', - url="http://millstonehill.haystack.mit.edu/"): + password=None, file_type='hdf5'): """Download data from Madrigal. Parameters @@ -203,9 +202,6 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None, Password for data download. (default=None) file_type : str File format for Madrigal data. (default='hdf5') - url : str - URL for the specific Madrigal data site. - (default='http://millstonehill.haystack.mit.edu/') Note ---- @@ -221,5 +217,5 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None, """ general.download(date_array, inst_code=str(madrigal_inst_code), kindat=madrigal_tag[inst_id][tag], data_path=data_path, - user=user, password=password, file_type=file_type, url=url) + user=user, password=password, file_type=file_type) return From f4c84af124091165c0666e37bb25f4d1e2299c4e Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 19 Dec 2023 03:11:38 -0500 Subject: [PATCH 108/139] BUG: added missing index Added a missing index for the test list. --- pysatMadrigal/tests/test_instruments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/tests/test_instruments.py b/pysatMadrigal/tests/test_instruments.py index 7bb6d8d..40a2110 100644 --- a/pysatMadrigal/tests/test_instruments.py +++ b/pysatMadrigal/tests/test_instruments.py @@ -69,7 +69,7 @@ def test_bad_los_value(self): if self.load_time not in inst.files.files.keys(): self.fake_file = os.path.join( inst.files.data_path, - self.inst_kwargs['inst_module'].supported_tags[inst.inst_id][ + self.inst_kwargs[0]['inst_module'].supported_tags[inst.inst_id][ inst.tag].format(file_type='hdf5').format( year=self.load_time.year, month=self.load_time.month, day=self.load_time.day, version=1)) From 96713d54bea10ba780999c4b630c78e6324e7e43 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 19 Dec 2023 14:43:38 -0500 Subject: [PATCH 109/139] REL: update version number Update version number in preparation for release. --- pyproject.toml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 74cdda8..31a4351 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pysatMadrigal" -version = "0.1.0" +version = "0.2.0" description = 'Madrigal instrument support for the pysat ecosystem' readme = "README.md" requires-python = ">=3.6" diff --git a/setup.cfg b/setup.cfg index a2f4510..aa9e557 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pysatMadrigal -version = 0.1.0 +version = 0.2.0 [flake8] max-line-length = 80 From bfd2ae3cee794665fab9847428a055bf7aa9f383 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 19 Dec 2023 14:44:04 -0500 Subject: [PATCH 110/139] REL: update changelog Update the changelog release version number and target date. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 094d7b8..e51b9a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -[0.1.X] - 2023-XX-XX +[0.2.0] - 2024-01-10 -------------------- * Enhancements * Moved the OMNI-2 IMF, Dst, and Geomagnetic index Instruments from the From 228af42abb86d18a99fd73b523da0619e93d7baa Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 31 Jan 2024 16:37:48 -0500 Subject: [PATCH 111/139] DOC: add acknowledgements Add acknowledgements to the documentation and the main directory. --- ACKNOWLEDGEMENTS.md | 28 ++++++++++++++++++++++++++++ docs/acknowledgements.rst | 1 + docs/index.rst | 5 +++++ 3 files changed, 34 insertions(+) create mode 100644 ACKNOWLEDGEMENTS.md create mode 100644 docs/acknowledgements.rst diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md new file mode 100644 index 0000000..a1b7ddd --- /dev/null +++ b/ACKNOWLEDGEMENTS.md @@ -0,0 +1,28 @@ +Funding +======= +The following institutions, missions, and programs have provided funding +for pysatSpaceWeather development. + +Institutions +------------ + - Cosmic Studio + - Defense Advanced Research Projects Agency (DARPA) Defense Sciences Office + - National Aeronautics and Space Administration (NASA) + - National Science Foundation (NSF) + - Office of Naval Research (ONR) + +Programs +-------- + - NSF 125908, AGS-1651393 + - Naval Research Laboratory N00173191G016 and N0017322P0744 + +Disclaimers +=========== +Any opinions, findings, and conclusions or recommendations expressed in this +material are those of the author(s) and do not necessarily reflect the views +of the funding agencies. + +Support and Thanks +================== +Thanks to Bill Rideout, who provided help when learning how to use the +madrigalWeb package. diff --git a/docs/acknowledgements.rst b/docs/acknowledgements.rst new file mode 100644 index 0000000..dd1f917 --- /dev/null +++ b/docs/acknowledgements.rst @@ -0,0 +1 @@ +.. mdinclude:: ../ACKNOWLEDGEMENTS.md diff --git a/docs/index.rst b/docs/index.rst index 4787575..899f928 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,6 +19,11 @@ data base as pysat.Instrument objects. examples.rst develop_guide.rst history.rst + acknowledgements.rst + + +.. admonition:: DISTRIBUTION STATEMENT A: Approved for public release. + Distribution is unlimited. Indices and tables From 0db87d16d6b72a152215ba8a96add52a0969ed8a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 31 Jan 2024 16:38:13 -0500 Subject: [PATCH 112/139] DOC: update file headers Update the code file headers to include the NRL pub release. --- pysatMadrigal/instruments/dmsp_ivm.py | 7 ++++--- pysatMadrigal/instruments/dmsp_ssj.py | 5 ++++- pysatMadrigal/instruments/gnss_tec.py | 10 +++++++++- pysatMadrigal/instruments/jro_isr.py | 5 ++++- pysatMadrigal/instruments/madrigal_dst.py | 5 ++++- pysatMadrigal/instruments/madrigal_geoind.py | 5 ++++- pysatMadrigal/instruments/methods/dmsp.py | 5 ++++- pysatMadrigal/instruments/methods/general.py | 5 ++++- pysatMadrigal/instruments/methods/gnss.py | 5 ++++- pysatMadrigal/instruments/methods/jro.py | 5 ++++- pysatMadrigal/instruments/ngdc_ae.py | 5 ++++- pysatMadrigal/instruments/omni2_imf.py | 5 ++++- pysatMadrigal/tests/test_instruments.py | 4 ++++ pysatMadrigal/tests/test_methods_dmsp.py | 6 +++++- pysatMadrigal/tests/test_methods_general.py | 4 ++++ pysatMadrigal/tests/test_methods_gnss.py | 4 ++++ pysatMadrigal/tests/test_methods_jro.py | 4 ++++ pysatMadrigal/tests/test_utils_coords.py | 9 +++++++++ pysatMadrigal/utils/coords.py | 4 ++++ 19 files changed, 87 insertions(+), 15 deletions(-) diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index 1855e5a..980718a 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*- """Supports the Defense Meteorological Satellite Program (DMSP) IVM instruments. The Ion Velocity Meter (IVM) is comprised of the Retarding Potential Analyzer @@ -45,8 +48,6 @@ ---- Please provide name and email when downloading data with this routine. -Code development supported by NSF grant 1259508 - """ import datetime as dt diff --git a/pysatMadrigal/instruments/dmsp_ssj.py b/pysatMadrigal/instruments/dmsp_ssj.py index 21c4cb6..fcdc313 100644 --- a/pysatMadrigal/instruments/dmsp_ssj.py +++ b/pysatMadrigal/instruments/dmsp_ssj.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*- """Support the DMSP Special Sensor-J (SSJ) instrument and derived products. The Defense Meteorological Satellite Program (DMSP) SSJ measures precipitating diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py index a51b496..b922096 100644 --- a/pysatMadrigal/instruments/gnss_tec.py +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -1,4 +1,12 @@ -# -*- coding: utf-8 -*-. +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# ---------------------------------------------------------------------------- """Supports the MIT Haystack GNSS TEC data products. The Global Navigation Satellite System (GNSS) is used in conjunction with a diff --git a/pysatMadrigal/instruments/jro_isr.py b/pysatMadrigal/instruments/jro_isr.py index b500a72..f4bf99f 100644 --- a/pysatMadrigal/instruments/jro_isr.py +++ b/pysatMadrigal/instruments/jro_isr.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*-. """Supports the Incoherent Scatter Radar at the Jicamarca Radio Observatory. Properties diff --git a/pysatMadrigal/instruments/madrigal_dst.py b/pysatMadrigal/instruments/madrigal_dst.py index 8dea5a5..7324c59 100644 --- a/pysatMadrigal/instruments/madrigal_dst.py +++ b/pysatMadrigal/instruments/madrigal_dst.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*-. """Supports access to Dst data archieved at Madrigal. Properties diff --git a/pysatMadrigal/instruments/madrigal_geoind.py b/pysatMadrigal/instruments/madrigal_geoind.py index 0cf19d3..be288b0 100644 --- a/pysatMadrigal/instruments/madrigal_geoind.py +++ b/pysatMadrigal/instruments/madrigal_geoind.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*-. """Supports access to F10.7, Kp, and Ap data archieved at Madrigal. Properties diff --git a/pysatMadrigal/instruments/methods/dmsp.py b/pysatMadrigal/instruments/methods/dmsp.py index 42be397..0eb4c56 100644 --- a/pysatMadrigal/instruments/methods/dmsp.py +++ b/pysatMadrigal/instruments/methods/dmsp.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*- """Methods supporting the Defense Meteorological Satellite Program (DMSP).""" import numpy as np diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py index 7eb04fa..4ca23bd 100644 --- a/pysatMadrigal/instruments/methods/general.py +++ b/pysatMadrigal/instruments/methods/general.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*-. """General routines for integrating CEDAR Madrigal instruments into pysat.""" import datetime as dt diff --git a/pysatMadrigal/instruments/methods/gnss.py b/pysatMadrigal/instruments/methods/gnss.py index eb3b620..0283a18 100644 --- a/pysatMadrigal/instruments/methods/gnss.py +++ b/pysatMadrigal/instruments/methods/gnss.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*- """Methods supporting the Global Navigation Satellite System platform.""" import datetime as dt diff --git a/pysatMadrigal/instruments/methods/jro.py b/pysatMadrigal/instruments/methods/jro.py index 880f7bb..4d8dcf3 100644 --- a/pysatMadrigal/instruments/methods/jro.py +++ b/pysatMadrigal/instruments/methods/jro.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*- """Methods supporting the Jicamarca Radio Observatory (JRO) platform.""" import numpy as np diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py index 7cb1e54..67aa42e 100644 --- a/pysatMadrigal/instruments/ngdc_ae.py +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*-. """Supports access to taped data of AE from the World Data Center A (Boulder). Properties diff --git a/pysatMadrigal/instruments/omni2_imf.py b/pysatMadrigal/instruments/omni2_imf.py index d06296e..cbadac5 100644 --- a/pysatMadrigal/instruments/omni2_imf.py +++ b/pysatMadrigal/instruments/omni2_imf.py @@ -1,9 +1,12 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -# -*- coding: utf-8 -*-. """Supports access to OMNI 2 IMF data archieved at Madrigal. Properties diff --git a/pysatMadrigal/tests/test_instruments.py b/pysatMadrigal/tests/test_instruments.py index 40a2110..20277db 100644 --- a/pysatMadrigal/tests/test_instruments.py +++ b/pysatMadrigal/tests/test_instruments.py @@ -1,7 +1,11 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- """Unit tests for the Instruments.""" diff --git a/pysatMadrigal/tests/test_methods_dmsp.py b/pysatMadrigal/tests/test_methods_dmsp.py index cc0f1bc..1d0efd7 100644 --- a/pysatMadrigal/tests/test_methods_dmsp.py +++ b/pysatMadrigal/tests/test_methods_dmsp.py @@ -1,9 +1,13 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- -"""Test methods for `pysatMadrigal.instruments.methods.gnss`.""" +"""Test methods for `pysatMadrigal.instruments.methods.dmsp`.""" import logging import numpy as np diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index 8d817c7..500346a 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -1,7 +1,11 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- """Unit tests for the general instrument methods.""" diff --git a/pysatMadrigal/tests/test_methods_gnss.py b/pysatMadrigal/tests/test_methods_gnss.py index e50f1cf..6338974 100644 --- a/pysatMadrigal/tests/test_methods_gnss.py +++ b/pysatMadrigal/tests/test_methods_gnss.py @@ -1,7 +1,11 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- """Test methods for `pysatMadrigal.instruments.methods.gnss`.""" diff --git a/pysatMadrigal/tests/test_methods_jro.py b/pysatMadrigal/tests/test_methods_jro.py index b3278d2..3994649 100644 --- a/pysatMadrigal/tests/test_methods_jro.py +++ b/pysatMadrigal/tests/test_methods_jro.py @@ -1,7 +1,11 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- """Test methods for `pysatMadrigal.instruments.methods.jro`.""" diff --git a/pysatMadrigal/tests/test_utils_coords.py b/pysatMadrigal/tests/test_utils_coords.py index 28ec880..2664d8e 100644 --- a/pysatMadrigal/tests/test_utils_coords.py +++ b/pysatMadrigal/tests/test_utils_coords.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. +# ---------------------------------------------------------------------------- """Tests for the coordinate conversion functions.""" import numpy as np diff --git a/pysatMadrigal/utils/coords.py b/pysatMadrigal/utils/coords.py index e5db684..d95ba58 100644 --- a/pysatMadrigal/utils/coords.py +++ b/pysatMadrigal/utils/coords.py @@ -1,7 +1,11 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Full license can be found in License.md # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3824979 +# +# DISTRIBUTION STATEMENT A: Approved for public release. Distribution is +# unlimited. # ---------------------------------------------------------------------------- """Coordinate transformation functions.""" From fb4f357ab6b94272664d0b89df5e4f81f2de6926 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 1 Feb 2024 11:18:12 -0500 Subject: [PATCH 113/139] MAINT: updated readme Updated the readme install instructions and fixed a typo in the example. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0ae27ba..cbeb110 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,8 @@ a local install use the "--user" flag after "install". ``` cd pysatMadrigal/ -python setup.py install +python -m build . +pip install . ``` # Examples @@ -67,7 +68,7 @@ Afterward, pysat will identify them using the `platform` and `name` keywords. ``` pysat.utils.registry.register('pysatMadrigal.instruments.dmsp_ivm') -dst = pysat.Instrument('dmsp', 'ivm', tag='utd', inst_id='f15') +ivm = pysat.Instrument('dmsp', 'ivm', tag='utd', inst_id='f15') ``` The package also include analysis tools. Detailed examples are in the From d44601bd6300971b9538b5bf366f99f00db28356 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 21 Feb 2024 16:48:13 -0500 Subject: [PATCH 114/139] DOC: Update CHANGELOG.md Update the targeted release date to unspecified in preparation for a separate release candidate branch. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e51b9a3..361d67f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -[0.2.0] - 2024-01-10 +[0.2.0] - 2024-XX-XX -------------------- * Enhancements * Moved the OMNI-2 IMF, Dst, and Geomagnetic index Instruments from the From 48964fa4cc2db4b3c31d63876a2f87ada09003d9 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Sun, 25 Feb 2024 23:18:09 -0500 Subject: [PATCH 115/139] MAINT: update acknowledgements Update the acknowledgements. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- ACKNOWLEDGEMENTS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index a1b7ddd..4354ddc 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -1,10 +1,11 @@ Funding ======= The following institutions, missions, and programs have provided funding -for pysatSpaceWeather development. +for pysatMadrigal development. Institutions ------------ + - The Catholic University of America (CUA) - Cosmic Studio - Defense Advanced Research Projects Agency (DARPA) Defense Sciences Office - National Aeronautics and Space Administration (NASA) From 80b44fb53d708742a2211b025b5c9a35dc98c6ad Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 26 Feb 2024 13:42:26 -0500 Subject: [PATCH 116/139] TST: updated CI yamls Updated the CI yamls by: - upgrading the action versions, - adding support for Python 3.12, and - cycling versions performing doc/pypi tests. --- .github/workflows/docs.yml | 6 +++--- .github/workflows/main.yml | 8 ++++---- .github/workflows/pip_rc_install.yml | 6 +++--- .github/workflows/pysat_rc.yml | 6 +++--- .readthedocs.yml | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ec7a09e..8cf4ccf 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,13 +13,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9"] + python-version: ["3.11"] name: Documentation tests steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install with dependencies diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42c802e..36668b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,12 +13,12 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] test_config: ["latest"] include: # NEP29 compliance settings - python-version: "3.9" - numpy_ver: "1.21" + numpy_ver: "1.23" os: ubuntu-latest test_config: "NEP29" # Operational compliance settings @@ -30,9 +30,9 @@ jobs: name: ${{ matrix.test_config }} Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index d9d3f41..ca7b8a6 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -13,14 +13,14 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.11"] # Keep this version at the highest supported Python version + python-version: ["3.12"] # Keep this version at the highest supported Python version name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 176b59e..3b7a80f 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -12,14 +12,14 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10"] + python-version: ["3.11"] name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.readthedocs.yml b/.readthedocs.yml index e166702..b198cf8 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.10" + python: "3.11" # Build documentation in the docs/ directory with Sphinx sphinx: From c9515e95111dbfd9e37ec56b436fe52d678253be Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 26 Feb 2024 13:45:40 -0500 Subject: [PATCH 117/139] MAINT: updated the python version Updated the supported python versions. --- README.md | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cbeb110..3abb977 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ give some examples on how to use the routines. ## Prerequisites pysatMadrigal uses common Python modules, as well as modules developed by and -for the Space Physics community. This module officially supports Python 3.7+. +for the Space Physics community. This module officially supports Python 3.6+. | Common modules | Community modules | | -------------- | ----------------- | diff --git a/pyproject.toml b/pyproject.toml index 31a4351..17494ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows" From ea52b19bfce1ed0b35be7b8c7dc0692681a4380c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 26 Feb 2024 13:46:03 -0500 Subject: [PATCH 118/139] DOC: set target release date Set a target release date in the changelog. Also added maintenance details. --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 361d67f..ed96765 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -[0.2.0] - 2024-XX-XX +[0.2.0] - 2024-03-15 -------------------- * Enhancements * Moved the OMNI-2 IMF, Dst, and Geomagnetic index Instruments from the @@ -16,7 +16,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). were used for specific load functions * Maintenance * Add manual GitHub Actions tests for pysatMadrigal RC - * Update GitHub Actions workflows for newer versions of pip + * Update GitHub Actions workflows for newer versions of pip, updated actions + * Added support for Python 3.12 * Added clean warning test attributes to all Instruments * Updated documentation to comply with current Ecosystem guidelines * Replaced setup.py with pyproject.toml From 2ec1c40e0eddc30578f3714d5c68c4a8f30fa029 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 26 Feb 2024 13:58:42 -0500 Subject: [PATCH 119/139] DOC: change index depth Change the index depth to have a maximum of two. --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 899f928..5439d5e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,7 +9,7 @@ to download, load, and support analysis for data sets available at the Madrigal data base as pysat.Instrument objects. .. toctree:: - :maxdepth: -1 + :maxdepth: 2 overview.rst installation.rst From e60af333e29bd3c4914c23ef346478ded8ed096a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 26 Feb 2024 17:30:43 -0500 Subject: [PATCH 120/139] BUG: bug fixes for pysat RC Fixes tests that are failing in the pysat RC. Co-authored-by: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> --- pyproject.toml | 1 + pysatMadrigal/instruments/dmsp_ivm.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 17494ff..4d77d3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ markers = [ "download", "no_download", "load_options", + "new_tests", "first", "second" ] diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index 980718a..ad24c8f 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -105,6 +105,8 @@ 'f17': {tag: dt.datetime(2009, 1, 1) for tag in inst_ids['f17']}, 'f18': {tag: dt.datetime(2017, 12, 30) for tag in inst_ids['f18']}} +# TODO(#99): Remove when compliant with multi-day load tests +_new_tests = {'f18': {'': False}} _clean_warn = {inst_id: {tag: {clean_level: [('logger', 'WARN', 'this level 1 data has no qual', clean_level)] From 8417fa30a321782d44336caae4d2582dcbaf412b Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 27 Feb 2024 10:57:57 -0500 Subject: [PATCH 121/139] DOC: added comment Added a comment to clarify which part is involved in the TODO statement. --- pysatMadrigal/instruments/dmsp_ivm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index ad24c8f..e61c2d6 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -107,6 +107,8 @@ # TODO(#99): Remove when compliant with multi-day load tests _new_tests = {'f18': {'': False}} + +# Set the clean warnings for testing _clean_warn = {inst_id: {tag: {clean_level: [('logger', 'WARN', 'this level 1 data has no qual', clean_level)] From abcfc2b0f63ff8879af50d7f28905d4e7b3a9da0 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 27 Feb 2024 10:58:35 -0500 Subject: [PATCH 122/139] TST: added skip for 'drifts' and 'drifts_ave' The multi-day load tests are failing for the drift data, this may be a test day issue. --- pysatMadrigal/instruments/jro_isr.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysatMadrigal/instruments/jro_isr.py b/pysatMadrigal/instruments/jro_isr.py index f4bf99f..f1c4d7b 100644 --- a/pysatMadrigal/instruments/jro_isr.py +++ b/pysatMadrigal/instruments/jro_isr.py @@ -91,6 +91,10 @@ 'oblique_rand': dt.datetime(2000, 11, 9), 'oblique_long': dt.datetime(2010, 4, 12)}} +# TODO(#99): Remove when compliant with multi-day load tests +_new_tests = {'': {'drifts': False, 'drifts_ave': False}} + +# Set the clean warnings for testing _clean_warn = {'': {tag: {clean_level: [('logger', 'WARN', 'this level 2 data has no quality ', clean_level)] From 10468737e4154d12e73bb23fb524eb5ed530bf19 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 27 Feb 2024 13:28:57 -0500 Subject: [PATCH 123/139] TST: extend JRO new test skipping Extending the JRO ISR tags that skip the new tests. --- pysatMadrigal/instruments/jro_isr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/jro_isr.py b/pysatMadrigal/instruments/jro_isr.py index f1c4d7b..996a94b 100644 --- a/pysatMadrigal/instruments/jro_isr.py +++ b/pysatMadrigal/instruments/jro_isr.py @@ -92,7 +92,8 @@ 'oblique_long': dt.datetime(2010, 4, 12)}} # TODO(#99): Remove when compliant with multi-day load tests -_new_tests = {'': {'drifts': False, 'drifts_ave': False}} +_new_tests = {'': {'drifts': False, 'drifts_ave': False, 'oblique_stan': False, + 'oblique_rand': False, 'oblique_long': False}} # Set the clean warnings for testing _clean_warn = {'': {tag: {clean_level: [('logger', 'WARN', From 31567b96c6b42038d12acda083b2f0902823e40f Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 27 Feb 2024 15:55:20 -0500 Subject: [PATCH 124/139] BUG: added flag for kindat 7800 Added a flag to not perform the new tests with the general pandas Madrigal instrument for the 7800 dataset. --- pysatMadrigal/instruments/madrigal_pandas.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pysatMadrigal/instruments/madrigal_pandas.py b/pysatMadrigal/instruments/madrigal_pandas.py index 1a2c283..574f09f 100644 --- a/pysatMadrigal/instruments/madrigal_pandas.py +++ b/pysatMadrigal/instruments/madrigal_pandas.py @@ -104,6 +104,11 @@ _test_dates = {'': {tag: tag_dates[tag] if tag in tag_dates.keys() else tag_dates['7800'] for tag in tags.keys()}} _test_download = {'': {tag: True for tag in tags.keys()}} + +# TODO(#99): Remove when compliant with multi-day load tests +_new_tests = {'': {'7800': False}} + +# Set the clean warnings for testing _clean_warn = {'': {tag: {clean_level: [('logger', 'WARN', "can't support instrument-specific", clean_level)] From 3390871775571d3b3705179303ef0bc0f3b35596 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 28 Feb 2024 15:26:09 -0500 Subject: [PATCH 125/139] BUG: fixed bug in warn dict Fixed a badly named variable in the clean warning definition dict. --- pysatMadrigal/instruments/dmsp_ivm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index e61c2d6..d225ca1 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -113,7 +113,7 @@ 'this level 1 data has no qual', clean_level)] for clean_level in ['clean', 'dusty', 'dirty']} - for tag in inst_ids[inst_id] if inst_id != 'utd'} + for tag in inst_ids[inst_id] if tag != 'utd'} for inst_id in inst_ids.keys()} # ---------------------------------------------------------------------------- From 6a8cf1f363b222674718d58451e366e948aee9e3 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Thu, 29 Feb 2024 19:17:42 -0500 Subject: [PATCH 126/139] BUG: second fix for clean warnings Test failing due to missing key, try empty list. --- pysatMadrigal/instruments/dmsp_ivm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index d225ca1..ea0bc8a 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -112,8 +112,9 @@ _clean_warn = {inst_id: {tag: {clean_level: [('logger', 'WARN', 'this level 1 data has no qual', clean_level)] + if tag != 'utd' else [] for clean_level in ['clean', 'dusty', 'dirty']} - for tag in inst_ids[inst_id] if tag != 'utd'} + for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} # ---------------------------------------------------------------------------- From 606df56cfe7189314212d564bb067ff7dc75f4a2 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Mar 2024 11:07:03 -0500 Subject: [PATCH 127/139] BUG: fixed AE clean warning Fixed the expected AE clean warning. --- pysatMadrigal/instruments/ngdc_ae.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py index 67aa42e..d280222 100644 --- a/pysatMadrigal/instruments/ngdc_ae.py +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -90,11 +90,11 @@ for inst_id in inst_ids.keys()} _test_download = {inst_id: {tag: True for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} -_clean_warn = {inst_id: {tag: {'dusty': [('logger', 'WARN', - "'dusty' and 'clean' are the same", - 'dusty')]} - for tag in inst_ids[inst_id]} - for inst_id in inst_ids.keys()} +_clean_warn = { + inst_id: {tag: {'dusty': [('logger', 'WARN', + "'dusty' and 'clean' levels are the same", + 'dusty')]} + for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} # ---------------------------------------------------------------------------- # Instrument methods From ded3b1b185a50ce11120b5c9e23667bf5b316191 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Mar 2024 11:07:33 -0500 Subject: [PATCH 128/139] STY: removed trailing whitespace Removed trailing whitespace from a line. --- pysatMadrigal/instruments/dmsp_ivm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index ea0bc8a..c0eec7c 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -112,7 +112,7 @@ _clean_warn = {inst_id: {tag: {clean_level: [('logger', 'WARN', 'this level 1 data has no qual', clean_level)] - if tag != 'utd' else [] + if tag != 'utd' else [] for clean_level in ['clean', 'dusty', 'dirty']} for tag in inst_ids[inst_id]} for inst_id in inst_ids.keys()} From 4e3db617904e9741fee58374378a6f124ce75c51 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Mar 2024 15:37:00 -0500 Subject: [PATCH 129/139] BUG: for debugging, removing test flag Temporarily remove test flag for RC tests. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4d77d3f..3c5f76b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,7 @@ Source = "https://github.com/pysat/pysatMadrigal" [tool.coverage.report] [tool.pytest.ini_options] -addopts = "-x --cov=pysatMadrigal" +addopts = "--cov=pysatMadrigal" # HERE: add back -x markers = [ "all_inst", "download", From 33c3b1ea4cbe05fc8829c4a2d13eb9036327f6ff Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Mar 2024 15:37:21 -0500 Subject: [PATCH 130/139] TST: update test flags Add another inst_id/tag combo to the skips. --- pysatMadrigal/instruments/dmsp_ivm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index c0eec7c..8f9bea1 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -106,7 +106,7 @@ 'f18': {tag: dt.datetime(2017, 12, 30) for tag in inst_ids['f18']}} # TODO(#99): Remove when compliant with multi-day load tests -_new_tests = {'f18': {'': False}} +_new_tests = {'f18': {'': False}, 'f14': {'utd': False}} # Set the clean warnings for testing _clean_warn = {inst_id: {tag: {clean_level: [('logger', 'WARN', From ca6d0d023e18c3ffd3d0692e0959ed2305103d31 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 1 Mar 2024 22:49:57 -0500 Subject: [PATCH 131/139] TST: update JRO methods tests Update the test Instrument used in the JRO methods test. --- pysatMadrigal/tests/test_methods_jro.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatMadrigal/tests/test_methods_jro.py b/pysatMadrigal/tests/test_methods_jro.py index 3994649..273ed68 100644 --- a/pysatMadrigal/tests/test_methods_jro.py +++ b/pysatMadrigal/tests/test_methods_jro.py @@ -46,8 +46,8 @@ class TestJROCalcLoc(object): def setup_method(self): """Run before every method to create a clean testing setup.""" - self.inst = pysat.Instrument('pysat', 'testing_xarray', num_samples=100) - self.stime = pysat.instruments.pysat_testing_xarray._test_dates[''][''] + self.inst = pysat.Instrument('pysat', 'ndtesting', num_samples=100) + self.stime = pysat.instruments.pysat_ndtesting._test_dates[''][''] # Set the hard-coded values self.az = 206.0 @@ -66,7 +66,7 @@ def teardown_method(self): return def transform_testing_to_jro(self, azel_type=''): - """Alter `testing_xarray` to mirror the JRO-ISR data.""" + """Alter `ndtesting` to mirror the JRO-ISR data.""" # Load the data self.inst.load(date=self.stime) From 7ba8463832192cf391610fa2ab12ac69dc673bb9 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 4 Mar 2024 06:38:54 -0500 Subject: [PATCH 132/139] TST: re-added quick fail Added the quick fail back into the tests. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3c5f76b..4d77d3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,7 @@ Source = "https://github.com/pysat/pysatMadrigal" [tool.coverage.report] [tool.pytest.ini_options] -addopts = "--cov=pysatMadrigal" # HERE: add back -x +addopts = "-x --cov=pysatMadrigal" markers = [ "all_inst", "download", From 72d6224764bf6efb373df99f29cadf0b22ee4d98 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Mar 2024 17:36:40 -0400 Subject: [PATCH 133/139] MAINT: applied suggestions Applied suggestions from code review. Co-authored-by: Russell Stoneback --- ACKNOWLEDGEMENTS.md | 6 ++++-- docs/supported_instruments.rst | 6 +++--- pysatMadrigal/instruments/ngdc_ae.py | 13 +++---------- pysatMadrigal/instruments/omni2_imf.py | 15 ++++----------- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index 4354ddc..9d12401 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -19,8 +19,10 @@ Programs Disclaimers =========== -Any opinions, findings, and conclusions or recommendations expressed in this -material are those of the author(s) and do not necessarily reflect the views +Any opinions or actions taken by the listed funding institutions are those of +the institutions and do not necessarily reflect the views of the pysat development +team or individual authors. Any opinions, findings, and conclusions or recommendations +expressed in this material are those of the author(s) and do not necessarily reflect the views of the funding agencies. Support and Thanks diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 3708be5..75f52a5 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -14,7 +14,7 @@ Meter (IVM) Madrigal data. DMSP_SSJ -------- -Supports the Defense Meteorological Satelitte Program (DMSP) Special Sensor J +Supports the Defense Meteorological Satellite Program (DMSP) Special Sensor J (SSJ) Madrigal data. @@ -88,7 +88,7 @@ support (e.g., cleaning methods, experiment acknowledgements, and references). NGDC_AE ------- -An instrument for the Geophysical indices from NGDC, which include AE AL, AU, +An instrument for the Geophysical indices from NGDC, which include AE, AL, AU, and AO. The :py:attr:`name` is AE due to the Madrigal naming conventions. The data set spans the years of 1978 through 1987, will all data saved in a single file. Because of this, you only need to download the data once and any desired @@ -101,7 +101,7 @@ time period may be loaded. OMNI2_IMF --------- -An instrument for the interplanetary magnetic field (IMF) data from Omni 2. The +An instrument for the interplanetary magnetic field (IMF) data from OMNI 2. The data starts in 1963 and the entire data set is contained in a single file. The file is occasionally updated, and so obtaining the most recent data means that all historic data must also be downloaded (or re-downloaded). OMNI data may diff --git a/pysatMadrigal/instruments/ngdc_ae.py b/pysatMadrigal/instruments/ngdc_ae.py index d280222..e711fe5 100644 --- a/pysatMadrigal/instruments/ngdc_ae.py +++ b/pysatMadrigal/instruments/ngdc_ae.py @@ -101,14 +101,7 @@ def init(self): - """Initialize the Instrument object in support of Madrigal access. - - Parameters - ---------- - kindat : str - Madrigal instrument experiment code(s). (default='') - - """ + """Initialize the Instrument object in support of Madrigal access.""" # Set the standard pysat attributes self.acknowledgements = general.cedar_rules() self.references = ''.join(['Davis, T. Neil and Masahisa Sugiura. “Auroral', @@ -126,8 +119,8 @@ def clean(self): Note ---- - Supports 'clean', 'dusty', 'dirty' in the sense that it prints - a message noting there is no cleaning. + Supports 'clean', 'dusty', 'dirty' in the sense that all + levels use the flag to clean data the same way. 'None' is also supported as it signifies no cleaning. Routine is called by pysat, and not by the end user directly. diff --git a/pysatMadrigal/instruments/omni2_imf.py b/pysatMadrigal/instruments/omni2_imf.py index cbadac5..5df6e66 100644 --- a/pysatMadrigal/instruments/omni2_imf.py +++ b/pysatMadrigal/instruments/omni2_imf.py @@ -7,7 +7,7 @@ # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. # ---------------------------------------------------------------------------- -"""Supports access to OMNI 2 IMF data archieved at Madrigal. +"""Supports access to OMNI 2 IMF data archived at Madrigal. Properties ---------- @@ -109,14 +109,7 @@ def init(self): - """Initialize the Instrument object in support of Madrigal access. - - Parameters - ---------- - kindat : str - Madrigal instrument experiment code(s). (default='') - - """ + """Initialize the Instrument object in support of Madrigal access.""" # Set the standard pysat attributes self.acknowledgements = ''.join([general.cedar_rules(), '\nFor full ', 'acknowledgement info, please see: ', @@ -225,9 +218,9 @@ def load(fnames, tag='', inst_id=''): if data.index[-1] > madrigal_end: pysat.logger.critical(''.join(['There is data beyond ', '{:}'.format(madrigal_end), ' in the ', - 'Omni2 IMF file, please notify the ', + 'OMNI2 IMF file, please notify the ', 'pysatMadrigal developers so that they ', - 'can update this Instrument'])) + 'can update this Instrument.'])) # Select the data for the desired time period data = data[fstart:fstop] From 6c2fc0d805e013505dd4ee37ae66bfc33055c374 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Mar 2024 17:45:27 -0400 Subject: [PATCH 134/139] DOC: improve DMSP SSJ init Improve the user output for the DMSP SSJ instrument. --- pysatMadrigal/instruments/dmsp_ssj.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pysatMadrigal/instruments/dmsp_ssj.py b/pysatMadrigal/instruments/dmsp_ssj.py index fcdc313..52287b7 100644 --- a/pysatMadrigal/instruments/dmsp_ssj.py +++ b/pysatMadrigal/instruments/dmsp_ssj.py @@ -108,7 +108,10 @@ def init(self): self.acknowledgements = ''.join([ general.cedar_rules(), '\nThe Air Force Research Laboratory Auroral ', 'Boundary Index (ABI) was provided by the United States Air Force ', - 'Research Laboratory, Kirtland Air Force Base, New Mexico.']) + 'Research Laboratory, Kirtland Air Force Base, New Mexico.\n', + 'Please send a copy of all publications that use the ABI to Dr. ', + 'Gordon Wilson at: gordon.wilson@kirtland.af.mil\n', + 'Dr. Wilson considers this data set deprecated.']) logger.info(self.acknowledgements) self.references = dmsp.references(self.name) From ea2ff926ef754b9ebaf61680fd7f6d4d44055a09 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 11 Mar 2024 17:54:40 -0400 Subject: [PATCH 135/139] MAINT: applied code review Found more code review comments and made changes. Co-authored-by: Russell Stoneback --- pysatMadrigal/instruments/madrigal_dst.py | 2 +- pysatMadrigal/instruments/madrigal_geoind.py | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/pysatMadrigal/instruments/madrigal_dst.py b/pysatMadrigal/instruments/madrigal_dst.py index 7324c59..1df06e1 100644 --- a/pysatMadrigal/instruments/madrigal_dst.py +++ b/pysatMadrigal/instruments/madrigal_dst.py @@ -7,7 +7,7 @@ # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. # ---------------------------------------------------------------------------- -"""Supports access to Dst data archieved at Madrigal. +"""Supports access to Dst data archived at Madrigal. Properties ---------- diff --git a/pysatMadrigal/instruments/madrigal_geoind.py b/pysatMadrigal/instruments/madrigal_geoind.py index be288b0..97e888e 100644 --- a/pysatMadrigal/instruments/madrigal_geoind.py +++ b/pysatMadrigal/instruments/madrigal_geoind.py @@ -7,7 +7,7 @@ # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. # ---------------------------------------------------------------------------- -"""Supports access to F10.7, Kp, and Ap data archieved at Madrigal. +"""Supports access to F10.7, Kp, and Ap data archived at Madrigal. Properties ---------- @@ -98,14 +98,7 @@ def init(self): - """Initialize the Instrument object in support of Madrigal access. - - Parameters - ---------- - kindat : str - Madrigal instrument experiment code(s). (default='') - - """ + """Initialize the Instrument object in support of Madrigal access.""" # Set the standard pysat attributes self.acknowledgements = general.cedar_rules() self.references = "\n".join( @@ -179,7 +172,7 @@ def clean(self): def load(fnames, tag='', inst_id=''): - """Load the Madrigal Dst data. + """Load the Madrigal geoindex data. Parameters ----------- @@ -195,7 +188,7 @@ def load(fnames, tag='', inst_id=''): Returns -------- data : pds.DataFrame - Object containing IMF data + Object containing geoindex data meta : pysat.Meta Object containing metadata such as column names and units From a2d7a3ff6c6e8c9dacc8a2a29b13a8251f3f4be3 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 12 Mar 2024 11:18:08 -0400 Subject: [PATCH 136/139] DOC: add TODO statements Added TODO statements for new issues. Co-authored-by: Russell Stoneback --- pysatMadrigal/tests/test_methods_general.py | 2 ++ pysatMadrigal/tests/test_methods_gnss.py | 1 + 2 files changed, 3 insertions(+) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index 500346a..2710a5e 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -60,6 +60,7 @@ def test_sort_file_format(self, caplog): ['hdf5', 'netCDF4', 'simple']) # Evaluate the logger warning + # TODO(#101) Use pysat warnings test assert len(caplog.records) == 1, "unexpected number of warnings" assert caplog.records[0].levelname == "WARNING" assert caplog.records[0].message.find( @@ -357,6 +358,7 @@ def test_get_remote_filenames_bad_date_array(self): "unknown date_array supplied", input_kwargs=self.kwargs) return + # (TODO #102) Expand testing of pandas to xarray method below def test_convert_pandas_to_xarray_bad_data_vars(self): """Test raises ValueError for unexpected date_array input.""" self.kwargs = [{('time', ): ['bad_var']}, pds.DataFrame([0]), diff --git a/pysatMadrigal/tests/test_methods_gnss.py b/pysatMadrigal/tests/test_methods_gnss.py index 6338974..432d732 100644 --- a/pysatMadrigal/tests/test_methods_gnss.py +++ b/pysatMadrigal/tests/test_methods_gnss.py @@ -62,6 +62,7 @@ def test_bad_file_type_warning(self, caplog): gnss.load_los(self.bad_fnames, "site", "zzon") # Test the logger warning + # TODO(#101) Use pysat eval warnings assert len(caplog.records) == 2, "unexpected number of warnings" for record in caplog.records: From d09a808e7a1fe509a4a5b42ae77af2491f57c03c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 12 Mar 2024 11:21:16 -0400 Subject: [PATCH 137/139] BUG: fix TODO formatting Use the correct TODO formatting. --- pysatMadrigal/tests/test_methods_general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py index 2710a5e..dffee57 100644 --- a/pysatMadrigal/tests/test_methods_general.py +++ b/pysatMadrigal/tests/test_methods_general.py @@ -358,7 +358,7 @@ def test_get_remote_filenames_bad_date_array(self): "unknown date_array supplied", input_kwargs=self.kwargs) return - # (TODO #102) Expand testing of pandas to xarray method below + # TODO(#102) Expand testing of pandas to xarray method below def test_convert_pandas_to_xarray_bad_data_vars(self): """Test raises ValueError for unexpected date_array input.""" self.kwargs = [{('time', ): ['bad_var']}, pds.DataFrame([0]), From 5af4b311fbf4ba9339d33adf5498f73bd0f7438b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 13 Mar 2024 16:07:17 -0400 Subject: [PATCH 138/139] REV: python 3.12 failing on mac Remove support for 3.12 from this release. --- .github/workflows/main.yml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 36668b6..5765314 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11"] test_config: ["latest"] include: # NEP29 compliance settings diff --git a/pyproject.toml b/pyproject.toml index 4d77d3f..956cd86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows" From 3d512a2899954c1b5c85d9627a640accb2e23a98 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 15 Mar 2024 11:23:35 -0400 Subject: [PATCH 139/139] MAINT: Update CHANGELOG.md Support for python 3.12 was removed until a documented issue can be solved. --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed96765..799fb67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,6 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Maintenance * Add manual GitHub Actions tests for pysatMadrigal RC * Update GitHub Actions workflows for newer versions of pip, updated actions - * Added support for Python 3.12 * Added clean warning test attributes to all Instruments * Updated documentation to comply with current Ecosystem guidelines * Replaced setup.py with pyproject.toml