diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index df7debb8..818115d4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,12 +11,12 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest] - python-version: ["3.8", "3.9"] + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.9", "3.10"] numpy_ver: [latest] include: - - python-version: "3.7" - numpy_ver: "1.17" + - python-version: "3.8" + numpy_ver: "1.19" os: ubuntu-latest name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} @@ -42,10 +42,10 @@ jobs: - name: Set up pysat run: | mkdir pysatData - python -c "import pysat; pysat.params['data_dirs'] = './pysatData'" + python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" - name: Test PEP8 compliance - run: flake8 . --count --select=E,F,W --show-source --statistics + run: flake8 . --count --select=D,E,F,H,W --show-source --statistics - name: Evaluate complexity run: flake8 . --count --exit-zero --max-complexity=10 --statistics diff --git a/.zenodo.json b/.zenodo.json index 75db3780..88e91af8 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,4 +1,17 @@ { + "keywords": [ + "pysat", + "ionosphere", + "atmosphere", + "thermosphere", + "magnetosphere", + "heliosphere", + "observations", + "space-weather", + "solar-index", + "geomagnetic-index", + "forecasts" + ], "creators": [ { "affiliation": "U.S. Naval Research Laboratory", @@ -11,10 +24,10 @@ "orcid": "0000-0001-8321-6074" }, { - "affiliation": "The University of Texas at Dallas", + "affiliation": "Stoneris LLC", "name": "Stoneback, Russell", "orcid": "0000-0001-7216-4336" - }, + }, { "affiliation": "Predictive Science", "name": "Pembroke, Asher" @@ -22,6 +35,11 @@ { "name": "Spence, Carey", "orcid": "0000-0001-8340-5625" + }, + { + "affiliation": "Goddard Space Flight Center", + "name": "Smith, Jonathon M.", + "orcid": "0000-0002-8191-4765" } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ad6364f..f85a100a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,24 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +[0.0.5] - 2022-06-10 +-------------------- +* Updated the docstrings to conform to pysat standards +* Added docstring tests to Flake8 portion of CI testing +* Fixed bug in `combine_kp` that occurs if no times are provided +* Improved unit test style and expanded unit test coverage +* Updated package organization documentation +* Added a function to normalize ACE SWEPAM variables as described in the OMNI + processing guide +* Deprecated `load_csv_data` method, which was moved to pysat +* Added the LASP predicted Dst to the Dst Instrument +* Updated pandas usage to remove existing deprecation warnings +* Updated `pysat.Instrument.load` calls to remove `use_header` deprecation + warning + [0.0.4] - 2021-05-19 -------------------- +* New Logo * Implements GitHub Actions for primary CI testing * Updated tested python versions * Removed non-document testing from Travis-CI and updated installation method diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eeec5863..7c82d968 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,6 +5,11 @@ Bug reports, feature suggestions and other contributions are greatly appreciated! pysat and pysatSpaceWeather are community-driven projects and welcomes both feedback and contributions. +Come join us on Slack! An invitation to the pysat workspace is available +in the 'About' section of the +[pysat GitHub Repository.](https://github.com/pysat/pysat) +Development meetings are generally held fortnightly. + Short version ------------- @@ -40,17 +45,26 @@ Development To set up `pysatSpaceWeather` for local development: -1. `Fork pysatSpaceWeather on GitHub `_. -2. Clone your fork locally:: +1. [Fork pysatSpaceWeather on GitHub](https://github.com/pysat/pysatSpaceWeather/fork>). + +2. Clone your fork locally: + ``` git clone git@github.com:your_name_here/pysatSpaceWeather.git + ``` -3. Create a branch for local development:: +3. Create a branch for local development: + ``` git checkout -b name-of-your-bugfix-or-feature + ``` Now you can make your changes locally. Tests for new instruments are - performed automatically. Tests for custom functions should be added to the + performed automatically. See discussion + [here](https://pysat.readthedocs.io/en/main/new_instrument.html#testing-support) + for more information on triggering these standard tests. + + Tests for custom functions should be added to the appropriately named file in ``pysatSpaceWeather/tests``. For example, space weather methods should be named ``pysatSpaceWeather/tests/test_methods_sw.py``. If no test file exists, @@ -59,20 +73,28 @@ To set up `pysatSpaceWeather` for local development: must begin with ``Test``, and test methods must also begin with ``test``. 4. When you're done making changes, run all the checks to ensure that nothing - is broken on your local system:: + is broken on your local system: + ``` pytest -vs pysatSpaceWeather + ``` 5. Update/add documentation (in ``docs``). Even if you don't think it's relevant, check to see if any existing examples have changed. 6. Add your name to the .zenodo.json file as an author -7. Commit your changes and push your branch to GitHub:: +7. Commit your changes and push your branch to GitHub: + ``` git add . - git commit -m "Brief description of your changes" + git commit -m "CODE: Brief description of your changes" git push origin name-of-your-bugfix-or-feature + ``` + + Where CODE is a standard shorthand for the type of change (eg, BUG or DOC). + `pysat` follows the [numpy development workflow](https://numpy.org/doc/stable/dev/development_workflow.html), + see the discussion there for a full list of this shorthand notation. 8. Submit a pull request through the GitHub website. Pull requests should be made to the ``develop`` branch. @@ -87,8 +109,8 @@ For merging, you should: 1. Include an example for use 2. Add a note to ``CHANGELOG.md`` about the changes -3. Ensure that all checks passed (current checks include Travis-CI - and Coveralls) [1]_ +3. Ensure that all checks passed (current checks include GitHub Actions, + Coveralls, and ReadTheDocs) [1]_ .. [1] If you don't have all the necessary Python versions available locally or have trouble building all the testing environments, you can rely on @@ -118,6 +140,8 @@ These include: * `import numpy as np` * `import pandas as pds` * `import xarray as xr` +* When incrementing a timestamp, use `dt.timedelta` instead of `pds.DateOffset` + when possible to reduce program runtime * All classes should have `__repr__` and `__str__` functions * Docstrings use `Note` instead of `Notes` * Try to avoid creating a try/except statement where except passes diff --git a/docs/conf.py b/docs/conf.py index faec5328..e23094c6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,6 +16,8 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +"""Configuration script for Sphinx documentation.""" + import json import os import sys diff --git a/docs/figures/pysatSpaceWeather.png b/docs/figures/pysatSpaceWeather.png index 4777efb8..de2b3d38 100644 Binary files a/docs/figures/pysatSpaceWeather.png and b/docs/figures/pysatSpaceWeather.png differ diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 3507ff8b..33aa8aee 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -75,7 +75,10 @@ The Disturbance Storm Time (Dst) Index is a measure of magnetic activity associated with the ring current. The National Geophysical Data Center (NGDC) maintains the `current database `_ from which -the Dst is downloaded. You can learn more about the Dst Index at the +the historic Dst is downloaded. +`LASP `_ +performs the calculates and provides the predicted Dst for the last 96 hours. +You can learn more about the Dst Index at the `WDC Kyoto Observatory page `_. diff --git a/pysatSpaceWeather/__init__.py b/pysatSpaceWeather/__init__.py index f980edcd..b518840d 100644 --- a/pysatSpaceWeather/__init__.py +++ b/pysatSpaceWeather/__init__.py @@ -1,3 +1,5 @@ +"""Initialization file for pysatSpaceWeather module.""" + import os from pysatSpaceWeather import instruments # noqa F401 diff --git a/pysatSpaceWeather/instruments/ace_epam.py b/pysatSpaceWeather/instruments/ace_epam.py index bb996634..913c19ac 100644 --- a/pysatSpaceWeather/instruments/ace_epam.py +++ b/pysatSpaceWeather/instruments/ace_epam.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Supports ACE Electron, Proton, and Alpha Monitor data +"""Supports ACE Electron, Proton, and Alpha Monitor data. Properties ---------- @@ -17,6 +17,8 @@ ---- This is not the ACE scientific data set, which will be available at pysatNASA +Examples +-------- The real-time data is stored by generation date, where each file contains the data for the current day. If you leave download dates empty, though, it will grab today's file three times and assign dates from yesterday, today, and @@ -42,6 +44,7 @@ import functools import numpy as np +from pysat.instruments.methods.general import load_csv_data from pysat import logger from pysatSpaceWeather.instruments.methods import ace as mm_ace @@ -74,11 +77,7 @@ def init(self): - """Initializes the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ + """Initialize the Instrument object with instrument specific values.""" # Set the appropriate acknowledgements and references self.acknowledgements = mm_ace.acknowledgements() @@ -90,7 +89,7 @@ def init(self): def clean(self): - """Routine to clean real-time ACE data using the status flag + """Clean the real-time ACE data using the status flag. Note ---- @@ -132,17 +131,17 @@ def clean(self): list_files = functools.partial(mm_ace.list_files, name=name) -def load(fnames, tag=None, inst_id=None): - """Load the ACE space weather prediction data +def load(fnames, tag='', inst_id=''): + """Load the ACE space weather prediction data. Parameters ---------- fnames : array-like - Series, list, or array of filenames - tag : str or NoneType - tag or None (default=None) - inst_id : str or NoneType - ACE instrument or None (default=None) + Series, list, or array of filenames. + tag : str + Instrument tag (default=''). + inst_id : str + ACE instrument ID (default=''). Returns ------- @@ -151,10 +150,9 @@ def load(fnames, tag=None, inst_id=None): meta : pysat.Meta Object containing metadata such as column names and units - Raises - ------ - ValueError - When unknown inst_id is supplied. + See Also + -------- + pysat.instruments.methods.general.load_csv_data Note ---- @@ -163,8 +161,8 @@ def load(fnames, tag=None, inst_id=None): """ # Save each file to the output DataFrame - data = mm_ace.load_csv_data(fnames, read_csv_kwargs={'index_col': 0, - 'parse_dates': True}) + data = load_csv_data(fnames, read_csv_kwargs={'index_col': 0, + 'parse_dates': True}) # Assign the meta data meta, status_desc = mm_ace.common_metadata() diff --git a/pysatSpaceWeather/instruments/ace_mag.py b/pysatSpaceWeather/instruments/ace_mag.py index 0300a36c..b8cc1c8c 100644 --- a/pysatSpaceWeather/instruments/ace_mag.py +++ b/pysatSpaceWeather/instruments/ace_mag.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Supports ACE Magnetometer data +"""Supports ACE Magnetometer data. Properties ---------- @@ -17,6 +17,8 @@ ---- This is not the ACE scientific data set, which will be available at pysatNASA +Examples +-------- The real-time data is stored by generation date, where each file contains the data for the current day. If you leave download dates empty, though, it will grab today's file three times and assign dates from yesterday, today, and @@ -42,6 +44,7 @@ import functools import numpy as np +from pysat.instruments.methods.general import load_csv_data from pysat import logger from pysatSpaceWeather.instruments.methods import ace as mm_ace @@ -74,11 +77,7 @@ def init(self): - """Initializes the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ + """Initialize the Instrument object with instrument specific values.""" # Set the appropraite acknowledgements and references self.acknowledgements = mm_ace.acknowledgements() @@ -90,7 +89,7 @@ def init(self): def clean(self): - """Routine to clean real-time ACE data using the status flag + """Clean real-time ACE data using the status flag. Note ---- @@ -115,17 +114,17 @@ def clean(self): list_files = functools.partial(mm_ace.list_files, name=name) -def load(fnames, tag=None, inst_id=None): - """Load the ACE space weather prediction data +def load(fnames, tag='', inst_id=''): + """Load the ACE space weather prediction data. Parameters ---------- fnames : array-like Series, list, or array of filenames - tag : str or NoneType - tag or None (default=None) - inst_id : str or NoneType - ACE instrument or None (default=None) + tag : str + Instrument tag, not used. (default='') + inst_id : str + ACE instrument ID, not used. (default='') Returns ------- @@ -134,10 +133,9 @@ def load(fnames, tag=None, inst_id=None): meta : pysat.Meta Object containing metadata such as column names and units - Raises - ------ - ValueError - When unknown inst_id is supplied. + See Also + -------- + pysat.instruments.methods.general.load_csv_data Note ---- @@ -146,8 +144,8 @@ def load(fnames, tag=None, inst_id=None): """ # Save each file to the output DataFrame - data = mm_ace.load_csv_data(fnames, read_csv_kwargs={'index_col': 0, - 'parse_dates': True}) + data = load_csv_data(fnames, read_csv_kwargs={'index_col': 0, + 'parse_dates': True}) # Assign the meta data meta, status_desc = mm_ace.common_metadata() diff --git a/pysatSpaceWeather/instruments/ace_sis.py b/pysatSpaceWeather/instruments/ace_sis.py index 15a3d32a..3337a8bf 100644 --- a/pysatSpaceWeather/instruments/ace_sis.py +++ b/pysatSpaceWeather/instruments/ace_sis.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Supports ACE Solar Isotope Spectrometer data +"""Supports ACE Solar Isotope Spectrometer data. Properties ---------- @@ -17,6 +17,8 @@ ---- This is not the ACE scientific data set, which will be available at pysatNASA +Examples +-------- The real-time data is stored by generation date, where each file contains the data for the current day. If you leave download dates empty, though, it will grab today's file three times and assign dates from yesterday, today, and @@ -42,6 +44,7 @@ import functools import numpy as np +from pysat.instruments.methods.general import load_csv_data from pysat import logger from pysatSpaceWeather.instruments.methods import ace as mm_ace @@ -74,11 +77,7 @@ def init(self): - """Initializes the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ + """Initialize the Instrument object with instrument specific values.""" # Set the appropraite acknowledgements and references self.acknowledgements = mm_ace.acknowledgements() @@ -90,7 +89,7 @@ def init(self): def clean(self): - """Routine to clean real-time ACE data using the status flag + """Clean real-time ACE data using the status flag. Note ---- @@ -124,17 +123,17 @@ def clean(self): list_files = functools.partial(mm_ace.list_files, name=name) -def load(fnames, tag=None, inst_id=None): - """Load the ACE space weather prediction data +def load(fnames, tag='', inst_id=''): + """Load the ACE space weather prediction data. Parameters ---------- fnames : array-like - Series, list, or array of filenames - tag : str or NoneType - tag or None (default=None) - inst_id : str or NoneType - ACE instrument or None (default=None) + Series, list, or array of filenames. + tag : str + Instrument tag, not used. (default='') + inst_id : str + ACE instrument ID, not used. (default='') Returns ------- @@ -143,10 +142,9 @@ def load(fnames, tag=None, inst_id=None): meta : pysat.Meta Object containing metadata such as column names and units - Raises - ------ - ValueError - When unknown inst_id is supplied. + See Also + -------- + pysat.instruments.methods.general.load_csv_data Note ---- @@ -155,8 +153,8 @@ def load(fnames, tag=None, inst_id=None): """ # Save each file to the output DataFrame - data = mm_ace.load_csv_data(fnames, read_csv_kwargs={'index_col': 0, - 'parse_dates': True}) + data = load_csv_data(fnames, read_csv_kwargs={'index_col': 0, + 'parse_dates': True}) # Assign the meta data meta, status_desc = mm_ace.common_metadata() diff --git a/pysatSpaceWeather/instruments/ace_swepam.py b/pysatSpaceWeather/instruments/ace_swepam.py index 805bdbac..02cde92f 100644 --- a/pysatSpaceWeather/instruments/ace_swepam.py +++ b/pysatSpaceWeather/instruments/ace_swepam.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Supports ACE Solar Wind Electron Proton Alpha Monitor data +"""Supports ACE Solar Wind Electron Proton Alpha Monitor data. Properties ---------- @@ -17,6 +17,8 @@ ---- This is not the ACE scientific data set, which will be available at pysatNASA +Examples +-------- The real-time data is stored by generation date, where each file contains the data for the current day. If you leave download dates empty, though, it will grab today's file three times and assign dates from yesterday, today, and @@ -41,6 +43,7 @@ import functools import numpy as np +from pysat.instruments.methods.general import load_csv_data from pysat import logger from pysatSpaceWeather.instruments.methods import ace as mm_ace @@ -73,11 +76,7 @@ def init(self): - """Initializes the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ + """Initialize the Instrument object with instrument specific values.""" # Set the appropraite acknowledgements and references self.acknowledgements = mm_ace.acknowledgements() @@ -89,7 +88,7 @@ def init(self): def clean(self): - """Routine to clean real-time ACE data using the status flag + """Clean real-time ACE data using the status flag. Note ---- @@ -114,17 +113,17 @@ def clean(self): list_files = functools.partial(mm_ace.list_files, name=name) -def load(fnames, tag=None, inst_id=None): - """Load the ACE space weather prediction data +def load(fnames, tag='', inst_id=''): + """Load the ACE space weather prediction data. Parameters ---------- fnames : array-like - Series, list, or array of filenames - tag : str or NoneType - tag or None (default=None) - inst_id : str or NoneType - ACE instrument or None (default=None) + Series, list, or array of filenames. + tag : str + Instrument tag, not used. (default='') + inst_id : str + ACE instrument ID, not used. (default='') Returns ------- @@ -133,10 +132,9 @@ def load(fnames, tag=None, inst_id=None): meta : pysat.Meta Object containing metadata such as column names and units - Raises - ------ - ValueError - When unknown inst_id is supplied. + See Also + -------- + pysat.instruments.methods.general.load_csv_data Note ---- @@ -145,8 +143,8 @@ def load(fnames, tag=None, inst_id=None): """ # Save each file to the output DataFrame - data = mm_ace.load_csv_data(fnames, read_csv_kwargs={'index_col': 0, - 'parse_dates': True}) + data = load_csv_data(fnames, read_csv_kwargs={'index_col': 0, + 'parse_dates': True}) # Assign the meta data meta, status_desc = mm_ace.common_metadata() diff --git a/pysatSpaceWeather/instruments/methods/ace.py b/pysatSpaceWeather/instruments/methods/ace.py index 4711c74d..a71a9671 100644 --- a/pysatSpaceWeather/instruments/methods/ace.py +++ b/pysatSpaceWeather/instruments/methods/ace.py @@ -1,12 +1,17 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*-. -"""Provides general routines for the ACE space weather instruments -""" +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Provides general routines for the ACE space weather instruments.""" import datetime as dt import numpy as np import os import pandas as pds import requests +import warnings import pysat @@ -14,11 +19,11 @@ def acknowledgements(): - """Returns acknowledgements for the specified ACE instrument + """Define the acknowledgements for the specified ACE instrument. Returns ------- - ackn : string + ackn : str Acknowledgements for the ACE instrument """ @@ -33,16 +38,16 @@ def acknowledgements(): def references(name): - """Returns references for the specified ACE instrument + """Define the references for the specified ACE instrument. Parameters ---------- - name : string + name : str Instrument name of the ACE instrument Returns ------- - ref : string + ref : str Reference for the ACE instrument paper """ @@ -77,7 +82,7 @@ def references(name): def clean(inst): - """Common aspects of the ACE space weather data cleaning + """Clean the common aspects of the ACE space weather data. Parameters ---------- @@ -113,26 +118,26 @@ def clean(inst): return max_status -def list_files(name='', tag='', inst_id='', data_path='', format_str=None): - """Return a Pandas Series of every file for ACE data +def list_files(name, tag='', inst_id='', data_path='', format_str=None): + """List the local ACE data files. Parameters ---------- name : str - ACE Instrument name. (default='') + ACE Instrument name. tag : str - Denotes type of file to load. (default='') + ACE Instrument tag. (default='') inst_id : str Specifies the ACE instrument ID. (default='') data_path : str Path to data directory. (default='') - format_str : string or NoneType + format_str : str or NoneType User specified file format. If None is specified, the default formats associated with the supplied tags are used. (default=None) Returns ------- - pysat.Files.from_os : pysat.utils.files.Files + files : pysat.Files A class containing the verified available files Note @@ -149,20 +154,19 @@ def list_files(name='', tag='', inst_id='', data_path='', format_str=None): return files -def download(date_array, name='', tag='', inst_id='', data_path='', now=None): - """Routine to download ACE Space Weather data +def download(date_array, name, tag='', inst_id='', data_path='', now=None): + """Download the requested ACE Space Weather data. Parameters ---------- date_array : array-like Array of datetime values name : str - ACE Instrument name. (default='') + ACE Instrument name. tag : str - Denotes type of file to load. Accepted types are 'realtime' and - 'historic'. (default='') + ACE Instrument tag. (default='') inst_id : str - Specifies the ACE instrument ID. (default='') + ACE instrument ID. (default='') data_path : str Path to data directory. (default='') now : dt.datetime or NoneType @@ -179,6 +183,7 @@ def download(date_array, name='', tag='', inst_id='', data_path='', now=None): - File requested not available on server """ + # Ensure now is up-to-date, if desired if now is None: now = dt.datetime.utcnow() @@ -258,7 +263,7 @@ def download(date_array, name='', tag='', inst_id='', data_path='', now=None): def common_metadata(): - """Provides common metadata information for all ACE instruments + """Define the common metadata information for all ACE instruments. Returns ------- @@ -296,7 +301,11 @@ def common_metadata(): def load_csv_data(fnames, read_csv_kwargs=None): - """Load CSV data from a list of files into a single DataFrame + """Load CSV data from a list of files into a single DataFrame. + + .. deprecated:: 0.0.5 + `load_csv_data` will be removed in pysatSpaceWeather 0.0.6+, as it has + been moved to `pysat.instruments.methods.general` as of pysat 3.0.1. Parameters ---------- @@ -312,9 +321,15 @@ def load_csv_data(fnames, read_csv_kwargs=None): See Also -------- - pds.read_csv + pds.read_csv, pysat.instruments.methods.general.load_csv_data """ + + warnings.warn("".join(["Moved to pysat.instruments.methods.general.", + "load_csv_data in pysat version 3.0.1. This method ", + "will be removed at the 0.0.6+ release."]), + DeprecationWarning) + # Ensure the filename input is array-like fnames = np.asarray(fnames) if fnames.shape == (): @@ -331,3 +346,75 @@ def load_csv_data(fnames, read_csv_kwargs=None): data = pds.DataFrame() if len(fdata) == 0 else pds.concat(fdata, axis=0) return data + + +def ace_swepam_hourly_omni_norm(as_inst, speed_key='sw_bulk_speed', + dens_key='sw_proton_dens', + temp_key='sw_ion_temp'): + """Normalize ACE SWEPAM variables as described in the OMNI processing _[1]. + + Parameters + ---------- + as_inst : pysat.Instrument + pysat Instrument object with ACE SWEPAM data. + speed_key : str + Data key for bulk solar wind speed data in km/s + (default='sw_bulk_speed') + dens_key : str + Data key for solar wind proton density data in P/cm^3 + (default='sw_proton_dens') + temp_key : str + Data key for solar wind ion temperature data in K + (default='sw_ion_temp') + + References + ---------- + [1] https://omniweb.gsfc.nasa.gov/html/omni_min_data.html + + """ + + # Check the input to make sure all the necessary data variables are present + for var in [speed_key, dens_key, temp_key]: + if var not in as_inst.variables: + raise ValueError('instrument missing variable: {:}'.format(var)) + + # Let yt be the fractional years since 1998.0 + yt = np.array([pysat.utils.time.datetime_to_dec_year(itime) - 1998.0 + for itime in as_inst.index]) + + # Get the masks for the different velocity limits + ilow = as_inst[speed_key] < 395 + imid = (as_inst[speed_key] >= 395) & (as_inst[speed_key] <= 405) + ihigh = as_inst[speed_key] > 405 + + # Calculate the normalized plasma density + norm_n = np.array(as_inst[dens_key]) + norm_n[ilow] *= (0.925 + 0.0039 * yt[ilow]) + norm_n[imid] *= (74.02 - 0.164 * as_inst[speed_key][imid] + + 0.0171 * as_inst[speed_key][imid] * yt[imid] + - 6.72 * yt[imid]) / 10.0 + norm_n[ihigh] *= (0.761 + 0.0210 * yt[ihigh]) + + # Normalize the temperature + norm_t = np.power(10.0, -0.069 + 1.024 * np.log10(as_inst[temp_key])) + + # Update the instrument data + as_inst['sw_proton_dens_norm'] = pds.Series(norm_n, index=as_inst.index) + as_inst['sw_ion_temp_norm'] = pds.Series(norm_t, index=as_inst.index) + + # Add meta data + for dkey in [dens_key, temp_key]: + nkey = '{:s}_norm'.format(dkey) + meta_dict = {} + + for mkey in as_inst.meta[dkey].keys(): + if mkey == as_inst.meta.labels.notes: + meta_dict[mkey] = ''.join([ + 'Normalized for hourly OMNI as described in ', + 'https://omniweb.gsfc.nasa.gov/html/omni_min_data.html']) + elif mkey != "children": + meta_dict[mkey] = as_inst.meta[dkey, mkey] + + as_inst.meta[nkey] = meta_dict + + return diff --git a/pysatSpaceWeather/instruments/methods/dst.py b/pysatSpaceWeather/instruments/methods/dst.py index cf1c2a9c..0eee65ae 100644 --- a/pysatSpaceWeather/instruments/methods/dst.py +++ b/pysatSpaceWeather/instruments/methods/dst.py @@ -1,42 +1,53 @@ # -*- coding: utf-8 -*-. -"""Provides default routines for Dst +"""Provides default routines for Dst.""" -""" - -def acknowledgements(name, tag): - """Returns acknowledgements for space weather dataset +def acknowledgements(tag): + """Define the acknowledgements for the Dst data. Parameters ---------- - name : string - Name of space weather index, eg, dst, f107, kp - tag : string + tag : str Tag of the space weather index + Returns + ------- + ackn : str + Acknowledgements string associated with the appropriate Dst tag. + """ - ackn = {'dst': - {'noaa': 'Dst is maintained at NCEI (formerly NGDC) at NOAA'}} + ackn = {'noaa': 'Dst is maintained at NCEI (formerly NGDC) at NOAA', + 'lasp': ''.join(['Preliminary Dst predictions are provided by ', + 'LASP, contact Xinlin Li for more details ', + ''])} - return ackn[name][tag] + return ackn[tag] -def references(name, tag): - """Returns references for space weather dataset +def references(tag): + """Define the references for the Dst data. Parameters ---------- - name : string - Name of space weather index, eg, dst, f107, kp tag : string Tag of the space weather index - """ + Returns + ------- + refs : str + Reference string associated with the appropriate Dst tag. - refs = {'dst': {'noaa': ''.join([ - 'See referenece list and publication at: Sugiura M. and T. Kamei, ' - 'http://wdc.kugi.kyoto-u.ac.jp/dstdir/dst2/onDstindex.html, ', - 'last updated June 1991, accessed Dec 2020'])}} + """ - return refs[name][tag] + refs = {'noaa': ''.join(['See referenece list and publication at: ', + 'Sugiura M. and T. Kamei, http://', + 'wdc.kugi.kyoto-u.ac.jp/dstdir/dst2/', + 'onDstindex.html, last updated June 1991, ', + 'accessed Dec 2020']), + 'lasp': ''.join(['A New Model for the Prediction of Dst on the ', + 'Basis of the Solar Wind [Temerin and Li, 2002] ', + 'and Dst model for 1995-2002 [Temerin and Li, ', + '2006]'])} + + return refs[tag] diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index 4476e1a5..206f774b 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -1,47 +1,58 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*-. -"""Provides default routines for solar wind and geospace indices +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- -""" +"""Routines for the F10.7 solar index.""" import datetime as dt -import pandas as pds import numpy as np +from packaging.version import Version +import pandas as pds import pysat import pysatSpaceWeather as pysat_sw -def acknowledgements(name, tag): - """Returns acknowledgements for space weather dataset +def acknowledgements(tag): + """Define the acknowledgements for the F10.7 data. Parameters ---------- - name : string - Name of space weather index, eg, dst, f107, kp - tag : string + tag : str Tag of the space waether index + Returns + ------- + ackn : str + Acknowledgements string associated with the appropriate F10.7 tag. + """ lisird = 'NOAA radio flux obtained through LISIRD' swpc = ''.join(['Prepared by the U.S. Dept. of Commerce, NOAA, Space ', 'Weather Prediction Center']) - ackn = {'f107': {'historic': lisird, 'prelim': swpc, - 'daily': swpc, 'forecast': swpc, '45day': swpc}} + ackn = {'historic': lisird, 'prelim': swpc, 'daily': swpc, + 'forecast': swpc, '45day': swpc} - return ackn[name][tag] + return ackn[tag] -def references(name, tag): - """Returns references for space weather dataset +def references(tag): + """Define the references for the F10.7 data. Parameters ---------- - name : string - Name of space weather index, eg, dst, f107, kp - tag : string - Tag of the space weather index + tag : str + Instrument tag for the F10.7 data. + + Returns + ------- + refs : str + Reference string associated with the appropriate F10.7 tag. """ noaa_desc = ''.join(['Dataset description: ', @@ -56,17 +67,17 @@ def references(name, tag): swpc_desc = ''.join(['Dataset description: https://www.swpc.noaa.gov/', 'sites/default/files/images/u2/Usr_guide.pdf']) - refs = {'f107': {'historic': "\n".join([noaa_desc, orig_ref]), - 'prelim': "\n".join([swpc_desc, orig_ref]), - 'daily': "\n".join([swpc_desc, orig_ref]), - 'forecast': "\n".join([swpc_desc, orig_ref]), - '45day': "\n".join([swpc_desc, orig_ref])}} + refs = {'historic': "\n".join([noaa_desc, orig_ref]), + 'prelim': "\n".join([swpc_desc, orig_ref]), + 'daily': "\n".join([swpc_desc, orig_ref]), + 'forecast': "\n".join([swpc_desc, orig_ref]), + '45day': "\n".join([swpc_desc, orig_ref])} - return refs[name][tag] + return refs[tag] def combine_f107(standard_inst, forecast_inst, start=None, stop=None): - """ Combine the output from the measured and forecasted F10.7 sources + """Combine the output from the measured and forecasted F10.7 sources. Parameters ---------- @@ -79,7 +90,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): start : dt.datetime or NoneType Starting time for combining data, or None to use earliest loaded date from the pysat Instruments (default=None) - stop : dt.datetime + stop : dt.datetime or NoneType Ending time for combining data, or None to use the latest loaded date from the pysat Instruments (default=None) @@ -90,6 +101,12 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): of time, merging the standard, 45day, and forecasted values based on their reliability + Raises + ------ + ValueError + If appropriate time data is not supplied, or if the date range is badly + formed. + Notes ----- Merging prioritizes the standard data, then the 45day data, and finally @@ -142,11 +159,17 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): if inst_flag == 'standard': # Test to see if data loading is needed if not np.any(standard_inst.index == itime): + # Set the load kwargs, which vary by pysat version and tag + load_kwargs = {'date': itime} + + if Version(pysat.__version__) > Version('3.0.1'): + load_kwargs['use_header'] = True + if standard_inst.tag == 'daily': # Add 30 days - standard_inst.load(date=itime + pds.DateOffset(days=30)) - else: - standard_inst.load(date=itime) + load_kwargs['date'] += dt.timedelta(days=30) + + standard_inst.load(**load_kwargs) good_times = ((standard_inst.index >= itime) & (standard_inst.index < stop)) @@ -183,7 +206,11 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): # data for filename in files: if filename is not None: - forecast_inst.load(fname=filename) + load_kwargs = {'fname': filename} + if Version(pysat.__version__) > Version('3.0.1'): + load_kwargs['use_header'] = True + + forecast_inst.load(**load_kwargs) if notes.find("forecast") < 0: notes += " the {:} source ({:} to ".format(inst_flag, @@ -201,11 +228,14 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): good_vals = forecast_inst['f107'][good_times] != fill_val # Save desired data and cycle time - new_times = list(forecast_inst.index[good_times][good_vals]) - f107_times.extend(new_times) - new_vals = list(forecast_inst['f107'][good_times][good_vals]) - f107_values.extend(new_vals) - itime = f107_times[-1] + pds.DateOffset(days=1) + if len(good_vals) > 0: + new_times = list( + forecast_inst.index[good_times][good_vals]) + f107_times.extend(new_times) + new_vals = list( + forecast_inst['f107'][good_times][good_vals]) + f107_values.extend(new_vals) + itime = f107_times[-1] + pds.DateOffset(days=1) notes += "{:})".format(itime.date()) @@ -266,8 +296,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): def parse_45day_block(block_lines): - """ Parse the data blocks used in the 45-day Ap and F10.7 Flux Forecast - file + """Parse the data blocks used in the 45-day Ap and F10.7 Flux Forecast file. Parameters ---------- @@ -303,7 +332,7 @@ def parse_45day_block(block_lines): def rewrite_daily_file(year, outfile, lines): - """ Rewrite the SWPC Daily Solar Data files + """Rewrite the SWPC Daily Solar Data files. Parameters ---------- @@ -316,7 +345,7 @@ def rewrite_daily_file(year, outfile, lines): """ - # get to the solar index data + # Get to the solar index data if year > 2000: raw_data = lines.split('#---------------------------------')[-1] raw_data = raw_data.split('\n')[1:-1] @@ -329,21 +358,21 @@ def rewrite_daily_file(year, outfile, lines): istart = 7 if year < 2000 else 1 raw_data = raw_data[istart:-1] - # parse the data + # Parse the data solar_times, data_dict = parse_daily_solar_data(raw_data, year, optical) - # collect into DataFrame + # Collect into DataFrame data = pds.DataFrame(data_dict, index=solar_times, columns=data_dict.keys()) - # write out as a file + # Write out as a file data.to_csv(outfile, header=True) return def parse_daily_solar_data(data_lines, year, optical): - """ Parse the data in the SWPC daily solar index file + """Parse the data in the SWPC daily solar index file. Parameters ---------- @@ -406,7 +435,7 @@ def parse_daily_solar_data(data_lines, year, optical): def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41): - """ Calculate the 81 day mean F10.7 + """Calculate the 81 day mean F10.7. Parameters ---------- @@ -465,7 +494,7 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41): freq = pysat.utils.time.calc_freq(f107_inst.index) if freq != "86400S": # Resample to the desired frequency - f107_fill = f107_fill.resample(freq).pad() + f107_fill = f107_fill.resample(freq).ffill() # Save the output in a list f107a = list(f107_fill[f107a_name]) diff --git a/pysatSpaceWeather/instruments/methods/general.py b/pysatSpaceWeather/instruments/methods/general.py index 51467565..3bf7729e 100644 --- a/pysatSpaceWeather/instruments/methods/general.py +++ b/pysatSpaceWeather/instruments/methods/general.py @@ -1,13 +1,11 @@ # -*- coding: utf-8 -*-. -"""Provides default routines for the space weather instruments - -""" +"""Provides routines that support general space weather instruments.""" import numpy as np def preprocess(inst): - """Preprocess the meta data by replacing the file fill values with NaN + """Preprocess the meta data by replacing the file fill values with NaN. Parameters ---------- @@ -15,9 +13,11 @@ def preprocess(inst): pysat.Instrument object """ + # Replace all fill values with NaN - for col in inst.data.columns: - fill_val = inst.meta[col][inst.meta.labels.fill_val] + for col in inst.variables: + fill_val = inst.meta[col, inst.meta.labels.fill_val] + # Ensure we are dealing with a float for future nan comparison if isinstance(fill_val, np.floating) or isinstance(fill_val, float): if ~np.isnan(fill_val): diff --git a/pysatSpaceWeather/instruments/methods/kp_ap.py b/pysatSpaceWeather/instruments/methods/kp_ap.py index 9fe69ec0..4cff1922 100644 --- a/pysatSpaceWeather/instruments/methods/kp_ap.py +++ b/pysatSpaceWeather/instruments/methods/kp_ap.py @@ -1,10 +1,15 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*-. -"""Provides default routines for solar wind and geospace indices +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Provides routines to support the geomagnetic indeces, Kp and Ap.""" -""" - -import pandas as pds +import datetime as dt import numpy as np +from packaging.version import Version +import pandas as pds import pysat @@ -15,14 +20,14 @@ # Instrument utilities def acknowledgements(name, tag): - """Returns acknowledgements for space weather dataset + """Define the acknowledgements for the geomagnetic data sets. Parameters ---------- - name : string - Name of space weather index, eg, dst, f107, kp - tag : string - Tag of the space waether index + name : str + Instrument name of space weather index, accepts 'kp' or 'ap'. + tag : str + Instrument tag. """ swpc = ''.join(['Prepared by the U.S. Dept. of Commerce, NOAA, Space ', @@ -35,14 +40,14 @@ def acknowledgements(name, tag): def references(name, tag): - """Returns references for space weather dataset + """Define the references for the geomagnetic data sets. Parameters ---------- - name : string - Name of space weather index, eg, dst, f107, kp - tag : string - Tag of the space waether index + name : str + Instrument name of space weather index, accepts kp or ap. + tag : str + Instrument tag. """ @@ -73,7 +78,7 @@ def references(name, tag): def initialize_kp_metadata(meta, data_key, fill_val=-1): - """ Initialize the Kp meta data using our knowledge of the index + """Initialize the Kp meta data using our knowledge of the index. Parameters ---------- @@ -99,19 +104,28 @@ def initialize_kp_metadata(meta, data_key, fill_val=-1): # Common custom functions -def convert_3hr_kp_to_ap(kp_inst): - """ Calculate 3 hour ap from 3 hour Kp index +def convert_3hr_kp_to_ap(kp_inst, var_name='Kp'): + """Calculate 3 hour ap from 3 hour Kp index. Parameters ---------- kp_inst : pysat.Instrument - Pysat instrument containing Kp data + Instrument containing Kp data + var_name : str + Variable name for the Kp data (default='Kp') - Notes - ----- + Raises + ------ + ValueError + If `var_name` is not present in `kp_inst`. + + Note + ---- Conversion between ap and Kp indices is described at: https://www.ngdc.noaa.gov/stp/GEOMAG/kp_ap.html + Assigns new data to '3hr_ap'. + """ # Kp are keys, where n.3 = n+ and n.6 = (n+1)-. E.g., 0.6 = 1- @@ -127,13 +141,14 @@ def ap(kk): return np.nan # Test the input - if 'Kp' not in kp_inst.data.columns: - raise ValueError('unable to locate Kp data') + if var_name not in kp_inst.variables: + raise ValueError('Variable name for Kp data is missing: {:}'.format( + var_name)) # Convert from Kp to ap - fill_val = kp_inst.meta['Kp'][kp_inst.meta.labels.fill_val] + fill_val = kp_inst.meta[var_name][kp_inst.meta.labels.fill_val] ap_data = np.array([ap(kp) if kp != fill_val else fill_val - for kp in kp_inst['Kp']]) + for kp in kp_inst[var_name]]) # Append the output to the pysat instrument kp_inst['3hr_ap'] = pds.Series(ap_data, index=kp_inst.index) @@ -155,7 +170,7 @@ def ap(kk): def calc_daily_Ap(ap_inst, ap_name='3hr_ap', daily_name='Ap', running_name=None): - """ Calculate the daily Ap index from the 3hr ap index + """Calculate the daily Ap index from the 3hr ap index. Parameters ---------- @@ -169,6 +184,11 @@ def calc_daily_Ap(ap_inst, ap_name='3hr_ap', daily_name='Ap', Column name for daily running average of ap, not output if None (default=None) + Raises + ------ + ValueError + If `ap_name` or `daily_name` aren't present in `ap_inst` + Note ---- Ap is the mean of the 3hr ap indices measured for a given day @@ -179,11 +199,11 @@ def calc_daily_Ap(ap_inst, ap_name='3hr_ap', daily_name='Ap', """ # Test that the necessary data is available - if ap_name not in ap_inst.data.columns: + if ap_name not in ap_inst.variables: raise ValueError("bad 3-hourly ap column name: {:}".format(ap_name)) # Test to see that we will not be overwritting data - if daily_name in ap_inst.data.columns: + if daily_name in ap_inst.variables: raise ValueError("daily Ap column name already exists: " + daily_name) # Calculate the daily mean value @@ -209,12 +229,14 @@ def calc_daily_Ap(ap_inst, ap_name='3hr_ap', daily_name='Ap', # Pad the data so the first day will be backfilled ap_pad = pds.Series(np.full(shape=(1,), fill_value=np.nan), index=[ap_mean.index[0] - pds.DateOffset(hours=3)]) + # Extract the mean that only uses data for one day ap_sel = ap_pad.combine_first(ap_mean[[i for i, tt in enumerate(ap_mean.index) if tt.hour == 21]]) + # Backfill this data - ap_data = ap_sel.resample('3H').backfill() + ap_data = ap_sel.resample('3H').bfill() # Save the output for the original time range ap_inst[daily_name] = pds.Series(ap_data[1:], index=ap_data.index[1:]) @@ -236,8 +258,9 @@ def calc_daily_Ap(ap_inst, ap_name='3hr_ap', daily_name='Ap', return -def filter_geomag(inst, min_kp=0, max_kp=9, filter_time=24, kp_inst=None): - """Filters pysat.Instrument data for given time after Kp drops below gate. +def filter_geomag(inst, min_kp=0, max_kp=9, filter_time=24, kp_inst=None, + var_name='Kp'): + """Filter pysat.Instrument data for given time after Kp drops below gate. Parameters ---------- @@ -254,6 +277,8 @@ def filter_geomag(inst, min_kp=0, max_kp=9, filter_time=24, kp_inst=None): kp_inst : pysat.Instrument or NoneType Kp pysat.Instrument object with or without data already loaded. If None, will load GFZ historic kp data for the instrument date (default=None) + var_name : str + String providing the variable name for the Kp data (default='Kp') Note ---- @@ -267,9 +292,6 @@ def filter_geomag(inst, min_kp=0, max_kp=9, filter_time=24, kp_inst=None): This routine is written for standard Kp data (tag=''), not the forecast or recent data. However, it will work with these Kp data if they are supplied. - This routine is designed to be used with the 'modify' flag if applied as - a custom routine. - """ # Load the desired data if kp_inst is None: @@ -277,13 +299,21 @@ def filter_geomag(inst, min_kp=0, max_kp=9, filter_time=24, kp_inst=None): tag='', pad=pds.DateOffset(days=1)) if kp_inst.empty: - kp_inst.load(date=inst.date, verifyPad=True) + load_kwargs = {'date': inst.index[0], 'end_date': inst.index[-1], + 'verifyPad': True} + if Version(pysat.__version__) > Version('3.0.1'): + load_kwargs['use_header'] = True + + kp_inst.load(**load_kwargs) # Begin filtering, starting at the beginning of the instrument data - sel_data = kp_inst[(inst.date - pds.DateOffset(days=1)): - (inst.date + pds.DateOffset(days=1))] - ind, = np.where((sel_data['Kp'] > max_kp) | (sel_data['Kp'] < min_kp)) + sel_data = kp_inst[(inst.index[0] - pds.DateOffset(days=1)): + (inst.index[-1] + pds.DateOffset(days=1))] + ind, = np.where((sel_data[var_name] > max_kp) + | (sel_data[var_name] < min_kp)) + for lind in ind: + # Determine the time filter range for removing each flagged data sind = sel_data.index[lind] eind = sind + pds.DateOffset(hours=filter_time) inst[sind:eind] = np.nan @@ -295,9 +325,8 @@ def filter_geomag(inst, min_kp=0, max_kp=9, filter_time=24, kp_inst=None): # -------------------------------------------------------------------------- # Common analysis functions - -def convert_ap_to_kp(ap_data, fill_val=-1, ap_name='ap'): - """ Convert Ap into Kp +def convert_ap_to_kp(ap_data, fill_val=-1, ap_name='ap', kp_name='Kp'): + """Convert Ap into Kp. Parameters ---------- @@ -306,7 +335,9 @@ def convert_ap_to_kp(ap_data, fill_val=-1, ap_name='ap'): fill_val : int, float, NoneType Fill value for the data set (default=-1) ap_name : str - Name of the input ap + Name of the input ap (default='ap') + kp_name : str + Name of the output Kp (default='Kp') Returns ------- @@ -317,60 +348,80 @@ def convert_ap_to_kp(ap_data, fill_val=-1, ap_name='ap'): """ - # Ap are keys, Kp returned as double (N- = N.6667, N+=N.3333333) - one_third = 1.0 / 3.0 - two_third = 2.0 / 3.0 - ap_to_kp = {0: 0, 2: one_third, 3: two_third, - 4: 1, 5: 1.0 + one_third, 6: 1.0 + two_third, - 7: 2, 9: 2.0 + one_third, 12: 2.0 + two_third, - 15: 3, 18: 3.0 + one_third, 22: 3.0 + two_third, - 27: 4, 32: 4.0 + one_third, 39: 4.0 + two_third, - 48: 5, 56: 5.0 + one_third, 67: 5.0 + two_third, - 80: 6, 94: 6.0 + one_third, 111: 6.0 + two_third, - 132: 7, 154: 7.0 + one_third, 179: 7.0 + two_third, - 207: 8, 236: 8.0 + one_third, 300: 8.0 + two_third, - 400: 9} - ap_keys = sorted([akey for akey in ap_to_kp.keys()]) - - # If the ap falls between two Kp indices, assign it to the lower Kp value - def round_ap(ap_in, fill_val=fill_val): - """ Round an ap value to the nearest Kp value - """ - if not np.isfinite(ap_in): - return fill_val - - i = 0 - while ap_keys[i] <= ap_in: - i += 1 - i -= 1 - - if i >= len(ap_keys) or ap_keys[i] > ap_in: - return fill_val - - return ap_to_kp[ap_keys[i]] - # Convert from ap to Kp - kp_data = np.array([ap_to_kp[aa] if aa in ap_keys else - round_ap(aa, fill_val=fill_val) for aa in ap_data]) + kp_data = np.array([round_ap(aa, fill_val=fill_val) for aa in ap_data]) # Set the metadata meta = pysat.Meta() - meta['Kp'] = {meta.labels.name: 'Kp', - meta.labels.desc: 'Kp converted from {:}'.format(ap_name), - meta.labels.min_val: 0, - meta.labels.max_val: 9, - meta.labels.fill_val: fill_val, - meta.labels.notes: ''.join( - ['Kp converted from ', ap_name, 'as described at: ', - 'https://www.ngdc.noaa.gov/stp/GEOMAG/kp_ap.html'])} + meta[kp_name] = {meta.labels.name: 'Kp', + meta.labels.desc: 'Kp converted from {:}'.format(ap_name), + meta.labels.min_val: 0, + meta.labels.max_val: 9, + meta.labels.fill_val: fill_val, + meta.labels.notes: ''.join( + ['Kp converted from ', ap_name, 'as described at: ', + 'https://www.ngdc.noaa.gov/stp/GEOMAG/kp_ap.html'])} # Return data and metadata return kp_data, meta +def round_ap(ap_in, fill_val=np.nan): + """Round an ap value to the nearest Kp value. + + Parameters + ---------- + ap_in : float + Ap value as a floating point number. + fill_val : float + Value for unassigned or bad indices. (default=np.nan) + + Returns + ------- + float + Fill value for infinite or out-of-range data, otherwise the best kp + index is provided. + + """ + + # Define the ap to kp conversion + one_third = 1.0 / 3.0 + two_third = 2.0 / 3.0 + ap_to_kp = {0: 0, 2: one_third, 3: two_third, 4: 1, 5: 1.0 + one_third, + 6: 1.0 + two_third, 7: 2, 9: 2.0 + one_third, + 12: 2.0 + two_third, 15: 3, 18: 3.0 + one_third, + 22: 3.0 + two_third, 27: 4, 32: 4.0 + one_third, + 39: 4.0 + two_third, 48: 5, 56: 5.0 + one_third, + 67: 5.0 + two_third, 80: 6, 94: 6.0 + one_third, + 111: 6.0 + two_third, 132: 7, 154: 7.0 + one_third, + 179: 7.0 + two_third, 207: 8, 236: 8.0 + one_third, + 300: 8.0 + two_third, 400: 9} + max_ap = 400 + + # Infinite or NaN values will return the fill value + if not np.isfinite(ap_in): + return fill_val + + # Ap values that correspond exactly to a Kp value will return that value + if ap_in in ap_to_kp.keys(): + return ap_to_kp[ap_in] + + # The input Ap value may be appropriate, but does not correspond directly + # to a Kp index. Get a sorted list of directly corresponding Ap indices, + # with Kp returned as double (N- = N.6667, N+ = N.3333333) + ap_keys = sorted([akey for akey in ap_to_kp.keys() if akey <= ap_in]) + + # If the value is too large or too small, return the fill value + if len(ap_keys) == 0 or (ap_keys[-1] < ap_in and ap_keys[-1] == max_ap): + return fill_val + + # The value is realistic, return the Kp value + return ap_to_kp[ap_keys[-1]] + + def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, start=None, stop=None, fill_val=np.nan): - """ Combine the output from the different Kp sources for a range of dates + """Combine the output from the different Kp sources for a range of dates. Parameters ---------- @@ -400,6 +451,11 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, time, merging the standard, recent, and forecasted values based on their reliability + Raises + ------ + ValueError + If only one Kp instrument or bad times are provided + Note ---- Merging prioritizes the standard data, then the recent data, and finally @@ -443,8 +499,7 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, if stop is None: stimes = [inst.index.max() for inst in all_inst if len(inst.index) > 0] - stop = max(stimes) if len(stimes) > 0 else None - stop += pds.DateOffset(days=1) + stop = max(stimes) + dt.timedelta(days=1) if len(stimes) > 0 else None if start is None or stop is None: raise ValueError(' '.join(("must either load in Instrument objects or", @@ -467,7 +522,11 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, while itime < stop and inst_flag is not None: # Load and save the standard data for as many times as possible if inst_flag == 'standard': - standard_inst.load(date=itime) + load_kwargs = {'date': itime} + if Version(pysat.__version__) > Version('3.0.1'): + load_kwargs['use_header'] = True + + standard_inst.load(**load_kwargs) if notes.find("standard") < 0: notes += " the {:} source ({:} to ".format(inst_flag, @@ -493,7 +552,10 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, # data for filename in files: if filename is not None: - recent_inst.load(fname=filename) + load_kwargs = {'fname': filename} + if Version(pysat.__version__) > Version('3.0.1'): + load_kwargs['use_header'] = True + recent_inst.load(**load_kwargs) if notes.find("recent") < 0: notes += " the {:} source ({:} to ".format(inst_flag, @@ -526,7 +588,10 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, # data for filename in files: if filename is not None: - forecast_inst.load(fname=filename) + load_kwargs = {'fname': filename} + if Version(pysat.__version__) > Version('3.0.1'): + load_kwargs['use_header'] = True + forecast_inst.load(**load_kwargs) if notes.find("forecast") < 0: notes += " the {:} source ({:} to ".format(inst_flag, @@ -553,7 +618,6 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, notes += "{:})".format(itime.date()) # Determine if the beginning or end of the time series needs to be padded - freq = None if len(kp_times) < 2 else pysat.utils.time.calc_freq(kp_times) end_date = stop - pds.DateOffset(days=1) date_range = pds.date_range(start=start, end=end_date, freq=freq) diff --git a/pysatSpaceWeather/instruments/sw_dst.py b/pysatSpaceWeather/instruments/sw_dst.py index 0bf15531..a59fe03a 100644 --- a/pysatSpaceWeather/instruments/sw_dst.py +++ b/pysatSpaceWeather/instruments/sw_dst.py @@ -9,6 +9,7 @@ 'dst' tag 'noaa' - Historic Dst data coalated by and maintained by NOAA/NCEI + 'lasp' - Predicted Dst from real-time ACE or DSCOVR provided by LASP inst_id '' @@ -30,25 +31,29 @@ import numpy as np import os import pandas as pds +import requests import pysat from pysatSpaceWeather.instruments.methods import dst as mm_dst -logger = pysat.logger - # ---------------------------------------------------------------------------- # Instrument attributes platform = 'sw' name = 'dst' -tags = {'noaa': 'Historic Dst data coalated by and maintained by NOAA/NCEI'} -inst_ids = {'': [tag for tag in tags]} +tags = {'noaa': 'Historic Dst data coalated by and maintained by NOAA/NCEI', + 'lasp': 'Predicted Dst from real-time ACE or DSCOVR provided by LASP'} +inst_ids = {'': [tag for tag in tags.keys()]} + +# Generate today's date to support loading predicted data sets +today = pysat.utils.time.today() +tomorrow = today + dt.timedelta(days=1) # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'noaa': dt.datetime(2007, 1, 1)}} +_test_dates = {'': {'noaa': dt.datetime(2007, 1, 1), 'lasp': today}} # Other tags assumed to be True _test_download_travis = {'': {'noaa': False}} @@ -58,23 +63,17 @@ def init(self): - """Initializes the Instrument object with instrument specific values. - """ + """Initialize the Instrument object with instrument specific values.""" - self.acknowledgements = mm_dst.acknowledgements(self.name, self.tag) - self.references = mm_dst.references(self.name, self.tag) - logger.info(self.acknowledgements) + self.acknowledgements = mm_dst.acknowledgements(self.tag) + self.references = mm_dst.references(self.tag) + pysat.logger.info(self.acknowledgements) return def clean(self): - """ Cleaning function for Dst - - Note - ---- - No necessary for the Dst index + """Clean the Dst index, empty function.""" - """ return @@ -82,17 +81,17 @@ def clean(self): # Instrument functions -def load(fnames, tag=None, inst_id=None): - """Load Kp index files +def load(fnames, tag='', inst_id=''): + """Load the Dst index files. Parameters ---------- fnames : pandas.Series Series of filenames - tag : str or NoneType - tag or None (default=None) - inst_id : str or NoneType - satellite id or None (default=None) + tag : str + Instrument tag string. (default='') + inst_id : str + Instrument ID, not used. (default='') Returns ------- @@ -109,70 +108,74 @@ def load(fnames, tag=None, inst_id=None): all_data = [] - # Dst data is actually stored by year but users can load by day. - # Extract the actual dates from the input list of filenames as - # well as the names of the actual files. - fdates = [] - ufnames = [] - for filename in fnames: - fdates.append(dt.datetime.strptime(filename[-10:], '%Y-%m-%d')) - ufnames.append(filename[0:-11]) - - # Get unique filenames that map to actual data - ufnames = np.unique(ufnames).tolist() - - # Load unique files - for fname in ufnames: - with open(fname) as open_f: - lines = open_f.readlines() - idx = 0 - - # Check if all lines are good - max_lines = 0 - for line in lines: - if len(line) > 1: - max_lines += 1 - - # Prep memory - yr = np.zeros(max_lines * 24, dtype=int) - mo = np.zeros(max_lines * 24, dtype=int) - day = np.zeros(max_lines * 24, dtype=int) - ut = np.zeros(max_lines * 24, dtype=int) - dst = np.zeros(max_lines * 24, dtype=int) - - # Read data - for line in lines: - if len(line) > 1: - temp_year = int(line[14:16] + line[3:5]) - if temp_year > 57: - temp_year += 1900 - else: - temp_year += 2000 - - yr[idx:idx + 24] = temp_year - mo[idx:idx + 24] = int(line[5:7]) - day[idx:idx + 24] = int(line[8:10]) - ut[idx:idx + 24] = np.arange(24) - temp = line.strip()[20:-4] - temp2 = [temp[4 * i:4 * (i + 1)] for i in np.arange(24)] - dst[idx:idx + 24] = temp2 - idx += 24 - - # Prep datetime index for the data and create DataFrame - start = dt.datetime(yr[0], mo[0], day[0], ut[0]) - stop = dt.datetime(yr[-1], mo[-1], day[-1], ut[-1]) - dates = pds.date_range(start, stop, freq='H') - new_data = pds.DataFrame(dst, index=dates, columns=['dst']) - - # Add to all data loaded for filenames - all_data.append(new_data) - - # Combine data together - data = pds.concat(all_data, sort=True, axis=0) - - # Pull out requested days - data = data.iloc[data.index >= fdates[0], :] - data = data.iloc[data.index < fdates[-1] + pds.DateOffset(days=1), :] + if tag == 'noaa': + # NOAA Dst data is actually stored by year, but users can load by day. + # Extract the actual dates from the input list of filenames as well as + # the names of the actual files. + fdates = [] + ufnames = [] + for filename in fnames: + fdates.append(dt.datetime.strptime(filename[-10:], '%Y-%m-%d')) + ufnames.append(filename[0:-11]) + + # Get unique filenames that map to actual data + ufnames = np.unique(ufnames).tolist() + + # Load unique files + for fname in ufnames: + with open(fname) as open_f: + lines = open_f.readlines() + idx = 0 + + # Check if all lines are good + max_lines = 0 + for line in lines: + if len(line) > 1: + max_lines += 1 + + # Prep memory + yr = np.zeros(max_lines * 24, dtype=int) + mo = np.zeros(max_lines * 24, dtype=int) + day = np.zeros(max_lines * 24, dtype=int) + ut = np.zeros(max_lines * 24, dtype=int) + dst = np.zeros(max_lines * 24, dtype=int) + + # Read data + for line in lines: + if len(line) > 1: + temp_year = int(line[14:16] + line[3:5]) + if temp_year > 57: + temp_year += 1900 + else: + temp_year += 2000 + + yr[idx:idx + 24] = temp_year + mo[idx:idx + 24] = int(line[5:7]) + day[idx:idx + 24] = int(line[8:10]) + ut[idx:idx + 24] = np.arange(24) + temp = line.strip()[20:-4] + temp2 = [temp[4 * i:4 * (i + 1)] for i in np.arange(24)] + dst[idx:idx + 24] = temp2 + idx += 24 + + # Prep datetime index for the data and create DataFrame + start = dt.datetime(yr[0], mo[0], day[0], ut[0]) + stop = dt.datetime(yr[-1], mo[-1], day[-1], ut[-1]) + dates = pds.date_range(start, stop, freq='H') + new_data = pds.DataFrame(dst, index=dates, columns=['dst']) + + # Add to all data loaded for filenames + all_data.append(new_data) + + # Combine data together + data = pds.concat(all_data, sort=True, axis=0) + + # Pull out requested days + data = data.iloc[data.index >= fdates[0], :] + data = data.iloc[data.index < fdates[-1] + pds.DateOffset(days=1), :] + else: + data = pysat.instruments.methods.general.load_csv_data( + fnames, read_csv_kwargs={'index_col': 0, 'parse_dates': True}) # Create metadata meta = pysat.Meta() @@ -187,27 +190,24 @@ def load(fnames, tag=None, inst_id=None): return data, meta -def list_files(tag=None, inst_id=None, data_path=None, format_str=None): - """Return a Pandas Series of every file for chosen satellite data +def list_files(tag='', inst_id='', data_path='', format_str=None): + """List local data files for Dst data. Parameters ---------- - tag : string or NoneType - Denotes type of file to load. Accepted types are '1min' and '5min'. - (default=None) - inst_id : string or NoneType - Specifies the satellite ID for a constellation. Not used. - (default=None) - data_path : string or NoneType - Path to data directory. If None is specified, the value previously - set in Instrument.files.data_path is used. (default=None) - format_str : string or NoneType + tag : str + Instrument tag, accepts any value from `tags`. (default='') + inst_id : str + Instrument ID, not used. (default='') + data_path : str + Path to data directory. (default='') + format_str : str or NoneType User specified file format. If None is specified, the default formats associated with the supplied tags are used. (default=None) Returns ------- - pysat.Files.from_os : pysat._files.Files + files : pysat.Files A class containing the verified available files Note @@ -215,76 +215,119 @@ def list_files(tag=None, inst_id=None, data_path=None, format_str=None): Called by pysat. Not intended for direct use by user. """ - - if data_path is not None: + # Get the format string, if not supplied by the user + if format_str is None: if tag == 'noaa': - # files are by year, going to add date to yearly filename for - # each day of the month. The load routine will load a month of - # data and use the appended date to select out appropriate data. - if format_str is None: - format_str = 'dst{year:4d}.txt' - out = pysat.Files.from_os(data_path=data_path, - format_str=format_str) - if not out.empty: - out.loc[out.index[-1] + pds.DateOffset(years=1) - - pds.DateOffset(days=1)] = out.iloc[-1] - out = out.asfreq('D', 'pad') - out = out + '_' + out.index.strftime('%Y-%m-%d') - return out + format_str = 'dst{year:4d}.txt' else: - raise ValueError(''.join(('Unrecognized tag name for Space ', - 'Weather Dst Index'))) - else: - raise ValueError(''.join(('A data_path must be passed to the loading ', - 'routine for Dst'))) - return + format_str = ''.join(['sw_dst_', tag, '_{year:4d}-{month:2d}-', + '{day:2d}.txt']) + # Get the desired files + files = pysat.Files.from_os(data_path=data_path, format_str=format_str) -def download(date_array, tag, inst_id, data_path, user=None, password=None): - """Routine to download Dst index data + if tag == 'noaa': + # NOAA files yearly, so we need to add daily dates to the yearly + # filenames. The load routine will load a month of data and use + # the appended date to select out appropriate data. + if not files.empty: + files.loc[files.index[-1] + pds.DateOffset(years=1) + - pds.DateOffset(days=1)] = files.iloc[-1] + files = files.asfreq('D', 'pad') + files = files + '_' + files.index.strftime('%Y-%m-%d') + + return files + + +def download(date_array, tag, inst_id, data_path): + """Download the Dst index data from the appropriate repository. Parameters ---------- - tag : string or NoneType - Denotes type of file to load. - (default=None) - inst_id : string or NoneType - Specifies the satellite ID for a constellation. Not used. - (default=None) - data_path : string or NoneType - Path to data directory. If None is specified, the value previously - set in Instrument.files.data_path is used. (default=None) + date_array : array-like or pandas.DatetimeIndex + Array-like or index of datetimes for which files will be downloaded. + tag : str + Instrument tag, used to determine download location. + inst_id : str + Instrument ID, not used. + data_path : str + Path to data directory. Note ---- Called by pysat. Not intended for direct use by user. """ - # Connect to host, default port - ftp = ftplib.FTP('ftp.ngdc.noaa.gov') - - # User anonymous, passwd anonymous@ - ftp.login() - ftp.cwd('/STP/GEOMAGNETIC_DATA/INDICES/DST') - - # Data stored by year. Only download for unique set of input years. - years = np.array([date.year for date in date_array]) - years = np.unique(years) - for year in years: - fname_root = 'dst{year:04d}.txt' - fname = fname_root.format(year=year) - saved_fname = os.path.join(data_path, fname) - try: - logger.info('Downloading file for {year:04d}'.format(year=year)) - with open(saved_fname, 'wb') as fp: - ftp.retrbinary('RETR ' + fname, fp.write) - except ftplib.error_perm as exception: - if str(exception.args[0]).split(" ", 1)[0] != '550': - raise - else: - # file not present - os.remove(saved_fname) - logger.info('File not available for {:04d}'.format(year)) - - ftp.close() + if tag == 'noaa': + # Connect to host, default port + ftp = ftplib.FTP('ftp.ngdc.noaa.gov') + + # User anonymous, passwd anonymous@ + ftp.login() + ftp.cwd('/STP/GEOMAGNETIC_DATA/INDICES/DST') + + # Data stored by year. Only download for unique set of input years. + years = np.array([date.year for date in date_array]) + years = np.unique(years) + for year in years: + fname_root = 'dst{year:04d}.txt' + fname = fname_root.format(year=year) + saved_fname = os.path.join(data_path, fname) + try: + pysat.logger.info('Downloading file for {year:04d}'.format( + year=year)) + with open(saved_fname, 'wb') as fp: + ftp.retrbinary('RETR ' + fname, fp.write) + except ftplib.error_perm as exception: + if str(exception.args[0]).split(" ", 1)[0] != '550': + raise + else: + # File not present + os.remove(saved_fname) + pysat.logger.info('File not available for {:04d}'.format( + year)) + + ftp.close() + elif tag == 'lasp': + # Set the remote data variables + url = ''.join(['https://lasp.colorado.edu/space_weather/dsttemerin/', + 'dst_last_96_hrs.txt']) + times = list() + data_dict = {'dst': []} + + # Download the webpage + req = requests.get(url) + + # Test to see if the file was found on the server + if req.text.find('not found on this server') > 0: + pysat.logger.warning(''.join(['LASP last 96 hour Dst file not ', + 'found on server: ', url])) + else: + # Split the file into lines, removing the header and + # trailing empty line + file_lines = req.text.split('\n')[1:-1] + + # Format the data + for line in file_lines: + # Split the line on whitespace + line_cols = line.split() + + if len(line_cols) != 2: + raise IOError(''.join(['unexpected line encountered in ', + 'file retrieved from ', url, ':\n', + line])) + + # Format the time and Dst values + times.append(dt.datetime.strptime(line_cols[0], + '%Y/%j-%H:%M:%S')) + data_dict['dst'].append(float(line_cols[1])) + + # Re-cast the data as a pandas DataFrame + data = pds.DataFrame(data_dict, index=times) + + # Write out as a file + file_base = '_'.join(['sw', 'dst', tag, today.strftime('%Y-%m-%d')]) + file_name = os.path.join(data_path, '{:s}.txt'.format(file_base)) + data.to_csv(file_name) + return diff --git a/pysatSpaceWeather/instruments/sw_f107.py b/pysatSpaceWeather/instruments/sw_f107.py index db00bdc9..5034930d 100644 --- a/pysatSpaceWeather/instruments/sw_f107.py +++ b/pysatSpaceWeather/instruments/sw_f107.py @@ -14,8 +14,8 @@ - 'forecast' Grab forecast data from SWPC (next 3 days) - '45day' 45-Day Forecast data from the Air Force -Example -------- +Examples +-------- Download and load all of the historic F10.7 data. Note that it will not stop on the current date, but a point in the past when post-processing has been successfully completed. @@ -60,15 +60,13 @@ import json import numpy as np import os +import pandas as pds +import pysat import requests import sys import warnings -import pandas as pds -import pysat - from pysatSpaceWeather.instruments.methods import f107 as mm_f107 -from pysatSpaceWeather.instruments.methods.ace import load_csv_data from pysatSpaceWeather.instruments.methods import general logger = pysat.logger @@ -115,14 +113,11 @@ def init(self): - """Initializes the Instrument object with instrument specific values. + """Initialize the Instrument object with instrument specific values.""" - Runs once upon instantiation. - - """ - - self.acknowledgements = mm_f107.acknowledgements(self.name, self.tag) - self.references = mm_f107.references(self.name, self.tag) + # Set the required Instrument attributes + self.acknowledgements = mm_f107.acknowledgements(self.tag) + self.references = mm_f107.references(self.tag) logger.info(self.acknowledgements) # Define the historic F10.7 starting time @@ -133,12 +128,8 @@ def init(self): def clean(self): - """ Cleaning function for Space Weather indices + """Clean the F10.7 data, empty function as this is not necessary.""" - Note - ---- - F10.7 doesn't require cleaning - """ return @@ -146,30 +137,35 @@ def clean(self): # Instrument functions -def load(fnames, tag=None, inst_id=None): - """Load F10.7 index files +def load(fnames, tag='', inst_id=''): + """Load F10.7 index files. Parameters ---------- fnames : pandas.Series - Series of filenames - tag : str or NoneType - tag or None (default=None) - inst_id : str or NoneType - satellite id or None (default=None) + Series of filenames. + tag : str + Instrument tag. (default='') + inst_id : str + Instrument ID, not used. (default='') Returns ------- data : pandas.DataFrame - Object containing satellite data + Object containing satellite data. meta : pysat.Meta - Object containing metadata such as column names and units + Object containing metadata such as column names and units. + + See Also + -------- + pysat.instruments.methods.general.load_csv_data Note ---- Called by pysat. Not intended for direct use by user. """ + # Get the desired file dates and file names from the daily indexed list file_dates = list() if tag in ['historic', 'prelim']: @@ -181,8 +177,8 @@ def load(fnames, tag=None, inst_id=None): fnames = unique_files # Load the CSV data files - data = load_csv_data(fnames, read_csv_kwargs={"index_col": 0, - "parse_dates": True}) + data = pysat.instruments.methods.general.load_csv_data( + fnames, read_csv_kwargs={"index_col": 0, "parse_dates": True}) # If there is a date range, downselect here if len(file_dates) > 0: @@ -294,21 +290,18 @@ def load(fnames, tag=None, inst_id=None): return data, meta -def list_files(tag=None, inst_id=None, data_path=None, format_str=None): - """Return a Pandas Series of every file for F10.7 data +def list_files(tag='', inst_id='', data_path='', format_str=None): + """List local F10.7 data files. Parameters ---------- - tag : string or NoneType - Denotes type of file to load. - (default=None) - inst_id : string or NoneType - Specifies the satellite ID for a constellation. Not used. - (default=None) - data_path : string or NoneType - Path to data directory. If None is specified, the value previously - set in Instrument.files.data_path is used. (default=None) - format_str : string or NoneType + tag : str + Instrument tag, accepts any value from `tags`. (default='') + inst_id : str + Instrument ID, not used. (default='') + data_path : str + Path to data directory. (default='') + format_str : str or NoneType User specified file format. If None is specified, the default formats associated with the supplied tags are used. (default=None) @@ -323,116 +316,111 @@ def list_files(tag=None, inst_id=None, data_path=None, format_str=None): """ - if data_path is not None: - if tag == 'historic': - # Files are by month, going to add date to monthly filename for - # each day of the month. The load routine will load a month of - # data and use the appended date to select out appropriate data. - if format_str is None: - format_str = 'f107_monthly_{year:04d}-{month:02d}.txt' - out_files = pysat.Files.from_os(data_path=data_path, - format_str=format_str) - if not out_files.empty: - out_files.loc[out_files.index[-1] + pds.DateOffset(months=1) - - pds.DateOffset(days=1)] = out_files.iloc[-1] - out_files = out_files.asfreq('D', 'pad') - out_files = out_files + '_' + out_files.index.strftime( - '%Y-%m-%d') - - elif tag == 'prelim': - # Files are by year (and quarter) - if format_str is None: - format_str = ''.join(['f107_prelim_{year:04d}_{month:02d}', - '_v{version:01d}.txt']) - out_files = pysat.Files.from_os(data_path=data_path, - format_str=format_str) - - if not out_files.empty: - # Set each file's valid length at a 1-day resolution - orig_files = out_files.sort_index().copy() - new_files = list() - - for orig in orig_files.iteritems(): - # Version determines each file's valid length - version = int(orig[1].split("_v")[1][0]) - doff = pds.DateOffset(years=1) if version == 2 \ - else pds.DateOffset(months=3) - istart = orig[0] - iend = istart + doff - pds.DateOffset(days=1) - - # Ensure the end time does not extend past the number of - # possible days included based on the file's download time - fname = os.path.join(data_path, orig[1]) - dend = dt.datetime.utcfromtimestamp(os.path.getctime(fname)) - dend = dend - pds.DateOffset(days=1) - if dend < iend: - iend = dend - - # Pad the original file index - out_files.loc[iend] = orig[1] - out_files = out_files.sort_index() - - # Save the files at a daily cadence over the desired period - new_files.append(out_files.loc[istart: - iend].asfreq('D', 'pad')) - # Add the newly indexed files to the file output - out_files = pds.concat(new_files, sort=True) - out_files = out_files.dropna() + if tag == 'historic': + # Files are by month, going to add date to monthly filename for + # each day of the month. The load routine will load a month of + # data and use the appended date to select out appropriate data. + if format_str is None: + format_str = 'f107_monthly_{year:04d}-{month:02d}.txt' + out_files = pysat.Files.from_os(data_path=data_path, + format_str=format_str) + if not out_files.empty: + out_files.loc[out_files.index[-1] + pds.DateOffset(months=1) + - pds.DateOffset(days=1)] = out_files.iloc[-1] + out_files = out_files.asfreq('D', 'pad') + out_files = out_files + '_' + out_files.index.strftime( + '%Y-%m-%d') + + elif tag == 'prelim': + # Files are by year (and quarter) + if format_str is None: + format_str = ''.join(['f107_prelim_{year:04d}_{month:02d}', + '_v{version:01d}.txt']) + out_files = pysat.Files.from_os(data_path=data_path, + format_str=format_str) + + if not out_files.empty: + # Set each file's valid length at a 1-day resolution + orig_files = out_files.sort_index().copy() + new_files = list() + + for orig in orig_files.iteritems(): + # Version determines each file's valid length + version = int(orig[1].split("_v")[1][0]) + doff = pds.DateOffset(years=1) if version == 2 \ + else pds.DateOffset(months=3) + istart = orig[0] + iend = istart + doff - pds.DateOffset(days=1) + + # Ensure the end time does not extend past the number of + # possible days included based on the file's download time + fname = os.path.join(data_path, orig[1]) + dend = dt.datetime.utcfromtimestamp(os.path.getctime(fname)) + dend = dend - pds.DateOffset(days=1) + if dend < iend: + iend = dend + + # Pad the original file index + out_files.loc[iend] = orig[1] out_files = out_files.sort_index() - out_files = out_files + '_' + out_files.index.strftime( - '%Y-%m-%d') - - elif tag in ['daily', 'forecast', '45day']: - format_str = ''.join(['f107_', tag, - '_{year:04d}-{month:02d}-{day:02d}.txt']) - out_files = pysat.Files.from_os(data_path=data_path, - format_str=format_str) - - # Pad list of files data to include most recent file under tomorrow - if not out_files.empty: - pds_off = pds.DateOffset(days=1) - out_files.loc[out_files.index[-1] - + pds_off] = out_files.values[-1] - out_files.loc[out_files.index[-1] - + pds_off] = out_files.values[-1] - - else: - raise ValueError(' '.join(('Unrecognized tag name for Space', - 'Weather Index F107:', tag))) - else: - raise ValueError(' '.join(('A data_path must be passed to the loading', - 'routine for F107'))) + + # Save the files at a daily cadence over the desired period + new_files.append(out_files.loc[istart: + iend].asfreq('D', 'pad')) + # Add the newly indexed files to the file output + out_files = pds.concat(new_files, sort=True) + out_files = out_files.dropna() + out_files = out_files.sort_index() + out_files = out_files + '_' + out_files.index.strftime('%Y-%m-%d') + + elif tag in ['daily', 'forecast', '45day']: + format_str = ''.join(['f107_', tag, + '_{year:04d}-{month:02d}-{day:02d}.txt']) + out_files = pysat.Files.from_os(data_path=data_path, + format_str=format_str) + + # Pad list of files data to include most recent file under tomorrow + if not out_files.empty: + pds_off = pds.DateOffset(days=1) + out_files.loc[out_files.index[-1] + pds_off] = out_files.values[-1] + out_files.loc[out_files.index[-1] + pds_off] = out_files.values[-1] return out_files def download(date_array, tag, inst_id, data_path, update_files=False): - """Routine to download F107 index data + """Download F107 index data from the appropriate repository. Parameters - ----------- - date_array : list-like - Sequence of dates to download date for. - tag : string or NoneType + ---------- + date_array : array-like + Sequence of dates for which files will be downloaded. + tag : str Denotes type of file to load. - inst_id : string or NoneType + inst_id : str Specifies the satellite ID for a constellation. - data_path : string or NoneType + data_path : str Path to data directory. update_files : bool Re-download data for files that already exist if True (default=False) - Note - ---- - Called by pysat. Not intended for direct use by user. + Raises + ------ + IOError + If a problem is encountered connecting to the gateway or retrieving + data from the repository. Warnings -------- Only able to download current forecast data, not archived forecasts. + Note + ---- + Called by pysat. Not intended for direct use by user. + """ - # download standard F107 data + # Download standard F107 data if tag == 'historic': # Test the date array, updating it if necessary if date_array.freq != 'MS': @@ -463,6 +451,10 @@ def download(date_array, tag, inst_id, data_path, update_files=False): req = requests.get(dstr) # Process the JSON file + if req.text.find('Gateway Timeout') >= 0: + raise IOError(''.join(['Gateway timeout when requesting ', + 'file using command: ', dstr])) + raw_dict = json.loads(req.text)['noaa_radio_flux'] data = pds.DataFrame.from_dict(raw_dict['samples']) if data.empty: @@ -487,8 +479,8 @@ def download(date_array, tag, inst_id, data_path, update_files=False): # Create a local CSV file data.to_csv(data_file, header=True) elif tag == 'prelim': - ftp = ftplib.FTP('ftp.swpc.noaa.gov') # connect to host, default port - ftp.login() # user anonymous, passwd anonymous@ + ftp = ftplib.FTP('ftp.swpc.noaa.gov') # Connect to host, default port + ftp.login() # User anonymous, passwd anonymous ftp.cwd('/pub/indices/old_indices') bad_fname = list() @@ -563,7 +555,7 @@ def download(date_array, tag, inst_id, data_path, update_files=False): # Test for an error if str(exception.args[0]).split(" ", 1)[0] != '550': - raise RuntimeError(exception) + raise IOError(exception) else: # file isn't actually there, try the next name os.remove(saved_fname) diff --git a/pysatSpaceWeather/instruments/sw_kp.py b/pysatSpaceWeather/instruments/sw_kp.py index db81a3a2..2013b443 100644 --- a/pysatSpaceWeather/instruments/sw_kp.py +++ b/pysatSpaceWeather/instruments/sw_kp.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Supports Kp index values. Downloads data from ftp.gfz-potsdam.de or SWPC. +"""Supports Kp index values. Properties ---------- @@ -11,9 +11,13 @@ - '' Standard Kp data - 'forecast' Grab forecast data from SWPC (next 3 days) - 'recent' Grab last 30 days of Kp data from SWPC +inst_id + '' Note ---- +Downloads data from ftp.gfz-potsdam.de or SWPC. + Standard Kp files are stored by the first day of each month. When downloading use kp.download(start, stop, freq='MS') to only download days that could possibly have data. 'MS' gives a monthly start frequency. @@ -24,15 +28,19 @@ load command is the date the forecast was generated. The data loaded will span three days. To always ensure you are loading the most recent data, load the data with tomorrow's date. + +Recent data is also stored by the generation date from the SWPC. Each file +contains 30 days of Kp measurements. The load date issued to pysat corresponds +to the generation date. + +Examples +-------- :: kp = pysat.Instrument('sw', 'kp', tag='recent') kp.download() kp.load(date=kp.tomorrow()) -Recent data is also stored by the generation date from the SWPC. Each file -contains 30 days of Kp measurements. The load date issued to pysat corresponds -to the generation date. Warnings -------- @@ -60,8 +68,8 @@ import pysat -from pysatSpaceWeather.instruments.methods import kp_ap from pysatSpaceWeather.instruments.methods import general +from pysatSpaceWeather.instruments.methods import kp_ap logger = pysat.logger @@ -75,7 +83,7 @@ 'recent': 'SWPC provided Kp for past 30 days'} inst_ids = {'': ['', 'forecast', 'recent']} -# generate todays date to support loading forecast data +# Generate todays date to support loading forecast data now = dt.datetime.utcnow() today = dt.datetime(now.year, now.month, now.day) @@ -97,11 +105,7 @@ def init(self): - """Initializes the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ + """Initialize the Instrument object with instrument specific values.""" self.acknowledgements = kp_ap.acknowledgements(self.name, self.tag) self.references = kp_ap.references(self.name, self.tag) @@ -110,12 +114,8 @@ def init(self): def clean(self): - """ Cleaning function for Space Weather indices + """Clean the Kp, not required for this index (empty function).""" - Note - ---- - Kp doesn't require cleaning - """ return @@ -123,17 +123,17 @@ def clean(self): # Instrument functions -def load(fnames, tag=None, inst_id=None): - """Load Kp index files +def load(fnames, tag='', inst_id=''): + """Load Kp index files. Parameters ---------- fnames : pandas.Series Series of filenames - tag : str or NoneType - tag or None (default=None) - inst_id : str or NoneType - satellite id or None (default=None) + tag : str + Instrument tag (default='') + inst_id : str + Instrument ID, not used. (default='') Returns ------- @@ -214,28 +214,34 @@ def load(fnames, tag=None, inst_id=None): # Combine data together data = pds.concat(all_data, axis=0, sort=True) - # Each column increments UT by three hours. Produce a single data - # series that has Kp value monotonically increasing in time with - # appropriate datetime indices - data_series = pds.Series(dtype='float64') - for i in np.arange(8): - tind = data.index + pds.DateOffset(hours=int(3 * i)) - temp = pds.Series(data.iloc[:, i].values, index=tind) - data_series = data_series.append(temp) - data_series = data_series.sort_index() - data_series.index.name = 'time' - - # Kp comes in non-user friendly values like 2-, 2o, and 2+. Relate - # these to 1.667, 2.0, 2.333 for processing and user friendliness - first = np.array([float(str_val[0]) for str_val in data_series]) - flag = np.array([str_val[1] for str_val in data_series]) - - ind, = np.where(flag == '+') - first[ind] += 1.0 / 3.0 - ind, = np.where(flag == '-') - first[ind] -= 1.0 / 3.0 - - result = pds.DataFrame(first, columns=['Kp'], index=data_series.index) + if len(data.index) > 0: + # Each column increments UT by three hours. Produce a single data + # series that has Kp value monotonically increasing in time with + # appropriate datetime indices + data_series = pds.Series(dtype='float64') + for i in np.arange(8): + tind = data.index + pds.DateOffset(hours=int(3 * i)) + temp = pds.Series(data.iloc[:, i].values, index=tind) + data_series = pds.concat([data_series, temp]) + + data_series = data_series.sort_index() + data_series.index.name = 'time' + + # Kp comes in non-user friendly values like 2-, 2o, and 2+. Relate + # these to 1.667, 2.0, 2.333 for processing and user friendliness + first = np.array([float(str_val[0]) for str_val in data_series]) + flag = np.array([str_val[1] for str_val in data_series]) + + ind, = np.where(flag == '+') + first[ind] += 1.0 / 3.0 + ind, = np.where(flag == '-') + first[ind] -= 1.0 / 3.0 + + result = pds.DataFrame(first, columns=['Kp'], + index=data_series.index) + else: + result = pds.DataFrame() + fill_val = np.nan elif tag == 'forecast': # Load forecast data @@ -243,6 +249,7 @@ def load(fnames, tag=None, inst_id=None): for fname in fnames: result = pds.read_csv(fname, index_col=0, parse_dates=True) all_data.append(result) + result = pds.concat(all_data) fill_val = -1 elif tag == 'recent': @@ -251,6 +258,7 @@ def load(fnames, tag=None, inst_id=None): for fname in fnames: result = pds.read_csv(fname, index_col=0, parse_dates=True) all_data.append(result) + result = pds.concat(all_data) fill_val = -1 @@ -261,27 +269,24 @@ def load(fnames, tag=None, inst_id=None): return result, meta -def list_files(tag=None, inst_id=None, data_path=None, format_str=None): - """Return a Pandas Series of every file for chosen satellite data +def list_files(tag='', inst_id='', data_path='', format_str=None): + """List local files for the requested Instrument tag. Parameters ----------- - tag : string or NoneType - Denotes type of file to load. - (default=None) - inst_id : string or NoneType - Specifies the satellite ID for a constellation. Not used. - (default=None) - data_path : string or NoneType - Path to data directory. If None is specified, the value previously - set in Instrument.files.data_path is used. (default=None) - format_str : string or NoneType + tag : str + Instrument tag, accepts any value from `tags`. (default='') + inst_id : str + Instrument ID, not used. (default='') + data_path : str + Path to data directory. (default='') + format_str : str or NoneType User specified file format. If None is specified, the default formats associated with the supplied tags are used. (default=None) Returns ------- - pysat.Files.from_os : pysat._files.Files + files : pysat._files.Files A class containing the verified available files Note @@ -290,66 +295,54 @@ def list_files(tag=None, inst_id=None, data_path=None, format_str=None): """ - if data_path is not None: - if tag == '': - # Files are by month, going to add date to monthly filename for - # each day of the month. The load routine will load a month of - # data and use the appended date to select out appropriate data. - if format_str is None: - format_str = 'kp{year:2d}{month:02d}.tab' - out = pysat.Files.from_os(data_path=data_path, - format_str=format_str, - two_digit_year_break=99) - if not out.empty: - out.loc[out.index[-1] + pds.DateOffset(months=1) - - pds.DateOffset(days=1)] = out.iloc[-1] - out = out.asfreq('D', 'pad') - out = out + '_' + out.index.strftime('%Y-%m-%d') - return out - elif tag == 'forecast': - format_str = 'kp_forecast_{year:04d}-{month:02d}-{day:02d}.txt' - files = pysat.Files.from_os(data_path=data_path, - format_str=format_str) - # Pad list of files data to include most recent file under tomorrow - if not files.empty: - pds_offset = pds.DateOffset(days=1) - files.loc[files.index[-1] + pds_offset] = files.values[-1] - files.loc[files.index[-1] + pds_offset] = files.values[-1] - return files - elif tag == 'recent': - format_str = 'kp_recent_{year:04d}-{month:02d}-{day:02d}.txt' - files = pysat.Files.from_os(data_path=data_path, - format_str=format_str) - - # Pad list of files data to include most recent file under tomorrow - if not files.empty: - pds_offset = pds.DateOffset(days=1) - files.loc[files.index[-1] + pds_offset] = files.values[-1] - files.loc[files.index[-1] + pds_offset] = files.values[-1] - return files + if tag == '': + # Files are by month, going to add date to monthly filename for + # each day of the month. The load routine will load a month of + # data and use the appended date to select out appropriate data. + if format_str is None: + format_str = 'kp{year:2d}{month:02d}.tab' + files = pysat.Files.from_os(data_path=data_path, + format_str=format_str, + two_digit_year_break=99) + if not files.empty: + files.loc[files.index[-1] + pds.DateOffset(months=1) + - pds.DateOffset(days=1)] = files.iloc[-1] + files = files.asfreq('D', 'pad') + files = files + '_' + files.index.strftime('%Y-%m-%d') - else: - raise ValueError(' '.join(('Unrecognized tag name for Space', - 'Weather Index Kp'))) else: - raise ValueError(' '.join(('A data_path must be passed to the loading', - 'routine for Kp'))) + format_str = '_'.join(['kp', tag, + '{year:04d}-{month:02d}-{day:02d}.txt']) + files = pysat.Files.from_os(data_path=data_path, + format_str=format_str) + + # Pad list of files data to include most recent file under tomorrow + if not files.empty: + pds_offset = pds.DateOffset(days=1) + files.loc[files.index[-1] + pds_offset] = files.values[-1] + files.loc[files.index[-1] + pds_offset] = files.values[-1] + + return files def download(date_array, tag, inst_id, data_path): - """Routine to download Kp index data + """Download the Kp index data from the appropriate repository. Parameters - ----------- - tag : string or NoneType - Denotes type of file to load. Accepted types are '' and 'forecast'. - (default=None) - inst_id : string or NoneType - Specifies the satellite ID for a constellation. Not used. - (default=None) - data_path : string or NoneType - Path to data directory. If None is specified, the value previously - set in Instrument.files.data_path is used. (default=None) + ---------- + date_array : array-like or pandas.DatetimeIndex + Array-like or index of datetimes to be downloaded. + tag : str + Denotes type of file to load. + inst_id : str + Specifies the instrument identification, not used. + data_path : str + Path to data directory. + + Raises + ------ + Exception + Bare raise upon FTP failure, facilitating useful error messages. Note ---- @@ -405,6 +398,7 @@ def download(date_array, tag, inst_id, data_path): elif tag == 'forecast': logger.info(' '.join(('This routine can only download the current', 'forecast, not archived forecasts'))) + # Download webpage furl = 'https://services.swpc.noaa.gov/text/3-day-geomag-forecast.txt' r = requests.get(furl) @@ -434,6 +428,7 @@ def download(date_array, tag, inst_id, data_path): day1.append(int(raw[0:10])) day2.append(int(raw[10:20])) day3.append(int(raw[20:])) + times = pds.date_range(forecast_date, periods=24, freq='3H') day = [] for dd in [day1, day2, day3]: @@ -453,14 +448,14 @@ def download(date_array, tag, inst_id, data_path): # Download webpage rurl = ''.join(('https://services.swpc.noaa.gov/text/', 'daily-geomagnetic-indices.txt')) - r = requests.get(rurl) + req = requests.get(rurl) # Parse text to get the date the prediction was generated - date_str = r.text.split(':Issued: ')[-1].split('\n')[0] + date_str = req.text.split(':Issued: ')[-1].split('\n')[0] dl_date = dt.datetime.strptime(date_str, '%H%M UT %d %b %Y') # Data is the forecast value for the next three days - raw_data = r.text.split('# Date ')[-1] + raw_data = req.text.split('# Date ')[-1] # Keep only the middle bits that matter raw_data = raw_data.split('\n')[1:-1] diff --git a/pysatSpaceWeather/tests/__init__.py b/pysatSpaceWeather/tests/__init__.py index e69de29b..722d4ca9 100644 --- a/pysatSpaceWeather/tests/__init__.py +++ b/pysatSpaceWeather/tests/__init__.py @@ -0,0 +1,26 @@ +"""Unit and Integration Tests for pysatSpaceWeather. + +Note +---- +This file must remain empty of code for pytest to function. + +Example +------- +To run all tests: +:: + + pytest + + +Or, if you do not have the executable installed: +:: + + python -m pytest + +To run a specific test, include the filename (with the full path if not in the +test directory), class, and test name: +:: + + pytest test_utils.py::TestScaleUnits::test_scale_units_angles + +""" diff --git a/pysatSpaceWeather/tests/test_instruments.py b/pysatSpaceWeather/tests/test_instruments.py index 9d49d0a3..520070e5 100644 --- a/pysatSpaceWeather/tests/test_instruments.py +++ b/pysatSpaceWeather/tests/test_instruments.py @@ -1,11 +1,18 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Standard pysat tests for pysatSpaceWeather Instruments.""" + import tempfile import pytest # Import the test classes from pysat import pysat -from pysat.utils import generate_instrument_list from pysat.tests.instrument_test_class import InstTestClass +from pysat.utils import generate_instrument_list # Make sure to import your instrument library here import pysatSpaceWeather @@ -40,8 +47,10 @@ class TestInstruments(InstTestClass): + """Test class for pysatSpaceWeather Instruments.""" + def setup_class(self): - """Runs once before the tests to initialize the testing setup.""" + """Create a clean the testing setup.""" # Make sure to use a temporary directory so that the user's setup is # not altered self.tempdir = tempfile.TemporaryDirectory() @@ -52,15 +61,19 @@ def setup_class(self): # to point to their own subpackage location, e.g., # self.inst_loc = mypackage.instruments self.inst_loc = pysatSpaceWeather.instruments + return def teardown_class(self): - """Runs after every method to clean up previous testing.""" + """Clean up previous testing setup.""" pysat.params.data['data_dirs'] = self.saved_path self.tempdir.cleanup() del self.inst_loc, self.saved_path, self.tempdir + return def setup_method(self): - """Runs before every method to create a clean testing setup.""" + """Create a clean testing setup.""" + return def teardown_method(self): - """Runs after every method to clean up previous testing.""" + """Clean up previous testing setup.""" + return diff --git a/pysatSpaceWeather/tests/test_methods_ace.py b/pysatSpaceWeather/tests/test_methods_ace.py index be389054..fcb0c344 100644 --- a/pysatSpaceWeather/tests/test_methods_ace.py +++ b/pysatSpaceWeather/tests/test_methods_ace.py @@ -1,35 +1,106 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Integration and unit test suite for ACE methods.""" + +from packaging.version import Version import pytest +import warnings + +import pysat from pysatSpaceWeather.instruments.methods import ace as mm_ace -class TestACEMethods(): +class TestACEMethods(object): + """Test class for ACE methods.""" + def setup(self): - """Runs before every method to create a clean testing setup""" + """Create a clean testing setup.""" self.out = None + return def teardown(self): - """Runs after every method to clean up previous testing.""" + """Clean up previous testing setup.""" del self.out + return def test_acknowledgements(self): - """ Test the ACE acknowledgements """ + """Test the ACE acknowledgements.""" self.out = mm_ace.acknowledgements() assert self.out.find('ACE') >= 0 return @pytest.mark.parametrize('name', ['mag', 'epam', 'swepam', 'sis']) def test_references(self, name): - """ Test the references for an ACE instrument""" + """Test the references for an ACE instrument.""" self.out = mm_ace.references(name) assert self.out.find('Space Sci. Rev.') > 0 return def test_references_bad_name(self): - """ Test the references raise an informative error for bad instrument - """ + """Test the references raise an informative error for bad instrument.""" with pytest.raises(KeyError) as kerr: mm_ace.references('ace') assert str(kerr.value).find('unknown ACE instrument') >= 0 return + + def test_load_csv_data_dep_warning(self): + """Test `load_csv_data` raises a DeprecationWarning.""" + + with warnings.catch_warnings(record=True) as war: + mm_ace.load_csv_data([]) + + assert len(war) == 1 + assert war[0].category == DeprecationWarning + assert str(war[0].message).find( + "Moved to pysat.instruments.methods.general.load_csv_data") >= 0 + return + + +@pytest.mark.skipif(Version(pysat.__version__) < Version('3.0.2'), + reason="Requires time routine available in pysat 3.0.2+") +class TestACESWEPAMMethods(object): + """Test class for ACE SWEPAM methods.""" + + def setup(self): + """Create a clean testing setup.""" + self.testInst = pysat.Instrument('pysat', 'testing') + self.testInst.load(date=self.testInst.inst_module._test_dates['']['']) + + self.omni_keys = ['sw_proton_dens_norm', 'sw_ion_temp_norm'] + return + + def teardown(self): + """Clean up previous testing setup.""" + del self.testInst + return + + def test_ace_swepam_hourly_omni_norm(self): + """Test ACE SWEPAM conversion to OMNI hourly normalized standards.""" + + self.testInst['slt'] *= 400.0 + mm_ace.ace_swepam_hourly_omni_norm(self.testInst, speed_key='slt', + dens_key='mlt', temp_key='dummy3') + + # Test that normalized data and metadata are present and realistic + for okey in self.omni_keys: + assert okey in self.testInst.variables + assert okey in self.testInst.meta.keys() + assert (self.testInst[okey].values >= 0.0).all() + + return + + def test_ace_swepam_hourly_omni_norm_bad_keys(self): + """Test ACE SWEPAM conversion to OMNI hourly normalized standards.""" + + with pytest.raises(ValueError) as verr: + mm_ace.ace_swepam_hourly_omni_norm(self.testInst) + + # Test the error message for missing data variables + assert str(verr).find("instrument missing variable") >= 0 + + return diff --git a/pysatSpaceWeather/tests/test_methods_f107.py b/pysatSpaceWeather/tests/test_methods_f107.py index d6f675c5..f550a205 100644 --- a/pysatSpaceWeather/tests/test_methods_f107.py +++ b/pysatSpaceWeather/tests/test_methods_f107.py @@ -1,43 +1,64 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Test suite for F10.7 methods.""" + import datetime as dt import numpy as np +from packaging.version import Version import pandas as pds -import pytest import pysat +import pytest import pysatSpaceWeather as pysat_sw -from pysatSpaceWeather.instruments import sw_f107 from pysatSpaceWeather.instruments.methods import f107 as mm_f107 +from pysatSpaceWeather.instruments import sw_f107 -class TestSWF107(): +class TestSWF107(object): + """Test class for F10.7 methods.""" + def setup(self): - """Runs before every method to create a clean testing setup""" + """Create a clean testing setup.""" # Load a test instrument self.testInst = pysat.Instrument() self.testInst.data = pds.DataFrame({'f107': np.linspace(70, 200, 160)}, index=[dt.datetime(2009, 1, 1) + pds.DateOffset(days=i) for i in range(160)]) + return def teardown(self): - """Runs after every method to clean up previous testing.""" + """Clean up previous testing setup.""" del self.testInst + return + + @pytest.mark.parametrize("inargs,vmsg", [ + (["bad"], "unknown input data column"), + (['f107', 'f107'], "output data column already exists")]) + def test_calc_f107a_bad_inputs(self, inargs, vmsg): + """Test the calc_f107a with a bad inputs. - def test_calc_f107a_bad_inname(self): - """ Test the calc_f107a with a bad input name """ + Parameters + ---------- + inargs : list + List of input arguements that should raise a ValueError + vmsg : str + Expected ValueError message - with pytest.raises(ValueError): - mm_f107.calc_f107a(self.testInst, 'bad') + """ - def test_calc_f107a_bad_outname(self): - """ Test the calc_f107a with a bad output name """ + with pytest.raises(ValueError) as verr: + mm_f107.calc_f107a(self.testInst, *inargs) - with pytest.raises(ValueError): - mm_f107.calc_f107a(self.testInst, 'f107', 'f107') + assert str(verr).find(vmsg) >= 0 + return def test_calc_f107a_daily(self): - """ Test the calc_f107a routine with daily data""" + """Test the calc_f107a routine with daily data.""" mm_f107.calc_f107a(self.testInst, f107_name='f107', f107a_name='f107a') @@ -49,9 +70,10 @@ def test_calc_f107a_daily(self): assert np.all(np.isfinite(self.testInst['f107a'])) assert self.testInst['f107a'].min() > self.testInst['f107'].min() assert self.testInst['f107a'].max() < self.testInst['f107'].max() + return def test_calc_f107a_high_rate(self): - """ Test the calc_f107a routine with sub-daily data""" + """Test the calc_f107a routine with sub-daily data.""" self.testInst.data = pds.DataFrame({'f107': np.linspace(70, 200, 3840)}, index=[dt.datetime(2009, 1, 1) @@ -70,9 +92,10 @@ def test_calc_f107a_high_rate(self): # Assert the same mean value is used for a day assert len(np.unique(self.testInst['f107a'][:24])) == 1 + return def test_calc_f107a_daily_missing(self): - """ Test the calc_f107a routine with some daily data missing""" + """Test the calc_f107a routine with some daily data missing.""" self.testInst.data = pds.DataFrame({'f107': np.linspace(70, 200, 160)}, index=[dt.datetime(2009, 1, 1) @@ -94,11 +117,14 @@ def test_calc_f107a_daily_missing(self): # Assert the expected number of fill values assert(len(self.testInst['f107a'][np.isnan(self.testInst['f107a'])]) == 40) + return + +class TestSWF107Combine(object): + """Test class for the `combine_f107` method.""" -class TestSWF107Combine(): def setup(self): - """Runs before every method to create a clean testing setup""" + """Create a clean testing setup.""" # Switch to test_data directory self.saved_path = pysat.params['data_dirs'] pysat.params.data['data_dirs'] = [pysat_sw.test_data_path] @@ -110,27 +136,52 @@ def setup(self): for tag in sw_f107.tags.keys()} self.combine_times = {"start": self.test_day - dt.timedelta(days=30), "stop": self.test_day + dt.timedelta(days=3)} + self.load_kwargs = {} + if Version(pysat.__version__) > Version('3.0.1'): + self.load_kwargs['use_header'] = True + + return def teardown(self): - """Runs after every method to clean up previous testing.""" + """Clean up previous testing setup.""" pysat.params.data['data_dirs'] = self.saved_path del self.combine_inst, self.test_day, self.combine_times + del self.load_kwargs + return def test_combine_f107_none(self): - """ Test combine_f107 failure when no input is provided""" + """Test `combine_f107` failure when no input is provided.""" - with pytest.raises(TypeError): + with pytest.raises(TypeError) as terr: mm_f107.combine_f107() + assert str(terr).find("missing 2 required positional arguments") >= 0 + return + def test_combine_f107_no_time(self): - """Test combine_f107 failure when no times are provided""" + """Test `combine_f107` failure when no times are provided.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError) as verr: mm_f107.combine_f107(self.combine_inst['historic'], self.combine_inst['forecast']) + assert str(verr).find("must either load in Instrument objects") >= 0 + return + + def test_combine_f107_bad_time(self): + """Test `combine_f107` failure when bad times are provided.""" + + with pytest.raises(ValueError) as verr: + mm_f107.combine_f107(self.combine_inst['historic'], + self.combine_inst['forecast'], + start=self.combine_times['stop'], + stop=self.combine_times['start']) + + assert str(verr).find("date range is zero or negative") >= 0 + return + def test_combine_f107_no_data(self): - """Test combine_f107 when no data is present for specified times""" + """Test `combine_f107` when no data is present for specified times.""" combo_in = {kk: self.combine_inst['forecast'] for kk in ['standard_inst', 'forecast_inst']} @@ -141,14 +192,16 @@ def test_combine_f107_no_data(self): assert f107_inst.data.isnull().all()["f107"] del combo_in, f107_inst + return def test_combine_f107_inst_time(self): - """Test combine_f107 with times provided through datasets""" + """Test `combine_f107` with times provided through datasets.""" self.combine_inst['historic'].load( date=self.combine_inst['historic'].lasp_stime, - end_date=self.combine_times['start']) - self.combine_inst['forecast'].load(date=self.test_day) + end_date=self.combine_times['start'], **self.load_kwargs) + self.combine_inst['forecast'].load(date=self.test_day, + **self.load_kwargs) f107_inst = mm_f107.combine_f107(self.combine_inst['historic'], self.combine_inst['forecast']) @@ -159,10 +212,10 @@ def test_combine_f107_inst_time(self): assert f107_inst.data.columns[0] == 'f107' del f107_inst + return def test_combine_f107_all(self): - """Test combine_f107 when input is provided with 'historic' and '45day' - """ + """Test `combine_f107` with 'historic' and '45day' input.""" f107_inst = mm_f107.combine_f107(self.combine_inst['historic'], self.combine_inst['45day'], @@ -174,3 +227,4 @@ def test_combine_f107_all(self): assert f107_inst.data.columns[0] == 'f107' del f107_inst + return diff --git a/pysatSpaceWeather/tests/test_methods_general.py b/pysatSpaceWeather/tests/test_methods_general.py new file mode 100644 index 00000000..70f5885a --- /dev/null +++ b/pysatSpaceWeather/tests/test_methods_general.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Integration and unit test suite for ACE methods.""" + +import numpy as np +from packaging.version import Version +import pysat +import pytest + +from pysatSpaceWeather.instruments.methods import general + + +@pytest.mark.skipif(Version(pysat.__version__) < Version('3.0.2'), + reason="Test setup requires pysat 3.0.2+") +class TestGeneralMethods(object): + """Test class for general methods.""" + + def setup(self): + """Create a clean testing setup.""" + self.testInst = pysat.Instrument('pysat', 'testing') + self.testInst.load(date=self.testInst.inst_module._test_dates['']['']) + return + + def teardown(self): + """Clean up previous testing setup.""" + del self.testInst + return + + def test_preprocess(self): + """Test the preprocessing routine updates all fill values to be NaN.""" + + # Make sure at least one fill value is not already NaN + var = self.testInst.variables[0] + self.testInst.meta[var] = {self.testInst.meta.labels.fill_val: 0.0} + + # Update the meta data using the general preprocess routine + general.preprocess(self.testInst) + + # Test the output + for var in self.testInst.meta.keys(): + assert np.isnan( + self.testInst.meta[var, self.testInst.meta.labels.fill_val]) + return diff --git a/pysatSpaceWeather/tests/test_methods_kp.py b/pysatSpaceWeather/tests/test_methods_kp.py index 38c183e5..8eb36358 100644 --- a/pysatSpaceWeather/tests/test_methods_kp.py +++ b/pysatSpaceWeather/tests/test_methods_kp.py @@ -3,32 +3,42 @@ # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3986138 # ---------------------------------------------------------------------------- +"""Test suite for Kp and Ap methods.""" import datetime as dt import numpy as np +from packaging.version import Version import pandas as pds -import pytest import pysat +import pytest import pysatSpaceWeather as pysat_sw -from pysatSpaceWeather.instruments import sw_kp from pysatSpaceWeather.instruments.methods import kp_ap +from pysatSpaceWeather.instruments import sw_kp + +class TestSWKp(object): + """Test class for Kp methods.""" -class TestSWKp(): def setup(self): - """Runs before every method to create a clean testing setup""" + """Create a clean testing setup.""" # Load a test instrument - self.testInst = pysat.Instrument() - self.testInst.data = \ - pds.DataFrame({'Kp': np.arange(0, 4, 1.0 / 3.0), - 'ap_nan': np.full(shape=12, fill_value=np.nan), - 'ap_inf': np.full(shape=12, fill_value=np.inf)}, - index=[dt.datetime(2009, 1, 1) - + pds.DateOffset(hours=3 * i) - for i in range(12)]) - self.testInst.meta = pysat.Meta() + self.testInst = pysat.Instrument('pysat', 'testing', num_samples=12) + self.test_time = pysat.instruments.pysat_testing._test_dates[''][''] + + load_kwargs = {'date': self.test_time} + if Version(pysat.__version__) > Version('3.0.1'): + load_kwargs['use_header'] = True + + self.testInst.load(**load_kwargs) + + # Create Kp data + self.testInst.data.index = pds.DatetimeIndex(data=[ + self.test_time + dt.timedelta(hours=3 * i) for i in range(12)]) + self.testInst['Kp'] = np.arange(0, 4, 1.0 / 3.0) + self.testInst['ap_nan'] = np.full(shape=12, fill_value=np.nan) + self.testInst['ap_inf'] = np.full(shape=12, fill_value=np.inf) self.testInst.meta['Kp'] = {self.testInst.meta.labels.fill_val: np.nan} self.testInst.meta['ap_nan'] = {self.testInst.meta.labels.fill_val: np.nan} @@ -37,13 +47,15 @@ def setup(self): # Load a test Metadata self.testMeta = pysat.Meta() + return def teardown(self): - """Runs after every method to clean up previous testing.""" - del self.testInst, self.testMeta + """Clean up previous testing setup.""" + del self.testInst, self.testMeta, self.test_time + return def test_convert_kp_to_ap(self): - """ Test conversion of Kp to ap""" + """Test conversion of Kp to ap.""" kp_ap.convert_3hr_kp_to_ap(self.testInst) @@ -53,13 +65,15 @@ def test_convert_kp_to_ap(self): '3hr_ap'][self.testInst.meta.labels.min_val] assert self.testInst['3hr_ap'].max() <= self.testInst.meta[ '3hr_ap'][self.testInst.meta.labels.max_val] + return def test_convert_kp_to_ap_fill_val(self): - """ Test conversion of Kp to ap with fill values""" + """Test conversion of Kp to ap with fill values.""" # Set the first value to a fill value, then calculate ap - fill_val = self.testInst.meta.labels.fill_val - self.testInst['Kp'][0] = self.testInst.meta['Kp'][fill_val] + fill_label = self.testInst.meta.labels.fill_val + fill_value = self.testInst.meta['Kp', fill_label] + self.testInst.data.at[self.testInst.index[0], 'Kp'] = fill_value kp_ap.convert_3hr_kp_to_ap(self.testInst) # Test non-fill ap values @@ -72,20 +86,24 @@ def test_convert_kp_to_ap_fill_val(self): # Test the fill value in the data and metadata assert np.isnan(self.testInst['3hr_ap'][0]) - assert np.isnan(self.testInst.meta['3hr_ap'][fill_val]) + assert np.isnan(self.testInst.meta['3hr_ap'][fill_label]) - del fill_val + return def test_convert_kp_to_ap_bad_input(self): - """ Test conversion of Kp to ap with bad input""" + """Test conversion of Kp to ap with bad input.""" - self.testInst.data.rename(columns={"Kp": "bad"}, inplace=True) + self.testInst.data = self.testInst.data.rename(columns={"Kp": "bad"}) - with pytest.raises(ValueError): + with pytest.raises(ValueError) as verr: kp_ap.convert_3hr_kp_to_ap(self.testInst) + assert str(verr).find("Variable name for Kp data is missing") >= 0 + return + def test_initialize_kp_metadata(self): - """Test default Kp metadata initialization""" + """Test default Kp metadata initialization.""" + kp_ap.initialize_kp_metadata(self.testInst.meta, 'Kp') assert self.testInst.meta['Kp'][self.testInst.meta.labels.units] == '' @@ -96,9 +114,10 @@ def test_initialize_kp_metadata(self): assert self.testInst.meta['Kp'][self.testInst.meta.labels.max_val] == 9 assert self.testInst.meta['Kp'][ self.testInst.meta.labels.fill_val] == -1 + return def test_uninit_kp_metadata(self): - """Test Kp metadata initialization with uninitialized Metadata""" + """Test Kp metadata initialization with uninitialized Metadata.""" kp_ap.initialize_kp_metadata(self.testMeta, 'Kp') assert self.testMeta['Kp'][self.testMeta.labels.units] == '' @@ -108,16 +127,18 @@ def test_uninit_kp_metadata(self): assert self.testMeta['Kp'][self.testMeta.labels.min_val] == 0 assert self.testMeta['Kp'][self.testMeta.labels.max_val] == 9 assert self.testMeta['Kp'][self.testMeta.labels.fill_val] == -1 + return def test_fill_kp_metadata(self): - """Test Kp metadata initialization with user-specified fill value""" + """Test Kp metadata initialization with user-specified fill value.""" kp_ap.initialize_kp_metadata(self.testInst.meta, 'Kp', fill_val=666) assert self.testInst.meta['Kp'][ self.testInst.meta.labels.fill_val] == 666 + return def test_long_name_kp_metadata(self): - """Test Kp metadata initialization with a long name""" + """Test Kp metadata initialization with a long name.""" dkey = 'high_lat_Kp' kp_ap.initialize_kp_metadata(self.testInst.meta, dkey) @@ -125,9 +146,10 @@ def test_long_name_kp_metadata(self): assert(self.testInst.meta[dkey][self.testInst.meta.labels.desc] == 'Planetary K-index') del dkey + return def test_convert_ap_to_kp(self): - """ Test conversion of ap to Kp""" + """Test conversion of ap to Kp.""" kp_ap.convert_3hr_kp_to_ap(self.testInst) kp_out, kp_meta = kp_ap.convert_ap_to_kp(self.testInst['3hr_ap']) @@ -140,12 +162,14 @@ def test_convert_ap_to_kp(self): assert kp_meta['Kp'][kp_meta.labels.fill_val] == -1 del kp_out, kp_meta + return def test_convert_ap_to_kp_middle(self): - """ Test conversion of ap to Kp where ap is not an exact Kp value""" + """Test conversion of ap to Kp where ap is not an exact Kp value.""" kp_ap.convert_3hr_kp_to_ap(self.testInst) - self.testInst['3hr_ap'][8] += 1 + new_val = self.testInst['3hr_ap'][8] + 1 + self.testInst.data.at[self.testInst.index[8], '3hr_ap'] = new_val kp_out, kp_meta = kp_ap.convert_ap_to_kp(self.testInst['3hr_ap']) # Assert original and coverted there and back Kp are equal @@ -155,10 +179,10 @@ def test_convert_ap_to_kp_middle(self): assert 'Kp' in kp_meta.keys() assert(kp_meta['Kp'][kp_meta.labels.fill_val] == -1) - del kp_out, kp_meta + return def test_convert_ap_to_kp_nan_input(self): - """ Test conversion of ap to Kp where ap is NaN""" + """Test conversion of ap to Kp where ap is NaN.""" kp_out, kp_meta = kp_ap.convert_ap_to_kp(self.testInst['ap_nan']) @@ -170,9 +194,10 @@ def test_convert_ap_to_kp_nan_input(self): assert(kp_meta['Kp'][kp_meta.labels.fill_val] == -1) del kp_out, kp_meta + return def test_convert_ap_to_kp_inf_input(self): - """ Test conversion of ap to Kp where ap is Inf""" + """Test conversion of ap to Kp where ap is Inf.""" kp_out, kp_meta = kp_ap.convert_ap_to_kp(self.testInst['ap_inf']) @@ -184,35 +209,72 @@ def test_convert_ap_to_kp_inf_input(self): assert(kp_meta['Kp'][kp_meta.labels.fill_val] == -1) del kp_out, kp_meta + return def test_convert_ap_to_kp_fill_val(self): - """ Test conversion of ap to Kp with fill values""" + """Test conversion of ap to Kp with fill values.""" # Set the first Kp value to a fill value - fill_val = self.testInst.meta.labels.fill_val - self.testInst['Kp'][0] = self.testInst.meta['Kp', fill_val] + fill_label = self.testInst.meta.labels.fill_val + fill_value = self.testInst.meta['Kp', fill_label] + self.testInst.data.at[self.testInst.index[0], 'Kp'] = fill_value # Calculate ap kp_ap.convert_3hr_kp_to_ap(self.testInst) # Recalculate Kp from ap - kp_out, kp_meta = kp_ap.convert_ap_to_kp( - self.testInst['3hr_ap'], - fill_val=self.testInst.meta['Kp', fill_val]) + kp_out, kp_meta = kp_ap.convert_ap_to_kp(self.testInst['3hr_ap'], + fill_val=fill_value) # Test non-fill ap values assert all(abs(kp_out[1:] - self.testInst.data['Kp'][1:]) < 1.0e-4) # Test the fill value in the data and metadata assert np.isnan(kp_out[0]) - assert np.isnan(kp_meta['Kp'][fill_val]) + assert np.isnan(kp_meta['Kp'][fill_label]) + + return + + @pytest.mark.parametrize("filter_kwargs,ngood", [ + ({"min_kp": 2, 'filter_time': 0}, 6), + ({"max_kp": 2, 'filter_time': 0}, 7), + ({"min_kp": 2, "filter_time": 12}, 2), + ({"min_kp": 2, "max_kp": 3, 'filter_time': 0}, 4)]) + def test_filter_geomag(self, filter_kwargs, ngood): + """Test geomag_filter success for different limits. + + Parameters + ---------- + filter_kwargs : dict + Dict with kwarg input for `filter_geomag` + ngood : int + Expected number of good samples + + """ + + kp_ap.filter_geomag(self.testInst, kp_inst=self.testInst, + **filter_kwargs) + assert len(self.testInst.index) == ngood, \ + 'Incorrect filtering using {:} of {:}'.format(filter_kwargs, + self.testInst['Kp']) + return + + def test_filter_geomag_load_kp(self): + """Test geomag_filter loading the Kp instrument.""" - del fill_val, kp_out, kp_meta + try: + kp_ap.filter_geomag(self.testInst) + assert len(self.testInst.index) == 12 # No filtering with defaults + except KeyError: + pass # Routine failed on filtering, after loading w/o Kp data + return -class TestSwKpCombine(): +class TestSwKpCombine(object): + """Tests for the `combine_kp` method.""" + def setup(self): - """Runs before every method to create a clean testing setup""" + """Create a clean testing setup.""" # Switch to test_data directory self.saved_path = pysat.params['data_dirs'] pysat.params.data['data_dirs'] = [pysat_sw.test_data_path] @@ -231,20 +293,29 @@ def setup(self): "start": self.test_day - dt.timedelta(days=30), "stop": self.test_day + dt.timedelta(days=3), "fill_val": -1} + self.load_kwargs = {} + if Version(pysat.__version__) > Version('3.0.1'): + self.load_kwargs['use_header'] = True + + return def teardown(self): - """Runs after every method to clean up previous testing.""" + """Clean up previous testing.""" pysat.params.data['data_dirs'] = self.saved_path - del self.combine, self.test_day, self.saved_path + del self.combine, self.test_day, self.saved_path, self.load_kwargs + return def test_combine_kp_none(self): - """ Test combine_kp failure when no input is provided""" + """Test combine_kp failure when no input is provided.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError) as verr: kp_ap.combine_kp() + assert str(verr).find("need at two Kp Instrument objects to") >= 0 + return + def test_combine_kp_one(self): - """ Test combine_kp failure when only one instrument is provided""" + """Test combine_kp raises ValueError with only one instrument.""" # Load a test instrument testInst = pysat.Instrument() @@ -256,24 +327,31 @@ def test_combine_kp_one(self): testInst.meta['Kp'] = {testInst.meta.labels.fill_val: np.nan} combo_in = {"standard_inst": testInst} - with pytest.raises(ValueError): + with pytest.raises(ValueError) as verr: kp_ap.combine_kp(combo_in) + assert str(verr).find("need at two Kp Instrument objects to") >= 0 + del combo_in, testInst + return def test_combine_kp_no_time(self): - """Test combine_kp failure when no times are provided""" + """Test combine_kp raises ValueError when no times are provided.""" - combo_in = {kk: self.combine[kk] for kk in - ['standard_inst', 'recent_inst', 'forecast_inst']} + # Remove the start times from the input dict + del self.combine['start'], self.combine['stop'] - with pytest.raises(ValueError): - kp_ap.combine_kp(combo_in) + # Raise a value error + with pytest.raises(ValueError) as verr: + kp_ap.combine_kp(**self.combine) + + # Test the error message + assert str(verr).find("must either load in Instrument objects or") >= 0 - del combo_in + return def test_combine_kp_no_data(self): - """Test combine_kp when no data is present for specified times""" + """Test combine_kp when no data is present for specified times.""" combo_in = {kk: self.combine['forecast_inst'] for kk in ['standard_inst', 'recent_inst', 'forecast_inst']} @@ -284,16 +362,20 @@ def test_combine_kp_no_data(self): assert kp_inst.data.isnull().all()["Kp"] del combo_in, kp_inst + return def test_combine_kp_inst_time(self): - """Test combine_kp when times are provided through the instruments""" + """Test combine_kp when times are provided through the instruments.""" combo_in = {kk: self.combine[kk] for kk in ['standard_inst', 'recent_inst', 'forecast_inst']} - combo_in['standard_inst'].load(date=self.combine['start']) - combo_in['recent_inst'].load(date=self.test_day) - combo_in['forecast_inst'].load(date=self.test_day) + combo_in['standard_inst'].load(date=self.combine['start'], + **self.load_kwargs) + combo_in['recent_inst'].load(date=self.test_day, + **self.load_kwargs) + combo_in['forecast_inst'].load(date=self.test_day, + **self.load_kwargs) combo_in['stop'] = combo_in['forecast_inst'].index[-1] kp_inst = kp_ap.combine_kp(**combo_in) @@ -308,9 +390,10 @@ def test_combine_kp_inst_time(self): assert len(kp_inst['Kp'][np.isnan(kp_inst['Kp'])]) == 0 del combo_in, kp_inst + return def test_combine_kp_all(self): - """Test combine_kp when all input is provided""" + """Test combine_kp when all input is provided.""" kp_inst = kp_ap.combine_kp(**self.combine) @@ -325,9 +408,10 @@ def test_combine_kp_all(self): assert (kp_inst['Kp'] != self.combine['fill_val']).all() del kp_inst + return def test_combine_kp_no_forecast(self): - """Test combine_kp when forecasted data is not provided""" + """Test combine_kp when forecasted data is not provided.""" combo_in = {kk: self.combine[kk] for kk in self.combine.keys() if kk != 'forecast_inst'} @@ -342,9 +426,10 @@ def test_combine_kp_no_forecast(self): assert (kp_inst['Kp'] == self.combine['fill_val']).any() del kp_inst, combo_in + return def test_combine_kp_no_recent(self): - """Test combine_kp when recent data is not provided""" + """Test combine_kp when recent data is not provided.""" combo_in = {kk: self.combine[kk] for kk in self.combine.keys() if kk != 'recent_inst'} @@ -359,9 +444,10 @@ def test_combine_kp_no_recent(self): assert (kp_inst['Kp'] == self.combine['fill_val']).any() del kp_inst, combo_in + return def test_combine_kp_no_standard(self): - """Test combine_kp when standard data is not provided""" + """Test combine_kp when standard data is not provided.""" combo_in = {kk: self.combine[kk] for kk in self.combine.keys() if kk != 'standard_inst'} @@ -377,35 +463,45 @@ def test_combine_kp_no_standard(self): == self.combine['fill_val']) > 0 del kp_inst, combo_in + return -class TestSWAp(): +class TestSWAp(object): + """Test class for Ap methods.""" + def setup(self): - """Runs before every method to create a clean testing setup""" - # Load a test instrument with 3hr ap data - self.testInst = pysat.Instrument() - self.testInst.data = pds.DataFrame({'3hr_ap': [0, 2, 3, 4, 5, 6, 7, 9, - 12, 15]}, - index=[dt.datetime(2009, 1, 1) - + pds.DateOffset(hours=3 * i) - for i in range(10)]) - self.testInst.meta = pysat.Meta() - self.meta_dict = {self.testInst.meta.labels.units: '', - self.testInst.meta.labels.name: 'ap', - self.testInst.meta.labels.desc: - "3-hour ap (equivalent range) index", - self.testInst.meta.labels.min_val: 0, - self.testInst.meta.labels.max_val: 400, - self.testInst.meta.labels.fill_val: np.nan, - self.testInst.meta.labels.notes: 'test ap'} - self.testInst.meta['3hr_ap'] = self.meta_dict + """Create a clean testing setup.""" + self.testInst = pysat.Instrument('pysat', 'testing', num_samples=10) + self.test_time = pysat.instruments.pysat_testing._test_dates[''][''] + + load_kwargs = {'date': self.test_time} + if Version(pysat.__version__) > Version('3.0.1'): + load_kwargs['use_header'] = True + + self.testInst.load(**load_kwargs) + + # Create 3 hr Ap data + self.testInst.data.index = pds.DatetimeIndex(data=[ + self.test_time + pds.DateOffset(hours=3 * i) for i in range(10)]) + self.testInst['3hr_ap'] = np.array([0, 2, 3, 4, 5, 6, 7, 9, 12, 15]) + self.testInst.meta['3hr_ap'] = { + self.testInst.meta.labels.units: '', + self.testInst.meta.labels.name: 'ap', + self.testInst.meta.labels.desc: + "3-hour ap (equivalent range) index", + self.testInst.meta.labels.min_val: 0, + self.testInst.meta.labels.max_val: 400, + self.testInst.meta.labels.fill_val: np.nan, + self.testInst.meta.labels.notes: 'test ap'} + return def teardown(self): - """Runs after every method to clean up previous testing.""" - del self.testInst, self.meta_dict + """Clean up previous testing.""" + del self.testInst, self.test_time + return def test_calc_daily_Ap(self): - """ Test daily Ap calculation""" + """Test daily Ap calculation.""" kp_ap.calc_daily_Ap(self.testInst) @@ -417,15 +513,66 @@ def test_calc_daily_Ap(self): # Test fill values (partial days) assert np.all(np.isnan(self.testInst['Ap'][8:])) + return + + def test_calc_daily_Ap_w_running(self): + """Test daily Ap calculation with running mean.""" - def test_calc_daily_Ap_bad_3hr(self): - """ Test daily Ap calculation with bad input key""" + kp_ap.calc_daily_Ap(self.testInst, running_name="running_ap") - with pytest.raises(ValueError): - kp_ap.calc_daily_Ap(self.testInst, "no") + assert 'Ap' in self.testInst.data.columns + assert 'Ap' in self.testInst.meta.keys() + assert 'running_ap' in self.testInst.data.columns + assert 'running_ap' in self.testInst.meta.keys() - def test_calc_daily_Ap_bad_daily(self): - """ Test daily Ap calculation with bad output key""" + # Test unfilled values (full days) + assert np.all(self.testInst['Ap'][:8].min() == 4.5) + assert np.all(self.testInst['running_ap'][6:].min() == 4.5) - with pytest.raises(ValueError): - kp_ap.calc_daily_Ap(self.testInst, "3hr_ap", "3hr_ap") + # Test fill values (partial days) + assert np.all(np.isnan(self.testInst['Ap'][8:])) + assert np.all(np.isnan(self.testInst['running_ap'][:6])) + return + + @pytest.mark.parametrize("inargs,vmsg", [ + (["no"], "bad 3-hourly ap column name"), + (["3hr_ap", "3hr_ap"], "daily Ap column name already exists")]) + def test_calc_daily_Ap_bad_3hr(self, inargs, vmsg): + """Test bad inputs raise ValueError for daily Ap calculation. + + Parameters + ---------- + inargs : list + Input arguements that should raise a ValueError + vmsg : str + Expected ValueError message + + """ + + with pytest.raises(ValueError) as verr: + kp_ap.calc_daily_Ap(self.testInst, *inargs) + + assert str(verr).find(vmsg) >= 0 + return + + @pytest.mark.parametrize("ap,out", [(0, 0), (1, 0), (153, 7), (-1, None), + (460, None), (np.nan, None), + (np.inf, None), (-np.inf, None)]) + def test_round_ap(self, ap, out): + """Test `round_ap` returns expected value for successes and failures. + + Parameters + ---------- + ap : float + Input ap + out : float or NoneType + Expected output kp or None to use fill_value + + """ + + fill_value = -47.0 + if out is None: + out = fill_value + + assert out == kp_ap.round_ap(ap, fill_val=fill_value) + return diff --git a/pysatSpaceWeather/version.txt b/pysatSpaceWeather/version.txt index 81340c7e..bbdeab62 100644 --- a/pysatSpaceWeather/version.txt +++ b/pysatSpaceWeather/version.txt @@ -1 +1 @@ -0.0.4 +0.0.5 diff --git a/requirements.txt b/requirements.txt index e02d8dd6..c6f6e153 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ netCDF4 numpy +packaging pandas -pysat >= 3.0.0 +pysat >= 3.0.1 requests xarray diff --git a/setup.cfg b/setup.cfg index 2902b1f1..386a17eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,13 +2,15 @@ name = pysatSpaceWeather version = file: pysatSpaceWeather/version.txt url = https://github.com/pysat/pysatSpaceWeather -author = Angeline Burrell +author = Angeline Burrell, et al. author_email = pysat.developers@gmail.com description = 'pysat support for Space Weather Indices' keywords = pysat ionosphere - space weather + heliophysics + magnetosphere + space-weather forecasting indexes classifiers = @@ -23,6 +25,7 @@ classifiers = Programming Language :: Python :: 3.9 Operating System :: MacOS :: MacOS X Operating System :: POSIX :: Linux + Operating System :: Microsoft :: Windows license_file = LICENSE long_description = file: README.md long_description_content_type = text/markdown @@ -35,22 +38,23 @@ zip_safe = False packages = find: install_requires = netCDF4 numpy + packaging pandas pysat requests xarray -[coverage:report] -omit = */instruments/templates/ - [flake8] max-line-length = 80 ignore = W503 + D200 + D202 [tool:pytest] markers = all_inst: tests all instruments download: tests for downloadable instruments no_download: tests for instruments without download support + load_options: tests for instruments including optional load kwargs first: first tests to run second: second tests to run diff --git a/setup.py b/setup.py index fc273bcb..ef2bf98b 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ # Copyright (C) 2020, Authors # Full license can be found in License.md # ----------------------------------------------------------------------------- +"""Setup script for pysatSpaceWeather module.""" from setuptools import setup diff --git a/test_requirements.txt b/test_requirements.txt index 02dae128..b8a41858 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,8 +1,10 @@ coveralls +flake8 +flake8-docstrings +hacking>=1.0 m2r2 numpydoc pytest-cov pytest-ordering -pytest-flake8 sphinx sphinx_rtd_theme